{"bundle_type":"pith_open_graph_bundle","bundle_version":"1.0","pith_number":"pith:2023:OSZMK7G3OHK7AGUQBSDTASJ6Y2","short_pith_number":"pith:OSZMK7G3","canonical_record":{"source":{"id":"2311.09692","kind":"arxiv","version":1},"metadata":{"license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.LG","submitted_at":"2023-11-16T09:07:34Z","cross_cats_sorted":["cs.AI","cs.RO"],"title_canon_sha256":"81875b50e8208a25bfb618a13fe78be1e990c8393219b2a12c65f0514fe931b4","abstract_canon_sha256":"7cc4ba2da7596b50da4ad8e606aab98193c9d4c79f1a8d5e4a382c302aa75941"},"schema_version":"1.0"},"canonical_sha256":"74b2c57cdb71d5f01a900c8730493ec69e418b96cdacebb5d3f3b6eeed649659","source":{"kind":"arxiv","id":"2311.09692","version":1},"source_aliases":[{"alias_kind":"arxiv","alias_value":"2311.09692","created_at":"2026-07-05T07:13:31Z"},{"alias_kind":"arxiv_version","alias_value":"2311.09692v1","created_at":"2026-07-05T07:13:31Z"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.2311.09692","created_at":"2026-07-05T07:13:31Z"},{"alias_kind":"pith_short_12","alias_value":"OSZMK7G3OHK7","created_at":"2026-07-05T07:13:31Z"},{"alias_kind":"pith_short_16","alias_value":"OSZMK7G3OHK7AGUQ","created_at":"2026-07-05T07:13:31Z"},{"alias_kind":"pith_short_8","alias_value":"OSZMK7G3","created_at":"2026-07-05T07:13:31Z"}],"events":[{"event_type":"record_created","subject_pith_number":"pith:2023:OSZMK7G3OHK7AGUQBSDTASJ6Y2","target":"record","payload":{"canonical_record":{"source":{"id":"2311.09692","kind":"arxiv","version":1},"metadata":{"license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.LG","submitted_at":"2023-11-16T09:07:34Z","cross_cats_sorted":["cs.AI","cs.RO"],"title_canon_sha256":"81875b50e8208a25bfb618a13fe78be1e990c8393219b2a12c65f0514fe931b4","abstract_canon_sha256":"7cc4ba2da7596b50da4ad8e606aab98193c9d4c79f1a8d5e4a382c302aa75941"},"schema_version":"1.0"},"canonical_sha256":"74b2c57cdb71d5f01a900c8730493ec69e418b96cdacebb5d3f3b6eeed649659","receipt":{"kind":"pith_receipt","key_id":"pith-v1-2026-05","algorithm":"ed25519","signed_at":"2026-07-05T07:13:31.478404Z","signature_b64":"k6KrS/lOE+Dwov4mJLegQGT+5dV1w/KtrdOFxbAjxyHge5xAcXSbmlrCdtulObtkqYsLs/+rkogobBoV8EEoAA==","signed_message":"canonical_sha256_bytes","builder_version":"pith-number-builder-2026-05-17-v1","receipt_version":"0.3","canonical_sha256":"74b2c57cdb71d5f01a900c8730493ec69e418b96cdacebb5d3f3b6eeed649659","last_reissued_at":"2026-07-05T07:13:31.478065Z","signature_status":"signed_v1","first_computed_at":"2026-07-05T07:13:31.478065Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"source_kind":"arxiv","source_id":"2311.09692","source_version":1,"attestation_state":"computed"},"signer":{"signer_id":"pith.science","signer_type":"pith_registry","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"created_at":"2026-07-05T07:13:31Z","supersedes":[],"prev_event":null,"signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"houVZ7vIoYLOF2DuJC0JMAyV3XRiAmBCIvYEz0bfG2A1t5ppE4oGiZaiyz8gHtwXALRYKWHVSlIrsIpWrOIDBQ==","signed_message":"open_graph_event_sha256_bytes","signed_at":"2026-08-11T02:12:45.099824Z"},"content_sha256":"cd9a7037555e48e75f78db45f61122dd7ba38815333221be5786255883397971","schema_version":"1.0","event_id":"sha256:cd9a7037555e48e75f78db45f61122dd7ba38815333221be5786255883397971"},{"event_type":"graph_snapshot","subject_pith_number":"pith:2023:OSZMK7G3OHK7AGUQBSDTASJ6Y2","target":"graph","payload":{"graph_snapshot":{"paper":{"title":"Augmenting Unsupervised Reinforcement Learning with Self-Reference","license":"http://creativecommons.org/licenses/by/4.0/","headline":"","cross_cats":["cs.AI","cs.RO"],"primary_cat":"cs.LG","authors_text":"Andrew Zhao, Erle Zhu, Gao Huang, Matthieu Lin, Rui Lu, Yong-jin Liu","submitted_at":"2023-11-16T09:07:34Z","abstract_excerpt":"Humans possess the ability to draw on past experiences explicitly when learning new tasks and applying them accordingly. We believe this capacity for self-referencing is especially advantageous for reinforcement learning agents in the unsupervised pretrain-then-finetune setting. During pretraining, an agent's past experiences can be explicitly utilized to mitigate the nonstationarity of intrinsic rewards. In the finetuning phase, referencing historical trajectories prevents the unlearning of valuable exploratory behaviors. Motivated by these benefits, we propose the Self-Reference (SR) approac"},"claims":{"count":0,"items":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"2311.09692","kind":"arxiv","version":1},"verdict":{"id":null,"model_set":{},"created_at":null,"strongest_claim":"","one_line_summary":"","pipeline_version":null,"weakest_assumption":"","pith_extraction_headline":""},"integrity":{"clean":true,"summary":{"advisory":0,"critical":0,"by_detector":{},"informational":0},"endpoint":"/pith/2311.09692/integrity.json","findings":[],"available":true,"detectors_run":[],"snapshot_sha256":"c28c3603d3b5d939e8dc4c7e95fa8dfce3d595e45f758748cecf8e644a296938"},"references":{"count":0,"sample":[],"resolved_work":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","internal_anchors":0},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"author_claims":{"count":0,"strong_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"builder_version":"pith-number-builder-2026-05-17-v1"},"verdict_id":null},"signer":{"signer_id":"pith.science","signer_type":"pith_registry","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"created_at":"2026-07-05T07:13:31Z","supersedes":[],"prev_event":null,"signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"tzhb+amNWhz21RKyiborWgZ86tnr7hgJwq9BW+51xy8z7JQ8DomQ2320nIm3TBeGpmoUI84tMrWpKwrhK9UzBQ==","signed_message":"open_graph_event_sha256_bytes","signed_at":"2026-08-11T02:12:45.100758Z"},"content_sha256":"650aa2be19aebac3d6d9e34985cee99bcf1da766ba22eb1a4161740c9b4c412f","schema_version":"1.0","event_id":"sha256:650aa2be19aebac3d6d9e34985cee99bcf1da766ba22eb1a4161740c9b4c412f"}],"timestamp_proofs":[],"mirror_hints":[{"mirror_type":"https","name":"Pith Resolver","base_url":"https://pith.science","bundle_url":"https://pith.science/pith/OSZMK7G3OHK7AGUQBSDTASJ6Y2/bundle.json","state_url":"https://pith.science/pith/OSZMK7G3OHK7AGUQBSDTASJ6Y2/state.json","well_known_bundle_url":"https://pith.science/.well-known/pith/OSZMK7G3OHK7AGUQBSDTASJ6Y2/bundle.json","status":"primary"}],"public_keys":[{"key_id":"pith-v1-2026-05","algorithm":"ed25519","format":"raw","public_key_b64":"stVStoiQhXFxp4s2pdzPNoqVNBMojDU/fJ2db5S3CbM=","public_key_hex":"b2d552b68890857171a78b36a5dccf368a953413288c353f7c9d9d6f94b709b3","fingerprint_sha256_b32_first128bits":"RVFV5Z2OI2J3ZUO7ERDEBCYNKS","fingerprint_sha256_hex":"8d4b5ee74e4693bcd1df2446408b0d54","rotates_at":null,"url":"https://pith.science/pith-signing-key.json","notes":"Pith uses this Ed25519 key to sign canonical record SHA-256 digests. Verify with: ed25519_verify(public_key, message=canonical_sha256_bytes, signature=base64decode(signature_b64))."}],"merge_version":"pith-open-graph-merge-v1","built_at":"2026-08-11T02:12:45Z","links":{"resolver":"https://pith.science/pith/OSZMK7G3OHK7AGUQBSDTASJ6Y2","bundle":"https://pith.science/pith/OSZMK7G3OHK7AGUQBSDTASJ6Y2/bundle.json","state":"https://pith.science/pith/OSZMK7G3OHK7AGUQBSDTASJ6Y2/state.json","well_known_bundle":"https://pith.science/.well-known/pith/OSZMK7G3OHK7AGUQBSDTASJ6Y2/bundle.json"},"state":{"state_type":"pith_open_graph_state","state_version":"1.0","pith_number":"pith:2023:OSZMK7G3OHK7AGUQBSDTASJ6Y2","merge_version":"pith-open-graph-merge-v1","event_count":2,"valid_event_count":2,"invalid_event_count":0,"equivocation_count":0,"current":{"canonical_record":{"metadata":{"abstract_canon_sha256":"7cc4ba2da7596b50da4ad8e606aab98193c9d4c79f1a8d5e4a382c302aa75941","cross_cats_sorted":["cs.AI","cs.RO"],"license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.LG","submitted_at":"2023-11-16T09:07:34Z","title_canon_sha256":"81875b50e8208a25bfb618a13fe78be1e990c8393219b2a12c65f0514fe931b4"},"schema_version":"1.0","source":{"id":"2311.09692","kind":"arxiv","version":1}},"source_aliases":[{"alias_kind":"arxiv","alias_value":"2311.09692","created_at":"2026-07-05T07:13:31Z"},{"alias_kind":"arxiv_version","alias_value":"2311.09692v1","created_at":"2026-07-05T07:13:31Z"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.2311.09692","created_at":"2026-07-05T07:13:31Z"},{"alias_kind":"pith_short_12","alias_value":"OSZMK7G3OHK7","created_at":"2026-07-05T07:13:31Z"},{"alias_kind":"pith_short_16","alias_value":"OSZMK7G3OHK7AGUQ","created_at":"2026-07-05T07:13:31Z"},{"alias_kind":"pith_short_8","alias_value":"OSZMK7G3","created_at":"2026-07-05T07:13:31Z"}],"graph_snapshots":[{"event_id":"sha256:650aa2be19aebac3d6d9e34985cee99bcf1da766ba22eb1a4161740c9b4c412f","target":"graph","created_at":"2026-07-05T07:13:31Z","signer":{"key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signer_id":"pith.science","signer_type":"pith_registry"},"payload":{"graph_snapshot":{"author_claims":{"count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","strong_count":0},"builder_version":"pith-number-builder-2026-05-17-v1","claims":{"count":0,"items":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"integrity":{"available":true,"clean":true,"detectors_run":[],"endpoint":"/pith/2311.09692/integrity.json","findings":[],"snapshot_sha256":"c28c3603d3b5d939e8dc4c7e95fa8dfce3d595e45f758748cecf8e644a296938","summary":{"advisory":0,"by_detector":{},"critical":0,"informational":0}},"paper":{"abstract_excerpt":"Humans possess the ability to draw on past experiences explicitly when learning new tasks and applying them accordingly. We believe this capacity for self-referencing is especially advantageous for reinforcement learning agents in the unsupervised pretrain-then-finetune setting. During pretraining, an agent's past experiences can be explicitly utilized to mitigate the nonstationarity of intrinsic rewards. In the finetuning phase, referencing historical trajectories prevents the unlearning of valuable exploratory behaviors. Motivated by these benefits, we propose the Self-Reference (SR) approac","authors_text":"Andrew Zhao, Erle Zhu, Gao Huang, Matthieu Lin, Rui Lu, Yong-jin Liu","cross_cats":["cs.AI","cs.RO"],"headline":"","license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.LG","submitted_at":"2023-11-16T09:07:34Z","title":"Augmenting Unsupervised Reinforcement Learning with Self-Reference"},"references":{"count":0,"internal_anchors":0,"resolved_work":0,"sample":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"2311.09692","kind":"arxiv","version":1},"verdict":{"created_at":null,"id":null,"model_set":{},"one_line_summary":"","pipeline_version":null,"pith_extraction_headline":"","strongest_claim":"","weakest_assumption":""}},"verdict_id":null}}],"author_attestations":[],"timestamp_anchors":[],"storage_attestations":[],"citation_signatures":[],"replication_records":[],"corrections":[],"mirror_hints":[],"record_created":{"event_id":"sha256:cd9a7037555e48e75f78db45f61122dd7ba38815333221be5786255883397971","target":"record","created_at":"2026-07-05T07:13:31Z","signer":{"key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signer_id":"pith.science","signer_type":"pith_registry"},"payload":{"attestation_state":"computed","canonical_record":{"metadata":{"abstract_canon_sha256":"7cc4ba2da7596b50da4ad8e606aab98193c9d4c79f1a8d5e4a382c302aa75941","cross_cats_sorted":["cs.AI","cs.RO"],"license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.LG","submitted_at":"2023-11-16T09:07:34Z","title_canon_sha256":"81875b50e8208a25bfb618a13fe78be1e990c8393219b2a12c65f0514fe931b4"},"schema_version":"1.0","source":{"id":"2311.09692","kind":"arxiv","version":1}},"canonical_sha256":"74b2c57cdb71d5f01a900c8730493ec69e418b96cdacebb5d3f3b6eeed649659","receipt":{"algorithm":"ed25519","builder_version":"pith-number-builder-2026-05-17-v1","canonical_sha256":"74b2c57cdb71d5f01a900c8730493ec69e418b96cdacebb5d3f3b6eeed649659","first_computed_at":"2026-07-05T07:13:31.478065Z","key_id":"pith-v1-2026-05","kind":"pith_receipt","last_reissued_at":"2026-07-05T07:13:31.478065Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","receipt_version":"0.3","signature_b64":"k6KrS/lOE+Dwov4mJLegQGT+5dV1w/KtrdOFxbAjxyHge5xAcXSbmlrCdtulObtkqYsLs/+rkogobBoV8EEoAA==","signature_status":"signed_v1","signed_at":"2026-07-05T07:13:31.478404Z","signed_message":"canonical_sha256_bytes"},"source_id":"2311.09692","source_kind":"arxiv","source_version":1}}},"equivocations":[],"invalid_events":[],"applied_event_ids":["sha256:cd9a7037555e48e75f78db45f61122dd7ba38815333221be5786255883397971","sha256:650aa2be19aebac3d6d9e34985cee99bcf1da766ba22eb1a4161740c9b4c412f"],"state_sha256":"e6051bdd1ff442472a11247c02a50c83ddb182df0bc474943ece88c68b80c423"},"bundle_signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"7BJm5uqfPl1sSP8kF40fAdiDl9XmMBq5SY1WJ23hhRDEApx2ovlxQJxQGzTqAcuv2S+aEU3QFnEX0UTMSYSuDg==","signed_message":"bundle_sha256_bytes","signed_at":"2026-08-11T02:12:45.111657Z","bundle_sha256":"7ececf53c60baa90f2ec508fbc03c3c19bc82337113f26088d88c4b20cd3bf7a"}}