{"bundle_type":"pith_open_graph_bundle","bundle_version":"1.0","pith_number":"pith:2019:GBP2Q6KOMO3ZXMGSCQIP7BORYV","short_pith_number":"pith:GBP2Q6KO","canonical_record":{"source":{"id":"1912.05104","kind":"arxiv","version":1},"metadata":{"license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","primary_cat":"cs.LG","submitted_at":"2019-12-11T03:40:46Z","cross_cats_sorted":["cs.AI","stat.ML"],"title_canon_sha256":"c2e7f6279c91c4cf006221f89f94fe6187d609a5fb1ddd3c7945f81e82404000","abstract_canon_sha256":"49243c772f9968ffadcd868abec86743b784adb31b03adc8ce854e41eb1efd81"},"schema_version":"1.0"},"canonical_sha256":"305fa8794e63b79bb0d21410ff85d1c55d7a71a2a3fb02926ee02f88315d6fbe","source":{"kind":"arxiv","id":"1912.05104","version":1},"source_aliases":[{"alias_kind":"arxiv","alias_value":"1912.05104","created_at":"2026-07-05T00:25:32Z"},{"alias_kind":"arxiv_version","alias_value":"1912.05104v1","created_at":"2026-07-05T00:25:32Z"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.1912.05104","created_at":"2026-07-05T00:25:32Z"},{"alias_kind":"pith_short_12","alias_value":"GBP2Q6KOMO3Z","created_at":"2026-07-05T00:25:32Z"},{"alias_kind":"pith_short_16","alias_value":"GBP2Q6KOMO3ZXMGS","created_at":"2026-07-05T00:25:32Z"},{"alias_kind":"pith_short_8","alias_value":"GBP2Q6KO","created_at":"2026-07-05T00:25:32Z"}],"events":[{"event_type":"record_created","subject_pith_number":"pith:2019:GBP2Q6KOMO3ZXMGSCQIP7BORYV","target":"record","payload":{"canonical_record":{"source":{"id":"1912.05104","kind":"arxiv","version":1},"metadata":{"license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","primary_cat":"cs.LG","submitted_at":"2019-12-11T03:40:46Z","cross_cats_sorted":["cs.AI","stat.ML"],"title_canon_sha256":"c2e7f6279c91c4cf006221f89f94fe6187d609a5fb1ddd3c7945f81e82404000","abstract_canon_sha256":"49243c772f9968ffadcd868abec86743b784adb31b03adc8ce854e41eb1efd81"},"schema_version":"1.0"},"canonical_sha256":"305fa8794e63b79bb0d21410ff85d1c55d7a71a2a3fb02926ee02f88315d6fbe","receipt":{"kind":"pith_receipt","key_id":"pith-v1-2026-05","algorithm":"ed25519","signed_at":"2026-07-05T00:25:32.139128Z","signature_b64":"to2gFc3yKsrMIdjXcPCXMT9Z0aC7G1PteWm4WT+arpYuR3O8EvAma358brUit86N870GN0ct7fgdibst4kSPAA==","signed_message":"canonical_sha256_bytes","builder_version":"pith-number-builder-2026-05-17-v1","receipt_version":"0.3","canonical_sha256":"305fa8794e63b79bb0d21410ff85d1c55d7a71a2a3fb02926ee02f88315d6fbe","last_reissued_at":"2026-07-05T00:25:32.138662Z","signature_status":"signed_v1","first_computed_at":"2026-07-05T00:25:32.138662Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"source_kind":"arxiv","source_id":"1912.05104","source_version":1,"attestation_state":"computed"},"signer":{"signer_id":"pith.science","signer_type":"pith_registry","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"created_at":"2026-07-05T00:25:32Z","supersedes":[],"prev_event":null,"signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"J1eJ1SlB3QsAZ8b5GhUYAb8cRkmP6MBLlXhgHc2B1XBcO5oqmtj0ouIcptLoBnQOL7dsy0ngws8zhRo6w7BsAg==","signed_message":"open_graph_event_sha256_bytes","signed_at":"2026-08-06T06:46:33.484525Z"},"content_sha256":"55c4d813c3a89c97f368df39ba3f9b19027b9aa2bb95b6a9d03b98609d424e59","schema_version":"1.0","event_id":"sha256:55c4d813c3a89c97f368df39ba3f9b19027b9aa2bb95b6a9d03b98609d424e59"},{"event_type":"graph_snapshot","subject_pith_number":"pith:2019:GBP2Q6KOMO3ZXMGSCQIP7BORYV","target":"graph","payload":{"graph_snapshot":{"paper":{"title":"Entropy Regularization with Discounted Future State Distribution in Policy Gradient Methods","license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","headline":"","cross_cats":["cs.AI","stat.ML"],"primary_cat":"cs.LG","authors_text":"Doina Precup, Pierre-Luc Bacon, Raihan Seraj, Riashat Islam","submitted_at":"2019-12-11T03:40:46Z","abstract_excerpt":"The policy gradient theorem is defined based on an objective with respect to the initial distribution over states. In the discounted case, this results in policies that are optimal for one distribution over initial states, but may not be uniformly optimal for others, no matter where the agent starts from. Furthermore, to obtain unbiased gradient estimates, the starting point of the policy gradient estimator requires sampling states from a normalized discounted weighting of states. However, the difficulty of estimating the normalized discounted weighting of states, or the stationary state distr"},"claims":{"count":0,"items":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"1912.05104","kind":"arxiv","version":1},"verdict":{"id":null,"model_set":{},"created_at":null,"strongest_claim":"","one_line_summary":"","pipeline_version":null,"weakest_assumption":"","pith_extraction_headline":""},"integrity":{"clean":true,"summary":{"advisory":0,"critical":0,"by_detector":{},"informational":0},"endpoint":"/pith/1912.05104/integrity.json","findings":[],"available":true,"detectors_run":[],"snapshot_sha256":"c28c3603d3b5d939e8dc4c7e95fa8dfce3d595e45f758748cecf8e644a296938"},"references":{"count":0,"sample":[],"resolved_work":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","internal_anchors":0},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"author_claims":{"count":0,"strong_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"builder_version":"pith-number-builder-2026-05-17-v1"},"verdict_id":null},"signer":{"signer_id":"pith.science","signer_type":"pith_registry","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"created_at":"2026-07-05T00:25:32Z","supersedes":[],"prev_event":null,"signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"fpag5iTUXQNMJdClrbHTxhyyV1UKFkME5G5BFDgwzPSORppp06dadivtf5+kqzalGN3D94h/eSDQEgEDiBWYCw==","signed_message":"open_graph_event_sha256_bytes","signed_at":"2026-08-06T06:46:33.485057Z"},"content_sha256":"6eaacc78e11d18fe4effdfbc8781d57244e3a02a7a75e65d53c05d14a9cf14a5","schema_version":"1.0","event_id":"sha256:6eaacc78e11d18fe4effdfbc8781d57244e3a02a7a75e65d53c05d14a9cf14a5"}],"timestamp_proofs":[],"mirror_hints":[{"mirror_type":"https","name":"Pith Resolver","base_url":"https://pith.science","bundle_url":"https://pith.science/pith/GBP2Q6KOMO3ZXMGSCQIP7BORYV/bundle.json","state_url":"https://pith.science/pith/GBP2Q6KOMO3ZXMGSCQIP7BORYV/state.json","well_known_bundle_url":"https://pith.science/.well-known/pith/GBP2Q6KOMO3ZXMGSCQIP7BORYV/bundle.json","status":"primary"}],"public_keys":[{"key_id":"pith-v1-2026-05","algorithm":"ed25519","format":"raw","public_key_b64":"stVStoiQhXFxp4s2pdzPNoqVNBMojDU/fJ2db5S3CbM=","public_key_hex":"b2d552b68890857171a78b36a5dccf368a953413288c353f7c9d9d6f94b709b3","fingerprint_sha256_b32_first128bits":"RVFV5Z2OI2J3ZUO7ERDEBCYNKS","fingerprint_sha256_hex":"8d4b5ee74e4693bcd1df2446408b0d54","rotates_at":null,"url":"https://pith.science/pith-signing-key.json","notes":"Pith uses this Ed25519 key to sign canonical record SHA-256 digests. Verify with: ed25519_verify(public_key, message=canonical_sha256_bytes, signature=base64decode(signature_b64))."}],"merge_version":"pith-open-graph-merge-v1","built_at":"2026-08-06T06:46:33Z","links":{"resolver":"https://pith.science/pith/GBP2Q6KOMO3ZXMGSCQIP7BORYV","bundle":"https://pith.science/pith/GBP2Q6KOMO3ZXMGSCQIP7BORYV/bundle.json","state":"https://pith.science/pith/GBP2Q6KOMO3ZXMGSCQIP7BORYV/state.json","well_known_bundle":"https://pith.science/.well-known/pith/GBP2Q6KOMO3ZXMGSCQIP7BORYV/bundle.json"},"state":{"state_type":"pith_open_graph_state","state_version":"1.0","pith_number":"pith:2019:GBP2Q6KOMO3ZXMGSCQIP7BORYV","merge_version":"pith-open-graph-merge-v1","event_count":2,"valid_event_count":2,"invalid_event_count":0,"equivocation_count":0,"current":{"canonical_record":{"metadata":{"abstract_canon_sha256":"49243c772f9968ffadcd868abec86743b784adb31b03adc8ce854e41eb1efd81","cross_cats_sorted":["cs.AI","stat.ML"],"license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","primary_cat":"cs.LG","submitted_at":"2019-12-11T03:40:46Z","title_canon_sha256":"c2e7f6279c91c4cf006221f89f94fe6187d609a5fb1ddd3c7945f81e82404000"},"schema_version":"1.0","source":{"id":"1912.05104","kind":"arxiv","version":1}},"source_aliases":[{"alias_kind":"arxiv","alias_value":"1912.05104","created_at":"2026-07-05T00:25:32Z"},{"alias_kind":"arxiv_version","alias_value":"1912.05104v1","created_at":"2026-07-05T00:25:32Z"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.1912.05104","created_at":"2026-07-05T00:25:32Z"},{"alias_kind":"pith_short_12","alias_value":"GBP2Q6KOMO3Z","created_at":"2026-07-05T00:25:32Z"},{"alias_kind":"pith_short_16","alias_value":"GBP2Q6KOMO3ZXMGS","created_at":"2026-07-05T00:25:32Z"},{"alias_kind":"pith_short_8","alias_value":"GBP2Q6KO","created_at":"2026-07-05T00:25:32Z"}],"graph_snapshots":[{"event_id":"sha256:6eaacc78e11d18fe4effdfbc8781d57244e3a02a7a75e65d53c05d14a9cf14a5","target":"graph","created_at":"2026-07-05T00:25:32Z","signer":{"key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signer_id":"pith.science","signer_type":"pith_registry"},"payload":{"graph_snapshot":{"author_claims":{"count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","strong_count":0},"builder_version":"pith-number-builder-2026-05-17-v1","claims":{"count":0,"items":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"integrity":{"available":true,"clean":true,"detectors_run":[],"endpoint":"/pith/1912.05104/integrity.json","findings":[],"snapshot_sha256":"c28c3603d3b5d939e8dc4c7e95fa8dfce3d595e45f758748cecf8e644a296938","summary":{"advisory":0,"by_detector":{},"critical":0,"informational":0}},"paper":{"abstract_excerpt":"The policy gradient theorem is defined based on an objective with respect to the initial distribution over states. In the discounted case, this results in policies that are optimal for one distribution over initial states, but may not be uniformly optimal for others, no matter where the agent starts from. Furthermore, to obtain unbiased gradient estimates, the starting point of the policy gradient estimator requires sampling states from a normalized discounted weighting of states. However, the difficulty of estimating the normalized discounted weighting of states, or the stationary state distr","authors_text":"Doina Precup, Pierre-Luc Bacon, Raihan Seraj, Riashat Islam","cross_cats":["cs.AI","stat.ML"],"headline":"","license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","primary_cat":"cs.LG","submitted_at":"2019-12-11T03:40:46Z","title":"Entropy Regularization with Discounted Future State Distribution in Policy Gradient Methods"},"references":{"count":0,"internal_anchors":0,"resolved_work":0,"sample":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"1912.05104","kind":"arxiv","version":1},"verdict":{"created_at":null,"id":null,"model_set":{},"one_line_summary":"","pipeline_version":null,"pith_extraction_headline":"","strongest_claim":"","weakest_assumption":""}},"verdict_id":null}}],"author_attestations":[],"timestamp_anchors":[],"storage_attestations":[],"citation_signatures":[],"replication_records":[],"corrections":[],"mirror_hints":[],"record_created":{"event_id":"sha256:55c4d813c3a89c97f368df39ba3f9b19027b9aa2bb95b6a9d03b98609d424e59","target":"record","created_at":"2026-07-05T00:25:32Z","signer":{"key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signer_id":"pith.science","signer_type":"pith_registry"},"payload":{"attestation_state":"computed","canonical_record":{"metadata":{"abstract_canon_sha256":"49243c772f9968ffadcd868abec86743b784adb31b03adc8ce854e41eb1efd81","cross_cats_sorted":["cs.AI","stat.ML"],"license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","primary_cat":"cs.LG","submitted_at":"2019-12-11T03:40:46Z","title_canon_sha256":"c2e7f6279c91c4cf006221f89f94fe6187d609a5fb1ddd3c7945f81e82404000"},"schema_version":"1.0","source":{"id":"1912.05104","kind":"arxiv","version":1}},"canonical_sha256":"305fa8794e63b79bb0d21410ff85d1c55d7a71a2a3fb02926ee02f88315d6fbe","receipt":{"algorithm":"ed25519","builder_version":"pith-number-builder-2026-05-17-v1","canonical_sha256":"305fa8794e63b79bb0d21410ff85d1c55d7a71a2a3fb02926ee02f88315d6fbe","first_computed_at":"2026-07-05T00:25:32.138662Z","key_id":"pith-v1-2026-05","kind":"pith_receipt","last_reissued_at":"2026-07-05T00:25:32.138662Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","receipt_version":"0.3","signature_b64":"to2gFc3yKsrMIdjXcPCXMT9Z0aC7G1PteWm4WT+arpYuR3O8EvAma358brUit86N870GN0ct7fgdibst4kSPAA==","signature_status":"signed_v1","signed_at":"2026-07-05T00:25:32.139128Z","signed_message":"canonical_sha256_bytes"},"source_id":"1912.05104","source_kind":"arxiv","source_version":1}}},"equivocations":[],"invalid_events":[],"applied_event_ids":["sha256:55c4d813c3a89c97f368df39ba3f9b19027b9aa2bb95b6a9d03b98609d424e59","sha256:6eaacc78e11d18fe4effdfbc8781d57244e3a02a7a75e65d53c05d14a9cf14a5"],"state_sha256":"73e67941e3be8a423b99e12ea204b2fc11f88826a0a5b1d59b8869183679f0f6"},"bundle_signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"RfPbCGzsOkYwue8T8S63aIWFdMpfas7cJFABtYVSTwwqGe3FW24QTp0o/DF1GwYVGpeaISv5BLPj2AvawzISDg==","signed_message":"bundle_sha256_bytes","signed_at":"2026-08-06T06:46:33.491147Z","bundle_sha256":"a9982d373ce160ccc3b2c9ef3658e69fcf3cda76555570eec92c18a94c333506"}}