{"bundle_type":"pith_open_graph_bundle","bundle_version":"1.0","pith_number":"pith:2021:UGJ5BVICGYW74C4DCOR46VIJHY","short_pith_number":"pith:UGJ5BVIC","canonical_record":{"source":{"id":"2107.12216","kind":"arxiv","version":2},"metadata":{"license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","primary_cat":"cs.LG","submitted_at":"2021-07-26T13:48:23Z","cross_cats_sorted":["cs.AI"],"title_canon_sha256":"e279daf5892255a0e96bda2a422f91005ea7f6e55e70794f00d73960c1b14e76","abstract_canon_sha256":"b5ff69a89bfc561825e31a8795c742c78867a1170265ae595fa95681838e5f30"},"schema_version":"1.0"},"canonical_sha256":"a193d0d502362dfe0b8313a3cf55093e225bdcf039b60a98abeb69bec7cfe7a5","source":{"kind":"arxiv","id":"2107.12216","version":2},"source_aliases":[{"alias_kind":"arxiv","alias_value":"2107.12216","created_at":"2026-07-05T03:03:29Z"},{"alias_kind":"arxiv_version","alias_value":"2107.12216v2","created_at":"2026-07-05T03:03:29Z"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.2107.12216","created_at":"2026-07-05T03:03:29Z"},{"alias_kind":"pith_short_12","alias_value":"UGJ5BVICGYW7","created_at":"2026-07-05T03:03:29Z"},{"alias_kind":"pith_short_16","alias_value":"UGJ5BVICGYW74C4D","created_at":"2026-07-05T03:03:29Z"},{"alias_kind":"pith_short_8","alias_value":"UGJ5BVIC","created_at":"2026-07-05T03:03:29Z"}],"events":[{"event_type":"record_created","subject_pith_number":"pith:2021:UGJ5BVICGYW74C4DCOR46VIJHY","target":"record","payload":{"canonical_record":{"source":{"id":"2107.12216","kind":"arxiv","version":2},"metadata":{"license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","primary_cat":"cs.LG","submitted_at":"2021-07-26T13:48:23Z","cross_cats_sorted":["cs.AI"],"title_canon_sha256":"e279daf5892255a0e96bda2a422f91005ea7f6e55e70794f00d73960c1b14e76","abstract_canon_sha256":"b5ff69a89bfc561825e31a8795c742c78867a1170265ae595fa95681838e5f30"},"schema_version":"1.0"},"canonical_sha256":"a193d0d502362dfe0b8313a3cf55093e225bdcf039b60a98abeb69bec7cfe7a5","receipt":{"kind":"pith_receipt","key_id":"pith-v1-2026-05","algorithm":"ed25519","signed_at":"2026-07-05T03:03:29.683881Z","signature_b64":"IwVOd4EVFc4cemfjZ4fKez77iMcXdvdacFV4rNyryqH1me9p2k/3L9LuptMsZ8qEgRgrnduL7/HdeKlXTTlYDQ==","signed_message":"canonical_sha256_bytes","builder_version":"pith-number-builder-2026-05-17-v1","receipt_version":"0.3","canonical_sha256":"a193d0d502362dfe0b8313a3cf55093e225bdcf039b60a98abeb69bec7cfe7a5","last_reissued_at":"2026-07-05T03:03:29.683330Z","signature_status":"signed_v1","first_computed_at":"2026-07-05T03:03:29.683330Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"source_kind":"arxiv","source_id":"2107.12216","source_version":2,"attestation_state":"computed"},"signer":{"signer_id":"pith.science","signer_type":"pith_registry","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"created_at":"2026-07-05T03:03:29Z","supersedes":[],"prev_event":null,"signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"yAPqqEIwyYalVL7gm93VO34wxVXFnxBA2QeUeoC/ixutRdqfMWxjrb9R/rAkMnBrNZufSGQiwyxr3AcPU5A7CQ==","signed_message":"open_graph_event_sha256_bytes","signed_at":"2026-07-07T09:20:12.147450Z"},"content_sha256":"68de7b8cce31c7fe435571fd5dd1c676aabc4d274726a156d17944d9601b29f7","schema_version":"1.0","event_id":"sha256:68de7b8cce31c7fe435571fd5dd1c676aabc4d274726a156d17944d9601b29f7"},{"event_type":"graph_snapshot","subject_pith_number":"pith:2021:UGJ5BVICGYW74C4DCOR46VIJHY","target":"graph","payload":{"graph_snapshot":{"paper":{"title":"Hindsight Value Function for Variance Reduction in Stochastic Dynamic Environment","license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","headline":"","cross_cats":["cs.AI"],"primary_cat":"cs.LG","authors_text":"Jiaming Guo, Qi Guo, Qi Yi, Rui Zhang, Shaohui Peng, Xing Hu, Xishan Zhang, Yunji Chen, Zidong Du","submitted_at":"2021-07-26T13:48:23Z","abstract_excerpt":"Policy gradient methods are appealing in deep reinforcement learning but suffer from high variance of gradient estimate. To reduce the variance, the state value function is applied commonly. However, the effect of the state value function becomes limited in stochastic dynamic environments, where the unexpected state dynamics and rewards will increase the variance. In this paper, we propose to replace the state value function with a novel hindsight value function, which leverages the information from the future to reduce the variance of the gradient estimate for stochastic dynamic environments."},"claims":{"count":0,"items":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"2107.12216","kind":"arxiv","version":2},"verdict":{"id":null,"model_set":{},"created_at":null,"strongest_claim":"","one_line_summary":"","pipeline_version":null,"weakest_assumption":"","pith_extraction_headline":""},"integrity":{"clean":true,"summary":{"advisory":0,"critical":0,"by_detector":{},"informational":0},"endpoint":"/pith/2107.12216/integrity.json","findings":[],"available":true,"detectors_run":[],"snapshot_sha256":"c28c3603d3b5d939e8dc4c7e95fa8dfce3d595e45f758748cecf8e644a296938"},"references":{"count":0,"sample":[],"resolved_work":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","internal_anchors":0},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"author_claims":{"count":0,"strong_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"builder_version":"pith-number-builder-2026-05-17-v1"},"verdict_id":null},"signer":{"signer_id":"pith.science","signer_type":"pith_registry","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"created_at":"2026-07-05T03:03:29Z","supersedes":[],"prev_event":null,"signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"nURvThCKIzzOSIGDlr5gFSDoTO4TNPU35a7mpFtw/AIzA0fTcUPrODag6RhGpcSw3S8crrmzWWE7RjBgb1TIAA==","signed_message":"open_graph_event_sha256_bytes","signed_at":"2026-07-07T09:20:12.147829Z"},"content_sha256":"e26f3db1325eb0262a168072502b06e08c87a4fc0694233e842d1159ddf244f5","schema_version":"1.0","event_id":"sha256:e26f3db1325eb0262a168072502b06e08c87a4fc0694233e842d1159ddf244f5"}],"timestamp_proofs":[],"mirror_hints":[{"mirror_type":"https","name":"Pith Resolver","base_url":"https://pith.science","bundle_url":"https://pith.science/pith/UGJ5BVICGYW74C4DCOR46VIJHY/bundle.json","state_url":"https://pith.science/pith/UGJ5BVICGYW74C4DCOR46VIJHY/state.json","well_known_bundle_url":"https://pith.science/.well-known/pith/UGJ5BVICGYW74C4DCOR46VIJHY/bundle.json","status":"primary"}],"public_keys":[{"key_id":"pith-v1-2026-05","algorithm":"ed25519","format":"raw","public_key_b64":"stVStoiQhXFxp4s2pdzPNoqVNBMojDU/fJ2db5S3CbM=","public_key_hex":"b2d552b68890857171a78b36a5dccf368a953413288c353f7c9d9d6f94b709b3","fingerprint_sha256_b32_first128bits":"RVFV5Z2OI2J3ZUO7ERDEBCYNKS","fingerprint_sha256_hex":"8d4b5ee74e4693bcd1df2446408b0d54","rotates_at":null,"url":"https://pith.science/pith-signing-key.json","notes":"Pith uses this Ed25519 key to sign canonical record SHA-256 digests. Verify with: ed25519_verify(public_key, message=canonical_sha256_bytes, signature=base64decode(signature_b64))."}],"merge_version":"pith-open-graph-merge-v1","built_at":"2026-07-07T09:20:12Z","links":{"resolver":"https://pith.science/pith/UGJ5BVICGYW74C4DCOR46VIJHY","bundle":"https://pith.science/pith/UGJ5BVICGYW74C4DCOR46VIJHY/bundle.json","state":"https://pith.science/pith/UGJ5BVICGYW74C4DCOR46VIJHY/state.json","well_known_bundle":"https://pith.science/.well-known/pith/UGJ5BVICGYW74C4DCOR46VIJHY/bundle.json"},"state":{"state_type":"pith_open_graph_state","state_version":"1.0","pith_number":"pith:2021:UGJ5BVICGYW74C4DCOR46VIJHY","merge_version":"pith-open-graph-merge-v1","event_count":2,"valid_event_count":2,"invalid_event_count":0,"equivocation_count":0,"current":{"canonical_record":{"metadata":{"abstract_canon_sha256":"b5ff69a89bfc561825e31a8795c742c78867a1170265ae595fa95681838e5f30","cross_cats_sorted":["cs.AI"],"license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","primary_cat":"cs.LG","submitted_at":"2021-07-26T13:48:23Z","title_canon_sha256":"e279daf5892255a0e96bda2a422f91005ea7f6e55e70794f00d73960c1b14e76"},"schema_version":"1.0","source":{"id":"2107.12216","kind":"arxiv","version":2}},"source_aliases":[{"alias_kind":"arxiv","alias_value":"2107.12216","created_at":"2026-07-05T03:03:29Z"},{"alias_kind":"arxiv_version","alias_value":"2107.12216v2","created_at":"2026-07-05T03:03:29Z"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.2107.12216","created_at":"2026-07-05T03:03:29Z"},{"alias_kind":"pith_short_12","alias_value":"UGJ5BVICGYW7","created_at":"2026-07-05T03:03:29Z"},{"alias_kind":"pith_short_16","alias_value":"UGJ5BVICGYW74C4D","created_at":"2026-07-05T03:03:29Z"},{"alias_kind":"pith_short_8","alias_value":"UGJ5BVIC","created_at":"2026-07-05T03:03:29Z"}],"graph_snapshots":[{"event_id":"sha256:e26f3db1325eb0262a168072502b06e08c87a4fc0694233e842d1159ddf244f5","target":"graph","created_at":"2026-07-05T03:03:29Z","signer":{"key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signer_id":"pith.science","signer_type":"pith_registry"},"payload":{"graph_snapshot":{"author_claims":{"count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","strong_count":0},"builder_version":"pith-number-builder-2026-05-17-v1","claims":{"count":0,"items":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"integrity":{"available":true,"clean":true,"detectors_run":[],"endpoint":"/pith/2107.12216/integrity.json","findings":[],"snapshot_sha256":"c28c3603d3b5d939e8dc4c7e95fa8dfce3d595e45f758748cecf8e644a296938","summary":{"advisory":0,"by_detector":{},"critical":0,"informational":0}},"paper":{"abstract_excerpt":"Policy gradient methods are appealing in deep reinforcement learning but suffer from high variance of gradient estimate. To reduce the variance, the state value function is applied commonly. However, the effect of the state value function becomes limited in stochastic dynamic environments, where the unexpected state dynamics and rewards will increase the variance. In this paper, we propose to replace the state value function with a novel hindsight value function, which leverages the information from the future to reduce the variance of the gradient estimate for stochastic dynamic environments.","authors_text":"Jiaming Guo, Qi Guo, Qi Yi, Rui Zhang, Shaohui Peng, Xing Hu, Xishan Zhang, Yunji Chen, Zidong Du","cross_cats":["cs.AI"],"headline":"","license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","primary_cat":"cs.LG","submitted_at":"2021-07-26T13:48:23Z","title":"Hindsight Value Function for Variance Reduction in Stochastic Dynamic Environment"},"references":{"count":0,"internal_anchors":0,"resolved_work":0,"sample":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"2107.12216","kind":"arxiv","version":2},"verdict":{"created_at":null,"id":null,"model_set":{},"one_line_summary":"","pipeline_version":null,"pith_extraction_headline":"","strongest_claim":"","weakest_assumption":""}},"verdict_id":null}}],"author_attestations":[],"timestamp_anchors":[],"storage_attestations":[],"citation_signatures":[],"replication_records":[],"corrections":[],"mirror_hints":[],"record_created":{"event_id":"sha256:68de7b8cce31c7fe435571fd5dd1c676aabc4d274726a156d17944d9601b29f7","target":"record","created_at":"2026-07-05T03:03:29Z","signer":{"key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signer_id":"pith.science","signer_type":"pith_registry"},"payload":{"attestation_state":"computed","canonical_record":{"metadata":{"abstract_canon_sha256":"b5ff69a89bfc561825e31a8795c742c78867a1170265ae595fa95681838e5f30","cross_cats_sorted":["cs.AI"],"license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","primary_cat":"cs.LG","submitted_at":"2021-07-26T13:48:23Z","title_canon_sha256":"e279daf5892255a0e96bda2a422f91005ea7f6e55e70794f00d73960c1b14e76"},"schema_version":"1.0","source":{"id":"2107.12216","kind":"arxiv","version":2}},"canonical_sha256":"a193d0d502362dfe0b8313a3cf55093e225bdcf039b60a98abeb69bec7cfe7a5","receipt":{"algorithm":"ed25519","builder_version":"pith-number-builder-2026-05-17-v1","canonical_sha256":"a193d0d502362dfe0b8313a3cf55093e225bdcf039b60a98abeb69bec7cfe7a5","first_computed_at":"2026-07-05T03:03:29.683330Z","key_id":"pith-v1-2026-05","kind":"pith_receipt","last_reissued_at":"2026-07-05T03:03:29.683330Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","receipt_version":"0.3","signature_b64":"IwVOd4EVFc4cemfjZ4fKez77iMcXdvdacFV4rNyryqH1me9p2k/3L9LuptMsZ8qEgRgrnduL7/HdeKlXTTlYDQ==","signature_status":"signed_v1","signed_at":"2026-07-05T03:03:29.683881Z","signed_message":"canonical_sha256_bytes"},"source_id":"2107.12216","source_kind":"arxiv","source_version":2}}},"equivocations":[],"invalid_events":[],"applied_event_ids":["sha256:68de7b8cce31c7fe435571fd5dd1c676aabc4d274726a156d17944d9601b29f7","sha256:e26f3db1325eb0262a168072502b06e08c87a4fc0694233e842d1159ddf244f5"],"state_sha256":"3b8260bc7ad7537940cb1cd66e95b64d0e480e310e590c8af5d32ca09ae90f6d"},"bundle_signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"0tfaRWw4zoh3Ogj8BiuHqHbnAc/N7C4Nl9GTTDddogZ1zJfZlMfZ/hnfN6CpLNxodV6t2K4nPLgIiNV/weJkBg==","signed_message":"bundle_sha256_bytes","signed_at":"2026-07-07T09:20:12.149798Z","bundle_sha256":"229e5549368ce5968ad0ac7e8e98c8c9b065aa6a8b602216ea82658336e46744"}}