{"bundle_type":"pith_open_graph_bundle","bundle_version":"1.0","pith_number":"pith:2020:VAIRWT347ZGV25OE55ZQ76VEMR","short_pith_number":"pith:VAIRWT34","canonical_record":{"source":{"id":"2005.08158","kind":"arxiv","version":4},"metadata":{"license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","primary_cat":"cs.LG","submitted_at":"2020-05-17T03:41:19Z","cross_cats_sorted":["stat.ML"],"title_canon_sha256":"53d2ee26695fdf8822654d8aad2e6143d9c742dd0ca02591f3bf74c879d759fb","abstract_canon_sha256":"c0948194d5cdb13543a385ac2e492407f2465bfa82cd8b21f49b6335d1f51caa"},"schema_version":"1.0"},"canonical_sha256":"a8111b4f7cfe4d5d75c4ef730ffaa4647fd6aa4a55a77eaea76f9616822e8ee5","source":{"kind":"arxiv","id":"2005.08158","version":4},"source_aliases":[{"alias_kind":"arxiv","alias_value":"2005.08158","created_at":"2026-07-05T01:37:10Z"},{"alias_kind":"arxiv_version","alias_value":"2005.08158v4","created_at":"2026-07-05T01:37:10Z"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.2005.08158","created_at":"2026-07-05T01:37:10Z"},{"alias_kind":"pith_short_12","alias_value":"VAIRWT347ZGV","created_at":"2026-07-05T01:37:10Z"},{"alias_kind":"pith_short_16","alias_value":"VAIRWT347ZGV25OE","created_at":"2026-07-05T01:37:10Z"},{"alias_kind":"pith_short_8","alias_value":"VAIRWT34","created_at":"2026-07-05T01:37:10Z"}],"events":[{"event_type":"record_created","subject_pith_number":"pith:2020:VAIRWT347ZGV25OE55ZQ76VEMR","target":"record","payload":{"canonical_record":{"source":{"id":"2005.08158","kind":"arxiv","version":4},"metadata":{"license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","primary_cat":"cs.LG","submitted_at":"2020-05-17T03:41:19Z","cross_cats_sorted":["stat.ML"],"title_canon_sha256":"53d2ee26695fdf8822654d8aad2e6143d9c742dd0ca02591f3bf74c879d759fb","abstract_canon_sha256":"c0948194d5cdb13543a385ac2e492407f2465bfa82cd8b21f49b6335d1f51caa"},"schema_version":"1.0"},"canonical_sha256":"a8111b4f7cfe4d5d75c4ef730ffaa4647fd6aa4a55a77eaea76f9616822e8ee5","receipt":{"kind":"pith_receipt","key_id":"pith-v1-2026-05","algorithm":"ed25519","signed_at":"2026-07-05T01:37:10.426539Z","signature_b64":"homNElbaPs9ltyy5U5L5nH7xy88xyy2So0K/W8FB2zw3jxPwNaMl+ioZI/v30whPA8xMjhyKlhCjaCAcGhJqBg==","signed_message":"canonical_sha256_bytes","builder_version":"pith-number-builder-2026-05-17-v1","receipt_version":"0.3","canonical_sha256":"a8111b4f7cfe4d5d75c4ef730ffaa4647fd6aa4a55a77eaea76f9616822e8ee5","last_reissued_at":"2026-07-05T01:37:10.426034Z","signature_status":"signed_v1","first_computed_at":"2026-07-05T01:37:10.426034Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"source_kind":"arxiv","source_id":"2005.08158","source_version":4,"attestation_state":"computed"},"signer":{"signer_id":"pith.science","signer_type":"pith_registry","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"created_at":"2026-07-05T01:37:10Z","supersedes":[],"prev_event":null,"signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"ARwAXAVKRSlRdXVr8nAvxTNXKF8WqpSX9FoS2boaaGfw8gi89V0zt0xEyqUbaa/uj6f/RXZXQ5hE3+r4wQD6Cg==","signed_message":"open_graph_event_sha256_bytes","signed_at":"2026-07-31T22:15:47.495750Z"},"content_sha256":"51880c3734e7efb19474991eecbaaed6bc2ba076e87ed31bda118ac8e0685b9b","schema_version":"1.0","event_id":"sha256:51880c3734e7efb19474991eecbaaed6bc2ba076e87ed31bda118ac8e0685b9b"},{"event_type":"graph_snapshot","subject_pith_number":"pith:2020:VAIRWT347ZGV25OE55ZQ76VEMR","target":"graph","payload":{"graph_snapshot":{"paper":{"title":"Optimizing for the Future in Non-Stationary MDPs","license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","headline":"","cross_cats":["stat.ML"],"primary_cat":"cs.LG","authors_text":"Georgios Theocharous, Martha White, Philip S. Thomas, Shiv Shankar, Sridhar Mahadevan, Yash Chandak","submitted_at":"2020-05-17T03:41:19Z","abstract_excerpt":"Most reinforcement learning methods are based upon the key assumption that the transition dynamics and reward functions are fixed, that is, the underlying Markov decision process is stationary. However, in many real-world applications, this assumption is violated, and using existing algorithms may result in a performance lag. To proactively search for a good future policy, we present a policy gradient algorithm that maximizes a forecast of future performance. This forecast is obtained by fitting a curve to the counter-factual estimates of policy performance over time, without explicitly modeli"},"claims":{"count":0,"items":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"2005.08158","kind":"arxiv","version":4},"verdict":{"id":null,"model_set":{},"created_at":null,"strongest_claim":"","one_line_summary":"","pipeline_version":null,"weakest_assumption":"","pith_extraction_headline":""},"integrity":{"clean":true,"summary":{"advisory":0,"critical":0,"by_detector":{},"informational":0},"endpoint":"/pith/2005.08158/integrity.json","findings":[],"available":true,"detectors_run":[],"snapshot_sha256":"c28c3603d3b5d939e8dc4c7e95fa8dfce3d595e45f758748cecf8e644a296938"},"references":{"count":0,"sample":[],"resolved_work":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","internal_anchors":0},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"author_claims":{"count":0,"strong_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"builder_version":"pith-number-builder-2026-05-17-v1"},"verdict_id":null},"signer":{"signer_id":"pith.science","signer_type":"pith_registry","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"created_at":"2026-07-05T01:37:10Z","supersedes":[],"prev_event":null,"signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"dFHrVQfyP3NcCtkFSroPiNEY7DKYsoPME+Kwr86Vx7aUxAm2mfHaMPiH1Ah6ngcQAQtCUZBbk20lIPArZxKhAA==","signed_message":"open_graph_event_sha256_bytes","signed_at":"2026-07-31T22:15:47.496247Z"},"content_sha256":"3c4e69d23cccbf6a9f2bb7fa2f78cc4871c71653adb0f7e023bc1edc14ed0be6","schema_version":"1.0","event_id":"sha256:3c4e69d23cccbf6a9f2bb7fa2f78cc4871c71653adb0f7e023bc1edc14ed0be6"}],"timestamp_proofs":[],"mirror_hints":[{"mirror_type":"https","name":"Pith Resolver","base_url":"https://pith.science","bundle_url":"https://pith.science/pith/VAIRWT347ZGV25OE55ZQ76VEMR/bundle.json","state_url":"https://pith.science/pith/VAIRWT347ZGV25OE55ZQ76VEMR/state.json","well_known_bundle_url":"https://pith.science/.well-known/pith/VAIRWT347ZGV25OE55ZQ76VEMR/bundle.json","status":"primary"}],"public_keys":[{"key_id":"pith-v1-2026-05","algorithm":"ed25519","format":"raw","public_key_b64":"stVStoiQhXFxp4s2pdzPNoqVNBMojDU/fJ2db5S3CbM=","public_key_hex":"b2d552b68890857171a78b36a5dccf368a953413288c353f7c9d9d6f94b709b3","fingerprint_sha256_b32_first128bits":"RVFV5Z2OI2J3ZUO7ERDEBCYNKS","fingerprint_sha256_hex":"8d4b5ee74e4693bcd1df2446408b0d54","rotates_at":null,"url":"https://pith.science/pith-signing-key.json","notes":"Pith uses this Ed25519 key to sign canonical record SHA-256 digests. Verify with: ed25519_verify(public_key, message=canonical_sha256_bytes, signature=base64decode(signature_b64))."}],"merge_version":"pith-open-graph-merge-v1","built_at":"2026-07-31T22:15:47Z","links":{"resolver":"https://pith.science/pith/VAIRWT347ZGV25OE55ZQ76VEMR","bundle":"https://pith.science/pith/VAIRWT347ZGV25OE55ZQ76VEMR/bundle.json","state":"https://pith.science/pith/VAIRWT347ZGV25OE55ZQ76VEMR/state.json","well_known_bundle":"https://pith.science/.well-known/pith/VAIRWT347ZGV25OE55ZQ76VEMR/bundle.json"},"state":{"state_type":"pith_open_graph_state","state_version":"1.0","pith_number":"pith:2020:VAIRWT347ZGV25OE55ZQ76VEMR","merge_version":"pith-open-graph-merge-v1","event_count":2,"valid_event_count":2,"invalid_event_count":0,"equivocation_count":0,"current":{"canonical_record":{"metadata":{"abstract_canon_sha256":"c0948194d5cdb13543a385ac2e492407f2465bfa82cd8b21f49b6335d1f51caa","cross_cats_sorted":["stat.ML"],"license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","primary_cat":"cs.LG","submitted_at":"2020-05-17T03:41:19Z","title_canon_sha256":"53d2ee26695fdf8822654d8aad2e6143d9c742dd0ca02591f3bf74c879d759fb"},"schema_version":"1.0","source":{"id":"2005.08158","kind":"arxiv","version":4}},"source_aliases":[{"alias_kind":"arxiv","alias_value":"2005.08158","created_at":"2026-07-05T01:37:10Z"},{"alias_kind":"arxiv_version","alias_value":"2005.08158v4","created_at":"2026-07-05T01:37:10Z"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.2005.08158","created_at":"2026-07-05T01:37:10Z"},{"alias_kind":"pith_short_12","alias_value":"VAIRWT347ZGV","created_at":"2026-07-05T01:37:10Z"},{"alias_kind":"pith_short_16","alias_value":"VAIRWT347ZGV25OE","created_at":"2026-07-05T01:37:10Z"},{"alias_kind":"pith_short_8","alias_value":"VAIRWT34","created_at":"2026-07-05T01:37:10Z"}],"graph_snapshots":[{"event_id":"sha256:3c4e69d23cccbf6a9f2bb7fa2f78cc4871c71653adb0f7e023bc1edc14ed0be6","target":"graph","created_at":"2026-07-05T01:37:10Z","signer":{"key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signer_id":"pith.science","signer_type":"pith_registry"},"payload":{"graph_snapshot":{"author_claims":{"count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","strong_count":0},"builder_version":"pith-number-builder-2026-05-17-v1","claims":{"count":0,"items":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"integrity":{"available":true,"clean":true,"detectors_run":[],"endpoint":"/pith/2005.08158/integrity.json","findings":[],"snapshot_sha256":"c28c3603d3b5d939e8dc4c7e95fa8dfce3d595e45f758748cecf8e644a296938","summary":{"advisory":0,"by_detector":{},"critical":0,"informational":0}},"paper":{"abstract_excerpt":"Most reinforcement learning methods are based upon the key assumption that the transition dynamics and reward functions are fixed, that is, the underlying Markov decision process is stationary. However, in many real-world applications, this assumption is violated, and using existing algorithms may result in a performance lag. To proactively search for a good future policy, we present a policy gradient algorithm that maximizes a forecast of future performance. This forecast is obtained by fitting a curve to the counter-factual estimates of policy performance over time, without explicitly modeli","authors_text":"Georgios Theocharous, Martha White, Philip S. Thomas, Shiv Shankar, Sridhar Mahadevan, Yash Chandak","cross_cats":["stat.ML"],"headline":"","license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","primary_cat":"cs.LG","submitted_at":"2020-05-17T03:41:19Z","title":"Optimizing for the Future in Non-Stationary MDPs"},"references":{"count":0,"internal_anchors":0,"resolved_work":0,"sample":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"2005.08158","kind":"arxiv","version":4},"verdict":{"created_at":null,"id":null,"model_set":{},"one_line_summary":"","pipeline_version":null,"pith_extraction_headline":"","strongest_claim":"","weakest_assumption":""}},"verdict_id":null}}],"author_attestations":[],"timestamp_anchors":[],"storage_attestations":[],"citation_signatures":[],"replication_records":[],"corrections":[],"mirror_hints":[],"record_created":{"event_id":"sha256:51880c3734e7efb19474991eecbaaed6bc2ba076e87ed31bda118ac8e0685b9b","target":"record","created_at":"2026-07-05T01:37:10Z","signer":{"key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signer_id":"pith.science","signer_type":"pith_registry"},"payload":{"attestation_state":"computed","canonical_record":{"metadata":{"abstract_canon_sha256":"c0948194d5cdb13543a385ac2e492407f2465bfa82cd8b21f49b6335d1f51caa","cross_cats_sorted":["stat.ML"],"license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","primary_cat":"cs.LG","submitted_at":"2020-05-17T03:41:19Z","title_canon_sha256":"53d2ee26695fdf8822654d8aad2e6143d9c742dd0ca02591f3bf74c879d759fb"},"schema_version":"1.0","source":{"id":"2005.08158","kind":"arxiv","version":4}},"canonical_sha256":"a8111b4f7cfe4d5d75c4ef730ffaa4647fd6aa4a55a77eaea76f9616822e8ee5","receipt":{"algorithm":"ed25519","builder_version":"pith-number-builder-2026-05-17-v1","canonical_sha256":"a8111b4f7cfe4d5d75c4ef730ffaa4647fd6aa4a55a77eaea76f9616822e8ee5","first_computed_at":"2026-07-05T01:37:10.426034Z","key_id":"pith-v1-2026-05","kind":"pith_receipt","last_reissued_at":"2026-07-05T01:37:10.426034Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","receipt_version":"0.3","signature_b64":"homNElbaPs9ltyy5U5L5nH7xy88xyy2So0K/W8FB2zw3jxPwNaMl+ioZI/v30whPA8xMjhyKlhCjaCAcGhJqBg==","signature_status":"signed_v1","signed_at":"2026-07-05T01:37:10.426539Z","signed_message":"canonical_sha256_bytes"},"source_id":"2005.08158","source_kind":"arxiv","source_version":4}}},"equivocations":[],"invalid_events":[],"applied_event_ids":["sha256:51880c3734e7efb19474991eecbaaed6bc2ba076e87ed31bda118ac8e0685b9b","sha256:3c4e69d23cccbf6a9f2bb7fa2f78cc4871c71653adb0f7e023bc1edc14ed0be6"],"state_sha256":"fb04ba32f931a4d28690121b3bc214a4be7b5d88057884e62663c7fbd77ff1c3"},"bundle_signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"5oAaVIVGALNTO/EEOMdmTR+0K0myWdu24rnBQEqHIbaLlKrPtKcPA1/Hl1mZ2fYVdx4DUb6zDxyxQzhMRVYeBQ==","signed_message":"bundle_sha256_bytes","signed_at":"2026-07-31T22:15:47.501990Z","bundle_sha256":"857d880208d2f2df9048f90b29b44c246e7050843c8b3ed12fc1bfc88f5bcaf6"}}