{"bundle_type":"pith_open_graph_bundle","bundle_version":"1.0","pith_number":"pith:2019:WYF4VB3MPF7BWKOWBIDKGH5RPZ","short_pith_number":"pith:WYF4VB3M","canonical_record":{"source":{"id":"1909.12238","kind":"arxiv","version":1},"metadata":{"license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","primary_cat":"cs.AI","submitted_at":"2019-09-26T16:34:22Z","cross_cats_sorted":["cs.LG"],"title_canon_sha256":"96d3437d089f1d08f592e44f0881b3be56bb73624d1b78feedd5289db17b4ed3","abstract_canon_sha256":"bd35b505c322e8052a95e137f0f98705dfc156736e4f1ab3e98a2a987044e1da"},"schema_version":"1.0"},"canonical_sha256":"b60bca876c797e1b29d60a06a31fb17e7a988b6a85248204ec024b84873eea80","source":{"kind":"arxiv","id":"1909.12238","version":1},"source_aliases":[{"alias_kind":"arxiv","alias_value":"1909.12238","created_at":"2026-07-05T00:07:32Z"},{"alias_kind":"arxiv_version","alias_value":"1909.12238v1","created_at":"2026-07-05T00:07:32Z"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.1909.12238","created_at":"2026-07-05T00:07:32Z"},{"alias_kind":"pith_short_12","alias_value":"WYF4VB3MPF7B","created_at":"2026-07-05T00:07:32Z"},{"alias_kind":"pith_short_16","alias_value":"WYF4VB3MPF7BWKOW","created_at":"2026-07-05T00:07:32Z"},{"alias_kind":"pith_short_8","alias_value":"WYF4VB3M","created_at":"2026-07-05T00:07:32Z"}],"events":[{"event_type":"record_created","subject_pith_number":"pith:2019:WYF4VB3MPF7BWKOWBIDKGH5RPZ","target":"record","payload":{"canonical_record":{"source":{"id":"1909.12238","kind":"arxiv","version":1},"metadata":{"license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","primary_cat":"cs.AI","submitted_at":"2019-09-26T16:34:22Z","cross_cats_sorted":["cs.LG"],"title_canon_sha256":"96d3437d089f1d08f592e44f0881b3be56bb73624d1b78feedd5289db17b4ed3","abstract_canon_sha256":"bd35b505c322e8052a95e137f0f98705dfc156736e4f1ab3e98a2a987044e1da"},"schema_version":"1.0"},"canonical_sha256":"b60bca876c797e1b29d60a06a31fb17e7a988b6a85248204ec024b84873eea80","receipt":{"kind":"pith_receipt","key_id":"pith-v1-2026-05","algorithm":"ed25519","signed_at":"2026-07-05T00:07:32.262039Z","signature_b64":"rnAOhVPvLCHYU9hXwRRETkRy2cD7mXQtxwirTtwJ7XyRgQgzL0WsyAQUJ7Ym7uZKI7Mxrjhhq4wNUTAriGTJAA==","signed_message":"canonical_sha256_bytes","builder_version":"pith-number-builder-2026-05-17-v1","receipt_version":"0.3","canonical_sha256":"b60bca876c797e1b29d60a06a31fb17e7a988b6a85248204ec024b84873eea80","last_reissued_at":"2026-07-05T00:07:32.261570Z","signature_status":"signed_v1","first_computed_at":"2026-07-05T00:07:32.261570Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"source_kind":"arxiv","source_id":"1909.12238","source_version":1,"attestation_state":"computed"},"signer":{"signer_id":"pith.science","signer_type":"pith_registry","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"created_at":"2026-07-05T00:07:32Z","supersedes":[],"prev_event":null,"signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"EF5WCImGq+INsZvr+RPFBWqc+t+29xCaKACbvJiVEuI7mW3lSR+mT8uwy0UaoqSlETT/Hh1RyCGeIx4RWVC6Ag==","signed_message":"open_graph_event_sha256_bytes","signed_at":"2026-07-16T11:39:04.275353Z"},"content_sha256":"18bcc56d6cc8d8df067e6df90356d30469036007feff3eb8919e255688c470cb","schema_version":"1.0","event_id":"sha256:18bcc56d6cc8d8df067e6df90356d30469036007feff3eb8919e255688c470cb"},{"event_type":"graph_snapshot","subject_pith_number":"pith:2019:WYF4VB3MPF7BWKOWBIDKGH5RPZ","target":"graph","payload":{"graph_snapshot":{"paper":{"title":"V-MPO: On-Policy Maximum a Posteriori Policy Optimization for Discrete and Continuous Control","license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","headline":"","cross_cats":["cs.LG"],"primary_cat":"cs.AI","authors_text":"Abbas Abdolmaleki, Aidan Clark, Arun Ahuja, Dan Belov, Dhruva Tirumala, H. Francis Song, Hubert Soyer, Jack W. Rae, Jost Tobias Springenberg, Martin Riedmiller, Matthew M. Botvinick, Nicolas Heess, Seb Noury, Siqi Liu","submitted_at":"2019-09-26T16:34:22Z","abstract_excerpt":"Some of the most successful applications of deep reinforcement learning to challenging domains in discrete and continuous control have used policy gradient methods in the on-policy setting. However, policy gradients can suffer from large variance that may limit performance, and in practice require carefully tuned entropy regularization to prevent policy collapse. As an alternative to policy gradient algorithms, we introduce V-MPO, an on-policy adaptation of Maximum a Posteriori Policy Optimization (MPO) that performs policy iteration based on a learned state-value function. We show that V-MPO "},"claims":{"count":0,"items":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"1909.12238","kind":"arxiv","version":1},"verdict":{"id":null,"model_set":{},"created_at":null,"strongest_claim":"","one_line_summary":"","pipeline_version":null,"weakest_assumption":"","pith_extraction_headline":""},"integrity":{"clean":true,"summary":{"advisory":0,"critical":0,"by_detector":{},"informational":0},"endpoint":"/pith/1909.12238/integrity.json","findings":[],"available":true,"detectors_run":[],"snapshot_sha256":"c28c3603d3b5d939e8dc4c7e95fa8dfce3d595e45f758748cecf8e644a296938"},"references":{"count":0,"sample":[],"resolved_work":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","internal_anchors":0},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"author_claims":{"count":0,"strong_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"builder_version":"pith-number-builder-2026-05-17-v1"},"verdict_id":null},"signer":{"signer_id":"pith.science","signer_type":"pith_registry","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"created_at":"2026-07-05T00:07:32Z","supersedes":[],"prev_event":null,"signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"WaHbHBrrwFXb63teAiGxV+RBFvB1Flo8orA0/pSc6tkVtOW0mjTKcDMLhqsCTbYImILkbysG4zFnOmZF3/JoCg==","signed_message":"open_graph_event_sha256_bytes","signed_at":"2026-07-16T11:39:04.275807Z"},"content_sha256":"e065c13ccbb110ec6c0dd7b6776129ffdd3c078884c55a7729245e6439f357c8","schema_version":"1.0","event_id":"sha256:e065c13ccbb110ec6c0dd7b6776129ffdd3c078884c55a7729245e6439f357c8"}],"timestamp_proofs":[],"mirror_hints":[{"mirror_type":"https","name":"Pith Resolver","base_url":"https://pith.science","bundle_url":"https://pith.science/pith/WYF4VB3MPF7BWKOWBIDKGH5RPZ/bundle.json","state_url":"https://pith.science/pith/WYF4VB3MPF7BWKOWBIDKGH5RPZ/state.json","well_known_bundle_url":"https://pith.science/.well-known/pith/WYF4VB3MPF7BWKOWBIDKGH5RPZ/bundle.json","status":"primary"}],"public_keys":[{"key_id":"pith-v1-2026-05","algorithm":"ed25519","format":"raw","public_key_b64":"stVStoiQhXFxp4s2pdzPNoqVNBMojDU/fJ2db5S3CbM=","public_key_hex":"b2d552b68890857171a78b36a5dccf368a953413288c353f7c9d9d6f94b709b3","fingerprint_sha256_b32_first128bits":"RVFV5Z2OI2J3ZUO7ERDEBCYNKS","fingerprint_sha256_hex":"8d4b5ee74e4693bcd1df2446408b0d54","rotates_at":null,"url":"https://pith.science/pith-signing-key.json","notes":"Pith uses this Ed25519 key to sign canonical record SHA-256 digests. Verify with: ed25519_verify(public_key, message=canonical_sha256_bytes, signature=base64decode(signature_b64))."}],"merge_version":"pith-open-graph-merge-v1","built_at":"2026-07-16T11:39:04Z","links":{"resolver":"https://pith.science/pith/WYF4VB3MPF7BWKOWBIDKGH5RPZ","bundle":"https://pith.science/pith/WYF4VB3MPF7BWKOWBIDKGH5RPZ/bundle.json","state":"https://pith.science/pith/WYF4VB3MPF7BWKOWBIDKGH5RPZ/state.json","well_known_bundle":"https://pith.science/.well-known/pith/WYF4VB3MPF7BWKOWBIDKGH5RPZ/bundle.json"},"state":{"state_type":"pith_open_graph_state","state_version":"1.0","pith_number":"pith:2019:WYF4VB3MPF7BWKOWBIDKGH5RPZ","merge_version":"pith-open-graph-merge-v1","event_count":2,"valid_event_count":2,"invalid_event_count":0,"equivocation_count":0,"current":{"canonical_record":{"metadata":{"abstract_canon_sha256":"bd35b505c322e8052a95e137f0f98705dfc156736e4f1ab3e98a2a987044e1da","cross_cats_sorted":["cs.LG"],"license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","primary_cat":"cs.AI","submitted_at":"2019-09-26T16:34:22Z","title_canon_sha256":"96d3437d089f1d08f592e44f0881b3be56bb73624d1b78feedd5289db17b4ed3"},"schema_version":"1.0","source":{"id":"1909.12238","kind":"arxiv","version":1}},"source_aliases":[{"alias_kind":"arxiv","alias_value":"1909.12238","created_at":"2026-07-05T00:07:32Z"},{"alias_kind":"arxiv_version","alias_value":"1909.12238v1","created_at":"2026-07-05T00:07:32Z"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.1909.12238","created_at":"2026-07-05T00:07:32Z"},{"alias_kind":"pith_short_12","alias_value":"WYF4VB3MPF7B","created_at":"2026-07-05T00:07:32Z"},{"alias_kind":"pith_short_16","alias_value":"WYF4VB3MPF7BWKOW","created_at":"2026-07-05T00:07:32Z"},{"alias_kind":"pith_short_8","alias_value":"WYF4VB3M","created_at":"2026-07-05T00:07:32Z"}],"graph_snapshots":[{"event_id":"sha256:e065c13ccbb110ec6c0dd7b6776129ffdd3c078884c55a7729245e6439f357c8","target":"graph","created_at":"2026-07-05T00:07:32Z","signer":{"key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signer_id":"pith.science","signer_type":"pith_registry"},"payload":{"graph_snapshot":{"author_claims":{"count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","strong_count":0},"builder_version":"pith-number-builder-2026-05-17-v1","claims":{"count":0,"items":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"integrity":{"available":true,"clean":true,"detectors_run":[],"endpoint":"/pith/1909.12238/integrity.json","findings":[],"snapshot_sha256":"c28c3603d3b5d939e8dc4c7e95fa8dfce3d595e45f758748cecf8e644a296938","summary":{"advisory":0,"by_detector":{},"critical":0,"informational":0}},"paper":{"abstract_excerpt":"Some of the most successful applications of deep reinforcement learning to challenging domains in discrete and continuous control have used policy gradient methods in the on-policy setting. However, policy gradients can suffer from large variance that may limit performance, and in practice require carefully tuned entropy regularization to prevent policy collapse. As an alternative to policy gradient algorithms, we introduce V-MPO, an on-policy adaptation of Maximum a Posteriori Policy Optimization (MPO) that performs policy iteration based on a learned state-value function. We show that V-MPO ","authors_text":"Abbas Abdolmaleki, Aidan Clark, Arun Ahuja, Dan Belov, Dhruva Tirumala, H. Francis Song, Hubert Soyer, Jack W. Rae, Jost Tobias Springenberg, Martin Riedmiller, Matthew M. Botvinick, Nicolas Heess, Seb Noury, Siqi Liu","cross_cats":["cs.LG"],"headline":"","license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","primary_cat":"cs.AI","submitted_at":"2019-09-26T16:34:22Z","title":"V-MPO: On-Policy Maximum a Posteriori Policy Optimization for Discrete and Continuous Control"},"references":{"count":0,"internal_anchors":0,"resolved_work":0,"sample":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"1909.12238","kind":"arxiv","version":1},"verdict":{"created_at":null,"id":null,"model_set":{},"one_line_summary":"","pipeline_version":null,"pith_extraction_headline":"","strongest_claim":"","weakest_assumption":""}},"verdict_id":null}}],"author_attestations":[],"timestamp_anchors":[],"storage_attestations":[],"citation_signatures":[],"replication_records":[],"corrections":[],"mirror_hints":[],"record_created":{"event_id":"sha256:18bcc56d6cc8d8df067e6df90356d30469036007feff3eb8919e255688c470cb","target":"record","created_at":"2026-07-05T00:07:32Z","signer":{"key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signer_id":"pith.science","signer_type":"pith_registry"},"payload":{"attestation_state":"computed","canonical_record":{"metadata":{"abstract_canon_sha256":"bd35b505c322e8052a95e137f0f98705dfc156736e4f1ab3e98a2a987044e1da","cross_cats_sorted":["cs.LG"],"license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","primary_cat":"cs.AI","submitted_at":"2019-09-26T16:34:22Z","title_canon_sha256":"96d3437d089f1d08f592e44f0881b3be56bb73624d1b78feedd5289db17b4ed3"},"schema_version":"1.0","source":{"id":"1909.12238","kind":"arxiv","version":1}},"canonical_sha256":"b60bca876c797e1b29d60a06a31fb17e7a988b6a85248204ec024b84873eea80","receipt":{"algorithm":"ed25519","builder_version":"pith-number-builder-2026-05-17-v1","canonical_sha256":"b60bca876c797e1b29d60a06a31fb17e7a988b6a85248204ec024b84873eea80","first_computed_at":"2026-07-05T00:07:32.261570Z","key_id":"pith-v1-2026-05","kind":"pith_receipt","last_reissued_at":"2026-07-05T00:07:32.261570Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","receipt_version":"0.3","signature_b64":"rnAOhVPvLCHYU9hXwRRETkRy2cD7mXQtxwirTtwJ7XyRgQgzL0WsyAQUJ7Ym7uZKI7Mxrjhhq4wNUTAriGTJAA==","signature_status":"signed_v1","signed_at":"2026-07-05T00:07:32.262039Z","signed_message":"canonical_sha256_bytes"},"source_id":"1909.12238","source_kind":"arxiv","source_version":1}}},"equivocations":[],"invalid_events":[],"applied_event_ids":["sha256:18bcc56d6cc8d8df067e6df90356d30469036007feff3eb8919e255688c470cb","sha256:e065c13ccbb110ec6c0dd7b6776129ffdd3c078884c55a7729245e6439f357c8"],"state_sha256":"9db850dbae9fc9dcbd661e0064c4ce0d342729bcefb63afcb83fc87fa86cb05e"},"bundle_signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"cj4ql5+XiE/8vreyi5I6rb2K3lHto56pZ49w5MngeyBvSGU99bW1kYUbIMjGvNGBPBF6d0erQVIiKXzjYrrWAw==","signed_message":"bundle_sha256_bytes","signed_at":"2026-07-16T11:39:04.279288Z","bundle_sha256":"c191828e7a66542e24b59b9d98222c25e975882e937931d0b139a927605740f8"}}