{"record_type":"pith_number_record","schema_url":"https://pith.science/schemas/pith-number/v1.json","pith_number":"pith:2018:UAXZQNFHJR5FBXNSMCYSPEJNKL","short_pith_number":"pith:UAXZQNFH","schema_version":"1.0","canonical_sha256":"a02f9834a74c7a50ddb260b127912d52c5a6ed30ac329fc99226664ba943c29f","source":{"kind":"arxiv","id":"1806.06920","version":1},"attestation_state":"computed","paper":{"title":"Maximum a Posteriori Policy Optimisation","license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","headline":"","cross_cats":["cs.AI","cs.IT","cs.RO","math.IT","stat.ML"],"primary_cat":"cs.LG","authors_text":"Abbas Abdolmaleki, Jost Tobias Springenberg, Martin Riedmiller, Nicolas Heess, Remi Munos, Yuval Tassa","submitted_at":"2018-06-14T12:46:23Z","abstract_excerpt":"We introduce a new algorithm for reinforcement learning called Maximum aposteriori Policy Optimisation (MPO) based on coordinate ascent on a relative entropy objective. We show that several existing methods can directly be related to our derivation. We develop two off-policy algorithms and demonstrate that they are competitive with the state-of-the-art in deep reinforcement learning. In particular, for continuous control, our method outperforms existing methods with respect to sample efficiency, premature convergence and robustness to hyperparameter settings while achieving similar or better f"},"verification_status":{"content_addressed":true,"pith_receipt":true,"author_attested":false,"weak_author_claims":0,"strong_author_claims":0,"externally_anchored":false,"storage_verified":false,"citation_signatures":0,"replication_records":0,"graph_snapshot":true,"references_resolved":false,"formal_links_present":false},"canonical_record":{"source":{"id":"1806.06920","kind":"arxiv","version":1},"metadata":{"license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","primary_cat":"cs.LG","submitted_at":"2018-06-14T12:46:23Z","cross_cats_sorted":["cs.AI","cs.IT","cs.RO","math.IT","stat.ML"],"title_canon_sha256":"eccde276c3f8de0e1b551d861c94e59d7836421b50ccd048e66805a67671b88e","abstract_canon_sha256":"80ef65e0114b46cbcd1411b8c1bcd125aef9f7e5ba657e9e84db07dc6e66f65a"},"schema_version":"1.0"},"receipt":{"kind":"pith_receipt","key_id":"pith-v1-2026-05","algorithm":"ed25519","signed_at":"2026-05-18T00:12:37.726177Z","signature_b64":"1WfTboq4Z5Ahm7CCgEFLa2XpC2CAYBpNDOn3bk0J16XZHfJbarLQcKAm8l+V/7NSOrbyHhVBxIIZ108M7Ta/Aw==","signed_message":"canonical_sha256_bytes","builder_version":"pith-number-builder-2026-05-17-v1","receipt_version":"0.3","canonical_sha256":"a02f9834a74c7a50ddb260b127912d52c5a6ed30ac329fc99226664ba943c29f","last_reissued_at":"2026-05-18T00:12:37.725340Z","signature_status":"signed_v1","first_computed_at":"2026-05-18T00:12:37.725340Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"graph_snapshot":{"paper":{"title":"Maximum a Posteriori Policy Optimisation","license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","headline":"","cross_cats":["cs.AI","cs.IT","cs.RO","math.IT","stat.ML"],"primary_cat":"cs.LG","authors_text":"Abbas Abdolmaleki, Jost Tobias Springenberg, Martin Riedmiller, Nicolas Heess, Remi Munos, Yuval Tassa","submitted_at":"2018-06-14T12:46:23Z","abstract_excerpt":"We introduce a new algorithm for reinforcement learning called Maximum aposteriori Policy Optimisation (MPO) based on coordinate ascent on a relative entropy objective. We show that several existing methods can directly be related to our derivation. We develop two off-policy algorithms and demonstrate that they are competitive with the state-of-the-art in deep reinforcement learning. In particular, for continuous control, our method outperforms existing methods with respect to sample efficiency, premature convergence and robustness to hyperparameter settings while achieving similar or better f"},"claims":{"count":0,"items":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"1806.06920","kind":"arxiv","version":1},"verdict":{"id":null,"model_set":{},"created_at":null,"strongest_claim":"","one_line_summary":"","pipeline_version":null,"weakest_assumption":"","pith_extraction_headline":""},"references":{"count":0,"sample":[],"resolved_work":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","internal_anchors":0},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"author_claims":{"count":0,"strong_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"builder_version":"pith-number-builder-2026-05-17-v1"},"aliases":[{"alias_kind":"arxiv","alias_value":"1806.06920","created_at":"2026-05-18T00:12:37.725450+00:00"},{"alias_kind":"arxiv_version","alias_value":"1806.06920v1","created_at":"2026-05-18T00:12:37.725450+00:00"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.1806.06920","created_at":"2026-05-18T00:12:37.725450+00:00"},{"alias_kind":"pith_short_12","alias_value":"UAXZQNFHJR5F","created_at":"2026-05-18T12:32:56.356000+00:00"},{"alias_kind":"pith_short_16","alias_value":"UAXZQNFHJR5FBXNS","created_at":"2026-05-18T12:32:56.356000+00:00"},{"alias_kind":"pith_short_8","alias_value":"UAXZQNFH","created_at":"2026-05-18T12:32:56.356000+00:00"}],"events":[],"event_summary":{},"paper_claims":[],"inbound_citations":{"count":20,"internal_anchor_count":11,"sample":[{"citing_arxiv_id":"1906.09223","citing_title":"Disentangled Skill Embeddings for Reinforcement Learning","ref_index":28,"is_internal_anchor":true},{"citing_arxiv_id":"1907.00456","citing_title":"Way Off-Policy Batch Deep Reinforcement Learning of Implicit Human Preferences in Dialog","ref_index":1,"is_internal_anchor":true},{"citing_arxiv_id":"2510.03508","citing_title":"D2 Actor Critic: Diffusion Actor Meets Distributional Critic","ref_index":1,"is_internal_anchor":true},{"citing_arxiv_id":"2211.14275","citing_title":"Solving math word problems with process- and outcome-based feedback","ref_index":1,"is_internal_anchor":true},{"citing_arxiv_id":"2509.22963","citing_title":"Reinforcement Learning with Discrete Diffusion Policies for Combinatorial Action Spaces","ref_index":1,"is_internal_anchor":true},{"citing_arxiv_id":"2605.06139","citing_title":"Listwise Policy Optimization: Group-based RLVR as Target-Projection on the LLM Response Simplex","ref_index":2,"is_internal_anchor":true},{"citing_arxiv_id":"2605.15935","citing_title":"Dynamic Plasma Shape Control with Arbitrary Sensor Subsets","ref_index":17,"is_internal_anchor":true},{"citing_arxiv_id":"2605.18591","citing_title":"Randomized Advantage Transformation (RAT): Computing Natural Policy Gradients via Direct Backpropagation","ref_index":138,"is_internal_anchor":true},{"citing_arxiv_id":"2605.19215","citing_title":"Not all uncertainty is alike: volatility, stochasticity, and exploration","ref_index":28,"is_internal_anchor":true},{"citing_arxiv_id":"2509.20265","citing_title":"Failure Modes of Maximum Entropy RLHF","ref_index":1,"is_internal_anchor":true},{"citing_arxiv_id":"2211.15657","citing_title":"Is Conditional Generative Modeling all you need for Decision-Making?","ref_index":249,"is_internal_anchor":true},{"citing_arxiv_id":"1911.11361","citing_title":"Behavior Regularized Offline Reinforcement Learning","ref_index":1,"is_internal_anchor":false},{"citing_arxiv_id":"1812.05905","citing_title":"Soft Actor-Critic Algorithms and Applications","ref_index":1,"is_internal_anchor":false},{"citing_arxiv_id":"2205.06175","citing_title":"A Generalist Agent","ref_index":1,"is_internal_anchor":false},{"citing_arxiv_id":"2605.11387","citing_title":"Behavioral Mode Discovery for Fine-tuning Multimodal Generative Policies","ref_index":36,"is_internal_anchor":false},{"citing_arxiv_id":"2605.06139","citing_title":"Listwise Policy Optimization: Group-based RLVR as Target-Projection on the LLM Response Simplex","ref_index":2,"is_internal_anchor":false},{"citing_arxiv_id":"2605.05481","citing_title":"Approximate Next Policy Sampling: Replacing Conservative Target Policy Updates in Deep RL","ref_index":1,"is_internal_anchor":false},{"citing_arxiv_id":"2301.04104","citing_title":"Mastering Diverse Domains through World Models","ref_index":33,"is_internal_anchor":false},{"citing_arxiv_id":"2604.14895","citing_title":"Beyond Importance Sampling: Rejection-Gated Policy Optimization","ref_index":1,"is_internal_anchor":false},{"citing_arxiv_id":"2605.02112","citing_title":"An adaptive variance estimator for relative sparsity","ref_index":80,"is_internal_anchor":false}]},"formal_canon":{"evidence_count":0,"sample":[],"anchors":[]},"links":{"html":"https://pith.science/pith/UAXZQNFHJR5FBXNSMCYSPEJNKL","json":"https://pith.science/pith/UAXZQNFHJR5FBXNSMCYSPEJNKL.json","graph_json":"https://pith.science/api/pith-number/UAXZQNFHJR5FBXNSMCYSPEJNKL/graph.json","events_json":"https://pith.science/api/pith-number/UAXZQNFHJR5FBXNSMCYSPEJNKL/events.json","paper":"https://pith.science/paper/UAXZQNFH"},"agent_actions":{"view_html":"https://pith.science/pith/UAXZQNFHJR5FBXNSMCYSPEJNKL","download_json":"https://pith.science/pith/UAXZQNFHJR5FBXNSMCYSPEJNKL.json","view_paper":"https://pith.science/paper/UAXZQNFH","resolve_alias":"https://pith.science/api/pith-number/resolve?arxiv=1806.06920&json=true","fetch_graph":"https://pith.science/api/pith-number/UAXZQNFHJR5FBXNSMCYSPEJNKL/graph.json","fetch_events":"https://pith.science/api/pith-number/UAXZQNFHJR5FBXNSMCYSPEJNKL/events.json","actions":{"anchor_timestamp":"https://pith.science/pith/UAXZQNFHJR5FBXNSMCYSPEJNKL/action/timestamp_anchor","attest_storage":"https://pith.science/pith/UAXZQNFHJR5FBXNSMCYSPEJNKL/action/storage_attestation","attest_author":"https://pith.science/pith/UAXZQNFHJR5FBXNSMCYSPEJNKL/action/author_attestation","sign_citation":"https://pith.science/pith/UAXZQNFHJR5FBXNSMCYSPEJNKL/action/citation_signature","submit_replication":"https://pith.science/pith/UAXZQNFHJR5FBXNSMCYSPEJNKL/action/replication_record"}},"created_at":"2026-05-18T00:12:37.725450+00:00","updated_at":"2026-05-18T00:12:37.725450+00:00"}