{"record_type":"pith_number_record","schema_url":"https://pith.science/schemas/pith-number/v1.json","pith_number":"pith:2020:TB24RQRM3IW24Y5PTEPSU2D24W","short_pith_number":"pith:TB24RQRM","schema_version":"1.0","canonical_sha256":"9875c8c22cda2dae63af991f2a687ae5bd378b58dffe70dbee032eb30986d95f","source":{"kind":"arxiv","id":"2009.10897","version":1},"attestation_state":"computed","paper":{"title":"Revisiting Design Choices in Proximal Policy Optimization","license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","headline":"","cross_cats":["stat.ML"],"primary_cat":"cs.LG","authors_text":"Celestine Mendler-D\\\"unner, Chloe Ching-Yun Hsu, Moritz Hardt","submitted_at":"2020-09-23T02:00:34Z","abstract_excerpt":"Proximal Policy Optimization (PPO) is a popular deep policy gradient algorithm. In standard implementations, PPO regularizes policy updates with clipped probability ratios, and parameterizes policies with either continuous Gaussian distributions or discrete Softmax distributions. These design choices are widely accepted, and motivated by empirical performance comparisons on MuJoCo and Atari benchmarks.\n  We revisit these practices outside the regime of current benchmarks, and expose three failure modes of standard PPO. We explain why standard design choices are problematic in these cases, and "},"verification_status":{"content_addressed":true,"pith_receipt":true,"author_attested":false,"weak_author_claims":0,"strong_author_claims":0,"externally_anchored":false,"storage_verified":false,"citation_signatures":0,"replication_records":0,"graph_snapshot":true,"references_resolved":false,"formal_links_present":false},"canonical_record":{"source":{"id":"2009.10897","kind":"arxiv","version":1},"metadata":{"license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","primary_cat":"cs.LG","submitted_at":"2020-09-23T02:00:34Z","cross_cats_sorted":["stat.ML"],"title_canon_sha256":"90f0ac4009f078769c70cd605c0aa05cc4e30f5fe04e27d7e84d75fbf6e651f3","abstract_canon_sha256":"60643a5f596cac16961d358153f6af9c7738a84363cdf736bdd7e09499f4acd6"},"schema_version":"1.0"},"receipt":{"kind":"pith_receipt","key_id":"pith-v1-2026-05","algorithm":"ed25519","signed_at":"2026-07-05T01:37:27.177391Z","signature_b64":"n3GEmHZMlDGda9QSAbTDa3d6K1bMPNzqd5iicuq3bea4egDBavTeVV8XdbA/qdT9dUFTxIHSIIzyAIgLZTaNCg==","signed_message":"canonical_sha256_bytes","builder_version":"pith-number-builder-2026-05-17-v1","receipt_version":"0.3","canonical_sha256":"9875c8c22cda2dae63af991f2a687ae5bd378b58dffe70dbee032eb30986d95f","last_reissued_at":"2026-07-05T01:37:27.176559Z","signature_status":"signed_v1","first_computed_at":"2026-07-05T01:37:27.176559Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"graph_snapshot":{"paper":{"title":"Revisiting Design Choices in Proximal Policy Optimization","license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","headline":"","cross_cats":["stat.ML"],"primary_cat":"cs.LG","authors_text":"Celestine Mendler-D\\\"unner, Chloe Ching-Yun Hsu, Moritz Hardt","submitted_at":"2020-09-23T02:00:34Z","abstract_excerpt":"Proximal Policy Optimization (PPO) is a popular deep policy gradient algorithm. In standard implementations, PPO regularizes policy updates with clipped probability ratios, and parameterizes policies with either continuous Gaussian distributions or discrete Softmax distributions. These design choices are widely accepted, and motivated by empirical performance comparisons on MuJoCo and Atari benchmarks.\n  We revisit these practices outside the regime of current benchmarks, and expose three failure modes of standard PPO. We explain why standard design choices are problematic in these cases, and "},"claims":{"count":0,"items":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"2009.10897","kind":"arxiv","version":1},"verdict":{"id":null,"model_set":{},"created_at":null,"strongest_claim":"","one_line_summary":"","pipeline_version":null,"weakest_assumption":"","pith_extraction_headline":""},"integrity":{"clean":true,"summary":{"advisory":0,"critical":0,"by_detector":{},"informational":0},"endpoint":"/pith/2009.10897/integrity.json","findings":[],"available":true,"detectors_run":[],"snapshot_sha256":"c28c3603d3b5d939e8dc4c7e95fa8dfce3d595e45f758748cecf8e644a296938"},"references":{"count":0,"sample":[],"resolved_work":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","internal_anchors":0},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"author_claims":{"count":0,"strong_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"builder_version":"pith-number-builder-2026-05-17-v1"},"aliases":[{"alias_kind":"arxiv","alias_value":"2009.10897","created_at":"2026-07-05T01:37:27.177026+00:00"},{"alias_kind":"arxiv_version","alias_value":"2009.10897v1","created_at":"2026-07-05T01:37:27.177026+00:00"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.2009.10897","created_at":"2026-07-05T01:37:27.177026+00:00"},{"alias_kind":"pith_short_12","alias_value":"TB24RQRM3IW2","created_at":"2026-07-05T01:37:27.177026+00:00"},{"alias_kind":"pith_short_16","alias_value":"TB24RQRM3IW24Y5P","created_at":"2026-07-05T01:37:27.177026+00:00"},{"alias_kind":"pith_short_8","alias_value":"TB24RQRM","created_at":"2026-07-05T01:37:27.177026+00:00"}],"events":[],"event_summary":{},"paper_claims":[],"inbound_citations":{"count":3,"internal_anchor_count":0,"sample":[{"citing_arxiv_id":"2606.23932","citing_title":"KLip-PPO: A per-sample KL perspective on PPO-Clip","ref_index":6,"is_internal_anchor":false},{"citing_arxiv_id":"2606.08153","citing_title":"LogNEO: A GPT-Neo Reinforcement Learning Framework for Accurate Real-Time Log Anomaly Detection","ref_index":23,"is_internal_anchor":false},{"citing_arxiv_id":"2605.06032","citing_title":"Does Synthetic Data Help? Empirical Evidence from Deep Learning Time Series Forecasters","ref_index":262,"is_internal_anchor":false}]},"formal_canon":{"evidence_count":0,"sample":[],"anchors":[]},"links":{"html":"https://pith.science/pith/TB24RQRM3IW24Y5PTEPSU2D24W","json":"https://pith.science/pith/TB24RQRM3IW24Y5PTEPSU2D24W.json","graph_json":"https://pith.science/api/pith-number/TB24RQRM3IW24Y5PTEPSU2D24W/graph.json","events_json":"https://pith.science/api/pith-number/TB24RQRM3IW24Y5PTEPSU2D24W/events.json","paper":"https://pith.science/paper/TB24RQRM"},"agent_actions":{"view_html":"https://pith.science/pith/TB24RQRM3IW24Y5PTEPSU2D24W","download_json":"https://pith.science/pith/TB24RQRM3IW24Y5PTEPSU2D24W.json","view_paper":"https://pith.science/paper/TB24RQRM","resolve_alias":"https://pith.science/api/pith-number/resolve?arxiv=2009.10897&json=true","fetch_graph":"https://pith.science/api/pith-number/TB24RQRM3IW24Y5PTEPSU2D24W/graph.json","fetch_events":"https://pith.science/api/pith-number/TB24RQRM3IW24Y5PTEPSU2D24W/events.json","actions":{"anchor_timestamp":"https://pith.science/pith/TB24RQRM3IW24Y5PTEPSU2D24W/action/timestamp_anchor","attest_storage":"https://pith.science/pith/TB24RQRM3IW24Y5PTEPSU2D24W/action/storage_attestation","attest_author":"https://pith.science/pith/TB24RQRM3IW24Y5PTEPSU2D24W/action/author_attestation","sign_citation":"https://pith.science/pith/TB24RQRM3IW24Y5PTEPSU2D24W/action/citation_signature","submit_replication":"https://pith.science/pith/TB24RQRM3IW24Y5PTEPSU2D24W/action/replication_record"}},"created_at":"2026-07-05T01:37:27.177026+00:00","updated_at":"2026-07-05T01:37:27.177026+00:00"}