{"bundle_type":"pith_open_graph_bundle","bundle_version":"1.0","pith_number":"pith:2021:T6J7V4CSBWI6343UWCEQ26QHRQ","short_pith_number":"pith:T6J7V4CS","canonical_record":{"source":{"id":"2102.10264","kind":"arxiv","version":2},"metadata":{"license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.LG","submitted_at":"2021-02-20T05:51:28Z","cross_cats_sorted":["cs.RO","stat.ML"],"title_canon_sha256":"b53c3ebc829d8104eb9b58a8bce4f6018400f0dcfc227cd50569b8773942f551","abstract_canon_sha256":"d3504ac11db094ea64627de284f34e6d61bfcebeb107163dbbffe8429bd48ec6"},"schema_version":"1.0"},"canonical_sha256":"9f93faf0520d91edf374b0890d7a078c1b3038910044e904fb926f97cbd516c9","source":{"kind":"arxiv","id":"2102.10264","version":2},"source_aliases":[{"alias_kind":"arxiv","alias_value":"2102.10264","created_at":"2026-07-05T02:57:15Z"},{"alias_kind":"arxiv_version","alias_value":"2102.10264v2","created_at":"2026-07-05T02:57:15Z"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.2102.10264","created_at":"2026-07-05T02:57:15Z"},{"alias_kind":"pith_short_12","alias_value":"T6J7V4CSBWI6","created_at":"2026-07-05T02:57:15Z"},{"alias_kind":"pith_short_16","alias_value":"T6J7V4CSBWI6343U","created_at":"2026-07-05T02:57:15Z"},{"alias_kind":"pith_short_8","alias_value":"T6J7V4CS","created_at":"2026-07-05T02:57:15Z"}],"events":[{"event_type":"record_created","subject_pith_number":"pith:2021:T6J7V4CSBWI6343UWCEQ26QHRQ","target":"record","payload":{"canonical_record":{"source":{"id":"2102.10264","kind":"arxiv","version":2},"metadata":{"license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.LG","submitted_at":"2021-02-20T05:51:28Z","cross_cats_sorted":["cs.RO","stat.ML"],"title_canon_sha256":"b53c3ebc829d8104eb9b58a8bce4f6018400f0dcfc227cd50569b8773942f551","abstract_canon_sha256":"d3504ac11db094ea64627de284f34e6d61bfcebeb107163dbbffe8429bd48ec6"},"schema_version":"1.0"},"canonical_sha256":"9f93faf0520d91edf374b0890d7a078c1b3038910044e904fb926f97cbd516c9","receipt":{"kind":"pith_receipt","key_id":"pith-v1-2026-05","algorithm":"ed25519","signed_at":"2026-07-05T02:57:15.800209Z","signature_b64":"90o/tvB3CKyMPwBZLtxYay1PiJ1lQgjuTdP41MIeuUeRjE1AI2LOoJeMndgQOnMjEmk9/u1RPwoJo6HXKVJ7Aw==","signed_message":"canonical_sha256_bytes","builder_version":"pith-number-builder-2026-05-17-v1","receipt_version":"0.3","canonical_sha256":"9f93faf0520d91edf374b0890d7a078c1b3038910044e904fb926f97cbd516c9","last_reissued_at":"2026-07-05T02:57:15.799684Z","signature_status":"signed_v1","first_computed_at":"2026-07-05T02:57:15.799684Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"source_kind":"arxiv","source_id":"2102.10264","source_version":2,"attestation_state":"computed"},"signer":{"signer_id":"pith.science","signer_type":"pith_registry","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"created_at":"2026-07-05T02:57:15Z","supersedes":[],"prev_event":null,"signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"LolHIrRZa+DgmYx4DLFim5Bwb2RyVXe7Yi1eWSr1l/V1XtlgGKYuI3sYQUdw4oZcNqmTW54HHI6Vh1FqXmrCCA==","signed_message":"open_graph_event_sha256_bytes","signed_at":"2026-08-04T04:53:33.572467Z"},"content_sha256":"9fdb3767dc72a6a874a5302d933c442920a6cd0462c3a3117b586d4d1c59ac49","schema_version":"1.0","event_id":"sha256:9fdb3767dc72a6a874a5302d933c442920a6cd0462c3a3117b586d4d1c59ac49"},{"event_type":"graph_snapshot","subject_pith_number":"pith:2021:T6J7V4CSBWI6343UWCEQ26QHRQ","target":"graph","payload":{"graph_snapshot":{"paper":{"title":"On Proximal Policy Optimization's Heavy-tailed Gradients","license":"http://creativecommons.org/licenses/by/4.0/","headline":"","cross_cats":["cs.RO","stat.ML"],"primary_cat":"cs.LG","authors_text":"Adarsh Prasad, Emilio Parisotto, Joshua Zhanson, J. Zico Kolter, Pradeep Ravikumar, Ruslan Salakhutdinov, Saurabh Garg, Sivaraman Balakrishnan, Zachary C. Lipton","submitted_at":"2021-02-20T05:51:28Z","abstract_excerpt":"Modern policy gradient algorithms such as Proximal Policy Optimization (PPO) rely on an arsenal of heuristics, including loss clipping and gradient clipping, to ensure successful learning. These heuristics are reminiscent of techniques from robust statistics, commonly used for estimation in outlier-rich (``heavy-tailed'') regimes. In this paper, we present a detailed empirical study to characterize the heavy-tailed nature of the gradients of the PPO surrogate reward function. We demonstrate that the gradients, especially for the actor network, exhibit pronounced heavy-tailedness and that it in"},"claims":{"count":0,"items":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"2102.10264","kind":"arxiv","version":2},"verdict":{"id":null,"model_set":{},"created_at":null,"strongest_claim":"","one_line_summary":"","pipeline_version":null,"weakest_assumption":"","pith_extraction_headline":""},"integrity":{"clean":true,"summary":{"advisory":0,"critical":0,"by_detector":{},"informational":0},"endpoint":"/pith/2102.10264/integrity.json","findings":[],"available":true,"detectors_run":[],"snapshot_sha256":"c28c3603d3b5d939e8dc4c7e95fa8dfce3d595e45f758748cecf8e644a296938"},"references":{"count":0,"sample":[],"resolved_work":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","internal_anchors":0},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"author_claims":{"count":0,"strong_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"builder_version":"pith-number-builder-2026-05-17-v1"},"verdict_id":null},"signer":{"signer_id":"pith.science","signer_type":"pith_registry","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"created_at":"2026-07-05T02:57:15Z","supersedes":[],"prev_event":null,"signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"Nl07PIwSsy4UQM9kU80tS15oBqurR4CzoxvfcCw6F+J2P0DK+YQ7olbL1gV3+4DC0hp2galWznTTMwOSkNLHBA==","signed_message":"open_graph_event_sha256_bytes","signed_at":"2026-08-04T04:53:33.572874Z"},"content_sha256":"3c257de2618e79d038fc2fa58f855f4e9feba65529bcbbf0e8b69ffecb212a66","schema_version":"1.0","event_id":"sha256:3c257de2618e79d038fc2fa58f855f4e9feba65529bcbbf0e8b69ffecb212a66"}],"timestamp_proofs":[],"mirror_hints":[{"mirror_type":"https","name":"Pith Resolver","base_url":"https://pith.science","bundle_url":"https://pith.science/pith/T6J7V4CSBWI6343UWCEQ26QHRQ/bundle.json","state_url":"https://pith.science/pith/T6J7V4CSBWI6343UWCEQ26QHRQ/state.json","well_known_bundle_url":"https://pith.science/.well-known/pith/T6J7V4CSBWI6343UWCEQ26QHRQ/bundle.json","status":"primary"}],"public_keys":[{"key_id":"pith-v1-2026-05","algorithm":"ed25519","format":"raw","public_key_b64":"stVStoiQhXFxp4s2pdzPNoqVNBMojDU/fJ2db5S3CbM=","public_key_hex":"b2d552b68890857171a78b36a5dccf368a953413288c353f7c9d9d6f94b709b3","fingerprint_sha256_b32_first128bits":"RVFV5Z2OI2J3ZUO7ERDEBCYNKS","fingerprint_sha256_hex":"8d4b5ee74e4693bcd1df2446408b0d54","rotates_at":null,"url":"https://pith.science/pith-signing-key.json","notes":"Pith uses this Ed25519 key to sign canonical record SHA-256 digests. Verify with: ed25519_verify(public_key, message=canonical_sha256_bytes, signature=base64decode(signature_b64))."}],"merge_version":"pith-open-graph-merge-v1","built_at":"2026-08-04T04:53:33Z","links":{"resolver":"https://pith.science/pith/T6J7V4CSBWI6343UWCEQ26QHRQ","bundle":"https://pith.science/pith/T6J7V4CSBWI6343UWCEQ26QHRQ/bundle.json","state":"https://pith.science/pith/T6J7V4CSBWI6343UWCEQ26QHRQ/state.json","well_known_bundle":"https://pith.science/.well-known/pith/T6J7V4CSBWI6343UWCEQ26QHRQ/bundle.json"},"state":{"state_type":"pith_open_graph_state","state_version":"1.0","pith_number":"pith:2021:T6J7V4CSBWI6343UWCEQ26QHRQ","merge_version":"pith-open-graph-merge-v1","event_count":2,"valid_event_count":2,"invalid_event_count":0,"equivocation_count":0,"current":{"canonical_record":{"metadata":{"abstract_canon_sha256":"d3504ac11db094ea64627de284f34e6d61bfcebeb107163dbbffe8429bd48ec6","cross_cats_sorted":["cs.RO","stat.ML"],"license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.LG","submitted_at":"2021-02-20T05:51:28Z","title_canon_sha256":"b53c3ebc829d8104eb9b58a8bce4f6018400f0dcfc227cd50569b8773942f551"},"schema_version":"1.0","source":{"id":"2102.10264","kind":"arxiv","version":2}},"source_aliases":[{"alias_kind":"arxiv","alias_value":"2102.10264","created_at":"2026-07-05T02:57:15Z"},{"alias_kind":"arxiv_version","alias_value":"2102.10264v2","created_at":"2026-07-05T02:57:15Z"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.2102.10264","created_at":"2026-07-05T02:57:15Z"},{"alias_kind":"pith_short_12","alias_value":"T6J7V4CSBWI6","created_at":"2026-07-05T02:57:15Z"},{"alias_kind":"pith_short_16","alias_value":"T6J7V4CSBWI6343U","created_at":"2026-07-05T02:57:15Z"},{"alias_kind":"pith_short_8","alias_value":"T6J7V4CS","created_at":"2026-07-05T02:57:15Z"}],"graph_snapshots":[{"event_id":"sha256:3c257de2618e79d038fc2fa58f855f4e9feba65529bcbbf0e8b69ffecb212a66","target":"graph","created_at":"2026-07-05T02:57:15Z","signer":{"key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signer_id":"pith.science","signer_type":"pith_registry"},"payload":{"graph_snapshot":{"author_claims":{"count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","strong_count":0},"builder_version":"pith-number-builder-2026-05-17-v1","claims":{"count":0,"items":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"integrity":{"available":true,"clean":true,"detectors_run":[],"endpoint":"/pith/2102.10264/integrity.json","findings":[],"snapshot_sha256":"c28c3603d3b5d939e8dc4c7e95fa8dfce3d595e45f758748cecf8e644a296938","summary":{"advisory":0,"by_detector":{},"critical":0,"informational":0}},"paper":{"abstract_excerpt":"Modern policy gradient algorithms such as Proximal Policy Optimization (PPO) rely on an arsenal of heuristics, including loss clipping and gradient clipping, to ensure successful learning. These heuristics are reminiscent of techniques from robust statistics, commonly used for estimation in outlier-rich (``heavy-tailed'') regimes. In this paper, we present a detailed empirical study to characterize the heavy-tailed nature of the gradients of the PPO surrogate reward function. We demonstrate that the gradients, especially for the actor network, exhibit pronounced heavy-tailedness and that it in","authors_text":"Adarsh Prasad, Emilio Parisotto, Joshua Zhanson, J. Zico Kolter, Pradeep Ravikumar, Ruslan Salakhutdinov, Saurabh Garg, Sivaraman Balakrishnan, Zachary C. Lipton","cross_cats":["cs.RO","stat.ML"],"headline":"","license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.LG","submitted_at":"2021-02-20T05:51:28Z","title":"On Proximal Policy Optimization's Heavy-tailed Gradients"},"references":{"count":0,"internal_anchors":0,"resolved_work":0,"sample":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"2102.10264","kind":"arxiv","version":2},"verdict":{"created_at":null,"id":null,"model_set":{},"one_line_summary":"","pipeline_version":null,"pith_extraction_headline":"","strongest_claim":"","weakest_assumption":""}},"verdict_id":null}}],"author_attestations":[],"timestamp_anchors":[],"storage_attestations":[],"citation_signatures":[],"replication_records":[],"corrections":[],"mirror_hints":[],"record_created":{"event_id":"sha256:9fdb3767dc72a6a874a5302d933c442920a6cd0462c3a3117b586d4d1c59ac49","target":"record","created_at":"2026-07-05T02:57:15Z","signer":{"key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signer_id":"pith.science","signer_type":"pith_registry"},"payload":{"attestation_state":"computed","canonical_record":{"metadata":{"abstract_canon_sha256":"d3504ac11db094ea64627de284f34e6d61bfcebeb107163dbbffe8429bd48ec6","cross_cats_sorted":["cs.RO","stat.ML"],"license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.LG","submitted_at":"2021-02-20T05:51:28Z","title_canon_sha256":"b53c3ebc829d8104eb9b58a8bce4f6018400f0dcfc227cd50569b8773942f551"},"schema_version":"1.0","source":{"id":"2102.10264","kind":"arxiv","version":2}},"canonical_sha256":"9f93faf0520d91edf374b0890d7a078c1b3038910044e904fb926f97cbd516c9","receipt":{"algorithm":"ed25519","builder_version":"pith-number-builder-2026-05-17-v1","canonical_sha256":"9f93faf0520d91edf374b0890d7a078c1b3038910044e904fb926f97cbd516c9","first_computed_at":"2026-07-05T02:57:15.799684Z","key_id":"pith-v1-2026-05","kind":"pith_receipt","last_reissued_at":"2026-07-05T02:57:15.799684Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","receipt_version":"0.3","signature_b64":"90o/tvB3CKyMPwBZLtxYay1PiJ1lQgjuTdP41MIeuUeRjE1AI2LOoJeMndgQOnMjEmk9/u1RPwoJo6HXKVJ7Aw==","signature_status":"signed_v1","signed_at":"2026-07-05T02:57:15.800209Z","signed_message":"canonical_sha256_bytes"},"source_id":"2102.10264","source_kind":"arxiv","source_version":2}}},"equivocations":[],"invalid_events":[],"applied_event_ids":["sha256:9fdb3767dc72a6a874a5302d933c442920a6cd0462c3a3117b586d4d1c59ac49","sha256:3c257de2618e79d038fc2fa58f855f4e9feba65529bcbbf0e8b69ffecb212a66"],"state_sha256":"476273917a3820b6d3281f6cd20b59330f93d242d8e2e1b41a4f890c91c5f064"},"bundle_signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"UEw/HziNAVu23zF5O13t1WHrWsnStsHcRBLOFkeuJ8lG64V1rM/DZBO418+8hn/9JejPGX6O0FAwTPjsD8Z6Bw==","signed_message":"bundle_sha256_bytes","signed_at":"2026-08-04T04:53:33.575357Z","bundle_sha256":"f716c13d549b4083e9cac3c07087fde1dde394d199b629d8ef6792f2cda5d28c"}}