{"bundle_type":"pith_open_graph_bundle","bundle_version":"1.0","pith_number":"pith:2024:CSEWCPPIXNN5X7KAYHDMX5TF5T","short_pith_number":"pith:CSEWCPPI","canonical_record":{"source":{"id":"2401.13662","kind":"arxiv","version":2},"metadata":{"license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.LG","submitted_at":"2024-01-24T18:56:53Z","cross_cats_sorted":["cs.AI"],"title_canon_sha256":"5973d7eae3f1cf8d8dc5c63cdc6384e79a8fa369a211fc15202b63b2d2070ec1","abstract_canon_sha256":"9344fd39b2ce3eaab01fb1027b21b13cc624bc3f0eeaeb8cf3650faa5555fcb0"},"schema_version":"1.0"},"canonical_sha256":"1489613de8bb5bdbfd40c1c6cbf665ecf73b673199297dfef9952ffbbd13bb10","source":{"kind":"arxiv","id":"2401.13662","version":2},"source_aliases":[{"alias_kind":"arxiv","alias_value":"2401.13662","created_at":"2026-07-05T07:50:55Z"},{"alias_kind":"arxiv_version","alias_value":"2401.13662v2","created_at":"2026-07-05T07:50:55Z"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.2401.13662","created_at":"2026-07-05T07:50:55Z"},{"alias_kind":"pith_short_12","alias_value":"CSEWCPPIXNN5","created_at":"2026-07-05T07:50:55Z"},{"alias_kind":"pith_short_16","alias_value":"CSEWCPPIXNN5X7KA","created_at":"2026-07-05T07:50:55Z"},{"alias_kind":"pith_short_8","alias_value":"CSEWCPPI","created_at":"2026-07-05T07:50:55Z"}],"events":[{"event_type":"record_created","subject_pith_number":"pith:2024:CSEWCPPIXNN5X7KAYHDMX5TF5T","target":"record","payload":{"canonical_record":{"source":{"id":"2401.13662","kind":"arxiv","version":2},"metadata":{"license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.LG","submitted_at":"2024-01-24T18:56:53Z","cross_cats_sorted":["cs.AI"],"title_canon_sha256":"5973d7eae3f1cf8d8dc5c63cdc6384e79a8fa369a211fc15202b63b2d2070ec1","abstract_canon_sha256":"9344fd39b2ce3eaab01fb1027b21b13cc624bc3f0eeaeb8cf3650faa5555fcb0"},"schema_version":"1.0"},"canonical_sha256":"1489613de8bb5bdbfd40c1c6cbf665ecf73b673199297dfef9952ffbbd13bb10","receipt":{"kind":"pith_receipt","key_id":"pith-v1-2026-05","algorithm":"ed25519","signed_at":"2026-07-05T07:50:55.094119Z","signature_b64":"viNgv6WLl5oXNG+BPlRjdUL0LBWmyWc6L6LXPgUJ0fCC1LKvNCo8JOHsee3d4zKT2S3J8CQAumQcT5Nj0fXIBg==","signed_message":"canonical_sha256_bytes","builder_version":"pith-number-builder-2026-05-17-v1","receipt_version":"0.3","canonical_sha256":"1489613de8bb5bdbfd40c1c6cbf665ecf73b673199297dfef9952ffbbd13bb10","last_reissued_at":"2026-07-05T07:50:55.093527Z","signature_status":"signed_v1","first_computed_at":"2026-07-05T07:50:55.093527Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"source_kind":"arxiv","source_id":"2401.13662","source_version":2,"attestation_state":"computed"},"signer":{"signer_id":"pith.science","signer_type":"pith_registry","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"created_at":"2026-07-05T07:50:55Z","supersedes":[],"prev_event":null,"signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"k03+QgR1Nqf1KSIoqdl33xqCwGUMXU3/VKE2ulej/7ruy3IDarkt8wR/cuH1IshJOSn1FmpGt5XC5pj5tbWEBA==","signed_message":"open_graph_event_sha256_bytes","signed_at":"2026-08-13T22:53:39.254208Z"},"content_sha256":"cff042d2e3a1d33e53449b2bd49eda6293f01c3173ff3f186d2eb1c701fbed68","schema_version":"1.0","event_id":"sha256:cff042d2e3a1d33e53449b2bd49eda6293f01c3173ff3f186d2eb1c701fbed68"},{"event_type":"graph_snapshot","subject_pith_number":"pith:2024:CSEWCPPIXNN5X7KAYHDMX5TF5T","target":"graph","payload":{"graph_snapshot":{"paper":{"title":"The Definitive Guide to Policy Gradients in Deep Reinforcement Learning: Theory, Algorithms and Implementations","license":"http://creativecommons.org/licenses/by/4.0/","headline":"","cross_cats":["cs.AI"],"primary_cat":"cs.LG","authors_text":"Matthias Lehmann","submitted_at":"2024-01-24T18:56:53Z","abstract_excerpt":"In recent years, various powerful policy gradient algorithms have been proposed in deep reinforcement learning. While all these algorithms build on the Policy Gradient Theorem, the specific design choices differ significantly across algorithms. We provide a holistic overview of on-policy policy gradient algorithms to facilitate the understanding of both their theoretical foundations and their practical implementations. In this overview, we include a detailed proof of the continuous version of the Policy Gradient Theorem, convergence results and a comprehensive discussion of practical algorithm"},"claims":{"count":0,"items":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"2401.13662","kind":"arxiv","version":2},"verdict":{"id":null,"model_set":{},"created_at":null,"strongest_claim":"","one_line_summary":"","pipeline_version":null,"weakest_assumption":"","pith_extraction_headline":""},"integrity":{"clean":true,"summary":{"advisory":0,"critical":0,"by_detector":{},"informational":0},"endpoint":"/pith/2401.13662/integrity.json","findings":[],"available":true,"detectors_run":[],"snapshot_sha256":"c28c3603d3b5d939e8dc4c7e95fa8dfce3d595e45f758748cecf8e644a296938"},"references":{"count":0,"sample":[],"resolved_work":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","internal_anchors":0},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"author_claims":{"count":0,"strong_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"builder_version":"pith-number-builder-2026-05-17-v1"},"verdict_id":null},"signer":{"signer_id":"pith.science","signer_type":"pith_registry","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"created_at":"2026-07-05T07:50:55Z","supersedes":[],"prev_event":null,"signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"u7rxLq95smUeCOEJ4zv+EBFCFOeLISKE/TY74q+qJl5PH3HBAo0GvI6ZSNKOlaQ4fxPpeyXFIgvZLxmD3AWcBw==","signed_message":"open_graph_event_sha256_bytes","signed_at":"2026-08-13T22:53:39.254985Z"},"content_sha256":"c28cd2d94edeba24419eb40ec283120f2af20e57783990e1c9d2d9bc392ecfec","schema_version":"1.0","event_id":"sha256:c28cd2d94edeba24419eb40ec283120f2af20e57783990e1c9d2d9bc392ecfec"}],"timestamp_proofs":[],"mirror_hints":[{"mirror_type":"https","name":"Pith Resolver","base_url":"https://pith.science","bundle_url":"https://pith.science/pith/CSEWCPPIXNN5X7KAYHDMX5TF5T/bundle.json","state_url":"https://pith.science/pith/CSEWCPPIXNN5X7KAYHDMX5TF5T/state.json","well_known_bundle_url":"https://pith.science/.well-known/pith/CSEWCPPIXNN5X7KAYHDMX5TF5T/bundle.json","status":"primary"}],"public_keys":[{"key_id":"pith-v1-2026-05","algorithm":"ed25519","format":"raw","public_key_b64":"stVStoiQhXFxp4s2pdzPNoqVNBMojDU/fJ2db5S3CbM=","public_key_hex":"b2d552b68890857171a78b36a5dccf368a953413288c353f7c9d9d6f94b709b3","fingerprint_sha256_b32_first128bits":"RVFV5Z2OI2J3ZUO7ERDEBCYNKS","fingerprint_sha256_hex":"8d4b5ee74e4693bcd1df2446408b0d54","rotates_at":null,"url":"https://pith.science/pith-signing-key.json","notes":"Pith uses this Ed25519 key to sign canonical record SHA-256 digests. Verify with: ed25519_verify(public_key, message=canonical_sha256_bytes, signature=base64decode(signature_b64))."}],"merge_version":"pith-open-graph-merge-v1","built_at":"2026-08-13T22:53:39Z","links":{"resolver":"https://pith.science/pith/CSEWCPPIXNN5X7KAYHDMX5TF5T","bundle":"https://pith.science/pith/CSEWCPPIXNN5X7KAYHDMX5TF5T/bundle.json","state":"https://pith.science/pith/CSEWCPPIXNN5X7KAYHDMX5TF5T/state.json","well_known_bundle":"https://pith.science/.well-known/pith/CSEWCPPIXNN5X7KAYHDMX5TF5T/bundle.json"},"state":{"state_type":"pith_open_graph_state","state_version":"1.0","pith_number":"pith:2024:CSEWCPPIXNN5X7KAYHDMX5TF5T","merge_version":"pith-open-graph-merge-v1","event_count":2,"valid_event_count":2,"invalid_event_count":0,"equivocation_count":0,"current":{"canonical_record":{"metadata":{"abstract_canon_sha256":"9344fd39b2ce3eaab01fb1027b21b13cc624bc3f0eeaeb8cf3650faa5555fcb0","cross_cats_sorted":["cs.AI"],"license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.LG","submitted_at":"2024-01-24T18:56:53Z","title_canon_sha256":"5973d7eae3f1cf8d8dc5c63cdc6384e79a8fa369a211fc15202b63b2d2070ec1"},"schema_version":"1.0","source":{"id":"2401.13662","kind":"arxiv","version":2}},"source_aliases":[{"alias_kind":"arxiv","alias_value":"2401.13662","created_at":"2026-07-05T07:50:55Z"},{"alias_kind":"arxiv_version","alias_value":"2401.13662v2","created_at":"2026-07-05T07:50:55Z"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.2401.13662","created_at":"2026-07-05T07:50:55Z"},{"alias_kind":"pith_short_12","alias_value":"CSEWCPPIXNN5","created_at":"2026-07-05T07:50:55Z"},{"alias_kind":"pith_short_16","alias_value":"CSEWCPPIXNN5X7KA","created_at":"2026-07-05T07:50:55Z"},{"alias_kind":"pith_short_8","alias_value":"CSEWCPPI","created_at":"2026-07-05T07:50:55Z"}],"graph_snapshots":[{"event_id":"sha256:c28cd2d94edeba24419eb40ec283120f2af20e57783990e1c9d2d9bc392ecfec","target":"graph","created_at":"2026-07-05T07:50:55Z","signer":{"key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signer_id":"pith.science","signer_type":"pith_registry"},"payload":{"graph_snapshot":{"author_claims":{"count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","strong_count":0},"builder_version":"pith-number-builder-2026-05-17-v1","claims":{"count":0,"items":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"integrity":{"available":true,"clean":true,"detectors_run":[],"endpoint":"/pith/2401.13662/integrity.json","findings":[],"snapshot_sha256":"c28c3603d3b5d939e8dc4c7e95fa8dfce3d595e45f758748cecf8e644a296938","summary":{"advisory":0,"by_detector":{},"critical":0,"informational":0}},"paper":{"abstract_excerpt":"In recent years, various powerful policy gradient algorithms have been proposed in deep reinforcement learning. While all these algorithms build on the Policy Gradient Theorem, the specific design choices differ significantly across algorithms. We provide a holistic overview of on-policy policy gradient algorithms to facilitate the understanding of both their theoretical foundations and their practical implementations. In this overview, we include a detailed proof of the continuous version of the Policy Gradient Theorem, convergence results and a comprehensive discussion of practical algorithm","authors_text":"Matthias Lehmann","cross_cats":["cs.AI"],"headline":"","license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.LG","submitted_at":"2024-01-24T18:56:53Z","title":"The Definitive Guide to Policy Gradients in Deep Reinforcement Learning: Theory, Algorithms and Implementations"},"references":{"count":0,"internal_anchors":0,"resolved_work":0,"sample":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"2401.13662","kind":"arxiv","version":2},"verdict":{"created_at":null,"id":null,"model_set":{},"one_line_summary":"","pipeline_version":null,"pith_extraction_headline":"","strongest_claim":"","weakest_assumption":""}},"verdict_id":null}}],"author_attestations":[],"timestamp_anchors":[],"storage_attestations":[],"citation_signatures":[],"replication_records":[],"corrections":[],"mirror_hints":[],"record_created":{"event_id":"sha256:cff042d2e3a1d33e53449b2bd49eda6293f01c3173ff3f186d2eb1c701fbed68","target":"record","created_at":"2026-07-05T07:50:55Z","signer":{"key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signer_id":"pith.science","signer_type":"pith_registry"},"payload":{"attestation_state":"computed","canonical_record":{"metadata":{"abstract_canon_sha256":"9344fd39b2ce3eaab01fb1027b21b13cc624bc3f0eeaeb8cf3650faa5555fcb0","cross_cats_sorted":["cs.AI"],"license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.LG","submitted_at":"2024-01-24T18:56:53Z","title_canon_sha256":"5973d7eae3f1cf8d8dc5c63cdc6384e79a8fa369a211fc15202b63b2d2070ec1"},"schema_version":"1.0","source":{"id":"2401.13662","kind":"arxiv","version":2}},"canonical_sha256":"1489613de8bb5bdbfd40c1c6cbf665ecf73b673199297dfef9952ffbbd13bb10","receipt":{"algorithm":"ed25519","builder_version":"pith-number-builder-2026-05-17-v1","canonical_sha256":"1489613de8bb5bdbfd40c1c6cbf665ecf73b673199297dfef9952ffbbd13bb10","first_computed_at":"2026-07-05T07:50:55.093527Z","key_id":"pith-v1-2026-05","kind":"pith_receipt","last_reissued_at":"2026-07-05T07:50:55.093527Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","receipt_version":"0.3","signature_b64":"viNgv6WLl5oXNG+BPlRjdUL0LBWmyWc6L6LXPgUJ0fCC1LKvNCo8JOHsee3d4zKT2S3J8CQAumQcT5Nj0fXIBg==","signature_status":"signed_v1","signed_at":"2026-07-05T07:50:55.094119Z","signed_message":"canonical_sha256_bytes"},"source_id":"2401.13662","source_kind":"arxiv","source_version":2}}},"equivocations":[],"invalid_events":[],"applied_event_ids":["sha256:cff042d2e3a1d33e53449b2bd49eda6293f01c3173ff3f186d2eb1c701fbed68","sha256:c28cd2d94edeba24419eb40ec283120f2af20e57783990e1c9d2d9bc392ecfec"],"state_sha256":"20e40605c608195d171ca3d9bc8dcfaef22718dc64c52bce5f9c0ac96fcb3359"},"bundle_signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"qRPXmyLW9sH+a/0X81ZZ/P9mXYNArnMPZu0FQLcJfmZTEIFcLTOkRnRXTGIRBOk+UUGQacG1NyigQJ4KTjQWBQ==","signed_message":"bundle_sha256_bytes","signed_at":"2026-08-13T22:53:39.260707Z","bundle_sha256":"a6cd4a984d8c190982d2df306a4a4f7e004d42dc4cbc661ea175d313be799bbf"}}