{"record_type":"pith_number_record","schema_url":"https://pith.science/schemas/pith-number/v1.json","pith_number":"pith:2019:2R6FP2DZ4OAOHPC4WLGYRHMXQH","short_pith_number":"pith:2R6FP2DZ","schema_version":"1.0","canonical_sha256":"d47c57e879e380e3bc5cb2cd889d9781db0698ba1d8fd2b3b3cdcfaeb67865fb","source":{"kind":"arxiv","id":"1906.07073","version":2},"attestation_state":"computed","paper":{"title":"Is the Policy Gradient a Gradient?","license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","headline":"","cross_cats":["stat.ML"],"primary_cat":"cs.LG","authors_text":"Chris Nota, Philip S. Thomas","submitted_at":"2019-06-17T15:00:42Z","abstract_excerpt":"The policy gradient theorem describes the gradient of the expected discounted return with respect to an agent's policy parameters. However, most policy gradient methods drop the discount factor from the state distribution and therefore do not optimize the discounted objective. What do they optimize instead? This has been an open question for several years, and this lack of theoretical clarity has lead to an abundance of misstatements in the literature. We answer this question by proving that the update direction approximated by most methods is not the gradient of any function. Further, we argu"},"verification_status":{"content_addressed":true,"pith_receipt":true,"author_attested":false,"weak_author_claims":0,"strong_author_claims":0,"externally_anchored":false,"storage_verified":false,"citation_signatures":0,"replication_records":0,"graph_snapshot":true,"references_resolved":false,"formal_links_present":false},"canonical_record":{"source":{"id":"1906.07073","kind":"arxiv","version":2},"metadata":{"license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","primary_cat":"cs.LG","submitted_at":"2019-06-17T15:00:42Z","cross_cats_sorted":["stat.ML"],"title_canon_sha256":"6f8dfb420c5c42b1e3aa67edb39dfccbb018d66cf47b15f8730f5506ee61bf83","abstract_canon_sha256":"dd714e2c42ca437a892911ea68f105b939fe3aecc82c9a1c2dde62bd2965c401"},"schema_version":"1.0"},"receipt":{"kind":"pith_receipt","key_id":"pith-v1-2026-05","algorithm":"ed25519","signed_at":"2026-07-05T00:44:25.909185Z","signature_b64":"4ksqtjlfKtTicdKCHsuCggmPcYg3Qhm1O0bV8sPf9pAgQ73KjCqz4hzvhjCsm2XR0r7k0MEqM28Og9sHFbidDQ==","signed_message":"canonical_sha256_bytes","builder_version":"pith-number-builder-2026-05-17-v1","receipt_version":"0.3","canonical_sha256":"d47c57e879e380e3bc5cb2cd889d9781db0698ba1d8fd2b3b3cdcfaeb67865fb","last_reissued_at":"2026-07-05T00:44:25.908755Z","signature_status":"signed_v1","first_computed_at":"2026-07-05T00:44:25.908755Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"graph_snapshot":{"paper":{"title":"Is the Policy Gradient a Gradient?","license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","headline":"","cross_cats":["stat.ML"],"primary_cat":"cs.LG","authors_text":"Chris Nota, Philip S. Thomas","submitted_at":"2019-06-17T15:00:42Z","abstract_excerpt":"The policy gradient theorem describes the gradient of the expected discounted return with respect to an agent's policy parameters. However, most policy gradient methods drop the discount factor from the state distribution and therefore do not optimize the discounted objective. What do they optimize instead? This has been an open question for several years, and this lack of theoretical clarity has lead to an abundance of misstatements in the literature. We answer this question by proving that the update direction approximated by most methods is not the gradient of any function. Further, we argu"},"claims":{"count":0,"items":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"1906.07073","kind":"arxiv","version":2},"verdict":{"id":null,"model_set":{},"created_at":null,"strongest_claim":"","one_line_summary":"","pipeline_version":null,"weakest_assumption":"","pith_extraction_headline":""},"integrity":{"clean":true,"summary":{"advisory":0,"critical":0,"by_detector":{},"informational":0},"endpoint":"/pith/1906.07073/integrity.json","findings":[],"available":true,"detectors_run":[],"snapshot_sha256":"c28c3603d3b5d939e8dc4c7e95fa8dfce3d595e45f758748cecf8e644a296938"},"references":{"count":0,"sample":[],"resolved_work":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","internal_anchors":0},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"author_claims":{"count":0,"strong_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"builder_version":"pith-number-builder-2026-05-17-v1"},"aliases":[{"alias_kind":"arxiv","alias_value":"1906.07073","created_at":"2026-07-05T00:44:25.908811+00:00"},{"alias_kind":"arxiv_version","alias_value":"1906.07073v2","created_at":"2026-07-05T00:44:25.908811+00:00"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.1906.07073","created_at":"2026-07-05T00:44:25.908811+00:00"},{"alias_kind":"pith_short_12","alias_value":"2R6FP2DZ4OAO","created_at":"2026-07-05T00:44:25.908811+00:00"},{"alias_kind":"pith_short_16","alias_value":"2R6FP2DZ4OAOHPC4","created_at":"2026-07-05T00:44:25.908811+00:00"},{"alias_kind":"pith_short_8","alias_value":"2R6FP2DZ","created_at":"2026-07-05T00:44:25.908811+00:00"}],"events":[],"event_summary":{},"paper_claims":[],"inbound_citations":{"count":0,"internal_anchor_count":0,"sample":[]},"formal_canon":{"evidence_count":0,"sample":[],"anchors":[]},"links":{"html":"https://pith.science/pith/2R6FP2DZ4OAOHPC4WLGYRHMXQH","json":"https://pith.science/pith/2R6FP2DZ4OAOHPC4WLGYRHMXQH.json","graph_json":"https://pith.science/api/pith-number/2R6FP2DZ4OAOHPC4WLGYRHMXQH/graph.json","events_json":"https://pith.science/api/pith-number/2R6FP2DZ4OAOHPC4WLGYRHMXQH/events.json","paper":"https://pith.science/paper/2R6FP2DZ"},"agent_actions":{"view_html":"https://pith.science/pith/2R6FP2DZ4OAOHPC4WLGYRHMXQH","download_json":"https://pith.science/pith/2R6FP2DZ4OAOHPC4WLGYRHMXQH.json","view_paper":"https://pith.science/paper/2R6FP2DZ","resolve_alias":"https://pith.science/api/pith-number/resolve?arxiv=1906.07073&json=true","fetch_graph":"https://pith.science/api/pith-number/2R6FP2DZ4OAOHPC4WLGYRHMXQH/graph.json","fetch_events":"https://pith.science/api/pith-number/2R6FP2DZ4OAOHPC4WLGYRHMXQH/events.json","actions":{"anchor_timestamp":"https://pith.science/pith/2R6FP2DZ4OAOHPC4WLGYRHMXQH/action/timestamp_anchor","attest_storage":"https://pith.science/pith/2R6FP2DZ4OAOHPC4WLGYRHMXQH/action/storage_attestation","attest_author":"https://pith.science/pith/2R6FP2DZ4OAOHPC4WLGYRHMXQH/action/author_attestation","sign_citation":"https://pith.science/pith/2R6FP2DZ4OAOHPC4WLGYRHMXQH/action/citation_signature","submit_replication":"https://pith.science/pith/2R6FP2DZ4OAOHPC4WLGYRHMXQH/action/replication_record"}},"created_at":"2026-07-05T00:44:25.908811+00:00","updated_at":"2026-07-05T00:44:25.908811+00:00"}