{"record_type":"pith_number_record","schema_url":"https://pith.science/schemas/pith-number/v1.json","pith_number":"pith:2020:F2YQCZKHHBCT6IHQQ2QNADWJO4","short_pith_number":"pith:F2YQCZKH","schema_version":"1.0","canonical_sha256":"2eb101654738453f20f086a0d00ec977337266d31fc78800d49e34f04b3179e6","source":{"kind":"arxiv","id":"2011.09464","version":2},"attestation_state":"computed","paper":{"title":"Counterfactual Credit Assignment in Model-Free Reinforcement Learning","license":"http://creativecommons.org/licenses/by/4.0/","headline":"","cross_cats":[],"primary_cat":"cs.LG","authors_text":"Alaa Saade, Anna Harutyunyan, Arthur Guez, \\'Eric Moulines, Fabio Viola, Lars Buesing, Marcus Hutter, Nicolas Heess, R\\'emi Munos, Shantanu Thakoor, Th\\'eophane Weber, Thomas Mesnard, Tom Stepleton, Will Dabney","submitted_at":"2020-11-18T18:41:44Z","abstract_excerpt":"Credit assignment in reinforcement learning is the problem of measuring an action's influence on future rewards. In particular, this requires separating skill from luck, i.e. disentangling the effect of an action on rewards from that of external factors and subsequent actions. To achieve this, we adapt the notion of counterfactuals from causality theory to a model-free RL setup. The key idea is to condition value functions on future events, by learning to extract relevant information from a trajectory. We formulate a family of policy gradient algorithms that use these future-conditional value "},"verification_status":{"content_addressed":true,"pith_receipt":true,"author_attested":false,"weak_author_claims":0,"strong_author_claims":0,"externally_anchored":false,"storage_verified":false,"citation_signatures":0,"replication_records":0,"graph_snapshot":true,"references_resolved":false,"formal_links_present":false},"canonical_record":{"source":{"id":"2011.09464","kind":"arxiv","version":2},"metadata":{"license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.LG","submitted_at":"2020-11-18T18:41:44Z","cross_cats_sorted":[],"title_canon_sha256":"495b38648bce874725aaceef93f8e43c863c590f855bd8356742760369f7b806","abstract_canon_sha256":"0e7524f6c61935c2804c9cdb1f43a93e92cbee87bc85638af21fe31a06dc9ff3"},"schema_version":"1.0"},"receipt":{"kind":"pith_receipt","key_id":"pith-v1-2026-05","algorithm":"ed25519","signed_at":"2026-07-05T03:40:28.797243Z","signature_b64":"fDL/LlHEXa5H5Fuz3D2u3pzA/i+14aGW/CMKZ95lzuVdG7WeZl69IqHAyvgTZvfqVG6VlEmwLZy57x45SyZXAQ==","signed_message":"canonical_sha256_bytes","builder_version":"pith-number-builder-2026-05-17-v1","receipt_version":"0.3","canonical_sha256":"2eb101654738453f20f086a0d00ec977337266d31fc78800d49e34f04b3179e6","last_reissued_at":"2026-07-05T03:40:28.796779Z","signature_status":"signed_v1","first_computed_at":"2026-07-05T03:40:28.796779Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"graph_snapshot":{"paper":{"title":"Counterfactual Credit Assignment in Model-Free Reinforcement Learning","license":"http://creativecommons.org/licenses/by/4.0/","headline":"","cross_cats":[],"primary_cat":"cs.LG","authors_text":"Alaa Saade, Anna Harutyunyan, Arthur Guez, \\'Eric Moulines, Fabio Viola, Lars Buesing, Marcus Hutter, Nicolas Heess, R\\'emi Munos, Shantanu Thakoor, Th\\'eophane Weber, Thomas Mesnard, Tom Stepleton, Will Dabney","submitted_at":"2020-11-18T18:41:44Z","abstract_excerpt":"Credit assignment in reinforcement learning is the problem of measuring an action's influence on future rewards. In particular, this requires separating skill from luck, i.e. disentangling the effect of an action on rewards from that of external factors and subsequent actions. To achieve this, we adapt the notion of counterfactuals from causality theory to a model-free RL setup. The key idea is to condition value functions on future events, by learning to extract relevant information from a trajectory. We formulate a family of policy gradient algorithms that use these future-conditional value "},"claims":{"count":0,"items":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"2011.09464","kind":"arxiv","version":2},"verdict":{"id":null,"model_set":{},"created_at":null,"strongest_claim":"","one_line_summary":"","pipeline_version":null,"weakest_assumption":"","pith_extraction_headline":""},"integrity":{"clean":true,"summary":{"advisory":0,"critical":0,"by_detector":{},"informational":0},"endpoint":"/pith/2011.09464/integrity.json","findings":[],"available":true,"detectors_run":[],"snapshot_sha256":"c28c3603d3b5d939e8dc4c7e95fa8dfce3d595e45f758748cecf8e644a296938"},"references":{"count":0,"sample":[],"resolved_work":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","internal_anchors":0},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"author_claims":{"count":0,"strong_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"builder_version":"pith-number-builder-2026-05-17-v1"},"aliases":[{"alias_kind":"arxiv","alias_value":"2011.09464","created_at":"2026-07-05T03:40:28.796832+00:00"},{"alias_kind":"arxiv_version","alias_value":"2011.09464v2","created_at":"2026-07-05T03:40:28.796832+00:00"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.2011.09464","created_at":"2026-07-05T03:40:28.796832+00:00"},{"alias_kind":"pith_short_12","alias_value":"F2YQCZKHHBCT","created_at":"2026-07-05T03:40:28.796832+00:00"},{"alias_kind":"pith_short_16","alias_value":"F2YQCZKHHBCT6IHQ","created_at":"2026-07-05T03:40:28.796832+00:00"},{"alias_kind":"pith_short_8","alias_value":"F2YQCZKH","created_at":"2026-07-05T03:40:28.796832+00:00"}],"events":[],"event_summary":{},"paper_claims":[],"inbound_citations":{"count":3,"internal_anchor_count":0,"sample":[{"citing_arxiv_id":"2606.08275","citing_title":"Causal Agent Replay: Counterfactual Attribution for LLM-Agent Failures","ref_index":4,"is_internal_anchor":false},{"citing_arxiv_id":"2502.16156","citing_title":"A Review of Causal Decision Making","ref_index":26,"is_internal_anchor":false},{"citing_arxiv_id":"2106.01345","citing_title":"Decision Transformer: Reinforcement Learning via Sequence Modeling","ref_index":12,"is_internal_anchor":false}]},"formal_canon":{"evidence_count":0,"sample":[],"anchors":[]},"links":{"html":"https://pith.science/pith/F2YQCZKHHBCT6IHQQ2QNADWJO4","json":"https://pith.science/pith/F2YQCZKHHBCT6IHQQ2QNADWJO4.json","graph_json":"https://pith.science/api/pith-number/F2YQCZKHHBCT6IHQQ2QNADWJO4/graph.json","events_json":"https://pith.science/api/pith-number/F2YQCZKHHBCT6IHQQ2QNADWJO4/events.json","paper":"https://pith.science/paper/F2YQCZKH"},"agent_actions":{"view_html":"https://pith.science/pith/F2YQCZKHHBCT6IHQQ2QNADWJO4","download_json":"https://pith.science/pith/F2YQCZKHHBCT6IHQQ2QNADWJO4.json","view_paper":"https://pith.science/paper/F2YQCZKH","resolve_alias":"https://pith.science/api/pith-number/resolve?arxiv=2011.09464&json=true","fetch_graph":"https://pith.science/api/pith-number/F2YQCZKHHBCT6IHQQ2QNADWJO4/graph.json","fetch_events":"https://pith.science/api/pith-number/F2YQCZKHHBCT6IHQQ2QNADWJO4/events.json","actions":{"anchor_timestamp":"https://pith.science/pith/F2YQCZKHHBCT6IHQQ2QNADWJO4/action/timestamp_anchor","attest_storage":"https://pith.science/pith/F2YQCZKHHBCT6IHQQ2QNADWJO4/action/storage_attestation","attest_author":"https://pith.science/pith/F2YQCZKHHBCT6IHQQ2QNADWJO4/action/author_attestation","sign_citation":"https://pith.science/pith/F2YQCZKHHBCT6IHQQ2QNADWJO4/action/citation_signature","submit_replication":"https://pith.science/pith/F2YQCZKHHBCT6IHQQ2QNADWJO4/action/replication_record"}},"created_at":"2026-07-05T03:40:28.796832+00:00","updated_at":"2026-07-05T03:40:28.796832+00:00"}