{"record_type":"pith_number_record","schema_url":"https://pith.science/schemas/pith-number/v1.json","pith_number":"pith:2025:4454ABHNZUFUCNFO2DAHERCE4N","short_pith_number":"pith:4454ABHN","schema_version":"1.0","canonical_sha256":"e73bc004edcd0b4134aed0c0724444e3706684d041270066afa00c6ef2060554","source":{"kind":"arxiv","id":"2506.11912","version":1},"attestation_state":"computed","paper":{"title":"Breaking Habits: On the Role of the Advantage Function in Learning Causal State Representations","license":"http://creativecommons.org/licenses/by/4.0/","headline":"","cross_cats":["cs.AI"],"primary_cat":"cs.LG","authors_text":"Miguel Suau","submitted_at":"2025-06-13T16:06:47Z","abstract_excerpt":"Recent work has shown that reinforcement learning agents can develop policies that exploit spurious correlations between rewards and observations. This phenomenon, known as policy confounding, arises because the agent's policy influences both past and future observation variables, creating a feedback loop that can hinder the agent's ability to generalize beyond its usual trajectories. In this paper, we show that the advantage function, commonly used in policy gradient methods, not only reduces the variance of gradient estimates but also mitigates the effects of policy confounding. By adjusting"},"verification_status":{"content_addressed":true,"pith_receipt":true,"author_attested":false,"weak_author_claims":0,"strong_author_claims":0,"externally_anchored":false,"storage_verified":false,"citation_signatures":0,"replication_records":0,"graph_snapshot":true,"references_resolved":false,"formal_links_present":false},"canonical_record":{"source":{"id":"2506.11912","kind":"arxiv","version":1},"metadata":{"license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.LG","submitted_at":"2025-06-13T16:06:47Z","cross_cats_sorted":["cs.AI"],"title_canon_sha256":"6d4a8b787dac01bb3fceeeb30ab458bf8d29c26237cdd4e122ca537a043963a8","abstract_canon_sha256":"e2e4bd25745f8aa101669dfe3c65778d0ce34b8874ad9e31fff0a287bfaf4d9e"},"schema_version":"1.0"},"receipt":{"kind":"pith_receipt","key_id":"pith-v1-2026-05","algorithm":"ed25519","signed_at":"2026-07-05T11:21:12.696740Z","signature_b64":"4WU/9lpHZW3KIueOYclyezn8POBr/M/q32PWZ4lZ81XNUiBadWlvrMHQtLDkf4nyFrZWXeGh4ExEejRhs7q8BA==","signed_message":"canonical_sha256_bytes","builder_version":"pith-number-builder-2026-05-17-v1","receipt_version":"0.3","canonical_sha256":"e73bc004edcd0b4134aed0c0724444e3706684d041270066afa00c6ef2060554","last_reissued_at":"2026-07-05T11:21:12.696245Z","signature_status":"signed_v1","first_computed_at":"2026-07-05T11:21:12.696245Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"graph_snapshot":{"paper":{"title":"Breaking Habits: On the Role of the Advantage Function in Learning Causal State Representations","license":"http://creativecommons.org/licenses/by/4.0/","headline":"","cross_cats":["cs.AI"],"primary_cat":"cs.LG","authors_text":"Miguel Suau","submitted_at":"2025-06-13T16:06:47Z","abstract_excerpt":"Recent work has shown that reinforcement learning agents can develop policies that exploit spurious correlations between rewards and observations. This phenomenon, known as policy confounding, arises because the agent's policy influences both past and future observation variables, creating a feedback loop that can hinder the agent's ability to generalize beyond its usual trajectories. In this paper, we show that the advantage function, commonly used in policy gradient methods, not only reduces the variance of gradient estimates but also mitigates the effects of policy confounding. By adjusting"},"claims":{"count":0,"items":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"2506.11912","kind":"arxiv","version":1},"verdict":{"id":null,"model_set":{},"created_at":null,"strongest_claim":"","one_line_summary":"","pipeline_version":null,"weakest_assumption":"","pith_extraction_headline":""},"integrity":{"clean":true,"summary":{"advisory":0,"critical":0,"by_detector":{},"informational":0},"endpoint":"/pith/2506.11912/integrity.json","findings":[],"available":true,"detectors_run":[],"snapshot_sha256":"c28c3603d3b5d939e8dc4c7e95fa8dfce3d595e45f758748cecf8e644a296938"},"references":{"count":0,"sample":[],"resolved_work":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","internal_anchors":0},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"author_claims":{"count":0,"strong_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"builder_version":"pith-number-builder-2026-05-17-v1"},"aliases":[{"alias_kind":"arxiv","alias_value":"2506.11912","created_at":"2026-07-05T11:21:12.696302+00:00"},{"alias_kind":"arxiv_version","alias_value":"2506.11912v1","created_at":"2026-07-05T11:21:12.696302+00:00"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.2506.11912","created_at":"2026-07-05T11:21:12.696302+00:00"},{"alias_kind":"pith_short_12","alias_value":"4454ABHNZUFU","created_at":"2026-07-05T11:21:12.696302+00:00"},{"alias_kind":"pith_short_16","alias_value":"4454ABHNZUFUCNFO","created_at":"2026-07-05T11:21:12.696302+00:00"},{"alias_kind":"pith_short_8","alias_value":"4454ABHN","created_at":"2026-07-05T11:21:12.696302+00:00"}],"events":[],"event_summary":{},"paper_claims":[],"inbound_citations":{"count":0,"internal_anchor_count":0,"sample":[]},"formal_canon":{"evidence_count":0,"sample":[],"anchors":[]},"links":{"html":"https://pith.science/pith/4454ABHNZUFUCNFO2DAHERCE4N","json":"https://pith.science/pith/4454ABHNZUFUCNFO2DAHERCE4N.json","graph_json":"https://pith.science/api/pith-number/4454ABHNZUFUCNFO2DAHERCE4N/graph.json","events_json":"https://pith.science/api/pith-number/4454ABHNZUFUCNFO2DAHERCE4N/events.json","paper":"https://pith.science/paper/4454ABHN"},"agent_actions":{"view_html":"https://pith.science/pith/4454ABHNZUFUCNFO2DAHERCE4N","download_json":"https://pith.science/pith/4454ABHNZUFUCNFO2DAHERCE4N.json","view_paper":"https://pith.science/paper/4454ABHN","resolve_alias":"https://pith.science/api/pith-number/resolve?arxiv=2506.11912&json=true","fetch_graph":"https://pith.science/api/pith-number/4454ABHNZUFUCNFO2DAHERCE4N/graph.json","fetch_events":"https://pith.science/api/pith-number/4454ABHNZUFUCNFO2DAHERCE4N/events.json","actions":{"anchor_timestamp":"https://pith.science/pith/4454ABHNZUFUCNFO2DAHERCE4N/action/timestamp_anchor","attest_storage":"https://pith.science/pith/4454ABHNZUFUCNFO2DAHERCE4N/action/storage_attestation","attest_author":"https://pith.science/pith/4454ABHNZUFUCNFO2DAHERCE4N/action/author_attestation","sign_citation":"https://pith.science/pith/4454ABHNZUFUCNFO2DAHERCE4N/action/citation_signature","submit_replication":"https://pith.science/pith/4454ABHNZUFUCNFO2DAHERCE4N/action/replication_record"}},"created_at":"2026-07-05T11:21:12.696302+00:00","updated_at":"2026-07-05T11:21:12.696302+00:00"}