{"bundle_type":"pith_open_graph_bundle","bundle_version":"1.0","pith_number":"pith:2025:4454ABHNZUFUCNFO2DAHERCE4N","short_pith_number":"pith:4454ABHN","canonical_record":{"source":{"id":"2506.11912","kind":"arxiv","version":1},"metadata":{"license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.LG","submitted_at":"2025-06-13T16:06:47Z","cross_cats_sorted":["cs.AI"],"title_canon_sha256":"6d4a8b787dac01bb3fceeeb30ab458bf8d29c26237cdd4e122ca537a043963a8","abstract_canon_sha256":"e2e4bd25745f8aa101669dfe3c65778d0ce34b8874ad9e31fff0a287bfaf4d9e"},"schema_version":"1.0"},"canonical_sha256":"e73bc004edcd0b4134aed0c0724444e3706684d041270066afa00c6ef2060554","source":{"kind":"arxiv","id":"2506.11912","version":1},"source_aliases":[{"alias_kind":"arxiv","alias_value":"2506.11912","created_at":"2026-07-05T11:21:12Z"},{"alias_kind":"arxiv_version","alias_value":"2506.11912v1","created_at":"2026-07-05T11:21:12Z"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.2506.11912","created_at":"2026-07-05T11:21:12Z"},{"alias_kind":"pith_short_12","alias_value":"4454ABHNZUFU","created_at":"2026-07-05T11:21:12Z"},{"alias_kind":"pith_short_16","alias_value":"4454ABHNZUFUCNFO","created_at":"2026-07-05T11:21:12Z"},{"alias_kind":"pith_short_8","alias_value":"4454ABHN","created_at":"2026-07-05T11:21:12Z"}],"events":[{"event_type":"record_created","subject_pith_number":"pith:2025:4454ABHNZUFUCNFO2DAHERCE4N","target":"record","payload":{"canonical_record":{"source":{"id":"2506.11912","kind":"arxiv","version":1},"metadata":{"license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.LG","submitted_at":"2025-06-13T16:06:47Z","cross_cats_sorted":["cs.AI"],"title_canon_sha256":"6d4a8b787dac01bb3fceeeb30ab458bf8d29c26237cdd4e122ca537a043963a8","abstract_canon_sha256":"e2e4bd25745f8aa101669dfe3c65778d0ce34b8874ad9e31fff0a287bfaf4d9e"},"schema_version":"1.0"},"canonical_sha256":"e73bc004edcd0b4134aed0c0724444e3706684d041270066afa00c6ef2060554","receipt":{"kind":"pith_receipt","key_id":"pith-v1-2026-05","algorithm":"ed25519","signed_at":"2026-07-05T11:21:12.696740Z","signature_b64":"4WU/9lpHZW3KIueOYclyezn8POBr/M/q32PWZ4lZ81XNUiBadWlvrMHQtLDkf4nyFrZWXeGh4ExEejRhs7q8BA==","signed_message":"canonical_sha256_bytes","builder_version":"pith-number-builder-2026-05-17-v1","receipt_version":"0.3","canonical_sha256":"e73bc004edcd0b4134aed0c0724444e3706684d041270066afa00c6ef2060554","last_reissued_at":"2026-07-05T11:21:12.696245Z","signature_status":"signed_v1","first_computed_at":"2026-07-05T11:21:12.696245Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"source_kind":"arxiv","source_id":"2506.11912","source_version":1,"attestation_state":"computed"},"signer":{"signer_id":"pith.science","signer_type":"pith_registry","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"created_at":"2026-07-05T11:21:12Z","supersedes":[],"prev_event":null,"signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"h4lyg+Wn3KOn84iq3MAuC72HD+uUJJuN5Ji7Acnltt+65UG0DxceF+vLotiRGk5GUClgXn+L/adFohYzUO+fDg==","signed_message":"open_graph_event_sha256_bytes","signed_at":"2026-08-09T07:30:43.667712Z"},"content_sha256":"2d47b11068933b448342e93f6641a9fa70c8d293e2be06aff0946e303b9f592f","schema_version":"1.0","event_id":"sha256:2d47b11068933b448342e93f6641a9fa70c8d293e2be06aff0946e303b9f592f"},{"event_type":"graph_snapshot","subject_pith_number":"pith:2025:4454ABHNZUFUCNFO2DAHERCE4N","target":"graph","payload":{"graph_snapshot":{"paper":{"title":"Breaking Habits: On the Role of the Advantage Function in Learning Causal State Representations","license":"http://creativecommons.org/licenses/by/4.0/","headline":"","cross_cats":["cs.AI"],"primary_cat":"cs.LG","authors_text":"Miguel Suau","submitted_at":"2025-06-13T16:06:47Z","abstract_excerpt":"Recent work has shown that reinforcement learning agents can develop policies that exploit spurious correlations between rewards and observations. This phenomenon, known as policy confounding, arises because the agent's policy influences both past and future observation variables, creating a feedback loop that can hinder the agent's ability to generalize beyond its usual trajectories. In this paper, we show that the advantage function, commonly used in policy gradient methods, not only reduces the variance of gradient estimates but also mitigates the effects of policy confounding. By adjusting"},"claims":{"count":0,"items":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"2506.11912","kind":"arxiv","version":1},"verdict":{"id":null,"model_set":{},"created_at":null,"strongest_claim":"","one_line_summary":"","pipeline_version":null,"weakest_assumption":"","pith_extraction_headline":""},"integrity":{"clean":true,"summary":{"advisory":0,"critical":0,"by_detector":{},"informational":0},"endpoint":"/pith/2506.11912/integrity.json","findings":[],"available":true,"detectors_run":[],"snapshot_sha256":"c28c3603d3b5d939e8dc4c7e95fa8dfce3d595e45f758748cecf8e644a296938"},"references":{"count":0,"sample":[],"resolved_work":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","internal_anchors":0},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"author_claims":{"count":0,"strong_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"builder_version":"pith-number-builder-2026-05-17-v1"},"verdict_id":null},"signer":{"signer_id":"pith.science","signer_type":"pith_registry","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"created_at":"2026-07-05T11:21:12Z","supersedes":[],"prev_event":null,"signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"BVAzTA+abkA/9qsKTBz7Vu06lLPxR+OyIsgh7EDv8U80ihn1iAvmKd4eSuchFcr9du5OUOC3AGFZyIiH0URYDg==","signed_message":"open_graph_event_sha256_bytes","signed_at":"2026-08-09T07:30:43.668226Z"},"content_sha256":"52a63e103bf6e5338526ee094057911c3092021c43e88175e7b882f4f6686a5b","schema_version":"1.0","event_id":"sha256:52a63e103bf6e5338526ee094057911c3092021c43e88175e7b882f4f6686a5b"}],"timestamp_proofs":[],"mirror_hints":[{"mirror_type":"https","name":"Pith Resolver","base_url":"https://pith.science","bundle_url":"https://pith.science/pith/4454ABHNZUFUCNFO2DAHERCE4N/bundle.json","state_url":"https://pith.science/pith/4454ABHNZUFUCNFO2DAHERCE4N/state.json","well_known_bundle_url":"https://pith.science/.well-known/pith/4454ABHNZUFUCNFO2DAHERCE4N/bundle.json","status":"primary"}],"public_keys":[{"key_id":"pith-v1-2026-05","algorithm":"ed25519","format":"raw","public_key_b64":"stVStoiQhXFxp4s2pdzPNoqVNBMojDU/fJ2db5S3CbM=","public_key_hex":"b2d552b68890857171a78b36a5dccf368a953413288c353f7c9d9d6f94b709b3","fingerprint_sha256_b32_first128bits":"RVFV5Z2OI2J3ZUO7ERDEBCYNKS","fingerprint_sha256_hex":"8d4b5ee74e4693bcd1df2446408b0d54","rotates_at":null,"url":"https://pith.science/pith-signing-key.json","notes":"Pith uses this Ed25519 key to sign canonical record SHA-256 digests. Verify with: ed25519_verify(public_key, message=canonical_sha256_bytes, signature=base64decode(signature_b64))."}],"merge_version":"pith-open-graph-merge-v1","built_at":"2026-08-09T07:30:43Z","links":{"resolver":"https://pith.science/pith/4454ABHNZUFUCNFO2DAHERCE4N","bundle":"https://pith.science/pith/4454ABHNZUFUCNFO2DAHERCE4N/bundle.json","state":"https://pith.science/pith/4454ABHNZUFUCNFO2DAHERCE4N/state.json","well_known_bundle":"https://pith.science/.well-known/pith/4454ABHNZUFUCNFO2DAHERCE4N/bundle.json"},"state":{"state_type":"pith_open_graph_state","state_version":"1.0","pith_number":"pith:2025:4454ABHNZUFUCNFO2DAHERCE4N","merge_version":"pith-open-graph-merge-v1","event_count":2,"valid_event_count":2,"invalid_event_count":0,"equivocation_count":0,"current":{"canonical_record":{"metadata":{"abstract_canon_sha256":"e2e4bd25745f8aa101669dfe3c65778d0ce34b8874ad9e31fff0a287bfaf4d9e","cross_cats_sorted":["cs.AI"],"license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.LG","submitted_at":"2025-06-13T16:06:47Z","title_canon_sha256":"6d4a8b787dac01bb3fceeeb30ab458bf8d29c26237cdd4e122ca537a043963a8"},"schema_version":"1.0","source":{"id":"2506.11912","kind":"arxiv","version":1}},"source_aliases":[{"alias_kind":"arxiv","alias_value":"2506.11912","created_at":"2026-07-05T11:21:12Z"},{"alias_kind":"arxiv_version","alias_value":"2506.11912v1","created_at":"2026-07-05T11:21:12Z"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.2506.11912","created_at":"2026-07-05T11:21:12Z"},{"alias_kind":"pith_short_12","alias_value":"4454ABHNZUFU","created_at":"2026-07-05T11:21:12Z"},{"alias_kind":"pith_short_16","alias_value":"4454ABHNZUFUCNFO","created_at":"2026-07-05T11:21:12Z"},{"alias_kind":"pith_short_8","alias_value":"4454ABHN","created_at":"2026-07-05T11:21:12Z"}],"graph_snapshots":[{"event_id":"sha256:52a63e103bf6e5338526ee094057911c3092021c43e88175e7b882f4f6686a5b","target":"graph","created_at":"2026-07-05T11:21:12Z","signer":{"key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signer_id":"pith.science","signer_type":"pith_registry"},"payload":{"graph_snapshot":{"author_claims":{"count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","strong_count":0},"builder_version":"pith-number-builder-2026-05-17-v1","claims":{"count":0,"items":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"integrity":{"available":true,"clean":true,"detectors_run":[],"endpoint":"/pith/2506.11912/integrity.json","findings":[],"snapshot_sha256":"c28c3603d3b5d939e8dc4c7e95fa8dfce3d595e45f758748cecf8e644a296938","summary":{"advisory":0,"by_detector":{},"critical":0,"informational":0}},"paper":{"abstract_excerpt":"Recent work has shown that reinforcement learning agents can develop policies that exploit spurious correlations between rewards and observations. This phenomenon, known as policy confounding, arises because the agent's policy influences both past and future observation variables, creating a feedback loop that can hinder the agent's ability to generalize beyond its usual trajectories. In this paper, we show that the advantage function, commonly used in policy gradient methods, not only reduces the variance of gradient estimates but also mitigates the effects of policy confounding. By adjusting","authors_text":"Miguel Suau","cross_cats":["cs.AI"],"headline":"","license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.LG","submitted_at":"2025-06-13T16:06:47Z","title":"Breaking Habits: On the Role of the Advantage Function in Learning Causal State Representations"},"references":{"count":0,"internal_anchors":0,"resolved_work":0,"sample":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"2506.11912","kind":"arxiv","version":1},"verdict":{"created_at":null,"id":null,"model_set":{},"one_line_summary":"","pipeline_version":null,"pith_extraction_headline":"","strongest_claim":"","weakest_assumption":""}},"verdict_id":null}}],"author_attestations":[],"timestamp_anchors":[],"storage_attestations":[],"citation_signatures":[],"replication_records":[],"corrections":[],"mirror_hints":[],"record_created":{"event_id":"sha256:2d47b11068933b448342e93f6641a9fa70c8d293e2be06aff0946e303b9f592f","target":"record","created_at":"2026-07-05T11:21:12Z","signer":{"key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signer_id":"pith.science","signer_type":"pith_registry"},"payload":{"attestation_state":"computed","canonical_record":{"metadata":{"abstract_canon_sha256":"e2e4bd25745f8aa101669dfe3c65778d0ce34b8874ad9e31fff0a287bfaf4d9e","cross_cats_sorted":["cs.AI"],"license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.LG","submitted_at":"2025-06-13T16:06:47Z","title_canon_sha256":"6d4a8b787dac01bb3fceeeb30ab458bf8d29c26237cdd4e122ca537a043963a8"},"schema_version":"1.0","source":{"id":"2506.11912","kind":"arxiv","version":1}},"canonical_sha256":"e73bc004edcd0b4134aed0c0724444e3706684d041270066afa00c6ef2060554","receipt":{"algorithm":"ed25519","builder_version":"pith-number-builder-2026-05-17-v1","canonical_sha256":"e73bc004edcd0b4134aed0c0724444e3706684d041270066afa00c6ef2060554","first_computed_at":"2026-07-05T11:21:12.696245Z","key_id":"pith-v1-2026-05","kind":"pith_receipt","last_reissued_at":"2026-07-05T11:21:12.696245Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","receipt_version":"0.3","signature_b64":"4WU/9lpHZW3KIueOYclyezn8POBr/M/q32PWZ4lZ81XNUiBadWlvrMHQtLDkf4nyFrZWXeGh4ExEejRhs7q8BA==","signature_status":"signed_v1","signed_at":"2026-07-05T11:21:12.696740Z","signed_message":"canonical_sha256_bytes"},"source_id":"2506.11912","source_kind":"arxiv","source_version":1}}},"equivocations":[],"invalid_events":[],"applied_event_ids":["sha256:2d47b11068933b448342e93f6641a9fa70c8d293e2be06aff0946e303b9f592f","sha256:52a63e103bf6e5338526ee094057911c3092021c43e88175e7b882f4f6686a5b"],"state_sha256":"6f930cab83f5057be3974c40a0a2169c8b9c7f47083f5351449947c9e185c482"},"bundle_signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"oD/2TmfhB7m1Zb3je9OIalZakllpi70a6YpiITKEwfUkXa251HGDDm7/J2lHs5+46J5TpMbIlFe+HolatWVfBA==","signed_message":"bundle_sha256_bytes","signed_at":"2026-08-09T07:30:43.671807Z","bundle_sha256":"f98fa17bbfa2922d3676592542e18ddf1761957ef0394616873b8bc1ae3009f6"}}