{"bundle_type":"pith_open_graph_bundle","bundle_version":"1.0","pith_number":"pith:2024:3LPH36JOMYMUG76AAR5IZJRKIV","short_pith_number":"pith:3LPH36JO","canonical_record":{"source":{"id":"2403.03185","kind":"arxiv","version":4},"metadata":{"license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","primary_cat":"cs.LG","submitted_at":"2024-03-05T18:22:15Z","cross_cats_sorted":["cs.AI"],"title_canon_sha256":"80efa56f65cd8ee73b5ce41f39c4d81f6a6f366927960f7e49b3c07bac55b21f","abstract_canon_sha256":"4fcd0503fb5dbf2ee89de3ec3ef96ec48d24d40046bfcf351612693a305e0685"},"schema_version":"1.0"},"canonical_sha256":"dade7df92e6619437fc0047a8ca62a455ea56e1ad951ddf68d4d9773a07c2306","source":{"kind":"arxiv","id":"2403.03185","version":4},"source_aliases":[{"alias_kind":"arxiv","alias_value":"2403.03185","created_at":"2026-07-05T10:30:19Z"},{"alias_kind":"arxiv_version","alias_value":"2403.03185v4","created_at":"2026-07-05T10:30:19Z"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.2403.03185","created_at":"2026-07-05T10:30:19Z"},{"alias_kind":"pith_short_12","alias_value":"3LPH36JOMYMU","created_at":"2026-07-05T10:30:19Z"},{"alias_kind":"pith_short_16","alias_value":"3LPH36JOMYMUG76A","created_at":"2026-07-05T10:30:19Z"},{"alias_kind":"pith_short_8","alias_value":"3LPH36JO","created_at":"2026-07-05T10:30:19Z"}],"events":[{"event_type":"record_created","subject_pith_number":"pith:2024:3LPH36JOMYMUG76AAR5IZJRKIV","target":"record","payload":{"canonical_record":{"source":{"id":"2403.03185","kind":"arxiv","version":4},"metadata":{"license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","primary_cat":"cs.LG","submitted_at":"2024-03-05T18:22:15Z","cross_cats_sorted":["cs.AI"],"title_canon_sha256":"80efa56f65cd8ee73b5ce41f39c4d81f6a6f366927960f7e49b3c07bac55b21f","abstract_canon_sha256":"4fcd0503fb5dbf2ee89de3ec3ef96ec48d24d40046bfcf351612693a305e0685"},"schema_version":"1.0"},"canonical_sha256":"dade7df92e6619437fc0047a8ca62a455ea56e1ad951ddf68d4d9773a07c2306","receipt":{"kind":"pith_receipt","key_id":"pith-v1-2026-05","algorithm":"ed25519","signed_at":"2026-07-05T10:30:19.788512Z","signature_b64":"yaKcKhgeXn/cYEHNTPDakK4qaI5yneUruoY6rYnXKtmK0RnLziEcPCpbssACupdDCXSxVEbZ/3PUmptF5OqxCQ==","signed_message":"canonical_sha256_bytes","builder_version":"pith-number-builder-2026-05-17-v1","receipt_version":"0.3","canonical_sha256":"dade7df92e6619437fc0047a8ca62a455ea56e1ad951ddf68d4d9773a07c2306","last_reissued_at":"2026-07-05T10:30:19.787934Z","signature_status":"signed_v1","first_computed_at":"2026-07-05T10:30:19.787934Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"source_kind":"arxiv","source_id":"2403.03185","source_version":4,"attestation_state":"computed"},"signer":{"signer_id":"pith.science","signer_type":"pith_registry","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"created_at":"2026-07-05T10:30:19Z","supersedes":[],"prev_event":null,"signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"CpqJMuiLpjJPI60vD9u5GRQF6yr1WgQua3l2hhXnGrtwldtgcs4rDiF7R30dVkWY32nxpNsm7gw90LTP/ElSDg==","signed_message":"open_graph_event_sha256_bytes","signed_at":"2026-08-04T23:04:10.921378Z"},"content_sha256":"d54de4038ba23ce305b9db1aad6a0b9e9c4953035e14bcdf0006553f0baf8395","schema_version":"1.0","event_id":"sha256:d54de4038ba23ce305b9db1aad6a0b9e9c4953035e14bcdf0006553f0baf8395"},{"event_type":"graph_snapshot","subject_pith_number":"pith:2024:3LPH36JOMYMUG76AAR5IZJRKIV","target":"graph","payload":{"graph_snapshot":{"paper":{"title":"Correlated Proxies: A New Definition and Improved Mitigation for Reward Hacking","license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","headline":"","cross_cats":["cs.AI"],"primary_cat":"cs.LG","authors_text":"Anca Dragan, Cassidy Laidlaw, Shivam Singhal","submitted_at":"2024-03-05T18:22:15Z","abstract_excerpt":"Because it is difficult to precisely specify complex objectives, reinforcement learning policies are often optimized using proxy reward functions that only approximate the true goal. However, optimizing proxy rewards frequently leads to reward hacking: the optimized reward function ceases to be a good proxy and the resulting policy performs poorly with respect to the unspecified true reward. Principled solutions to reward hacking have been impeded by the lack of a good definition for the problem. To address this gap, we introduce a definition of reward hacking based on the correlation between "},"claims":{"count":0,"items":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"2403.03185","kind":"arxiv","version":4},"verdict":{"id":null,"model_set":{},"created_at":null,"strongest_claim":"","one_line_summary":"","pipeline_version":null,"weakest_assumption":"","pith_extraction_headline":""},"integrity":{"clean":true,"summary":{"advisory":0,"critical":0,"by_detector":{},"informational":0},"endpoint":"/pith/2403.03185/integrity.json","findings":[],"available":true,"detectors_run":[],"snapshot_sha256":"c28c3603d3b5d939e8dc4c7e95fa8dfce3d595e45f758748cecf8e644a296938"},"references":{"count":0,"sample":[],"resolved_work":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","internal_anchors":0},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"author_claims":{"count":0,"strong_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"builder_version":"pith-number-builder-2026-05-17-v1"},"verdict_id":null},"signer":{"signer_id":"pith.science","signer_type":"pith_registry","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"created_at":"2026-07-05T10:30:19Z","supersedes":[],"prev_event":null,"signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"osVVeGNE+m6Hl3VDlNTAHkMqT9y3o0LrXbjsX/JOnMFtNG7D5wQY8SI2d0VnyrK7bfREgHN20QPuytaWpmfkDQ==","signed_message":"open_graph_event_sha256_bytes","signed_at":"2026-08-04T23:04:10.922326Z"},"content_sha256":"fa15c5cf86c99ee96b29a8d3bc5040aeeab4f08f9061c68707e925ec8cb27a13","schema_version":"1.0","event_id":"sha256:fa15c5cf86c99ee96b29a8d3bc5040aeeab4f08f9061c68707e925ec8cb27a13"}],"timestamp_proofs":[],"mirror_hints":[{"mirror_type":"https","name":"Pith Resolver","base_url":"https://pith.science","bundle_url":"https://pith.science/pith/3LPH36JOMYMUG76AAR5IZJRKIV/bundle.json","state_url":"https://pith.science/pith/3LPH36JOMYMUG76AAR5IZJRKIV/state.json","well_known_bundle_url":"https://pith.science/.well-known/pith/3LPH36JOMYMUG76AAR5IZJRKIV/bundle.json","status":"primary"}],"public_keys":[{"key_id":"pith-v1-2026-05","algorithm":"ed25519","format":"raw","public_key_b64":"stVStoiQhXFxp4s2pdzPNoqVNBMojDU/fJ2db5S3CbM=","public_key_hex":"b2d552b68890857171a78b36a5dccf368a953413288c353f7c9d9d6f94b709b3","fingerprint_sha256_b32_first128bits":"RVFV5Z2OI2J3ZUO7ERDEBCYNKS","fingerprint_sha256_hex":"8d4b5ee74e4693bcd1df2446408b0d54","rotates_at":null,"url":"https://pith.science/pith-signing-key.json","notes":"Pith uses this Ed25519 key to sign canonical record SHA-256 digests. Verify with: ed25519_verify(public_key, message=canonical_sha256_bytes, signature=base64decode(signature_b64))."}],"merge_version":"pith-open-graph-merge-v1","built_at":"2026-08-04T23:04:10Z","links":{"resolver":"https://pith.science/pith/3LPH36JOMYMUG76AAR5IZJRKIV","bundle":"https://pith.science/pith/3LPH36JOMYMUG76AAR5IZJRKIV/bundle.json","state":"https://pith.science/pith/3LPH36JOMYMUG76AAR5IZJRKIV/state.json","well_known_bundle":"https://pith.science/.well-known/pith/3LPH36JOMYMUG76AAR5IZJRKIV/bundle.json"},"state":{"state_type":"pith_open_graph_state","state_version":"1.0","pith_number":"pith:2024:3LPH36JOMYMUG76AAR5IZJRKIV","merge_version":"pith-open-graph-merge-v1","event_count":2,"valid_event_count":2,"invalid_event_count":0,"equivocation_count":0,"current":{"canonical_record":{"metadata":{"abstract_canon_sha256":"4fcd0503fb5dbf2ee89de3ec3ef96ec48d24d40046bfcf351612693a305e0685","cross_cats_sorted":["cs.AI"],"license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","primary_cat":"cs.LG","submitted_at":"2024-03-05T18:22:15Z","title_canon_sha256":"80efa56f65cd8ee73b5ce41f39c4d81f6a6f366927960f7e49b3c07bac55b21f"},"schema_version":"1.0","source":{"id":"2403.03185","kind":"arxiv","version":4}},"source_aliases":[{"alias_kind":"arxiv","alias_value":"2403.03185","created_at":"2026-07-05T10:30:19Z"},{"alias_kind":"arxiv_version","alias_value":"2403.03185v4","created_at":"2026-07-05T10:30:19Z"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.2403.03185","created_at":"2026-07-05T10:30:19Z"},{"alias_kind":"pith_short_12","alias_value":"3LPH36JOMYMU","created_at":"2026-07-05T10:30:19Z"},{"alias_kind":"pith_short_16","alias_value":"3LPH36JOMYMUG76A","created_at":"2026-07-05T10:30:19Z"},{"alias_kind":"pith_short_8","alias_value":"3LPH36JO","created_at":"2026-07-05T10:30:19Z"}],"graph_snapshots":[{"event_id":"sha256:fa15c5cf86c99ee96b29a8d3bc5040aeeab4f08f9061c68707e925ec8cb27a13","target":"graph","created_at":"2026-07-05T10:30:19Z","signer":{"key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signer_id":"pith.science","signer_type":"pith_registry"},"payload":{"graph_snapshot":{"author_claims":{"count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","strong_count":0},"builder_version":"pith-number-builder-2026-05-17-v1","claims":{"count":0,"items":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"integrity":{"available":true,"clean":true,"detectors_run":[],"endpoint":"/pith/2403.03185/integrity.json","findings":[],"snapshot_sha256":"c28c3603d3b5d939e8dc4c7e95fa8dfce3d595e45f758748cecf8e644a296938","summary":{"advisory":0,"by_detector":{},"critical":0,"informational":0}},"paper":{"abstract_excerpt":"Because it is difficult to precisely specify complex objectives, reinforcement learning policies are often optimized using proxy reward functions that only approximate the true goal. However, optimizing proxy rewards frequently leads to reward hacking: the optimized reward function ceases to be a good proxy and the resulting policy performs poorly with respect to the unspecified true reward. Principled solutions to reward hacking have been impeded by the lack of a good definition for the problem. To address this gap, we introduce a definition of reward hacking based on the correlation between ","authors_text":"Anca Dragan, Cassidy Laidlaw, Shivam Singhal","cross_cats":["cs.AI"],"headline":"","license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","primary_cat":"cs.LG","submitted_at":"2024-03-05T18:22:15Z","title":"Correlated Proxies: A New Definition and Improved Mitigation for Reward Hacking"},"references":{"count":0,"internal_anchors":0,"resolved_work":0,"sample":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"2403.03185","kind":"arxiv","version":4},"verdict":{"created_at":null,"id":null,"model_set":{},"one_line_summary":"","pipeline_version":null,"pith_extraction_headline":"","strongest_claim":"","weakest_assumption":""}},"verdict_id":null}}],"author_attestations":[],"timestamp_anchors":[],"storage_attestations":[],"citation_signatures":[],"replication_records":[],"corrections":[],"mirror_hints":[],"record_created":{"event_id":"sha256:d54de4038ba23ce305b9db1aad6a0b9e9c4953035e14bcdf0006553f0baf8395","target":"record","created_at":"2026-07-05T10:30:19Z","signer":{"key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signer_id":"pith.science","signer_type":"pith_registry"},"payload":{"attestation_state":"computed","canonical_record":{"metadata":{"abstract_canon_sha256":"4fcd0503fb5dbf2ee89de3ec3ef96ec48d24d40046bfcf351612693a305e0685","cross_cats_sorted":["cs.AI"],"license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","primary_cat":"cs.LG","submitted_at":"2024-03-05T18:22:15Z","title_canon_sha256":"80efa56f65cd8ee73b5ce41f39c4d81f6a6f366927960f7e49b3c07bac55b21f"},"schema_version":"1.0","source":{"id":"2403.03185","kind":"arxiv","version":4}},"canonical_sha256":"dade7df92e6619437fc0047a8ca62a455ea56e1ad951ddf68d4d9773a07c2306","receipt":{"algorithm":"ed25519","builder_version":"pith-number-builder-2026-05-17-v1","canonical_sha256":"dade7df92e6619437fc0047a8ca62a455ea56e1ad951ddf68d4d9773a07c2306","first_computed_at":"2026-07-05T10:30:19.787934Z","key_id":"pith-v1-2026-05","kind":"pith_receipt","last_reissued_at":"2026-07-05T10:30:19.787934Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","receipt_version":"0.3","signature_b64":"yaKcKhgeXn/cYEHNTPDakK4qaI5yneUruoY6rYnXKtmK0RnLziEcPCpbssACupdDCXSxVEbZ/3PUmptF5OqxCQ==","signature_status":"signed_v1","signed_at":"2026-07-05T10:30:19.788512Z","signed_message":"canonical_sha256_bytes"},"source_id":"2403.03185","source_kind":"arxiv","source_version":4}}},"equivocations":[],"invalid_events":[],"applied_event_ids":["sha256:d54de4038ba23ce305b9db1aad6a0b9e9c4953035e14bcdf0006553f0baf8395","sha256:fa15c5cf86c99ee96b29a8d3bc5040aeeab4f08f9061c68707e925ec8cb27a13"],"state_sha256":"05c878a0bc591e819c4ca864ac33286de2b056f1818033fed0b4a74a6b849023"},"bundle_signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"eU7E4BQRhgx6+hIWkG0uDs2w5ZsM92CHpViGlb6fJ6Z+EitiwREOPQMSZ0dcOliP+CDQbgS2bN4oSakxhe8ABg==","signed_message":"bundle_sha256_bytes","signed_at":"2026-08-04T23:04:10.929373Z","bundle_sha256":"78620670d0cedffae5f54497d113f91f23f180266b5a3f826303a88a2141afc6"}}