{"bundle_type":"pith_open_graph_bundle","bundle_version":"1.0","pith_number":"pith:2025:VPWA75BNUQNBY7ATIKAPFLOTBK","short_pith_number":"pith:VPWA75BN","canonical_record":{"source":{"id":"2503.06810","kind":"arxiv","version":1},"metadata":{"license":"http://creativecommons.org/licenses/by-nc-sa/4.0/","primary_cat":"cs.LG","submitted_at":"2025-03-10T00:13:19Z","cross_cats_sorted":["cs.AI"],"title_canon_sha256":"0bfbfcc8a915be1499e1a0af94b4c1e5af80f13d6fa7c28931e88d4f24f98da6","abstract_canon_sha256":"69fae6b811058d5ba899a4d9944f38098cdc23a982db1e686dafd31970b2e4ca"},"schema_version":"1.0"},"canonical_sha256":"abec0ff42da41a1c7c134280f2add30a9a6927dd65aec5d6327dc400ea18b4fa","source":{"kind":"arxiv","id":"2503.06810","version":1},"source_aliases":[{"alias_kind":"arxiv","alias_value":"2503.06810","created_at":"2026-07-05T10:27:43Z"},{"alias_kind":"arxiv_version","alias_value":"2503.06810v1","created_at":"2026-07-05T10:27:43Z"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.2503.06810","created_at":"2026-07-05T10:27:43Z"},{"alias_kind":"pith_short_12","alias_value":"VPWA75BNUQNB","created_at":"2026-07-05T10:27:43Z"},{"alias_kind":"pith_short_16","alias_value":"VPWA75BNUQNBY7AT","created_at":"2026-07-05T10:27:43Z"},{"alias_kind":"pith_short_8","alias_value":"VPWA75BN","created_at":"2026-07-05T10:27:43Z"}],"events":[{"event_type":"record_created","subject_pith_number":"pith:2025:VPWA75BNUQNBY7ATIKAPFLOTBK","target":"record","payload":{"canonical_record":{"source":{"id":"2503.06810","kind":"arxiv","version":1},"metadata":{"license":"http://creativecommons.org/licenses/by-nc-sa/4.0/","primary_cat":"cs.LG","submitted_at":"2025-03-10T00:13:19Z","cross_cats_sorted":["cs.AI"],"title_canon_sha256":"0bfbfcc8a915be1499e1a0af94b4c1e5af80f13d6fa7c28931e88d4f24f98da6","abstract_canon_sha256":"69fae6b811058d5ba899a4d9944f38098cdc23a982db1e686dafd31970b2e4ca"},"schema_version":"1.0"},"canonical_sha256":"abec0ff42da41a1c7c134280f2add30a9a6927dd65aec5d6327dc400ea18b4fa","receipt":{"kind":"pith_receipt","key_id":"pith-v1-2026-05","algorithm":"ed25519","signed_at":"2026-07-05T10:27:43.055724Z","signature_b64":"Lv0I1VhWVrMVtqQ+U4KcjAUoC1f1yit/Axm6pg6pXQMcKeJ0xy8SIcA2jsmxY92ZfRKaKoWanpXRFzdmPOfJCw==","signed_message":"canonical_sha256_bytes","builder_version":"pith-number-builder-2026-05-17-v1","receipt_version":"0.3","canonical_sha256":"abec0ff42da41a1c7c134280f2add30a9a6927dd65aec5d6327dc400ea18b4fa","last_reissued_at":"2026-07-05T10:27:43.055104Z","signature_status":"signed_v1","first_computed_at":"2026-07-05T10:27:43.055104Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"source_kind":"arxiv","source_id":"2503.06810","source_version":1,"attestation_state":"computed"},"signer":{"signer_id":"pith.science","signer_type":"pith_registry","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"created_at":"2026-07-05T10:27:43Z","supersedes":[],"prev_event":null,"signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"jqZpQXv+4GjAjxnHFQ/izitCTQnQEo7DWoOrkGkX+P3oCvH8pUZ6i37b4yPAe8ig2pbvpMtJK/YlIzYeTZ8jBg==","signed_message":"open_graph_event_sha256_bytes","signed_at":"2026-08-07T03:40:21.674729Z"},"content_sha256":"d14e446b0ec63240b7c900e97f6078984bc18ae9d54c09a22096732078c7ce32","schema_version":"1.0","event_id":"sha256:d14e446b0ec63240b7c900e97f6078984bc18ae9d54c09a22096732078c7ce32"},{"event_type":"graph_snapshot","subject_pith_number":"pith:2025:VPWA75BNUQNBY7ATIKAPFLOTBK","target":"graph","payload":{"graph_snapshot":{"paper":{"title":"Mitigating Preference Hacking in Policy Optimization with Pessimism","license":"http://creativecommons.org/licenses/by-nc-sa/4.0/","headline":"","cross_cats":["cs.AI"],"primary_cat":"cs.LG","authors_text":"Adam Fisch, Alekh Agarwal, Christoph Dann, Dhawal Gupta","submitted_at":"2025-03-10T00:13:19Z","abstract_excerpt":"This work tackles the problem of overoptimization in reinforcement learning from human feedback (RLHF), a prevalent technique for aligning models with human preferences. RLHF relies on reward or preference models trained on \\emph{fixed preference datasets}, and these models are unreliable when evaluated outside the support of this preference data, leading to the common reward or preference hacking phenomenon. We propose novel, pessimistic objectives for RLHF which are provably robust to overoptimization through the use of pessimism in the face of uncertainty, and design practical algorithms, P"},"claims":{"count":0,"items":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"2503.06810","kind":"arxiv","version":1},"verdict":{"id":null,"model_set":{},"created_at":null,"strongest_claim":"","one_line_summary":"","pipeline_version":null,"weakest_assumption":"","pith_extraction_headline":""},"integrity":{"clean":true,"summary":{"advisory":0,"critical":0,"by_detector":{},"informational":0},"endpoint":"/pith/2503.06810/integrity.json","findings":[],"available":true,"detectors_run":[],"snapshot_sha256":"c28c3603d3b5d939e8dc4c7e95fa8dfce3d595e45f758748cecf8e644a296938"},"references":{"count":0,"sample":[],"resolved_work":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","internal_anchors":0},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"author_claims":{"count":0,"strong_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"builder_version":"pith-number-builder-2026-05-17-v1"},"verdict_id":null},"signer":{"signer_id":"pith.science","signer_type":"pith_registry","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"created_at":"2026-07-05T10:27:43Z","supersedes":[],"prev_event":null,"signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"eFUh43SqKkfYMGNd6Y43orHkeDCzoyzsKnO/MJis7RAKymti6sXyro6dpI9nMiJlgK0JWqvxqTDarnzUD/VMBw==","signed_message":"open_graph_event_sha256_bytes","signed_at":"2026-08-07T03:40:21.675654Z"},"content_sha256":"5c16ef4833c424914ba831211e0a09af4b98f47402d94164a47c5bc2e1b39f4d","schema_version":"1.0","event_id":"sha256:5c16ef4833c424914ba831211e0a09af4b98f47402d94164a47c5bc2e1b39f4d"}],"timestamp_proofs":[],"mirror_hints":[{"mirror_type":"https","name":"Pith Resolver","base_url":"https://pith.science","bundle_url":"https://pith.science/pith/VPWA75BNUQNBY7ATIKAPFLOTBK/bundle.json","state_url":"https://pith.science/pith/VPWA75BNUQNBY7ATIKAPFLOTBK/state.json","well_known_bundle_url":"https://pith.science/.well-known/pith/VPWA75BNUQNBY7ATIKAPFLOTBK/bundle.json","status":"primary"}],"public_keys":[{"key_id":"pith-v1-2026-05","algorithm":"ed25519","format":"raw","public_key_b64":"stVStoiQhXFxp4s2pdzPNoqVNBMojDU/fJ2db5S3CbM=","public_key_hex":"b2d552b68890857171a78b36a5dccf368a953413288c353f7c9d9d6f94b709b3","fingerprint_sha256_b32_first128bits":"RVFV5Z2OI2J3ZUO7ERDEBCYNKS","fingerprint_sha256_hex":"8d4b5ee74e4693bcd1df2446408b0d54","rotates_at":null,"url":"https://pith.science/pith-signing-key.json","notes":"Pith uses this Ed25519 key to sign canonical record SHA-256 digests. Verify with: ed25519_verify(public_key, message=canonical_sha256_bytes, signature=base64decode(signature_b64))."}],"merge_version":"pith-open-graph-merge-v1","built_at":"2026-08-07T03:40:21Z","links":{"resolver":"https://pith.science/pith/VPWA75BNUQNBY7ATIKAPFLOTBK","bundle":"https://pith.science/pith/VPWA75BNUQNBY7ATIKAPFLOTBK/bundle.json","state":"https://pith.science/pith/VPWA75BNUQNBY7ATIKAPFLOTBK/state.json","well_known_bundle":"https://pith.science/.well-known/pith/VPWA75BNUQNBY7ATIKAPFLOTBK/bundle.json"},"state":{"state_type":"pith_open_graph_state","state_version":"1.0","pith_number":"pith:2025:VPWA75BNUQNBY7ATIKAPFLOTBK","merge_version":"pith-open-graph-merge-v1","event_count":2,"valid_event_count":2,"invalid_event_count":0,"equivocation_count":0,"current":{"canonical_record":{"metadata":{"abstract_canon_sha256":"69fae6b811058d5ba899a4d9944f38098cdc23a982db1e686dafd31970b2e4ca","cross_cats_sorted":["cs.AI"],"license":"http://creativecommons.org/licenses/by-nc-sa/4.0/","primary_cat":"cs.LG","submitted_at":"2025-03-10T00:13:19Z","title_canon_sha256":"0bfbfcc8a915be1499e1a0af94b4c1e5af80f13d6fa7c28931e88d4f24f98da6"},"schema_version":"1.0","source":{"id":"2503.06810","kind":"arxiv","version":1}},"source_aliases":[{"alias_kind":"arxiv","alias_value":"2503.06810","created_at":"2026-07-05T10:27:43Z"},{"alias_kind":"arxiv_version","alias_value":"2503.06810v1","created_at":"2026-07-05T10:27:43Z"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.2503.06810","created_at":"2026-07-05T10:27:43Z"},{"alias_kind":"pith_short_12","alias_value":"VPWA75BNUQNB","created_at":"2026-07-05T10:27:43Z"},{"alias_kind":"pith_short_16","alias_value":"VPWA75BNUQNBY7AT","created_at":"2026-07-05T10:27:43Z"},{"alias_kind":"pith_short_8","alias_value":"VPWA75BN","created_at":"2026-07-05T10:27:43Z"}],"graph_snapshots":[{"event_id":"sha256:5c16ef4833c424914ba831211e0a09af4b98f47402d94164a47c5bc2e1b39f4d","target":"graph","created_at":"2026-07-05T10:27:43Z","signer":{"key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signer_id":"pith.science","signer_type":"pith_registry"},"payload":{"graph_snapshot":{"author_claims":{"count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","strong_count":0},"builder_version":"pith-number-builder-2026-05-17-v1","claims":{"count":0,"items":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"integrity":{"available":true,"clean":true,"detectors_run":[],"endpoint":"/pith/2503.06810/integrity.json","findings":[],"snapshot_sha256":"c28c3603d3b5d939e8dc4c7e95fa8dfce3d595e45f758748cecf8e644a296938","summary":{"advisory":0,"by_detector":{},"critical":0,"informational":0}},"paper":{"abstract_excerpt":"This work tackles the problem of overoptimization in reinforcement learning from human feedback (RLHF), a prevalent technique for aligning models with human preferences. RLHF relies on reward or preference models trained on \\emph{fixed preference datasets}, and these models are unreliable when evaluated outside the support of this preference data, leading to the common reward or preference hacking phenomenon. We propose novel, pessimistic objectives for RLHF which are provably robust to overoptimization through the use of pessimism in the face of uncertainty, and design practical algorithms, P","authors_text":"Adam Fisch, Alekh Agarwal, Christoph Dann, Dhawal Gupta","cross_cats":["cs.AI"],"headline":"","license":"http://creativecommons.org/licenses/by-nc-sa/4.0/","primary_cat":"cs.LG","submitted_at":"2025-03-10T00:13:19Z","title":"Mitigating Preference Hacking in Policy Optimization with Pessimism"},"references":{"count":0,"internal_anchors":0,"resolved_work":0,"sample":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"2503.06810","kind":"arxiv","version":1},"verdict":{"created_at":null,"id":null,"model_set":{},"one_line_summary":"","pipeline_version":null,"pith_extraction_headline":"","strongest_claim":"","weakest_assumption":""}},"verdict_id":null}}],"author_attestations":[],"timestamp_anchors":[],"storage_attestations":[],"citation_signatures":[],"replication_records":[],"corrections":[],"mirror_hints":[],"record_created":{"event_id":"sha256:d14e446b0ec63240b7c900e97f6078984bc18ae9d54c09a22096732078c7ce32","target":"record","created_at":"2026-07-05T10:27:43Z","signer":{"key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signer_id":"pith.science","signer_type":"pith_registry"},"payload":{"attestation_state":"computed","canonical_record":{"metadata":{"abstract_canon_sha256":"69fae6b811058d5ba899a4d9944f38098cdc23a982db1e686dafd31970b2e4ca","cross_cats_sorted":["cs.AI"],"license":"http://creativecommons.org/licenses/by-nc-sa/4.0/","primary_cat":"cs.LG","submitted_at":"2025-03-10T00:13:19Z","title_canon_sha256":"0bfbfcc8a915be1499e1a0af94b4c1e5af80f13d6fa7c28931e88d4f24f98da6"},"schema_version":"1.0","source":{"id":"2503.06810","kind":"arxiv","version":1}},"canonical_sha256":"abec0ff42da41a1c7c134280f2add30a9a6927dd65aec5d6327dc400ea18b4fa","receipt":{"algorithm":"ed25519","builder_version":"pith-number-builder-2026-05-17-v1","canonical_sha256":"abec0ff42da41a1c7c134280f2add30a9a6927dd65aec5d6327dc400ea18b4fa","first_computed_at":"2026-07-05T10:27:43.055104Z","key_id":"pith-v1-2026-05","kind":"pith_receipt","last_reissued_at":"2026-07-05T10:27:43.055104Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","receipt_version":"0.3","signature_b64":"Lv0I1VhWVrMVtqQ+U4KcjAUoC1f1yit/Axm6pg6pXQMcKeJ0xy8SIcA2jsmxY92ZfRKaKoWanpXRFzdmPOfJCw==","signature_status":"signed_v1","signed_at":"2026-07-05T10:27:43.055724Z","signed_message":"canonical_sha256_bytes"},"source_id":"2503.06810","source_kind":"arxiv","source_version":1}}},"equivocations":[],"invalid_events":[],"applied_event_ids":["sha256:d14e446b0ec63240b7c900e97f6078984bc18ae9d54c09a22096732078c7ce32","sha256:5c16ef4833c424914ba831211e0a09af4b98f47402d94164a47c5bc2e1b39f4d"],"state_sha256":"cac905ff309e6284352c6f9b91fe1717a0a26ee790753012629443aa93d42074"},"bundle_signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"i6EYYvg+KzZ/ge74K8XEeXzC2OFufrZ2yTWsNQgBt1aNZZxSoixJQ4H1wcE/DpVVI4v6sHFAOBXp6O8RhZ0xAg==","signed_message":"bundle_sha256_bytes","signed_at":"2026-08-07T03:40:21.693978Z","bundle_sha256":"23b0fc88956f5655475bd061061540e3342fc79494e95ab9c1bc6230917d6caf"}}