{"bundle_type":"pith_open_graph_bundle","bundle_version":"1.0","pith_number":"pith:2024:LDGMNZ3EZ4JNCP4N26ASDST4VZ","short_pith_number":"pith:LDGMNZ3E","canonical_record":{"source":{"id":"2405.16436","kind":"arxiv","version":3},"metadata":{"license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.LG","submitted_at":"2024-05-26T05:38:50Z","cross_cats_sorted":["cs.AI","stat.ML"],"title_canon_sha256":"df47d6abe0e3ef65c704adddc48b8669394b86ad2b49ebff8485ccc5d80b4f85","abstract_canon_sha256":"39c45de216948dfac00c5dc3d77090f19d93075de7c8411802f26fbe1d58094d"},"schema_version":"1.0"},"canonical_sha256":"58ccc6e764cf12d13f8dd78121ca7cae480ff50873a694b769f261f36054ad81","source":{"kind":"arxiv","id":"2405.16436","version":3},"source_aliases":[{"alias_kind":"arxiv","alias_value":"2405.16436","created_at":"2026-07-05T09:44:09Z"},{"alias_kind":"arxiv_version","alias_value":"2405.16436v3","created_at":"2026-07-05T09:44:09Z"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.2405.16436","created_at":"2026-07-05T09:44:09Z"},{"alias_kind":"pith_short_12","alias_value":"LDGMNZ3EZ4JN","created_at":"2026-07-05T09:44:09Z"},{"alias_kind":"pith_short_16","alias_value":"LDGMNZ3EZ4JNCP4N","created_at":"2026-07-05T09:44:09Z"},{"alias_kind":"pith_short_8","alias_value":"LDGMNZ3E","created_at":"2026-07-05T09:44:09Z"}],"events":[{"event_type":"record_created","subject_pith_number":"pith:2024:LDGMNZ3EZ4JNCP4N26ASDST4VZ","target":"record","payload":{"canonical_record":{"source":{"id":"2405.16436","kind":"arxiv","version":3},"metadata":{"license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.LG","submitted_at":"2024-05-26T05:38:50Z","cross_cats_sorted":["cs.AI","stat.ML"],"title_canon_sha256":"df47d6abe0e3ef65c704adddc48b8669394b86ad2b49ebff8485ccc5d80b4f85","abstract_canon_sha256":"39c45de216948dfac00c5dc3d77090f19d93075de7c8411802f26fbe1d58094d"},"schema_version":"1.0"},"canonical_sha256":"58ccc6e764cf12d13f8dd78121ca7cae480ff50873a694b769f261f36054ad81","receipt":{"kind":"pith_receipt","key_id":"pith-v1-2026-05","algorithm":"ed25519","signed_at":"2026-07-05T09:44:09.461867Z","signature_b64":"xg+4xLCpTEJaMQLk6ykRBsfa8kksqr2h4YcMQKJB5D6XSVzpQrm7nqHafg77l1qaF4ILkbEpCqbd8rNXDA2dCw==","signed_message":"canonical_sha256_bytes","builder_version":"pith-number-builder-2026-05-17-v1","receipt_version":"0.3","canonical_sha256":"58ccc6e764cf12d13f8dd78121ca7cae480ff50873a694b769f261f36054ad81","last_reissued_at":"2026-07-05T09:44:09.461364Z","signature_status":"signed_v1","first_computed_at":"2026-07-05T09:44:09.461364Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"source_kind":"arxiv","source_id":"2405.16436","source_version":3,"attestation_state":"computed"},"signer":{"signer_id":"pith.science","signer_type":"pith_registry","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"created_at":"2026-07-05T09:44:09Z","supersedes":[],"prev_event":null,"signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"L86ooUelH1jDn3DnnLDV6Plj2PwE5VBYhmum+6EIYD5yfY93lhKfeQuRu896MhBz2tKmfB+3Pwq/ZWyW0wV7CA==","signed_message":"open_graph_event_sha256_bytes","signed_at":"2026-08-07T16:01:54.741952Z"},"content_sha256":"64f1d4fa85c78f48050fee6b70975ed7013730d25fe85f8981e40dba9f899738","schema_version":"1.0","event_id":"sha256:64f1d4fa85c78f48050fee6b70975ed7013730d25fe85f8981e40dba9f899738"},{"event_type":"graph_snapshot","subject_pith_number":"pith:2024:LDGMNZ3EZ4JNCP4N26ASDST4VZ","target":"graph","payload":{"graph_snapshot":{"paper":{"title":"Provably Mitigating Overoptimization in RLHF: Your SFT Loss is Implicitly an Adversarial Regularizer","license":"http://creativecommons.org/licenses/by/4.0/","headline":"","cross_cats":["cs.AI","stat.ML"],"primary_cat":"cs.LG","authors_text":"Boyi Liu, Hongyi Guo, Jose Blanchet, Miao Lu, Shenao Zhang, Yingxiang Yang, Zhaoran Wang, Zhihan Liu","submitted_at":"2024-05-26T05:38:50Z","abstract_excerpt":"Aligning generative models with human preference via RLHF typically suffers from overoptimization, where an imperfectly learned reward model can misguide the generative model to output undesired responses. We investigate this problem in a principled manner by identifying the source of the misalignment as a form of distributional shift and uncertainty in learning human preferences. To mitigate overoptimization, we first propose a theoretical algorithm that chooses the best policy for an adversarially chosen reward model; one that simultaneously minimizes the maximum likelihood estimation of the"},"claims":{"count":0,"items":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"2405.16436","kind":"arxiv","version":3},"verdict":{"id":null,"model_set":{},"created_at":null,"strongest_claim":"","one_line_summary":"","pipeline_version":null,"weakest_assumption":"","pith_extraction_headline":""},"integrity":{"clean":true,"summary":{"advisory":0,"critical":0,"by_detector":{},"informational":0},"endpoint":"/pith/2405.16436/integrity.json","findings":[],"available":true,"detectors_run":[],"snapshot_sha256":"c28c3603d3b5d939e8dc4c7e95fa8dfce3d595e45f758748cecf8e644a296938"},"references":{"count":0,"sample":[],"resolved_work":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","internal_anchors":0},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"author_claims":{"count":0,"strong_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"builder_version":"pith-number-builder-2026-05-17-v1"},"verdict_id":null},"signer":{"signer_id":"pith.science","signer_type":"pith_registry","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"created_at":"2026-07-05T09:44:09Z","supersedes":[],"prev_event":null,"signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"H2sEbEuxHvFwJSNV0FL3dkGigbUXMi0kbMWcoLpG1JZnH+0MWsAA2tRPbymIJPl5MoB3sgCAPYs1Y+Mt8eHVCA==","signed_message":"open_graph_event_sha256_bytes","signed_at":"2026-08-07T16:01:54.742468Z"},"content_sha256":"8a367999528e3c4ae64833e0d0c2b2167566602f34317a5ce94aab2281cd7342","schema_version":"1.0","event_id":"sha256:8a367999528e3c4ae64833e0d0c2b2167566602f34317a5ce94aab2281cd7342"}],"timestamp_proofs":[],"mirror_hints":[{"mirror_type":"https","name":"Pith Resolver","base_url":"https://pith.science","bundle_url":"https://pith.science/pith/LDGMNZ3EZ4JNCP4N26ASDST4VZ/bundle.json","state_url":"https://pith.science/pith/LDGMNZ3EZ4JNCP4N26ASDST4VZ/state.json","well_known_bundle_url":"https://pith.science/.well-known/pith/LDGMNZ3EZ4JNCP4N26ASDST4VZ/bundle.json","status":"primary"}],"public_keys":[{"key_id":"pith-v1-2026-05","algorithm":"ed25519","format":"raw","public_key_b64":"stVStoiQhXFxp4s2pdzPNoqVNBMojDU/fJ2db5S3CbM=","public_key_hex":"b2d552b68890857171a78b36a5dccf368a953413288c353f7c9d9d6f94b709b3","fingerprint_sha256_b32_first128bits":"RVFV5Z2OI2J3ZUO7ERDEBCYNKS","fingerprint_sha256_hex":"8d4b5ee74e4693bcd1df2446408b0d54","rotates_at":null,"url":"https://pith.science/pith-signing-key.json","notes":"Pith uses this Ed25519 key to sign canonical record SHA-256 digests. Verify with: ed25519_verify(public_key, message=canonical_sha256_bytes, signature=base64decode(signature_b64))."}],"merge_version":"pith-open-graph-merge-v1","built_at":"2026-08-07T16:01:54Z","links":{"resolver":"https://pith.science/pith/LDGMNZ3EZ4JNCP4N26ASDST4VZ","bundle":"https://pith.science/pith/LDGMNZ3EZ4JNCP4N26ASDST4VZ/bundle.json","state":"https://pith.science/pith/LDGMNZ3EZ4JNCP4N26ASDST4VZ/state.json","well_known_bundle":"https://pith.science/.well-known/pith/LDGMNZ3EZ4JNCP4N26ASDST4VZ/bundle.json"},"state":{"state_type":"pith_open_graph_state","state_version":"1.0","pith_number":"pith:2024:LDGMNZ3EZ4JNCP4N26ASDST4VZ","merge_version":"pith-open-graph-merge-v1","event_count":2,"valid_event_count":2,"invalid_event_count":0,"equivocation_count":0,"current":{"canonical_record":{"metadata":{"abstract_canon_sha256":"39c45de216948dfac00c5dc3d77090f19d93075de7c8411802f26fbe1d58094d","cross_cats_sorted":["cs.AI","stat.ML"],"license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.LG","submitted_at":"2024-05-26T05:38:50Z","title_canon_sha256":"df47d6abe0e3ef65c704adddc48b8669394b86ad2b49ebff8485ccc5d80b4f85"},"schema_version":"1.0","source":{"id":"2405.16436","kind":"arxiv","version":3}},"source_aliases":[{"alias_kind":"arxiv","alias_value":"2405.16436","created_at":"2026-07-05T09:44:09Z"},{"alias_kind":"arxiv_version","alias_value":"2405.16436v3","created_at":"2026-07-05T09:44:09Z"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.2405.16436","created_at":"2026-07-05T09:44:09Z"},{"alias_kind":"pith_short_12","alias_value":"LDGMNZ3EZ4JN","created_at":"2026-07-05T09:44:09Z"},{"alias_kind":"pith_short_16","alias_value":"LDGMNZ3EZ4JNCP4N","created_at":"2026-07-05T09:44:09Z"},{"alias_kind":"pith_short_8","alias_value":"LDGMNZ3E","created_at":"2026-07-05T09:44:09Z"}],"graph_snapshots":[{"event_id":"sha256:8a367999528e3c4ae64833e0d0c2b2167566602f34317a5ce94aab2281cd7342","target":"graph","created_at":"2026-07-05T09:44:09Z","signer":{"key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signer_id":"pith.science","signer_type":"pith_registry"},"payload":{"graph_snapshot":{"author_claims":{"count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","strong_count":0},"builder_version":"pith-number-builder-2026-05-17-v1","claims":{"count":0,"items":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"integrity":{"available":true,"clean":true,"detectors_run":[],"endpoint":"/pith/2405.16436/integrity.json","findings":[],"snapshot_sha256":"c28c3603d3b5d939e8dc4c7e95fa8dfce3d595e45f758748cecf8e644a296938","summary":{"advisory":0,"by_detector":{},"critical":0,"informational":0}},"paper":{"abstract_excerpt":"Aligning generative models with human preference via RLHF typically suffers from overoptimization, where an imperfectly learned reward model can misguide the generative model to output undesired responses. We investigate this problem in a principled manner by identifying the source of the misalignment as a form of distributional shift and uncertainty in learning human preferences. To mitigate overoptimization, we first propose a theoretical algorithm that chooses the best policy for an adversarially chosen reward model; one that simultaneously minimizes the maximum likelihood estimation of the","authors_text":"Boyi Liu, Hongyi Guo, Jose Blanchet, Miao Lu, Shenao Zhang, Yingxiang Yang, Zhaoran Wang, Zhihan Liu","cross_cats":["cs.AI","stat.ML"],"headline":"","license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.LG","submitted_at":"2024-05-26T05:38:50Z","title":"Provably Mitigating Overoptimization in RLHF: Your SFT Loss is Implicitly an Adversarial Regularizer"},"references":{"count":0,"internal_anchors":0,"resolved_work":0,"sample":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"2405.16436","kind":"arxiv","version":3},"verdict":{"created_at":null,"id":null,"model_set":{},"one_line_summary":"","pipeline_version":null,"pith_extraction_headline":"","strongest_claim":"","weakest_assumption":""}},"verdict_id":null}}],"author_attestations":[],"timestamp_anchors":[],"storage_attestations":[],"citation_signatures":[],"replication_records":[],"corrections":[],"mirror_hints":[],"record_created":{"event_id":"sha256:64f1d4fa85c78f48050fee6b70975ed7013730d25fe85f8981e40dba9f899738","target":"record","created_at":"2026-07-05T09:44:09Z","signer":{"key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signer_id":"pith.science","signer_type":"pith_registry"},"payload":{"attestation_state":"computed","canonical_record":{"metadata":{"abstract_canon_sha256":"39c45de216948dfac00c5dc3d77090f19d93075de7c8411802f26fbe1d58094d","cross_cats_sorted":["cs.AI","stat.ML"],"license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.LG","submitted_at":"2024-05-26T05:38:50Z","title_canon_sha256":"df47d6abe0e3ef65c704adddc48b8669394b86ad2b49ebff8485ccc5d80b4f85"},"schema_version":"1.0","source":{"id":"2405.16436","kind":"arxiv","version":3}},"canonical_sha256":"58ccc6e764cf12d13f8dd78121ca7cae480ff50873a694b769f261f36054ad81","receipt":{"algorithm":"ed25519","builder_version":"pith-number-builder-2026-05-17-v1","canonical_sha256":"58ccc6e764cf12d13f8dd78121ca7cae480ff50873a694b769f261f36054ad81","first_computed_at":"2026-07-05T09:44:09.461364Z","key_id":"pith-v1-2026-05","kind":"pith_receipt","last_reissued_at":"2026-07-05T09:44:09.461364Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","receipt_version":"0.3","signature_b64":"xg+4xLCpTEJaMQLk6ykRBsfa8kksqr2h4YcMQKJB5D6XSVzpQrm7nqHafg77l1qaF4ILkbEpCqbd8rNXDA2dCw==","signature_status":"signed_v1","signed_at":"2026-07-05T09:44:09.461867Z","signed_message":"canonical_sha256_bytes"},"source_id":"2405.16436","source_kind":"arxiv","source_version":3}}},"equivocations":[],"invalid_events":[],"applied_event_ids":["sha256:64f1d4fa85c78f48050fee6b70975ed7013730d25fe85f8981e40dba9f899738","sha256:8a367999528e3c4ae64833e0d0c2b2167566602f34317a5ce94aab2281cd7342"],"state_sha256":"377c8a274b2a44142f3ce4f0d54a42a29d73af3d4674e6cd066e57af43a1b165"},"bundle_signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"uLZFKcg0Aj3qll7781ycbcvHRhbMh6Xn85SCxB4u0ppCZr1kx5lOIJjTZaiUX4VzHHrGOscmO76dxu/srTCrAQ==","signed_message":"bundle_sha256_bytes","signed_at":"2026-08-07T16:01:54.752578Z","bundle_sha256":"4d7ba5e35838606e26412a52bf6b8a1f5d9d6a58dcac193d0dc4eab6142f759c"}}