{"record_type":"pith_number_record","schema_url":"https://pith.science/schemas/pith-number/v1.json","pith_number":"pith:2024:SCZ3F52OSTLS6446BDXACNCAHG","short_pith_number":"pith:SCZ3F52O","schema_version":"1.0","canonical_sha256":"90b3b2f74e94d72f739e08ee01344039b4eacab6bf22636e3f349d5a195f774d","source":{"kind":"arxiv","id":"2410.20187","version":1},"attestation_state":"computed","paper":{"title":"Uncertainty-Penalized Direct Preference Optimization","license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","headline":"","cross_cats":["cs.AI","stat.ML"],"primary_cat":"cs.LG","authors_text":"Alexander Immer, Aliz\\'ee Pace, Gunnar R\\\"atsch, Sam Houliston","submitted_at":"2024-10-26T14:24:37Z","abstract_excerpt":"Aligning Large Language Models (LLMs) to human preferences in content, style, and presentation is challenging, in part because preferences are varied, context-dependent, and sometimes inherently ambiguous. While successful, Reinforcement Learning from Human Feedback (RLHF) and Direct Preference Optimization (DPO) are prone to the issue of proxy reward overoptimization. Analysis of the DPO loss reveals a critical need for regularization for mislabeled or ambiguous preference pairs to avoid reward hacking. In this work, we develop a pessimistic framework for DPO by introducing preference uncerta"},"verification_status":{"content_addressed":true,"pith_receipt":true,"author_attested":false,"weak_author_claims":0,"strong_author_claims":0,"externally_anchored":false,"storage_verified":false,"citation_signatures":0,"replication_records":0,"graph_snapshot":true,"references_resolved":false,"formal_links_present":false},"canonical_record":{"source":{"id":"2410.20187","kind":"arxiv","version":1},"metadata":{"license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","primary_cat":"cs.LG","submitted_at":"2024-10-26T14:24:37Z","cross_cats_sorted":["cs.AI","stat.ML"],"title_canon_sha256":"0e6b7567a8087172bf8f6fe572ffa2db9e2c7dd41628733fc07315fa535dcc8b","abstract_canon_sha256":"feb5e3ca4ad342631d6f0708aff9a301e703c42ed36a55e9a3b25ef56d418097"},"schema_version":"1.0"},"receipt":{"kind":"pith_receipt","key_id":"pith-v1-2026-05","algorithm":"ed25519","signed_at":"2026-07-05T09:26:40.593252Z","signature_b64":"0z/Dbptf468ILltjvGeO5XnG2fhLHbp6Fv5VP4y0kKD2Wj3gPcpWBaaKHCeB5tFVqaQPuDMByHtZjORDq7RHAw==","signed_message":"canonical_sha256_bytes","builder_version":"pith-number-builder-2026-05-17-v1","receipt_version":"0.3","canonical_sha256":"90b3b2f74e94d72f739e08ee01344039b4eacab6bf22636e3f349d5a195f774d","last_reissued_at":"2026-07-05T09:26:40.592734Z","signature_status":"signed_v1","first_computed_at":"2026-07-05T09:26:40.592734Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"graph_snapshot":{"paper":{"title":"Uncertainty-Penalized Direct Preference Optimization","license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","headline":"","cross_cats":["cs.AI","stat.ML"],"primary_cat":"cs.LG","authors_text":"Alexander Immer, Aliz\\'ee Pace, Gunnar R\\\"atsch, Sam Houliston","submitted_at":"2024-10-26T14:24:37Z","abstract_excerpt":"Aligning Large Language Models (LLMs) to human preferences in content, style, and presentation is challenging, in part because preferences are varied, context-dependent, and sometimes inherently ambiguous. While successful, Reinforcement Learning from Human Feedback (RLHF) and Direct Preference Optimization (DPO) are prone to the issue of proxy reward overoptimization. Analysis of the DPO loss reveals a critical need for regularization for mislabeled or ambiguous preference pairs to avoid reward hacking. In this work, we develop a pessimistic framework for DPO by introducing preference uncerta"},"claims":{"count":0,"items":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"2410.20187","kind":"arxiv","version":1},"verdict":{"id":null,"model_set":{},"created_at":null,"strongest_claim":"","one_line_summary":"","pipeline_version":null,"weakest_assumption":"","pith_extraction_headline":""},"integrity":{"clean":true,"summary":{"advisory":0,"critical":0,"by_detector":{},"informational":0},"endpoint":"/pith/2410.20187/integrity.json","findings":[],"available":true,"detectors_run":[],"snapshot_sha256":"c28c3603d3b5d939e8dc4c7e95fa8dfce3d595e45f758748cecf8e644a296938"},"references":{"count":0,"sample":[],"resolved_work":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","internal_anchors":0},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"author_claims":{"count":0,"strong_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"builder_version":"pith-number-builder-2026-05-17-v1"},"aliases":[{"alias_kind":"arxiv","alias_value":"2410.20187","created_at":"2026-07-05T09:26:40.592802+00:00"},{"alias_kind":"arxiv_version","alias_value":"2410.20187v1","created_at":"2026-07-05T09:26:40.592802+00:00"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.2410.20187","created_at":"2026-07-05T09:26:40.592802+00:00"},{"alias_kind":"pith_short_12","alias_value":"SCZ3F52OSTLS","created_at":"2026-07-05T09:26:40.592802+00:00"},{"alias_kind":"pith_short_16","alias_value":"SCZ3F52OSTLS6446","created_at":"2026-07-05T09:26:40.592802+00:00"},{"alias_kind":"pith_short_8","alias_value":"SCZ3F52O","created_at":"2026-07-05T09:26:40.592802+00:00"}],"events":[],"event_summary":{},"paper_claims":[],"inbound_citations":{"count":2,"internal_anchor_count":0,"sample":[{"citing_arxiv_id":"2606.20662","citing_title":"Confidence Laundering in Agent Systems: Why Uncertainty Needs a Latent Carrier","ref_index":38,"is_internal_anchor":false},{"citing_arxiv_id":"2605.20696","citing_title":"Distributed Direct Preference Optimization","ref_index":5,"is_internal_anchor":false}]},"formal_canon":{"evidence_count":0,"sample":[],"anchors":[]},"links":{"html":"https://pith.science/pith/SCZ3F52OSTLS6446BDXACNCAHG","json":"https://pith.science/pith/SCZ3F52OSTLS6446BDXACNCAHG.json","graph_json":"https://pith.science/api/pith-number/SCZ3F52OSTLS6446BDXACNCAHG/graph.json","events_json":"https://pith.science/api/pith-number/SCZ3F52OSTLS6446BDXACNCAHG/events.json","paper":"https://pith.science/paper/SCZ3F52O"},"agent_actions":{"view_html":"https://pith.science/pith/SCZ3F52OSTLS6446BDXACNCAHG","download_json":"https://pith.science/pith/SCZ3F52OSTLS6446BDXACNCAHG.json","view_paper":"https://pith.science/paper/SCZ3F52O","resolve_alias":"https://pith.science/api/pith-number/resolve?arxiv=2410.20187&json=true","fetch_graph":"https://pith.science/api/pith-number/SCZ3F52OSTLS6446BDXACNCAHG/graph.json","fetch_events":"https://pith.science/api/pith-number/SCZ3F52OSTLS6446BDXACNCAHG/events.json","actions":{"anchor_timestamp":"https://pith.science/pith/SCZ3F52OSTLS6446BDXACNCAHG/action/timestamp_anchor","attest_storage":"https://pith.science/pith/SCZ3F52OSTLS6446BDXACNCAHG/action/storage_attestation","attest_author":"https://pith.science/pith/SCZ3F52OSTLS6446BDXACNCAHG/action/author_attestation","sign_citation":"https://pith.science/pith/SCZ3F52OSTLS6446BDXACNCAHG/action/citation_signature","submit_replication":"https://pith.science/pith/SCZ3F52OSTLS6446BDXACNCAHG/action/replication_record"}},"created_at":"2026-07-05T09:26:40.592802+00:00","updated_at":"2026-07-05T09:26:40.592802+00:00"}