{"record_type":"pith_number_record","schema_url":"https://pith.science/schemas/pith-number/v1.json","pith_number":"pith:2025:F5DB5DEJMZAPXAHPUW2VOHRUMH","short_pith_number":"pith:F5DB5DEJ","schema_version":"1.0","canonical_sha256":"2f461e8c896640fb80efa5b5571e3461e327f5e1a73a4570d1f95eaab42bc0a9","source":{"kind":"arxiv","id":"2506.08681","version":2},"attestation_state":"computed","paper":{"title":"Mitigating Reward Over-optimization in Direct Alignment Algorithms with Importance Sampling","license":"http://creativecommons.org/licenses/by/4.0/","headline":"","cross_cats":[],"primary_cat":"cs.LG","authors_text":"Anji Liu, An Mai, Binh T. Nguyen, Daniel Sonntag, Duy H. M. Nguyen, Khoa D. Doan, Ngoc-Hieu Nguyen, Phuc Minh Nguyen","submitted_at":"2025-06-10T10:45:26Z","abstract_excerpt":"Direct Alignment Algorithms (DAAs) such as Direct Preference Optimization (DPO) have emerged as alternatives to the standard Reinforcement Learning from Human Feedback (RLHF) for aligning large language models (LLMs) with human values. However, these methods are more susceptible to over-optimization, in which the model drifts away from the reference policy, leading to degraded performance as training progresses. This paper proposes a novel importance-sampling approach to mitigate the over-optimization problem of offline DAAs. This approach, called (IS-DAAs), multiplies the DAA objective with a"},"verification_status":{"content_addressed":true,"pith_receipt":true,"author_attested":false,"weak_author_claims":0,"strong_author_claims":0,"externally_anchored":false,"storage_verified":false,"citation_signatures":0,"replication_records":0,"graph_snapshot":true,"references_resolved":false,"formal_links_present":false},"canonical_record":{"source":{"id":"2506.08681","kind":"arxiv","version":2},"metadata":{"license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.LG","submitted_at":"2025-06-10T10:45:26Z","cross_cats_sorted":[],"title_canon_sha256":"be73c854a369f74bc838f6c32458345307e08de22a5f5e596ebe4ada6382d82e","abstract_canon_sha256":"f276d3ae07fdbe77bacd745c01542bb645e6de711fecd264c9e43350c945b897"},"schema_version":"1.0"},"receipt":{"kind":"pith_receipt","key_id":"pith-v1-2026-05","algorithm":"ed25519","signed_at":"2026-07-05T11:19:30.039143Z","signature_b64":"4leh+3zMRdxMVD7awpmiFoXH42iLUJ7pTroIVFHM8EzL9lEN2tWzqASOxYHUvKMMHZ7KchW2U3JbJIqqpXrKDA==","signed_message":"canonical_sha256_bytes","builder_version":"pith-number-builder-2026-05-17-v1","receipt_version":"0.3","canonical_sha256":"2f461e8c896640fb80efa5b5571e3461e327f5e1a73a4570d1f95eaab42bc0a9","last_reissued_at":"2026-07-05T11:19:30.038541Z","signature_status":"signed_v1","first_computed_at":"2026-07-05T11:19:30.038541Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"graph_snapshot":{"paper":{"title":"Mitigating Reward Over-optimization in Direct Alignment Algorithms with Importance Sampling","license":"http://creativecommons.org/licenses/by/4.0/","headline":"","cross_cats":[],"primary_cat":"cs.LG","authors_text":"Anji Liu, An Mai, Binh T. Nguyen, Daniel Sonntag, Duy H. M. Nguyen, Khoa D. Doan, Ngoc-Hieu Nguyen, Phuc Minh Nguyen","submitted_at":"2025-06-10T10:45:26Z","abstract_excerpt":"Direct Alignment Algorithms (DAAs) such as Direct Preference Optimization (DPO) have emerged as alternatives to the standard Reinforcement Learning from Human Feedback (RLHF) for aligning large language models (LLMs) with human values. However, these methods are more susceptible to over-optimization, in which the model drifts away from the reference policy, leading to degraded performance as training progresses. This paper proposes a novel importance-sampling approach to mitigate the over-optimization problem of offline DAAs. This approach, called (IS-DAAs), multiplies the DAA objective with a"},"claims":{"count":0,"items":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"2506.08681","kind":"arxiv","version":2},"verdict":{"id":null,"model_set":{},"created_at":null,"strongest_claim":"","one_line_summary":"","pipeline_version":null,"weakest_assumption":"","pith_extraction_headline":""},"integrity":{"clean":true,"summary":{"advisory":0,"critical":0,"by_detector":{},"informational":0},"endpoint":"/pith/2506.08681/integrity.json","findings":[],"available":true,"detectors_run":[],"snapshot_sha256":"c28c3603d3b5d939e8dc4c7e95fa8dfce3d595e45f758748cecf8e644a296938"},"references":{"count":0,"sample":[],"resolved_work":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","internal_anchors":0},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"author_claims":{"count":0,"strong_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"builder_version":"pith-number-builder-2026-05-17-v1"},"aliases":[{"alias_kind":"arxiv","alias_value":"2506.08681","created_at":"2026-07-05T11:19:30.038622+00:00"},{"alias_kind":"arxiv_version","alias_value":"2506.08681v2","created_at":"2026-07-05T11:19:30.038622+00:00"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.2506.08681","created_at":"2026-07-05T11:19:30.038622+00:00"},{"alias_kind":"pith_short_12","alias_value":"F5DB5DEJMZAP","created_at":"2026-07-05T11:19:30.038622+00:00"},{"alias_kind":"pith_short_16","alias_value":"F5DB5DEJMZAPXAHP","created_at":"2026-07-05T11:19:30.038622+00:00"},{"alias_kind":"pith_short_8","alias_value":"F5DB5DEJ","created_at":"2026-07-05T11:19:30.038622+00:00"}],"events":[],"event_summary":{},"paper_claims":[],"inbound_citations":{"count":1,"internal_anchor_count":0,"sample":[{"citing_arxiv_id":"2605.23398","citing_title":"TPMM-DPO: Trajectory-aware Preference-guided Model Merging for Iterative Direct Preference Optimization","ref_index":7,"is_internal_anchor":false}]},"formal_canon":{"evidence_count":0,"sample":[],"anchors":[]},"links":{"html":"https://pith.science/pith/F5DB5DEJMZAPXAHPUW2VOHRUMH","json":"https://pith.science/pith/F5DB5DEJMZAPXAHPUW2VOHRUMH.json","graph_json":"https://pith.science/api/pith-number/F5DB5DEJMZAPXAHPUW2VOHRUMH/graph.json","events_json":"https://pith.science/api/pith-number/F5DB5DEJMZAPXAHPUW2VOHRUMH/events.json","paper":"https://pith.science/paper/F5DB5DEJ"},"agent_actions":{"view_html":"https://pith.science/pith/F5DB5DEJMZAPXAHPUW2VOHRUMH","download_json":"https://pith.science/pith/F5DB5DEJMZAPXAHPUW2VOHRUMH.json","view_paper":"https://pith.science/paper/F5DB5DEJ","resolve_alias":"https://pith.science/api/pith-number/resolve?arxiv=2506.08681&json=true","fetch_graph":"https://pith.science/api/pith-number/F5DB5DEJMZAPXAHPUW2VOHRUMH/graph.json","fetch_events":"https://pith.science/api/pith-number/F5DB5DEJMZAPXAHPUW2VOHRUMH/events.json","actions":{"anchor_timestamp":"https://pith.science/pith/F5DB5DEJMZAPXAHPUW2VOHRUMH/action/timestamp_anchor","attest_storage":"https://pith.science/pith/F5DB5DEJMZAPXAHPUW2VOHRUMH/action/storage_attestation","attest_author":"https://pith.science/pith/F5DB5DEJMZAPXAHPUW2VOHRUMH/action/author_attestation","sign_citation":"https://pith.science/pith/F5DB5DEJMZAPXAHPUW2VOHRUMH/action/citation_signature","submit_replication":"https://pith.science/pith/F5DB5DEJMZAPXAHPUW2VOHRUMH/action/replication_record"}},"created_at":"2026-07-05T11:19:30.038622+00:00","updated_at":"2026-07-05T11:19:30.038622+00:00"}