{"bundle_type":"pith_open_graph_bundle","bundle_version":"1.0","pith_number":"pith:2025:F5DB5DEJMZAPXAHPUW2VOHRUMH","short_pith_number":"pith:F5DB5DEJ","canonical_record":{"source":{"id":"2506.08681","kind":"arxiv","version":2},"metadata":{"license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.LG","submitted_at":"2025-06-10T10:45:26Z","cross_cats_sorted":[],"title_canon_sha256":"be73c854a369f74bc838f6c32458345307e08de22a5f5e596ebe4ada6382d82e","abstract_canon_sha256":"f276d3ae07fdbe77bacd745c01542bb645e6de711fecd264c9e43350c945b897"},"schema_version":"1.0"},"canonical_sha256":"2f461e8c896640fb80efa5b5571e3461e327f5e1a73a4570d1f95eaab42bc0a9","source":{"kind":"arxiv","id":"2506.08681","version":2},"source_aliases":[{"alias_kind":"arxiv","alias_value":"2506.08681","created_at":"2026-07-05T11:19:30Z"},{"alias_kind":"arxiv_version","alias_value":"2506.08681v2","created_at":"2026-07-05T11:19:30Z"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.2506.08681","created_at":"2026-07-05T11:19:30Z"},{"alias_kind":"pith_short_12","alias_value":"F5DB5DEJMZAP","created_at":"2026-07-05T11:19:30Z"},{"alias_kind":"pith_short_16","alias_value":"F5DB5DEJMZAPXAHP","created_at":"2026-07-05T11:19:30Z"},{"alias_kind":"pith_short_8","alias_value":"F5DB5DEJ","created_at":"2026-07-05T11:19:30Z"}],"events":[{"event_type":"record_created","subject_pith_number":"pith:2025:F5DB5DEJMZAPXAHPUW2VOHRUMH","target":"record","payload":{"canonical_record":{"source":{"id":"2506.08681","kind":"arxiv","version":2},"metadata":{"license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.LG","submitted_at":"2025-06-10T10:45:26Z","cross_cats_sorted":[],"title_canon_sha256":"be73c854a369f74bc838f6c32458345307e08de22a5f5e596ebe4ada6382d82e","abstract_canon_sha256":"f276d3ae07fdbe77bacd745c01542bb645e6de711fecd264c9e43350c945b897"},"schema_version":"1.0"},"canonical_sha256":"2f461e8c896640fb80efa5b5571e3461e327f5e1a73a4570d1f95eaab42bc0a9","receipt":{"kind":"pith_receipt","key_id":"pith-v1-2026-05","algorithm":"ed25519","signed_at":"2026-07-05T11:19:30.039143Z","signature_b64":"4leh+3zMRdxMVD7awpmiFoXH42iLUJ7pTroIVFHM8EzL9lEN2tWzqASOxYHUvKMMHZ7KchW2U3JbJIqqpXrKDA==","signed_message":"canonical_sha256_bytes","builder_version":"pith-number-builder-2026-05-17-v1","receipt_version":"0.3","canonical_sha256":"2f461e8c896640fb80efa5b5571e3461e327f5e1a73a4570d1f95eaab42bc0a9","last_reissued_at":"2026-07-05T11:19:30.038541Z","signature_status":"signed_v1","first_computed_at":"2026-07-05T11:19:30.038541Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"source_kind":"arxiv","source_id":"2506.08681","source_version":2,"attestation_state":"computed"},"signer":{"signer_id":"pith.science","signer_type":"pith_registry","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"created_at":"2026-07-05T11:19:30Z","supersedes":[],"prev_event":null,"signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"0Jw9GQ5ZAmxcFf2tV331KRs8oTXXkKTZkAipLgHiZPNOh4r2eAIJ7gTUgTpXFIG/uDQKUIJq+dTbSlrcL6Z+DQ==","signed_message":"open_graph_event_sha256_bytes","signed_at":"2026-08-07T22:41:33.862456Z"},"content_sha256":"260a061eed4c03d0950023d3c582b7917f65ff48354e90f176a2505acfc326b2","schema_version":"1.0","event_id":"sha256:260a061eed4c03d0950023d3c582b7917f65ff48354e90f176a2505acfc326b2"},{"event_type":"graph_snapshot","subject_pith_number":"pith:2025:F5DB5DEJMZAPXAHPUW2VOHRUMH","target":"graph","payload":{"graph_snapshot":{"paper":{"title":"Mitigating Reward Over-optimization in Direct Alignment Algorithms with Importance Sampling","license":"http://creativecommons.org/licenses/by/4.0/","headline":"","cross_cats":[],"primary_cat":"cs.LG","authors_text":"Anji Liu, An Mai, Binh T. Nguyen, Daniel Sonntag, Duy H. M. Nguyen, Khoa D. Doan, Ngoc-Hieu Nguyen, Phuc Minh Nguyen","submitted_at":"2025-06-10T10:45:26Z","abstract_excerpt":"Direct Alignment Algorithms (DAAs) such as Direct Preference Optimization (DPO) have emerged as alternatives to the standard Reinforcement Learning from Human Feedback (RLHF) for aligning large language models (LLMs) with human values. However, these methods are more susceptible to over-optimization, in which the model drifts away from the reference policy, leading to degraded performance as training progresses. This paper proposes a novel importance-sampling approach to mitigate the over-optimization problem of offline DAAs. This approach, called (IS-DAAs), multiplies the DAA objective with a"},"claims":{"count":0,"items":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"2506.08681","kind":"arxiv","version":2},"verdict":{"id":null,"model_set":{},"created_at":null,"strongest_claim":"","one_line_summary":"","pipeline_version":null,"weakest_assumption":"","pith_extraction_headline":""},"integrity":{"clean":true,"summary":{"advisory":0,"critical":0,"by_detector":{},"informational":0},"endpoint":"/pith/2506.08681/integrity.json","findings":[],"available":true,"detectors_run":[],"snapshot_sha256":"c28c3603d3b5d939e8dc4c7e95fa8dfce3d595e45f758748cecf8e644a296938"},"references":{"count":0,"sample":[],"resolved_work":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","internal_anchors":0},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"author_claims":{"count":0,"strong_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"builder_version":"pith-number-builder-2026-05-17-v1"},"verdict_id":null},"signer":{"signer_id":"pith.science","signer_type":"pith_registry","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"created_at":"2026-07-05T11:19:30Z","supersedes":[],"prev_event":null,"signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"16q/JUs04FPBTWfCNpUwus1r1+BMGBU3/TQmEwUCCbGpsps0POxHN3S5uZ7vfka3x18pVhoAbknbV7VAPKxjAA==","signed_message":"open_graph_event_sha256_bytes","signed_at":"2026-08-07T22:41:33.862991Z"},"content_sha256":"4cd51a1b17789452c63b90d8d0b2f4abd2803e108d00b6a20ed9b8ba4302415a","schema_version":"1.0","event_id":"sha256:4cd51a1b17789452c63b90d8d0b2f4abd2803e108d00b6a20ed9b8ba4302415a"}],"timestamp_proofs":[],"mirror_hints":[{"mirror_type":"https","name":"Pith Resolver","base_url":"https://pith.science","bundle_url":"https://pith.science/pith/F5DB5DEJMZAPXAHPUW2VOHRUMH/bundle.json","state_url":"https://pith.science/pith/F5DB5DEJMZAPXAHPUW2VOHRUMH/state.json","well_known_bundle_url":"https://pith.science/.well-known/pith/F5DB5DEJMZAPXAHPUW2VOHRUMH/bundle.json","status":"primary"}],"public_keys":[{"key_id":"pith-v1-2026-05","algorithm":"ed25519","format":"raw","public_key_b64":"stVStoiQhXFxp4s2pdzPNoqVNBMojDU/fJ2db5S3CbM=","public_key_hex":"b2d552b68890857171a78b36a5dccf368a953413288c353f7c9d9d6f94b709b3","fingerprint_sha256_b32_first128bits":"RVFV5Z2OI2J3ZUO7ERDEBCYNKS","fingerprint_sha256_hex":"8d4b5ee74e4693bcd1df2446408b0d54","rotates_at":null,"url":"https://pith.science/pith-signing-key.json","notes":"Pith uses this Ed25519 key to sign canonical record SHA-256 digests. Verify with: ed25519_verify(public_key, message=canonical_sha256_bytes, signature=base64decode(signature_b64))."}],"merge_version":"pith-open-graph-merge-v1","built_at":"2026-08-07T22:41:33Z","links":{"resolver":"https://pith.science/pith/F5DB5DEJMZAPXAHPUW2VOHRUMH","bundle":"https://pith.science/pith/F5DB5DEJMZAPXAHPUW2VOHRUMH/bundle.json","state":"https://pith.science/pith/F5DB5DEJMZAPXAHPUW2VOHRUMH/state.json","well_known_bundle":"https://pith.science/.well-known/pith/F5DB5DEJMZAPXAHPUW2VOHRUMH/bundle.json"},"state":{"state_type":"pith_open_graph_state","state_version":"1.0","pith_number":"pith:2025:F5DB5DEJMZAPXAHPUW2VOHRUMH","merge_version":"pith-open-graph-merge-v1","event_count":2,"valid_event_count":2,"invalid_event_count":0,"equivocation_count":0,"current":{"canonical_record":{"metadata":{"abstract_canon_sha256":"f276d3ae07fdbe77bacd745c01542bb645e6de711fecd264c9e43350c945b897","cross_cats_sorted":[],"license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.LG","submitted_at":"2025-06-10T10:45:26Z","title_canon_sha256":"be73c854a369f74bc838f6c32458345307e08de22a5f5e596ebe4ada6382d82e"},"schema_version":"1.0","source":{"id":"2506.08681","kind":"arxiv","version":2}},"source_aliases":[{"alias_kind":"arxiv","alias_value":"2506.08681","created_at":"2026-07-05T11:19:30Z"},{"alias_kind":"arxiv_version","alias_value":"2506.08681v2","created_at":"2026-07-05T11:19:30Z"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.2506.08681","created_at":"2026-07-05T11:19:30Z"},{"alias_kind":"pith_short_12","alias_value":"F5DB5DEJMZAP","created_at":"2026-07-05T11:19:30Z"},{"alias_kind":"pith_short_16","alias_value":"F5DB5DEJMZAPXAHP","created_at":"2026-07-05T11:19:30Z"},{"alias_kind":"pith_short_8","alias_value":"F5DB5DEJ","created_at":"2026-07-05T11:19:30Z"}],"graph_snapshots":[{"event_id":"sha256:4cd51a1b17789452c63b90d8d0b2f4abd2803e108d00b6a20ed9b8ba4302415a","target":"graph","created_at":"2026-07-05T11:19:30Z","signer":{"key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signer_id":"pith.science","signer_type":"pith_registry"},"payload":{"graph_snapshot":{"author_claims":{"count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","strong_count":0},"builder_version":"pith-number-builder-2026-05-17-v1","claims":{"count":0,"items":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"integrity":{"available":true,"clean":true,"detectors_run":[],"endpoint":"/pith/2506.08681/integrity.json","findings":[],"snapshot_sha256":"c28c3603d3b5d939e8dc4c7e95fa8dfce3d595e45f758748cecf8e644a296938","summary":{"advisory":0,"by_detector":{},"critical":0,"informational":0}},"paper":{"abstract_excerpt":"Direct Alignment Algorithms (DAAs) such as Direct Preference Optimization (DPO) have emerged as alternatives to the standard Reinforcement Learning from Human Feedback (RLHF) for aligning large language models (LLMs) with human values. However, these methods are more susceptible to over-optimization, in which the model drifts away from the reference policy, leading to degraded performance as training progresses. This paper proposes a novel importance-sampling approach to mitigate the over-optimization problem of offline DAAs. This approach, called (IS-DAAs), multiplies the DAA objective with a","authors_text":"Anji Liu, An Mai, Binh T. Nguyen, Daniel Sonntag, Duy H. M. Nguyen, Khoa D. Doan, Ngoc-Hieu Nguyen, Phuc Minh Nguyen","cross_cats":[],"headline":"","license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.LG","submitted_at":"2025-06-10T10:45:26Z","title":"Mitigating Reward Over-optimization in Direct Alignment Algorithms with Importance Sampling"},"references":{"count":0,"internal_anchors":0,"resolved_work":0,"sample":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"2506.08681","kind":"arxiv","version":2},"verdict":{"created_at":null,"id":null,"model_set":{},"one_line_summary":"","pipeline_version":null,"pith_extraction_headline":"","strongest_claim":"","weakest_assumption":""}},"verdict_id":null}}],"author_attestations":[],"timestamp_anchors":[],"storage_attestations":[],"citation_signatures":[],"replication_records":[],"corrections":[],"mirror_hints":[],"record_created":{"event_id":"sha256:260a061eed4c03d0950023d3c582b7917f65ff48354e90f176a2505acfc326b2","target":"record","created_at":"2026-07-05T11:19:30Z","signer":{"key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signer_id":"pith.science","signer_type":"pith_registry"},"payload":{"attestation_state":"computed","canonical_record":{"metadata":{"abstract_canon_sha256":"f276d3ae07fdbe77bacd745c01542bb645e6de711fecd264c9e43350c945b897","cross_cats_sorted":[],"license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.LG","submitted_at":"2025-06-10T10:45:26Z","title_canon_sha256":"be73c854a369f74bc838f6c32458345307e08de22a5f5e596ebe4ada6382d82e"},"schema_version":"1.0","source":{"id":"2506.08681","kind":"arxiv","version":2}},"canonical_sha256":"2f461e8c896640fb80efa5b5571e3461e327f5e1a73a4570d1f95eaab42bc0a9","receipt":{"algorithm":"ed25519","builder_version":"pith-number-builder-2026-05-17-v1","canonical_sha256":"2f461e8c896640fb80efa5b5571e3461e327f5e1a73a4570d1f95eaab42bc0a9","first_computed_at":"2026-07-05T11:19:30.038541Z","key_id":"pith-v1-2026-05","kind":"pith_receipt","last_reissued_at":"2026-07-05T11:19:30.038541Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","receipt_version":"0.3","signature_b64":"4leh+3zMRdxMVD7awpmiFoXH42iLUJ7pTroIVFHM8EzL9lEN2tWzqASOxYHUvKMMHZ7KchW2U3JbJIqqpXrKDA==","signature_status":"signed_v1","signed_at":"2026-07-05T11:19:30.039143Z","signed_message":"canonical_sha256_bytes"},"source_id":"2506.08681","source_kind":"arxiv","source_version":2}}},"equivocations":[],"invalid_events":[],"applied_event_ids":["sha256:260a061eed4c03d0950023d3c582b7917f65ff48354e90f176a2505acfc326b2","sha256:4cd51a1b17789452c63b90d8d0b2f4abd2803e108d00b6a20ed9b8ba4302415a"],"state_sha256":"cb1b36b1d79bbf9c4af8cf70b304401c3fbde159d71a9927fa924bd1f46c513f"},"bundle_signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"YN2War2WJssgPYWOb+QR4ycH65D6cG5RxGh/1Y1sjdMI3hhUBah8VuiyBtgGN1vupQWXdDNX0krbozcvFw5TBw==","signed_message":"bundle_sha256_bytes","signed_at":"2026-08-07T22:41:33.867158Z","bundle_sha256":"6cdeeca76c8b6010b1197b34d57985f8377c0f5f33bf8d22fef825490da71414"}}