{"bundle_type":"pith_open_graph_bundle","bundle_version":"1.0","pith_number":"pith:2024:K235WED2K2BIJYEDFWZE2CHJTA","short_pith_number":"pith:K235WED2","canonical_record":{"source":{"id":"2409.06411","kind":"arxiv","version":2},"metadata":{"license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.LG","submitted_at":"2024-09-10T10:49:38Z","cross_cats_sorted":["cs.CL"],"title_canon_sha256":"2e69cb7969f28230c802a487487c0c281e82a8f55950b814dcf893efcb713b7c","abstract_canon_sha256":"84a526a0252f42fc4af7318266a79aa58c03caa4d01eb5ab5a6dce9a3c1a7a02"},"schema_version":"1.0"},"canonical_sha256":"56b7db107a568284e0832db24d08e99805f2b3ed33d58c5a2d176ac2ae1d5cdd","source":{"kind":"arxiv","id":"2409.06411","version":2},"source_aliases":[{"alias_kind":"arxiv","alias_value":"2409.06411","created_at":"2026-07-05T09:41:41Z"},{"alias_kind":"arxiv_version","alias_value":"2409.06411v2","created_at":"2026-07-05T09:41:41Z"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.2409.06411","created_at":"2026-07-05T09:41:41Z"},{"alias_kind":"pith_short_12","alias_value":"K235WED2K2BI","created_at":"2026-07-05T09:41:41Z"},{"alias_kind":"pith_short_16","alias_value":"K235WED2K2BIJYED","created_at":"2026-07-05T09:41:41Z"},{"alias_kind":"pith_short_8","alias_value":"K235WED2","created_at":"2026-07-05T09:41:41Z"}],"events":[{"event_type":"record_created","subject_pith_number":"pith:2024:K235WED2K2BIJYEDFWZE2CHJTA","target":"record","payload":{"canonical_record":{"source":{"id":"2409.06411","kind":"arxiv","version":2},"metadata":{"license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.LG","submitted_at":"2024-09-10T10:49:38Z","cross_cats_sorted":["cs.CL"],"title_canon_sha256":"2e69cb7969f28230c802a487487c0c281e82a8f55950b814dcf893efcb713b7c","abstract_canon_sha256":"84a526a0252f42fc4af7318266a79aa58c03caa4d01eb5ab5a6dce9a3c1a7a02"},"schema_version":"1.0"},"canonical_sha256":"56b7db107a568284e0832db24d08e99805f2b3ed33d58c5a2d176ac2ae1d5cdd","receipt":{"kind":"pith_receipt","key_id":"pith-v1-2026-05","algorithm":"ed25519","signed_at":"2026-07-05T09:41:41.419976Z","signature_b64":"AMVi4L8Fj21CEXBQI0blC0/PVaBvYHG+UvO7NQQ0W1I+4mQj+yiFfa3NB4cLL4JCbwu0sryOl81Jf5h2th/eCw==","signed_message":"canonical_sha256_bytes","builder_version":"pith-number-builder-2026-05-17-v1","receipt_version":"0.3","canonical_sha256":"56b7db107a568284e0832db24d08e99805f2b3ed33d58c5a2d176ac2ae1d5cdd","last_reissued_at":"2026-07-05T09:41:41.419500Z","signature_status":"signed_v1","first_computed_at":"2026-07-05T09:41:41.419500Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"source_kind":"arxiv","source_id":"2409.06411","source_version":2,"attestation_state":"computed"},"signer":{"signer_id":"pith.science","signer_type":"pith_registry","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"created_at":"2026-07-05T09:41:41Z","supersedes":[],"prev_event":null,"signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"uddpOsXcMVN6mNACLtskRwojh4g4AOP4LRZk+f6oYd+ZqA82QwUi4yhQLNcrrBgMpfvCyuv8gbpTqrt9jxxYAg==","signed_message":"open_graph_event_sha256_bytes","signed_at":"2026-08-14T09:15:05.150417Z"},"content_sha256":"918c245faaea6b334ee52f37a30ee4fa6269b8326a59c1b2abfa93f389e0f34f","schema_version":"1.0","event_id":"sha256:918c245faaea6b334ee52f37a30ee4fa6269b8326a59c1b2abfa93f389e0f34f"},{"event_type":"graph_snapshot","subject_pith_number":"pith:2024:K235WED2K2BIJYEDFWZE2CHJTA","target":"graph","payload":{"graph_snapshot":{"paper":{"title":"Length Desensitization in Direct Preference Optimization","license":"http://creativecommons.org/licenses/by/4.0/","headline":"","cross_cats":["cs.CL"],"primary_cat":"cs.LG","authors_text":"Chengcheng Han, Jingang Wang, Jun Xu, Rongxiang Weng, Wei Liu, Xuezhi Cao, Xunliang Cai, Yang Bai","submitted_at":"2024-09-10T10:49:38Z","abstract_excerpt":"Direct Preference Optimization (DPO) is widely utilized in the Reinforcement Learning from Human Feedback (RLHF) phase to align Large Language Models (LLMs) with human preferences, thereby enhancing both their harmlessness and efficacy. However, it has been observed that DPO tends to over-optimize for verbosity, which can detrimentally affect both performance and user experience. In this paper, we conduct an in-depth theoretical analysis of DPO's optimization objective and reveal a strong correlation between its implicit reward and data length. This correlation misguides the optimization direc"},"claims":{"count":0,"items":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"2409.06411","kind":"arxiv","version":2},"verdict":{"id":null,"model_set":{},"created_at":null,"strongest_claim":"","one_line_summary":"","pipeline_version":null,"weakest_assumption":"","pith_extraction_headline":""},"integrity":{"clean":true,"summary":{"advisory":0,"critical":0,"by_detector":{},"informational":0},"endpoint":"/pith/2409.06411/integrity.json","findings":[],"available":true,"detectors_run":[],"snapshot_sha256":"c28c3603d3b5d939e8dc4c7e95fa8dfce3d595e45f758748cecf8e644a296938"},"references":{"count":0,"sample":[],"resolved_work":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","internal_anchors":0},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"author_claims":{"count":0,"strong_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"builder_version":"pith-number-builder-2026-05-17-v1"},"verdict_id":null},"signer":{"signer_id":"pith.science","signer_type":"pith_registry","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"created_at":"2026-07-05T09:41:41Z","supersedes":[],"prev_event":null,"signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"QFH1Vb6ZETvTsBREYX7DngqEnSkwPo4YUZ/XQ3LA4eiTbaVYWREeiTPyNH4RyTYxQ2/v5hvZY171Z9U32EHBBA==","signed_message":"open_graph_event_sha256_bytes","signed_at":"2026-08-14T09:15:05.151298Z"},"content_sha256":"e2902f16ffdcac93be937b1fac8d8983685c07f14d6373a531024dce6244afbc","schema_version":"1.0","event_id":"sha256:e2902f16ffdcac93be937b1fac8d8983685c07f14d6373a531024dce6244afbc"}],"timestamp_proofs":[],"mirror_hints":[{"mirror_type":"https","name":"Pith Resolver","base_url":"https://pith.science","bundle_url":"https://pith.science/pith/K235WED2K2BIJYEDFWZE2CHJTA/bundle.json","state_url":"https://pith.science/pith/K235WED2K2BIJYEDFWZE2CHJTA/state.json","well_known_bundle_url":"https://pith.science/.well-known/pith/K235WED2K2BIJYEDFWZE2CHJTA/bundle.json","status":"primary"}],"public_keys":[{"key_id":"pith-v1-2026-05","algorithm":"ed25519","format":"raw","public_key_b64":"stVStoiQhXFxp4s2pdzPNoqVNBMojDU/fJ2db5S3CbM=","public_key_hex":"b2d552b68890857171a78b36a5dccf368a953413288c353f7c9d9d6f94b709b3","fingerprint_sha256_b32_first128bits":"RVFV5Z2OI2J3ZUO7ERDEBCYNKS","fingerprint_sha256_hex":"8d4b5ee74e4693bcd1df2446408b0d54","rotates_at":null,"url":"https://pith.science/pith-signing-key.json","notes":"Pith uses this Ed25519 key to sign canonical record SHA-256 digests. Verify with: ed25519_verify(public_key, message=canonical_sha256_bytes, signature=base64decode(signature_b64))."}],"merge_version":"pith-open-graph-merge-v1","built_at":"2026-08-14T09:15:05Z","links":{"resolver":"https://pith.science/pith/K235WED2K2BIJYEDFWZE2CHJTA","bundle":"https://pith.science/pith/K235WED2K2BIJYEDFWZE2CHJTA/bundle.json","state":"https://pith.science/pith/K235WED2K2BIJYEDFWZE2CHJTA/state.json","well_known_bundle":"https://pith.science/.well-known/pith/K235WED2K2BIJYEDFWZE2CHJTA/bundle.json"},"state":{"state_type":"pith_open_graph_state","state_version":"1.0","pith_number":"pith:2024:K235WED2K2BIJYEDFWZE2CHJTA","merge_version":"pith-open-graph-merge-v1","event_count":2,"valid_event_count":2,"invalid_event_count":0,"equivocation_count":0,"current":{"canonical_record":{"metadata":{"abstract_canon_sha256":"84a526a0252f42fc4af7318266a79aa58c03caa4d01eb5ab5a6dce9a3c1a7a02","cross_cats_sorted":["cs.CL"],"license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.LG","submitted_at":"2024-09-10T10:49:38Z","title_canon_sha256":"2e69cb7969f28230c802a487487c0c281e82a8f55950b814dcf893efcb713b7c"},"schema_version":"1.0","source":{"id":"2409.06411","kind":"arxiv","version":2}},"source_aliases":[{"alias_kind":"arxiv","alias_value":"2409.06411","created_at":"2026-07-05T09:41:41Z"},{"alias_kind":"arxiv_version","alias_value":"2409.06411v2","created_at":"2026-07-05T09:41:41Z"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.2409.06411","created_at":"2026-07-05T09:41:41Z"},{"alias_kind":"pith_short_12","alias_value":"K235WED2K2BI","created_at":"2026-07-05T09:41:41Z"},{"alias_kind":"pith_short_16","alias_value":"K235WED2K2BIJYED","created_at":"2026-07-05T09:41:41Z"},{"alias_kind":"pith_short_8","alias_value":"K235WED2","created_at":"2026-07-05T09:41:41Z"}],"graph_snapshots":[{"event_id":"sha256:e2902f16ffdcac93be937b1fac8d8983685c07f14d6373a531024dce6244afbc","target":"graph","created_at":"2026-07-05T09:41:41Z","signer":{"key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signer_id":"pith.science","signer_type":"pith_registry"},"payload":{"graph_snapshot":{"author_claims":{"count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","strong_count":0},"builder_version":"pith-number-builder-2026-05-17-v1","claims":{"count":0,"items":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"integrity":{"available":true,"clean":true,"detectors_run":[],"endpoint":"/pith/2409.06411/integrity.json","findings":[],"snapshot_sha256":"c28c3603d3b5d939e8dc4c7e95fa8dfce3d595e45f758748cecf8e644a296938","summary":{"advisory":0,"by_detector":{},"critical":0,"informational":0}},"paper":{"abstract_excerpt":"Direct Preference Optimization (DPO) is widely utilized in the Reinforcement Learning from Human Feedback (RLHF) phase to align Large Language Models (LLMs) with human preferences, thereby enhancing both their harmlessness and efficacy. However, it has been observed that DPO tends to over-optimize for verbosity, which can detrimentally affect both performance and user experience. In this paper, we conduct an in-depth theoretical analysis of DPO's optimization objective and reveal a strong correlation between its implicit reward and data length. This correlation misguides the optimization direc","authors_text":"Chengcheng Han, Jingang Wang, Jun Xu, Rongxiang Weng, Wei Liu, Xuezhi Cao, Xunliang Cai, Yang Bai","cross_cats":["cs.CL"],"headline":"","license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.LG","submitted_at":"2024-09-10T10:49:38Z","title":"Length Desensitization in Direct Preference Optimization"},"references":{"count":0,"internal_anchors":0,"resolved_work":0,"sample":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"2409.06411","kind":"arxiv","version":2},"verdict":{"created_at":null,"id":null,"model_set":{},"one_line_summary":"","pipeline_version":null,"pith_extraction_headline":"","strongest_claim":"","weakest_assumption":""}},"verdict_id":null}}],"author_attestations":[],"timestamp_anchors":[],"storage_attestations":[],"citation_signatures":[],"replication_records":[],"corrections":[],"mirror_hints":[],"record_created":{"event_id":"sha256:918c245faaea6b334ee52f37a30ee4fa6269b8326a59c1b2abfa93f389e0f34f","target":"record","created_at":"2026-07-05T09:41:41Z","signer":{"key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signer_id":"pith.science","signer_type":"pith_registry"},"payload":{"attestation_state":"computed","canonical_record":{"metadata":{"abstract_canon_sha256":"84a526a0252f42fc4af7318266a79aa58c03caa4d01eb5ab5a6dce9a3c1a7a02","cross_cats_sorted":["cs.CL"],"license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.LG","submitted_at":"2024-09-10T10:49:38Z","title_canon_sha256":"2e69cb7969f28230c802a487487c0c281e82a8f55950b814dcf893efcb713b7c"},"schema_version":"1.0","source":{"id":"2409.06411","kind":"arxiv","version":2}},"canonical_sha256":"56b7db107a568284e0832db24d08e99805f2b3ed33d58c5a2d176ac2ae1d5cdd","receipt":{"algorithm":"ed25519","builder_version":"pith-number-builder-2026-05-17-v1","canonical_sha256":"56b7db107a568284e0832db24d08e99805f2b3ed33d58c5a2d176ac2ae1d5cdd","first_computed_at":"2026-07-05T09:41:41.419500Z","key_id":"pith-v1-2026-05","kind":"pith_receipt","last_reissued_at":"2026-07-05T09:41:41.419500Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","receipt_version":"0.3","signature_b64":"AMVi4L8Fj21CEXBQI0blC0/PVaBvYHG+UvO7NQQ0W1I+4mQj+yiFfa3NB4cLL4JCbwu0sryOl81Jf5h2th/eCw==","signature_status":"signed_v1","signed_at":"2026-07-05T09:41:41.419976Z","signed_message":"canonical_sha256_bytes"},"source_id":"2409.06411","source_kind":"arxiv","source_version":2}}},"equivocations":[],"invalid_events":[],"applied_event_ids":["sha256:918c245faaea6b334ee52f37a30ee4fa6269b8326a59c1b2abfa93f389e0f34f","sha256:e2902f16ffdcac93be937b1fac8d8983685c07f14d6373a531024dce6244afbc"],"state_sha256":"8363e9722caf8c900d2ee0d56ac42f964e119375823536d1ea21a6ca5bbc57d0"},"bundle_signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"J8A22261mjIS2qimN9iCoedkvalz93kMlUazWcXXKK64rldgBOGHIUqSEhGM9wxyVbh9Oggs5OKGvnbIRFdrCw==","signed_message":"bundle_sha256_bytes","signed_at":"2026-08-14T09:15:05.163239Z","bundle_sha256":"36527c1ea02a8abab07dfe1880866eff3d19fc8f69a0c6a64288853674ae2921"}}