{"record_type":"pith_number_record","schema_url":"https://pith.science/schemas/pith-number/v1.json","pith_number":"pith:2025:7VGEXAL5VHE5PS7TIGZCX4UFQP","short_pith_number":"pith:7VGEXAL5","schema_version":"1.0","canonical_sha256":"fd4c4b817da9c9d7cbf341b22bf28583d8cff08b2a9bbad16d1108cc3af6c640","source":{"kind":"arxiv","id":"2502.20847","version":1},"attestation_state":"computed","paper":{"title":"Gradient Imbalance in Direct Preference Optimization","license":"http://creativecommons.org/licenses/by/4.0/","headline":"","cross_cats":[],"primary_cat":"cs.LG","authors_text":"Can Jin, Jenq-Neng Hwang, Jingzhe Shi, Lei Li, Qinwei Ma, Serge Belongie","submitted_at":"2025-02-28T08:47:03Z","abstract_excerpt":"Direct Preference Optimization (DPO) has been proposed as a promising alternative to Proximal Policy Optimization (PPO) based Reinforcement Learning with Human Feedback (RLHF). However, empirical evaluations consistently reveal suboptimal performance in DPO compared to common RLHF pipelines. In this work, we conduct a systematic analysis of DPO's training dynamics and identify gradient imbalance as a critical limitation. We demonstrate theoretically and empirically that this imbalance perturbs optimization trajectories, destabilizes learning, and induces suboptimal convergence. To address this"},"verification_status":{"content_addressed":true,"pith_receipt":true,"author_attested":false,"weak_author_claims":0,"strong_author_claims":0,"externally_anchored":false,"storage_verified":false,"citation_signatures":0,"replication_records":0,"graph_snapshot":true,"references_resolved":false,"formal_links_present":false},"canonical_record":{"source":{"id":"2502.20847","kind":"arxiv","version":1},"metadata":{"license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.LG","submitted_at":"2025-02-28T08:47:03Z","cross_cats_sorted":[],"title_canon_sha256":"f7c0f99c752d9249bb2941d4b6004819fc7151312d96b4f3d41b96b3c5b897e3","abstract_canon_sha256":"6edb87d0bd995b2e3c5539a87105feb77e073fa69e7f31a2cded466df307069e"},"schema_version":"1.0"},"receipt":{"kind":"pith_receipt","key_id":"pith-v1-2026-05","algorithm":"ed25519","signed_at":"2026-07-05T10:21:40.895282Z","signature_b64":"OMVcq9d4tD+EP4cU9VDSrcAZpe4y5dIePG/zWpqmEE8G4aAYHN0LiZiccmIJ4XM2awHaSRNCo+0YFTVgFoULCw==","signed_message":"canonical_sha256_bytes","builder_version":"pith-number-builder-2026-05-17-v1","receipt_version":"0.3","canonical_sha256":"fd4c4b817da9c9d7cbf341b22bf28583d8cff08b2a9bbad16d1108cc3af6c640","last_reissued_at":"2026-07-05T10:21:40.894769Z","signature_status":"signed_v1","first_computed_at":"2026-07-05T10:21:40.894769Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"graph_snapshot":{"paper":{"title":"Gradient Imbalance in Direct Preference Optimization","license":"http://creativecommons.org/licenses/by/4.0/","headline":"","cross_cats":[],"primary_cat":"cs.LG","authors_text":"Can Jin, Jenq-Neng Hwang, Jingzhe Shi, Lei Li, Qinwei Ma, Serge Belongie","submitted_at":"2025-02-28T08:47:03Z","abstract_excerpt":"Direct Preference Optimization (DPO) has been proposed as a promising alternative to Proximal Policy Optimization (PPO) based Reinforcement Learning with Human Feedback (RLHF). However, empirical evaluations consistently reveal suboptimal performance in DPO compared to common RLHF pipelines. In this work, we conduct a systematic analysis of DPO's training dynamics and identify gradient imbalance as a critical limitation. We demonstrate theoretically and empirically that this imbalance perturbs optimization trajectories, destabilizes learning, and induces suboptimal convergence. To address this"},"claims":{"count":0,"items":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"2502.20847","kind":"arxiv","version":1},"verdict":{"id":null,"model_set":{},"created_at":null,"strongest_claim":"","one_line_summary":"","pipeline_version":null,"weakest_assumption":"","pith_extraction_headline":""},"integrity":{"clean":true,"summary":{"advisory":0,"critical":0,"by_detector":{},"informational":0},"endpoint":"/pith/2502.20847/integrity.json","findings":[],"available":true,"detectors_run":[],"snapshot_sha256":"c28c3603d3b5d939e8dc4c7e95fa8dfce3d595e45f758748cecf8e644a296938"},"references":{"count":0,"sample":[],"resolved_work":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","internal_anchors":0},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"author_claims":{"count":0,"strong_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"builder_version":"pith-number-builder-2026-05-17-v1"},"aliases":[{"alias_kind":"arxiv","alias_value":"2502.20847","created_at":"2026-07-05T10:21:40.894829+00:00"},{"alias_kind":"arxiv_version","alias_value":"2502.20847v1","created_at":"2026-07-05T10:21:40.894829+00:00"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.2502.20847","created_at":"2026-07-05T10:21:40.894829+00:00"},{"alias_kind":"pith_short_12","alias_value":"7VGEXAL5VHE5","created_at":"2026-07-05T10:21:40.894829+00:00"},{"alias_kind":"pith_short_16","alias_value":"7VGEXAL5VHE5PS7T","created_at":"2026-07-05T10:21:40.894829+00:00"},{"alias_kind":"pith_short_8","alias_value":"7VGEXAL5","created_at":"2026-07-05T10:21:40.894829+00:00"}],"events":[],"event_summary":{},"paper_claims":[],"inbound_citations":{"count":2,"internal_anchor_count":0,"sample":[{"citing_arxiv_id":"2605.28440","citing_title":"AdaDPO: Self-Adaptive Direct Preference Optimization with Balanced Gradient Updates","ref_index":6,"is_internal_anchor":false},{"citing_arxiv_id":"2605.15012","citing_title":"Boosting Reinforcement Learning with Verifiable Rewards via Randomly Selected Few-Shot Guidance","ref_index":57,"is_internal_anchor":false}]},"formal_canon":{"evidence_count":0,"sample":[],"anchors":[]},"links":{"html":"https://pith.science/pith/7VGEXAL5VHE5PS7TIGZCX4UFQP","json":"https://pith.science/pith/7VGEXAL5VHE5PS7TIGZCX4UFQP.json","graph_json":"https://pith.science/api/pith-number/7VGEXAL5VHE5PS7TIGZCX4UFQP/graph.json","events_json":"https://pith.science/api/pith-number/7VGEXAL5VHE5PS7TIGZCX4UFQP/events.json","paper":"https://pith.science/paper/7VGEXAL5"},"agent_actions":{"view_html":"https://pith.science/pith/7VGEXAL5VHE5PS7TIGZCX4UFQP","download_json":"https://pith.science/pith/7VGEXAL5VHE5PS7TIGZCX4UFQP.json","view_paper":"https://pith.science/paper/7VGEXAL5","resolve_alias":"https://pith.science/api/pith-number/resolve?arxiv=2502.20847&json=true","fetch_graph":"https://pith.science/api/pith-number/7VGEXAL5VHE5PS7TIGZCX4UFQP/graph.json","fetch_events":"https://pith.science/api/pith-number/7VGEXAL5VHE5PS7TIGZCX4UFQP/events.json","actions":{"anchor_timestamp":"https://pith.science/pith/7VGEXAL5VHE5PS7TIGZCX4UFQP/action/timestamp_anchor","attest_storage":"https://pith.science/pith/7VGEXAL5VHE5PS7TIGZCX4UFQP/action/storage_attestation","attest_author":"https://pith.science/pith/7VGEXAL5VHE5PS7TIGZCX4UFQP/action/author_attestation","sign_citation":"https://pith.science/pith/7VGEXAL5VHE5PS7TIGZCX4UFQP/action/citation_signature","submit_replication":"https://pith.science/pith/7VGEXAL5VHE5PS7TIGZCX4UFQP/action/replication_record"}},"created_at":"2026-07-05T10:21:40.894829+00:00","updated_at":"2026-07-05T10:21:40.894829+00:00"}