{"record_type":"pith_number_record","schema_url":"https://pith.science/schemas/pith-number/v1.json","pith_number":"pith:2025:JECZIIMDDOJYK437VGPVEBNJOL","short_pith_number":"pith:JECZIIMD","schema_version":"1.0","canonical_sha256":"49059421831b9385737fa99f5205a972fefc95e9f9afe4f324f55df33ddaac18","source":{"kind":"arxiv","id":"2502.16852","version":1},"attestation_state":"computed","paper":{"title":"Improving LLM General Preference Alignment via Optimistic Online Mirror Descent","license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","headline":"","cross_cats":["cs.AI","cs.CL"],"primary_cat":"cs.LG","authors_text":"Dian Yu, Dong Yu, Haitao Mi, Linfeng Song, Nan Jiang, Tao Ge, Yuheng Zhang, Zhichen Zeng","submitted_at":"2025-02-24T05:24:52Z","abstract_excerpt":"Reinforcement learning from human feedback (RLHF) has demonstrated remarkable effectiveness in aligning large language models (LLMs) with human preferences. Many existing alignment approaches rely on the Bradley-Terry (BT) model assumption, which assumes the existence of a ground-truth reward for each prompt-response pair. However, this assumption can be overly restrictive when modeling complex human preferences. In this paper, we drop the BT model assumption and study LLM alignment under general preferences, formulated as a two-player game. Drawing on theoretical insights from learning in gam"},"verification_status":{"content_addressed":true,"pith_receipt":true,"author_attested":false,"weak_author_claims":0,"strong_author_claims":0,"externally_anchored":false,"storage_verified":false,"citation_signatures":0,"replication_records":0,"graph_snapshot":true,"references_resolved":false,"formal_links_present":false},"canonical_record":{"source":{"id":"2502.16852","kind":"arxiv","version":1},"metadata":{"license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","primary_cat":"cs.LG","submitted_at":"2025-02-24T05:24:52Z","cross_cats_sorted":["cs.AI","cs.CL"],"title_canon_sha256":"f3e112b569265803411ada93a1b8b2c8c7d8e7a39336ffd2b7689cc5887fe7c0","abstract_canon_sha256":"469d80476c992b3a30b6e32a23b7cbb5886278d00db28bc31f00201debe01e52"},"schema_version":"1.0"},"receipt":{"kind":"pith_receipt","key_id":"pith-v1-2026-05","algorithm":"ed25519","signed_at":"2026-07-05T10:19:00.887287Z","signature_b64":"e6T4MbXHbVslVyKpWayoAmfcR2bOBSMwOm1eqt8KWkMiZV0PnkZgvqI5j6Gx2Wtb2pP9wmx7ZS8UB5Z20NCGDQ==","signed_message":"canonical_sha256_bytes","builder_version":"pith-number-builder-2026-05-17-v1","receipt_version":"0.3","canonical_sha256":"49059421831b9385737fa99f5205a972fefc95e9f9afe4f324f55df33ddaac18","last_reissued_at":"2026-07-05T10:19:00.886825Z","signature_status":"signed_v1","first_computed_at":"2026-07-05T10:19:00.886825Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"graph_snapshot":{"paper":{"title":"Improving LLM General Preference Alignment via Optimistic Online Mirror Descent","license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","headline":"","cross_cats":["cs.AI","cs.CL"],"primary_cat":"cs.LG","authors_text":"Dian Yu, Dong Yu, Haitao Mi, Linfeng Song, Nan Jiang, Tao Ge, Yuheng Zhang, Zhichen Zeng","submitted_at":"2025-02-24T05:24:52Z","abstract_excerpt":"Reinforcement learning from human feedback (RLHF) has demonstrated remarkable effectiveness in aligning large language models (LLMs) with human preferences. Many existing alignment approaches rely on the Bradley-Terry (BT) model assumption, which assumes the existence of a ground-truth reward for each prompt-response pair. However, this assumption can be overly restrictive when modeling complex human preferences. In this paper, we drop the BT model assumption and study LLM alignment under general preferences, formulated as a two-player game. Drawing on theoretical insights from learning in gam"},"claims":{"count":0,"items":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"2502.16852","kind":"arxiv","version":1},"verdict":{"id":null,"model_set":{},"created_at":null,"strongest_claim":"","one_line_summary":"","pipeline_version":null,"weakest_assumption":"","pith_extraction_headline":""},"integrity":{"clean":true,"summary":{"advisory":0,"critical":0,"by_detector":{},"informational":0},"endpoint":"/pith/2502.16852/integrity.json","findings":[],"available":true,"detectors_run":[],"snapshot_sha256":"c28c3603d3b5d939e8dc4c7e95fa8dfce3d595e45f758748cecf8e644a296938"},"references":{"count":0,"sample":[],"resolved_work":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","internal_anchors":0},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"author_claims":{"count":0,"strong_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"builder_version":"pith-number-builder-2026-05-17-v1"},"aliases":[{"alias_kind":"arxiv","alias_value":"2502.16852","created_at":"2026-07-05T10:19:00.886880+00:00"},{"alias_kind":"arxiv_version","alias_value":"2502.16852v1","created_at":"2026-07-05T10:19:00.886880+00:00"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.2502.16852","created_at":"2026-07-05T10:19:00.886880+00:00"},{"alias_kind":"pith_short_12","alias_value":"JECZIIMDDOJY","created_at":"2026-07-05T10:19:00.886880+00:00"},{"alias_kind":"pith_short_16","alias_value":"JECZIIMDDOJYK437","created_at":"2026-07-05T10:19:00.886880+00:00"},{"alias_kind":"pith_short_8","alias_value":"JECZIIMD","created_at":"2026-07-05T10:19:00.886880+00:00"}],"events":[],"event_summary":{},"paper_claims":[],"inbound_citations":{"count":5,"internal_anchor_count":0,"sample":[{"citing_arxiv_id":"2504.12501","citing_title":"Reinforcement Learning from Human Feedback","ref_index":147,"is_internal_anchor":false},{"citing_arxiv_id":"2605.17342","citing_title":"Transitivity Meets Cyclicity: Explicit Preference Decomposition for Dynamic Large Language Model Alignment","ref_index":104,"is_internal_anchor":false},{"citing_arxiv_id":"2509.23102","citing_title":"Multiplayer Nash Preference Optimization","ref_index":39,"is_internal_anchor":false},{"citing_arxiv_id":"2605.04494","citing_title":"Towards General Preference Alignment: Diffusion Models at Nash Equilibrium","ref_index":53,"is_internal_anchor":false},{"citing_arxiv_id":"2605.07331","citing_title":"Rethinking Importance Sampling in LLM Policy Optimization: A Cumulative Token Perspective","ref_index":27,"is_internal_anchor":false}]},"formal_canon":{"evidence_count":0,"sample":[],"anchors":[]},"links":{"html":"https://pith.science/pith/JECZIIMDDOJYK437VGPVEBNJOL","json":"https://pith.science/pith/JECZIIMDDOJYK437VGPVEBNJOL.json","graph_json":"https://pith.science/api/pith-number/JECZIIMDDOJYK437VGPVEBNJOL/graph.json","events_json":"https://pith.science/api/pith-number/JECZIIMDDOJYK437VGPVEBNJOL/events.json","paper":"https://pith.science/paper/JECZIIMD"},"agent_actions":{"view_html":"https://pith.science/pith/JECZIIMDDOJYK437VGPVEBNJOL","download_json":"https://pith.science/pith/JECZIIMDDOJYK437VGPVEBNJOL.json","view_paper":"https://pith.science/paper/JECZIIMD","resolve_alias":"https://pith.science/api/pith-number/resolve?arxiv=2502.16852&json=true","fetch_graph":"https://pith.science/api/pith-number/JECZIIMDDOJYK437VGPVEBNJOL/graph.json","fetch_events":"https://pith.science/api/pith-number/JECZIIMDDOJYK437VGPVEBNJOL/events.json","actions":{"anchor_timestamp":"https://pith.science/pith/JECZIIMDDOJYK437VGPVEBNJOL/action/timestamp_anchor","attest_storage":"https://pith.science/pith/JECZIIMDDOJYK437VGPVEBNJOL/action/storage_attestation","attest_author":"https://pith.science/pith/JECZIIMDDOJYK437VGPVEBNJOL/action/author_attestation","sign_citation":"https://pith.science/pith/JECZIIMDDOJYK437VGPVEBNJOL/action/citation_signature","submit_replication":"https://pith.science/pith/JECZIIMDDOJYK437VGPVEBNJOL/action/replication_record"}},"created_at":"2026-07-05T10:19:00.886880+00:00","updated_at":"2026-07-05T10:19:00.886880+00:00"}