{"bundle_type":"pith_open_graph_bundle","bundle_version":"1.0","pith_number":"pith:2025:JECZIIMDDOJYK437VGPVEBNJOL","short_pith_number":"pith:JECZIIMD","canonical_record":{"source":{"id":"2502.16852","kind":"arxiv","version":1},"metadata":{"license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","primary_cat":"cs.LG","submitted_at":"2025-02-24T05:24:52Z","cross_cats_sorted":["cs.AI","cs.CL"],"title_canon_sha256":"f3e112b569265803411ada93a1b8b2c8c7d8e7a39336ffd2b7689cc5887fe7c0","abstract_canon_sha256":"469d80476c992b3a30b6e32a23b7cbb5886278d00db28bc31f00201debe01e52"},"schema_version":"1.0"},"canonical_sha256":"49059421831b9385737fa99f5205a972fefc95e9f9afe4f324f55df33ddaac18","source":{"kind":"arxiv","id":"2502.16852","version":1},"source_aliases":[{"alias_kind":"arxiv","alias_value":"2502.16852","created_at":"2026-07-05T10:19:00Z"},{"alias_kind":"arxiv_version","alias_value":"2502.16852v1","created_at":"2026-07-05T10:19:00Z"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.2502.16852","created_at":"2026-07-05T10:19:00Z"},{"alias_kind":"pith_short_12","alias_value":"JECZIIMDDOJY","created_at":"2026-07-05T10:19:00Z"},{"alias_kind":"pith_short_16","alias_value":"JECZIIMDDOJYK437","created_at":"2026-07-05T10:19:00Z"},{"alias_kind":"pith_short_8","alias_value":"JECZIIMD","created_at":"2026-07-05T10:19:00Z"}],"events":[{"event_type":"record_created","subject_pith_number":"pith:2025:JECZIIMDDOJYK437VGPVEBNJOL","target":"record","payload":{"canonical_record":{"source":{"id":"2502.16852","kind":"arxiv","version":1},"metadata":{"license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","primary_cat":"cs.LG","submitted_at":"2025-02-24T05:24:52Z","cross_cats_sorted":["cs.AI","cs.CL"],"title_canon_sha256":"f3e112b569265803411ada93a1b8b2c8c7d8e7a39336ffd2b7689cc5887fe7c0","abstract_canon_sha256":"469d80476c992b3a30b6e32a23b7cbb5886278d00db28bc31f00201debe01e52"},"schema_version":"1.0"},"canonical_sha256":"49059421831b9385737fa99f5205a972fefc95e9f9afe4f324f55df33ddaac18","receipt":{"kind":"pith_receipt","key_id":"pith-v1-2026-05","algorithm":"ed25519","signed_at":"2026-07-05T10:19:00.887287Z","signature_b64":"e6T4MbXHbVslVyKpWayoAmfcR2bOBSMwOm1eqt8KWkMiZV0PnkZgvqI5j6Gx2Wtb2pP9wmx7ZS8UB5Z20NCGDQ==","signed_message":"canonical_sha256_bytes","builder_version":"pith-number-builder-2026-05-17-v1","receipt_version":"0.3","canonical_sha256":"49059421831b9385737fa99f5205a972fefc95e9f9afe4f324f55df33ddaac18","last_reissued_at":"2026-07-05T10:19:00.886825Z","signature_status":"signed_v1","first_computed_at":"2026-07-05T10:19:00.886825Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"source_kind":"arxiv","source_id":"2502.16852","source_version":1,"attestation_state":"computed"},"signer":{"signer_id":"pith.science","signer_type":"pith_registry","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"created_at":"2026-07-05T10:19:00Z","supersedes":[],"prev_event":null,"signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"yUkRS7iuZOmdfVigZRa3FSBtf01lSy7lrB2rpOLZ5H69sJ+lsvv1OneOhQYODvvILMKWEyBWxcRv6bea+w30AQ==","signed_message":"open_graph_event_sha256_bytes","signed_at":"2026-08-04T23:20:00.380745Z"},"content_sha256":"470d98009b17f1a4d4589210ff30e2a25f1ef4e2188a08beb08f3b3fc1fc7154","schema_version":"1.0","event_id":"sha256:470d98009b17f1a4d4589210ff30e2a25f1ef4e2188a08beb08f3b3fc1fc7154"},{"event_type":"graph_snapshot","subject_pith_number":"pith:2025:JECZIIMDDOJYK437VGPVEBNJOL","target":"graph","payload":{"graph_snapshot":{"paper":{"title":"Improving LLM General Preference Alignment via Optimistic Online Mirror Descent","license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","headline":"","cross_cats":["cs.AI","cs.CL"],"primary_cat":"cs.LG","authors_text":"Dian Yu, Dong Yu, Haitao Mi, Linfeng Song, Nan Jiang, Tao Ge, Yuheng Zhang, Zhichen Zeng","submitted_at":"2025-02-24T05:24:52Z","abstract_excerpt":"Reinforcement learning from human feedback (RLHF) has demonstrated remarkable effectiveness in aligning large language models (LLMs) with human preferences. Many existing alignment approaches rely on the Bradley-Terry (BT) model assumption, which assumes the existence of a ground-truth reward for each prompt-response pair. However, this assumption can be overly restrictive when modeling complex human preferences. In this paper, we drop the BT model assumption and study LLM alignment under general preferences, formulated as a two-player game. Drawing on theoretical insights from learning in gam"},"claims":{"count":0,"items":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"2502.16852","kind":"arxiv","version":1},"verdict":{"id":null,"model_set":{},"created_at":null,"strongest_claim":"","one_line_summary":"","pipeline_version":null,"weakest_assumption":"","pith_extraction_headline":""},"integrity":{"clean":true,"summary":{"advisory":0,"critical":0,"by_detector":{},"informational":0},"endpoint":"/pith/2502.16852/integrity.json","findings":[],"available":true,"detectors_run":[],"snapshot_sha256":"c28c3603d3b5d939e8dc4c7e95fa8dfce3d595e45f758748cecf8e644a296938"},"references":{"count":0,"sample":[],"resolved_work":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","internal_anchors":0},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"author_claims":{"count":0,"strong_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"builder_version":"pith-number-builder-2026-05-17-v1"},"verdict_id":null},"signer":{"signer_id":"pith.science","signer_type":"pith_registry","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"created_at":"2026-07-05T10:19:00Z","supersedes":[],"prev_event":null,"signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"WL90DP8Qxikulrgn1V0pedmL4G6o3d4XGlw4bVe/NlNdsOvFK7KvN7Pop06ukPRd8nyy1nLXqnH5lJSrYqosBw==","signed_message":"open_graph_event_sha256_bytes","signed_at":"2026-08-04T23:20:00.381326Z"},"content_sha256":"c36a229a11c86cd940c4e76187cb27db73bb805faf0c9d7f769bc729dc7e042d","schema_version":"1.0","event_id":"sha256:c36a229a11c86cd940c4e76187cb27db73bb805faf0c9d7f769bc729dc7e042d"}],"timestamp_proofs":[],"mirror_hints":[{"mirror_type":"https","name":"Pith Resolver","base_url":"https://pith.science","bundle_url":"https://pith.science/pith/JECZIIMDDOJYK437VGPVEBNJOL/bundle.json","state_url":"https://pith.science/pith/JECZIIMDDOJYK437VGPVEBNJOL/state.json","well_known_bundle_url":"https://pith.science/.well-known/pith/JECZIIMDDOJYK437VGPVEBNJOL/bundle.json","status":"primary"}],"public_keys":[{"key_id":"pith-v1-2026-05","algorithm":"ed25519","format":"raw","public_key_b64":"stVStoiQhXFxp4s2pdzPNoqVNBMojDU/fJ2db5S3CbM=","public_key_hex":"b2d552b68890857171a78b36a5dccf368a953413288c353f7c9d9d6f94b709b3","fingerprint_sha256_b32_first128bits":"RVFV5Z2OI2J3ZUO7ERDEBCYNKS","fingerprint_sha256_hex":"8d4b5ee74e4693bcd1df2446408b0d54","rotates_at":null,"url":"https://pith.science/pith-signing-key.json","notes":"Pith uses this Ed25519 key to sign canonical record SHA-256 digests. Verify with: ed25519_verify(public_key, message=canonical_sha256_bytes, signature=base64decode(signature_b64))."}],"merge_version":"pith-open-graph-merge-v1","built_at":"2026-08-04T23:20:00Z","links":{"resolver":"https://pith.science/pith/JECZIIMDDOJYK437VGPVEBNJOL","bundle":"https://pith.science/pith/JECZIIMDDOJYK437VGPVEBNJOL/bundle.json","state":"https://pith.science/pith/JECZIIMDDOJYK437VGPVEBNJOL/state.json","well_known_bundle":"https://pith.science/.well-known/pith/JECZIIMDDOJYK437VGPVEBNJOL/bundle.json"},"state":{"state_type":"pith_open_graph_state","state_version":"1.0","pith_number":"pith:2025:JECZIIMDDOJYK437VGPVEBNJOL","merge_version":"pith-open-graph-merge-v1","event_count":2,"valid_event_count":2,"invalid_event_count":0,"equivocation_count":0,"current":{"canonical_record":{"metadata":{"abstract_canon_sha256":"469d80476c992b3a30b6e32a23b7cbb5886278d00db28bc31f00201debe01e52","cross_cats_sorted":["cs.AI","cs.CL"],"license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","primary_cat":"cs.LG","submitted_at":"2025-02-24T05:24:52Z","title_canon_sha256":"f3e112b569265803411ada93a1b8b2c8c7d8e7a39336ffd2b7689cc5887fe7c0"},"schema_version":"1.0","source":{"id":"2502.16852","kind":"arxiv","version":1}},"source_aliases":[{"alias_kind":"arxiv","alias_value":"2502.16852","created_at":"2026-07-05T10:19:00Z"},{"alias_kind":"arxiv_version","alias_value":"2502.16852v1","created_at":"2026-07-05T10:19:00Z"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.2502.16852","created_at":"2026-07-05T10:19:00Z"},{"alias_kind":"pith_short_12","alias_value":"JECZIIMDDOJY","created_at":"2026-07-05T10:19:00Z"},{"alias_kind":"pith_short_16","alias_value":"JECZIIMDDOJYK437","created_at":"2026-07-05T10:19:00Z"},{"alias_kind":"pith_short_8","alias_value":"JECZIIMD","created_at":"2026-07-05T10:19:00Z"}],"graph_snapshots":[{"event_id":"sha256:c36a229a11c86cd940c4e76187cb27db73bb805faf0c9d7f769bc729dc7e042d","target":"graph","created_at":"2026-07-05T10:19:00Z","signer":{"key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signer_id":"pith.science","signer_type":"pith_registry"},"payload":{"graph_snapshot":{"author_claims":{"count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","strong_count":0},"builder_version":"pith-number-builder-2026-05-17-v1","claims":{"count":0,"items":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"integrity":{"available":true,"clean":true,"detectors_run":[],"endpoint":"/pith/2502.16852/integrity.json","findings":[],"snapshot_sha256":"c28c3603d3b5d939e8dc4c7e95fa8dfce3d595e45f758748cecf8e644a296938","summary":{"advisory":0,"by_detector":{},"critical":0,"informational":0}},"paper":{"abstract_excerpt":"Reinforcement learning from human feedback (RLHF) has demonstrated remarkable effectiveness in aligning large language models (LLMs) with human preferences. Many existing alignment approaches rely on the Bradley-Terry (BT) model assumption, which assumes the existence of a ground-truth reward for each prompt-response pair. However, this assumption can be overly restrictive when modeling complex human preferences. In this paper, we drop the BT model assumption and study LLM alignment under general preferences, formulated as a two-player game. Drawing on theoretical insights from learning in gam","authors_text":"Dian Yu, Dong Yu, Haitao Mi, Linfeng Song, Nan Jiang, Tao Ge, Yuheng Zhang, Zhichen Zeng","cross_cats":["cs.AI","cs.CL"],"headline":"","license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","primary_cat":"cs.LG","submitted_at":"2025-02-24T05:24:52Z","title":"Improving LLM General Preference Alignment via Optimistic Online Mirror Descent"},"references":{"count":0,"internal_anchors":0,"resolved_work":0,"sample":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"2502.16852","kind":"arxiv","version":1},"verdict":{"created_at":null,"id":null,"model_set":{},"one_line_summary":"","pipeline_version":null,"pith_extraction_headline":"","strongest_claim":"","weakest_assumption":""}},"verdict_id":null}}],"author_attestations":[],"timestamp_anchors":[],"storage_attestations":[],"citation_signatures":[],"replication_records":[],"corrections":[],"mirror_hints":[],"record_created":{"event_id":"sha256:470d98009b17f1a4d4589210ff30e2a25f1ef4e2188a08beb08f3b3fc1fc7154","target":"record","created_at":"2026-07-05T10:19:00Z","signer":{"key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signer_id":"pith.science","signer_type":"pith_registry"},"payload":{"attestation_state":"computed","canonical_record":{"metadata":{"abstract_canon_sha256":"469d80476c992b3a30b6e32a23b7cbb5886278d00db28bc31f00201debe01e52","cross_cats_sorted":["cs.AI","cs.CL"],"license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","primary_cat":"cs.LG","submitted_at":"2025-02-24T05:24:52Z","title_canon_sha256":"f3e112b569265803411ada93a1b8b2c8c7d8e7a39336ffd2b7689cc5887fe7c0"},"schema_version":"1.0","source":{"id":"2502.16852","kind":"arxiv","version":1}},"canonical_sha256":"49059421831b9385737fa99f5205a972fefc95e9f9afe4f324f55df33ddaac18","receipt":{"algorithm":"ed25519","builder_version":"pith-number-builder-2026-05-17-v1","canonical_sha256":"49059421831b9385737fa99f5205a972fefc95e9f9afe4f324f55df33ddaac18","first_computed_at":"2026-07-05T10:19:00.886825Z","key_id":"pith-v1-2026-05","kind":"pith_receipt","last_reissued_at":"2026-07-05T10:19:00.886825Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","receipt_version":"0.3","signature_b64":"e6T4MbXHbVslVyKpWayoAmfcR2bOBSMwOm1eqt8KWkMiZV0PnkZgvqI5j6Gx2Wtb2pP9wmx7ZS8UB5Z20NCGDQ==","signature_status":"signed_v1","signed_at":"2026-07-05T10:19:00.887287Z","signed_message":"canonical_sha256_bytes"},"source_id":"2502.16852","source_kind":"arxiv","source_version":1}}},"equivocations":[],"invalid_events":[],"applied_event_ids":["sha256:470d98009b17f1a4d4589210ff30e2a25f1ef4e2188a08beb08f3b3fc1fc7154","sha256:c36a229a11c86cd940c4e76187cb27db73bb805faf0c9d7f769bc729dc7e042d"],"state_sha256":"20d12454bb652ea912b1d426971d496f0e0f8f6b4e9f9912cc4706184e096d98"},"bundle_signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"VBbA3i8j+dDSNs4hPN67QT39VKjPQT5szAjhkQ2yY+u7auMPFke1zHUxnQPJcFcN0gcBeRVGBZ24KZzknwnKAg==","signed_message":"bundle_sha256_bytes","signed_at":"2026-08-04T23:20:00.385041Z","bundle_sha256":"c7364dba8e3d9bb4970f260d9a4618c5887ae8b1bbb887ffae646d7fbd691877"}}