{"bundle_type":"pith_open_graph_bundle","bundle_version":"1.0","pith_number":"pith:2024:DI7CNBK3UJ2LX7IP4KQYAVKIRE","short_pith_number":"pith:DI7CNBK3","canonical_record":{"source":{"id":"2410.04350","kind":"arxiv","version":3},"metadata":{"license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.CL","submitted_at":"2024-10-06T04:03:00Z","cross_cats_sorted":[],"title_canon_sha256":"c1f671c0df6d2537ec3eaf135a64453e9813463889ff0bb2bf641b942e5368a5","abstract_canon_sha256":"fd15fe158d7ec79c88aa618177be172a8543d14151cc30a23920c56795221496"},"schema_version":"1.0"},"canonical_sha256":"1a3e26855ba274bbfd0fe2a1805548893e168b71b7ee3891e5a8e5b2e304ccbb","source":{"kind":"arxiv","id":"2410.04350","version":3},"source_aliases":[{"alias_kind":"arxiv","alias_value":"2410.04350","created_at":"2026-07-05T10:49:09Z"},{"alias_kind":"arxiv_version","alias_value":"2410.04350v3","created_at":"2026-07-05T10:49:09Z"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.2410.04350","created_at":"2026-07-05T10:49:09Z"},{"alias_kind":"pith_short_12","alias_value":"DI7CNBK3UJ2L","created_at":"2026-07-05T10:49:09Z"},{"alias_kind":"pith_short_16","alias_value":"DI7CNBK3UJ2LX7IP","created_at":"2026-07-05T10:49:09Z"},{"alias_kind":"pith_short_8","alias_value":"DI7CNBK3","created_at":"2026-07-05T10:49:09Z"}],"events":[{"event_type":"record_created","subject_pith_number":"pith:2024:DI7CNBK3UJ2LX7IP4KQYAVKIRE","target":"record","payload":{"canonical_record":{"source":{"id":"2410.04350","kind":"arxiv","version":3},"metadata":{"license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.CL","submitted_at":"2024-10-06T04:03:00Z","cross_cats_sorted":[],"title_canon_sha256":"c1f671c0df6d2537ec3eaf135a64453e9813463889ff0bb2bf641b942e5368a5","abstract_canon_sha256":"fd15fe158d7ec79c88aa618177be172a8543d14151cc30a23920c56795221496"},"schema_version":"1.0"},"canonical_sha256":"1a3e26855ba274bbfd0fe2a1805548893e168b71b7ee3891e5a8e5b2e304ccbb","receipt":{"kind":"pith_receipt","key_id":"pith-v1-2026-05","algorithm":"ed25519","signed_at":"2026-07-05T10:49:09.665433Z","signature_b64":"tTFrjcfkPW0KjCZYt10eDWXED2I1GPMe3sPoohFCk6xrEyu1g0296Afx9ORN8IGvp0cDq2W2FQCAgfUC5J4ZDw==","signed_message":"canonical_sha256_bytes","builder_version":"pith-number-builder-2026-05-17-v1","receipt_version":"0.3","canonical_sha256":"1a3e26855ba274bbfd0fe2a1805548893e168b71b7ee3891e5a8e5b2e304ccbb","last_reissued_at":"2026-07-05T10:49:09.664911Z","signature_status":"signed_v1","first_computed_at":"2026-07-05T10:49:09.664911Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"source_kind":"arxiv","source_id":"2410.04350","source_version":3,"attestation_state":"computed"},"signer":{"signer_id":"pith.science","signer_type":"pith_registry","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"created_at":"2026-07-05T10:49:09Z","supersedes":[],"prev_event":null,"signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"xptT3ZqoA0FbGNzEXUqnv3onO9iG8q8E0VSGKNkkZ2Q5nnJUTCFiPTEfxqRMvdW4DE2F0wheuFT7OhRKdymyAw==","signed_message":"open_graph_event_sha256_bytes","signed_at":"2026-07-06T15:06:05.469078Z"},"content_sha256":"5b0bd5488c673876f3483789e9ca54502137ac7fe21364b0865be9adadd2151a","schema_version":"1.0","event_id":"sha256:5b0bd5488c673876f3483789e9ca54502137ac7fe21364b0865be9adadd2151a"},{"event_type":"graph_snapshot","subject_pith_number":"pith:2024:DI7CNBK3UJ2LX7IP4KQYAVKIRE","target":"graph","payload":{"graph_snapshot":{"paper":{"title":"TIS-DPO: Token-level Importance Sampling for Direct Preference Optimization With Estimated Weights","license":"http://creativecommons.org/licenses/by/4.0/","headline":"","cross_cats":[],"primary_cat":"cs.CL","authors_text":"Aiwei Liu, Albin Madappally Jose, Haoping Bai, Jiulong Shan, Lijie Wen, Meng Cao, Philip S. Yu, Simon Wang, Xiang Kong, Xiaojiang Liu, Yanchao Sun, Zhiyun Lu","submitted_at":"2024-10-06T04:03:00Z","abstract_excerpt":"Direct Preference Optimization (DPO) has been widely adopted for preference alignment of Large Language Models (LLMs) due to its simplicity and effectiveness. However, DPO is derived as a bandit problem in which the whole response is treated as a single arm, ignoring the importance differences between tokens, which may affect optimization efficiency and make it difficult to achieve optimal results. In this work, we propose that the optimal data for DPO has equal expected rewards for each token in winning and losing responses, as there is no difference in token importance. However, since the op"},"claims":{"count":0,"items":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"2410.04350","kind":"arxiv","version":3},"verdict":{"id":null,"model_set":{},"created_at":null,"strongest_claim":"","one_line_summary":"","pipeline_version":null,"weakest_assumption":"","pith_extraction_headline":""},"integrity":{"clean":true,"summary":{"advisory":0,"critical":0,"by_detector":{},"informational":0},"endpoint":"/pith/2410.04350/integrity.json","findings":[],"available":true,"detectors_run":[],"snapshot_sha256":"c28c3603d3b5d939e8dc4c7e95fa8dfce3d595e45f758748cecf8e644a296938"},"references":{"count":0,"sample":[],"resolved_work":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","internal_anchors":0},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"author_claims":{"count":0,"strong_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"builder_version":"pith-number-builder-2026-05-17-v1"},"verdict_id":null},"signer":{"signer_id":"pith.science","signer_type":"pith_registry","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"created_at":"2026-07-05T10:49:09Z","supersedes":[],"prev_event":null,"signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"eyigjLqG8wRa9DsrYGrTDBoE1PEyBjuCNCHhp0IwxaWXZJpNASTx8d8iR4xl/bPsp1IXILBPWtBMXhLQAclDAA==","signed_message":"open_graph_event_sha256_bytes","signed_at":"2026-07-06T15:06:05.469471Z"},"content_sha256":"aa34f68855b96d83780a5117b6280ae020021714124486f13163c2c3be04b703","schema_version":"1.0","event_id":"sha256:aa34f68855b96d83780a5117b6280ae020021714124486f13163c2c3be04b703"}],"timestamp_proofs":[],"mirror_hints":[{"mirror_type":"https","name":"Pith Resolver","base_url":"https://pith.science","bundle_url":"https://pith.science/pith/DI7CNBK3UJ2LX7IP4KQYAVKIRE/bundle.json","state_url":"https://pith.science/pith/DI7CNBK3UJ2LX7IP4KQYAVKIRE/state.json","well_known_bundle_url":"https://pith.science/.well-known/pith/DI7CNBK3UJ2LX7IP4KQYAVKIRE/bundle.json","status":"primary"}],"public_keys":[{"key_id":"pith-v1-2026-05","algorithm":"ed25519","format":"raw","public_key_b64":"stVStoiQhXFxp4s2pdzPNoqVNBMojDU/fJ2db5S3CbM=","public_key_hex":"b2d552b68890857171a78b36a5dccf368a953413288c353f7c9d9d6f94b709b3","fingerprint_sha256_b32_first128bits":"RVFV5Z2OI2J3ZUO7ERDEBCYNKS","fingerprint_sha256_hex":"8d4b5ee74e4693bcd1df2446408b0d54","rotates_at":null,"url":"https://pith.science/pith-signing-key.json","notes":"Pith uses this Ed25519 key to sign canonical record SHA-256 digests. Verify with: ed25519_verify(public_key, message=canonical_sha256_bytes, signature=base64decode(signature_b64))."}],"merge_version":"pith-open-graph-merge-v1","built_at":"2026-07-06T15:06:05Z","links":{"resolver":"https://pith.science/pith/DI7CNBK3UJ2LX7IP4KQYAVKIRE","bundle":"https://pith.science/pith/DI7CNBK3UJ2LX7IP4KQYAVKIRE/bundle.json","state":"https://pith.science/pith/DI7CNBK3UJ2LX7IP4KQYAVKIRE/state.json","well_known_bundle":"https://pith.science/.well-known/pith/DI7CNBK3UJ2LX7IP4KQYAVKIRE/bundle.json"},"state":{"state_type":"pith_open_graph_state","state_version":"1.0","pith_number":"pith:2024:DI7CNBK3UJ2LX7IP4KQYAVKIRE","merge_version":"pith-open-graph-merge-v1","event_count":2,"valid_event_count":2,"invalid_event_count":0,"equivocation_count":0,"current":{"canonical_record":{"metadata":{"abstract_canon_sha256":"fd15fe158d7ec79c88aa618177be172a8543d14151cc30a23920c56795221496","cross_cats_sorted":[],"license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.CL","submitted_at":"2024-10-06T04:03:00Z","title_canon_sha256":"c1f671c0df6d2537ec3eaf135a64453e9813463889ff0bb2bf641b942e5368a5"},"schema_version":"1.0","source":{"id":"2410.04350","kind":"arxiv","version":3}},"source_aliases":[{"alias_kind":"arxiv","alias_value":"2410.04350","created_at":"2026-07-05T10:49:09Z"},{"alias_kind":"arxiv_version","alias_value":"2410.04350v3","created_at":"2026-07-05T10:49:09Z"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.2410.04350","created_at":"2026-07-05T10:49:09Z"},{"alias_kind":"pith_short_12","alias_value":"DI7CNBK3UJ2L","created_at":"2026-07-05T10:49:09Z"},{"alias_kind":"pith_short_16","alias_value":"DI7CNBK3UJ2LX7IP","created_at":"2026-07-05T10:49:09Z"},{"alias_kind":"pith_short_8","alias_value":"DI7CNBK3","created_at":"2026-07-05T10:49:09Z"}],"graph_snapshots":[{"event_id":"sha256:aa34f68855b96d83780a5117b6280ae020021714124486f13163c2c3be04b703","target":"graph","created_at":"2026-07-05T10:49:09Z","signer":{"key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signer_id":"pith.science","signer_type":"pith_registry"},"payload":{"graph_snapshot":{"author_claims":{"count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","strong_count":0},"builder_version":"pith-number-builder-2026-05-17-v1","claims":{"count":0,"items":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"integrity":{"available":true,"clean":true,"detectors_run":[],"endpoint":"/pith/2410.04350/integrity.json","findings":[],"snapshot_sha256":"c28c3603d3b5d939e8dc4c7e95fa8dfce3d595e45f758748cecf8e644a296938","summary":{"advisory":0,"by_detector":{},"critical":0,"informational":0}},"paper":{"abstract_excerpt":"Direct Preference Optimization (DPO) has been widely adopted for preference alignment of Large Language Models (LLMs) due to its simplicity and effectiveness. However, DPO is derived as a bandit problem in which the whole response is treated as a single arm, ignoring the importance differences between tokens, which may affect optimization efficiency and make it difficult to achieve optimal results. In this work, we propose that the optimal data for DPO has equal expected rewards for each token in winning and losing responses, as there is no difference in token importance. However, since the op","authors_text":"Aiwei Liu, Albin Madappally Jose, Haoping Bai, Jiulong Shan, Lijie Wen, Meng Cao, Philip S. Yu, Simon Wang, Xiang Kong, Xiaojiang Liu, Yanchao Sun, Zhiyun Lu","cross_cats":[],"headline":"","license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.CL","submitted_at":"2024-10-06T04:03:00Z","title":"TIS-DPO: Token-level Importance Sampling for Direct Preference Optimization With Estimated Weights"},"references":{"count":0,"internal_anchors":0,"resolved_work":0,"sample":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"2410.04350","kind":"arxiv","version":3},"verdict":{"created_at":null,"id":null,"model_set":{},"one_line_summary":"","pipeline_version":null,"pith_extraction_headline":"","strongest_claim":"","weakest_assumption":""}},"verdict_id":null}}],"author_attestations":[],"timestamp_anchors":[],"storage_attestations":[],"citation_signatures":[],"replication_records":[],"corrections":[],"mirror_hints":[],"record_created":{"event_id":"sha256:5b0bd5488c673876f3483789e9ca54502137ac7fe21364b0865be9adadd2151a","target":"record","created_at":"2026-07-05T10:49:09Z","signer":{"key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signer_id":"pith.science","signer_type":"pith_registry"},"payload":{"attestation_state":"computed","canonical_record":{"metadata":{"abstract_canon_sha256":"fd15fe158d7ec79c88aa618177be172a8543d14151cc30a23920c56795221496","cross_cats_sorted":[],"license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.CL","submitted_at":"2024-10-06T04:03:00Z","title_canon_sha256":"c1f671c0df6d2537ec3eaf135a64453e9813463889ff0bb2bf641b942e5368a5"},"schema_version":"1.0","source":{"id":"2410.04350","kind":"arxiv","version":3}},"canonical_sha256":"1a3e26855ba274bbfd0fe2a1805548893e168b71b7ee3891e5a8e5b2e304ccbb","receipt":{"algorithm":"ed25519","builder_version":"pith-number-builder-2026-05-17-v1","canonical_sha256":"1a3e26855ba274bbfd0fe2a1805548893e168b71b7ee3891e5a8e5b2e304ccbb","first_computed_at":"2026-07-05T10:49:09.664911Z","key_id":"pith-v1-2026-05","kind":"pith_receipt","last_reissued_at":"2026-07-05T10:49:09.664911Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","receipt_version":"0.3","signature_b64":"tTFrjcfkPW0KjCZYt10eDWXED2I1GPMe3sPoohFCk6xrEyu1g0296Afx9ORN8IGvp0cDq2W2FQCAgfUC5J4ZDw==","signature_status":"signed_v1","signed_at":"2026-07-05T10:49:09.665433Z","signed_message":"canonical_sha256_bytes"},"source_id":"2410.04350","source_kind":"arxiv","source_version":3}}},"equivocations":[],"invalid_events":[],"applied_event_ids":["sha256:5b0bd5488c673876f3483789e9ca54502137ac7fe21364b0865be9adadd2151a","sha256:aa34f68855b96d83780a5117b6280ae020021714124486f13163c2c3be04b703"],"state_sha256":"6e9b652938694b51ec80097adcd3865960c377dc0831553156787b44397e18b4"},"bundle_signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"2zuw8SK7BCop7RKf2RuR3xn4wgL0sFekkoXAGokfboDApl0yCjYDAJOEU0KXwErnWQ4AUMzDF7PA7OmC2cfeDg==","signed_message":"bundle_sha256_bytes","signed_at":"2026-07-06T15:06:05.471571Z","bundle_sha256":"b05dc75e272477f921cf5a48c376cf7a6c402a9f4ab26952df0a56f1ea94abac"}}