{"bundle_type":"pith_open_graph_bundle","bundle_version":"1.0","pith_number":"pith:2025:H2WOAKZ76FOVDZO323N64WTDR4","short_pith_number":"pith:H2WOAKZ7","canonical_record":{"source":{"id":"2502.04567","kind":"arxiv","version":1},"metadata":{"license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.AI","submitted_at":"2025-02-06T23:45:08Z","cross_cats_sorted":[],"title_canon_sha256":"f3db60d7a80781cf861043efb4ff1c9a8372bdac24379065be8bd5975b335325","abstract_canon_sha256":"fc1b28b04bb92f17469695071d1a58843f301c6fcbecd5d6e90ec1d200ead159"},"schema_version":"1.0"},"canonical_sha256":"3eace02b3ff15d51e5dbd6dbee5a638f1b2687ead19720bb897bac5c361ae83d","source":{"kind":"arxiv","id":"2502.04567","version":1},"source_aliases":[{"alias_kind":"arxiv","alias_value":"2502.04567","created_at":"2026-07-05T10:10:53Z"},{"alias_kind":"arxiv_version","alias_value":"2502.04567v1","created_at":"2026-07-05T10:10:53Z"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.2502.04567","created_at":"2026-07-05T10:10:53Z"},{"alias_kind":"pith_short_12","alias_value":"H2WOAKZ76FOV","created_at":"2026-07-05T10:10:53Z"},{"alias_kind":"pith_short_16","alias_value":"H2WOAKZ76FOVDZO3","created_at":"2026-07-05T10:10:53Z"},{"alias_kind":"pith_short_8","alias_value":"H2WOAKZ7","created_at":"2026-07-05T10:10:53Z"}],"events":[{"event_type":"record_created","subject_pith_number":"pith:2025:H2WOAKZ76FOVDZO323N64WTDR4","target":"record","payload":{"canonical_record":{"source":{"id":"2502.04567","kind":"arxiv","version":1},"metadata":{"license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.AI","submitted_at":"2025-02-06T23:45:08Z","cross_cats_sorted":[],"title_canon_sha256":"f3db60d7a80781cf861043efb4ff1c9a8372bdac24379065be8bd5975b335325","abstract_canon_sha256":"fc1b28b04bb92f17469695071d1a58843f301c6fcbecd5d6e90ec1d200ead159"},"schema_version":"1.0"},"canonical_sha256":"3eace02b3ff15d51e5dbd6dbee5a638f1b2687ead19720bb897bac5c361ae83d","receipt":{"kind":"pith_receipt","key_id":"pith-v1-2026-05","algorithm":"ed25519","signed_at":"2026-07-05T10:10:53.560202Z","signature_b64":"Fz+iucaV3QvqRjZtCSX8Gu0FGRkKw4lG6a1l3ZrUZLMpnHTJGItG1Ivhua+n+J/9aWf4qHJnhxJKjYpPXl5LCQ==","signed_message":"canonical_sha256_bytes","builder_version":"pith-number-builder-2026-05-17-v1","receipt_version":"0.3","canonical_sha256":"3eace02b3ff15d51e5dbd6dbee5a638f1b2687ead19720bb897bac5c361ae83d","last_reissued_at":"2026-07-05T10:10:53.559671Z","signature_status":"signed_v1","first_computed_at":"2026-07-05T10:10:53.559671Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"source_kind":"arxiv","source_id":"2502.04567","source_version":1,"attestation_state":"computed"},"signer":{"signer_id":"pith.science","signer_type":"pith_registry","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"created_at":"2026-07-05T10:10:53Z","supersedes":[],"prev_event":null,"signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"m9CVAdz4WSCnpgzgSU1UEG/JXMKhDN+VC+BaGFP8cVBioTfxgbRpL3aowjMH5Y4dWHybsbj/i066b0mcj8HFDA==","signed_message":"open_graph_event_sha256_bytes","signed_at":"2026-08-07T14:11:34.894941Z"},"content_sha256":"237a1596f2351f76e2a2933d0f1da4ce21622057949b839aa0cfd10bc2d29fd1","schema_version":"1.0","event_id":"sha256:237a1596f2351f76e2a2933d0f1da4ce21622057949b839aa0cfd10bc2d29fd1"},{"event_type":"graph_snapshot","subject_pith_number":"pith:2025:H2WOAKZ76FOVDZO323N64WTDR4","target":"graph","payload":{"graph_snapshot":{"paper":{"title":"Preference Optimization via Contrastive Divergence: Your Reward Model is Secretly an NLL Estimator","license":"http://creativecommons.org/licenses/by/4.0/","headline":"","cross_cats":[],"primary_cat":"cs.AI","authors_text":"Fang Liu, Mohammad Ghavamzadeh, Xuan Zhu, Yanjun Qi, Zhuotong Chen","submitted_at":"2025-02-06T23:45:08Z","abstract_excerpt":"Existing studies on preference optimization (PO) have centered on constructing pairwise preference data following simple heuristics, such as maximizing the margin between preferred and dispreferred completions based on human (or AI) ranked scores. However, none of these heuristics has a full theoretical justification. In this work, we develop a novel PO framework that provides theoretical guidance to effectively sample dispreferred completions. To achieve this, we formulate PO as minimizing the negative log-likelihood (NLL) of a probability model and propose to estimate its normalization const"},"claims":{"count":0,"items":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"2502.04567","kind":"arxiv","version":1},"verdict":{"id":null,"model_set":{},"created_at":null,"strongest_claim":"","one_line_summary":"","pipeline_version":null,"weakest_assumption":"","pith_extraction_headline":""},"integrity":{"clean":true,"summary":{"advisory":0,"critical":0,"by_detector":{},"informational":0},"endpoint":"/pith/2502.04567/integrity.json","findings":[],"available":true,"detectors_run":[],"snapshot_sha256":"c28c3603d3b5d939e8dc4c7e95fa8dfce3d595e45f758748cecf8e644a296938"},"references":{"count":0,"sample":[],"resolved_work":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","internal_anchors":0},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"author_claims":{"count":0,"strong_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"builder_version":"pith-number-builder-2026-05-17-v1"},"verdict_id":null},"signer":{"signer_id":"pith.science","signer_type":"pith_registry","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"created_at":"2026-07-05T10:10:53Z","supersedes":[],"prev_event":null,"signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"8Naqjbw/rlH/oON4M+8rnYvkZOgDy9RhUI+pASd5EJg4YJBOHfBvUjy/ASa2azrZvhZ3/vwuLxvFbSPj+RLdBw==","signed_message":"open_graph_event_sha256_bytes","signed_at":"2026-08-07T14:11:34.895692Z"},"content_sha256":"fd4bcf0eb22eb04bb230b2a17160f86b2b74fc76b0d79753981eeb3ff01ae124","schema_version":"1.0","event_id":"sha256:fd4bcf0eb22eb04bb230b2a17160f86b2b74fc76b0d79753981eeb3ff01ae124"}],"timestamp_proofs":[],"mirror_hints":[{"mirror_type":"https","name":"Pith Resolver","base_url":"https://pith.science","bundle_url":"https://pith.science/pith/H2WOAKZ76FOVDZO323N64WTDR4/bundle.json","state_url":"https://pith.science/pith/H2WOAKZ76FOVDZO323N64WTDR4/state.json","well_known_bundle_url":"https://pith.science/.well-known/pith/H2WOAKZ76FOVDZO323N64WTDR4/bundle.json","status":"primary"}],"public_keys":[{"key_id":"pith-v1-2026-05","algorithm":"ed25519","format":"raw","public_key_b64":"stVStoiQhXFxp4s2pdzPNoqVNBMojDU/fJ2db5S3CbM=","public_key_hex":"b2d552b68890857171a78b36a5dccf368a953413288c353f7c9d9d6f94b709b3","fingerprint_sha256_b32_first128bits":"RVFV5Z2OI2J3ZUO7ERDEBCYNKS","fingerprint_sha256_hex":"8d4b5ee74e4693bcd1df2446408b0d54","rotates_at":null,"url":"https://pith.science/pith-signing-key.json","notes":"Pith uses this Ed25519 key to sign canonical record SHA-256 digests. Verify with: ed25519_verify(public_key, message=canonical_sha256_bytes, signature=base64decode(signature_b64))."}],"merge_version":"pith-open-graph-merge-v1","built_at":"2026-08-07T14:11:34Z","links":{"resolver":"https://pith.science/pith/H2WOAKZ76FOVDZO323N64WTDR4","bundle":"https://pith.science/pith/H2WOAKZ76FOVDZO323N64WTDR4/bundle.json","state":"https://pith.science/pith/H2WOAKZ76FOVDZO323N64WTDR4/state.json","well_known_bundle":"https://pith.science/.well-known/pith/H2WOAKZ76FOVDZO323N64WTDR4/bundle.json"},"state":{"state_type":"pith_open_graph_state","state_version":"1.0","pith_number":"pith:2025:H2WOAKZ76FOVDZO323N64WTDR4","merge_version":"pith-open-graph-merge-v1","event_count":2,"valid_event_count":2,"invalid_event_count":0,"equivocation_count":0,"current":{"canonical_record":{"metadata":{"abstract_canon_sha256":"fc1b28b04bb92f17469695071d1a58843f301c6fcbecd5d6e90ec1d200ead159","cross_cats_sorted":[],"license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.AI","submitted_at":"2025-02-06T23:45:08Z","title_canon_sha256":"f3db60d7a80781cf861043efb4ff1c9a8372bdac24379065be8bd5975b335325"},"schema_version":"1.0","source":{"id":"2502.04567","kind":"arxiv","version":1}},"source_aliases":[{"alias_kind":"arxiv","alias_value":"2502.04567","created_at":"2026-07-05T10:10:53Z"},{"alias_kind":"arxiv_version","alias_value":"2502.04567v1","created_at":"2026-07-05T10:10:53Z"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.2502.04567","created_at":"2026-07-05T10:10:53Z"},{"alias_kind":"pith_short_12","alias_value":"H2WOAKZ76FOV","created_at":"2026-07-05T10:10:53Z"},{"alias_kind":"pith_short_16","alias_value":"H2WOAKZ76FOVDZO3","created_at":"2026-07-05T10:10:53Z"},{"alias_kind":"pith_short_8","alias_value":"H2WOAKZ7","created_at":"2026-07-05T10:10:53Z"}],"graph_snapshots":[{"event_id":"sha256:fd4bcf0eb22eb04bb230b2a17160f86b2b74fc76b0d79753981eeb3ff01ae124","target":"graph","created_at":"2026-07-05T10:10:53Z","signer":{"key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signer_id":"pith.science","signer_type":"pith_registry"},"payload":{"graph_snapshot":{"author_claims":{"count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","strong_count":0},"builder_version":"pith-number-builder-2026-05-17-v1","claims":{"count":0,"items":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"integrity":{"available":true,"clean":true,"detectors_run":[],"endpoint":"/pith/2502.04567/integrity.json","findings":[],"snapshot_sha256":"c28c3603d3b5d939e8dc4c7e95fa8dfce3d595e45f758748cecf8e644a296938","summary":{"advisory":0,"by_detector":{},"critical":0,"informational":0}},"paper":{"abstract_excerpt":"Existing studies on preference optimization (PO) have centered on constructing pairwise preference data following simple heuristics, such as maximizing the margin between preferred and dispreferred completions based on human (or AI) ranked scores. However, none of these heuristics has a full theoretical justification. In this work, we develop a novel PO framework that provides theoretical guidance to effectively sample dispreferred completions. To achieve this, we formulate PO as minimizing the negative log-likelihood (NLL) of a probability model and propose to estimate its normalization const","authors_text":"Fang Liu, Mohammad Ghavamzadeh, Xuan Zhu, Yanjun Qi, Zhuotong Chen","cross_cats":[],"headline":"","license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.AI","submitted_at":"2025-02-06T23:45:08Z","title":"Preference Optimization via Contrastive Divergence: Your Reward Model is Secretly an NLL Estimator"},"references":{"count":0,"internal_anchors":0,"resolved_work":0,"sample":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"2502.04567","kind":"arxiv","version":1},"verdict":{"created_at":null,"id":null,"model_set":{},"one_line_summary":"","pipeline_version":null,"pith_extraction_headline":"","strongest_claim":"","weakest_assumption":""}},"verdict_id":null}}],"author_attestations":[],"timestamp_anchors":[],"storage_attestations":[],"citation_signatures":[],"replication_records":[],"corrections":[],"mirror_hints":[],"record_created":{"event_id":"sha256:237a1596f2351f76e2a2933d0f1da4ce21622057949b839aa0cfd10bc2d29fd1","target":"record","created_at":"2026-07-05T10:10:53Z","signer":{"key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signer_id":"pith.science","signer_type":"pith_registry"},"payload":{"attestation_state":"computed","canonical_record":{"metadata":{"abstract_canon_sha256":"fc1b28b04bb92f17469695071d1a58843f301c6fcbecd5d6e90ec1d200ead159","cross_cats_sorted":[],"license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.AI","submitted_at":"2025-02-06T23:45:08Z","title_canon_sha256":"f3db60d7a80781cf861043efb4ff1c9a8372bdac24379065be8bd5975b335325"},"schema_version":"1.0","source":{"id":"2502.04567","kind":"arxiv","version":1}},"canonical_sha256":"3eace02b3ff15d51e5dbd6dbee5a638f1b2687ead19720bb897bac5c361ae83d","receipt":{"algorithm":"ed25519","builder_version":"pith-number-builder-2026-05-17-v1","canonical_sha256":"3eace02b3ff15d51e5dbd6dbee5a638f1b2687ead19720bb897bac5c361ae83d","first_computed_at":"2026-07-05T10:10:53.559671Z","key_id":"pith-v1-2026-05","kind":"pith_receipt","last_reissued_at":"2026-07-05T10:10:53.559671Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","receipt_version":"0.3","signature_b64":"Fz+iucaV3QvqRjZtCSX8Gu0FGRkKw4lG6a1l3ZrUZLMpnHTJGItG1Ivhua+n+J/9aWf4qHJnhxJKjYpPXl5LCQ==","signature_status":"signed_v1","signed_at":"2026-07-05T10:10:53.560202Z","signed_message":"canonical_sha256_bytes"},"source_id":"2502.04567","source_kind":"arxiv","source_version":1}}},"equivocations":[],"invalid_events":[],"applied_event_ids":["sha256:237a1596f2351f76e2a2933d0f1da4ce21622057949b839aa0cfd10bc2d29fd1","sha256:fd4bcf0eb22eb04bb230b2a17160f86b2b74fc76b0d79753981eeb3ff01ae124"],"state_sha256":"b4ce5873394304b92260efca09c1f8c08417c51dee9f5b1956def7d5af1cea85"},"bundle_signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"vAx5AQViD0WkDHu8PUVf9UccqxSPxdAwZ+wVuHztJCsXezryK+R8/bXCusdJuh98p9iBkfgvKilobdk+xZv6Dw==","signed_message":"bundle_sha256_bytes","signed_at":"2026-08-07T14:11:34.900683Z","bundle_sha256":"48e1ecb73497357d2885dcb98a7d42c1db21fec995c4d61e5a7f8456bcfc2b5f"}}