{"state_type":"pith_open_graph_state","state_version":"1.0","pith_number":"pith:2024:PNZRDWCIKUKCKVRIIBGOYDFKBX","merge_version":"pith-open-graph-merge-v1","event_count":2,"valid_event_count":2,"invalid_event_count":0,"equivocation_count":0,"current":{"canonical_record":{"metadata":{"abstract_canon_sha256":"df21458cbcc354bf6316235aa71bb0c5fc55ca12b6c002d93b75758393f08e65","cross_cats_sorted":["cs.AI"],"license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.LG","submitted_at":"2024-04-30T19:48:55Z","title_canon_sha256":"4888b73eef519d97e9addcb1d42abd67a44f3698d14867b3d53b25add6dc2eb4"},"schema_version":"1.0","source":{"id":"2405.00747","kind":"arxiv","version":4}},"source_aliases":[{"alias_kind":"arxiv","alias_value":"2405.00747","created_at":"2026-07-05T09:15:31Z"},{"alias_kind":"arxiv_version","alias_value":"2405.00747v4","created_at":"2026-07-05T09:15:31Z"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.2405.00747","created_at":"2026-07-05T09:15:31Z"},{"alias_kind":"pith_short_12","alias_value":"PNZRDWCIKUKC","created_at":"2026-07-05T09:15:31Z"},{"alias_kind":"pith_short_16","alias_value":"PNZRDWCIKUKCKVRI","created_at":"2026-07-05T09:15:31Z"},{"alias_kind":"pith_short_8","alias_value":"PNZRDWCI","created_at":"2026-07-05T09:15:31Z"}],"graph_snapshots":[{"event_id":"sha256:6b03e7e4c9f9d3936a41184bcbc0d70605f4ab2fb30de8424172e45ca5632cca","target":"graph","created_at":"2026-07-05T09:15:31Z","signer":{"key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signer_id":"pith.science","signer_type":"pith_registry"},"payload":{"graph_snapshot":{"author_claims":{"count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","strong_count":0},"builder_version":"pith-number-builder-2026-05-17-v1","claims":{"count":0,"items":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"integrity":{"available":true,"clean":true,"detectors_run":[],"endpoint":"/pith/2405.00747/integrity.json","findings":[],"snapshot_sha256":"c28c3603d3b5d939e8dc4c7e95fa8dfce3d595e45f758748cecf8e644a296938","summary":{"advisory":0,"by_detector":{},"critical":0,"informational":0}},"paper":{"abstract_excerpt":"We propose Soft Preference Optimization (SPO), a method for aligning generative models, such as Large Language Models (LLMs), with human preferences, without the need for a reward model. SPO optimizes model outputs directly over a preference dataset through a natural loss function that integrates preference loss with a regularization term across the model's entire output distribution rather than limiting it to the preference dataset. Although SPO does not require the assumption of an existing underlying reward model, we demonstrate that, under the Bradley-Terry (BT) model assumption, it conver","authors_text":"Arsalan Sharifnassab, Dale Schuurmans, Saber Salehkaleybar, Sina Ghiassian, Surya Kanoria","cross_cats":["cs.AI"],"headline":"","license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.LG","submitted_at":"2024-04-30T19:48:55Z","title":"Soft Preference Optimization: Aligning Language Models to Expert Distributions"},"references":{"count":0,"internal_anchors":0,"resolved_work":0,"sample":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"2405.00747","kind":"arxiv","version":4},"verdict":{"created_at":null,"id":null,"model_set":{},"one_line_summary":"","pipeline_version":null,"pith_extraction_headline":"","strongest_claim":"","weakest_assumption":""}},"verdict_id":null}}],"author_attestations":[],"timestamp_anchors":[],"storage_attestations":[],"citation_signatures":[],"replication_records":[],"corrections":[],"mirror_hints":[],"record_created":{"event_id":"sha256:30467294c5776b934c12fc481b8415105034df7089848ee72047ab262549315b","target":"record","created_at":"2026-07-05T09:15:31Z","signer":{"key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signer_id":"pith.science","signer_type":"pith_registry"},"payload":{"attestation_state":"computed","canonical_record":{"metadata":{"abstract_canon_sha256":"df21458cbcc354bf6316235aa71bb0c5fc55ca12b6c002d93b75758393f08e65","cross_cats_sorted":["cs.AI"],"license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.LG","submitted_at":"2024-04-30T19:48:55Z","title_canon_sha256":"4888b73eef519d97e9addcb1d42abd67a44f3698d14867b3d53b25add6dc2eb4"},"schema_version":"1.0","source":{"id":"2405.00747","kind":"arxiv","version":4}},"canonical_sha256":"7b7311d8485514255628404cec0caa0deaa3bbd7ae239c745ea817b0cf0f836a","receipt":{"algorithm":"ed25519","builder_version":"pith-number-builder-2026-05-17-v1","canonical_sha256":"7b7311d8485514255628404cec0caa0deaa3bbd7ae239c745ea817b0cf0f836a","first_computed_at":"2026-07-05T09:15:31.972572Z","key_id":"pith-v1-2026-05","kind":"pith_receipt","last_reissued_at":"2026-07-05T09:15:31.972572Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","receipt_version":"0.3","signature_b64":"GVe9ID6kOO03jDW9C+8v3eCIWQnLwq6JyXfXbe3D73Hfku34T25/TXU4hhNfnXwcjduszOf+0SFoDt7IcJt5CQ==","signature_status":"signed_v1","signed_at":"2026-07-05T09:15:31.973066Z","signed_message":"canonical_sha256_bytes"},"source_id":"2405.00747","source_kind":"arxiv","source_version":4}}},"equivocations":[],"invalid_events":[],"applied_event_ids":["sha256:30467294c5776b934c12fc481b8415105034df7089848ee72047ab262549315b","sha256:6b03e7e4c9f9d3936a41184bcbc0d70605f4ab2fb30de8424172e45ca5632cca"],"state_sha256":"d94acf45e0d51066ec54e5600acbb2e92f31ed7d41462fb426be2dcd63957c51"}