{"bundle_type":"pith_open_graph_bundle","bundle_version":"1.0","pith_number":"pith:2024:MNN2C64NX3ZVAUC3YMNHHCQEBM","short_pith_number":"pith:MNN2C64N","canonical_record":{"source":{"id":"2406.09279","kind":"arxiv","version":2},"metadata":{"license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.CL","submitted_at":"2024-06-13T16:17:21Z","cross_cats_sorted":[],"title_canon_sha256":"1cd3bfc79394695e3b7c836bf9648ef0bfd3715b4317f362c3f863db7f60a21e","abstract_canon_sha256":"fb57553bdf4dd687e6a7cfbddc2163e4f4b8a7d5751bc72c3d97c716b2928678"},"schema_version":"1.0"},"canonical_sha256":"635ba17b8dbef350505bc31a738a040b2eb53269c0a08972af54f9721d90d1c2","source":{"kind":"arxiv","id":"2406.09279","version":2},"source_aliases":[{"alias_kind":"arxiv","alias_value":"2406.09279","created_at":"2026-07-05T09:17:13Z"},{"alias_kind":"arxiv_version","alias_value":"2406.09279v2","created_at":"2026-07-05T09:17:13Z"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.2406.09279","created_at":"2026-07-05T09:17:13Z"},{"alias_kind":"pith_short_12","alias_value":"MNN2C64NX3ZV","created_at":"2026-07-05T09:17:13Z"},{"alias_kind":"pith_short_16","alias_value":"MNN2C64NX3ZVAUC3","created_at":"2026-07-05T09:17:13Z"},{"alias_kind":"pith_short_8","alias_value":"MNN2C64N","created_at":"2026-07-05T09:17:13Z"}],"events":[{"event_type":"record_created","subject_pith_number":"pith:2024:MNN2C64NX3ZVAUC3YMNHHCQEBM","target":"record","payload":{"canonical_record":{"source":{"id":"2406.09279","kind":"arxiv","version":2},"metadata":{"license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.CL","submitted_at":"2024-06-13T16:17:21Z","cross_cats_sorted":[],"title_canon_sha256":"1cd3bfc79394695e3b7c836bf9648ef0bfd3715b4317f362c3f863db7f60a21e","abstract_canon_sha256":"fb57553bdf4dd687e6a7cfbddc2163e4f4b8a7d5751bc72c3d97c716b2928678"},"schema_version":"1.0"},"canonical_sha256":"635ba17b8dbef350505bc31a738a040b2eb53269c0a08972af54f9721d90d1c2","receipt":{"kind":"pith_receipt","key_id":"pith-v1-2026-05","algorithm":"ed25519","signed_at":"2026-07-05T09:17:13.878882Z","signature_b64":"8++xmidi/3+ydkg/jezJ6FjjjysW2Diguj4zSHbjsYYHR/Kys69ykHjqPZ8K6NA1ZauwRFMIhMOZOhob8s2BCg==","signed_message":"canonical_sha256_bytes","builder_version":"pith-number-builder-2026-05-17-v1","receipt_version":"0.3","canonical_sha256":"635ba17b8dbef350505bc31a738a040b2eb53269c0a08972af54f9721d90d1c2","last_reissued_at":"2026-07-05T09:17:13.878437Z","signature_status":"signed_v1","first_computed_at":"2026-07-05T09:17:13.878437Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"source_kind":"arxiv","source_id":"2406.09279","source_version":2,"attestation_state":"computed"},"signer":{"signer_id":"pith.science","signer_type":"pith_registry","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"created_at":"2026-07-05T09:17:13Z","supersedes":[],"prev_event":null,"signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"af4iKiPlCguk6fhrISOC5EXRZHnPB3y7BukQ/CiWWSWSa0s6SCR66Bgew73c1aNlgjWTrj8v+87iNbvhoCxmCg==","signed_message":"open_graph_event_sha256_bytes","signed_at":"2026-08-09T05:56:26.821381Z"},"content_sha256":"29cb79fc65a810a55f11c92e67d29af6fcde1e25a529473e02da3eb15cca2a45","schema_version":"1.0","event_id":"sha256:29cb79fc65a810a55f11c92e67d29af6fcde1e25a529473e02da3eb15cca2a45"},{"event_type":"graph_snapshot","subject_pith_number":"pith:2024:MNN2C64NX3ZVAUC3YMNHHCQEBM","target":"graph","payload":{"graph_snapshot":{"paper":{"title":"Unpacking DPO and PPO: Disentangling Best Practices for Learning from Preference Feedback","license":"http://creativecommons.org/licenses/by/4.0/","headline":"","cross_cats":[],"primary_cat":"cs.CL","authors_text":"Hamish Ivison, Hannaneh Hajishirzi, Jiacheng Liu, Nathan Lambert, Noah A. Smith, Valentina Pyatkin, Yejin Choi, Yizhong Wang, Zeqiu Wu","submitted_at":"2024-06-13T16:17:21Z","abstract_excerpt":"Learning from preference feedback has emerged as an essential step for improving the generation quality and performance of modern language models (LMs). Despite its widespread use, the way preference-based learning is applied varies wildly, with differing data, learning algorithms, and evaluations used, making disentangling the impact of each aspect difficult. In this work, we identify four core aspects of preference-based learning: preference data, learning algorithm, reward model, and policy training prompts, systematically investigate the impact of these components on downstream model perfo"},"claims":{"count":0,"items":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"2406.09279","kind":"arxiv","version":2},"verdict":{"id":null,"model_set":{},"created_at":null,"strongest_claim":"","one_line_summary":"","pipeline_version":null,"weakest_assumption":"","pith_extraction_headline":""},"integrity":{"clean":true,"summary":{"advisory":0,"critical":0,"by_detector":{},"informational":0},"endpoint":"/pith/2406.09279/integrity.json","findings":[],"available":true,"detectors_run":[],"snapshot_sha256":"c28c3603d3b5d939e8dc4c7e95fa8dfce3d595e45f758748cecf8e644a296938"},"references":{"count":0,"sample":[],"resolved_work":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","internal_anchors":0},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"author_claims":{"count":0,"strong_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"builder_version":"pith-number-builder-2026-05-17-v1"},"verdict_id":null},"signer":{"signer_id":"pith.science","signer_type":"pith_registry","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"created_at":"2026-07-05T09:17:13Z","supersedes":[],"prev_event":null,"signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"4W8CaX34bIk/qMISnBFP6kflBCw+0dIuMjJHm/X8acnD8HfUWv4Mdky4ffyqPsP4pRBvDAHGW7O/nDnexBLpDQ==","signed_message":"open_graph_event_sha256_bytes","signed_at":"2026-08-09T05:56:26.821879Z"},"content_sha256":"9bcb2a97d3580bc666ff3c4a83eda9f56ceb2e5e3e2895621960320343c07e13","schema_version":"1.0","event_id":"sha256:9bcb2a97d3580bc666ff3c4a83eda9f56ceb2e5e3e2895621960320343c07e13"}],"timestamp_proofs":[],"mirror_hints":[{"mirror_type":"https","name":"Pith Resolver","base_url":"https://pith.science","bundle_url":"https://pith.science/pith/MNN2C64NX3ZVAUC3YMNHHCQEBM/bundle.json","state_url":"https://pith.science/pith/MNN2C64NX3ZVAUC3YMNHHCQEBM/state.json","well_known_bundle_url":"https://pith.science/.well-known/pith/MNN2C64NX3ZVAUC3YMNHHCQEBM/bundle.json","status":"primary"}],"public_keys":[{"key_id":"pith-v1-2026-05","algorithm":"ed25519","format":"raw","public_key_b64":"stVStoiQhXFxp4s2pdzPNoqVNBMojDU/fJ2db5S3CbM=","public_key_hex":"b2d552b68890857171a78b36a5dccf368a953413288c353f7c9d9d6f94b709b3","fingerprint_sha256_b32_first128bits":"RVFV5Z2OI2J3ZUO7ERDEBCYNKS","fingerprint_sha256_hex":"8d4b5ee74e4693bcd1df2446408b0d54","rotates_at":null,"url":"https://pith.science/pith-signing-key.json","notes":"Pith uses this Ed25519 key to sign canonical record SHA-256 digests. Verify with: ed25519_verify(public_key, message=canonical_sha256_bytes, signature=base64decode(signature_b64))."}],"merge_version":"pith-open-graph-merge-v1","built_at":"2026-08-09T05:56:26Z","links":{"resolver":"https://pith.science/pith/MNN2C64NX3ZVAUC3YMNHHCQEBM","bundle":"https://pith.science/pith/MNN2C64NX3ZVAUC3YMNHHCQEBM/bundle.json","state":"https://pith.science/pith/MNN2C64NX3ZVAUC3YMNHHCQEBM/state.json","well_known_bundle":"https://pith.science/.well-known/pith/MNN2C64NX3ZVAUC3YMNHHCQEBM/bundle.json"},"state":{"state_type":"pith_open_graph_state","state_version":"1.0","pith_number":"pith:2024:MNN2C64NX3ZVAUC3YMNHHCQEBM","merge_version":"pith-open-graph-merge-v1","event_count":2,"valid_event_count":2,"invalid_event_count":0,"equivocation_count":0,"current":{"canonical_record":{"metadata":{"abstract_canon_sha256":"fb57553bdf4dd687e6a7cfbddc2163e4f4b8a7d5751bc72c3d97c716b2928678","cross_cats_sorted":[],"license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.CL","submitted_at":"2024-06-13T16:17:21Z","title_canon_sha256":"1cd3bfc79394695e3b7c836bf9648ef0bfd3715b4317f362c3f863db7f60a21e"},"schema_version":"1.0","source":{"id":"2406.09279","kind":"arxiv","version":2}},"source_aliases":[{"alias_kind":"arxiv","alias_value":"2406.09279","created_at":"2026-07-05T09:17:13Z"},{"alias_kind":"arxiv_version","alias_value":"2406.09279v2","created_at":"2026-07-05T09:17:13Z"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.2406.09279","created_at":"2026-07-05T09:17:13Z"},{"alias_kind":"pith_short_12","alias_value":"MNN2C64NX3ZV","created_at":"2026-07-05T09:17:13Z"},{"alias_kind":"pith_short_16","alias_value":"MNN2C64NX3ZVAUC3","created_at":"2026-07-05T09:17:13Z"},{"alias_kind":"pith_short_8","alias_value":"MNN2C64N","created_at":"2026-07-05T09:17:13Z"}],"graph_snapshots":[{"event_id":"sha256:9bcb2a97d3580bc666ff3c4a83eda9f56ceb2e5e3e2895621960320343c07e13","target":"graph","created_at":"2026-07-05T09:17:13Z","signer":{"key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signer_id":"pith.science","signer_type":"pith_registry"},"payload":{"graph_snapshot":{"author_claims":{"count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","strong_count":0},"builder_version":"pith-number-builder-2026-05-17-v1","claims":{"count":0,"items":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"integrity":{"available":true,"clean":true,"detectors_run":[],"endpoint":"/pith/2406.09279/integrity.json","findings":[],"snapshot_sha256":"c28c3603d3b5d939e8dc4c7e95fa8dfce3d595e45f758748cecf8e644a296938","summary":{"advisory":0,"by_detector":{},"critical":0,"informational":0}},"paper":{"abstract_excerpt":"Learning from preference feedback has emerged as an essential step for improving the generation quality and performance of modern language models (LMs). Despite its widespread use, the way preference-based learning is applied varies wildly, with differing data, learning algorithms, and evaluations used, making disentangling the impact of each aspect difficult. In this work, we identify four core aspects of preference-based learning: preference data, learning algorithm, reward model, and policy training prompts, systematically investigate the impact of these components on downstream model perfo","authors_text":"Hamish Ivison, Hannaneh Hajishirzi, Jiacheng Liu, Nathan Lambert, Noah A. Smith, Valentina Pyatkin, Yejin Choi, Yizhong Wang, Zeqiu Wu","cross_cats":[],"headline":"","license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.CL","submitted_at":"2024-06-13T16:17:21Z","title":"Unpacking DPO and PPO: Disentangling Best Practices for Learning from Preference Feedback"},"references":{"count":0,"internal_anchors":0,"resolved_work":0,"sample":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"2406.09279","kind":"arxiv","version":2},"verdict":{"created_at":null,"id":null,"model_set":{},"one_line_summary":"","pipeline_version":null,"pith_extraction_headline":"","strongest_claim":"","weakest_assumption":""}},"verdict_id":null}}],"author_attestations":[],"timestamp_anchors":[],"storage_attestations":[],"citation_signatures":[],"replication_records":[],"corrections":[],"mirror_hints":[],"record_created":{"event_id":"sha256:29cb79fc65a810a55f11c92e67d29af6fcde1e25a529473e02da3eb15cca2a45","target":"record","created_at":"2026-07-05T09:17:13Z","signer":{"key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signer_id":"pith.science","signer_type":"pith_registry"},"payload":{"attestation_state":"computed","canonical_record":{"metadata":{"abstract_canon_sha256":"fb57553bdf4dd687e6a7cfbddc2163e4f4b8a7d5751bc72c3d97c716b2928678","cross_cats_sorted":[],"license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.CL","submitted_at":"2024-06-13T16:17:21Z","title_canon_sha256":"1cd3bfc79394695e3b7c836bf9648ef0bfd3715b4317f362c3f863db7f60a21e"},"schema_version":"1.0","source":{"id":"2406.09279","kind":"arxiv","version":2}},"canonical_sha256":"635ba17b8dbef350505bc31a738a040b2eb53269c0a08972af54f9721d90d1c2","receipt":{"algorithm":"ed25519","builder_version":"pith-number-builder-2026-05-17-v1","canonical_sha256":"635ba17b8dbef350505bc31a738a040b2eb53269c0a08972af54f9721d90d1c2","first_computed_at":"2026-07-05T09:17:13.878437Z","key_id":"pith-v1-2026-05","kind":"pith_receipt","last_reissued_at":"2026-07-05T09:17:13.878437Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","receipt_version":"0.3","signature_b64":"8++xmidi/3+ydkg/jezJ6FjjjysW2Diguj4zSHbjsYYHR/Kys69ykHjqPZ8K6NA1ZauwRFMIhMOZOhob8s2BCg==","signature_status":"signed_v1","signed_at":"2026-07-05T09:17:13.878882Z","signed_message":"canonical_sha256_bytes"},"source_id":"2406.09279","source_kind":"arxiv","source_version":2}}},"equivocations":[],"invalid_events":[],"applied_event_ids":["sha256:29cb79fc65a810a55f11c92e67d29af6fcde1e25a529473e02da3eb15cca2a45","sha256:9bcb2a97d3580bc666ff3c4a83eda9f56ceb2e5e3e2895621960320343c07e13"],"state_sha256":"5cd217b291ebcaf9600969cacfe4d992af2fb89bac789de4b0b4a5460ea76f76"},"bundle_signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"RJi8g259cO6cA4VnSmULZpPyTQi6fuMjC98EPz/c64RkADt2pgD4uSMIACdR0zzKsh2qvzNkNbhg5RB6/nPlDQ==","signed_message":"bundle_sha256_bytes","signed_at":"2026-08-09T05:56:26.827126Z","bundle_sha256":"4fd66024050f31bdc56216d126fe08a8c91c16bd9f638c0e2d3bcf77a38ec931"}}