{"bundle_type":"pith_open_graph_bundle","bundle_version":"1.0","pith_number":"pith:2023:ARMMOBV75W5WURD2ZQNO23P3PS","short_pith_number":"pith:ARMMOBV7","canonical_record":{"source":{"id":"2311.14115","kind":"arxiv","version":3},"metadata":{"license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.LG","submitted_at":"2023-11-23T17:20:36Z","cross_cats_sorted":["cs.AI","cs.CL"],"title_canon_sha256":"b10a9bd7415f891a14cf4157df52bd3c5cfe2384fc167125d5998514235e2259","abstract_canon_sha256":"2fda041d08e64cea03f302d8995eb7bca8fddb11966fa51c40cefc0fd2cf4318"},"schema_version":"1.0"},"canonical_sha256":"0458c706bfedbb6a447acc1aed6dfb7ca8cd9f2abd196c84000f22e6fa7d58a9","source":{"kind":"arxiv","id":"2311.14115","version":3},"source_aliases":[{"alias_kind":"arxiv","alias_value":"2311.14115","created_at":"2026-07-05T07:32:10Z"},{"alias_kind":"arxiv_version","alias_value":"2311.14115v3","created_at":"2026-07-05T07:32:10Z"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.2311.14115","created_at":"2026-07-05T07:32:10Z"},{"alias_kind":"pith_short_12","alias_value":"ARMMOBV75W5W","created_at":"2026-07-05T07:32:10Z"},{"alias_kind":"pith_short_16","alias_value":"ARMMOBV75W5WURD2","created_at":"2026-07-05T07:32:10Z"},{"alias_kind":"pith_short_8","alias_value":"ARMMOBV7","created_at":"2026-07-05T07:32:10Z"}],"events":[{"event_type":"record_created","subject_pith_number":"pith:2023:ARMMOBV75W5WURD2ZQNO23P3PS","target":"record","payload":{"canonical_record":{"source":{"id":"2311.14115","kind":"arxiv","version":3},"metadata":{"license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.LG","submitted_at":"2023-11-23T17:20:36Z","cross_cats_sorted":["cs.AI","cs.CL"],"title_canon_sha256":"b10a9bd7415f891a14cf4157df52bd3c5cfe2384fc167125d5998514235e2259","abstract_canon_sha256":"2fda041d08e64cea03f302d8995eb7bca8fddb11966fa51c40cefc0fd2cf4318"},"schema_version":"1.0"},"canonical_sha256":"0458c706bfedbb6a447acc1aed6dfb7ca8cd9f2abd196c84000f22e6fa7d58a9","receipt":{"kind":"pith_receipt","key_id":"pith-v1-2026-05","algorithm":"ed25519","signed_at":"2026-07-05T07:32:10.858283Z","signature_b64":"DiumpY+9+FPp1YEcixeToQrGTdn5hMac9DpQUZBvCKNIqZC94Y3I5sRD1X9S38cact/CcPHbWpkvRKqnUKaeCA==","signed_message":"canonical_sha256_bytes","builder_version":"pith-number-builder-2026-05-17-v1","receipt_version":"0.3","canonical_sha256":"0458c706bfedbb6a447acc1aed6dfb7ca8cd9f2abd196c84000f22e6fa7d58a9","last_reissued_at":"2026-07-05T07:32:10.857795Z","signature_status":"signed_v1","first_computed_at":"2026-07-05T07:32:10.857795Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"source_kind":"arxiv","source_id":"2311.14115","source_version":3,"attestation_state":"computed"},"signer":{"signer_id":"pith.science","signer_type":"pith_registry","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"created_at":"2026-07-05T07:32:10Z","supersedes":[],"prev_event":null,"signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"Kc7FS5qq8F9k6KeNfEMYBJRKWMIx0XDhRunQ7RVsGMwG+n5/FIKkunlEuRC9Mh0P3PXPaQtuViwg7xtNwSamCg==","signed_message":"open_graph_event_sha256_bytes","signed_at":"2026-07-27T15:19:56.394646Z"},"content_sha256":"1554985c2880159a725aa312fd191ebebade826bafdf407fad7a105d7cd5e7d0","schema_version":"1.0","event_id":"sha256:1554985c2880159a725aa312fd191ebebade826bafdf407fad7a105d7cd5e7d0"},{"event_type":"graph_snapshot","subject_pith_number":"pith:2023:ARMMOBV75W5WURD2ZQNO23P3PS","target":"graph","payload":{"graph_snapshot":{"paper":{"title":"A density estimation perspective on learning from pairwise human preferences","license":"http://creativecommons.org/licenses/by/4.0/","headline":"","cross_cats":["cs.AI","cs.CL"],"primary_cat":"cs.LG","authors_text":"Daniel D. Johnson, Hugo Larochelle, Pablo Samuel Castro, Vincent Dumoulin, Yann Dauphin","submitted_at":"2023-11-23T17:20:36Z","abstract_excerpt":"Learning from human feedback (LHF) -- and in particular learning from pairwise preferences -- has recently become a crucial ingredient in training large language models (LLMs), and has been the subject of much research. Most recent works frame it as a reinforcement learning problem, where a reward function is learned from pairwise preference data and the LLM is treated as a policy which is adapted to maximize the rewards, often under additional regularization constraints. We propose an alternative interpretation which centers on the generative process for pairwise preferences and treats LHF as"},"claims":{"count":0,"items":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"2311.14115","kind":"arxiv","version":3},"verdict":{"id":null,"model_set":{},"created_at":null,"strongest_claim":"","one_line_summary":"","pipeline_version":null,"weakest_assumption":"","pith_extraction_headline":""},"integrity":{"clean":true,"summary":{"advisory":0,"critical":0,"by_detector":{},"informational":0},"endpoint":"/pith/2311.14115/integrity.json","findings":[],"available":true,"detectors_run":[],"snapshot_sha256":"c28c3603d3b5d939e8dc4c7e95fa8dfce3d595e45f758748cecf8e644a296938"},"references":{"count":0,"sample":[],"resolved_work":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","internal_anchors":0},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"author_claims":{"count":0,"strong_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"builder_version":"pith-number-builder-2026-05-17-v1"},"verdict_id":null},"signer":{"signer_id":"pith.science","signer_type":"pith_registry","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"created_at":"2026-07-05T07:32:10Z","supersedes":[],"prev_event":null,"signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"vuxsogdVTxfWyXwG1g5omJn67vlOZpz8vENiJ/PAJX7lMSWg8FR9bfXnp+4g9KVVoek8wsLt5nWNhODA04SuCw==","signed_message":"open_graph_event_sha256_bytes","signed_at":"2026-07-27T15:19:56.395037Z"},"content_sha256":"73c5d1b0330b36143a587497529ad125c2bb8fac4b2c8472ae935a4ca3d254de","schema_version":"1.0","event_id":"sha256:73c5d1b0330b36143a587497529ad125c2bb8fac4b2c8472ae935a4ca3d254de"}],"timestamp_proofs":[],"mirror_hints":[{"mirror_type":"https","name":"Pith Resolver","base_url":"https://pith.science","bundle_url":"https://pith.science/pith/ARMMOBV75W5WURD2ZQNO23P3PS/bundle.json","state_url":"https://pith.science/pith/ARMMOBV75W5WURD2ZQNO23P3PS/state.json","well_known_bundle_url":"https://pith.science/.well-known/pith/ARMMOBV75W5WURD2ZQNO23P3PS/bundle.json","status":"primary"}],"public_keys":[{"key_id":"pith-v1-2026-05","algorithm":"ed25519","format":"raw","public_key_b64":"stVStoiQhXFxp4s2pdzPNoqVNBMojDU/fJ2db5S3CbM=","public_key_hex":"b2d552b68890857171a78b36a5dccf368a953413288c353f7c9d9d6f94b709b3","fingerprint_sha256_b32_first128bits":"RVFV5Z2OI2J3ZUO7ERDEBCYNKS","fingerprint_sha256_hex":"8d4b5ee74e4693bcd1df2446408b0d54","rotates_at":null,"url":"https://pith.science/pith-signing-key.json","notes":"Pith uses this Ed25519 key to sign canonical record SHA-256 digests. Verify with: ed25519_verify(public_key, message=canonical_sha256_bytes, signature=base64decode(signature_b64))."}],"merge_version":"pith-open-graph-merge-v1","built_at":"2026-07-27T15:19:56Z","links":{"resolver":"https://pith.science/pith/ARMMOBV75W5WURD2ZQNO23P3PS","bundle":"https://pith.science/pith/ARMMOBV75W5WURD2ZQNO23P3PS/bundle.json","state":"https://pith.science/pith/ARMMOBV75W5WURD2ZQNO23P3PS/state.json","well_known_bundle":"https://pith.science/.well-known/pith/ARMMOBV75W5WURD2ZQNO23P3PS/bundle.json"},"state":{"state_type":"pith_open_graph_state","state_version":"1.0","pith_number":"pith:2023:ARMMOBV75W5WURD2ZQNO23P3PS","merge_version":"pith-open-graph-merge-v1","event_count":2,"valid_event_count":2,"invalid_event_count":0,"equivocation_count":0,"current":{"canonical_record":{"metadata":{"abstract_canon_sha256":"2fda041d08e64cea03f302d8995eb7bca8fddb11966fa51c40cefc0fd2cf4318","cross_cats_sorted":["cs.AI","cs.CL"],"license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.LG","submitted_at":"2023-11-23T17:20:36Z","title_canon_sha256":"b10a9bd7415f891a14cf4157df52bd3c5cfe2384fc167125d5998514235e2259"},"schema_version":"1.0","source":{"id":"2311.14115","kind":"arxiv","version":3}},"source_aliases":[{"alias_kind":"arxiv","alias_value":"2311.14115","created_at":"2026-07-05T07:32:10Z"},{"alias_kind":"arxiv_version","alias_value":"2311.14115v3","created_at":"2026-07-05T07:32:10Z"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.2311.14115","created_at":"2026-07-05T07:32:10Z"},{"alias_kind":"pith_short_12","alias_value":"ARMMOBV75W5W","created_at":"2026-07-05T07:32:10Z"},{"alias_kind":"pith_short_16","alias_value":"ARMMOBV75W5WURD2","created_at":"2026-07-05T07:32:10Z"},{"alias_kind":"pith_short_8","alias_value":"ARMMOBV7","created_at":"2026-07-05T07:32:10Z"}],"graph_snapshots":[{"event_id":"sha256:73c5d1b0330b36143a587497529ad125c2bb8fac4b2c8472ae935a4ca3d254de","target":"graph","created_at":"2026-07-05T07:32:10Z","signer":{"key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signer_id":"pith.science","signer_type":"pith_registry"},"payload":{"graph_snapshot":{"author_claims":{"count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","strong_count":0},"builder_version":"pith-number-builder-2026-05-17-v1","claims":{"count":0,"items":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"integrity":{"available":true,"clean":true,"detectors_run":[],"endpoint":"/pith/2311.14115/integrity.json","findings":[],"snapshot_sha256":"c28c3603d3b5d939e8dc4c7e95fa8dfce3d595e45f758748cecf8e644a296938","summary":{"advisory":0,"by_detector":{},"critical":0,"informational":0}},"paper":{"abstract_excerpt":"Learning from human feedback (LHF) -- and in particular learning from pairwise preferences -- has recently become a crucial ingredient in training large language models (LLMs), and has been the subject of much research. Most recent works frame it as a reinforcement learning problem, where a reward function is learned from pairwise preference data and the LLM is treated as a policy which is adapted to maximize the rewards, often under additional regularization constraints. We propose an alternative interpretation which centers on the generative process for pairwise preferences and treats LHF as","authors_text":"Daniel D. Johnson, Hugo Larochelle, Pablo Samuel Castro, Vincent Dumoulin, Yann Dauphin","cross_cats":["cs.AI","cs.CL"],"headline":"","license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.LG","submitted_at":"2023-11-23T17:20:36Z","title":"A density estimation perspective on learning from pairwise human preferences"},"references":{"count":0,"internal_anchors":0,"resolved_work":0,"sample":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"2311.14115","kind":"arxiv","version":3},"verdict":{"created_at":null,"id":null,"model_set":{},"one_line_summary":"","pipeline_version":null,"pith_extraction_headline":"","strongest_claim":"","weakest_assumption":""}},"verdict_id":null}}],"author_attestations":[],"timestamp_anchors":[],"storage_attestations":[],"citation_signatures":[],"replication_records":[],"corrections":[],"mirror_hints":[],"record_created":{"event_id":"sha256:1554985c2880159a725aa312fd191ebebade826bafdf407fad7a105d7cd5e7d0","target":"record","created_at":"2026-07-05T07:32:10Z","signer":{"key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signer_id":"pith.science","signer_type":"pith_registry"},"payload":{"attestation_state":"computed","canonical_record":{"metadata":{"abstract_canon_sha256":"2fda041d08e64cea03f302d8995eb7bca8fddb11966fa51c40cefc0fd2cf4318","cross_cats_sorted":["cs.AI","cs.CL"],"license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.LG","submitted_at":"2023-11-23T17:20:36Z","title_canon_sha256":"b10a9bd7415f891a14cf4157df52bd3c5cfe2384fc167125d5998514235e2259"},"schema_version":"1.0","source":{"id":"2311.14115","kind":"arxiv","version":3}},"canonical_sha256":"0458c706bfedbb6a447acc1aed6dfb7ca8cd9f2abd196c84000f22e6fa7d58a9","receipt":{"algorithm":"ed25519","builder_version":"pith-number-builder-2026-05-17-v1","canonical_sha256":"0458c706bfedbb6a447acc1aed6dfb7ca8cd9f2abd196c84000f22e6fa7d58a9","first_computed_at":"2026-07-05T07:32:10.857795Z","key_id":"pith-v1-2026-05","kind":"pith_receipt","last_reissued_at":"2026-07-05T07:32:10.857795Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","receipt_version":"0.3","signature_b64":"DiumpY+9+FPp1YEcixeToQrGTdn5hMac9DpQUZBvCKNIqZC94Y3I5sRD1X9S38cact/CcPHbWpkvRKqnUKaeCA==","signature_status":"signed_v1","signed_at":"2026-07-05T07:32:10.858283Z","signed_message":"canonical_sha256_bytes"},"source_id":"2311.14115","source_kind":"arxiv","source_version":3}}},"equivocations":[],"invalid_events":[],"applied_event_ids":["sha256:1554985c2880159a725aa312fd191ebebade826bafdf407fad7a105d7cd5e7d0","sha256:73c5d1b0330b36143a587497529ad125c2bb8fac4b2c8472ae935a4ca3d254de"],"state_sha256":"21ddf085ff4ba8536b0e0038c1af6bf7a699770ae05d944f208cc97e03919b06"},"bundle_signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"1GFlNa60EtvAa4dJM/opMv13TSjpic+3qFd1bH8Sab36g+SOSxGnOHK7c8k7CsNqihFDCbzTutQPi/EKUPgkCQ==","signed_message":"bundle_sha256_bytes","signed_at":"2026-07-27T15:19:56.397398Z","bundle_sha256":"7fef265240d731737eda15fc3228c6f672351934442c2905b9aa9efb7369fd0d"}}