{"bundle_type":"pith_open_graph_bundle","bundle_version":"1.0","pith_number":"pith:2024:RJIXMJEUFJPRPSAO6YC4TQZ6GM","short_pith_number":"pith:RJIXMJEU","canonical_record":{"source":{"id":"2410.17055","kind":"arxiv","version":2},"metadata":{"license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.LG","submitted_at":"2024-10-22T14:36:44Z","cross_cats_sorted":["stat.ML"],"title_canon_sha256":"22d4441c1210c0070219f7d76b17c1a53d85330e1b57e122d1f095ea230dda7a","abstract_canon_sha256":"7b28aa9edb653f14e7879d00bcb8df19598433e1749408c666c8b52dfe8ddbd5"},"schema_version":"1.0"},"canonical_sha256":"8a517624942a5f17c80ef605c9c33e332df346a09bb303512d92a3bed40462cc","source":{"kind":"arxiv","id":"2410.17055","version":2},"source_aliases":[{"alias_kind":"arxiv","alias_value":"2410.17055","created_at":"2026-07-05T09:24:40Z"},{"alias_kind":"arxiv_version","alias_value":"2410.17055v2","created_at":"2026-07-05T09:24:40Z"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.2410.17055","created_at":"2026-07-05T09:24:40Z"},{"alias_kind":"pith_short_12","alias_value":"RJIXMJEUFJPR","created_at":"2026-07-05T09:24:40Z"},{"alias_kind":"pith_short_16","alias_value":"RJIXMJEUFJPRPSAO","created_at":"2026-07-05T09:24:40Z"},{"alias_kind":"pith_short_8","alias_value":"RJIXMJEU","created_at":"2026-07-05T09:24:40Z"}],"events":[{"event_type":"record_created","subject_pith_number":"pith:2024:RJIXMJEUFJPRPSAO6YC4TQZ6GM","target":"record","payload":{"canonical_record":{"source":{"id":"2410.17055","kind":"arxiv","version":2},"metadata":{"license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.LG","submitted_at":"2024-10-22T14:36:44Z","cross_cats_sorted":["stat.ML"],"title_canon_sha256":"22d4441c1210c0070219f7d76b17c1a53d85330e1b57e122d1f095ea230dda7a","abstract_canon_sha256":"7b28aa9edb653f14e7879d00bcb8df19598433e1749408c666c8b52dfe8ddbd5"},"schema_version":"1.0"},"canonical_sha256":"8a517624942a5f17c80ef605c9c33e332df346a09bb303512d92a3bed40462cc","receipt":{"kind":"pith_receipt","key_id":"pith-v1-2026-05","algorithm":"ed25519","signed_at":"2026-07-05T09:24:40.111001Z","signature_b64":"f46RRHpO/VAztoHaqM9PiRQUuf9Z3zXadMdUyd8frP3dUR3e2rqXUFI9UZwdDmxHCp5g4QBapqu/P872ZjmcCw==","signed_message":"canonical_sha256_bytes","builder_version":"pith-number-builder-2026-05-17-v1","receipt_version":"0.3","canonical_sha256":"8a517624942a5f17c80ef605c9c33e332df346a09bb303512d92a3bed40462cc","last_reissued_at":"2026-07-05T09:24:40.110452Z","signature_status":"signed_v1","first_computed_at":"2026-07-05T09:24:40.110452Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"source_kind":"arxiv","source_id":"2410.17055","source_version":2,"attestation_state":"computed"},"signer":{"signer_id":"pith.science","signer_type":"pith_registry","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"created_at":"2026-07-05T09:24:40Z","supersedes":[],"prev_event":null,"signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"EeLUubKC7DkniMNBt03jNoMuadMtgOxLnAYWNvviO+b6jpCo82NzDlAQUU+bSS3szEYRvHFojkjecNaS503yCw==","signed_message":"open_graph_event_sha256_bytes","signed_at":"2026-07-16T15:37:46.376264Z"},"content_sha256":"ef21e8afe923cc3ebf5b70e18f35c1d4d84cc077c68031ffc220ff9ec897a1a7","schema_version":"1.0","event_id":"sha256:ef21e8afe923cc3ebf5b70e18f35c1d4d84cc077c68031ffc220ff9ec897a1a7"},{"event_type":"graph_snapshot","subject_pith_number":"pith:2024:RJIXMJEUFJPRPSAO6YC4TQZ6GM","target":"graph","payload":{"graph_snapshot":{"paper":{"title":"Optimal Design for Reward Modeling in RLHF","license":"http://creativecommons.org/licenses/by/4.0/","headline":"","cross_cats":["stat.ML"],"primary_cat":"cs.LG","authors_text":"Alain Durmus, Antoine Scheid, Eric Moulines, Etienne Boursier, Michael I. Jordan, Michal Valko, Pierre M\\'enard","submitted_at":"2024-10-22T14:36:44Z","abstract_excerpt":"Reinforcement Learning from Human Feedback (RLHF) has become a popular approach to align language models (LMs) with human preferences. This method involves collecting a large dataset of human pairwise preferences across various text generations and using it to infer (implicitly or explicitly) a reward model. Numerous methods have been proposed to learn the reward model and align a LM with it. However, the costly process of collecting human preferences has received little attention and could benefit from theoretical insights. This paper addresses this issue and aims to formalize the reward trai"},"claims":{"count":0,"items":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"2410.17055","kind":"arxiv","version":2},"verdict":{"id":null,"model_set":{},"created_at":null,"strongest_claim":"","one_line_summary":"","pipeline_version":null,"weakest_assumption":"","pith_extraction_headline":""},"integrity":{"clean":true,"summary":{"advisory":0,"critical":0,"by_detector":{},"informational":0},"endpoint":"/pith/2410.17055/integrity.json","findings":[],"available":true,"detectors_run":[],"snapshot_sha256":"c28c3603d3b5d939e8dc4c7e95fa8dfce3d595e45f758748cecf8e644a296938"},"references":{"count":0,"sample":[],"resolved_work":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","internal_anchors":0},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"author_claims":{"count":0,"strong_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"builder_version":"pith-number-builder-2026-05-17-v1"},"verdict_id":null},"signer":{"signer_id":"pith.science","signer_type":"pith_registry","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"created_at":"2026-07-05T09:24:40Z","supersedes":[],"prev_event":null,"signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"7HpsShuuTP84EcBmQfzofijmdW7Pn0h10txqPDQkK+6kIf6hnUEDT+Ps+5k8ENN2LJ2p61avdNS8aHztOEmABA==","signed_message":"open_graph_event_sha256_bytes","signed_at":"2026-07-16T15:37:46.376661Z"},"content_sha256":"b9e88ece20c517290a3b8e7d18fdf568726a879ed44da95111d7a4aa973ae650","schema_version":"1.0","event_id":"sha256:b9e88ece20c517290a3b8e7d18fdf568726a879ed44da95111d7a4aa973ae650"}],"timestamp_proofs":[],"mirror_hints":[{"mirror_type":"https","name":"Pith Resolver","base_url":"https://pith.science","bundle_url":"https://pith.science/pith/RJIXMJEUFJPRPSAO6YC4TQZ6GM/bundle.json","state_url":"https://pith.science/pith/RJIXMJEUFJPRPSAO6YC4TQZ6GM/state.json","well_known_bundle_url":"https://pith.science/.well-known/pith/RJIXMJEUFJPRPSAO6YC4TQZ6GM/bundle.json","status":"primary"}],"public_keys":[{"key_id":"pith-v1-2026-05","algorithm":"ed25519","format":"raw","public_key_b64":"stVStoiQhXFxp4s2pdzPNoqVNBMojDU/fJ2db5S3CbM=","public_key_hex":"b2d552b68890857171a78b36a5dccf368a953413288c353f7c9d9d6f94b709b3","fingerprint_sha256_b32_first128bits":"RVFV5Z2OI2J3ZUO7ERDEBCYNKS","fingerprint_sha256_hex":"8d4b5ee74e4693bcd1df2446408b0d54","rotates_at":null,"url":"https://pith.science/pith-signing-key.json","notes":"Pith uses this Ed25519 key to sign canonical record SHA-256 digests. Verify with: ed25519_verify(public_key, message=canonical_sha256_bytes, signature=base64decode(signature_b64))."}],"merge_version":"pith-open-graph-merge-v1","built_at":"2026-07-16T15:37:46Z","links":{"resolver":"https://pith.science/pith/RJIXMJEUFJPRPSAO6YC4TQZ6GM","bundle":"https://pith.science/pith/RJIXMJEUFJPRPSAO6YC4TQZ6GM/bundle.json","state":"https://pith.science/pith/RJIXMJEUFJPRPSAO6YC4TQZ6GM/state.json","well_known_bundle":"https://pith.science/.well-known/pith/RJIXMJEUFJPRPSAO6YC4TQZ6GM/bundle.json"},"state":{"state_type":"pith_open_graph_state","state_version":"1.0","pith_number":"pith:2024:RJIXMJEUFJPRPSAO6YC4TQZ6GM","merge_version":"pith-open-graph-merge-v1","event_count":2,"valid_event_count":2,"invalid_event_count":0,"equivocation_count":0,"current":{"canonical_record":{"metadata":{"abstract_canon_sha256":"7b28aa9edb653f14e7879d00bcb8df19598433e1749408c666c8b52dfe8ddbd5","cross_cats_sorted":["stat.ML"],"license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.LG","submitted_at":"2024-10-22T14:36:44Z","title_canon_sha256":"22d4441c1210c0070219f7d76b17c1a53d85330e1b57e122d1f095ea230dda7a"},"schema_version":"1.0","source":{"id":"2410.17055","kind":"arxiv","version":2}},"source_aliases":[{"alias_kind":"arxiv","alias_value":"2410.17055","created_at":"2026-07-05T09:24:40Z"},{"alias_kind":"arxiv_version","alias_value":"2410.17055v2","created_at":"2026-07-05T09:24:40Z"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.2410.17055","created_at":"2026-07-05T09:24:40Z"},{"alias_kind":"pith_short_12","alias_value":"RJIXMJEUFJPR","created_at":"2026-07-05T09:24:40Z"},{"alias_kind":"pith_short_16","alias_value":"RJIXMJEUFJPRPSAO","created_at":"2026-07-05T09:24:40Z"},{"alias_kind":"pith_short_8","alias_value":"RJIXMJEU","created_at":"2026-07-05T09:24:40Z"}],"graph_snapshots":[{"event_id":"sha256:b9e88ece20c517290a3b8e7d18fdf568726a879ed44da95111d7a4aa973ae650","target":"graph","created_at":"2026-07-05T09:24:40Z","signer":{"key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signer_id":"pith.science","signer_type":"pith_registry"},"payload":{"graph_snapshot":{"author_claims":{"count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","strong_count":0},"builder_version":"pith-number-builder-2026-05-17-v1","claims":{"count":0,"items":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"integrity":{"available":true,"clean":true,"detectors_run":[],"endpoint":"/pith/2410.17055/integrity.json","findings":[],"snapshot_sha256":"c28c3603d3b5d939e8dc4c7e95fa8dfce3d595e45f758748cecf8e644a296938","summary":{"advisory":0,"by_detector":{},"critical":0,"informational":0}},"paper":{"abstract_excerpt":"Reinforcement Learning from Human Feedback (RLHF) has become a popular approach to align language models (LMs) with human preferences. This method involves collecting a large dataset of human pairwise preferences across various text generations and using it to infer (implicitly or explicitly) a reward model. Numerous methods have been proposed to learn the reward model and align a LM with it. However, the costly process of collecting human preferences has received little attention and could benefit from theoretical insights. This paper addresses this issue and aims to formalize the reward trai","authors_text":"Alain Durmus, Antoine Scheid, Eric Moulines, Etienne Boursier, Michael I. Jordan, Michal Valko, Pierre M\\'enard","cross_cats":["stat.ML"],"headline":"","license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.LG","submitted_at":"2024-10-22T14:36:44Z","title":"Optimal Design for Reward Modeling in RLHF"},"references":{"count":0,"internal_anchors":0,"resolved_work":0,"sample":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"2410.17055","kind":"arxiv","version":2},"verdict":{"created_at":null,"id":null,"model_set":{},"one_line_summary":"","pipeline_version":null,"pith_extraction_headline":"","strongest_claim":"","weakest_assumption":""}},"verdict_id":null}}],"author_attestations":[],"timestamp_anchors":[],"storage_attestations":[],"citation_signatures":[],"replication_records":[],"corrections":[],"mirror_hints":[],"record_created":{"event_id":"sha256:ef21e8afe923cc3ebf5b70e18f35c1d4d84cc077c68031ffc220ff9ec897a1a7","target":"record","created_at":"2026-07-05T09:24:40Z","signer":{"key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signer_id":"pith.science","signer_type":"pith_registry"},"payload":{"attestation_state":"computed","canonical_record":{"metadata":{"abstract_canon_sha256":"7b28aa9edb653f14e7879d00bcb8df19598433e1749408c666c8b52dfe8ddbd5","cross_cats_sorted":["stat.ML"],"license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.LG","submitted_at":"2024-10-22T14:36:44Z","title_canon_sha256":"22d4441c1210c0070219f7d76b17c1a53d85330e1b57e122d1f095ea230dda7a"},"schema_version":"1.0","source":{"id":"2410.17055","kind":"arxiv","version":2}},"canonical_sha256":"8a517624942a5f17c80ef605c9c33e332df346a09bb303512d92a3bed40462cc","receipt":{"algorithm":"ed25519","builder_version":"pith-number-builder-2026-05-17-v1","canonical_sha256":"8a517624942a5f17c80ef605c9c33e332df346a09bb303512d92a3bed40462cc","first_computed_at":"2026-07-05T09:24:40.110452Z","key_id":"pith-v1-2026-05","kind":"pith_receipt","last_reissued_at":"2026-07-05T09:24:40.110452Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","receipt_version":"0.3","signature_b64":"f46RRHpO/VAztoHaqM9PiRQUuf9Z3zXadMdUyd8frP3dUR3e2rqXUFI9UZwdDmxHCp5g4QBapqu/P872ZjmcCw==","signature_status":"signed_v1","signed_at":"2026-07-05T09:24:40.111001Z","signed_message":"canonical_sha256_bytes"},"source_id":"2410.17055","source_kind":"arxiv","source_version":2}}},"equivocations":[],"invalid_events":[],"applied_event_ids":["sha256:ef21e8afe923cc3ebf5b70e18f35c1d4d84cc077c68031ffc220ff9ec897a1a7","sha256:b9e88ece20c517290a3b8e7d18fdf568726a879ed44da95111d7a4aa973ae650"],"state_sha256":"27a72e49192d736a8f68a2577442c4aedc2c6d46a69c0e6b8c83fb75a5014812"},"bundle_signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"DsuNMks9Lsz335BhPVh/KxOvjwbSkPpGXnbpXOgLZjmU81cgOXoxwUWpQlcYGIWBixQ/ITcVExw3D4oik8BSDw==","signed_message":"bundle_sha256_bytes","signed_at":"2026-07-16T15:37:46.378869Z","bundle_sha256":"675580bfdaec7af05675de57b04ba9bb32f1c16099f77ebe0777f0f8b53a0cc4"}}