{"bundle_type":"pith_open_graph_bundle","bundle_version":"1.0","pith_number":"pith:2024:YKI4JOUUOKNSKZHJKFOSJHOOZZ","short_pith_number":"pith:YKI4JOUU","canonical_record":{"source":{"id":"2402.00742","kind":"arxiv","version":2},"metadata":{"license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.CL","submitted_at":"2024-02-01T16:39:28Z","cross_cats_sorted":["cs.AI"],"title_canon_sha256":"25da635e53ed4ecf84da86c4cc307e90124b13a554eb0fa05d0cfb8dda896ea6","abstract_canon_sha256":"6aff8462b2d6ede8c1b86cda4b9bb747470734951cbef06ad8ac19d1ab2879d3"},"schema_version":"1.0"},"canonical_sha256":"c291c4ba94729b2564e9515d249dcece46b585cf2a5ad5998db9de13a3e440e0","source":{"kind":"arxiv","id":"2402.00742","version":2},"source_aliases":[{"alias_kind":"arxiv","alias_value":"2402.00742","created_at":"2026-07-05T08:45:49Z"},{"alias_kind":"arxiv_version","alias_value":"2402.00742v2","created_at":"2026-07-05T08:45:49Z"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.2402.00742","created_at":"2026-07-05T08:45:49Z"},{"alias_kind":"pith_short_12","alias_value":"YKI4JOUUOKNS","created_at":"2026-07-05T08:45:49Z"},{"alias_kind":"pith_short_16","alias_value":"YKI4JOUUOKNSKZHJ","created_at":"2026-07-05T08:45:49Z"},{"alias_kind":"pith_short_8","alias_value":"YKI4JOUU","created_at":"2026-07-05T08:45:49Z"}],"events":[{"event_type":"record_created","subject_pith_number":"pith:2024:YKI4JOUUOKNSKZHJKFOSJHOOZZ","target":"record","payload":{"canonical_record":{"source":{"id":"2402.00742","kind":"arxiv","version":2},"metadata":{"license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.CL","submitted_at":"2024-02-01T16:39:28Z","cross_cats_sorted":["cs.AI"],"title_canon_sha256":"25da635e53ed4ecf84da86c4cc307e90124b13a554eb0fa05d0cfb8dda896ea6","abstract_canon_sha256":"6aff8462b2d6ede8c1b86cda4b9bb747470734951cbef06ad8ac19d1ab2879d3"},"schema_version":"1.0"},"canonical_sha256":"c291c4ba94729b2564e9515d249dcece46b585cf2a5ad5998db9de13a3e440e0","receipt":{"kind":"pith_receipt","key_id":"pith-v1-2026-05","algorithm":"ed25519","signed_at":"2026-07-05T08:45:49.867809Z","signature_b64":"BSpWfHRHiiWEKObP4qYLi0XdFVEb35NohzniUoPxmbue/IS8UfvClV8wu60WKHMszMTKgpFMLnIu5tfTr0BzDA==","signed_message":"canonical_sha256_bytes","builder_version":"pith-number-builder-2026-05-17-v1","receipt_version":"0.3","canonical_sha256":"c291c4ba94729b2564e9515d249dcece46b585cf2a5ad5998db9de13a3e440e0","last_reissued_at":"2026-07-05T08:45:49.867335Z","signature_status":"signed_v1","first_computed_at":"2026-07-05T08:45:49.867335Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"source_kind":"arxiv","source_id":"2402.00742","source_version":2,"attestation_state":"computed"},"signer":{"signer_id":"pith.science","signer_type":"pith_registry","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"created_at":"2026-07-05T08:45:49Z","supersedes":[],"prev_event":null,"signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"VMz8mx+RF4U0zpndkPyVKJuLzoBJaTe6+jLjyu8MikC8ns109+cGegWW2BlkHEaTsRT5PjJGuYncF+rE6UwvBw==","signed_message":"open_graph_event_sha256_bytes","signed_at":"2026-08-08T21:34:41.955558Z"},"content_sha256":"79f034215941bb5f574677f57c4ae7adf900c321c03ab6ce88c74baf4a743590","schema_version":"1.0","event_id":"sha256:79f034215941bb5f574677f57c4ae7adf900c321c03ab6ce88c74baf4a743590"},{"event_type":"graph_snapshot","subject_pith_number":"pith:2024:YKI4JOUUOKNSKZHJKFOSJHOOZZ","target":"graph","payload":{"graph_snapshot":{"paper":{"title":"Transforming and Combining Rewards for Aligning Large Language Models","license":"http://creativecommons.org/licenses/by/4.0/","headline":"","cross_cats":["cs.AI"],"primary_cat":"cs.CL","authors_text":"Alex D'Amour, Chirag Nagpal, Jacob Eisenstein, Jonathan Berant, Sanmi Koyejo, Victor Veitch, Zihao Wang","submitted_at":"2024-02-01T16:39:28Z","abstract_excerpt":"A common approach for aligning language models to human preferences is to first learn a reward model from preference data, and then use this reward model to update the language model. We study two closely related problems that arise in this approach. First, any monotone transformation of the reward model preserves preference ranking; is there a choice that is ``better'' than others? Second, we often wish to align language models to multiple properties: how should we combine multiple reward models? Using a probabilistic interpretation of the alignment procedure, we identify a natural choice for"},"claims":{"count":0,"items":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"2402.00742","kind":"arxiv","version":2},"verdict":{"id":null,"model_set":{},"created_at":null,"strongest_claim":"","one_line_summary":"","pipeline_version":null,"weakest_assumption":"","pith_extraction_headline":""},"integrity":{"clean":true,"summary":{"advisory":0,"critical":0,"by_detector":{},"informational":0},"endpoint":"/pith/2402.00742/integrity.json","findings":[],"available":true,"detectors_run":[],"snapshot_sha256":"c28c3603d3b5d939e8dc4c7e95fa8dfce3d595e45f758748cecf8e644a296938"},"references":{"count":0,"sample":[],"resolved_work":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","internal_anchors":0},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"author_claims":{"count":0,"strong_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"builder_version":"pith-number-builder-2026-05-17-v1"},"verdict_id":null},"signer":{"signer_id":"pith.science","signer_type":"pith_registry","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"created_at":"2026-07-05T08:45:49Z","supersedes":[],"prev_event":null,"signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"1XL3uvry0hZ4tdERvzt+/jWkPIynnShtTTPvsP3mu6mkRbDKpNAFUAsH18oEaa2T68BrUWFKm8ZVE5qndQKjAg==","signed_message":"open_graph_event_sha256_bytes","signed_at":"2026-08-08T21:34:41.956066Z"},"content_sha256":"9aac1d3abff76037dcef54cdd35f6dddf6b3a2a048850f2cd8a8a105680cb8c9","schema_version":"1.0","event_id":"sha256:9aac1d3abff76037dcef54cdd35f6dddf6b3a2a048850f2cd8a8a105680cb8c9"}],"timestamp_proofs":[],"mirror_hints":[{"mirror_type":"https","name":"Pith Resolver","base_url":"https://pith.science","bundle_url":"https://pith.science/pith/YKI4JOUUOKNSKZHJKFOSJHOOZZ/bundle.json","state_url":"https://pith.science/pith/YKI4JOUUOKNSKZHJKFOSJHOOZZ/state.json","well_known_bundle_url":"https://pith.science/.well-known/pith/YKI4JOUUOKNSKZHJKFOSJHOOZZ/bundle.json","status":"primary"}],"public_keys":[{"key_id":"pith-v1-2026-05","algorithm":"ed25519","format":"raw","public_key_b64":"stVStoiQhXFxp4s2pdzPNoqVNBMojDU/fJ2db5S3CbM=","public_key_hex":"b2d552b68890857171a78b36a5dccf368a953413288c353f7c9d9d6f94b709b3","fingerprint_sha256_b32_first128bits":"RVFV5Z2OI2J3ZUO7ERDEBCYNKS","fingerprint_sha256_hex":"8d4b5ee74e4693bcd1df2446408b0d54","rotates_at":null,"url":"https://pith.science/pith-signing-key.json","notes":"Pith uses this Ed25519 key to sign canonical record SHA-256 digests. Verify with: ed25519_verify(public_key, message=canonical_sha256_bytes, signature=base64decode(signature_b64))."}],"merge_version":"pith-open-graph-merge-v1","built_at":"2026-08-08T21:34:41Z","links":{"resolver":"https://pith.science/pith/YKI4JOUUOKNSKZHJKFOSJHOOZZ","bundle":"https://pith.science/pith/YKI4JOUUOKNSKZHJKFOSJHOOZZ/bundle.json","state":"https://pith.science/pith/YKI4JOUUOKNSKZHJKFOSJHOOZZ/state.json","well_known_bundle":"https://pith.science/.well-known/pith/YKI4JOUUOKNSKZHJKFOSJHOOZZ/bundle.json"},"state":{"state_type":"pith_open_graph_state","state_version":"1.0","pith_number":"pith:2024:YKI4JOUUOKNSKZHJKFOSJHOOZZ","merge_version":"pith-open-graph-merge-v1","event_count":2,"valid_event_count":2,"invalid_event_count":0,"equivocation_count":0,"current":{"canonical_record":{"metadata":{"abstract_canon_sha256":"6aff8462b2d6ede8c1b86cda4b9bb747470734951cbef06ad8ac19d1ab2879d3","cross_cats_sorted":["cs.AI"],"license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.CL","submitted_at":"2024-02-01T16:39:28Z","title_canon_sha256":"25da635e53ed4ecf84da86c4cc307e90124b13a554eb0fa05d0cfb8dda896ea6"},"schema_version":"1.0","source":{"id":"2402.00742","kind":"arxiv","version":2}},"source_aliases":[{"alias_kind":"arxiv","alias_value":"2402.00742","created_at":"2026-07-05T08:45:49Z"},{"alias_kind":"arxiv_version","alias_value":"2402.00742v2","created_at":"2026-07-05T08:45:49Z"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.2402.00742","created_at":"2026-07-05T08:45:49Z"},{"alias_kind":"pith_short_12","alias_value":"YKI4JOUUOKNS","created_at":"2026-07-05T08:45:49Z"},{"alias_kind":"pith_short_16","alias_value":"YKI4JOUUOKNSKZHJ","created_at":"2026-07-05T08:45:49Z"},{"alias_kind":"pith_short_8","alias_value":"YKI4JOUU","created_at":"2026-07-05T08:45:49Z"}],"graph_snapshots":[{"event_id":"sha256:9aac1d3abff76037dcef54cdd35f6dddf6b3a2a048850f2cd8a8a105680cb8c9","target":"graph","created_at":"2026-07-05T08:45:49Z","signer":{"key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signer_id":"pith.science","signer_type":"pith_registry"},"payload":{"graph_snapshot":{"author_claims":{"count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","strong_count":0},"builder_version":"pith-number-builder-2026-05-17-v1","claims":{"count":0,"items":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"integrity":{"available":true,"clean":true,"detectors_run":[],"endpoint":"/pith/2402.00742/integrity.json","findings":[],"snapshot_sha256":"c28c3603d3b5d939e8dc4c7e95fa8dfce3d595e45f758748cecf8e644a296938","summary":{"advisory":0,"by_detector":{},"critical":0,"informational":0}},"paper":{"abstract_excerpt":"A common approach for aligning language models to human preferences is to first learn a reward model from preference data, and then use this reward model to update the language model. We study two closely related problems that arise in this approach. First, any monotone transformation of the reward model preserves preference ranking; is there a choice that is ``better'' than others? Second, we often wish to align language models to multiple properties: how should we combine multiple reward models? Using a probabilistic interpretation of the alignment procedure, we identify a natural choice for","authors_text":"Alex D'Amour, Chirag Nagpal, Jacob Eisenstein, Jonathan Berant, Sanmi Koyejo, Victor Veitch, Zihao Wang","cross_cats":["cs.AI"],"headline":"","license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.CL","submitted_at":"2024-02-01T16:39:28Z","title":"Transforming and Combining Rewards for Aligning Large Language Models"},"references":{"count":0,"internal_anchors":0,"resolved_work":0,"sample":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"2402.00742","kind":"arxiv","version":2},"verdict":{"created_at":null,"id":null,"model_set":{},"one_line_summary":"","pipeline_version":null,"pith_extraction_headline":"","strongest_claim":"","weakest_assumption":""}},"verdict_id":null}}],"author_attestations":[],"timestamp_anchors":[],"storage_attestations":[],"citation_signatures":[],"replication_records":[],"corrections":[],"mirror_hints":[],"record_created":{"event_id":"sha256:79f034215941bb5f574677f57c4ae7adf900c321c03ab6ce88c74baf4a743590","target":"record","created_at":"2026-07-05T08:45:49Z","signer":{"key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signer_id":"pith.science","signer_type":"pith_registry"},"payload":{"attestation_state":"computed","canonical_record":{"metadata":{"abstract_canon_sha256":"6aff8462b2d6ede8c1b86cda4b9bb747470734951cbef06ad8ac19d1ab2879d3","cross_cats_sorted":["cs.AI"],"license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.CL","submitted_at":"2024-02-01T16:39:28Z","title_canon_sha256":"25da635e53ed4ecf84da86c4cc307e90124b13a554eb0fa05d0cfb8dda896ea6"},"schema_version":"1.0","source":{"id":"2402.00742","kind":"arxiv","version":2}},"canonical_sha256":"c291c4ba94729b2564e9515d249dcece46b585cf2a5ad5998db9de13a3e440e0","receipt":{"algorithm":"ed25519","builder_version":"pith-number-builder-2026-05-17-v1","canonical_sha256":"c291c4ba94729b2564e9515d249dcece46b585cf2a5ad5998db9de13a3e440e0","first_computed_at":"2026-07-05T08:45:49.867335Z","key_id":"pith-v1-2026-05","kind":"pith_receipt","last_reissued_at":"2026-07-05T08:45:49.867335Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","receipt_version":"0.3","signature_b64":"BSpWfHRHiiWEKObP4qYLi0XdFVEb35NohzniUoPxmbue/IS8UfvClV8wu60WKHMszMTKgpFMLnIu5tfTr0BzDA==","signature_status":"signed_v1","signed_at":"2026-07-05T08:45:49.867809Z","signed_message":"canonical_sha256_bytes"},"source_id":"2402.00742","source_kind":"arxiv","source_version":2}}},"equivocations":[],"invalid_events":[],"applied_event_ids":["sha256:79f034215941bb5f574677f57c4ae7adf900c321c03ab6ce88c74baf4a743590","sha256:9aac1d3abff76037dcef54cdd35f6dddf6b3a2a048850f2cd8a8a105680cb8c9"],"state_sha256":"2d509f2a6e80cfd020b65b245816477b1805a1842404facf1d5eb6b44b73cdb1"},"bundle_signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"N0g9ZidDg7ZgBcS7G0Rg21ra66OAV6kCyad6YfP8NyGT+oxoYkvKeWHqkGMGy5yh6l6PW98yL4l9a+nZvVaZDg==","signed_message":"bundle_sha256_bytes","signed_at":"2026-08-08T21:34:41.960909Z","bundle_sha256":"e320a1448b4304e9ddd7aa0253c4b212241189b5149cb027b5c8782076000b3f"}}