{"bundle_type":"pith_open_graph_bundle","bundle_version":"1.0","pith_number":"pith:2023:EVCKQQYYDVUQUROJNB2IUXTKFG","short_pith_number":"pith:EVCKQQYY","canonical_record":{"source":{"id":"2306.04488","kind":"arxiv","version":2},"metadata":{"license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","primary_cat":"cs.LG","submitted_at":"2023-06-07T14:58:15Z","cross_cats_sorted":["cs.AI","cs.CV"],"title_canon_sha256":"26c339f847aece3d6e106a841627fdb62335a57c90a526972b9bcc7833220405","abstract_canon_sha256":"cabacdfc127ffc4d4b0a6d2b89dd7f81d3e4a4c41261f733d32778fa9254f4c2"},"schema_version":"1.0"},"canonical_sha256":"2544a843181d690a45c968748a5e6a29b7902d5016512bc568ca60d63cc9dcc1","source":{"kind":"arxiv","id":"2306.04488","version":2},"source_aliases":[{"alias_kind":"arxiv","alias_value":"2306.04488","created_at":"2026-07-05T07:01:02Z"},{"alias_kind":"arxiv_version","alias_value":"2306.04488v2","created_at":"2026-07-05T07:01:02Z"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.2306.04488","created_at":"2026-07-05T07:01:02Z"},{"alias_kind":"pith_short_12","alias_value":"EVCKQQYYDVUQ","created_at":"2026-07-05T07:01:02Z"},{"alias_kind":"pith_short_16","alias_value":"EVCKQQYYDVUQUROJ","created_at":"2026-07-05T07:01:02Z"},{"alias_kind":"pith_short_8","alias_value":"EVCKQQYY","created_at":"2026-07-05T07:01:02Z"}],"events":[{"event_type":"record_created","subject_pith_number":"pith:2023:EVCKQQYYDVUQUROJNB2IUXTKFG","target":"record","payload":{"canonical_record":{"source":{"id":"2306.04488","kind":"arxiv","version":2},"metadata":{"license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","primary_cat":"cs.LG","submitted_at":"2023-06-07T14:58:15Z","cross_cats_sorted":["cs.AI","cs.CV"],"title_canon_sha256":"26c339f847aece3d6e106a841627fdb62335a57c90a526972b9bcc7833220405","abstract_canon_sha256":"cabacdfc127ffc4d4b0a6d2b89dd7f81d3e4a4c41261f733d32778fa9254f4c2"},"schema_version":"1.0"},"canonical_sha256":"2544a843181d690a45c968748a5e6a29b7902d5016512bc568ca60d63cc9dcc1","receipt":{"kind":"pith_receipt","key_id":"pith-v1-2026-05","algorithm":"ed25519","signed_at":"2026-07-05T07:01:02.525934Z","signature_b64":"pwCZAFDw829z7zE8JF426JuROY6vK1UqLQIIeOyMkfBCDigPv97545Y1ZLx5OPHiVooA3bUJV58tPUKMuVBzCA==","signed_message":"canonical_sha256_bytes","builder_version":"pith-number-builder-2026-05-17-v1","receipt_version":"0.3","canonical_sha256":"2544a843181d690a45c968748a5e6a29b7902d5016512bc568ca60d63cc9dcc1","last_reissued_at":"2026-07-05T07:01:02.525558Z","signature_status":"signed_v1","first_computed_at":"2026-07-05T07:01:02.525558Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"source_kind":"arxiv","source_id":"2306.04488","source_version":2,"attestation_state":"computed"},"signer":{"signer_id":"pith.science","signer_type":"pith_registry","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"created_at":"2026-07-05T07:01:02Z","supersedes":[],"prev_event":null,"signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"SjMdS8pK1/6bsYYAL3V6HvjZ+ueDqz+mOs2eujUn+qqx4QKNRVitI2IaofVP5yQGBBZq2R7GwVKLVvNVkCUQDw==","signed_message":"open_graph_event_sha256_bytes","signed_at":"2026-07-07T14:43:31.977736Z"},"content_sha256":"8cf0636a05b663bc5244dca7724d1f6b762c9aafd8208646ea2f1491055c98ca","schema_version":"1.0","event_id":"sha256:8cf0636a05b663bc5244dca7724d1f6b762c9aafd8208646ea2f1491055c98ca"},{"event_type":"graph_snapshot","subject_pith_number":"pith:2023:EVCKQQYYDVUQUROJNB2IUXTKFG","target":"graph","payload":{"graph_snapshot":{"paper":{"title":"Rewarded soups: towards Pareto-optimal alignment by interpolating weights fine-tuned on diverse rewards","license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","headline":"","cross_cats":["cs.AI","cs.CV"],"primary_cat":"cs.LG","authors_text":"Alexandre Ram\\'e, Corentin Dancette, Guillaume Couairon, Jean-Baptiste Gaya, Laure Soulier, Matthieu Cord, Mustafa Shukor","submitted_at":"2023-06-07T14:58:15Z","abstract_excerpt":"Foundation models are first pre-trained on vast unsupervised datasets and then fine-tuned on labeled data. Reinforcement learning, notably from human feedback (RLHF), can further align the network with the intended usage. Yet the imperfections in the proxy reward may hinder the training and lead to suboptimal results; the diversity of objectives in real-world tasks and human opinions exacerbate the issue. This paper proposes embracing the heterogeneity of diverse rewards by following a multi-policy strategy. Rather than focusing on a single a priori reward, we aim for Pareto-optimal generaliza"},"claims":{"count":0,"items":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"2306.04488","kind":"arxiv","version":2},"verdict":{"id":null,"model_set":{},"created_at":null,"strongest_claim":"","one_line_summary":"","pipeline_version":null,"weakest_assumption":"","pith_extraction_headline":""},"integrity":{"clean":true,"summary":{"advisory":0,"critical":0,"by_detector":{},"informational":0},"endpoint":"/pith/2306.04488/integrity.json","findings":[],"available":true,"detectors_run":[],"snapshot_sha256":"c28c3603d3b5d939e8dc4c7e95fa8dfce3d595e45f758748cecf8e644a296938"},"references":{"count":0,"sample":[],"resolved_work":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","internal_anchors":0},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"author_claims":{"count":0,"strong_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"builder_version":"pith-number-builder-2026-05-17-v1"},"verdict_id":null},"signer":{"signer_id":"pith.science","signer_type":"pith_registry","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"created_at":"2026-07-05T07:01:02Z","supersedes":[],"prev_event":null,"signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"I/uz0UmZAvDm0w+Pxvg6KKNgIhnaZqyvdTIBvkWcCIPDEAnlSnElJzy3am+3KpvtFOwgAFgasWh4wCPUCQmlAg==","signed_message":"open_graph_event_sha256_bytes","signed_at":"2026-07-07T14:43:31.978111Z"},"content_sha256":"728f5b1e267055b965daa8ae12b8b07e4639d8d0f725da406fa9bfb7a0e13ad5","schema_version":"1.0","event_id":"sha256:728f5b1e267055b965daa8ae12b8b07e4639d8d0f725da406fa9bfb7a0e13ad5"}],"timestamp_proofs":[],"mirror_hints":[{"mirror_type":"https","name":"Pith Resolver","base_url":"https://pith.science","bundle_url":"https://pith.science/pith/EVCKQQYYDVUQUROJNB2IUXTKFG/bundle.json","state_url":"https://pith.science/pith/EVCKQQYYDVUQUROJNB2IUXTKFG/state.json","well_known_bundle_url":"https://pith.science/.well-known/pith/EVCKQQYYDVUQUROJNB2IUXTKFG/bundle.json","status":"primary"}],"public_keys":[{"key_id":"pith-v1-2026-05","algorithm":"ed25519","format":"raw","public_key_b64":"stVStoiQhXFxp4s2pdzPNoqVNBMojDU/fJ2db5S3CbM=","public_key_hex":"b2d552b68890857171a78b36a5dccf368a953413288c353f7c9d9d6f94b709b3","fingerprint_sha256_b32_first128bits":"RVFV5Z2OI2J3ZUO7ERDEBCYNKS","fingerprint_sha256_hex":"8d4b5ee74e4693bcd1df2446408b0d54","rotates_at":null,"url":"https://pith.science/pith-signing-key.json","notes":"Pith uses this Ed25519 key to sign canonical record SHA-256 digests. Verify with: ed25519_verify(public_key, message=canonical_sha256_bytes, signature=base64decode(signature_b64))."}],"merge_version":"pith-open-graph-merge-v1","built_at":"2026-07-07T14:43:31Z","links":{"resolver":"https://pith.science/pith/EVCKQQYYDVUQUROJNB2IUXTKFG","bundle":"https://pith.science/pith/EVCKQQYYDVUQUROJNB2IUXTKFG/bundle.json","state":"https://pith.science/pith/EVCKQQYYDVUQUROJNB2IUXTKFG/state.json","well_known_bundle":"https://pith.science/.well-known/pith/EVCKQQYYDVUQUROJNB2IUXTKFG/bundle.json"},"state":{"state_type":"pith_open_graph_state","state_version":"1.0","pith_number":"pith:2023:EVCKQQYYDVUQUROJNB2IUXTKFG","merge_version":"pith-open-graph-merge-v1","event_count":2,"valid_event_count":2,"invalid_event_count":0,"equivocation_count":0,"current":{"canonical_record":{"metadata":{"abstract_canon_sha256":"cabacdfc127ffc4d4b0a6d2b89dd7f81d3e4a4c41261f733d32778fa9254f4c2","cross_cats_sorted":["cs.AI","cs.CV"],"license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","primary_cat":"cs.LG","submitted_at":"2023-06-07T14:58:15Z","title_canon_sha256":"26c339f847aece3d6e106a841627fdb62335a57c90a526972b9bcc7833220405"},"schema_version":"1.0","source":{"id":"2306.04488","kind":"arxiv","version":2}},"source_aliases":[{"alias_kind":"arxiv","alias_value":"2306.04488","created_at":"2026-07-05T07:01:02Z"},{"alias_kind":"arxiv_version","alias_value":"2306.04488v2","created_at":"2026-07-05T07:01:02Z"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.2306.04488","created_at":"2026-07-05T07:01:02Z"},{"alias_kind":"pith_short_12","alias_value":"EVCKQQYYDVUQ","created_at":"2026-07-05T07:01:02Z"},{"alias_kind":"pith_short_16","alias_value":"EVCKQQYYDVUQUROJ","created_at":"2026-07-05T07:01:02Z"},{"alias_kind":"pith_short_8","alias_value":"EVCKQQYY","created_at":"2026-07-05T07:01:02Z"}],"graph_snapshots":[{"event_id":"sha256:728f5b1e267055b965daa8ae12b8b07e4639d8d0f725da406fa9bfb7a0e13ad5","target":"graph","created_at":"2026-07-05T07:01:02Z","signer":{"key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signer_id":"pith.science","signer_type":"pith_registry"},"payload":{"graph_snapshot":{"author_claims":{"count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","strong_count":0},"builder_version":"pith-number-builder-2026-05-17-v1","claims":{"count":0,"items":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"integrity":{"available":true,"clean":true,"detectors_run":[],"endpoint":"/pith/2306.04488/integrity.json","findings":[],"snapshot_sha256":"c28c3603d3b5d939e8dc4c7e95fa8dfce3d595e45f758748cecf8e644a296938","summary":{"advisory":0,"by_detector":{},"critical":0,"informational":0}},"paper":{"abstract_excerpt":"Foundation models are first pre-trained on vast unsupervised datasets and then fine-tuned on labeled data. Reinforcement learning, notably from human feedback (RLHF), can further align the network with the intended usage. Yet the imperfections in the proxy reward may hinder the training and lead to suboptimal results; the diversity of objectives in real-world tasks and human opinions exacerbate the issue. This paper proposes embracing the heterogeneity of diverse rewards by following a multi-policy strategy. Rather than focusing on a single a priori reward, we aim for Pareto-optimal generaliza","authors_text":"Alexandre Ram\\'e, Corentin Dancette, Guillaume Couairon, Jean-Baptiste Gaya, Laure Soulier, Matthieu Cord, Mustafa Shukor","cross_cats":["cs.AI","cs.CV"],"headline":"","license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","primary_cat":"cs.LG","submitted_at":"2023-06-07T14:58:15Z","title":"Rewarded soups: towards Pareto-optimal alignment by interpolating weights fine-tuned on diverse rewards"},"references":{"count":0,"internal_anchors":0,"resolved_work":0,"sample":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"2306.04488","kind":"arxiv","version":2},"verdict":{"created_at":null,"id":null,"model_set":{},"one_line_summary":"","pipeline_version":null,"pith_extraction_headline":"","strongest_claim":"","weakest_assumption":""}},"verdict_id":null}}],"author_attestations":[],"timestamp_anchors":[],"storage_attestations":[],"citation_signatures":[],"replication_records":[],"corrections":[],"mirror_hints":[],"record_created":{"event_id":"sha256:8cf0636a05b663bc5244dca7724d1f6b762c9aafd8208646ea2f1491055c98ca","target":"record","created_at":"2026-07-05T07:01:02Z","signer":{"key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signer_id":"pith.science","signer_type":"pith_registry"},"payload":{"attestation_state":"computed","canonical_record":{"metadata":{"abstract_canon_sha256":"cabacdfc127ffc4d4b0a6d2b89dd7f81d3e4a4c41261f733d32778fa9254f4c2","cross_cats_sorted":["cs.AI","cs.CV"],"license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","primary_cat":"cs.LG","submitted_at":"2023-06-07T14:58:15Z","title_canon_sha256":"26c339f847aece3d6e106a841627fdb62335a57c90a526972b9bcc7833220405"},"schema_version":"1.0","source":{"id":"2306.04488","kind":"arxiv","version":2}},"canonical_sha256":"2544a843181d690a45c968748a5e6a29b7902d5016512bc568ca60d63cc9dcc1","receipt":{"algorithm":"ed25519","builder_version":"pith-number-builder-2026-05-17-v1","canonical_sha256":"2544a843181d690a45c968748a5e6a29b7902d5016512bc568ca60d63cc9dcc1","first_computed_at":"2026-07-05T07:01:02.525558Z","key_id":"pith-v1-2026-05","kind":"pith_receipt","last_reissued_at":"2026-07-05T07:01:02.525558Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","receipt_version":"0.3","signature_b64":"pwCZAFDw829z7zE8JF426JuROY6vK1UqLQIIeOyMkfBCDigPv97545Y1ZLx5OPHiVooA3bUJV58tPUKMuVBzCA==","signature_status":"signed_v1","signed_at":"2026-07-05T07:01:02.525934Z","signed_message":"canonical_sha256_bytes"},"source_id":"2306.04488","source_kind":"arxiv","source_version":2}}},"equivocations":[],"invalid_events":[],"applied_event_ids":["sha256:8cf0636a05b663bc5244dca7724d1f6b762c9aafd8208646ea2f1491055c98ca","sha256:728f5b1e267055b965daa8ae12b8b07e4639d8d0f725da406fa9bfb7a0e13ad5"],"state_sha256":"beea3f3d4c3e037b9c7ed61447f68b08e235b55a4f68650eed214c53f8e6724a"},"bundle_signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"E+RiUk01MjBzsDEDIJWG6RiOkHMqyq5CGDb/7buQ6Loblyq73sky8TcyVlfClT2N7/DHYVFaHIFQgeWlqPpOBA==","signed_message":"bundle_sha256_bytes","signed_at":"2026-07-07T14:43:31.980018Z","bundle_sha256":"19a59041240c22b07ade4952f14ba505d1e82b165d8dd54deea70e4b240c6fd9"}}