{"bundle_type":"pith_open_graph_bundle","bundle_version":"1.0","pith_number":"pith:2023:IV77PKGLF4ZPDSHMMBFJJGWFLS","short_pith_number":"pith:IV77PKGL","canonical_record":{"source":{"id":"2310.11564","kind":"arxiv","version":1},"metadata":{"license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.CL","submitted_at":"2023-10-17T20:22:13Z","cross_cats_sorted":[],"title_canon_sha256":"617497f27a9edb91d92afa2f2d10b4ba1ccc1f3f23e75d66ed10865fd1ac7503","abstract_canon_sha256":"0e76bdd04b06fdc6424e58b9ec7dd68be89cfc4d268093813ba5db99827edf1e"},"schema_version":"1.0"},"canonical_sha256":"457ff7a8cb2f32f1c8ec604a949ac55c9d8227becb804ed648796e5a0fd8bdfa","source":{"kind":"arxiv","id":"2310.11564","version":1},"source_aliases":[{"alias_kind":"arxiv","alias_value":"2310.11564","created_at":"2026-07-05T07:02:09Z"},{"alias_kind":"arxiv_version","alias_value":"2310.11564v1","created_at":"2026-07-05T07:02:09Z"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.2310.11564","created_at":"2026-07-05T07:02:09Z"},{"alias_kind":"pith_short_12","alias_value":"IV77PKGLF4ZP","created_at":"2026-07-05T07:02:09Z"},{"alias_kind":"pith_short_16","alias_value":"IV77PKGLF4ZPDSHM","created_at":"2026-07-05T07:02:09Z"},{"alias_kind":"pith_short_8","alias_value":"IV77PKGL","created_at":"2026-07-05T07:02:09Z"}],"events":[{"event_type":"record_created","subject_pith_number":"pith:2023:IV77PKGLF4ZPDSHMMBFJJGWFLS","target":"record","payload":{"canonical_record":{"source":{"id":"2310.11564","kind":"arxiv","version":1},"metadata":{"license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.CL","submitted_at":"2023-10-17T20:22:13Z","cross_cats_sorted":[],"title_canon_sha256":"617497f27a9edb91d92afa2f2d10b4ba1ccc1f3f23e75d66ed10865fd1ac7503","abstract_canon_sha256":"0e76bdd04b06fdc6424e58b9ec7dd68be89cfc4d268093813ba5db99827edf1e"},"schema_version":"1.0"},"canonical_sha256":"457ff7a8cb2f32f1c8ec604a949ac55c9d8227becb804ed648796e5a0fd8bdfa","receipt":{"kind":"pith_receipt","key_id":"pith-v1-2026-05","algorithm":"ed25519","signed_at":"2026-07-05T07:02:09.123560Z","signature_b64":"v78lYODaTXzViSYJ8/2t6grJL4PJgiW3BXQBpEA35uobHDJtt98KZHybOcicQKYrfsw+Ou/A23+/WQx/4nO7DA==","signed_message":"canonical_sha256_bytes","builder_version":"pith-number-builder-2026-05-17-v1","receipt_version":"0.3","canonical_sha256":"457ff7a8cb2f32f1c8ec604a949ac55c9d8227becb804ed648796e5a0fd8bdfa","last_reissued_at":"2026-07-05T07:02:09.123199Z","signature_status":"signed_v1","first_computed_at":"2026-07-05T07:02:09.123199Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"source_kind":"arxiv","source_id":"2310.11564","source_version":1,"attestation_state":"computed"},"signer":{"signer_id":"pith.science","signer_type":"pith_registry","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"created_at":"2026-07-05T07:02:09Z","supersedes":[],"prev_event":null,"signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"TBWxKuSsik1xx8/bKlnRaFEz6Xqgy6kbGHn0+xdmpPo8Hml2OAfj+MRrvxQr3EOpdyR8d826fAjJjX5pMQoVDA==","signed_message":"open_graph_event_sha256_bytes","signed_at":"2026-07-15T08:37:45.280297Z"},"content_sha256":"fd6e4d0d98d4a368a20d67520f553154c403a2a464c83b0d20b8f8c90ae99760","schema_version":"1.0","event_id":"sha256:fd6e4d0d98d4a368a20d67520f553154c403a2a464c83b0d20b8f8c90ae99760"},{"event_type":"graph_snapshot","subject_pith_number":"pith:2023:IV77PKGLF4ZPDSHMMBFJJGWFLS","target":"graph","payload":{"graph_snapshot":{"paper":{"title":"Personalized Soups: Personalized Large Language Model Alignment via Post-hoc Parameter Merging","license":"http://creativecommons.org/licenses/by/4.0/","headline":"","cross_cats":[],"primary_cat":"cs.CL","authors_text":"Bill Yuchen Lin, Hannaneh Hajishirzi, Jack Hessel, Joel Jang, Luke Zettlemoyer, Prithviraj Ammanabrolu, Seungone Kim, Yejin Choi, Yizhong Wang","submitted_at":"2023-10-17T20:22:13Z","abstract_excerpt":"While Reinforcement Learning from Human Feedback (RLHF) aligns Large Language Models (LLMs) with general, aggregate human preferences, it is suboptimal for learning diverse, individual perspectives. In this work, we study Reinforcement Learning from Personalized Human Feedback (RLPHF) problem, wherein LLMs are aligned to multiple (sometimes conflicting) preferences by modeling alignment as a Multi-Objective Reinforcement Learning (MORL) problem. Compared to strong single-objective baselines, we show that we can achieve personalized alignment by decomposing preferences into multiple dimensions."},"claims":{"count":0,"items":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"2310.11564","kind":"arxiv","version":1},"verdict":{"id":null,"model_set":{},"created_at":null,"strongest_claim":"","one_line_summary":"","pipeline_version":null,"weakest_assumption":"","pith_extraction_headline":""},"integrity":{"clean":true,"summary":{"advisory":0,"critical":0,"by_detector":{},"informational":0},"endpoint":"/pith/2310.11564/integrity.json","findings":[],"available":true,"detectors_run":[],"snapshot_sha256":"c28c3603d3b5d939e8dc4c7e95fa8dfce3d595e45f758748cecf8e644a296938"},"references":{"count":0,"sample":[],"resolved_work":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","internal_anchors":0},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"author_claims":{"count":0,"strong_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"builder_version":"pith-number-builder-2026-05-17-v1"},"verdict_id":null},"signer":{"signer_id":"pith.science","signer_type":"pith_registry","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"created_at":"2026-07-05T07:02:09Z","supersedes":[],"prev_event":null,"signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"Vm3RQBSadS9JaPso2UUubhV167FxWD4vqOE9o5JR2XL5e2OPiw/JNaSNp4873oDKAwjlYNsHDOXn4wOeNZMuDg==","signed_message":"open_graph_event_sha256_bytes","signed_at":"2026-07-15T08:37:45.280709Z"},"content_sha256":"67bece12c7d3298075483efd6fd201146acf9dc8e05cb22fd35fb91e0d975c2b","schema_version":"1.0","event_id":"sha256:67bece12c7d3298075483efd6fd201146acf9dc8e05cb22fd35fb91e0d975c2b"}],"timestamp_proofs":[],"mirror_hints":[{"mirror_type":"https","name":"Pith Resolver","base_url":"https://pith.science","bundle_url":"https://pith.science/pith/IV77PKGLF4ZPDSHMMBFJJGWFLS/bundle.json","state_url":"https://pith.science/pith/IV77PKGLF4ZPDSHMMBFJJGWFLS/state.json","well_known_bundle_url":"https://pith.science/.well-known/pith/IV77PKGLF4ZPDSHMMBFJJGWFLS/bundle.json","status":"primary"}],"public_keys":[{"key_id":"pith-v1-2026-05","algorithm":"ed25519","format":"raw","public_key_b64":"stVStoiQhXFxp4s2pdzPNoqVNBMojDU/fJ2db5S3CbM=","public_key_hex":"b2d552b68890857171a78b36a5dccf368a953413288c353f7c9d9d6f94b709b3","fingerprint_sha256_b32_first128bits":"RVFV5Z2OI2J3ZUO7ERDEBCYNKS","fingerprint_sha256_hex":"8d4b5ee74e4693bcd1df2446408b0d54","rotates_at":null,"url":"https://pith.science/pith-signing-key.json","notes":"Pith uses this Ed25519 key to sign canonical record SHA-256 digests. Verify with: ed25519_verify(public_key, message=canonical_sha256_bytes, signature=base64decode(signature_b64))."}],"merge_version":"pith-open-graph-merge-v1","built_at":"2026-07-15T08:37:45Z","links":{"resolver":"https://pith.science/pith/IV77PKGLF4ZPDSHMMBFJJGWFLS","bundle":"https://pith.science/pith/IV77PKGLF4ZPDSHMMBFJJGWFLS/bundle.json","state":"https://pith.science/pith/IV77PKGLF4ZPDSHMMBFJJGWFLS/state.json","well_known_bundle":"https://pith.science/.well-known/pith/IV77PKGLF4ZPDSHMMBFJJGWFLS/bundle.json"},"state":{"state_type":"pith_open_graph_state","state_version":"1.0","pith_number":"pith:2023:IV77PKGLF4ZPDSHMMBFJJGWFLS","merge_version":"pith-open-graph-merge-v1","event_count":2,"valid_event_count":2,"invalid_event_count":0,"equivocation_count":0,"current":{"canonical_record":{"metadata":{"abstract_canon_sha256":"0e76bdd04b06fdc6424e58b9ec7dd68be89cfc4d268093813ba5db99827edf1e","cross_cats_sorted":[],"license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.CL","submitted_at":"2023-10-17T20:22:13Z","title_canon_sha256":"617497f27a9edb91d92afa2f2d10b4ba1ccc1f3f23e75d66ed10865fd1ac7503"},"schema_version":"1.0","source":{"id":"2310.11564","kind":"arxiv","version":1}},"source_aliases":[{"alias_kind":"arxiv","alias_value":"2310.11564","created_at":"2026-07-05T07:02:09Z"},{"alias_kind":"arxiv_version","alias_value":"2310.11564v1","created_at":"2026-07-05T07:02:09Z"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.2310.11564","created_at":"2026-07-05T07:02:09Z"},{"alias_kind":"pith_short_12","alias_value":"IV77PKGLF4ZP","created_at":"2026-07-05T07:02:09Z"},{"alias_kind":"pith_short_16","alias_value":"IV77PKGLF4ZPDSHM","created_at":"2026-07-05T07:02:09Z"},{"alias_kind":"pith_short_8","alias_value":"IV77PKGL","created_at":"2026-07-05T07:02:09Z"}],"graph_snapshots":[{"event_id":"sha256:67bece12c7d3298075483efd6fd201146acf9dc8e05cb22fd35fb91e0d975c2b","target":"graph","created_at":"2026-07-05T07:02:09Z","signer":{"key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signer_id":"pith.science","signer_type":"pith_registry"},"payload":{"graph_snapshot":{"author_claims":{"count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","strong_count":0},"builder_version":"pith-number-builder-2026-05-17-v1","claims":{"count":0,"items":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"integrity":{"available":true,"clean":true,"detectors_run":[],"endpoint":"/pith/2310.11564/integrity.json","findings":[],"snapshot_sha256":"c28c3603d3b5d939e8dc4c7e95fa8dfce3d595e45f758748cecf8e644a296938","summary":{"advisory":0,"by_detector":{},"critical":0,"informational":0}},"paper":{"abstract_excerpt":"While Reinforcement Learning from Human Feedback (RLHF) aligns Large Language Models (LLMs) with general, aggregate human preferences, it is suboptimal for learning diverse, individual perspectives. In this work, we study Reinforcement Learning from Personalized Human Feedback (RLPHF) problem, wherein LLMs are aligned to multiple (sometimes conflicting) preferences by modeling alignment as a Multi-Objective Reinforcement Learning (MORL) problem. Compared to strong single-objective baselines, we show that we can achieve personalized alignment by decomposing preferences into multiple dimensions.","authors_text":"Bill Yuchen Lin, Hannaneh Hajishirzi, Jack Hessel, Joel Jang, Luke Zettlemoyer, Prithviraj Ammanabrolu, Seungone Kim, Yejin Choi, Yizhong Wang","cross_cats":[],"headline":"","license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.CL","submitted_at":"2023-10-17T20:22:13Z","title":"Personalized Soups: Personalized Large Language Model Alignment via Post-hoc Parameter Merging"},"references":{"count":0,"internal_anchors":0,"resolved_work":0,"sample":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"2310.11564","kind":"arxiv","version":1},"verdict":{"created_at":null,"id":null,"model_set":{},"one_line_summary":"","pipeline_version":null,"pith_extraction_headline":"","strongest_claim":"","weakest_assumption":""}},"verdict_id":null}}],"author_attestations":[],"timestamp_anchors":[],"storage_attestations":[],"citation_signatures":[],"replication_records":[],"corrections":[],"mirror_hints":[],"record_created":{"event_id":"sha256:fd6e4d0d98d4a368a20d67520f553154c403a2a464c83b0d20b8f8c90ae99760","target":"record","created_at":"2026-07-05T07:02:09Z","signer":{"key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signer_id":"pith.science","signer_type":"pith_registry"},"payload":{"attestation_state":"computed","canonical_record":{"metadata":{"abstract_canon_sha256":"0e76bdd04b06fdc6424e58b9ec7dd68be89cfc4d268093813ba5db99827edf1e","cross_cats_sorted":[],"license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.CL","submitted_at":"2023-10-17T20:22:13Z","title_canon_sha256":"617497f27a9edb91d92afa2f2d10b4ba1ccc1f3f23e75d66ed10865fd1ac7503"},"schema_version":"1.0","source":{"id":"2310.11564","kind":"arxiv","version":1}},"canonical_sha256":"457ff7a8cb2f32f1c8ec604a949ac55c9d8227becb804ed648796e5a0fd8bdfa","receipt":{"algorithm":"ed25519","builder_version":"pith-number-builder-2026-05-17-v1","canonical_sha256":"457ff7a8cb2f32f1c8ec604a949ac55c9d8227becb804ed648796e5a0fd8bdfa","first_computed_at":"2026-07-05T07:02:09.123199Z","key_id":"pith-v1-2026-05","kind":"pith_receipt","last_reissued_at":"2026-07-05T07:02:09.123199Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","receipt_version":"0.3","signature_b64":"v78lYODaTXzViSYJ8/2t6grJL4PJgiW3BXQBpEA35uobHDJtt98KZHybOcicQKYrfsw+Ou/A23+/WQx/4nO7DA==","signature_status":"signed_v1","signed_at":"2026-07-05T07:02:09.123560Z","signed_message":"canonical_sha256_bytes"},"source_id":"2310.11564","source_kind":"arxiv","source_version":1}}},"equivocations":[],"invalid_events":[],"applied_event_ids":["sha256:fd6e4d0d98d4a368a20d67520f553154c403a2a464c83b0d20b8f8c90ae99760","sha256:67bece12c7d3298075483efd6fd201146acf9dc8e05cb22fd35fb91e0d975c2b"],"state_sha256":"3fefbbab4d0c3d57733c158a6350d0183cad31547b257f72d3f171dde7d98aa4"},"bundle_signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"iWBuUmvmcRnoCyrMt80IOhdt0wjUNv1cOO2eqVjtSpIIFwwGSBcKWcsz5ua+ziwoNVVGQGpiAa6Y1bZvdGVRCw==","signed_message":"bundle_sha256_bytes","signed_at":"2026-07-15T08:37:45.282798Z","bundle_sha256":"e4b38036aa6fb94b32162208dfaa416dd87bcdcc2c958dff42b4f46bd6ce74a6"}}