{"bundle_type":"pith_open_graph_bundle","bundle_version":"1.0","pith_number":"pith:2024:BXUE7VOQG4R4L7H775PEWR4CPS","short_pith_number":"pith:BXUE7VOQ","canonical_record":{"source":{"id":"2405.00254","kind":"arxiv","version":2},"metadata":{"license":"http://creativecommons.org/licenses/by-sa/4.0/","primary_cat":"cs.AI","submitted_at":"2024-04-30T23:57:23Z","cross_cats_sorted":["cs.LG"],"title_canon_sha256":"ee5d02ec59d823d0d4a4ad7c680212961c7bffeb356874cd840f4c0352b4102c","abstract_canon_sha256":"ebf509ae404865ae07cd71eacfff34203f219771bd57d68fdc75cd9589451d5f"},"schema_version":"1.0"},"canonical_sha256":"0de84fd5d03723c5fcffff5e4b47827c9c45489d3266d5a3c91bbda055be55ee","source":{"kind":"arxiv","id":"2405.00254","version":2},"source_aliases":[{"alias_kind":"arxiv","alias_value":"2405.00254","created_at":"2026-07-05T08:23:22Z"},{"alias_kind":"arxiv_version","alias_value":"2405.00254v2","created_at":"2026-07-05T08:23:22Z"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.2405.00254","created_at":"2026-07-05T08:23:22Z"},{"alias_kind":"pith_short_12","alias_value":"BXUE7VOQG4R4","created_at":"2026-07-05T08:23:22Z"},{"alias_kind":"pith_short_16","alias_value":"BXUE7VOQG4R4L7H7","created_at":"2026-07-05T08:23:22Z"},{"alias_kind":"pith_short_8","alias_value":"BXUE7VOQ","created_at":"2026-07-05T08:23:22Z"}],"events":[{"event_type":"record_created","subject_pith_number":"pith:2024:BXUE7VOQG4R4L7H775PEWR4CPS","target":"record","payload":{"canonical_record":{"source":{"id":"2405.00254","kind":"arxiv","version":2},"metadata":{"license":"http://creativecommons.org/licenses/by-sa/4.0/","primary_cat":"cs.AI","submitted_at":"2024-04-30T23:57:23Z","cross_cats_sorted":["cs.LG"],"title_canon_sha256":"ee5d02ec59d823d0d4a4ad7c680212961c7bffeb356874cd840f4c0352b4102c","abstract_canon_sha256":"ebf509ae404865ae07cd71eacfff34203f219771bd57d68fdc75cd9589451d5f"},"schema_version":"1.0"},"canonical_sha256":"0de84fd5d03723c5fcffff5e4b47827c9c45489d3266d5a3c91bbda055be55ee","receipt":{"kind":"pith_receipt","key_id":"pith-v1-2026-05","algorithm":"ed25519","signed_at":"2026-07-05T08:23:22.077499Z","signature_b64":"TZYP45wJajceksNhzqQYNjFzyKsrkvBEqrV0GllbPex4cSfbtfBy5hFQ8B2TtjAOVlPmx1bg5s4BGtpTqPPqBw==","signed_message":"canonical_sha256_bytes","builder_version":"pith-number-builder-2026-05-17-v1","receipt_version":"0.3","canonical_sha256":"0de84fd5d03723c5fcffff5e4b47827c9c45489d3266d5a3c91bbda055be55ee","last_reissued_at":"2026-07-05T08:23:22.077004Z","signature_status":"signed_v1","first_computed_at":"2026-07-05T08:23:22.077004Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"source_kind":"arxiv","source_id":"2405.00254","source_version":2,"attestation_state":"computed"},"signer":{"signer_id":"pith.science","signer_type":"pith_registry","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"created_at":"2026-07-05T08:23:22Z","supersedes":[],"prev_event":null,"signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"aJgM/t96FRJOPxGXL7Sv5RorSB51XopwlBRMlACfFgAyqjGxUt4dOS5iYOhicVWk8xotibmWVOdVN/401W5oDA==","signed_message":"open_graph_event_sha256_bytes","signed_at":"2026-08-09T14:12:04.283503Z"},"content_sha256":"76b042fbeeffdca5639e4e4eaa2ff60b23e37b4b296b232b47d542c98497882b","schema_version":"1.0","event_id":"sha256:76b042fbeeffdca5639e4e4eaa2ff60b23e37b4b296b232b47d542c98497882b"},{"event_type":"graph_snapshot","subject_pith_number":"pith:2024:BXUE7VOQG4R4L7H775PEWR4CPS","target":"graph","payload":{"graph_snapshot":{"paper":{"title":"RLHF from Heterogeneous Feedback via Personalization and Preference Aggregation","license":"http://creativecommons.org/licenses/by-sa/4.0/","headline":"","cross_cats":["cs.LG"],"primary_cat":"cs.AI","authors_text":"Asuman Ozdaglar, Chanwoo Park, Dingwen Kong, Kaiqing Zhang, Mingyang Liu","submitted_at":"2024-04-30T23:57:23Z","abstract_excerpt":"Reinforcement learning from human feedback (RLHF) has been an effective technique for aligning AI systems with human values, with remarkable successes in fine-tuning large-language models recently. Most existing RLHF paradigms make the underlying assumption that human preferences are relatively homogeneous, and can be encoded by a single reward model. In this paper, we focus on addressing the issues due to the inherent heterogeneity in human preferences, as well as their potential strategic behavior in providing feedback. Specifically, we propose two frameworks to address heterogeneous human f"},"claims":{"count":0,"items":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"2405.00254","kind":"arxiv","version":2},"verdict":{"id":null,"model_set":{},"created_at":null,"strongest_claim":"","one_line_summary":"","pipeline_version":null,"weakest_assumption":"","pith_extraction_headline":""},"integrity":{"clean":true,"summary":{"advisory":0,"critical":0,"by_detector":{},"informational":0},"endpoint":"/pith/2405.00254/integrity.json","findings":[],"available":true,"detectors_run":[],"snapshot_sha256":"c28c3603d3b5d939e8dc4c7e95fa8dfce3d595e45f758748cecf8e644a296938"},"references":{"count":0,"sample":[],"resolved_work":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","internal_anchors":0},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"author_claims":{"count":0,"strong_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"builder_version":"pith-number-builder-2026-05-17-v1"},"verdict_id":null},"signer":{"signer_id":"pith.science","signer_type":"pith_registry","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"created_at":"2026-07-05T08:23:22Z","supersedes":[],"prev_event":null,"signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"KKD07B9fmtbtHy/f082jqjpJuSMqGBG1r03/Yth/y1PJ49b70xoKqTeXjzlBlNo6sfJb5u3WSS6UjfMhPZKUDA==","signed_message":"open_graph_event_sha256_bytes","signed_at":"2026-08-09T14:12:04.284044Z"},"content_sha256":"3549d7cae175d7aa16a159db71283b2af5056b9b302917d0ae079024b809481b","schema_version":"1.0","event_id":"sha256:3549d7cae175d7aa16a159db71283b2af5056b9b302917d0ae079024b809481b"}],"timestamp_proofs":[],"mirror_hints":[{"mirror_type":"https","name":"Pith Resolver","base_url":"https://pith.science","bundle_url":"https://pith.science/pith/BXUE7VOQG4R4L7H775PEWR4CPS/bundle.json","state_url":"https://pith.science/pith/BXUE7VOQG4R4L7H775PEWR4CPS/state.json","well_known_bundle_url":"https://pith.science/.well-known/pith/BXUE7VOQG4R4L7H775PEWR4CPS/bundle.json","status":"primary"}],"public_keys":[{"key_id":"pith-v1-2026-05","algorithm":"ed25519","format":"raw","public_key_b64":"stVStoiQhXFxp4s2pdzPNoqVNBMojDU/fJ2db5S3CbM=","public_key_hex":"b2d552b68890857171a78b36a5dccf368a953413288c353f7c9d9d6f94b709b3","fingerprint_sha256_b32_first128bits":"RVFV5Z2OI2J3ZUO7ERDEBCYNKS","fingerprint_sha256_hex":"8d4b5ee74e4693bcd1df2446408b0d54","rotates_at":null,"url":"https://pith.science/pith-signing-key.json","notes":"Pith uses this Ed25519 key to sign canonical record SHA-256 digests. Verify with: ed25519_verify(public_key, message=canonical_sha256_bytes, signature=base64decode(signature_b64))."}],"merge_version":"pith-open-graph-merge-v1","built_at":"2026-08-09T14:12:04Z","links":{"resolver":"https://pith.science/pith/BXUE7VOQG4R4L7H775PEWR4CPS","bundle":"https://pith.science/pith/BXUE7VOQG4R4L7H775PEWR4CPS/bundle.json","state":"https://pith.science/pith/BXUE7VOQG4R4L7H775PEWR4CPS/state.json","well_known_bundle":"https://pith.science/.well-known/pith/BXUE7VOQG4R4L7H775PEWR4CPS/bundle.json"},"state":{"state_type":"pith_open_graph_state","state_version":"1.0","pith_number":"pith:2024:BXUE7VOQG4R4L7H775PEWR4CPS","merge_version":"pith-open-graph-merge-v1","event_count":2,"valid_event_count":2,"invalid_event_count":0,"equivocation_count":0,"current":{"canonical_record":{"metadata":{"abstract_canon_sha256":"ebf509ae404865ae07cd71eacfff34203f219771bd57d68fdc75cd9589451d5f","cross_cats_sorted":["cs.LG"],"license":"http://creativecommons.org/licenses/by-sa/4.0/","primary_cat":"cs.AI","submitted_at":"2024-04-30T23:57:23Z","title_canon_sha256":"ee5d02ec59d823d0d4a4ad7c680212961c7bffeb356874cd840f4c0352b4102c"},"schema_version":"1.0","source":{"id":"2405.00254","kind":"arxiv","version":2}},"source_aliases":[{"alias_kind":"arxiv","alias_value":"2405.00254","created_at":"2026-07-05T08:23:22Z"},{"alias_kind":"arxiv_version","alias_value":"2405.00254v2","created_at":"2026-07-05T08:23:22Z"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.2405.00254","created_at":"2026-07-05T08:23:22Z"},{"alias_kind":"pith_short_12","alias_value":"BXUE7VOQG4R4","created_at":"2026-07-05T08:23:22Z"},{"alias_kind":"pith_short_16","alias_value":"BXUE7VOQG4R4L7H7","created_at":"2026-07-05T08:23:22Z"},{"alias_kind":"pith_short_8","alias_value":"BXUE7VOQ","created_at":"2026-07-05T08:23:22Z"}],"graph_snapshots":[{"event_id":"sha256:3549d7cae175d7aa16a159db71283b2af5056b9b302917d0ae079024b809481b","target":"graph","created_at":"2026-07-05T08:23:22Z","signer":{"key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signer_id":"pith.science","signer_type":"pith_registry"},"payload":{"graph_snapshot":{"author_claims":{"count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","strong_count":0},"builder_version":"pith-number-builder-2026-05-17-v1","claims":{"count":0,"items":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"integrity":{"available":true,"clean":true,"detectors_run":[],"endpoint":"/pith/2405.00254/integrity.json","findings":[],"snapshot_sha256":"c28c3603d3b5d939e8dc4c7e95fa8dfce3d595e45f758748cecf8e644a296938","summary":{"advisory":0,"by_detector":{},"critical":0,"informational":0}},"paper":{"abstract_excerpt":"Reinforcement learning from human feedback (RLHF) has been an effective technique for aligning AI systems with human values, with remarkable successes in fine-tuning large-language models recently. Most existing RLHF paradigms make the underlying assumption that human preferences are relatively homogeneous, and can be encoded by a single reward model. In this paper, we focus on addressing the issues due to the inherent heterogeneity in human preferences, as well as their potential strategic behavior in providing feedback. Specifically, we propose two frameworks to address heterogeneous human f","authors_text":"Asuman Ozdaglar, Chanwoo Park, Dingwen Kong, Kaiqing Zhang, Mingyang Liu","cross_cats":["cs.LG"],"headline":"","license":"http://creativecommons.org/licenses/by-sa/4.0/","primary_cat":"cs.AI","submitted_at":"2024-04-30T23:57:23Z","title":"RLHF from Heterogeneous Feedback via Personalization and Preference Aggregation"},"references":{"count":0,"internal_anchors":0,"resolved_work":0,"sample":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"2405.00254","kind":"arxiv","version":2},"verdict":{"created_at":null,"id":null,"model_set":{},"one_line_summary":"","pipeline_version":null,"pith_extraction_headline":"","strongest_claim":"","weakest_assumption":""}},"verdict_id":null}}],"author_attestations":[],"timestamp_anchors":[],"storage_attestations":[],"citation_signatures":[],"replication_records":[],"corrections":[],"mirror_hints":[],"record_created":{"event_id":"sha256:76b042fbeeffdca5639e4e4eaa2ff60b23e37b4b296b232b47d542c98497882b","target":"record","created_at":"2026-07-05T08:23:22Z","signer":{"key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signer_id":"pith.science","signer_type":"pith_registry"},"payload":{"attestation_state":"computed","canonical_record":{"metadata":{"abstract_canon_sha256":"ebf509ae404865ae07cd71eacfff34203f219771bd57d68fdc75cd9589451d5f","cross_cats_sorted":["cs.LG"],"license":"http://creativecommons.org/licenses/by-sa/4.0/","primary_cat":"cs.AI","submitted_at":"2024-04-30T23:57:23Z","title_canon_sha256":"ee5d02ec59d823d0d4a4ad7c680212961c7bffeb356874cd840f4c0352b4102c"},"schema_version":"1.0","source":{"id":"2405.00254","kind":"arxiv","version":2}},"canonical_sha256":"0de84fd5d03723c5fcffff5e4b47827c9c45489d3266d5a3c91bbda055be55ee","receipt":{"algorithm":"ed25519","builder_version":"pith-number-builder-2026-05-17-v1","canonical_sha256":"0de84fd5d03723c5fcffff5e4b47827c9c45489d3266d5a3c91bbda055be55ee","first_computed_at":"2026-07-05T08:23:22.077004Z","key_id":"pith-v1-2026-05","kind":"pith_receipt","last_reissued_at":"2026-07-05T08:23:22.077004Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","receipt_version":"0.3","signature_b64":"TZYP45wJajceksNhzqQYNjFzyKsrkvBEqrV0GllbPex4cSfbtfBy5hFQ8B2TtjAOVlPmx1bg5s4BGtpTqPPqBw==","signature_status":"signed_v1","signed_at":"2026-07-05T08:23:22.077499Z","signed_message":"canonical_sha256_bytes"},"source_id":"2405.00254","source_kind":"arxiv","source_version":2}}},"equivocations":[],"invalid_events":[],"applied_event_ids":["sha256:76b042fbeeffdca5639e4e4eaa2ff60b23e37b4b296b232b47d542c98497882b","sha256:3549d7cae175d7aa16a159db71283b2af5056b9b302917d0ae079024b809481b"],"state_sha256":"d233793f92c5af9db29f7c4092b9ea017aca88084b211ff220e24f9b1ac414af"},"bundle_signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"HeH5AjmZwtHu7T4ANZZdAlG6R+3FArKjstn2tuw5SlFDCXsD1EmZjnXFQGoVBpMzV0sEQEvVaC24MJXVAErCCw==","signed_message":"bundle_sha256_bytes","signed_at":"2026-08-09T14:12:04.288456Z","bundle_sha256":"ae41114509fa7bf37d6e4a35e4a6f9038c6a30730a370595e0ac059a7395e919"}}