{"bundle_type":"pith_open_graph_bundle","bundle_version":"1.0","pith_number":"pith:2024:GBQR6C6CQ7L7LIGS5UXCS4ECFG","short_pith_number":"pith:GBQR6C6C","canonical_record":{"source":{"id":"2402.18571","kind":"arxiv","version":3},"metadata":{"license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","primary_cat":"cs.LG","submitted_at":"2024-02-28T18:58:25Z","cross_cats_sorted":["cs.AI","cs.CL","stat.ML"],"title_canon_sha256":"59a4b4d0719eb5a27a52c62832b52505fe573efe9e80838032089351eacfc2ea","abstract_canon_sha256":"caa8060672cfb6bb84a59213df3b7964e1984ba4f12c10e99ef5683a1ebfbba4"},"schema_version":"1.0"},"canonical_sha256":"30611f0bc287d7f5a0d2ed2e29708229acd0383b5c3f01d2488d5c7800dd93ad","source":{"kind":"arxiv","id":"2402.18571","version":3},"source_aliases":[{"alias_kind":"arxiv","alias_value":"2402.18571","created_at":"2026-07-05T07:52:44Z"},{"alias_kind":"arxiv_version","alias_value":"2402.18571v3","created_at":"2026-07-05T07:52:44Z"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.2402.18571","created_at":"2026-07-05T07:52:44Z"},{"alias_kind":"pith_short_12","alias_value":"GBQR6C6CQ7L7","created_at":"2026-07-05T07:52:44Z"},{"alias_kind":"pith_short_16","alias_value":"GBQR6C6CQ7L7LIGS","created_at":"2026-07-05T07:52:44Z"},{"alias_kind":"pith_short_8","alias_value":"GBQR6C6C","created_at":"2026-07-05T07:52:44Z"}],"events":[{"event_type":"record_created","subject_pith_number":"pith:2024:GBQR6C6CQ7L7LIGS5UXCS4ECFG","target":"record","payload":{"canonical_record":{"source":{"id":"2402.18571","kind":"arxiv","version":3},"metadata":{"license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","primary_cat":"cs.LG","submitted_at":"2024-02-28T18:58:25Z","cross_cats_sorted":["cs.AI","cs.CL","stat.ML"],"title_canon_sha256":"59a4b4d0719eb5a27a52c62832b52505fe573efe9e80838032089351eacfc2ea","abstract_canon_sha256":"caa8060672cfb6bb84a59213df3b7964e1984ba4f12c10e99ef5683a1ebfbba4"},"schema_version":"1.0"},"canonical_sha256":"30611f0bc287d7f5a0d2ed2e29708229acd0383b5c3f01d2488d5c7800dd93ad","receipt":{"kind":"pith_receipt","key_id":"pith-v1-2026-05","algorithm":"ed25519","signed_at":"2026-07-05T07:52:44.935232Z","signature_b64":"FC0HXaV32Z5z0w8Z4Fms5CrunUAfJ1Z8uf4cOu4pmVU6DO4lwyqFzDUT6ZtUdf8YwefJqrm6bl7WPPyDbFJLCg==","signed_message":"canonical_sha256_bytes","builder_version":"pith-number-builder-2026-05-17-v1","receipt_version":"0.3","canonical_sha256":"30611f0bc287d7f5a0d2ed2e29708229acd0383b5c3f01d2488d5c7800dd93ad","last_reissued_at":"2026-07-05T07:52:44.934761Z","signature_status":"signed_v1","first_computed_at":"2026-07-05T07:52:44.934761Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"source_kind":"arxiv","source_id":"2402.18571","source_version":3,"attestation_state":"computed"},"signer":{"signer_id":"pith.science","signer_type":"pith_registry","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"created_at":"2026-07-05T07:52:44Z","supersedes":[],"prev_event":null,"signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"q1b5Jqm/BPjJHiSEoDWzruvF28L9CKu+EdKdZl/yGRlhOuVhyLcFvhQKt3vkN7i+CUHSTAfol0RoGCY/274iCg==","signed_message":"open_graph_event_sha256_bytes","signed_at":"2026-08-09T03:03:05.687100Z"},"content_sha256":"0983a068d13571354b23daed3c994f890d2e60192dd92c53cc7c67ad98010bb6","schema_version":"1.0","event_id":"sha256:0983a068d13571354b23daed3c994f890d2e60192dd92c53cc7c67ad98010bb6"},{"event_type":"graph_snapshot","subject_pith_number":"pith:2024:GBQR6C6CQ7L7LIGS5UXCS4ECFG","target":"graph","payload":{"graph_snapshot":{"paper":{"title":"Arithmetic Control of LLMs for Diverse User Preferences: Directional Preference Alignment with Multi-Objective Rewards","license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","headline":"","cross_cats":["cs.AI","cs.CL","stat.ML"],"primary_cat":"cs.LG","authors_text":"Han Zhao, Haoxiang Wang, Rui Yang, Shizhe Diao, Shuang Qiu, Tong Zhang, Wei Xiong, Yong Lin","submitted_at":"2024-02-28T18:58:25Z","abstract_excerpt":"Fine-grained control over large language models (LLMs) remains a significant challenge, hindering their adaptability to diverse user needs. While Reinforcement Learning from Human Feedback (RLHF) shows promise in aligning LLMs, its reliance on scalar rewards often limits its ability to capture diverse user preferences in real-world applications. To address this limitation, we introduce the Directional Preference Alignment (DPA) framework. Unlike the scalar-reward RLHF, DPA incorporates multi-objective reward modeling to represent diverse preference profiles. Additionally, DPA models user prefe"},"claims":{"count":0,"items":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"2402.18571","kind":"arxiv","version":3},"verdict":{"id":null,"model_set":{},"created_at":null,"strongest_claim":"","one_line_summary":"","pipeline_version":null,"weakest_assumption":"","pith_extraction_headline":""},"integrity":{"clean":true,"summary":{"advisory":0,"critical":0,"by_detector":{},"informational":0},"endpoint":"/pith/2402.18571/integrity.json","findings":[],"available":true,"detectors_run":[],"snapshot_sha256":"c28c3603d3b5d939e8dc4c7e95fa8dfce3d595e45f758748cecf8e644a296938"},"references":{"count":0,"sample":[],"resolved_work":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","internal_anchors":0},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"author_claims":{"count":0,"strong_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"builder_version":"pith-number-builder-2026-05-17-v1"},"verdict_id":null},"signer":{"signer_id":"pith.science","signer_type":"pith_registry","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"created_at":"2026-07-05T07:52:44Z","supersedes":[],"prev_event":null,"signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"ksaZ+8F+DxchWI6/gO0jKFn5D1NbRYXfjy2ZI54qtaA2nLEWFuWljzGnKpdXuCUVvN1a0OZ5MzaMewOVhupgCg==","signed_message":"open_graph_event_sha256_bytes","signed_at":"2026-08-09T03:03:05.687694Z"},"content_sha256":"4e3b8f75e66f05440d282229ffdbf6b04e7bf4694b9ee6c8c3d976d73494805d","schema_version":"1.0","event_id":"sha256:4e3b8f75e66f05440d282229ffdbf6b04e7bf4694b9ee6c8c3d976d73494805d"}],"timestamp_proofs":[],"mirror_hints":[{"mirror_type":"https","name":"Pith Resolver","base_url":"https://pith.science","bundle_url":"https://pith.science/pith/GBQR6C6CQ7L7LIGS5UXCS4ECFG/bundle.json","state_url":"https://pith.science/pith/GBQR6C6CQ7L7LIGS5UXCS4ECFG/state.json","well_known_bundle_url":"https://pith.science/.well-known/pith/GBQR6C6CQ7L7LIGS5UXCS4ECFG/bundle.json","status":"primary"}],"public_keys":[{"key_id":"pith-v1-2026-05","algorithm":"ed25519","format":"raw","public_key_b64":"stVStoiQhXFxp4s2pdzPNoqVNBMojDU/fJ2db5S3CbM=","public_key_hex":"b2d552b68890857171a78b36a5dccf368a953413288c353f7c9d9d6f94b709b3","fingerprint_sha256_b32_first128bits":"RVFV5Z2OI2J3ZUO7ERDEBCYNKS","fingerprint_sha256_hex":"8d4b5ee74e4693bcd1df2446408b0d54","rotates_at":null,"url":"https://pith.science/pith-signing-key.json","notes":"Pith uses this Ed25519 key to sign canonical record SHA-256 digests. Verify with: ed25519_verify(public_key, message=canonical_sha256_bytes, signature=base64decode(signature_b64))."}],"merge_version":"pith-open-graph-merge-v1","built_at":"2026-08-09T03:03:05Z","links":{"resolver":"https://pith.science/pith/GBQR6C6CQ7L7LIGS5UXCS4ECFG","bundle":"https://pith.science/pith/GBQR6C6CQ7L7LIGS5UXCS4ECFG/bundle.json","state":"https://pith.science/pith/GBQR6C6CQ7L7LIGS5UXCS4ECFG/state.json","well_known_bundle":"https://pith.science/.well-known/pith/GBQR6C6CQ7L7LIGS5UXCS4ECFG/bundle.json"},"state":{"state_type":"pith_open_graph_state","state_version":"1.0","pith_number":"pith:2024:GBQR6C6CQ7L7LIGS5UXCS4ECFG","merge_version":"pith-open-graph-merge-v1","event_count":2,"valid_event_count":2,"invalid_event_count":0,"equivocation_count":0,"current":{"canonical_record":{"metadata":{"abstract_canon_sha256":"caa8060672cfb6bb84a59213df3b7964e1984ba4f12c10e99ef5683a1ebfbba4","cross_cats_sorted":["cs.AI","cs.CL","stat.ML"],"license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","primary_cat":"cs.LG","submitted_at":"2024-02-28T18:58:25Z","title_canon_sha256":"59a4b4d0719eb5a27a52c62832b52505fe573efe9e80838032089351eacfc2ea"},"schema_version":"1.0","source":{"id":"2402.18571","kind":"arxiv","version":3}},"source_aliases":[{"alias_kind":"arxiv","alias_value":"2402.18571","created_at":"2026-07-05T07:52:44Z"},{"alias_kind":"arxiv_version","alias_value":"2402.18571v3","created_at":"2026-07-05T07:52:44Z"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.2402.18571","created_at":"2026-07-05T07:52:44Z"},{"alias_kind":"pith_short_12","alias_value":"GBQR6C6CQ7L7","created_at":"2026-07-05T07:52:44Z"},{"alias_kind":"pith_short_16","alias_value":"GBQR6C6CQ7L7LIGS","created_at":"2026-07-05T07:52:44Z"},{"alias_kind":"pith_short_8","alias_value":"GBQR6C6C","created_at":"2026-07-05T07:52:44Z"}],"graph_snapshots":[{"event_id":"sha256:4e3b8f75e66f05440d282229ffdbf6b04e7bf4694b9ee6c8c3d976d73494805d","target":"graph","created_at":"2026-07-05T07:52:44Z","signer":{"key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signer_id":"pith.science","signer_type":"pith_registry"},"payload":{"graph_snapshot":{"author_claims":{"count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","strong_count":0},"builder_version":"pith-number-builder-2026-05-17-v1","claims":{"count":0,"items":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"integrity":{"available":true,"clean":true,"detectors_run":[],"endpoint":"/pith/2402.18571/integrity.json","findings":[],"snapshot_sha256":"c28c3603d3b5d939e8dc4c7e95fa8dfce3d595e45f758748cecf8e644a296938","summary":{"advisory":0,"by_detector":{},"critical":0,"informational":0}},"paper":{"abstract_excerpt":"Fine-grained control over large language models (LLMs) remains a significant challenge, hindering their adaptability to diverse user needs. While Reinforcement Learning from Human Feedback (RLHF) shows promise in aligning LLMs, its reliance on scalar rewards often limits its ability to capture diverse user preferences in real-world applications. To address this limitation, we introduce the Directional Preference Alignment (DPA) framework. Unlike the scalar-reward RLHF, DPA incorporates multi-objective reward modeling to represent diverse preference profiles. Additionally, DPA models user prefe","authors_text":"Han Zhao, Haoxiang Wang, Rui Yang, Shizhe Diao, Shuang Qiu, Tong Zhang, Wei Xiong, Yong Lin","cross_cats":["cs.AI","cs.CL","stat.ML"],"headline":"","license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","primary_cat":"cs.LG","submitted_at":"2024-02-28T18:58:25Z","title":"Arithmetic Control of LLMs for Diverse User Preferences: Directional Preference Alignment with Multi-Objective Rewards"},"references":{"count":0,"internal_anchors":0,"resolved_work":0,"sample":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"2402.18571","kind":"arxiv","version":3},"verdict":{"created_at":null,"id":null,"model_set":{},"one_line_summary":"","pipeline_version":null,"pith_extraction_headline":"","strongest_claim":"","weakest_assumption":""}},"verdict_id":null}}],"author_attestations":[],"timestamp_anchors":[],"storage_attestations":[],"citation_signatures":[],"replication_records":[],"corrections":[],"mirror_hints":[],"record_created":{"event_id":"sha256:0983a068d13571354b23daed3c994f890d2e60192dd92c53cc7c67ad98010bb6","target":"record","created_at":"2026-07-05T07:52:44Z","signer":{"key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signer_id":"pith.science","signer_type":"pith_registry"},"payload":{"attestation_state":"computed","canonical_record":{"metadata":{"abstract_canon_sha256":"caa8060672cfb6bb84a59213df3b7964e1984ba4f12c10e99ef5683a1ebfbba4","cross_cats_sorted":["cs.AI","cs.CL","stat.ML"],"license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","primary_cat":"cs.LG","submitted_at":"2024-02-28T18:58:25Z","title_canon_sha256":"59a4b4d0719eb5a27a52c62832b52505fe573efe9e80838032089351eacfc2ea"},"schema_version":"1.0","source":{"id":"2402.18571","kind":"arxiv","version":3}},"canonical_sha256":"30611f0bc287d7f5a0d2ed2e29708229acd0383b5c3f01d2488d5c7800dd93ad","receipt":{"algorithm":"ed25519","builder_version":"pith-number-builder-2026-05-17-v1","canonical_sha256":"30611f0bc287d7f5a0d2ed2e29708229acd0383b5c3f01d2488d5c7800dd93ad","first_computed_at":"2026-07-05T07:52:44.934761Z","key_id":"pith-v1-2026-05","kind":"pith_receipt","last_reissued_at":"2026-07-05T07:52:44.934761Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","receipt_version":"0.3","signature_b64":"FC0HXaV32Z5z0w8Z4Fms5CrunUAfJ1Z8uf4cOu4pmVU6DO4lwyqFzDUT6ZtUdf8YwefJqrm6bl7WPPyDbFJLCg==","signature_status":"signed_v1","signed_at":"2026-07-05T07:52:44.935232Z","signed_message":"canonical_sha256_bytes"},"source_id":"2402.18571","source_kind":"arxiv","source_version":3}}},"equivocations":[],"invalid_events":[],"applied_event_ids":["sha256:0983a068d13571354b23daed3c994f890d2e60192dd92c53cc7c67ad98010bb6","sha256:4e3b8f75e66f05440d282229ffdbf6b04e7bf4694b9ee6c8c3d976d73494805d"],"state_sha256":"867b845f711743d192d9e0b418aee152e9e081eb443292c227240feb80c156bb"},"bundle_signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"CKN1yy2+Ol5tBreZCSuu9rbKJFuOd9Mc+HG+QZQY8zeYThZjxyJExNPt2Whu3ryDwiwRzsFERWU1OrHbROD2AA==","signed_message":"bundle_sha256_bytes","signed_at":"2026-08-09T03:03:05.693513Z","bundle_sha256":"9c2efe3e2cfb699bf8bded6be441f51133cfb7d70f8e9dad8b00c7c6fa3bd923"}}