{"record_type":"pith_number_record","schema_url":"https://pith.science/schemas/pith-number/v1.json","pith_number":"pith:2024:GBQR6C6CQ7L7LIGS5UXCS4ECFG","short_pith_number":"pith:GBQR6C6C","schema_version":"1.0","canonical_sha256":"30611f0bc287d7f5a0d2ed2e29708229acd0383b5c3f01d2488d5c7800dd93ad","source":{"kind":"arxiv","id":"2402.18571","version":3},"attestation_state":"computed","paper":{"title":"Arithmetic Control of LLMs for Diverse User Preferences: Directional Preference Alignment with Multi-Objective Rewards","license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","headline":"","cross_cats":["cs.AI","cs.CL","stat.ML"],"primary_cat":"cs.LG","authors_text":"Han Zhao, Haoxiang Wang, Rui Yang, Shizhe Diao, Shuang Qiu, Tong Zhang, Wei Xiong, Yong Lin","submitted_at":"2024-02-28T18:58:25Z","abstract_excerpt":"Fine-grained control over large language models (LLMs) remains a significant challenge, hindering their adaptability to diverse user needs. While Reinforcement Learning from Human Feedback (RLHF) shows promise in aligning LLMs, its reliance on scalar rewards often limits its ability to capture diverse user preferences in real-world applications. To address this limitation, we introduce the Directional Preference Alignment (DPA) framework. Unlike the scalar-reward RLHF, DPA incorporates multi-objective reward modeling to represent diverse preference profiles. Additionally, DPA models user prefe"},"verification_status":{"content_addressed":true,"pith_receipt":true,"author_attested":false,"weak_author_claims":0,"strong_author_claims":0,"externally_anchored":false,"storage_verified":false,"citation_signatures":0,"replication_records":0,"graph_snapshot":true,"references_resolved":false,"formal_links_present":false},"canonical_record":{"source":{"id":"2402.18571","kind":"arxiv","version":3},"metadata":{"license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","primary_cat":"cs.LG","submitted_at":"2024-02-28T18:58:25Z","cross_cats_sorted":["cs.AI","cs.CL","stat.ML"],"title_canon_sha256":"59a4b4d0719eb5a27a52c62832b52505fe573efe9e80838032089351eacfc2ea","abstract_canon_sha256":"caa8060672cfb6bb84a59213df3b7964e1984ba4f12c10e99ef5683a1ebfbba4"},"schema_version":"1.0"},"receipt":{"kind":"pith_receipt","key_id":"pith-v1-2026-05","algorithm":"ed25519","signed_at":"2026-07-05T07:52:44.935232Z","signature_b64":"FC0HXaV32Z5z0w8Z4Fms5CrunUAfJ1Z8uf4cOu4pmVU6DO4lwyqFzDUT6ZtUdf8YwefJqrm6bl7WPPyDbFJLCg==","signed_message":"canonical_sha256_bytes","builder_version":"pith-number-builder-2026-05-17-v1","receipt_version":"0.3","canonical_sha256":"30611f0bc287d7f5a0d2ed2e29708229acd0383b5c3f01d2488d5c7800dd93ad","last_reissued_at":"2026-07-05T07:52:44.934761Z","signature_status":"signed_v1","first_computed_at":"2026-07-05T07:52:44.934761Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"graph_snapshot":{"paper":{"title":"Arithmetic Control of LLMs for Diverse User Preferences: Directional Preference Alignment with Multi-Objective Rewards","license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","headline":"","cross_cats":["cs.AI","cs.CL","stat.ML"],"primary_cat":"cs.LG","authors_text":"Han Zhao, Haoxiang Wang, Rui Yang, Shizhe Diao, Shuang Qiu, Tong Zhang, Wei Xiong, Yong Lin","submitted_at":"2024-02-28T18:58:25Z","abstract_excerpt":"Fine-grained control over large language models (LLMs) remains a significant challenge, hindering their adaptability to diverse user needs. While Reinforcement Learning from Human Feedback (RLHF) shows promise in aligning LLMs, its reliance on scalar rewards often limits its ability to capture diverse user preferences in real-world applications. To address this limitation, we introduce the Directional Preference Alignment (DPA) framework. Unlike the scalar-reward RLHF, DPA incorporates multi-objective reward modeling to represent diverse preference profiles. Additionally, DPA models user prefe"},"claims":{"count":0,"items":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"2402.18571","kind":"arxiv","version":3},"verdict":{"id":null,"model_set":{},"created_at":null,"strongest_claim":"","one_line_summary":"","pipeline_version":null,"weakest_assumption":"","pith_extraction_headline":""},"integrity":{"clean":true,"summary":{"advisory":0,"critical":0,"by_detector":{},"informational":0},"endpoint":"/pith/2402.18571/integrity.json","findings":[],"available":true,"detectors_run":[],"snapshot_sha256":"c28c3603d3b5d939e8dc4c7e95fa8dfce3d595e45f758748cecf8e644a296938"},"references":{"count":0,"sample":[],"resolved_work":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","internal_anchors":0},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"author_claims":{"count":0,"strong_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"builder_version":"pith-number-builder-2026-05-17-v1"},"aliases":[{"alias_kind":"arxiv","alias_value":"2402.18571","created_at":"2026-07-05T07:52:44.934820+00:00"},{"alias_kind":"arxiv_version","alias_value":"2402.18571v3","created_at":"2026-07-05T07:52:44.934820+00:00"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.2402.18571","created_at":"2026-07-05T07:52:44.934820+00:00"},{"alias_kind":"pith_short_12","alias_value":"GBQR6C6CQ7L7","created_at":"2026-07-05T07:52:44.934820+00:00"},{"alias_kind":"pith_short_16","alias_value":"GBQR6C6CQ7L7LIGS","created_at":"2026-07-05T07:52:44.934820+00:00"},{"alias_kind":"pith_short_8","alias_value":"GBQR6C6C","created_at":"2026-07-05T07:52:44.934820+00:00"}],"events":[],"event_summary":{},"paper_claims":[],"inbound_citations":{"count":8,"internal_anchor_count":0,"sample":[{"citing_arxiv_id":"2605.20408","citing_title":"Spectral Souping: A Unified Framework for Online Preference Alignment","ref_index":29,"is_internal_anchor":false},{"citing_arxiv_id":"2605.19330","citing_title":"MOCHA: Multi-Objective Chebyshev Annealing for Agent Skill Optimization","ref_index":26,"is_internal_anchor":false},{"citing_arxiv_id":"2605.11679","citing_title":"Explaining and Breaking the Safety-Helpfulness Ceiling via Preference Dimensional Expansion","ref_index":66,"is_internal_anchor":false},{"citing_arxiv_id":"2507.17746","citing_title":"Rubrics as Rewards: Reinforcement Learning Beyond Verifiable Domains","ref_index":34,"is_internal_anchor":false},{"citing_arxiv_id":"2605.11679","citing_title":"Explaining and Breaking the Safety-Helpfulness Ceiling via Preference Dimensional Expansion","ref_index":66,"is_internal_anchor":false},{"citing_arxiv_id":"2604.24536","citing_title":"Generating Place-Based Compromises Between Two Points of View","ref_index":70,"is_internal_anchor":false},{"citing_arxiv_id":"2605.06987","citing_title":"Response Time Enhances Alignment with Heterogeneous Preferences","ref_index":164,"is_internal_anchor":false},{"citing_arxiv_id":"2605.07162","citing_title":"CLIPer: Tailoring Diverse User Preference via Classifier-Guided Inference-Time Personalization","ref_index":3,"is_internal_anchor":false}]},"formal_canon":{"evidence_count":0,"sample":[],"anchors":[]},"links":{"html":"https://pith.science/pith/GBQR6C6CQ7L7LIGS5UXCS4ECFG","json":"https://pith.science/pith/GBQR6C6CQ7L7LIGS5UXCS4ECFG.json","graph_json":"https://pith.science/api/pith-number/GBQR6C6CQ7L7LIGS5UXCS4ECFG/graph.json","events_json":"https://pith.science/api/pith-number/GBQR6C6CQ7L7LIGS5UXCS4ECFG/events.json","paper":"https://pith.science/paper/GBQR6C6C"},"agent_actions":{"view_html":"https://pith.science/pith/GBQR6C6CQ7L7LIGS5UXCS4ECFG","download_json":"https://pith.science/pith/GBQR6C6CQ7L7LIGS5UXCS4ECFG.json","view_paper":"https://pith.science/paper/GBQR6C6C","resolve_alias":"https://pith.science/api/pith-number/resolve?arxiv=2402.18571&json=true","fetch_graph":"https://pith.science/api/pith-number/GBQR6C6CQ7L7LIGS5UXCS4ECFG/graph.json","fetch_events":"https://pith.science/api/pith-number/GBQR6C6CQ7L7LIGS5UXCS4ECFG/events.json","actions":{"anchor_timestamp":"https://pith.science/pith/GBQR6C6CQ7L7LIGS5UXCS4ECFG/action/timestamp_anchor","attest_storage":"https://pith.science/pith/GBQR6C6CQ7L7LIGS5UXCS4ECFG/action/storage_attestation","attest_author":"https://pith.science/pith/GBQR6C6CQ7L7LIGS5UXCS4ECFG/action/author_attestation","sign_citation":"https://pith.science/pith/GBQR6C6CQ7L7LIGS5UXCS4ECFG/action/citation_signature","submit_replication":"https://pith.science/pith/GBQR6C6CQ7L7LIGS5UXCS4ECFG/action/replication_record"}},"created_at":"2026-07-05T07:52:44.934820+00:00","updated_at":"2026-07-05T07:52:44.934820+00:00"}