{"record_type":"pith_number_record","schema_url":"https://pith.science/schemas/pith-number/v1.json","pith_number":"pith:2025:SHDWZORB6WOME3XUQCTRUNZS5V","short_pith_number":"pith:SHDWZORB","schema_version":"1.0","canonical_sha256":"91c76cba21f59cc26ef480a71a3732ed5f9b0b370cffe447aef72bed62a040a0","source":{"kind":"arxiv","id":"2504.11337","version":1},"attestation_state":"computed","paper":{"title":"REWARD CONSISTENCY: Improving Multi-Objective Alignment from a Data-Centric Perspective","license":"http://creativecommons.org/licenses/by-nc-sa/4.0/","headline":"","cross_cats":[],"primary_cat":"cs.CL","authors_text":"Jun Zhou, Xin Zhang, Xiting Wang, Yongqi Tong, Zhihao Xu","submitted_at":"2025-04-15T16:09:19Z","abstract_excerpt":"Multi-objective preference alignment in language models often encounters a challenging trade-off: optimizing for one human preference (e.g., helpfulness) frequently compromises others (e.g., harmlessness) due to the inherent conflicts between competing objectives. While prior work mainly focuses on algorithmic solutions, we explore a novel data-driven approach to uncover the types of data that can effectively mitigate these conflicts. Specifically, we propose the concept of Reward Consistency (RC), which identifies samples that align with multiple preference objectives, thereby reducing confli"},"verification_status":{"content_addressed":true,"pith_receipt":true,"author_attested":false,"weak_author_claims":0,"strong_author_claims":0,"externally_anchored":false,"storage_verified":false,"citation_signatures":0,"replication_records":0,"graph_snapshot":true,"references_resolved":false,"formal_links_present":false},"canonical_record":{"source":{"id":"2504.11337","kind":"arxiv","version":1},"metadata":{"license":"http://creativecommons.org/licenses/by-nc-sa/4.0/","primary_cat":"cs.CL","submitted_at":"2025-04-15T16:09:19Z","cross_cats_sorted":[],"title_canon_sha256":"ecc80b43937e5ec2ee2ef67e248b62edb20003c281901275dc35164cc4667ed5","abstract_canon_sha256":"3ff076815a99144e8812531bb143cd20df382c56ccde5db1d7e9f7b7becd6491"},"schema_version":"1.0"},"receipt":{"kind":"pith_receipt","key_id":"pith-v1-2026-05","algorithm":"ed25519","signed_at":"2026-07-05T10:49:32.417049Z","signature_b64":"UgMcDOnMW1lb6FXi+3W+fHmH8z/x6S1Wp0HtyfCusGASptndUooobILRgddkucSTTo9QhHwxvIpvBskhYkZPCw==","signed_message":"canonical_sha256_bytes","builder_version":"pith-number-builder-2026-05-17-v1","receipt_version":"0.3","canonical_sha256":"91c76cba21f59cc26ef480a71a3732ed5f9b0b370cffe447aef72bed62a040a0","last_reissued_at":"2026-07-05T10:49:32.416539Z","signature_status":"signed_v1","first_computed_at":"2026-07-05T10:49:32.416539Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"graph_snapshot":{"paper":{"title":"REWARD CONSISTENCY: Improving Multi-Objective Alignment from a Data-Centric Perspective","license":"http://creativecommons.org/licenses/by-nc-sa/4.0/","headline":"","cross_cats":[],"primary_cat":"cs.CL","authors_text":"Jun Zhou, Xin Zhang, Xiting Wang, Yongqi Tong, Zhihao Xu","submitted_at":"2025-04-15T16:09:19Z","abstract_excerpt":"Multi-objective preference alignment in language models often encounters a challenging trade-off: optimizing for one human preference (e.g., helpfulness) frequently compromises others (e.g., harmlessness) due to the inherent conflicts between competing objectives. While prior work mainly focuses on algorithmic solutions, we explore a novel data-driven approach to uncover the types of data that can effectively mitigate these conflicts. Specifically, we propose the concept of Reward Consistency (RC), which identifies samples that align with multiple preference objectives, thereby reducing confli"},"claims":{"count":0,"items":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"2504.11337","kind":"arxiv","version":1},"verdict":{"id":null,"model_set":{},"created_at":null,"strongest_claim":"","one_line_summary":"","pipeline_version":null,"weakest_assumption":"","pith_extraction_headline":""},"integrity":{"clean":true,"summary":{"advisory":0,"critical":0,"by_detector":{},"informational":0},"endpoint":"/pith/2504.11337/integrity.json","findings":[],"available":true,"detectors_run":[],"snapshot_sha256":"c28c3603d3b5d939e8dc4c7e95fa8dfce3d595e45f758748cecf8e644a296938"},"references":{"count":0,"sample":[],"resolved_work":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","internal_anchors":0},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"author_claims":{"count":0,"strong_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"builder_version":"pith-number-builder-2026-05-17-v1"},"aliases":[{"alias_kind":"arxiv","alias_value":"2504.11337","created_at":"2026-07-05T10:49:32.416599+00:00"},{"alias_kind":"arxiv_version","alias_value":"2504.11337v1","created_at":"2026-07-05T10:49:32.416599+00:00"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.2504.11337","created_at":"2026-07-05T10:49:32.416599+00:00"},{"alias_kind":"pith_short_12","alias_value":"SHDWZORB6WOM","created_at":"2026-07-05T10:49:32.416599+00:00"},{"alias_kind":"pith_short_16","alias_value":"SHDWZORB6WOME3XU","created_at":"2026-07-05T10:49:32.416599+00:00"},{"alias_kind":"pith_short_8","alias_value":"SHDWZORB","created_at":"2026-07-05T10:49:32.416599+00:00"}],"events":[],"event_summary":{},"paper_claims":[],"inbound_citations":{"count":3,"internal_anchor_count":0,"sample":[{"citing_arxiv_id":"2607.01392","citing_title":"Multi-Objective Exploration and Preference Optimization via Mutual Information","ref_index":100,"is_internal_anchor":false},{"citing_arxiv_id":"2605.11679","citing_title":"Explaining and Breaking the Safety-Helpfulness Ceiling via Preference Dimensional Expansion","ref_index":17,"is_internal_anchor":false},{"citing_arxiv_id":"2605.11679","citing_title":"Explaining and Breaking the Safety-Helpfulness Ceiling via Preference Dimensional Expansion","ref_index":17,"is_internal_anchor":false}]},"formal_canon":{"evidence_count":0,"sample":[],"anchors":[]},"links":{"html":"https://pith.science/pith/SHDWZORB6WOME3XUQCTRUNZS5V","json":"https://pith.science/pith/SHDWZORB6WOME3XUQCTRUNZS5V.json","graph_json":"https://pith.science/api/pith-number/SHDWZORB6WOME3XUQCTRUNZS5V/graph.json","events_json":"https://pith.science/api/pith-number/SHDWZORB6WOME3XUQCTRUNZS5V/events.json","paper":"https://pith.science/paper/SHDWZORB"},"agent_actions":{"view_html":"https://pith.science/pith/SHDWZORB6WOME3XUQCTRUNZS5V","download_json":"https://pith.science/pith/SHDWZORB6WOME3XUQCTRUNZS5V.json","view_paper":"https://pith.science/paper/SHDWZORB","resolve_alias":"https://pith.science/api/pith-number/resolve?arxiv=2504.11337&json=true","fetch_graph":"https://pith.science/api/pith-number/SHDWZORB6WOME3XUQCTRUNZS5V/graph.json","fetch_events":"https://pith.science/api/pith-number/SHDWZORB6WOME3XUQCTRUNZS5V/events.json","actions":{"anchor_timestamp":"https://pith.science/pith/SHDWZORB6WOME3XUQCTRUNZS5V/action/timestamp_anchor","attest_storage":"https://pith.science/pith/SHDWZORB6WOME3XUQCTRUNZS5V/action/storage_attestation","attest_author":"https://pith.science/pith/SHDWZORB6WOME3XUQCTRUNZS5V/action/author_attestation","sign_citation":"https://pith.science/pith/SHDWZORB6WOME3XUQCTRUNZS5V/action/citation_signature","submit_replication":"https://pith.science/pith/SHDWZORB6WOME3XUQCTRUNZS5V/action/replication_record"}},"created_at":"2026-07-05T10:49:32.416599+00:00","updated_at":"2026-07-05T10:49:32.416599+00:00"}