{"bundle_type":"pith_open_graph_bundle","bundle_version":"1.0","pith_number":"pith:2024:2GXS6JSKTNWOVQTNXGO5IKVW2B","short_pith_number":"pith:2GXS6JSK","canonical_record":{"source":{"id":"2405.17931","kind":"arxiv","version":1},"metadata":{"license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","primary_cat":"cs.CL","submitted_at":"2024-05-28T07:53:40Z","cross_cats_sorted":["cs.LG"],"title_canon_sha256":"12424b266f150fc1e3640abec34335244d5a0d6a1078e1b2693afaf06e25f133","abstract_canon_sha256":"19ff8076bde6ab34fae72b7127643d73b788aa27637b292e1e538c80ba1eeb49"},"schema_version":"1.0"},"canonical_sha256":"d1af2f264a9b6ceac26db99dd42ab6d04be3df68e46c3edc4f5c53574931f503","source":{"kind":"arxiv","id":"2405.17931","version":1},"source_aliases":[{"alias_kind":"arxiv","alias_value":"2405.17931","created_at":"2026-07-05T08:24:13Z"},{"alias_kind":"arxiv_version","alias_value":"2405.17931v1","created_at":"2026-07-05T08:24:13Z"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.2405.17931","created_at":"2026-07-05T08:24:13Z"},{"alias_kind":"pith_short_12","alias_value":"2GXS6JSKTNWO","created_at":"2026-07-05T08:24:13Z"},{"alias_kind":"pith_short_16","alias_value":"2GXS6JSKTNWOVQTN","created_at":"2026-07-05T08:24:13Z"},{"alias_kind":"pith_short_8","alias_value":"2GXS6JSK","created_at":"2026-07-05T08:24:13Z"}],"events":[{"event_type":"record_created","subject_pith_number":"pith:2024:2GXS6JSKTNWOVQTNXGO5IKVW2B","target":"record","payload":{"canonical_record":{"source":{"id":"2405.17931","kind":"arxiv","version":1},"metadata":{"license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","primary_cat":"cs.CL","submitted_at":"2024-05-28T07:53:40Z","cross_cats_sorted":["cs.LG"],"title_canon_sha256":"12424b266f150fc1e3640abec34335244d5a0d6a1078e1b2693afaf06e25f133","abstract_canon_sha256":"19ff8076bde6ab34fae72b7127643d73b788aa27637b292e1e538c80ba1eeb49"},"schema_version":"1.0"},"canonical_sha256":"d1af2f264a9b6ceac26db99dd42ab6d04be3df68e46c3edc4f5c53574931f503","receipt":{"kind":"pith_receipt","key_id":"pith-v1-2026-05","algorithm":"ed25519","signed_at":"2026-07-05T08:24:13.314354Z","signature_b64":"NxRQqeFdfYFugepgz7n7lGlO1VpAPwCqIBOde/pc+k5Uu3DldimvLqMZXXwWxQ05f/W4M+7LZeOif4ffXYSMAw==","signed_message":"canonical_sha256_bytes","builder_version":"pith-number-builder-2026-05-17-v1","receipt_version":"0.3","canonical_sha256":"d1af2f264a9b6ceac26db99dd42ab6d04be3df68e46c3edc4f5c53574931f503","last_reissued_at":"2026-07-05T08:24:13.313885Z","signature_status":"signed_v1","first_computed_at":"2026-07-05T08:24:13.313885Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"source_kind":"arxiv","source_id":"2405.17931","source_version":1,"attestation_state":"computed"},"signer":{"signer_id":"pith.science","signer_type":"pith_registry","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"created_at":"2026-07-05T08:24:13Z","supersedes":[],"prev_event":null,"signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"7t1fBdyK5rqU/rAM8qYM76PrjCeVXXmtp/rnbcU5w3sswy9lCzyuxxHk9OGdnAH+Wm7d/M4rEUEiS8ZLIb1/Cw==","signed_message":"open_graph_event_sha256_bytes","signed_at":"2026-08-22T01:16:32.038151Z"},"content_sha256":"4384b73a3ae95afcfe12a9103fa2e63c3ea39587a66132229ba6b5e0d425abe8","schema_version":"1.0","event_id":"sha256:4384b73a3ae95afcfe12a9103fa2e63c3ea39587a66132229ba6b5e0d425abe8"},{"event_type":"graph_snapshot","subject_pith_number":"pith:2024:2GXS6JSKTNWOVQTNXGO5IKVW2B","target":"graph","payload":{"graph_snapshot":{"paper":{"title":"Online Merging Optimizers for Boosting Rewards and Mitigating Tax in Alignment","license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","headline":"","cross_cats":["cs.LG"],"primary_cat":"cs.CL","authors_text":"Bowen Yu, Chang Zhou, Fei Huang, Keming Lu, Runji Lin, Yang Fan","submitted_at":"2024-05-28T07:53:40Z","abstract_excerpt":"Effectively aligning Large Language Models (LLMs) with human-centric values while preventing the degradation of abilities acquired through Pre-training and Supervised Fine-tuning (SFT) poses a central challenge in Reinforcement Learning from Human Feedback (RLHF). In this paper, we first discover that interpolating RLHF and SFT model parameters can adjust the trade-off between human preference and basic capabilities, thereby reducing the alignment tax at the cost of alignment reward. Inspired by this, we propose integrating the RL policy and SFT models at each optimization step in RLHF to cont"},"claims":{"count":0,"items":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"2405.17931","kind":"arxiv","version":1},"verdict":{"id":null,"model_set":{},"created_at":null,"strongest_claim":"","one_line_summary":"","pipeline_version":null,"weakest_assumption":"","pith_extraction_headline":""},"integrity":{"clean":true,"summary":{"advisory":0,"critical":0,"by_detector":{},"informational":0},"endpoint":"/pith/2405.17931/integrity.json","findings":[],"available":true,"detectors_run":[],"snapshot_sha256":"c28c3603d3b5d939e8dc4c7e95fa8dfce3d595e45f758748cecf8e644a296938"},"references":{"count":0,"sample":[],"resolved_work":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","internal_anchors":0},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"author_claims":{"count":0,"strong_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"builder_version":"pith-number-builder-2026-05-17-v1"},"verdict_id":null},"signer":{"signer_id":"pith.science","signer_type":"pith_registry","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"created_at":"2026-07-05T08:24:13Z","supersedes":[],"prev_event":null,"signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"3gXLAC/rY5jsooYInLKRQP8CDpQm5RkpmK4pRIGeBVbmnX5BuLgFzCPtYAKt62OlhsN+d5FV7Hs1l6iuCHWPAA==","signed_message":"open_graph_event_sha256_bytes","signed_at":"2026-08-22T01:16:32.038529Z"},"content_sha256":"69408cdc6442c9970b4ec2f5851827cc471106a85e34486fd86f1f185f9ab8f9","schema_version":"1.0","event_id":"sha256:69408cdc6442c9970b4ec2f5851827cc471106a85e34486fd86f1f185f9ab8f9"}],"timestamp_proofs":[],"mirror_hints":[{"mirror_type":"https","name":"Pith Resolver","base_url":"https://pith.science","bundle_url":"https://pith.science/pith/2GXS6JSKTNWOVQTNXGO5IKVW2B/bundle.json","state_url":"https://pith.science/pith/2GXS6JSKTNWOVQTNXGO5IKVW2B/state.json","well_known_bundle_url":"https://pith.science/.well-known/pith/2GXS6JSKTNWOVQTNXGO5IKVW2B/bundle.json","status":"primary"}],"public_keys":[{"key_id":"pith-v1-2026-05","algorithm":"ed25519","format":"raw","public_key_b64":"stVStoiQhXFxp4s2pdzPNoqVNBMojDU/fJ2db5S3CbM=","public_key_hex":"b2d552b68890857171a78b36a5dccf368a953413288c353f7c9d9d6f94b709b3","fingerprint_sha256_b32_first128bits":"RVFV5Z2OI2J3ZUO7ERDEBCYNKS","fingerprint_sha256_hex":"8d4b5ee74e4693bcd1df2446408b0d54","rotates_at":null,"url":"https://pith.science/pith-signing-key.json","notes":"Pith uses this Ed25519 key to sign canonical record SHA-256 digests. Verify with: ed25519_verify(public_key, message=canonical_sha256_bytes, signature=base64decode(signature_b64))."}],"merge_version":"pith-open-graph-merge-v1","built_at":"2026-08-22T01:16:32Z","links":{"resolver":"https://pith.science/pith/2GXS6JSKTNWOVQTNXGO5IKVW2B","bundle":"https://pith.science/pith/2GXS6JSKTNWOVQTNXGO5IKVW2B/bundle.json","state":"https://pith.science/pith/2GXS6JSKTNWOVQTNXGO5IKVW2B/state.json","well_known_bundle":"https://pith.science/.well-known/pith/2GXS6JSKTNWOVQTNXGO5IKVW2B/bundle.json"},"state":{"state_type":"pith_open_graph_state","state_version":"1.0","pith_number":"pith:2024:2GXS6JSKTNWOVQTNXGO5IKVW2B","merge_version":"pith-open-graph-merge-v1","event_count":2,"valid_event_count":2,"invalid_event_count":0,"equivocation_count":0,"current":{"canonical_record":{"metadata":{"abstract_canon_sha256":"19ff8076bde6ab34fae72b7127643d73b788aa27637b292e1e538c80ba1eeb49","cross_cats_sorted":["cs.LG"],"license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","primary_cat":"cs.CL","submitted_at":"2024-05-28T07:53:40Z","title_canon_sha256":"12424b266f150fc1e3640abec34335244d5a0d6a1078e1b2693afaf06e25f133"},"schema_version":"1.0","source":{"id":"2405.17931","kind":"arxiv","version":1}},"source_aliases":[{"alias_kind":"arxiv","alias_value":"2405.17931","created_at":"2026-07-05T08:24:13Z"},{"alias_kind":"arxiv_version","alias_value":"2405.17931v1","created_at":"2026-07-05T08:24:13Z"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.2405.17931","created_at":"2026-07-05T08:24:13Z"},{"alias_kind":"pith_short_12","alias_value":"2GXS6JSKTNWO","created_at":"2026-07-05T08:24:13Z"},{"alias_kind":"pith_short_16","alias_value":"2GXS6JSKTNWOVQTN","created_at":"2026-07-05T08:24:13Z"},{"alias_kind":"pith_short_8","alias_value":"2GXS6JSK","created_at":"2026-07-05T08:24:13Z"}],"graph_snapshots":[{"event_id":"sha256:69408cdc6442c9970b4ec2f5851827cc471106a85e34486fd86f1f185f9ab8f9","target":"graph","created_at":"2026-07-05T08:24:13Z","signer":{"key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signer_id":"pith.science","signer_type":"pith_registry"},"payload":{"graph_snapshot":{"author_claims":{"count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","strong_count":0},"builder_version":"pith-number-builder-2026-05-17-v1","claims":{"count":0,"items":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"integrity":{"available":true,"clean":true,"detectors_run":[],"endpoint":"/pith/2405.17931/integrity.json","findings":[],"snapshot_sha256":"c28c3603d3b5d939e8dc4c7e95fa8dfce3d595e45f758748cecf8e644a296938","summary":{"advisory":0,"by_detector":{},"critical":0,"informational":0}},"paper":{"abstract_excerpt":"Effectively aligning Large Language Models (LLMs) with human-centric values while preventing the degradation of abilities acquired through Pre-training and Supervised Fine-tuning (SFT) poses a central challenge in Reinforcement Learning from Human Feedback (RLHF). In this paper, we first discover that interpolating RLHF and SFT model parameters can adjust the trade-off between human preference and basic capabilities, thereby reducing the alignment tax at the cost of alignment reward. Inspired by this, we propose integrating the RL policy and SFT models at each optimization step in RLHF to cont","authors_text":"Bowen Yu, Chang Zhou, Fei Huang, Keming Lu, Runji Lin, Yang Fan","cross_cats":["cs.LG"],"headline":"","license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","primary_cat":"cs.CL","submitted_at":"2024-05-28T07:53:40Z","title":"Online Merging Optimizers for Boosting Rewards and Mitigating Tax in Alignment"},"references":{"count":0,"internal_anchors":0,"resolved_work":0,"sample":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"2405.17931","kind":"arxiv","version":1},"verdict":{"created_at":null,"id":null,"model_set":{},"one_line_summary":"","pipeline_version":null,"pith_extraction_headline":"","strongest_claim":"","weakest_assumption":""}},"verdict_id":null}}],"author_attestations":[],"timestamp_anchors":[],"storage_attestations":[],"citation_signatures":[],"replication_records":[],"corrections":[],"mirror_hints":[],"record_created":{"event_id":"sha256:4384b73a3ae95afcfe12a9103fa2e63c3ea39587a66132229ba6b5e0d425abe8","target":"record","created_at":"2026-07-05T08:24:13Z","signer":{"key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signer_id":"pith.science","signer_type":"pith_registry"},"payload":{"attestation_state":"computed","canonical_record":{"metadata":{"abstract_canon_sha256":"19ff8076bde6ab34fae72b7127643d73b788aa27637b292e1e538c80ba1eeb49","cross_cats_sorted":["cs.LG"],"license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","primary_cat":"cs.CL","submitted_at":"2024-05-28T07:53:40Z","title_canon_sha256":"12424b266f150fc1e3640abec34335244d5a0d6a1078e1b2693afaf06e25f133"},"schema_version":"1.0","source":{"id":"2405.17931","kind":"arxiv","version":1}},"canonical_sha256":"d1af2f264a9b6ceac26db99dd42ab6d04be3df68e46c3edc4f5c53574931f503","receipt":{"algorithm":"ed25519","builder_version":"pith-number-builder-2026-05-17-v1","canonical_sha256":"d1af2f264a9b6ceac26db99dd42ab6d04be3df68e46c3edc4f5c53574931f503","first_computed_at":"2026-07-05T08:24:13.313885Z","key_id":"pith-v1-2026-05","kind":"pith_receipt","last_reissued_at":"2026-07-05T08:24:13.313885Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","receipt_version":"0.3","signature_b64":"NxRQqeFdfYFugepgz7n7lGlO1VpAPwCqIBOde/pc+k5Uu3DldimvLqMZXXwWxQ05f/W4M+7LZeOif4ffXYSMAw==","signature_status":"signed_v1","signed_at":"2026-07-05T08:24:13.314354Z","signed_message":"canonical_sha256_bytes"},"source_id":"2405.17931","source_kind":"arxiv","source_version":1}}},"equivocations":[],"invalid_events":[],"applied_event_ids":["sha256:4384b73a3ae95afcfe12a9103fa2e63c3ea39587a66132229ba6b5e0d425abe8","sha256:69408cdc6442c9970b4ec2f5851827cc471106a85e34486fd86f1f185f9ab8f9"],"state_sha256":"e69d1cfcf056987f8235c3ddac60bfc4f8142be0c554f429fdd72dad60f6c8ed"},"bundle_signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"0Uyo5iCYBo4TcNAXmq4TAu/F0FIhqpi9leL1dG97Tejn3CKeRY3fc5oConaHlQizHPycczehc583Kfbx9iuOBg==","signed_message":"bundle_sha256_bytes","signed_at":"2026-08-22T01:16:32.041676Z","bundle_sha256":"70846527e454df95a7c722709b0eb5c828bc8b649312b13f4816ce9a9249b4ac"}}