{"bundle_type":"pith_open_graph_bundle","bundle_version":"1.0","pith_number":"pith:2023:ZGHDMJINES6QP4L72YYOOCBRMT","short_pith_number":"pith:ZGHDMJIN","canonical_record":{"source":{"id":"2309.06256","kind":"arxiv","version":4},"metadata":{"license":"http://creativecommons.org/publicdomain/zero/1.0/","primary_cat":"cs.LG","submitted_at":"2023-09-12T14:16:54Z","cross_cats_sorted":[],"title_canon_sha256":"c01d0956b9fdf046bcf7c956980d437c068d9b9c8a64f396fd7f4ce9d4f70a86","abstract_canon_sha256":"23a0baf6b0f2ca6e9953cca7704bc3e20254b763729480d4590eaa20715448ae"},"schema_version":"1.0"},"canonical_sha256":"c98e36250d24bd07f17fd630e7083164ee83aa323b39514abc1c025a0520a82c","source":{"kind":"arxiv","id":"2309.06256","version":4},"source_aliases":[{"alias_kind":"arxiv","alias_value":"2309.06256","created_at":"2026-07-05T09:19:51Z"},{"alias_kind":"arxiv_version","alias_value":"2309.06256v4","created_at":"2026-07-05T09:19:51Z"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.2309.06256","created_at":"2026-07-05T09:19:51Z"},{"alias_kind":"pith_short_12","alias_value":"ZGHDMJINES6Q","created_at":"2026-07-05T09:19:51Z"},{"alias_kind":"pith_short_16","alias_value":"ZGHDMJINES6QP4L7","created_at":"2026-07-05T09:19:51Z"},{"alias_kind":"pith_short_8","alias_value":"ZGHDMJIN","created_at":"2026-07-05T09:19:51Z"}],"events":[{"event_type":"record_created","subject_pith_number":"pith:2023:ZGHDMJINES6QP4L72YYOOCBRMT","target":"record","payload":{"canonical_record":{"source":{"id":"2309.06256","kind":"arxiv","version":4},"metadata":{"license":"http://creativecommons.org/publicdomain/zero/1.0/","primary_cat":"cs.LG","submitted_at":"2023-09-12T14:16:54Z","cross_cats_sorted":[],"title_canon_sha256":"c01d0956b9fdf046bcf7c956980d437c068d9b9c8a64f396fd7f4ce9d4f70a86","abstract_canon_sha256":"23a0baf6b0f2ca6e9953cca7704bc3e20254b763729480d4590eaa20715448ae"},"schema_version":"1.0"},"canonical_sha256":"c98e36250d24bd07f17fd630e7083164ee83aa323b39514abc1c025a0520a82c","receipt":{"kind":"pith_receipt","key_id":"pith-v1-2026-05","algorithm":"ed25519","signed_at":"2026-07-05T09:19:51.969818Z","signature_b64":"95PgP8ng/jCd2p9p8sAPGDDiHMarMznT3reNX4vb4611s4DobSC+NWfUfdoJyRuJs+vo4QAgeb4GsS8nL8sfBQ==","signed_message":"canonical_sha256_bytes","builder_version":"pith-number-builder-2026-05-17-v1","receipt_version":"0.3","canonical_sha256":"c98e36250d24bd07f17fd630e7083164ee83aa323b39514abc1c025a0520a82c","last_reissued_at":"2026-07-05T09:19:51.969308Z","signature_status":"signed_v1","first_computed_at":"2026-07-05T09:19:51.969308Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"source_kind":"arxiv","source_id":"2309.06256","source_version":4,"attestation_state":"computed"},"signer":{"signer_id":"pith.science","signer_type":"pith_registry","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"created_at":"2026-07-05T09:19:51Z","supersedes":[],"prev_event":null,"signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"JO0OuNmdWam2eueUQUQUTyP6vWJadsM5Lbw0sWwmRZIXmyD9i6VA2jvZM56zJWFf9eCx2mB//bpjPSSRrAqeBQ==","signed_message":"open_graph_event_sha256_bytes","signed_at":"2026-08-09T11:44:27.690864Z"},"content_sha256":"53b159113152e5dd6a73e3cceaabc412f06fd41dd0286e6435acb20566ccf735","schema_version":"1.0","event_id":"sha256:53b159113152e5dd6a73e3cceaabc412f06fd41dd0286e6435acb20566ccf735"},{"event_type":"graph_snapshot","subject_pith_number":"pith:2023:ZGHDMJINES6QP4L72YYOOCBRMT","target":"graph","payload":{"graph_snapshot":{"paper":{"title":"Mitigating the Alignment Tax of RLHF","license":"http://creativecommons.org/publicdomain/zero/1.0/","headline":"","cross_cats":[],"primary_cat":"cs.LG","authors_text":"Hangyu Lin, Hanning Zhang, HanZe Dong, Han Zhao, Haoxiang Wang, Heng Ji, Jianmeng Liu, Jipeng Zhang, Nan Jiang, Renjie Pi, Rui Pan, Shizhe Diao, Tong Zhang, Wei Xiong, Wenbin Hu, Yong Lin, Yuan Yao","submitted_at":"2023-09-12T14:16:54Z","abstract_excerpt":"LLMs acquire a wide range of abilities during pre-training, but aligning LLMs under Reinforcement Learning with Human Feedback (RLHF) can lead to forgetting pretrained abilities, which is also known as the alignment tax. To investigate alignment tax, we conducted experiments with existing RLHF algorithms using OpenLLaMA-3B, which revealed a pronounced alignment tax in NLP tasks. Whereas, despite various techniques to mitigate forgetting, they are often at odds with the RLHF performance, leading to a trade-off between alignment performance and forgetting mitigation, leading to an alignment-forg"},"claims":{"count":0,"items":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"2309.06256","kind":"arxiv","version":4},"verdict":{"id":null,"model_set":{},"created_at":null,"strongest_claim":"","one_line_summary":"","pipeline_version":null,"weakest_assumption":"","pith_extraction_headline":""},"integrity":{"clean":true,"summary":{"advisory":0,"critical":0,"by_detector":{},"informational":0},"endpoint":"/pith/2309.06256/integrity.json","findings":[],"available":true,"detectors_run":[],"snapshot_sha256":"c28c3603d3b5d939e8dc4c7e95fa8dfce3d595e45f758748cecf8e644a296938"},"references":{"count":0,"sample":[],"resolved_work":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","internal_anchors":0},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"author_claims":{"count":0,"strong_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"builder_version":"pith-number-builder-2026-05-17-v1"},"verdict_id":null},"signer":{"signer_id":"pith.science","signer_type":"pith_registry","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"created_at":"2026-07-05T09:19:51Z","supersedes":[],"prev_event":null,"signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"O5I6Z5gJmtU+v5fthPe8+vm8NYxh14dPYLuC/f0uod8QUKardsec1QVnHi+AMr+1Ugz0jMs4UuBDeHjSiEf9AQ==","signed_message":"open_graph_event_sha256_bytes","signed_at":"2026-08-09T11:44:27.691382Z"},"content_sha256":"81dd4c45c6e7eff8a9acc12dc2470e2f5b6b91e1f60001176af7c0894259c0d9","schema_version":"1.0","event_id":"sha256:81dd4c45c6e7eff8a9acc12dc2470e2f5b6b91e1f60001176af7c0894259c0d9"}],"timestamp_proofs":[],"mirror_hints":[{"mirror_type":"https","name":"Pith Resolver","base_url":"https://pith.science","bundle_url":"https://pith.science/pith/ZGHDMJINES6QP4L72YYOOCBRMT/bundle.json","state_url":"https://pith.science/pith/ZGHDMJINES6QP4L72YYOOCBRMT/state.json","well_known_bundle_url":"https://pith.science/.well-known/pith/ZGHDMJINES6QP4L72YYOOCBRMT/bundle.json","status":"primary"}],"public_keys":[{"key_id":"pith-v1-2026-05","algorithm":"ed25519","format":"raw","public_key_b64":"stVStoiQhXFxp4s2pdzPNoqVNBMojDU/fJ2db5S3CbM=","public_key_hex":"b2d552b68890857171a78b36a5dccf368a953413288c353f7c9d9d6f94b709b3","fingerprint_sha256_b32_first128bits":"RVFV5Z2OI2J3ZUO7ERDEBCYNKS","fingerprint_sha256_hex":"8d4b5ee74e4693bcd1df2446408b0d54","rotates_at":null,"url":"https://pith.science/pith-signing-key.json","notes":"Pith uses this Ed25519 key to sign canonical record SHA-256 digests. Verify with: ed25519_verify(public_key, message=canonical_sha256_bytes, signature=base64decode(signature_b64))."}],"merge_version":"pith-open-graph-merge-v1","built_at":"2026-08-09T11:44:27Z","links":{"resolver":"https://pith.science/pith/ZGHDMJINES6QP4L72YYOOCBRMT","bundle":"https://pith.science/pith/ZGHDMJINES6QP4L72YYOOCBRMT/bundle.json","state":"https://pith.science/pith/ZGHDMJINES6QP4L72YYOOCBRMT/state.json","well_known_bundle":"https://pith.science/.well-known/pith/ZGHDMJINES6QP4L72YYOOCBRMT/bundle.json"},"state":{"state_type":"pith_open_graph_state","state_version":"1.0","pith_number":"pith:2023:ZGHDMJINES6QP4L72YYOOCBRMT","merge_version":"pith-open-graph-merge-v1","event_count":2,"valid_event_count":2,"invalid_event_count":0,"equivocation_count":0,"current":{"canonical_record":{"metadata":{"abstract_canon_sha256":"23a0baf6b0f2ca6e9953cca7704bc3e20254b763729480d4590eaa20715448ae","cross_cats_sorted":[],"license":"http://creativecommons.org/publicdomain/zero/1.0/","primary_cat":"cs.LG","submitted_at":"2023-09-12T14:16:54Z","title_canon_sha256":"c01d0956b9fdf046bcf7c956980d437c068d9b9c8a64f396fd7f4ce9d4f70a86"},"schema_version":"1.0","source":{"id":"2309.06256","kind":"arxiv","version":4}},"source_aliases":[{"alias_kind":"arxiv","alias_value":"2309.06256","created_at":"2026-07-05T09:19:51Z"},{"alias_kind":"arxiv_version","alias_value":"2309.06256v4","created_at":"2026-07-05T09:19:51Z"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.2309.06256","created_at":"2026-07-05T09:19:51Z"},{"alias_kind":"pith_short_12","alias_value":"ZGHDMJINES6Q","created_at":"2026-07-05T09:19:51Z"},{"alias_kind":"pith_short_16","alias_value":"ZGHDMJINES6QP4L7","created_at":"2026-07-05T09:19:51Z"},{"alias_kind":"pith_short_8","alias_value":"ZGHDMJIN","created_at":"2026-07-05T09:19:51Z"}],"graph_snapshots":[{"event_id":"sha256:81dd4c45c6e7eff8a9acc12dc2470e2f5b6b91e1f60001176af7c0894259c0d9","target":"graph","created_at":"2026-07-05T09:19:51Z","signer":{"key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signer_id":"pith.science","signer_type":"pith_registry"},"payload":{"graph_snapshot":{"author_claims":{"count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","strong_count":0},"builder_version":"pith-number-builder-2026-05-17-v1","claims":{"count":0,"items":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"integrity":{"available":true,"clean":true,"detectors_run":[],"endpoint":"/pith/2309.06256/integrity.json","findings":[],"snapshot_sha256":"c28c3603d3b5d939e8dc4c7e95fa8dfce3d595e45f758748cecf8e644a296938","summary":{"advisory":0,"by_detector":{},"critical":0,"informational":0}},"paper":{"abstract_excerpt":"LLMs acquire a wide range of abilities during pre-training, but aligning LLMs under Reinforcement Learning with Human Feedback (RLHF) can lead to forgetting pretrained abilities, which is also known as the alignment tax. To investigate alignment tax, we conducted experiments with existing RLHF algorithms using OpenLLaMA-3B, which revealed a pronounced alignment tax in NLP tasks. Whereas, despite various techniques to mitigate forgetting, they are often at odds with the RLHF performance, leading to a trade-off between alignment performance and forgetting mitigation, leading to an alignment-forg","authors_text":"Hangyu Lin, Hanning Zhang, HanZe Dong, Han Zhao, Haoxiang Wang, Heng Ji, Jianmeng Liu, Jipeng Zhang, Nan Jiang, Renjie Pi, Rui Pan, Shizhe Diao, Tong Zhang, Wei Xiong, Wenbin Hu, Yong Lin, Yuan Yao","cross_cats":[],"headline":"","license":"http://creativecommons.org/publicdomain/zero/1.0/","primary_cat":"cs.LG","submitted_at":"2023-09-12T14:16:54Z","title":"Mitigating the Alignment Tax of RLHF"},"references":{"count":0,"internal_anchors":0,"resolved_work":0,"sample":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"2309.06256","kind":"arxiv","version":4},"verdict":{"created_at":null,"id":null,"model_set":{},"one_line_summary":"","pipeline_version":null,"pith_extraction_headline":"","strongest_claim":"","weakest_assumption":""}},"verdict_id":null}}],"author_attestations":[],"timestamp_anchors":[],"storage_attestations":[],"citation_signatures":[],"replication_records":[],"corrections":[],"mirror_hints":[],"record_created":{"event_id":"sha256:53b159113152e5dd6a73e3cceaabc412f06fd41dd0286e6435acb20566ccf735","target":"record","created_at":"2026-07-05T09:19:51Z","signer":{"key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signer_id":"pith.science","signer_type":"pith_registry"},"payload":{"attestation_state":"computed","canonical_record":{"metadata":{"abstract_canon_sha256":"23a0baf6b0f2ca6e9953cca7704bc3e20254b763729480d4590eaa20715448ae","cross_cats_sorted":[],"license":"http://creativecommons.org/publicdomain/zero/1.0/","primary_cat":"cs.LG","submitted_at":"2023-09-12T14:16:54Z","title_canon_sha256":"c01d0956b9fdf046bcf7c956980d437c068d9b9c8a64f396fd7f4ce9d4f70a86"},"schema_version":"1.0","source":{"id":"2309.06256","kind":"arxiv","version":4}},"canonical_sha256":"c98e36250d24bd07f17fd630e7083164ee83aa323b39514abc1c025a0520a82c","receipt":{"algorithm":"ed25519","builder_version":"pith-number-builder-2026-05-17-v1","canonical_sha256":"c98e36250d24bd07f17fd630e7083164ee83aa323b39514abc1c025a0520a82c","first_computed_at":"2026-07-05T09:19:51.969308Z","key_id":"pith-v1-2026-05","kind":"pith_receipt","last_reissued_at":"2026-07-05T09:19:51.969308Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","receipt_version":"0.3","signature_b64":"95PgP8ng/jCd2p9p8sAPGDDiHMarMznT3reNX4vb4611s4DobSC+NWfUfdoJyRuJs+vo4QAgeb4GsS8nL8sfBQ==","signature_status":"signed_v1","signed_at":"2026-07-05T09:19:51.969818Z","signed_message":"canonical_sha256_bytes"},"source_id":"2309.06256","source_kind":"arxiv","source_version":4}}},"equivocations":[],"invalid_events":[],"applied_event_ids":["sha256:53b159113152e5dd6a73e3cceaabc412f06fd41dd0286e6435acb20566ccf735","sha256:81dd4c45c6e7eff8a9acc12dc2470e2f5b6b91e1f60001176af7c0894259c0d9"],"state_sha256":"c29a243990538c38bc5f0eb80b7441031733d0954046c6d4894f48c392cffe1d"},"bundle_signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"W95wDOMkg9onq6trMM9txEcrU6xgYp7A/KT285UEZZwa2imxUeNTZ4JCRpq30zC96b71oSdZ6aYJTGER95P9Cw==","signed_message":"bundle_sha256_bytes","signed_at":"2026-08-09T11:44:27.695091Z","bundle_sha256":"dc925f875829c4a9c20ad4c6f91309dad06e294b61670b0fe15b624cd1d34f1e"}}