{"bundle_type":"pith_open_graph_bundle","bundle_version":"1.0","pith_number":"pith:2023:J5NBVNQVG7ZNNXBCZUMHOKPKK4","short_pith_number":"pith:J5NBVNQV","canonical_record":{"source":{"id":"2309.10202","kind":"arxiv","version":1},"metadata":{"license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.CL","submitted_at":"2023-09-18T23:06:32Z","cross_cats_sorted":["cs.AI"],"title_canon_sha256":"2c357fa4da585af568aae8a83140b5340c0816b882c5b3b99561ff4f769791ed","abstract_canon_sha256":"11dbc78bf54ce2eaebd2f95761d8c71bc48641b96ac1cc5194bcd53c0f2c0507"},"schema_version":"1.0"},"canonical_sha256":"4f5a1ab61537f2d6dc22cd187729ea572b7167f7bee96cfd60a9c1e436e63382","source":{"kind":"arxiv","id":"2309.10202","version":1},"source_aliases":[{"alias_kind":"arxiv","alias_value":"2309.10202","created_at":"2026-07-05T06:51:52Z"},{"alias_kind":"arxiv_version","alias_value":"2309.10202v1","created_at":"2026-07-05T06:51:52Z"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.2309.10202","created_at":"2026-07-05T06:51:52Z"},{"alias_kind":"pith_short_12","alias_value":"J5NBVNQVG7ZN","created_at":"2026-07-05T06:51:52Z"},{"alias_kind":"pith_short_16","alias_value":"J5NBVNQVG7ZNNXBC","created_at":"2026-07-05T06:51:52Z"},{"alias_kind":"pith_short_8","alias_value":"J5NBVNQV","created_at":"2026-07-05T06:51:52Z"}],"events":[{"event_type":"record_created","subject_pith_number":"pith:2023:J5NBVNQVG7ZNNXBCZUMHOKPKK4","target":"record","payload":{"canonical_record":{"source":{"id":"2309.10202","kind":"arxiv","version":1},"metadata":{"license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.CL","submitted_at":"2023-09-18T23:06:32Z","cross_cats_sorted":["cs.AI"],"title_canon_sha256":"2c357fa4da585af568aae8a83140b5340c0816b882c5b3b99561ff4f769791ed","abstract_canon_sha256":"11dbc78bf54ce2eaebd2f95761d8c71bc48641b96ac1cc5194bcd53c0f2c0507"},"schema_version":"1.0"},"canonical_sha256":"4f5a1ab61537f2d6dc22cd187729ea572b7167f7bee96cfd60a9c1e436e63382","receipt":{"kind":"pith_receipt","key_id":"pith-v1-2026-05","algorithm":"ed25519","signed_at":"2026-07-05T06:51:52.389888Z","signature_b64":"SQk62h34POC/qABxQti/rMLuwE4hKyZjPjjdZqXmuldErp7lJIkGQWZhb+Kn6nekROfLj3R7gz3fo4JZnemiBw==","signed_message":"canonical_sha256_bytes","builder_version":"pith-number-builder-2026-05-17-v1","receipt_version":"0.3","canonical_sha256":"4f5a1ab61537f2d6dc22cd187729ea572b7167f7bee96cfd60a9c1e436e63382","last_reissued_at":"2026-07-05T06:51:52.389407Z","signature_status":"signed_v1","first_computed_at":"2026-07-05T06:51:52.389407Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"source_kind":"arxiv","source_id":"2309.10202","source_version":1,"attestation_state":"computed"},"signer":{"signer_id":"pith.science","signer_type":"pith_registry","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"created_at":"2026-07-05T06:51:52Z","supersedes":[],"prev_event":null,"signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"UQ99ZNmzz43FMojBBlRrL7yniSsweIoPlBhVdcxwYTauCtQa+c+8x420VStv38XAThK1yrSQGEacpQlHWP6vDA==","signed_message":"open_graph_event_sha256_bytes","signed_at":"2026-08-04T21:04:30.346284Z"},"content_sha256":"2b2627e4409bef9a01f2dfd9204bb44c98f6863807fb29fb562621bd7af7c54b","schema_version":"1.0","event_id":"sha256:2b2627e4409bef9a01f2dfd9204bb44c98f6863807fb29fb562621bd7af7c54b"},{"event_type":"graph_snapshot","subject_pith_number":"pith:2023:J5NBVNQVG7ZNNXBCZUMHOKPKK4","target":"graph","payload":{"graph_snapshot":{"paper":{"title":"Stabilizing RLHF through Advantage Model and Selective Rehearsal","license":"http://creativecommons.org/licenses/by/4.0/","headline":"","cross_cats":["cs.AI"],"primary_cat":"cs.CL","authors_text":"Baolin Peng, Dong Yu, Haitao Mi, Lifeng Jin, Linfeng Song, Ye Tian","submitted_at":"2023-09-18T23:06:32Z","abstract_excerpt":"Large Language Models (LLMs) have revolutionized natural language processing, yet aligning these models with human values and preferences using RLHF remains a significant challenge. This challenge is characterized by various instabilities, such as reward hacking and catastrophic forgetting. In this technical report, we propose two innovations to stabilize RLHF training: 1) Advantage Model, which directly models advantage score i.e., extra reward compared to the expected rewards and regulates score distributions across tasks to prevent reward hacking. 2) Selective Rehearsal, which mitigates cat"},"claims":{"count":0,"items":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"2309.10202","kind":"arxiv","version":1},"verdict":{"id":null,"model_set":{},"created_at":null,"strongest_claim":"","one_line_summary":"","pipeline_version":null,"weakest_assumption":"","pith_extraction_headline":""},"integrity":{"clean":true,"summary":{"advisory":0,"critical":0,"by_detector":{},"informational":0},"endpoint":"/pith/2309.10202/integrity.json","findings":[],"available":true,"detectors_run":[],"snapshot_sha256":"c28c3603d3b5d939e8dc4c7e95fa8dfce3d595e45f758748cecf8e644a296938"},"references":{"count":0,"sample":[],"resolved_work":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","internal_anchors":0},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"author_claims":{"count":0,"strong_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"builder_version":"pith-number-builder-2026-05-17-v1"},"verdict_id":null},"signer":{"signer_id":"pith.science","signer_type":"pith_registry","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"created_at":"2026-07-05T06:51:52Z","supersedes":[],"prev_event":null,"signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"AVfI9guPHzYLFUerfhI0JWx8SfIjXQoKHH2OeTU+QzlZdFqfQKMZm4RAQbFfwJCt7PMFRhoYu+w5oDgKNQOiBg==","signed_message":"open_graph_event_sha256_bytes","signed_at":"2026-08-04T21:04:30.346796Z"},"content_sha256":"5efb0ff960ed4dbd2512e218aac2cb603cb4eaa604c23f61b68acd7b6def999e","schema_version":"1.0","event_id":"sha256:5efb0ff960ed4dbd2512e218aac2cb603cb4eaa604c23f61b68acd7b6def999e"}],"timestamp_proofs":[],"mirror_hints":[{"mirror_type":"https","name":"Pith Resolver","base_url":"https://pith.science","bundle_url":"https://pith.science/pith/J5NBVNQVG7ZNNXBCZUMHOKPKK4/bundle.json","state_url":"https://pith.science/pith/J5NBVNQVG7ZNNXBCZUMHOKPKK4/state.json","well_known_bundle_url":"https://pith.science/.well-known/pith/J5NBVNQVG7ZNNXBCZUMHOKPKK4/bundle.json","status":"primary"}],"public_keys":[{"key_id":"pith-v1-2026-05","algorithm":"ed25519","format":"raw","public_key_b64":"stVStoiQhXFxp4s2pdzPNoqVNBMojDU/fJ2db5S3CbM=","public_key_hex":"b2d552b68890857171a78b36a5dccf368a953413288c353f7c9d9d6f94b709b3","fingerprint_sha256_b32_first128bits":"RVFV5Z2OI2J3ZUO7ERDEBCYNKS","fingerprint_sha256_hex":"8d4b5ee74e4693bcd1df2446408b0d54","rotates_at":null,"url":"https://pith.science/pith-signing-key.json","notes":"Pith uses this Ed25519 key to sign canonical record SHA-256 digests. Verify with: ed25519_verify(public_key, message=canonical_sha256_bytes, signature=base64decode(signature_b64))."}],"merge_version":"pith-open-graph-merge-v1","built_at":"2026-08-04T21:04:30Z","links":{"resolver":"https://pith.science/pith/J5NBVNQVG7ZNNXBCZUMHOKPKK4","bundle":"https://pith.science/pith/J5NBVNQVG7ZNNXBCZUMHOKPKK4/bundle.json","state":"https://pith.science/pith/J5NBVNQVG7ZNNXBCZUMHOKPKK4/state.json","well_known_bundle":"https://pith.science/.well-known/pith/J5NBVNQVG7ZNNXBCZUMHOKPKK4/bundle.json"},"state":{"state_type":"pith_open_graph_state","state_version":"1.0","pith_number":"pith:2023:J5NBVNQVG7ZNNXBCZUMHOKPKK4","merge_version":"pith-open-graph-merge-v1","event_count":2,"valid_event_count":2,"invalid_event_count":0,"equivocation_count":0,"current":{"canonical_record":{"metadata":{"abstract_canon_sha256":"11dbc78bf54ce2eaebd2f95761d8c71bc48641b96ac1cc5194bcd53c0f2c0507","cross_cats_sorted":["cs.AI"],"license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.CL","submitted_at":"2023-09-18T23:06:32Z","title_canon_sha256":"2c357fa4da585af568aae8a83140b5340c0816b882c5b3b99561ff4f769791ed"},"schema_version":"1.0","source":{"id":"2309.10202","kind":"arxiv","version":1}},"source_aliases":[{"alias_kind":"arxiv","alias_value":"2309.10202","created_at":"2026-07-05T06:51:52Z"},{"alias_kind":"arxiv_version","alias_value":"2309.10202v1","created_at":"2026-07-05T06:51:52Z"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.2309.10202","created_at":"2026-07-05T06:51:52Z"},{"alias_kind":"pith_short_12","alias_value":"J5NBVNQVG7ZN","created_at":"2026-07-05T06:51:52Z"},{"alias_kind":"pith_short_16","alias_value":"J5NBVNQVG7ZNNXBC","created_at":"2026-07-05T06:51:52Z"},{"alias_kind":"pith_short_8","alias_value":"J5NBVNQV","created_at":"2026-07-05T06:51:52Z"}],"graph_snapshots":[{"event_id":"sha256:5efb0ff960ed4dbd2512e218aac2cb603cb4eaa604c23f61b68acd7b6def999e","target":"graph","created_at":"2026-07-05T06:51:52Z","signer":{"key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signer_id":"pith.science","signer_type":"pith_registry"},"payload":{"graph_snapshot":{"author_claims":{"count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","strong_count":0},"builder_version":"pith-number-builder-2026-05-17-v1","claims":{"count":0,"items":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"integrity":{"available":true,"clean":true,"detectors_run":[],"endpoint":"/pith/2309.10202/integrity.json","findings":[],"snapshot_sha256":"c28c3603d3b5d939e8dc4c7e95fa8dfce3d595e45f758748cecf8e644a296938","summary":{"advisory":0,"by_detector":{},"critical":0,"informational":0}},"paper":{"abstract_excerpt":"Large Language Models (LLMs) have revolutionized natural language processing, yet aligning these models with human values and preferences using RLHF remains a significant challenge. This challenge is characterized by various instabilities, such as reward hacking and catastrophic forgetting. In this technical report, we propose two innovations to stabilize RLHF training: 1) Advantage Model, which directly models advantage score i.e., extra reward compared to the expected rewards and regulates score distributions across tasks to prevent reward hacking. 2) Selective Rehearsal, which mitigates cat","authors_text":"Baolin Peng, Dong Yu, Haitao Mi, Lifeng Jin, Linfeng Song, Ye Tian","cross_cats":["cs.AI"],"headline":"","license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.CL","submitted_at":"2023-09-18T23:06:32Z","title":"Stabilizing RLHF through Advantage Model and Selective Rehearsal"},"references":{"count":0,"internal_anchors":0,"resolved_work":0,"sample":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"2309.10202","kind":"arxiv","version":1},"verdict":{"created_at":null,"id":null,"model_set":{},"one_line_summary":"","pipeline_version":null,"pith_extraction_headline":"","strongest_claim":"","weakest_assumption":""}},"verdict_id":null}}],"author_attestations":[],"timestamp_anchors":[],"storage_attestations":[],"citation_signatures":[],"replication_records":[],"corrections":[],"mirror_hints":[],"record_created":{"event_id":"sha256:2b2627e4409bef9a01f2dfd9204bb44c98f6863807fb29fb562621bd7af7c54b","target":"record","created_at":"2026-07-05T06:51:52Z","signer":{"key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signer_id":"pith.science","signer_type":"pith_registry"},"payload":{"attestation_state":"computed","canonical_record":{"metadata":{"abstract_canon_sha256":"11dbc78bf54ce2eaebd2f95761d8c71bc48641b96ac1cc5194bcd53c0f2c0507","cross_cats_sorted":["cs.AI"],"license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.CL","submitted_at":"2023-09-18T23:06:32Z","title_canon_sha256":"2c357fa4da585af568aae8a83140b5340c0816b882c5b3b99561ff4f769791ed"},"schema_version":"1.0","source":{"id":"2309.10202","kind":"arxiv","version":1}},"canonical_sha256":"4f5a1ab61537f2d6dc22cd187729ea572b7167f7bee96cfd60a9c1e436e63382","receipt":{"algorithm":"ed25519","builder_version":"pith-number-builder-2026-05-17-v1","canonical_sha256":"4f5a1ab61537f2d6dc22cd187729ea572b7167f7bee96cfd60a9c1e436e63382","first_computed_at":"2026-07-05T06:51:52.389407Z","key_id":"pith-v1-2026-05","kind":"pith_receipt","last_reissued_at":"2026-07-05T06:51:52.389407Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","receipt_version":"0.3","signature_b64":"SQk62h34POC/qABxQti/rMLuwE4hKyZjPjjdZqXmuldErp7lJIkGQWZhb+Kn6nekROfLj3R7gz3fo4JZnemiBw==","signature_status":"signed_v1","signed_at":"2026-07-05T06:51:52.389888Z","signed_message":"canonical_sha256_bytes"},"source_id":"2309.10202","source_kind":"arxiv","source_version":1}}},"equivocations":[],"invalid_events":[],"applied_event_ids":["sha256:2b2627e4409bef9a01f2dfd9204bb44c98f6863807fb29fb562621bd7af7c54b","sha256:5efb0ff960ed4dbd2512e218aac2cb603cb4eaa604c23f61b68acd7b6def999e"],"state_sha256":"c14acafb7b75299c4a641609866ef06e7efb2a5efaa68482eb87aca98aa46bb0"},"bundle_signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"AQ+oaS6ohIArwp+jvF2a+sUm9pS8v/X1p84EEA8itEbjfbyO4TtJvvvMcQjS0CryP4kGvRmnDYhAdBQDZLpEAQ==","signed_message":"bundle_sha256_bytes","signed_at":"2026-08-04T21:04:30.349906Z","bundle_sha256":"ddb9645c7636645010801c124c36f473aaab4c2736112224e1d2eb7ecefd1606"}}