{"record_type":"pith_number_record","schema_url":"https://pith.science/schemas/pith-number/v1.json","pith_number":"pith:2023:3U6WHGE54W42THGIJJXBVKD3MH","short_pith_number":"pith:3U6WHGE5","schema_version":"1.0","canonical_sha256":"dd3d63989de5b9a99cc84a6e1aa87b61df14f5ed5b85dc1afc065c8296e1c1ab","source":{"kind":"arxiv","id":"2309.00754","version":1},"attestation_state":"computed","paper":{"title":"Efficient RLHF: Reducing the Memory Usage of PPO","license":"http://creativecommons.org/licenses/by/4.0/","headline":"","cross_cats":["cs.AI","cs.CL"],"primary_cat":"cs.LG","authors_text":"Han Yu, Michael Santacroce, Yadong Lu, Yelong Shen, Yuanzhi Li","submitted_at":"2023-09-01T22:57:20Z","abstract_excerpt":"Reinforcement Learning with Human Feedback (RLHF) has revolutionized language modeling by aligning models with human preferences. However, the RL stage, Proximal Policy Optimization (PPO), requires over 3x the memory of Supervised Fine-Tuning (SFT), making it infeasible to use for most practitioners. To address this issue, we present a comprehensive analysis the memory usage, performance, and training time of memory-savings techniques for PPO. We introduce Hydra-RLHF by first integrating the SFT and Reward models and then dynamically turning LoRA \"off\" during training. Our experiments show: 1."},"verification_status":{"content_addressed":true,"pith_receipt":true,"author_attested":false,"weak_author_claims":0,"strong_author_claims":0,"externally_anchored":false,"storage_verified":false,"citation_signatures":0,"replication_records":0,"graph_snapshot":true,"references_resolved":false,"formal_links_present":false},"canonical_record":{"source":{"id":"2309.00754","kind":"arxiv","version":1},"metadata":{"license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.LG","submitted_at":"2023-09-01T22:57:20Z","cross_cats_sorted":["cs.AI","cs.CL"],"title_canon_sha256":"19dac73cdb0fdbbb55e1effde9053ca0bc4c0f245802b5c9110afbf2f1904a0d","abstract_canon_sha256":"7758e2b3178299f7a74a04ed40d9712be6105a9dd190d6730c29daa489c10590"},"schema_version":"1.0"},"receipt":{"kind":"pith_receipt","key_id":"pith-v1-2026-05","algorithm":"ed25519","signed_at":"2026-07-05T06:47:06.642988Z","signature_b64":"armpgkmuIYE5O6LDlsxFrKlwKrolPwcxymfWCgGfosHYMgtm2SjDMQXCIN5+2UWHDO4nDb+TnKzJyEr2+KMrBQ==","signed_message":"canonical_sha256_bytes","builder_version":"pith-number-builder-2026-05-17-v1","receipt_version":"0.3","canonical_sha256":"dd3d63989de5b9a99cc84a6e1aa87b61df14f5ed5b85dc1afc065c8296e1c1ab","last_reissued_at":"2026-07-05T06:47:06.642575Z","signature_status":"signed_v1","first_computed_at":"2026-07-05T06:47:06.642575Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"graph_snapshot":{"paper":{"title":"Efficient RLHF: Reducing the Memory Usage of PPO","license":"http://creativecommons.org/licenses/by/4.0/","headline":"","cross_cats":["cs.AI","cs.CL"],"primary_cat":"cs.LG","authors_text":"Han Yu, Michael Santacroce, Yadong Lu, Yelong Shen, Yuanzhi Li","submitted_at":"2023-09-01T22:57:20Z","abstract_excerpt":"Reinforcement Learning with Human Feedback (RLHF) has revolutionized language modeling by aligning models with human preferences. However, the RL stage, Proximal Policy Optimization (PPO), requires over 3x the memory of Supervised Fine-Tuning (SFT), making it infeasible to use for most practitioners. To address this issue, we present a comprehensive analysis the memory usage, performance, and training time of memory-savings techniques for PPO. We introduce Hydra-RLHF by first integrating the SFT and Reward models and then dynamically turning LoRA \"off\" during training. Our experiments show: 1."},"claims":{"count":0,"items":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"2309.00754","kind":"arxiv","version":1},"verdict":{"id":null,"model_set":{},"created_at":null,"strongest_claim":"","one_line_summary":"","pipeline_version":null,"weakest_assumption":"","pith_extraction_headline":""},"integrity":{"clean":true,"summary":{"advisory":0,"critical":0,"by_detector":{},"informational":0},"endpoint":"/pith/2309.00754/integrity.json","findings":[],"available":true,"detectors_run":[],"snapshot_sha256":"c28c3603d3b5d939e8dc4c7e95fa8dfce3d595e45f758748cecf8e644a296938"},"references":{"count":0,"sample":[],"resolved_work":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","internal_anchors":0},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"author_claims":{"count":0,"strong_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"builder_version":"pith-number-builder-2026-05-17-v1"},"aliases":[{"alias_kind":"arxiv","alias_value":"2309.00754","created_at":"2026-07-05T06:47:06.642634+00:00"},{"alias_kind":"arxiv_version","alias_value":"2309.00754v1","created_at":"2026-07-05T06:47:06.642634+00:00"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.2309.00754","created_at":"2026-07-05T06:47:06.642634+00:00"},{"alias_kind":"pith_short_12","alias_value":"3U6WHGE54W42","created_at":"2026-07-05T06:47:06.642634+00:00"},{"alias_kind":"pith_short_16","alias_value":"3U6WHGE54W42THGI","created_at":"2026-07-05T06:47:06.642634+00:00"},{"alias_kind":"pith_short_8","alias_value":"3U6WHGE5","created_at":"2026-07-05T06:47:06.642634+00:00"}],"events":[],"event_summary":{},"paper_claims":[],"inbound_citations":{"count":6,"internal_anchor_count":0,"sample":[{"citing_arxiv_id":"2605.12207","citing_title":"Not How Many, But Which: Parameter Placement in Low-Rank Adaptation","ref_index":73,"is_internal_anchor":false},{"citing_arxiv_id":"2605.08527","citing_title":"MARLaaS: Multi-Tenant Asynchronous Reinforcement Learning as a Service","ref_index":2,"is_internal_anchor":false},{"citing_arxiv_id":"2605.08378","citing_title":"Reinforcement Learning for Scalable and Trustworthy Intelligent Systems","ref_index":96,"is_internal_anchor":false},{"citing_arxiv_id":"2409.19256","citing_title":"HybridFlow: A Flexible and Efficient RLHF Framework","ref_index":75,"is_internal_anchor":false},{"citing_arxiv_id":"2604.17396","citing_title":"Representation-Guided Parameter-Efficient LLM Unlearning","ref_index":202,"is_internal_anchor":false},{"citing_arxiv_id":"2605.02469","citing_title":"Reference-Sampled Boltzmann Projection for KL-Regularized RLVR: Target-Matched Weighted SFT, Finite One-Shot Gaps, and Policy Mirror Descent","ref_index":40,"is_internal_anchor":false}]},"formal_canon":{"evidence_count":0,"sample":[],"anchors":[]},"links":{"html":"https://pith.science/pith/3U6WHGE54W42THGIJJXBVKD3MH","json":"https://pith.science/pith/3U6WHGE54W42THGIJJXBVKD3MH.json","graph_json":"https://pith.science/api/pith-number/3U6WHGE54W42THGIJJXBVKD3MH/graph.json","events_json":"https://pith.science/api/pith-number/3U6WHGE54W42THGIJJXBVKD3MH/events.json","paper":"https://pith.science/paper/3U6WHGE5"},"agent_actions":{"view_html":"https://pith.science/pith/3U6WHGE54W42THGIJJXBVKD3MH","download_json":"https://pith.science/pith/3U6WHGE54W42THGIJJXBVKD3MH.json","view_paper":"https://pith.science/paper/3U6WHGE5","resolve_alias":"https://pith.science/api/pith-number/resolve?arxiv=2309.00754&json=true","fetch_graph":"https://pith.science/api/pith-number/3U6WHGE54W42THGIJJXBVKD3MH/graph.json","fetch_events":"https://pith.science/api/pith-number/3U6WHGE54W42THGIJJXBVKD3MH/events.json","actions":{"anchor_timestamp":"https://pith.science/pith/3U6WHGE54W42THGIJJXBVKD3MH/action/timestamp_anchor","attest_storage":"https://pith.science/pith/3U6WHGE54W42THGIJJXBVKD3MH/action/storage_attestation","attest_author":"https://pith.science/pith/3U6WHGE54W42THGIJJXBVKD3MH/action/author_attestation","sign_citation":"https://pith.science/pith/3U6WHGE54W42THGIJJXBVKD3MH/action/citation_signature","submit_replication":"https://pith.science/pith/3U6WHGE54W42THGIJJXBVKD3MH/action/replication_record"}},"created_at":"2026-07-05T06:47:06.642634+00:00","updated_at":"2026-07-05T06:47:06.642634+00:00"}