{"state_type":"pith_open_graph_state","state_version":"1.0","pith_number":"pith:2023:3U6WHGE54W42THGIJJXBVKD3MH","merge_version":"pith-open-graph-merge-v1","event_count":2,"valid_event_count":2,"invalid_event_count":0,"equivocation_count":0,"current":{"canonical_record":{"metadata":{"abstract_canon_sha256":"7758e2b3178299f7a74a04ed40d9712be6105a9dd190d6730c29daa489c10590","cross_cats_sorted":["cs.AI","cs.CL"],"license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.LG","submitted_at":"2023-09-01T22:57:20Z","title_canon_sha256":"19dac73cdb0fdbbb55e1effde9053ca0bc4c0f245802b5c9110afbf2f1904a0d"},"schema_version":"1.0","source":{"id":"2309.00754","kind":"arxiv","version":1}},"source_aliases":[{"alias_kind":"arxiv","alias_value":"2309.00754","created_at":"2026-07-05T06:47:06Z"},{"alias_kind":"arxiv_version","alias_value":"2309.00754v1","created_at":"2026-07-05T06:47:06Z"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.2309.00754","created_at":"2026-07-05T06:47:06Z"},{"alias_kind":"pith_short_12","alias_value":"3U6WHGE54W42","created_at":"2026-07-05T06:47:06Z"},{"alias_kind":"pith_short_16","alias_value":"3U6WHGE54W42THGI","created_at":"2026-07-05T06:47:06Z"},{"alias_kind":"pith_short_8","alias_value":"3U6WHGE5","created_at":"2026-07-05T06:47:06Z"}],"graph_snapshots":[{"event_id":"sha256:aa9fcf3ae3bd9572c1e7ae47353191091492ed85c7ee13214c3c1e7afb694f45","target":"graph","created_at":"2026-07-05T06:47:06Z","signer":{"key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signer_id":"pith.science","signer_type":"pith_registry"},"payload":{"graph_snapshot":{"author_claims":{"count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","strong_count":0},"builder_version":"pith-number-builder-2026-05-17-v1","claims":{"count":0,"items":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"integrity":{"available":true,"clean":true,"detectors_run":[],"endpoint":"/pith/2309.00754/integrity.json","findings":[],"snapshot_sha256":"c28c3603d3b5d939e8dc4c7e95fa8dfce3d595e45f758748cecf8e644a296938","summary":{"advisory":0,"by_detector":{},"critical":0,"informational":0}},"paper":{"abstract_excerpt":"Reinforcement Learning with Human Feedback (RLHF) has revolutionized language modeling by aligning models with human preferences. However, the RL stage, Proximal Policy Optimization (PPO), requires over 3x the memory of Supervised Fine-Tuning (SFT), making it infeasible to use for most practitioners. To address this issue, we present a comprehensive analysis the memory usage, performance, and training time of memory-savings techniques for PPO. We introduce Hydra-RLHF by first integrating the SFT and Reward models and then dynamically turning LoRA \"off\" during training. Our experiments show: 1.","authors_text":"Han Yu, Michael Santacroce, Yadong Lu, Yelong Shen, Yuanzhi Li","cross_cats":["cs.AI","cs.CL"],"headline":"","license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.LG","submitted_at":"2023-09-01T22:57:20Z","title":"Efficient RLHF: Reducing the Memory Usage of PPO"},"references":{"count":0,"internal_anchors":0,"resolved_work":0,"sample":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"2309.00754","kind":"arxiv","version":1},"verdict":{"created_at":null,"id":null,"model_set":{},"one_line_summary":"","pipeline_version":null,"pith_extraction_headline":"","strongest_claim":"","weakest_assumption":""}},"verdict_id":null}}],"author_attestations":[],"timestamp_anchors":[],"storage_attestations":[],"citation_signatures":[],"replication_records":[],"corrections":[],"mirror_hints":[],"record_created":{"event_id":"sha256:9deb392eecb90c72b84b312c7f439a6f9c48ad795c9c4fd42e5352265f3bf590","target":"record","created_at":"2026-07-05T06:47:06Z","signer":{"key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signer_id":"pith.science","signer_type":"pith_registry"},"payload":{"attestation_state":"computed","canonical_record":{"metadata":{"abstract_canon_sha256":"7758e2b3178299f7a74a04ed40d9712be6105a9dd190d6730c29daa489c10590","cross_cats_sorted":["cs.AI","cs.CL"],"license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.LG","submitted_at":"2023-09-01T22:57:20Z","title_canon_sha256":"19dac73cdb0fdbbb55e1effde9053ca0bc4c0f245802b5c9110afbf2f1904a0d"},"schema_version":"1.0","source":{"id":"2309.00754","kind":"arxiv","version":1}},"canonical_sha256":"dd3d63989de5b9a99cc84a6e1aa87b61df14f5ed5b85dc1afc065c8296e1c1ab","receipt":{"algorithm":"ed25519","builder_version":"pith-number-builder-2026-05-17-v1","canonical_sha256":"dd3d63989de5b9a99cc84a6e1aa87b61df14f5ed5b85dc1afc065c8296e1c1ab","first_computed_at":"2026-07-05T06:47:06.642575Z","key_id":"pith-v1-2026-05","kind":"pith_receipt","last_reissued_at":"2026-07-05T06:47:06.642575Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","receipt_version":"0.3","signature_b64":"armpgkmuIYE5O6LDlsxFrKlwKrolPwcxymfWCgGfosHYMgtm2SjDMQXCIN5+2UWHDO4nDb+TnKzJyEr2+KMrBQ==","signature_status":"signed_v1","signed_at":"2026-07-05T06:47:06.642988Z","signed_message":"canonical_sha256_bytes"},"source_id":"2309.00754","source_kind":"arxiv","source_version":1}}},"equivocations":[],"invalid_events":[],"applied_event_ids":["sha256:9deb392eecb90c72b84b312c7f439a6f9c48ad795c9c4fd42e5352265f3bf590","sha256:aa9fcf3ae3bd9572c1e7ae47353191091492ed85c7ee13214c3c1e7afb694f45"],"state_sha256":"94fcdcdc3b1adee208183fad61844277081a78d6ad7b83d80c0ae539e1ecb616"}