{"record_type":"pith_number_record","schema_url":"https://pith.science/schemas/pith-number/v1.json","pith_number":"pith:2024:2TAQZXOZH2HNYCTC74RYD7GBVB","short_pith_number":"pith:2TAQZXOZ","schema_version":"1.0","canonical_sha256":"d4c10cddd93e8edc0a62ff2381fcc1a84774b253a6b9d08207c6992100a02a5d","source":{"kind":"arxiv","id":"2410.18252","version":3},"attestation_state":"computed","paper":{"title":"Asynchronous RLHF: Faster and More Efficient Off-Policy RL for Language Models","license":"http://creativecommons.org/licenses/by-nc-nd/4.0/","headline":"","cross_cats":["cs.AI","cs.CL"],"primary_cat":"cs.LG","authors_text":"Aaron Courville, Arian Hosseini, Michael Noukhovitch, Rishabh Agarwal, Shengyi Huang, Sophie Xhonneux","submitted_at":"2024-10-23T19:59:50Z","abstract_excerpt":"The dominant paradigm for RLHF is online and on-policy RL: synchronously generating from the large language model (LLM) policy, labelling with a reward model, and learning using feedback on the LLM's own outputs. While performant, this paradigm is computationally inefficient. Inspired by classical deep RL literature, we propose separating generation and learning in RLHF. This enables asynchronous generation of new samples while simultaneously training on old samples, leading to faster training and more compute-optimal scaling. However, asynchronous training relies on an underexplored regime, o"},"verification_status":{"content_addressed":true,"pith_receipt":true,"author_attested":false,"weak_author_claims":0,"strong_author_claims":0,"externally_anchored":false,"storage_verified":false,"citation_signatures":0,"replication_records":0,"graph_snapshot":true,"references_resolved":false,"formal_links_present":false},"canonical_record":{"source":{"id":"2410.18252","kind":"arxiv","version":3},"metadata":{"license":"http://creativecommons.org/licenses/by-nc-nd/4.0/","primary_cat":"cs.LG","submitted_at":"2024-10-23T19:59:50Z","cross_cats_sorted":["cs.AI","cs.CL"],"title_canon_sha256":"bf0402ab32c496401c009c0bcec4287787a0ec7d8a061d30967554184ef0ca05","abstract_canon_sha256":"df25b02154dc0a1279acf9171edfdfbe188d5b69b5a5cf99a70837cdbd5c1277"},"schema_version":"1.0"},"receipt":{"kind":"pith_receipt","key_id":"pith-v1-2026-05","algorithm":"ed25519","signed_at":"2026-07-05T10:54:13.559705Z","signature_b64":"zEJYMJ30+nrLbvxsYbYAUT25durOvtapNiwkxMJwEriWE34Yw/ENFjWsc7ao2fPLfKrP2y5qT7vli1T/csKVDA==","signed_message":"canonical_sha256_bytes","builder_version":"pith-number-builder-2026-05-17-v1","receipt_version":"0.3","canonical_sha256":"d4c10cddd93e8edc0a62ff2381fcc1a84774b253a6b9d08207c6992100a02a5d","last_reissued_at":"2026-07-05T10:54:13.559190Z","signature_status":"signed_v1","first_computed_at":"2026-07-05T10:54:13.559190Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"graph_snapshot":{"paper":{"title":"Asynchronous RLHF: Faster and More Efficient Off-Policy RL for Language Models","license":"http://creativecommons.org/licenses/by-nc-nd/4.0/","headline":"","cross_cats":["cs.AI","cs.CL"],"primary_cat":"cs.LG","authors_text":"Aaron Courville, Arian Hosseini, Michael Noukhovitch, Rishabh Agarwal, Shengyi Huang, Sophie Xhonneux","submitted_at":"2024-10-23T19:59:50Z","abstract_excerpt":"The dominant paradigm for RLHF is online and on-policy RL: synchronously generating from the large language model (LLM) policy, labelling with a reward model, and learning using feedback on the LLM's own outputs. While performant, this paradigm is computationally inefficient. Inspired by classical deep RL literature, we propose separating generation and learning in RLHF. This enables asynchronous generation of new samples while simultaneously training on old samples, leading to faster training and more compute-optimal scaling. However, asynchronous training relies on an underexplored regime, o"},"claims":{"count":0,"items":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"2410.18252","kind":"arxiv","version":3},"verdict":{"id":null,"model_set":{},"created_at":null,"strongest_claim":"","one_line_summary":"","pipeline_version":null,"weakest_assumption":"","pith_extraction_headline":""},"integrity":{"clean":true,"summary":{"advisory":0,"critical":0,"by_detector":{},"informational":0},"endpoint":"/pith/2410.18252/integrity.json","findings":[],"available":true,"detectors_run":[],"snapshot_sha256":"c28c3603d3b5d939e8dc4c7e95fa8dfce3d595e45f758748cecf8e644a296938"},"references":{"count":0,"sample":[],"resolved_work":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","internal_anchors":0},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"author_claims":{"count":0,"strong_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"builder_version":"pith-number-builder-2026-05-17-v1"},"aliases":[{"alias_kind":"arxiv","alias_value":"2410.18252","created_at":"2026-07-05T10:54:13.559248+00:00"},{"alias_kind":"arxiv_version","alias_value":"2410.18252v3","created_at":"2026-07-05T10:54:13.559248+00:00"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.2410.18252","created_at":"2026-07-05T10:54:13.559248+00:00"},{"alias_kind":"pith_short_12","alias_value":"2TAQZXOZH2HN","created_at":"2026-07-05T10:54:13.559248+00:00"},{"alias_kind":"pith_short_16","alias_value":"2TAQZXOZH2HNYCTC","created_at":"2026-07-05T10:54:13.559248+00:00"},{"alias_kind":"pith_short_8","alias_value":"2TAQZXOZ","created_at":"2026-07-05T10:54:13.559248+00:00"}],"events":[],"event_summary":{},"paper_claims":[],"inbound_citations":{"count":15,"internal_anchor_count":2,"sample":[{"citing_arxiv_id":"2607.06763","citing_title":"Trees from Marginals: Autoregressive drafting with factorized priors","ref_index":3,"is_internal_anchor":true},{"citing_arxiv_id":"2607.07508","citing_title":"Single-Rollout Asynchronous Optimization for Agentic Reinforcement Learning","ref_index":13,"is_internal_anchor":true},{"citing_arxiv_id":"2607.02390","citing_title":"DecompRL: Solving Harder Problems by Learning Modular Code Generation","ref_index":42,"is_internal_anchor":false},{"citing_arxiv_id":"2606.11867","citing_title":"Harnessing Routing Foresight for Micro-step-level MoE load balancing in RL Post-training","ref_index":40,"is_internal_anchor":false},{"citing_arxiv_id":"2606.05597","citing_title":"AsyncWebRL: Efficient Asynchronous Reinforcement Learning for Multi-Step Visual Web Agents","ref_index":24,"is_internal_anchor":false},{"citing_arxiv_id":"2606.03070","citing_title":"ASymPO: Asymmetric-Scale Policy Optimization for Asynchronous LLM Post-Training Without Behavior Information","ref_index":40,"is_internal_anchor":false},{"citing_arxiv_id":"2606.27580","citing_title":"Retroactive Advantage Correction: Closed-Form V-Trace Bias Correction for Delay-Aware RLHF","ref_index":13,"is_internal_anchor":false},{"citing_arxiv_id":"2502.12272","citing_title":"Learning to Reason at the Frontier of Learnability","ref_index":21,"is_internal_anchor":false},{"citing_arxiv_id":"2605.15565","citing_title":"AstraFlow: Dataflow-Oriented Reinforcement Learning for Agentic LLMs","ref_index":24,"is_internal_anchor":false},{"citing_arxiv_id":"2511.18871","citing_title":"Periodic Asynchrony: An On-Policy Approach for Accelerating LLM Reinforcement Learning","ref_index":13,"is_internal_anchor":false},{"citing_arxiv_id":"2605.14220","citing_title":"Diagnosing Training Inference Mismatch in LLM Reinforcement Learning","ref_index":50,"is_internal_anchor":false},{"citing_arxiv_id":"2604.26256","citing_title":"DORA: A Scalable Asynchronous Reinforcement Learning System for Language Model Training","ref_index":8,"is_internal_anchor":false},{"citing_arxiv_id":"2604.23838","citing_title":"JigsawRL: Assembling RL Pipelines for Efficient LLM Post-Training","ref_index":29,"is_internal_anchor":false},{"citing_arxiv_id":"2604.09107","citing_title":"TensorHub: Scalable and Elastic Weight Transfer for LLM RL Training","ref_index":22,"is_internal_anchor":false},{"citing_arxiv_id":"2604.16918","citing_title":"Freshness-Aware Prioritized Experience Replay for LLM/VLM Reinforcement Learning","ref_index":12,"is_internal_anchor":false}]},"formal_canon":{"evidence_count":0,"sample":[],"anchors":[]},"links":{"html":"https://pith.science/pith/2TAQZXOZH2HNYCTC74RYD7GBVB","json":"https://pith.science/pith/2TAQZXOZH2HNYCTC74RYD7GBVB.json","graph_json":"https://pith.science/api/pith-number/2TAQZXOZH2HNYCTC74RYD7GBVB/graph.json","events_json":"https://pith.science/api/pith-number/2TAQZXOZH2HNYCTC74RYD7GBVB/events.json","paper":"https://pith.science/paper/2TAQZXOZ"},"agent_actions":{"view_html":"https://pith.science/pith/2TAQZXOZH2HNYCTC74RYD7GBVB","download_json":"https://pith.science/pith/2TAQZXOZH2HNYCTC74RYD7GBVB.json","view_paper":"https://pith.science/paper/2TAQZXOZ","resolve_alias":"https://pith.science/api/pith-number/resolve?arxiv=2410.18252&json=true","fetch_graph":"https://pith.science/api/pith-number/2TAQZXOZH2HNYCTC74RYD7GBVB/graph.json","fetch_events":"https://pith.science/api/pith-number/2TAQZXOZH2HNYCTC74RYD7GBVB/events.json","actions":{"anchor_timestamp":"https://pith.science/pith/2TAQZXOZH2HNYCTC74RYD7GBVB/action/timestamp_anchor","attest_storage":"https://pith.science/pith/2TAQZXOZH2HNYCTC74RYD7GBVB/action/storage_attestation","attest_author":"https://pith.science/pith/2TAQZXOZH2HNYCTC74RYD7GBVB/action/author_attestation","sign_citation":"https://pith.science/pith/2TAQZXOZH2HNYCTC74RYD7GBVB/action/citation_signature","submit_replication":"https://pith.science/pith/2TAQZXOZH2HNYCTC74RYD7GBVB/action/replication_record"}},"created_at":"2026-07-05T10:54:13.559248+00:00","updated_at":"2026-07-05T10:54:13.559248+00:00"}