{"record_type":"pith_number_record","schema_url":"https://pith.science/schemas/pith-number/v1.json","pith_number":"pith:2025:LEGNLGLF34JZ4ZJ53CLHW7TWJI","short_pith_number":"pith:LEGNLGLF","schema_version":"1.0","canonical_sha256":"590cd59965df139e653dd8967b7e764a2f9f4e826de7f9ec26f739013c03a5b5","source":{"kind":"arxiv","id":"2505.24298","version":5},"attestation_state":"computed","paper":{"title":"AReaL: A Large-Scale Asynchronous Reinforcement Learning System for Language Reasoning","license":"http://creativecommons.org/licenses/by-nc-nd/4.0/","headline":"AReaL decouples generation from training in reinforcement learning to achieve up to 2.77 times faster training for language models on reasoning tasks.","cross_cats":["cs.AI"],"primary_cat":"cs.LG","authors_text":"Binhang Yuan, Chen Zhu, Chuyi He, Guo Wei, Jiashu Wang, Jiaxuan Gao, Jun Mei, Shusheng Xu, Tongkai Yang, Wei Fu, Xujie Shen, Yi Wu, Zhiyu Mei","submitted_at":"2025-05-30T07:18:25Z","abstract_excerpt":"Reinforcement learning (RL) has become a dominant paradigm for training large language models (LLMs), particularly for reasoning tasks. Effective RL for LLMs requires massive parallelization and poses an urgent need for efficient training systems. Most existing large-scale RL systems for LLMs are synchronous, alternating generation and training in a batch setting where rollouts in each training batch are generated by the same model. This approach stabilizes RL training but suffers from severe system-level inefficiency: generation must wait until the longest output in the batch is completed bef"},"verification_status":{"content_addressed":true,"pith_receipt":true,"author_attested":false,"weak_author_claims":0,"strong_author_claims":0,"externally_anchored":false,"storage_verified":false,"citation_signatures":0,"replication_records":0,"graph_snapshot":true,"references_resolved":true,"formal_links_present":false},"canonical_record":{"source":{"id":"2505.24298","kind":"arxiv","version":5},"metadata":{"license":"http://creativecommons.org/licenses/by-nc-nd/4.0/","primary_cat":"cs.LG","submitted_at":"2025-05-30T07:18:25Z","cross_cats_sorted":["cs.AI"],"title_canon_sha256":"ff58ae21a309efe1d147bd9a0bcb0a0041e651f751a57c5a6b49ee3abc1e73de","abstract_canon_sha256":"02100390d6c07329c4b3e7edcd670ebbfc27e4c0175403a6bccdd413babab26e"},"schema_version":"1.0"},"receipt":{"kind":"pith_receipt","key_id":"pith-v1-2026-05","algorithm":"ed25519","signed_at":"2026-05-17T23:38:52.357947Z","signature_b64":"DLHIb7B/htUILWcqbP5/TYFoFdoMR7r6zSqf9R3iEuUpS6S7reAON/cNcHiETnEbAp8AbkvsMu6vPmN7mosUDg==","signed_message":"canonical_sha256_bytes","builder_version":"pith-number-builder-2026-05-17-v1","receipt_version":"0.3","canonical_sha256":"590cd59965df139e653dd8967b7e764a2f9f4e826de7f9ec26f739013c03a5b5","last_reissued_at":"2026-05-17T23:38:52.357524Z","signature_status":"signed_v1","first_computed_at":"2026-05-17T23:38:52.357524Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"graph_snapshot":{"paper":{"title":"AReaL: A Large-Scale Asynchronous Reinforcement Learning System for Language Reasoning","license":"http://creativecommons.org/licenses/by-nc-nd/4.0/","headline":"AReaL decouples generation from training in reinforcement learning to achieve up to 2.77 times faster training for language models on reasoning tasks.","cross_cats":["cs.AI"],"primary_cat":"cs.LG","authors_text":"Binhang Yuan, Chen Zhu, Chuyi He, Guo Wei, Jiashu Wang, Jiaxuan Gao, Jun Mei, Shusheng Xu, Tongkai Yang, Wei Fu, Xujie Shen, Yi Wu, Zhiyu Mei","submitted_at":"2025-05-30T07:18:25Z","abstract_excerpt":"Reinforcement learning (RL) has become a dominant paradigm for training large language models (LLMs), particularly for reasoning tasks. Effective RL for LLMs requires massive parallelization and poses an urgent need for efficient training systems. Most existing large-scale RL systems for LLMs are synchronous, alternating generation and training in a batch setting where rollouts in each training batch are generated by the same model. This approach stabilizes RL training but suffers from severe system-level inefficiency: generation must wait until the longest output in the batch is completed bef"},"claims":{"count":4,"items":[{"kind":"strongest_claim","text":"AReaL achieves up to 2.77× training speedup compared to synchronous systems with the same number of GPUs and matched or improved final performance.","source":"verdict.strongest_claim","status":"machine_extracted","claim_id":"C1","attestation":"unclaimed"},{"kind":"weakest_assumption","text":"That workload balancing between rollout and training workers plus the staleness-enhanced PPO variant can keep training stable and effective despite using outdated samples.","source":"verdict.weakest_assumption","status":"machine_extracted","claim_id":"C2","attestation":"unclaimed"},{"kind":"one_line_summary","text":"AReaL decouples generation and training in LLM reinforcement learning to achieve up to 2.77x speedup with matched or better performance on math and code benchmarks.","source":"verdict.one_line_summary","status":"machine_extracted","claim_id":"C3","attestation":"unclaimed"},{"kind":"headline","text":"AReaL decouples generation from training in reinforcement learning to achieve up to 2.77 times faster training for language models on reasoning tasks.","source":"verdict.pith_extraction.headline","status":"machine_extracted","claim_id":"C4","attestation":"unclaimed"}],"snapshot_sha256":"8f7b78557dc18bf2f5c7402623e5ae37b7da41f48d7a52993445f4dedbc0ba71"},"source":{"id":"2505.24298","kind":"arxiv","version":5},"verdict":{"id":"74e10581-fed9-455c-9d62-a2dedfc2da9e","model_set":{"reader":"grok-4.3"},"created_at":"2026-05-15T14:19:43.363131Z","strongest_claim":"AReaL achieves up to 2.77× training speedup compared to synchronous systems with the same number of GPUs and matched or improved final performance.","one_line_summary":"AReaL decouples generation and training in LLM reinforcement learning to achieve up to 2.77x speedup with matched or better performance on math and code benchmarks.","pipeline_version":"pith-pipeline@v0.9.0","weakest_assumption":"That workload balancing between rollout and training workers plus the staleness-enhanced PPO variant can keep training stable and effective despite using outdated samples.","pith_extraction_headline":"AReaL decouples generation from training in reinforcement learning to achieve up to 2.77 times faster training for language models on reasoning tasks."},"references":{"count":50,"sample":[{"doi":"","year":1912,"title":"Dota 2 with Large Scale Deep Reinforcement Learning","work_id":"b047dc18-e9a3-4d11-8ff6-cd59d41a6357","ref_index":2,"cited_arxiv_id":"1912.06680","is_internal_anchor":true},{"doi":"","year":2021,"title":"Evaluating Large Language Models Trained on Code","work_id":"042493e9-b26f-4b4e-bbde-382072ca9b08","ref_index":3,"cited_arxiv_id":"2107.03374","is_internal_anchor":true},{"doi":"","year":2024,"title":"Z. Chen, A. May, R. Svirschevski, Y . Huang, M. Ryabinin, Z. Jia, and B. Chen. Se- quoia: Scalable and robust speculative decoding. In A. Globerson, L. Mackey, D. Bel- grave, A. Fan, U. Paquet, J. Tom","work_id":"6e8f2590-79af-491f-9142-599ba03cbebb","ref_index":4,"cited_arxiv_id":"","is_internal_anchor":false},{"doi":"","year":2021,"title":"Training Verifiers to Solve Math Word Problems","work_id":"acab1aa8-b4d6-40e0-a3ee-25341701dca2","ref_index":5,"cited_arxiv_id":"2110.14168","is_internal_anchor":true},{"doi":"","year":2018,"title":"L. Espeholt, H. Soyer, R. Munos, K. Simonyan, V . Mnih, T. Ward, Y . Doron, V . Firoiu, T. Harley, I. Dunning, S. Legg, and K. Kavukcuoglu. IMPALA: scalable distributed deep-rl with impor- tance weigh","work_id":"c8686f95-7ff2-4fbf-b5bc-f1243774d697","ref_index":7,"cited_arxiv_id":"","is_internal_anchor":false}],"resolved_work":50,"snapshot_sha256":"8333ebd236a17d4e7295c910e2838d186da222050b0e8e43c0b0565ddf934280","internal_anchors":10},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"author_claims":{"count":0,"strong_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"builder_version":"pith-number-builder-2026-05-17-v1"},"aliases":[{"alias_kind":"arxiv","alias_value":"2505.24298","created_at":"2026-05-17T23:38:52.357591+00:00"},{"alias_kind":"arxiv_version","alias_value":"2505.24298v5","created_at":"2026-05-17T23:38:52.357591+00:00"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.2505.24298","created_at":"2026-05-17T23:38:52.357591+00:00"},{"alias_kind":"pith_short_12","alias_value":"LEGNLGLF34JZ","created_at":"2026-05-18T12:33:37.589309+00:00"},{"alias_kind":"pith_short_16","alias_value":"LEGNLGLF34JZ4ZJ5","created_at":"2026-05-18T12:33:37.589309+00:00"},{"alias_kind":"pith_short_8","alias_value":"LEGNLGLF","created_at":"2026-05-18T12:33:37.589309+00:00"}],"events":[],"event_summary":{},"paper_claims":[],"inbound_citations":{"count":38,"internal_anchor_count":38,"sample":[{"citing_arxiv_id":"2605.20402","citing_title":"Decomposing MXFP4 quantization error for LLM reinforcement learning: reducible bias, recoverable deadzone, and an irreducible floor","ref_index":18,"is_internal_anchor":true},{"citing_arxiv_id":"2504.12501","citing_title":"Reinforcement Learning from Human Feedback","ref_index":134,"is_internal_anchor":true},{"citing_arxiv_id":"2605.20342","citing_title":"ParaVT: Taming the Tool Prior Paradox for Parallel Tool Use in Agentic Video Reinforcement Learning","ref_index":10,"is_internal_anchor":true},{"citing_arxiv_id":"2601.21484","citing_title":"ETS: Energy-Guided Test-Time Scaling for Training-Free RL Alignment","ref_index":10,"is_internal_anchor":true},{"citing_arxiv_id":"2605.20402","citing_title":"Decomposing MXFP4 quantization error for LLM reinforcement learning: reducible bias, recoverable deadzone, and an irreducible floor","ref_index":18,"is_internal_anchor":true},{"citing_arxiv_id":"2605.20342","citing_title":"ParaVT: Taming the Tool Prior Paradox for Parallel Tool Use in Agentic Video Reinforcement Learning","ref_index":10,"is_internal_anchor":true},{"citing_arxiv_id":"2605.18815","citing_title":"DynaTrain: Fast Online Parallelism Switching for Elastic LLM Training","ref_index":2,"is_internal_anchor":true},{"citing_arxiv_id":"2605.15565","citing_title":"AstraFlow: Dataflow-Oriented Reinforcement Learning for Agentic LLMs","ref_index":5,"is_internal_anchor":true},{"citing_arxiv_id":"2605.15224","citing_title":"ICRL: Learning to Internalize Self-Critique with Reinforcement Learning","ref_index":43,"is_internal_anchor":true},{"citing_arxiv_id":"2510.19225","citing_title":"RLBoost: Harvesting Preemptible Resources for Cost-Efficient Reinforcement Learning on LLMs","ref_index":10,"is_internal_anchor":true},{"citing_arxiv_id":"2509.08827","citing_title":"A Survey of Reinforcement Learning for Large Reasoning Models","ref_index":147,"is_internal_anchor":true},{"citing_arxiv_id":"2511.14617","citing_title":"Seer: Online Context Learning for Fast Synchronous LLM Reinforcement Learning","ref_index":6,"is_internal_anchor":true},{"citing_arxiv_id":"2512.12476","citing_title":"HetRL: Efficient Reinforcement Learning for LLMs in Heterogeneous Environments","ref_index":8,"is_internal_anchor":true},{"citing_arxiv_id":"2601.18150","citing_title":"FP8-RL: A Practical and Stable Low-Precision Stack for LLM Reinforcement Learning","ref_index":18,"is_internal_anchor":true},{"citing_arxiv_id":"2601.21484","citing_title":"ETS: Energy-Guided Test-Time Scaling for Training-Free RL Alignment","ref_index":10,"is_internal_anchor":true},{"citing_arxiv_id":"2602.05765","citing_title":"RL-VLA$^3$: A Flexible and Asynchronous Reinforcement Learning Framework for VLA Training","ref_index":4,"is_internal_anchor":true},{"citing_arxiv_id":"2602.06932","citing_title":"When RL Meets Adaptive Speculative Training: A Unified Training-Serving System","ref_index":11,"is_internal_anchor":true},{"citing_arxiv_id":"2603.05295","citing_title":"WebChain: A Large-Scale Human-Annotated Dataset of Real-World Web Interaction Traces","ref_index":6,"is_internal_anchor":true},{"citing_arxiv_id":"2603.10165","citing_title":"OpenClaw-RL: Train Any Agent Simply by Talking","ref_index":1,"is_internal_anchor":true},{"citing_arxiv_id":"2605.13907","citing_title":"AIS: Adaptive Importance Sampling for Quantized RL","ref_index":3,"is_internal_anchor":true},{"citing_arxiv_id":"2605.14220","citing_title":"Diagnosing Training Inference Mismatch in LLM Reinforcement Learning","ref_index":48,"is_internal_anchor":true},{"citing_arxiv_id":"2605.12966","citing_title":"Position: Agentic AI System Is a Foreseeable Pathway to AGI","ref_index":38,"is_internal_anchor":true},{"citing_arxiv_id":"2604.27083","citing_title":"Co-Evolving Policy Distillation","ref_index":21,"is_internal_anchor":true},{"citing_arxiv_id":"2604.26256","citing_title":"DORA: A Scalable Asynchronous Reinforcement Learning System for Language Model Training","ref_index":3,"is_internal_anchor":true},{"citing_arxiv_id":"2503.09567","citing_title":"Towards Reasoning Era: A Survey of Long Chain-of-Thought for Reasoning Large Language Models","ref_index":191,"is_internal_anchor":true}]},"formal_canon":{"evidence_count":0,"sample":[],"anchors":[]},"links":{"html":"https://pith.science/pith/LEGNLGLF34JZ4ZJ53CLHW7TWJI","json":"https://pith.science/pith/LEGNLGLF34JZ4ZJ53CLHW7TWJI.json","graph_json":"https://pith.science/api/pith-number/LEGNLGLF34JZ4ZJ53CLHW7TWJI/graph.json","events_json":"https://pith.science/api/pith-number/LEGNLGLF34JZ4ZJ53CLHW7TWJI/events.json","paper":"https://pith.science/paper/LEGNLGLF"},"agent_actions":{"view_html":"https://pith.science/pith/LEGNLGLF34JZ4ZJ53CLHW7TWJI","download_json":"https://pith.science/pith/LEGNLGLF34JZ4ZJ53CLHW7TWJI.json","view_paper":"https://pith.science/paper/LEGNLGLF","resolve_alias":"https://pith.science/api/pith-number/resolve?arxiv=2505.24298&json=true","fetch_graph":"https://pith.science/api/pith-number/LEGNLGLF34JZ4ZJ53CLHW7TWJI/graph.json","fetch_events":"https://pith.science/api/pith-number/LEGNLGLF34JZ4ZJ53CLHW7TWJI/events.json","actions":{"anchor_timestamp":"https://pith.science/pith/LEGNLGLF34JZ4ZJ53CLHW7TWJI/action/timestamp_anchor","attest_storage":"https://pith.science/pith/LEGNLGLF34JZ4ZJ53CLHW7TWJI/action/storage_attestation","attest_author":"https://pith.science/pith/LEGNLGLF34JZ4ZJ53CLHW7TWJI/action/author_attestation","sign_citation":"https://pith.science/pith/LEGNLGLF34JZ4ZJ53CLHW7TWJI/action/citation_signature","submit_replication":"https://pith.science/pith/LEGNLGLF34JZ4ZJ53CLHW7TWJI/action/replication_record"}},"created_at":"2026-05-17T23:38:52.357591+00:00","updated_at":"2026-05-17T23:38:52.357591+00:00"}