{"record_type":"pith_number_record","schema_url":"https://pith.science/schemas/pith-number/v1.json","pith_number":"pith:2019:LG76FO4FXIRQAKPLNJWYADOHLW","short_pith_number":"pith:LG76FO4F","schema_version":"1.0","canonical_sha256":"59bfe2bb85ba230029eb6a6d800dc75da176779950b8cf7ce12fe03970dfb98d","source":{"kind":"arxiv","id":"1910.02054","version":3},"attestation_state":"computed","paper":{"title":"ZeRO: Memory Optimizations Toward Training Trillion Parameter Models","license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","headline":"ZeRO partitions optimizer states and gradients across devices to remove memory redundancy in parallel training.","cross_cats":["cs.DC","stat.ML"],"primary_cat":"cs.LG","authors_text":"Jeff Rasley, Olatunji Ruwase, Samyam Rajbhandari, Yuxiong He","submitted_at":"2019-10-04T17:29:39Z","abstract_excerpt":"Large deep learning models offer significant accuracy gains, but training billions to trillions of parameters is challenging. Existing solutions such as data and model parallelisms exhibit fundamental limitations to fit these models into limited device memory, while obtaining computation, communication and development efficiency. We develop a novel solution, Zero Redundancy Optimizer (ZeRO), to optimize memory, vastly improving training speed while increasing the model size that can be efficiently trained. ZeRO eliminates memory redundancies in data- and model-parallel training while retaining"},"verification_status":{"content_addressed":true,"pith_receipt":true,"author_attested":false,"weak_author_claims":0,"strong_author_claims":0,"externally_anchored":false,"storage_verified":false,"citation_signatures":0,"replication_records":0,"graph_snapshot":true,"references_resolved":true,"formal_links_present":true},"canonical_record":{"source":{"id":"1910.02054","kind":"arxiv","version":3},"metadata":{"license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","primary_cat":"cs.LG","submitted_at":"2019-10-04T17:29:39Z","cross_cats_sorted":["cs.DC","stat.ML"],"title_canon_sha256":"5c51bb8d9d15dc00904edb477c9632c6ae88312b10fbfa1a9d71978551cf7643","abstract_canon_sha256":"769410855d6e6defbf18a87865b61cd2c4373b74c87a93f622ec300280dd1a77"},"schema_version":"1.0"},"receipt":{"kind":"pith_receipt","key_id":"pith-v1-2026-05","algorithm":"ed25519","signed_at":"2026-05-17T23:38:48.365001Z","signature_b64":"PBoYjcEAJHLwGKkGpkXMYLMxB1pvRcJkJUjL8YdmV4AXfhNqavUrrBhYxFBl83b2iH0QeKNCVbhpwCTkMRvSDg==","signed_message":"canonical_sha256_bytes","builder_version":"pith-number-builder-2026-05-17-v1","receipt_version":"0.3","canonical_sha256":"59bfe2bb85ba230029eb6a6d800dc75da176779950b8cf7ce12fe03970dfb98d","last_reissued_at":"2026-05-17T23:38:48.364346Z","signature_status":"signed_v1","first_computed_at":"2026-05-17T23:38:48.364346Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"graph_snapshot":{"paper":{"title":"ZeRO: Memory Optimizations Toward Training Trillion Parameter Models","license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","headline":"ZeRO partitions optimizer states and gradients across devices to remove memory redundancy in parallel training.","cross_cats":["cs.DC","stat.ML"],"primary_cat":"cs.LG","authors_text":"Jeff Rasley, Olatunji Ruwase, Samyam Rajbhandari, Yuxiong He","submitted_at":"2019-10-04T17:29:39Z","abstract_excerpt":"Large deep learning models offer significant accuracy gains, but training billions to trillions of parameters is challenging. Existing solutions such as data and model parallelisms exhibit fundamental limitations to fit these models into limited device memory, while obtaining computation, communication and development efficiency. We develop a novel solution, Zero Redundancy Optimizer (ZeRO), to optimize memory, vastly improving training speed while increasing the model size that can be efficiently trained. ZeRO eliminates memory redundancies in data- and model-parallel training while retaining"},"claims":{"count":4,"items":[{"kind":"strongest_claim","text":"ZeRO eliminates memory redundancies in data- and model-parallel training while retaining low communication volume and high computational granularity, allowing us to scale the model size proportional to the number of devices with sustained high efficiency. Our analysis demonstrates ZeRO has the potential to scale beyond 1 Trillion parameters using today's hardware.","source":"verdict.strongest_claim","status":"machine_extracted","claim_id":"C1","attestation":"unclaimed"},{"kind":"weakest_assumption","text":"The assumption that partitioning optimizer states and gradients will not introduce new communication bottlenecks or synchronization overheads that scale worse than linearly when moving to thousands of devices.","source":"verdict.weakest_assumption","status":"machine_extracted","claim_id":"C2","attestation":"unclaimed"},{"kind":"one_line_summary","text":"ZeRO removes memory redundancies in parallel training to scale deep learning models to over a trillion parameters with high throughput on current hardware.","source":"verdict.one_line_summary","status":"machine_extracted","claim_id":"C3","attestation":"unclaimed"},{"kind":"headline","text":"ZeRO partitions optimizer states and gradients across devices to remove memory redundancy in parallel training.","source":"verdict.pith_extraction.headline","status":"machine_extracted","claim_id":"C4","attestation":"unclaimed"}],"snapshot_sha256":"fed046d505278de78c538980ee62cc7728f9370deeb337a2015fab3ca89efd8c"},"source":{"id":"1910.02054","kind":"arxiv","version":3},"verdict":{"id":"96ad60f6-935f-44ea-a8c6-1d8168a2b1d0","model_set":{"reader":"grok-4.3"},"created_at":"2026-05-16T09:20:59.131371Z","strongest_claim":"ZeRO eliminates memory redundancies in data- and model-parallel training while retaining low communication volume and high computational granularity, allowing us to scale the model size proportional to the number of devices with sustained high efficiency. Our analysis demonstrates ZeRO has the potential to scale beyond 1 Trillion parameters using today's hardware.","one_line_summary":"ZeRO removes memory redundancies in parallel training to scale deep learning models to over a trillion parameters with high throughput on current hardware.","pipeline_version":"pith-pipeline@v0.9.0","weakest_assumption":"The assumption that partitioning optimizer states and gradients will not introduce new communication bottlenecks or synchronization overheads that scale worse than linearly when moving to thousands of devices.","pith_extraction_headline":"ZeRO partitions optimizer states and gradients across devices to remove memory redundancy in parallel training."},"references":{"count":26,"sample":[{"doi":"","year":2018,"title":"BERT: Pre-training of Deep Bidirectional Transformers for Language Understanding","work_id":"ed240a10-5b19-406c-baa5-30803f465785","ref_index":1,"cited_arxiv_id":"1810.04805","is_internal_anchor":true},{"doi":"","year":2019,"title":"Language models are unsupervised multitask learners","work_id":"47eda437-4651-49ed-bbe6-d63c3b7a78a9","ref_index":2,"cited_arxiv_id":"","is_internal_anchor":false},{"doi":"","year":2019,"title":"Megatron-lm: Training multi-billion parameter language models using model parallelism","work_id":"6dc163d1-d970-4fb3-9587-d5a203ab3150","ref_index":3,"cited_arxiv_id":"","is_internal_anchor":false},{"doi":"","year":2019,"title":"Colin Raffel, Noam Shazeer, Adam Roberts, Katherine Lee, Sharan Narang, Michael Matena, Yanqi Zhou, Wei Li, and Peter J. Liu. Exploring the limits of transfer learn- ing with a unified text-to-text tran","work_id":"a750effe-a64d-4fa6-b86b-425475f72f39","ref_index":4,"cited_arxiv_id":"","is_internal_anchor":false},{"doi":"","year":2018,"title":"Nimit Sharad Sohoni, Christopher Richard Aberger, Megan Leszczynski, Jian Zhang, and Christo- pher R´e","work_id":"b2e01cf2-a2ed-470f-879d-29d3ba18624f","ref_index":5,"cited_arxiv_id":"1811.02084","is_internal_anchor":true}],"resolved_work":26,"snapshot_sha256":"6ac7d75af76466d71edc9d6819b30d58f7e0540e2b455fe24106fd3131c35e45","internal_anchors":9},"formal_canon":{"evidence_count":2,"snapshot_sha256":"3615ab0fe103150bdf437367540d9e4304f066ef41e6d15fa1275ecfbbad8550"},"author_claims":{"count":0,"strong_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"builder_version":"pith-number-builder-2026-05-17-v1"},"aliases":[{"alias_kind":"arxiv","alias_value":"1910.02054","created_at":"2026-05-17T23:38:48.364458+00:00"},{"alias_kind":"arxiv_version","alias_value":"1910.02054v3","created_at":"2026-05-17T23:38:48.364458+00:00"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.1910.02054","created_at":"2026-05-17T23:38:48.364458+00:00"},{"alias_kind":"pith_short_12","alias_value":"LG76FO4FXIRQ","created_at":"2026-05-18T12:33:21.387695+00:00"},{"alias_kind":"pith_short_16","alias_value":"LG76FO4FXIRQAKPL","created_at":"2026-05-18T12:33:21.387695+00:00"},{"alias_kind":"pith_short_8","alias_value":"LG76FO4F","created_at":"2026-05-18T12:33:21.387695+00:00"}],"events":[],"event_summary":{},"paper_claims":[],"inbound_citations":{"count":31,"internal_anchor_count":31,"sample":[{"citing_arxiv_id":"2504.10013","citing_title":"Training LLMs on HPC Systems: Best Practices from the OpenGPT-X Project","ref_index":36,"is_internal_anchor":true},{"citing_arxiv_id":"2603.20421","citing_title":"Hawkeye: Reproducing GPU-Level Non-Determinism","ref_index":11,"is_internal_anchor":true},{"citing_arxiv_id":"2605.17164","citing_title":"Charon: A Unified and Fine-Grained Simulator for Large-Scale LLM Training and Inference","ref_index":17,"is_internal_anchor":true},{"citing_arxiv_id":"2605.20722","citing_title":"AGPO: Adaptive Group Policy Optimization with Dual Statistical Feedback","ref_index":13,"is_internal_anchor":true},{"citing_arxiv_id":"2605.17064","citing_title":"Towards Human-Level Book-Writing Capability","ref_index":23,"is_internal_anchor":true},{"citing_arxiv_id":"2605.17164","citing_title":"Charon: A Unified and Fine-Grained Simulator for Large-Scale LLM Training and Inference","ref_index":11,"is_internal_anchor":true},{"citing_arxiv_id":"2605.18750","citing_title":"A Readiness-Driven Runtime for Pipeline-Parallel Training under Runtime Variability","ref_index":45,"is_internal_anchor":true},{"citing_arxiv_id":"2605.20170","citing_title":"KoRe: Compact Knowledge Representations for Large Language Models","ref_index":26,"is_internal_anchor":true},{"citing_arxiv_id":"2508.21613","citing_title":"Chameleon: Adaptive Fault Tolerance for Distributed Training via Real-time Policy Selection","ref_index":19,"is_internal_anchor":true},{"citing_arxiv_id":"2105.04663","citing_title":"GSPMD: General and Scalable Parallelization for ML Computation Graphs","ref_index":30,"is_internal_anchor":true},{"citing_arxiv_id":"2512.20856","citing_title":"NVIDIA Nemotron 3: Efficient and Open Intelligence","ref_index":129,"is_internal_anchor":true},{"citing_arxiv_id":"2006.15704","citing_title":"PyTorch Distributed: Experiences on Accelerating Data Parallel Training","ref_index":40,"is_internal_anchor":true},{"citing_arxiv_id":"2601.02954","citing_title":"The World is Not Mono: Enabling Spatial Understanding in Large Audio-Language Models","ref_index":14,"is_internal_anchor":true},{"citing_arxiv_id":"2602.05695","citing_title":"SweetSpot: An Analytical Model for Predicting Energy Efficiency of LLM Inference","ref_index":17,"is_internal_anchor":true},{"citing_arxiv_id":"2605.13779","citing_title":"MinT: Managed Infrastructure for Training and Serving Millions of LLMs","ref_index":24,"is_internal_anchor":true},{"citing_arxiv_id":"2505.13211","citing_title":"MAGI-1: Autoregressive Video Generation at Scale","ref_index":37,"is_internal_anchor":true},{"citing_arxiv_id":"2309.16588","citing_title":"Vision Transformers Need Registers","ref_index":119,"is_internal_anchor":true},{"citing_arxiv_id":"2101.03961","citing_title":"Switch Transformers: Scaling to Trillion Parameter Models with Simple and Efficient Sparsity","ref_index":26,"is_internal_anchor":true},{"citing_arxiv_id":"2202.08906","citing_title":"ST-MoE: Designing Stable and Transferable Sparse Expert Models","ref_index":78,"is_internal_anchor":true},{"citing_arxiv_id":"2604.27085","citing_title":"Efficient Training on Multiple Consumer GPUs with RoundPipe","ref_index":41,"is_internal_anchor":true},{"citing_arxiv_id":"2605.09176","citing_title":"Navigating LLM Valley: From AdamW to Memory-Efficient and Matrix-Based Optimizers","ref_index":32,"is_internal_anchor":true},{"citing_arxiv_id":"2605.08766","citing_title":"UserGPT Technical Report","ref_index":80,"is_internal_anchor":true},{"citing_arxiv_id":"2605.08524","citing_title":"Unleashing Scalable Context Parallelism for Foundation Models Pre-Training via FCP","ref_index":54,"is_internal_anchor":true},{"citing_arxiv_id":"2604.24088","citing_title":"TACO: Efficient Communication Compression of Intermediate Tensors for Scalable Tensor-Parallel LLM Training","ref_index":43,"is_internal_anchor":true},{"citing_arxiv_id":"2312.06681","citing_title":"Steering Llama 2 via Contrastive Activation Addition","ref_index":60,"is_internal_anchor":true}]},"formal_canon":{"evidence_count":2,"sample":[],"anchors":[]},"links":{"html":"https://pith.science/pith/LG76FO4FXIRQAKPLNJWYADOHLW","json":"https://pith.science/pith/LG76FO4FXIRQAKPLNJWYADOHLW.json","graph_json":"https://pith.science/api/pith-number/LG76FO4FXIRQAKPLNJWYADOHLW/graph.json","events_json":"https://pith.science/api/pith-number/LG76FO4FXIRQAKPLNJWYADOHLW/events.json","paper":"https://pith.science/paper/LG76FO4F"},"agent_actions":{"view_html":"https://pith.science/pith/LG76FO4FXIRQAKPLNJWYADOHLW","download_json":"https://pith.science/pith/LG76FO4FXIRQAKPLNJWYADOHLW.json","view_paper":"https://pith.science/paper/LG76FO4F","resolve_alias":"https://pith.science/api/pith-number/resolve?arxiv=1910.02054&json=true","fetch_graph":"https://pith.science/api/pith-number/LG76FO4FXIRQAKPLNJWYADOHLW/graph.json","fetch_events":"https://pith.science/api/pith-number/LG76FO4FXIRQAKPLNJWYADOHLW/events.json","actions":{"anchor_timestamp":"https://pith.science/pith/LG76FO4FXIRQAKPLNJWYADOHLW/action/timestamp_anchor","attest_storage":"https://pith.science/pith/LG76FO4FXIRQAKPLNJWYADOHLW/action/storage_attestation","attest_author":"https://pith.science/pith/LG76FO4FXIRQAKPLNJWYADOHLW/action/author_attestation","sign_citation":"https://pith.science/pith/LG76FO4FXIRQAKPLNJWYADOHLW/action/citation_signature","submit_replication":"https://pith.science/pith/LG76FO4FXIRQAKPLNJWYADOHLW/action/replication_record"}},"created_at":"2026-05-17T23:38:48.364458+00:00","updated_at":"2026-05-17T23:38:48.364458+00:00"}