{"record_type":"pith_number_record","schema_url":"https://pith.science/schemas/pith-number/v1.json","pith_number":"pith:2021:YBK5PTAKOSCXY2WMLIIB7DMYH6","short_pith_number":"pith:YBK5PTAK","schema_version":"1.0","canonical_sha256":"c055d7cc0a74857c6acc5a101f8d983fa34422542eeacc6a32799b3cc6c9e5cf","source":{"kind":"arxiv","id":"2104.04473","version":5},"attestation_state":"computed","paper":{"title":"Efficient Large-Scale Language Model Training on GPU Clusters Using Megatron-LM","license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","headline":"","cross_cats":["cs.DC"],"primary_cat":"cs.CL","authors_text":"Amar Phanishayee, Bryan Catanzaro, Deepak Narayanan, Dmitri Vainbrand, Jared Casper, Julie Bernauer, Matei Zaharia, Mohammad Shoeybi, Mostofa Patwary, Patrick LeGresley, Prethvi Kashinkunti, Vijay Anand Korthikanti","submitted_at":"2021-04-09T16:43:11Z","abstract_excerpt":"Large language models have led to state-of-the-art accuracies across a range of tasks. However, training these models efficiently is challenging for two reasons: a) GPU memory capacity is limited, making it impossible to fit large models on even a multi-GPU server, and b) the number of compute operations required to train these models can result in unrealistically long training times. Consequently, new methods of model parallelism such as tensor and pipeline parallelism have been proposed. Unfortunately, naive usage of these methods leads to fundamental scaling issues at thousands of GPUs, e.g"},"verification_status":{"content_addressed":true,"pith_receipt":true,"author_attested":false,"weak_author_claims":0,"strong_author_claims":0,"externally_anchored":false,"storage_verified":false,"citation_signatures":0,"replication_records":0,"graph_snapshot":true,"references_resolved":false,"formal_links_present":false},"canonical_record":{"source":{"id":"2104.04473","kind":"arxiv","version":5},"metadata":{"license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","primary_cat":"cs.CL","submitted_at":"2021-04-09T16:43:11Z","cross_cats_sorted":["cs.DC"],"title_canon_sha256":"19b514ef339611c256cd138a2ea86db3856f0b133433ac17add07fa00008b717","abstract_canon_sha256":"cc326570f4c4a5dbe16f1bcd1a28cab0a09137b0133e0dd7803d2edaa1e46973"},"schema_version":"1.0"},"receipt":{"kind":"pith_receipt","key_id":"pith-v1-2026-05","algorithm":"ed25519","signed_at":"2026-07-05T03:08:07.280302Z","signature_b64":"5FRPzal+CsBQENarZOm3gtAbzLBKKvnug4hNb3sOiDvh2BmwObKJcPCWIhf03wMUfy8WERSUNoAcNxG8ajYOCA==","signed_message":"canonical_sha256_bytes","builder_version":"pith-number-builder-2026-05-17-v1","receipt_version":"0.3","canonical_sha256":"c055d7cc0a74857c6acc5a101f8d983fa34422542eeacc6a32799b3cc6c9e5cf","last_reissued_at":"2026-07-05T03:08:07.279717Z","signature_status":"signed_v1","first_computed_at":"2026-07-05T03:08:07.279717Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"graph_snapshot":{"paper":{"title":"Efficient Large-Scale Language Model Training on GPU Clusters Using Megatron-LM","license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","headline":"","cross_cats":["cs.DC"],"primary_cat":"cs.CL","authors_text":"Amar Phanishayee, Bryan Catanzaro, Deepak Narayanan, Dmitri Vainbrand, Jared Casper, Julie Bernauer, Matei Zaharia, Mohammad Shoeybi, Mostofa Patwary, Patrick LeGresley, Prethvi Kashinkunti, Vijay Anand Korthikanti","submitted_at":"2021-04-09T16:43:11Z","abstract_excerpt":"Large language models have led to state-of-the-art accuracies across a range of tasks. However, training these models efficiently is challenging for two reasons: a) GPU memory capacity is limited, making it impossible to fit large models on even a multi-GPU server, and b) the number of compute operations required to train these models can result in unrealistically long training times. Consequently, new methods of model parallelism such as tensor and pipeline parallelism have been proposed. Unfortunately, naive usage of these methods leads to fundamental scaling issues at thousands of GPUs, e.g"},"claims":{"count":0,"items":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"2104.04473","kind":"arxiv","version":5},"verdict":{"id":null,"model_set":{},"created_at":null,"strongest_claim":"","one_line_summary":"","pipeline_version":null,"weakest_assumption":"","pith_extraction_headline":""},"integrity":{"clean":true,"summary":{"advisory":0,"critical":0,"by_detector":{},"informational":0},"endpoint":"/pith/2104.04473/integrity.json","findings":[],"available":true,"detectors_run":[],"snapshot_sha256":"c28c3603d3b5d939e8dc4c7e95fa8dfce3d595e45f758748cecf8e644a296938"},"references":{"count":0,"sample":[],"resolved_work":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","internal_anchors":0},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"author_claims":{"count":0,"strong_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"builder_version":"pith-number-builder-2026-05-17-v1"},"aliases":[{"alias_kind":"arxiv","alias_value":"2104.04473","created_at":"2026-07-05T03:08:07.279806+00:00"},{"alias_kind":"arxiv_version","alias_value":"2104.04473v5","created_at":"2026-07-05T03:08:07.279806+00:00"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.2104.04473","created_at":"2026-07-05T03:08:07.279806+00:00"},{"alias_kind":"pith_short_12","alias_value":"YBK5PTAKOSCX","created_at":"2026-07-05T03:08:07.279806+00:00"},{"alias_kind":"pith_short_16","alias_value":"YBK5PTAKOSCXY2WM","created_at":"2026-07-05T03:08:07.279806+00:00"},{"alias_kind":"pith_short_8","alias_value":"YBK5PTAK","created_at":"2026-07-05T03:08:07.279806+00:00"}],"events":[],"event_summary":{},"paper_claims":[],"inbound_citations":{"count":28,"internal_anchor_count":1,"sample":[{"citing_arxiv_id":"2607.07494","citing_title":"GIFT: Geometry-Informed Low-precision Gradient Communication for LLM Pretraining","ref_index":14,"is_internal_anchor":true},{"citing_arxiv_id":"2606.24937","citing_title":"The Hitchhiker's Guide to Agentic AI: From Foundations to Systems","ref_index":225,"is_internal_anchor":false},{"citing_arxiv_id":"2607.01646","citing_title":"PHOENIX: Resilient LLM Training with Hot-Swapping via Zero-Overhead Checkpoint","ref_index":4,"is_internal_anchor":false},{"citing_arxiv_id":"2606.11169","citing_title":"Piper: A Programmable Distributed Training System","ref_index":30,"is_internal_anchor":false},{"citing_arxiv_id":"2606.07857","citing_title":"Model Multiplicity for Adversarial Detection in Small Language Model Training on Edge Devices","ref_index":17,"is_internal_anchor":false},{"citing_arxiv_id":"2606.07639","citing_title":"MOSS-Video-Preview: Toward Real-Time Video Understanding via Cross-Attention","ref_index":50,"is_internal_anchor":false},{"citing_arxiv_id":"2605.13779","citing_title":"MinT: Managed Infrastructure for Training and Serving Millions of LLMs","ref_index":20,"is_internal_anchor":false},{"citing_arxiv_id":"2605.24217","citing_title":"Identifying and Mitigating Systemic Measurement Bias in Production LLM Inference Benchmarks","ref_index":14,"is_internal_anchor":false},{"citing_arxiv_id":"2605.27678","citing_title":"Heterogeneous Parallelism for Multimodal Large Language Model Training","ref_index":10,"is_internal_anchor":false},{"citing_arxiv_id":"2201.11990","citing_title":"Using DeepSpeed and Megatron to Train Megatron-Turing NLG 530B, A Large-Scale Generative Language Model","ref_index":43,"is_internal_anchor":false},{"citing_arxiv_id":"2605.17164","citing_title":"Charon: A Unified and Fine-Grained Simulator for Large-Scale LLM Training and Inference","ref_index":16,"is_internal_anchor":false},{"citing_arxiv_id":"2603.15031","citing_title":"Attention Residuals","ref_index":34,"is_internal_anchor":false},{"citing_arxiv_id":"2605.17164","citing_title":"Charon: A Unified and Fine-Grained Simulator for Large-Scale LLM Training and Inference","ref_index":10,"is_internal_anchor":false},{"citing_arxiv_id":"2605.18750","citing_title":"A Readiness-Driven Runtime for Pipeline-Parallel Training under Runtime Variability","ref_index":33,"is_internal_anchor":false},{"citing_arxiv_id":"2509.05276","citing_title":"SpikingBrain: Spiking Brain-inspired Large Models","ref_index":27,"is_internal_anchor":false},{"citing_arxiv_id":"2509.07177","citing_title":"Towards EnergyGPT: A Large Language Model Specialized for the Energy Sector","ref_index":60,"is_internal_anchor":false},{"citing_arxiv_id":"2511.21613","citing_title":"Beyond URLs: Metadata Diversity and Position for Efficient LLM Pretraining","ref_index":15,"is_internal_anchor":false},{"citing_arxiv_id":"2512.11470","citing_title":"Rethinking Expert Trajectory Utilization in LLM Post-training for Mathematical Reasoning","ref_index":27,"is_internal_anchor":false},{"citing_arxiv_id":"2605.13779","citing_title":"MinT: Managed Infrastructure for Training and Serving Millions of LLMs","ref_index":19,"is_internal_anchor":false},{"citing_arxiv_id":"2604.03425","citing_title":"AEGIS: Scaling Long-Sequence Homomorphic Encrypted Transformer Inference via Hybrid Parallelism on Multi-GPU Systems","ref_index":51,"is_internal_anchor":false},{"citing_arxiv_id":"2604.27085","citing_title":"Efficient Training on Multiple Consumer GPUs with RoundPipe","ref_index":35,"is_internal_anchor":false},{"citing_arxiv_id":"2605.09176","citing_title":"Navigating LLM Valley: From AdamW to Memory-Efficient and Matrix-Based Optimizers","ref_index":26,"is_internal_anchor":false},{"citing_arxiv_id":"2604.17861","citing_title":"GPUOS: A GPU Operating System Primitive for Transparent Operation Fusion","ref_index":11,"is_internal_anchor":false},{"citing_arxiv_id":"2605.07726","citing_title":"A Scalable Recipe on SuperMUC-NG Phase 2: Efficient Large-Scale Training of Language Models","ref_index":16,"is_internal_anchor":false},{"citing_arxiv_id":"2205.01068","citing_title":"OPT: Open Pre-trained Transformer Language Models","ref_index":142,"is_internal_anchor":false}]},"formal_canon":{"evidence_count":0,"sample":[],"anchors":[]},"links":{"html":"https://pith.science/pith/YBK5PTAKOSCXY2WMLIIB7DMYH6","json":"https://pith.science/pith/YBK5PTAKOSCXY2WMLIIB7DMYH6.json","graph_json":"https://pith.science/api/pith-number/YBK5PTAKOSCXY2WMLIIB7DMYH6/graph.json","events_json":"https://pith.science/api/pith-number/YBK5PTAKOSCXY2WMLIIB7DMYH6/events.json","paper":"https://pith.science/paper/YBK5PTAK"},"agent_actions":{"view_html":"https://pith.science/pith/YBK5PTAKOSCXY2WMLIIB7DMYH6","download_json":"https://pith.science/pith/YBK5PTAKOSCXY2WMLIIB7DMYH6.json","view_paper":"https://pith.science/paper/YBK5PTAK","resolve_alias":"https://pith.science/api/pith-number/resolve?arxiv=2104.04473&json=true","fetch_graph":"https://pith.science/api/pith-number/YBK5PTAKOSCXY2WMLIIB7DMYH6/graph.json","fetch_events":"https://pith.science/api/pith-number/YBK5PTAKOSCXY2WMLIIB7DMYH6/events.json","actions":{"anchor_timestamp":"https://pith.science/pith/YBK5PTAKOSCXY2WMLIIB7DMYH6/action/timestamp_anchor","attest_storage":"https://pith.science/pith/YBK5PTAKOSCXY2WMLIIB7DMYH6/action/storage_attestation","attest_author":"https://pith.science/pith/YBK5PTAKOSCXY2WMLIIB7DMYH6/action/author_attestation","sign_citation":"https://pith.science/pith/YBK5PTAKOSCXY2WMLIIB7DMYH6/action/citation_signature","submit_replication":"https://pith.science/pith/YBK5PTAKOSCXY2WMLIIB7DMYH6/action/replication_record"}},"created_at":"2026-07-05T03:08:07.279806+00:00","updated_at":"2026-07-05T03:08:07.279806+00:00"}