{"bundle_type":"pith_open_graph_bundle","bundle_version":"1.0","pith_number":"pith:2024:XR2UHLWNHHFSD4ZJJXWRU6R6QP","short_pith_number":"pith:XR2UHLWN","canonical_record":{"source":{"id":"2402.15627","kind":"arxiv","version":1},"metadata":{"license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","primary_cat":"cs.LG","submitted_at":"2024-02-23T22:10:59Z","cross_cats_sorted":["cs.DC"],"title_canon_sha256":"965ecaee53eed9d5aab76f4c165b841a1f89677bff99362b775bbc76a9d427f7","abstract_canon_sha256":"ce0c3f4ade76bd25b3ab5b2cd3576980c8a1f3e0d142a7c6f3f912088a62ca84"},"schema_version":"1.0"},"canonical_sha256":"bc7543aecd39cb21f3294ded1a7a3e83e3d13c5d82445f0e623f0f8f8faf88f6","source":{"kind":"arxiv","id":"2402.15627","version":1},"source_aliases":[{"alias_kind":"arxiv","alias_value":"2402.15627","created_at":"2026-07-05T07:48:55Z"},{"alias_kind":"arxiv_version","alias_value":"2402.15627v1","created_at":"2026-07-05T07:48:55Z"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.2402.15627","created_at":"2026-07-05T07:48:55Z"},{"alias_kind":"pith_short_12","alias_value":"XR2UHLWNHHFS","created_at":"2026-07-05T07:48:55Z"},{"alias_kind":"pith_short_16","alias_value":"XR2UHLWNHHFSD4ZJ","created_at":"2026-07-05T07:48:55Z"},{"alias_kind":"pith_short_8","alias_value":"XR2UHLWN","created_at":"2026-07-05T07:48:55Z"}],"events":[{"event_type":"record_created","subject_pith_number":"pith:2024:XR2UHLWNHHFSD4ZJJXWRU6R6QP","target":"record","payload":{"canonical_record":{"source":{"id":"2402.15627","kind":"arxiv","version":1},"metadata":{"license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","primary_cat":"cs.LG","submitted_at":"2024-02-23T22:10:59Z","cross_cats_sorted":["cs.DC"],"title_canon_sha256":"965ecaee53eed9d5aab76f4c165b841a1f89677bff99362b775bbc76a9d427f7","abstract_canon_sha256":"ce0c3f4ade76bd25b3ab5b2cd3576980c8a1f3e0d142a7c6f3f912088a62ca84"},"schema_version":"1.0"},"canonical_sha256":"bc7543aecd39cb21f3294ded1a7a3e83e3d13c5d82445f0e623f0f8f8faf88f6","receipt":{"kind":"pith_receipt","key_id":"pith-v1-2026-05","algorithm":"ed25519","signed_at":"2026-07-05T07:48:55.792980Z","signature_b64":"F0IC0Up+BemLS818jLwPZmxApldJ0bFp0RopVT8Nz0rNNXInXdfmbz5LfJ81ZuhaKeKK9zRN/vNnfIz4Zs/eCA==","signed_message":"canonical_sha256_bytes","builder_version":"pith-number-builder-2026-05-17-v1","receipt_version":"0.3","canonical_sha256":"bc7543aecd39cb21f3294ded1a7a3e83e3d13c5d82445f0e623f0f8f8faf88f6","last_reissued_at":"2026-07-05T07:48:55.792483Z","signature_status":"signed_v1","first_computed_at":"2026-07-05T07:48:55.792483Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"source_kind":"arxiv","source_id":"2402.15627","source_version":1,"attestation_state":"computed"},"signer":{"signer_id":"pith.science","signer_type":"pith_registry","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"created_at":"2026-07-05T07:48:55Z","supersedes":[],"prev_event":null,"signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"VFR+dINRZUxxW2oIx1QIES4cijQjNCpl6r8n7pjn+AZSFhJaH8unWLtaAtlaK4GmCvzm961XjpeH+mSdeQALCQ==","signed_message":"open_graph_event_sha256_bytes","signed_at":"2026-08-04T23:04:30.116274Z"},"content_sha256":"31ca7a7cebdef5aa97d424ae9488d1e62dfe15d19a26d186d2c744da1a9e1838","schema_version":"1.0","event_id":"sha256:31ca7a7cebdef5aa97d424ae9488d1e62dfe15d19a26d186d2c744da1a9e1838"},{"event_type":"graph_snapshot","subject_pith_number":"pith:2024:XR2UHLWNHHFSD4ZJJXWRU6R6QP","target":"graph","payload":{"graph_snapshot":{"paper":{"title":"MegaScale: Scaling Large Language Model Training to More Than 10,000 GPUs","license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","headline":"","cross_cats":["cs.DC"],"primary_cat":"cs.LG","authors_text":"Cong Xie, Ding Zhou, Haibin Lin, Haohan Xu, Haoran Wei, Hongmin Chen, Jianxi Ye, Leqi Zou, Liang Xiang, Pengfei Nie, Qi Hou, Qi Huang, Shibiao Nong, Shipeng Yan, Sida Zhao, Sun He, Xiang Li, Xiaoying Jia, Xin Jin, Xin Liu, Yanghua Peng, Yangrui Chen, Yinmin Zhong, Yiyao Sheng, Yulu Jia, Zhang Zhang, Zhe Li, Zherui Liu, Zhihao Bai, Zhi Zhang, Zhuo Jiang, Ziheng Jiang","submitted_at":"2024-02-23T22:10:59Z","abstract_excerpt":"We present the design, implementation and engineering experience in building and deploying MegaScale, a production system for training large language models (LLMs) at the scale of more than 10,000 GPUs. Training LLMs at this scale brings unprecedented challenges to training efficiency and stability. We take a full-stack approach that co-designs the algorithmic and system components across model block and optimizer design, computation and communication overlapping, operator optimization, data pipeline, and network performance tuning. Maintaining high efficiency throughout the training process ("},"claims":{"count":0,"items":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"2402.15627","kind":"arxiv","version":1},"verdict":{"id":null,"model_set":{},"created_at":null,"strongest_claim":"","one_line_summary":"","pipeline_version":null,"weakest_assumption":"","pith_extraction_headline":""},"integrity":{"clean":true,"summary":{"advisory":0,"critical":0,"by_detector":{},"informational":0},"endpoint":"/pith/2402.15627/integrity.json","findings":[],"available":true,"detectors_run":[],"snapshot_sha256":"c28c3603d3b5d939e8dc4c7e95fa8dfce3d595e45f758748cecf8e644a296938"},"references":{"count":0,"sample":[],"resolved_work":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","internal_anchors":0},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"author_claims":{"count":0,"strong_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"builder_version":"pith-number-builder-2026-05-17-v1"},"verdict_id":null},"signer":{"signer_id":"pith.science","signer_type":"pith_registry","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"created_at":"2026-07-05T07:48:55Z","supersedes":[],"prev_event":null,"signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"k2jLsT7w+bW2gncEC21QMDaLa46o0OIK2qH0TrpTewfLw+5LAcJGEo8vpQn47jwun5ia1yjYArFEXTIvew4GBQ==","signed_message":"open_graph_event_sha256_bytes","signed_at":"2026-08-04T23:04:30.117264Z"},"content_sha256":"022914ae4a03d74221574ffd375c3c5882301ba1b833276c95e1f5072326a744","schema_version":"1.0","event_id":"sha256:022914ae4a03d74221574ffd375c3c5882301ba1b833276c95e1f5072326a744"}],"timestamp_proofs":[],"mirror_hints":[{"mirror_type":"https","name":"Pith Resolver","base_url":"https://pith.science","bundle_url":"https://pith.science/pith/XR2UHLWNHHFSD4ZJJXWRU6R6QP/bundle.json","state_url":"https://pith.science/pith/XR2UHLWNHHFSD4ZJJXWRU6R6QP/state.json","well_known_bundle_url":"https://pith.science/.well-known/pith/XR2UHLWNHHFSD4ZJJXWRU6R6QP/bundle.json","status":"primary"}],"public_keys":[{"key_id":"pith-v1-2026-05","algorithm":"ed25519","format":"raw","public_key_b64":"stVStoiQhXFxp4s2pdzPNoqVNBMojDU/fJ2db5S3CbM=","public_key_hex":"b2d552b68890857171a78b36a5dccf368a953413288c353f7c9d9d6f94b709b3","fingerprint_sha256_b32_first128bits":"RVFV5Z2OI2J3ZUO7ERDEBCYNKS","fingerprint_sha256_hex":"8d4b5ee74e4693bcd1df2446408b0d54","rotates_at":null,"url":"https://pith.science/pith-signing-key.json","notes":"Pith uses this Ed25519 key to sign canonical record SHA-256 digests. Verify with: ed25519_verify(public_key, message=canonical_sha256_bytes, signature=base64decode(signature_b64))."}],"merge_version":"pith-open-graph-merge-v1","built_at":"2026-08-04T23:04:30Z","links":{"resolver":"https://pith.science/pith/XR2UHLWNHHFSD4ZJJXWRU6R6QP","bundle":"https://pith.science/pith/XR2UHLWNHHFSD4ZJJXWRU6R6QP/bundle.json","state":"https://pith.science/pith/XR2UHLWNHHFSD4ZJJXWRU6R6QP/state.json","well_known_bundle":"https://pith.science/.well-known/pith/XR2UHLWNHHFSD4ZJJXWRU6R6QP/bundle.json"},"state":{"state_type":"pith_open_graph_state","state_version":"1.0","pith_number":"pith:2024:XR2UHLWNHHFSD4ZJJXWRU6R6QP","merge_version":"pith-open-graph-merge-v1","event_count":2,"valid_event_count":2,"invalid_event_count":0,"equivocation_count":0,"current":{"canonical_record":{"metadata":{"abstract_canon_sha256":"ce0c3f4ade76bd25b3ab5b2cd3576980c8a1f3e0d142a7c6f3f912088a62ca84","cross_cats_sorted":["cs.DC"],"license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","primary_cat":"cs.LG","submitted_at":"2024-02-23T22:10:59Z","title_canon_sha256":"965ecaee53eed9d5aab76f4c165b841a1f89677bff99362b775bbc76a9d427f7"},"schema_version":"1.0","source":{"id":"2402.15627","kind":"arxiv","version":1}},"source_aliases":[{"alias_kind":"arxiv","alias_value":"2402.15627","created_at":"2026-07-05T07:48:55Z"},{"alias_kind":"arxiv_version","alias_value":"2402.15627v1","created_at":"2026-07-05T07:48:55Z"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.2402.15627","created_at":"2026-07-05T07:48:55Z"},{"alias_kind":"pith_short_12","alias_value":"XR2UHLWNHHFS","created_at":"2026-07-05T07:48:55Z"},{"alias_kind":"pith_short_16","alias_value":"XR2UHLWNHHFSD4ZJ","created_at":"2026-07-05T07:48:55Z"},{"alias_kind":"pith_short_8","alias_value":"XR2UHLWN","created_at":"2026-07-05T07:48:55Z"}],"graph_snapshots":[{"event_id":"sha256:022914ae4a03d74221574ffd375c3c5882301ba1b833276c95e1f5072326a744","target":"graph","created_at":"2026-07-05T07:48:55Z","signer":{"key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signer_id":"pith.science","signer_type":"pith_registry"},"payload":{"graph_snapshot":{"author_claims":{"count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","strong_count":0},"builder_version":"pith-number-builder-2026-05-17-v1","claims":{"count":0,"items":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"integrity":{"available":true,"clean":true,"detectors_run":[],"endpoint":"/pith/2402.15627/integrity.json","findings":[],"snapshot_sha256":"c28c3603d3b5d939e8dc4c7e95fa8dfce3d595e45f758748cecf8e644a296938","summary":{"advisory":0,"by_detector":{},"critical":0,"informational":0}},"paper":{"abstract_excerpt":"We present the design, implementation and engineering experience in building and deploying MegaScale, a production system for training large language models (LLMs) at the scale of more than 10,000 GPUs. Training LLMs at this scale brings unprecedented challenges to training efficiency and stability. We take a full-stack approach that co-designs the algorithmic and system components across model block and optimizer design, computation and communication overlapping, operator optimization, data pipeline, and network performance tuning. Maintaining high efficiency throughout the training process (","authors_text":"Cong Xie, Ding Zhou, Haibin Lin, Haohan Xu, Haoran Wei, Hongmin Chen, Jianxi Ye, Leqi Zou, Liang Xiang, Pengfei Nie, Qi Hou, Qi Huang, Shibiao Nong, Shipeng Yan, Sida Zhao, Sun He, Xiang Li, Xiaoying Jia, Xin Jin, Xin Liu, Yanghua Peng, Yangrui Chen, Yinmin Zhong, Yiyao Sheng, Yulu Jia, Zhang Zhang, Zhe Li, Zherui Liu, Zhihao Bai, Zhi Zhang, Zhuo Jiang, Ziheng Jiang","cross_cats":["cs.DC"],"headline":"","license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","primary_cat":"cs.LG","submitted_at":"2024-02-23T22:10:59Z","title":"MegaScale: Scaling Large Language Model Training to More Than 10,000 GPUs"},"references":{"count":0,"internal_anchors":0,"resolved_work":0,"sample":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"2402.15627","kind":"arxiv","version":1},"verdict":{"created_at":null,"id":null,"model_set":{},"one_line_summary":"","pipeline_version":null,"pith_extraction_headline":"","strongest_claim":"","weakest_assumption":""}},"verdict_id":null}}],"author_attestations":[],"timestamp_anchors":[],"storage_attestations":[],"citation_signatures":[],"replication_records":[],"corrections":[],"mirror_hints":[],"record_created":{"event_id":"sha256:31ca7a7cebdef5aa97d424ae9488d1e62dfe15d19a26d186d2c744da1a9e1838","target":"record","created_at":"2026-07-05T07:48:55Z","signer":{"key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signer_id":"pith.science","signer_type":"pith_registry"},"payload":{"attestation_state":"computed","canonical_record":{"metadata":{"abstract_canon_sha256":"ce0c3f4ade76bd25b3ab5b2cd3576980c8a1f3e0d142a7c6f3f912088a62ca84","cross_cats_sorted":["cs.DC"],"license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","primary_cat":"cs.LG","submitted_at":"2024-02-23T22:10:59Z","title_canon_sha256":"965ecaee53eed9d5aab76f4c165b841a1f89677bff99362b775bbc76a9d427f7"},"schema_version":"1.0","source":{"id":"2402.15627","kind":"arxiv","version":1}},"canonical_sha256":"bc7543aecd39cb21f3294ded1a7a3e83e3d13c5d82445f0e623f0f8f8faf88f6","receipt":{"algorithm":"ed25519","builder_version":"pith-number-builder-2026-05-17-v1","canonical_sha256":"bc7543aecd39cb21f3294ded1a7a3e83e3d13c5d82445f0e623f0f8f8faf88f6","first_computed_at":"2026-07-05T07:48:55.792483Z","key_id":"pith-v1-2026-05","kind":"pith_receipt","last_reissued_at":"2026-07-05T07:48:55.792483Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","receipt_version":"0.3","signature_b64":"F0IC0Up+BemLS818jLwPZmxApldJ0bFp0RopVT8Nz0rNNXInXdfmbz5LfJ81ZuhaKeKK9zRN/vNnfIz4Zs/eCA==","signature_status":"signed_v1","signed_at":"2026-07-05T07:48:55.792980Z","signed_message":"canonical_sha256_bytes"},"source_id":"2402.15627","source_kind":"arxiv","source_version":1}}},"equivocations":[],"invalid_events":[],"applied_event_ids":["sha256:31ca7a7cebdef5aa97d424ae9488d1e62dfe15d19a26d186d2c744da1a9e1838","sha256:022914ae4a03d74221574ffd375c3c5882301ba1b833276c95e1f5072326a744"],"state_sha256":"a4ee2fa64ac15936ea476216c3aaab99d7f9307a609bccd192acdf5bbc3df709"},"bundle_signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"OD7Pntz6dUT3gOY3F0bt0w1pYND9WEZdQ7nY8B/BdyO9R29VVYzqGtXMp27oZELfPBf0bc2zPxluHNMwISscBA==","signed_message":"bundle_sha256_bytes","signed_at":"2026-08-04T23:04:30.123812Z","bundle_sha256":"e7069b44a1aaec07ee675bbf65843c5fec66d5e929207d462cc8c1e1e24e2011"}}