{"bundle_type":"pith_open_graph_bundle","bundle_version":"1.0","pith_number":"pith:2023:ENASDM3ZJ27EQUNF3RMCPK6XQE","short_pith_number":"pith:ENASDM3Z","canonical_record":{"source":{"id":"2306.10209","kind":"arxiv","version":1},"metadata":{"license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.DC","submitted_at":"2023-06-16T23:26:19Z","cross_cats_sorted":["cs.AI","cs.LG","cs.PF"],"title_canon_sha256":"822b30edd2b73b3fb3b27b88534f6a8d2637dea475a99c73d6cbfecea3b9ec8a","abstract_canon_sha256":"f0ab44c82c391d13d025136ea15cf97296000e9a6b938fdf4516f84ab8e2cded"},"schema_version":"1.0"},"canonical_sha256":"234121b3794ebe4851a5dc5827abd78130044ea8a6304f0a449790cb6fd4e577","source":{"kind":"arxiv","id":"2306.10209","version":1},"source_aliases":[{"alias_kind":"arxiv","alias_value":"2306.10209","created_at":"2026-07-05T06:21:35Z"},{"alias_kind":"arxiv_version","alias_value":"2306.10209v1","created_at":"2026-07-05T06:21:35Z"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.2306.10209","created_at":"2026-07-05T06:21:35Z"},{"alias_kind":"pith_short_12","alias_value":"ENASDM3ZJ27E","created_at":"2026-07-05T06:21:35Z"},{"alias_kind":"pith_short_16","alias_value":"ENASDM3ZJ27EQUNF","created_at":"2026-07-05T06:21:35Z"},{"alias_kind":"pith_short_8","alias_value":"ENASDM3Z","created_at":"2026-07-05T06:21:35Z"}],"events":[{"event_type":"record_created","subject_pith_number":"pith:2023:ENASDM3ZJ27EQUNF3RMCPK6XQE","target":"record","payload":{"canonical_record":{"source":{"id":"2306.10209","kind":"arxiv","version":1},"metadata":{"license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.DC","submitted_at":"2023-06-16T23:26:19Z","cross_cats_sorted":["cs.AI","cs.LG","cs.PF"],"title_canon_sha256":"822b30edd2b73b3fb3b27b88534f6a8d2637dea475a99c73d6cbfecea3b9ec8a","abstract_canon_sha256":"f0ab44c82c391d13d025136ea15cf97296000e9a6b938fdf4516f84ab8e2cded"},"schema_version":"1.0"},"canonical_sha256":"234121b3794ebe4851a5dc5827abd78130044ea8a6304f0a449790cb6fd4e577","receipt":{"kind":"pith_receipt","key_id":"pith-v1-2026-05","algorithm":"ed25519","signed_at":"2026-07-05T06:21:35.381339Z","signature_b64":"3XvYSBJOq3YAC3igIrTBS4bs/syf9ztWt0I5iPtYOQiJahXp+D5xucqrsyQaDr0E8/1LVptu7LpCQawiTamiAQ==","signed_message":"canonical_sha256_bytes","builder_version":"pith-number-builder-2026-05-17-v1","receipt_version":"0.3","canonical_sha256":"234121b3794ebe4851a5dc5827abd78130044ea8a6304f0a449790cb6fd4e577","last_reissued_at":"2026-07-05T06:21:35.380855Z","signature_status":"signed_v1","first_computed_at":"2026-07-05T06:21:35.380855Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"source_kind":"arxiv","source_id":"2306.10209","source_version":1,"attestation_state":"computed"},"signer":{"signer_id":"pith.science","signer_type":"pith_registry","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"created_at":"2026-07-05T06:21:35Z","supersedes":[],"prev_event":null,"signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"VRPD3Rwf897WKXtGnl8Tjb0IlNsslUR0pPKDXF78jpK1a7kv0+UbdvmNjQZAq9l1BEjnTwEzGFjm5No2br1TDA==","signed_message":"open_graph_event_sha256_bytes","signed_at":"2026-07-10T22:02:38.660665Z"},"content_sha256":"756c98d01692dc72cfeca39b9f5e34648fcfa5fc22125395c1638b100496e99b","schema_version":"1.0","event_id":"sha256:756c98d01692dc72cfeca39b9f5e34648fcfa5fc22125395c1638b100496e99b"},{"event_type":"graph_snapshot","subject_pith_number":"pith:2023:ENASDM3ZJ27EQUNF3RMCPK6XQE","target":"graph","payload":{"graph_snapshot":{"paper":{"title":"ZeRO++: Extremely Efficient Collective Communication for Giant Model Training","license":"http://creativecommons.org/licenses/by/4.0/","headline":"","cross_cats":["cs.AI","cs.LG","cs.PF"],"primary_cat":"cs.DC","authors_text":"Connor Holmes, Feng Yan, Guanhua Wang, Heyang Qin, Lei Yang, Olatunji Ruwase, Sam Ade Jacobs, Samyam Rajbhandari, Yuxiong He","submitted_at":"2023-06-16T23:26:19Z","abstract_excerpt":"Zero Redundancy Optimizer (ZeRO) has been used to train a wide range of large language models on massive GPUs clusters due to its ease of use, efficiency, and good scalability. However, when training on low-bandwidth clusters, or at scale which forces batch size per GPU to be small, ZeRO's effective throughput is limited because of high communication volume from gathering weights in forward pass, backward pass, and averaging gradients. This paper introduces three communication volume reduction techniques, which we collectively refer to as ZeRO++, targeting each of the communication collectives"},"claims":{"count":0,"items":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"2306.10209","kind":"arxiv","version":1},"verdict":{"id":null,"model_set":{},"created_at":null,"strongest_claim":"","one_line_summary":"","pipeline_version":null,"weakest_assumption":"","pith_extraction_headline":""},"integrity":{"clean":true,"summary":{"advisory":0,"critical":0,"by_detector":{},"informational":0},"endpoint":"/pith/2306.10209/integrity.json","findings":[],"available":true,"detectors_run":[],"snapshot_sha256":"c28c3603d3b5d939e8dc4c7e95fa8dfce3d595e45f758748cecf8e644a296938"},"references":{"count":0,"sample":[],"resolved_work":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","internal_anchors":0},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"author_claims":{"count":0,"strong_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"builder_version":"pith-number-builder-2026-05-17-v1"},"verdict_id":null},"signer":{"signer_id":"pith.science","signer_type":"pith_registry","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"created_at":"2026-07-05T06:21:35Z","supersedes":[],"prev_event":null,"signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"tEIkJzwEsTIluEwZakhGc2eqreM2s+zzOI+CYqmwIBIxNhnFUSfcLBEdLVfFhCu+0vWddOXsAtVcYUrLlSxZAw==","signed_message":"open_graph_event_sha256_bytes","signed_at":"2026-07-10T22:02:38.661399Z"},"content_sha256":"86111a96e216821877f16ac4bbb548ca697c92e7f1fe71bf32abc3e0a9695e0a","schema_version":"1.0","event_id":"sha256:86111a96e216821877f16ac4bbb548ca697c92e7f1fe71bf32abc3e0a9695e0a"}],"timestamp_proofs":[],"mirror_hints":[{"mirror_type":"https","name":"Pith Resolver","base_url":"https://pith.science","bundle_url":"https://pith.science/pith/ENASDM3ZJ27EQUNF3RMCPK6XQE/bundle.json","state_url":"https://pith.science/pith/ENASDM3ZJ27EQUNF3RMCPK6XQE/state.json","well_known_bundle_url":"https://pith.science/.well-known/pith/ENASDM3ZJ27EQUNF3RMCPK6XQE/bundle.json","status":"primary"}],"public_keys":[{"key_id":"pith-v1-2026-05","algorithm":"ed25519","format":"raw","public_key_b64":"stVStoiQhXFxp4s2pdzPNoqVNBMojDU/fJ2db5S3CbM=","public_key_hex":"b2d552b68890857171a78b36a5dccf368a953413288c353f7c9d9d6f94b709b3","fingerprint_sha256_b32_first128bits":"RVFV5Z2OI2J3ZUO7ERDEBCYNKS","fingerprint_sha256_hex":"8d4b5ee74e4693bcd1df2446408b0d54","rotates_at":null,"url":"https://pith.science/pith-signing-key.json","notes":"Pith uses this Ed25519 key to sign canonical record SHA-256 digests. Verify with: ed25519_verify(public_key, message=canonical_sha256_bytes, signature=base64decode(signature_b64))."}],"merge_version":"pith-open-graph-merge-v1","built_at":"2026-07-10T22:02:38Z","links":{"resolver":"https://pith.science/pith/ENASDM3ZJ27EQUNF3RMCPK6XQE","bundle":"https://pith.science/pith/ENASDM3ZJ27EQUNF3RMCPK6XQE/bundle.json","state":"https://pith.science/pith/ENASDM3ZJ27EQUNF3RMCPK6XQE/state.json","well_known_bundle":"https://pith.science/.well-known/pith/ENASDM3ZJ27EQUNF3RMCPK6XQE/bundle.json"},"state":{"state_type":"pith_open_graph_state","state_version":"1.0","pith_number":"pith:2023:ENASDM3ZJ27EQUNF3RMCPK6XQE","merge_version":"pith-open-graph-merge-v1","event_count":2,"valid_event_count":2,"invalid_event_count":0,"equivocation_count":0,"current":{"canonical_record":{"metadata":{"abstract_canon_sha256":"f0ab44c82c391d13d025136ea15cf97296000e9a6b938fdf4516f84ab8e2cded","cross_cats_sorted":["cs.AI","cs.LG","cs.PF"],"license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.DC","submitted_at":"2023-06-16T23:26:19Z","title_canon_sha256":"822b30edd2b73b3fb3b27b88534f6a8d2637dea475a99c73d6cbfecea3b9ec8a"},"schema_version":"1.0","source":{"id":"2306.10209","kind":"arxiv","version":1}},"source_aliases":[{"alias_kind":"arxiv","alias_value":"2306.10209","created_at":"2026-07-05T06:21:35Z"},{"alias_kind":"arxiv_version","alias_value":"2306.10209v1","created_at":"2026-07-05T06:21:35Z"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.2306.10209","created_at":"2026-07-05T06:21:35Z"},{"alias_kind":"pith_short_12","alias_value":"ENASDM3ZJ27E","created_at":"2026-07-05T06:21:35Z"},{"alias_kind":"pith_short_16","alias_value":"ENASDM3ZJ27EQUNF","created_at":"2026-07-05T06:21:35Z"},{"alias_kind":"pith_short_8","alias_value":"ENASDM3Z","created_at":"2026-07-05T06:21:35Z"}],"graph_snapshots":[{"event_id":"sha256:86111a96e216821877f16ac4bbb548ca697c92e7f1fe71bf32abc3e0a9695e0a","target":"graph","created_at":"2026-07-05T06:21:35Z","signer":{"key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signer_id":"pith.science","signer_type":"pith_registry"},"payload":{"graph_snapshot":{"author_claims":{"count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","strong_count":0},"builder_version":"pith-number-builder-2026-05-17-v1","claims":{"count":0,"items":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"integrity":{"available":true,"clean":true,"detectors_run":[],"endpoint":"/pith/2306.10209/integrity.json","findings":[],"snapshot_sha256":"c28c3603d3b5d939e8dc4c7e95fa8dfce3d595e45f758748cecf8e644a296938","summary":{"advisory":0,"by_detector":{},"critical":0,"informational":0}},"paper":{"abstract_excerpt":"Zero Redundancy Optimizer (ZeRO) has been used to train a wide range of large language models on massive GPUs clusters due to its ease of use, efficiency, and good scalability. However, when training on low-bandwidth clusters, or at scale which forces batch size per GPU to be small, ZeRO's effective throughput is limited because of high communication volume from gathering weights in forward pass, backward pass, and averaging gradients. This paper introduces three communication volume reduction techniques, which we collectively refer to as ZeRO++, targeting each of the communication collectives","authors_text":"Connor Holmes, Feng Yan, Guanhua Wang, Heyang Qin, Lei Yang, Olatunji Ruwase, Sam Ade Jacobs, Samyam Rajbhandari, Yuxiong He","cross_cats":["cs.AI","cs.LG","cs.PF"],"headline":"","license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.DC","submitted_at":"2023-06-16T23:26:19Z","title":"ZeRO++: Extremely Efficient Collective Communication for Giant Model Training"},"references":{"count":0,"internal_anchors":0,"resolved_work":0,"sample":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"2306.10209","kind":"arxiv","version":1},"verdict":{"created_at":null,"id":null,"model_set":{},"one_line_summary":"","pipeline_version":null,"pith_extraction_headline":"","strongest_claim":"","weakest_assumption":""}},"verdict_id":null}}],"author_attestations":[],"timestamp_anchors":[],"storage_attestations":[],"citation_signatures":[],"replication_records":[],"corrections":[],"mirror_hints":[],"record_created":{"event_id":"sha256:756c98d01692dc72cfeca39b9f5e34648fcfa5fc22125395c1638b100496e99b","target":"record","created_at":"2026-07-05T06:21:35Z","signer":{"key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signer_id":"pith.science","signer_type":"pith_registry"},"payload":{"attestation_state":"computed","canonical_record":{"metadata":{"abstract_canon_sha256":"f0ab44c82c391d13d025136ea15cf97296000e9a6b938fdf4516f84ab8e2cded","cross_cats_sorted":["cs.AI","cs.LG","cs.PF"],"license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.DC","submitted_at":"2023-06-16T23:26:19Z","title_canon_sha256":"822b30edd2b73b3fb3b27b88534f6a8d2637dea475a99c73d6cbfecea3b9ec8a"},"schema_version":"1.0","source":{"id":"2306.10209","kind":"arxiv","version":1}},"canonical_sha256":"234121b3794ebe4851a5dc5827abd78130044ea8a6304f0a449790cb6fd4e577","receipt":{"algorithm":"ed25519","builder_version":"pith-number-builder-2026-05-17-v1","canonical_sha256":"234121b3794ebe4851a5dc5827abd78130044ea8a6304f0a449790cb6fd4e577","first_computed_at":"2026-07-05T06:21:35.380855Z","key_id":"pith-v1-2026-05","kind":"pith_receipt","last_reissued_at":"2026-07-05T06:21:35.380855Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","receipt_version":"0.3","signature_b64":"3XvYSBJOq3YAC3igIrTBS4bs/syf9ztWt0I5iPtYOQiJahXp+D5xucqrsyQaDr0E8/1LVptu7LpCQawiTamiAQ==","signature_status":"signed_v1","signed_at":"2026-07-05T06:21:35.381339Z","signed_message":"canonical_sha256_bytes"},"source_id":"2306.10209","source_kind":"arxiv","source_version":1}}},"equivocations":[],"invalid_events":[],"applied_event_ids":["sha256:756c98d01692dc72cfeca39b9f5e34648fcfa5fc22125395c1638b100496e99b","sha256:86111a96e216821877f16ac4bbb548ca697c92e7f1fe71bf32abc3e0a9695e0a"],"state_sha256":"6a1b3a462d9f0a58a1822e01cd80544f82a2722e5d73c13ef9aef5015e63974f"},"bundle_signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"5edPiQc3sGxZmsGMxTdidCy7hhFlNFWVTpSZ6ySmkOcHcv9kFu/fRNUzx9EQ/cP9WZtiNb1t8eBd5TavXCzPAA==","signed_message":"bundle_sha256_bytes","signed_at":"2026-07-10T22:02:38.664770Z","bundle_sha256":"72e4cfc5767c234d6bdbd31ef1d5a17a6f880f7bf20902f37bc90834140f1b07"}}