{"bundle_type":"pith_open_graph_bundle","bundle_version":"1.0","pith_number":"pith:2021:JFLLIOJUI2M3KX3FLLGP4N2V65","short_pith_number":"pith:JFLLIOJU","canonical_record":{"source":{"id":"2106.02679","kind":"arxiv","version":1},"metadata":{"license":"http://creativecommons.org/licenses/by-sa/4.0/","primary_cat":"cs.LG","submitted_at":"2021-06-04T19:21:49Z","cross_cats_sorted":["cs.AI","cs.CL","cs.DC"],"title_canon_sha256":"434da334d2825ada3dac75e9e2b905314cb028892db0175cc031b036c7f49566","abstract_canon_sha256":"ebba9938e8dc4a66cfa5976ba9f46fdca098172c024e0cb498ac94ea0bcd5fe8"},"schema_version":"1.0"},"canonical_sha256":"4956b439344699b55f655accfe3755f76675906c3782b2b848781fbe13e87432","source":{"kind":"arxiv","id":"2106.02679","version":1},"source_aliases":[{"alias_kind":"arxiv","alias_value":"2106.02679","created_at":"2026-07-05T02:46:28Z"},{"alias_kind":"arxiv_version","alias_value":"2106.02679v1","created_at":"2026-07-05T02:46:28Z"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.2106.02679","created_at":"2026-07-05T02:46:28Z"},{"alias_kind":"pith_short_12","alias_value":"JFLLIOJUI2M3","created_at":"2026-07-05T02:46:28Z"},{"alias_kind":"pith_short_16","alias_value":"JFLLIOJUI2M3KX3F","created_at":"2026-07-05T02:46:28Z"},{"alias_kind":"pith_short_8","alias_value":"JFLLIOJU","created_at":"2026-07-05T02:46:28Z"}],"events":[{"event_type":"record_created","subject_pith_number":"pith:2021:JFLLIOJUI2M3KX3FLLGP4N2V65","target":"record","payload":{"canonical_record":{"source":{"id":"2106.02679","kind":"arxiv","version":1},"metadata":{"license":"http://creativecommons.org/licenses/by-sa/4.0/","primary_cat":"cs.LG","submitted_at":"2021-06-04T19:21:49Z","cross_cats_sorted":["cs.AI","cs.CL","cs.DC"],"title_canon_sha256":"434da334d2825ada3dac75e9e2b905314cb028892db0175cc031b036c7f49566","abstract_canon_sha256":"ebba9938e8dc4a66cfa5976ba9f46fdca098172c024e0cb498ac94ea0bcd5fe8"},"schema_version":"1.0"},"canonical_sha256":"4956b439344699b55f655accfe3755f76675906c3782b2b848781fbe13e87432","receipt":{"kind":"pith_receipt","key_id":"pith-v1-2026-05","algorithm":"ed25519","signed_at":"2026-07-05T02:46:28.981389Z","signature_b64":"LATkpIUXJMlWEJHZI4zmLK6En7GKKRe38ota61W5nAZgBk5mTKP3Hm5nLNtNsi4Jl3sfonfNhorZ6Csq0IPdDw==","signed_message":"canonical_sha256_bytes","builder_version":"pith-number-builder-2026-05-17-v1","receipt_version":"0.3","canonical_sha256":"4956b439344699b55f655accfe3755f76675906c3782b2b848781fbe13e87432","last_reissued_at":"2026-07-05T02:46:28.980929Z","signature_status":"signed_v1","first_computed_at":"2026-07-05T02:46:28.980929Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"source_kind":"arxiv","source_id":"2106.02679","source_version":1,"attestation_state":"computed"},"signer":{"signer_id":"pith.science","signer_type":"pith_registry","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"created_at":"2026-07-05T02:46:28Z","supersedes":[],"prev_event":null,"signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"Xk9Rmb6KcdcGaww4o/bzyu4Q5UVRoLSLinnNdEk6UhpmzTvv+hr7RPuTd8hPSVFDW2VDqjZTYjPZi9DNgB3KDA==","signed_message":"open_graph_event_sha256_bytes","signed_at":"2026-08-09T23:02:25.475056Z"},"content_sha256":"695326385e2fccb2b775e15b73a563208b68f84bd022cd9bf698066e11ea2bba","schema_version":"1.0","event_id":"sha256:695326385e2fccb2b775e15b73a563208b68f84bd022cd9bf698066e11ea2bba"},{"event_type":"graph_snapshot","subject_pith_number":"pith:2021:JFLLIOJUI2M3KX3FLLGP4N2V65","target":"graph","payload":{"graph_snapshot":{"paper":{"title":"Layered gradient accumulation and modular pipeline parallelism: fast and efficient training of large language models","license":"http://creativecommons.org/licenses/by-sa/4.0/","headline":"","cross_cats":["cs.AI","cs.CL","cs.DC"],"primary_cat":"cs.LG","authors_text":"Joel Lamy-Poirier","submitted_at":"2021-06-04T19:21:49Z","abstract_excerpt":"The advent of the transformer has sparked a quick growth in the size of language models, far outpacing hardware improvements. (Dense) transformers are expected to reach the trillion-parameter scale in the near future, for which training requires thousands or even tens of thousands of GPUs. We investigate the challenges of training at this scale and beyond on commercially available hardware. In particular, we analyse the shortest possible training time for different configurations of distributed training, leveraging empirical scaling laws for language models to estimate the optimal (critical) b"},"claims":{"count":0,"items":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"2106.02679","kind":"arxiv","version":1},"verdict":{"id":null,"model_set":{},"created_at":null,"strongest_claim":"","one_line_summary":"","pipeline_version":null,"weakest_assumption":"","pith_extraction_headline":""},"integrity":{"clean":true,"summary":{"advisory":0,"critical":0,"by_detector":{},"informational":0},"endpoint":"/pith/2106.02679/integrity.json","findings":[],"available":true,"detectors_run":[],"snapshot_sha256":"c28c3603d3b5d939e8dc4c7e95fa8dfce3d595e45f758748cecf8e644a296938"},"references":{"count":0,"sample":[],"resolved_work":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","internal_anchors":0},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"author_claims":{"count":0,"strong_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"builder_version":"pith-number-builder-2026-05-17-v1"},"verdict_id":null},"signer":{"signer_id":"pith.science","signer_type":"pith_registry","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"created_at":"2026-07-05T02:46:28Z","supersedes":[],"prev_event":null,"signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"4lk3R6DpMSMzE/zHhc42Y+1GS1htQLdlI2i0e36CyrfhwjIeI/zwGwCrzwLC4o+wGi15P5eZcK52XrxSPXELDA==","signed_message":"open_graph_event_sha256_bytes","signed_at":"2026-08-09T23:02:25.475959Z"},"content_sha256":"8780ac4fb61f07c7fe53c7f4b44cf3c8dc641d2f2742612bd8eddbabf36ef8d9","schema_version":"1.0","event_id":"sha256:8780ac4fb61f07c7fe53c7f4b44cf3c8dc641d2f2742612bd8eddbabf36ef8d9"}],"timestamp_proofs":[],"mirror_hints":[{"mirror_type":"https","name":"Pith Resolver","base_url":"https://pith.science","bundle_url":"https://pith.science/pith/JFLLIOJUI2M3KX3FLLGP4N2V65/bundle.json","state_url":"https://pith.science/pith/JFLLIOJUI2M3KX3FLLGP4N2V65/state.json","well_known_bundle_url":"https://pith.science/.well-known/pith/JFLLIOJUI2M3KX3FLLGP4N2V65/bundle.json","status":"primary"}],"public_keys":[{"key_id":"pith-v1-2026-05","algorithm":"ed25519","format":"raw","public_key_b64":"stVStoiQhXFxp4s2pdzPNoqVNBMojDU/fJ2db5S3CbM=","public_key_hex":"b2d552b68890857171a78b36a5dccf368a953413288c353f7c9d9d6f94b709b3","fingerprint_sha256_b32_first128bits":"RVFV5Z2OI2J3ZUO7ERDEBCYNKS","fingerprint_sha256_hex":"8d4b5ee74e4693bcd1df2446408b0d54","rotates_at":null,"url":"https://pith.science/pith-signing-key.json","notes":"Pith uses this Ed25519 key to sign canonical record SHA-256 digests. Verify with: ed25519_verify(public_key, message=canonical_sha256_bytes, signature=base64decode(signature_b64))."}],"merge_version":"pith-open-graph-merge-v1","built_at":"2026-08-09T23:02:25Z","links":{"resolver":"https://pith.science/pith/JFLLIOJUI2M3KX3FLLGP4N2V65","bundle":"https://pith.science/pith/JFLLIOJUI2M3KX3FLLGP4N2V65/bundle.json","state":"https://pith.science/pith/JFLLIOJUI2M3KX3FLLGP4N2V65/state.json","well_known_bundle":"https://pith.science/.well-known/pith/JFLLIOJUI2M3KX3FLLGP4N2V65/bundle.json"},"state":{"state_type":"pith_open_graph_state","state_version":"1.0","pith_number":"pith:2021:JFLLIOJUI2M3KX3FLLGP4N2V65","merge_version":"pith-open-graph-merge-v1","event_count":2,"valid_event_count":2,"invalid_event_count":0,"equivocation_count":0,"current":{"canonical_record":{"metadata":{"abstract_canon_sha256":"ebba9938e8dc4a66cfa5976ba9f46fdca098172c024e0cb498ac94ea0bcd5fe8","cross_cats_sorted":["cs.AI","cs.CL","cs.DC"],"license":"http://creativecommons.org/licenses/by-sa/4.0/","primary_cat":"cs.LG","submitted_at":"2021-06-04T19:21:49Z","title_canon_sha256":"434da334d2825ada3dac75e9e2b905314cb028892db0175cc031b036c7f49566"},"schema_version":"1.0","source":{"id":"2106.02679","kind":"arxiv","version":1}},"source_aliases":[{"alias_kind":"arxiv","alias_value":"2106.02679","created_at":"2026-07-05T02:46:28Z"},{"alias_kind":"arxiv_version","alias_value":"2106.02679v1","created_at":"2026-07-05T02:46:28Z"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.2106.02679","created_at":"2026-07-05T02:46:28Z"},{"alias_kind":"pith_short_12","alias_value":"JFLLIOJUI2M3","created_at":"2026-07-05T02:46:28Z"},{"alias_kind":"pith_short_16","alias_value":"JFLLIOJUI2M3KX3F","created_at":"2026-07-05T02:46:28Z"},{"alias_kind":"pith_short_8","alias_value":"JFLLIOJU","created_at":"2026-07-05T02:46:28Z"}],"graph_snapshots":[{"event_id":"sha256:8780ac4fb61f07c7fe53c7f4b44cf3c8dc641d2f2742612bd8eddbabf36ef8d9","target":"graph","created_at":"2026-07-05T02:46:28Z","signer":{"key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signer_id":"pith.science","signer_type":"pith_registry"},"payload":{"graph_snapshot":{"author_claims":{"count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","strong_count":0},"builder_version":"pith-number-builder-2026-05-17-v1","claims":{"count":0,"items":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"integrity":{"available":true,"clean":true,"detectors_run":[],"endpoint":"/pith/2106.02679/integrity.json","findings":[],"snapshot_sha256":"c28c3603d3b5d939e8dc4c7e95fa8dfce3d595e45f758748cecf8e644a296938","summary":{"advisory":0,"by_detector":{},"critical":0,"informational":0}},"paper":{"abstract_excerpt":"The advent of the transformer has sparked a quick growth in the size of language models, far outpacing hardware improvements. (Dense) transformers are expected to reach the trillion-parameter scale in the near future, for which training requires thousands or even tens of thousands of GPUs. We investigate the challenges of training at this scale and beyond on commercially available hardware. In particular, we analyse the shortest possible training time for different configurations of distributed training, leveraging empirical scaling laws for language models to estimate the optimal (critical) b","authors_text":"Joel Lamy-Poirier","cross_cats":["cs.AI","cs.CL","cs.DC"],"headline":"","license":"http://creativecommons.org/licenses/by-sa/4.0/","primary_cat":"cs.LG","submitted_at":"2021-06-04T19:21:49Z","title":"Layered gradient accumulation and modular pipeline parallelism: fast and efficient training of large language models"},"references":{"count":0,"internal_anchors":0,"resolved_work":0,"sample":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"2106.02679","kind":"arxiv","version":1},"verdict":{"created_at":null,"id":null,"model_set":{},"one_line_summary":"","pipeline_version":null,"pith_extraction_headline":"","strongest_claim":"","weakest_assumption":""}},"verdict_id":null}}],"author_attestations":[],"timestamp_anchors":[],"storage_attestations":[],"citation_signatures":[],"replication_records":[],"corrections":[],"mirror_hints":[],"record_created":{"event_id":"sha256:695326385e2fccb2b775e15b73a563208b68f84bd022cd9bf698066e11ea2bba","target":"record","created_at":"2026-07-05T02:46:28Z","signer":{"key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signer_id":"pith.science","signer_type":"pith_registry"},"payload":{"attestation_state":"computed","canonical_record":{"metadata":{"abstract_canon_sha256":"ebba9938e8dc4a66cfa5976ba9f46fdca098172c024e0cb498ac94ea0bcd5fe8","cross_cats_sorted":["cs.AI","cs.CL","cs.DC"],"license":"http://creativecommons.org/licenses/by-sa/4.0/","primary_cat":"cs.LG","submitted_at":"2021-06-04T19:21:49Z","title_canon_sha256":"434da334d2825ada3dac75e9e2b905314cb028892db0175cc031b036c7f49566"},"schema_version":"1.0","source":{"id":"2106.02679","kind":"arxiv","version":1}},"canonical_sha256":"4956b439344699b55f655accfe3755f76675906c3782b2b848781fbe13e87432","receipt":{"algorithm":"ed25519","builder_version":"pith-number-builder-2026-05-17-v1","canonical_sha256":"4956b439344699b55f655accfe3755f76675906c3782b2b848781fbe13e87432","first_computed_at":"2026-07-05T02:46:28.980929Z","key_id":"pith-v1-2026-05","kind":"pith_receipt","last_reissued_at":"2026-07-05T02:46:28.980929Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","receipt_version":"0.3","signature_b64":"LATkpIUXJMlWEJHZI4zmLK6En7GKKRe38ota61W5nAZgBk5mTKP3Hm5nLNtNsi4Jl3sfonfNhorZ6Csq0IPdDw==","signature_status":"signed_v1","signed_at":"2026-07-05T02:46:28.981389Z","signed_message":"canonical_sha256_bytes"},"source_id":"2106.02679","source_kind":"arxiv","source_version":1}}},"equivocations":[],"invalid_events":[],"applied_event_ids":["sha256:695326385e2fccb2b775e15b73a563208b68f84bd022cd9bf698066e11ea2bba","sha256:8780ac4fb61f07c7fe53c7f4b44cf3c8dc641d2f2742612bd8eddbabf36ef8d9"],"state_sha256":"37afd725873fd1007cec750f0066347491cf96d30b470cf46be971f84f3c265f"},"bundle_signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"5KZ9PDgzqN81kCzArdANOWkhadxPxBZeQpnA7hFYPLIDP/rRhiQcE3iMXY9YWZJ8N7+FGEDwrfpoHSxPS/TuCQ==","signed_message":"bundle_sha256_bytes","signed_at":"2026-08-09T23:02:25.482431Z","bundle_sha256":"2293a6c77651e8d97b6bb86251ab0b2b1e82a300da0cf6fef6b46c1e57f19175"}}