{"bundle_type":"pith_open_graph_bundle","bundle_version":"1.0","pith_number":"pith:2024:7R4PLNKGOPF3E5PIJ6MUJ5PAOL","short_pith_number":"pith:7R4PLNKG","canonical_record":{"source":{"id":"2411.05288","kind":"arxiv","version":2},"metadata":{"license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.DC","submitted_at":"2024-11-08T02:45:30Z","cross_cats_sorted":[],"title_canon_sha256":"d99ccdae091ed9566f0aa5b63e56ac872a5a2c868990e6fbdf45270fe4e07e3a","abstract_canon_sha256":"cafde4f98eb1e517a5bdd8b693c7a21f8b92bdea78320327b1ded662ff83b5d3"},"schema_version":"1.0"},"canonical_sha256":"fc78f5b54673cbb275e84f9944f5e072d2487a4ec49dae11534a1f86fa56d95b","source":{"kind":"arxiv","id":"2411.05288","version":2},"source_aliases":[{"alias_kind":"arxiv","alias_value":"2411.05288","created_at":"2026-07-05T10:58:20Z"},{"alias_kind":"arxiv_version","alias_value":"2411.05288v2","created_at":"2026-07-05T10:58:20Z"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.2411.05288","created_at":"2026-07-05T10:58:20Z"},{"alias_kind":"pith_short_12","alias_value":"7R4PLNKGOPF3","created_at":"2026-07-05T10:58:20Z"},{"alias_kind":"pith_short_16","alias_value":"7R4PLNKGOPF3E5PI","created_at":"2026-07-05T10:58:20Z"},{"alias_kind":"pith_short_8","alias_value":"7R4PLNKG","created_at":"2026-07-05T10:58:20Z"}],"events":[{"event_type":"record_created","subject_pith_number":"pith:2024:7R4PLNKGOPF3E5PIJ6MUJ5PAOL","target":"record","payload":{"canonical_record":{"source":{"id":"2411.05288","kind":"arxiv","version":2},"metadata":{"license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.DC","submitted_at":"2024-11-08T02:45:30Z","cross_cats_sorted":[],"title_canon_sha256":"d99ccdae091ed9566f0aa5b63e56ac872a5a2c868990e6fbdf45270fe4e07e3a","abstract_canon_sha256":"cafde4f98eb1e517a5bdd8b693c7a21f8b92bdea78320327b1ded662ff83b5d3"},"schema_version":"1.0"},"canonical_sha256":"fc78f5b54673cbb275e84f9944f5e072d2487a4ec49dae11534a1f86fa56d95b","receipt":{"kind":"pith_receipt","key_id":"pith-v1-2026-05","algorithm":"ed25519","signed_at":"2026-07-05T10:58:20.114629Z","signature_b64":"mdWLSda0P9ZfEXsFRmizto+WCF0aaFISDywsvyWpzZcAng1eW7+y3qdJlQ8SOP/IAHU5mC3FkgPxmCPxuJk+Dg==","signed_message":"canonical_sha256_bytes","builder_version":"pith-number-builder-2026-05-17-v1","receipt_version":"0.3","canonical_sha256":"fc78f5b54673cbb275e84f9944f5e072d2487a4ec49dae11534a1f86fa56d95b","last_reissued_at":"2026-07-05T10:58:20.114090Z","signature_status":"signed_v1","first_computed_at":"2026-07-05T10:58:20.114090Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"source_kind":"arxiv","source_id":"2411.05288","source_version":2,"attestation_state":"computed"},"signer":{"signer_id":"pith.science","signer_type":"pith_registry","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"created_at":"2026-07-05T10:58:20Z","supersedes":[],"prev_event":null,"signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"CsqMf4mo0bi86W1tx6l+uTAov9F6TmnfjphOtbcrZfYrzBrkzpP83kVfc46Z3eFSQ5KLjFkD+xHdbAYRJflFBA==","signed_message":"open_graph_event_sha256_bytes","signed_at":"2026-07-09T05:57:01.316999Z"},"content_sha256":"653e38602741e4aabfdbf04ee6cce3e91b870c2fd0268bee113e9d35e8af03ee","schema_version":"1.0","event_id":"sha256:653e38602741e4aabfdbf04ee6cce3e91b870c2fd0268bee113e9d35e8af03ee"},{"event_type":"graph_snapshot","subject_pith_number":"pith:2024:7R4PLNKGOPF3E5PIJ6MUJ5PAOL","target":"graph","payload":{"graph_snapshot":{"paper":{"title":"Balancing Pipeline Parallelism with Vocabulary Parallelism","license":"http://creativecommons.org/licenses/by/4.0/","headline":"","cross_cats":[],"primary_cat":"cs.DC","authors_text":"Man Tsung Yeung, Min Lin, Penghui Qi, Xinyi Wan","submitted_at":"2024-11-08T02:45:30Z","abstract_excerpt":"Pipeline parallelism is widely used to scale the training of transformer-based large language models, various works have been done to improve its throughput and memory footprint. In this paper, we address a frequently overlooked issue: the vocabulary layers can cause imbalanced computation and memory usage across pipeline stages, worsening pipeline bubbles and the memory bottleneck. To tackle this, we partition the vocabulary layers evenly across pipeline devices and group the computation into pipeline passes. To reduce the activation memory overhead, we propose several algorithms to reduce co"},"claims":{"count":0,"items":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"2411.05288","kind":"arxiv","version":2},"verdict":{"id":null,"model_set":{},"created_at":null,"strongest_claim":"","one_line_summary":"","pipeline_version":null,"weakest_assumption":"","pith_extraction_headline":""},"integrity":{"clean":true,"summary":{"advisory":0,"critical":0,"by_detector":{},"informational":0},"endpoint":"/pith/2411.05288/integrity.json","findings":[],"available":true,"detectors_run":[],"snapshot_sha256":"c28c3603d3b5d939e8dc4c7e95fa8dfce3d595e45f758748cecf8e644a296938"},"references":{"count":0,"sample":[],"resolved_work":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","internal_anchors":0},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"author_claims":{"count":0,"strong_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"builder_version":"pith-number-builder-2026-05-17-v1"},"verdict_id":null},"signer":{"signer_id":"pith.science","signer_type":"pith_registry","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"created_at":"2026-07-05T10:58:20Z","supersedes":[],"prev_event":null,"signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"bhtWxThWJIDVe/iZmtCGY3FlnnEVwG1cH3W9nkub1pH9kzx7C6+ewS8mESzn2rIo6MRVSLgl0Jhb91Pv/XStDA==","signed_message":"open_graph_event_sha256_bytes","signed_at":"2026-07-09T05:57:01.317648Z"},"content_sha256":"d4cdf6911ad8022ba77fd242c7b52977ece19acc66a46f96e87c5c00fa314389","schema_version":"1.0","event_id":"sha256:d4cdf6911ad8022ba77fd242c7b52977ece19acc66a46f96e87c5c00fa314389"}],"timestamp_proofs":[],"mirror_hints":[{"mirror_type":"https","name":"Pith Resolver","base_url":"https://pith.science","bundle_url":"https://pith.science/pith/7R4PLNKGOPF3E5PIJ6MUJ5PAOL/bundle.json","state_url":"https://pith.science/pith/7R4PLNKGOPF3E5PIJ6MUJ5PAOL/state.json","well_known_bundle_url":"https://pith.science/.well-known/pith/7R4PLNKGOPF3E5PIJ6MUJ5PAOL/bundle.json","status":"primary"}],"public_keys":[{"key_id":"pith-v1-2026-05","algorithm":"ed25519","format":"raw","public_key_b64":"stVStoiQhXFxp4s2pdzPNoqVNBMojDU/fJ2db5S3CbM=","public_key_hex":"b2d552b68890857171a78b36a5dccf368a953413288c353f7c9d9d6f94b709b3","fingerprint_sha256_b32_first128bits":"RVFV5Z2OI2J3ZUO7ERDEBCYNKS","fingerprint_sha256_hex":"8d4b5ee74e4693bcd1df2446408b0d54","rotates_at":null,"url":"https://pith.science/pith-signing-key.json","notes":"Pith uses this Ed25519 key to sign canonical record SHA-256 digests. Verify with: ed25519_verify(public_key, message=canonical_sha256_bytes, signature=base64decode(signature_b64))."}],"merge_version":"pith-open-graph-merge-v1","built_at":"2026-07-09T05:57:01Z","links":{"resolver":"https://pith.science/pith/7R4PLNKGOPF3E5PIJ6MUJ5PAOL","bundle":"https://pith.science/pith/7R4PLNKGOPF3E5PIJ6MUJ5PAOL/bundle.json","state":"https://pith.science/pith/7R4PLNKGOPF3E5PIJ6MUJ5PAOL/state.json","well_known_bundle":"https://pith.science/.well-known/pith/7R4PLNKGOPF3E5PIJ6MUJ5PAOL/bundle.json"},"state":{"state_type":"pith_open_graph_state","state_version":"1.0","pith_number":"pith:2024:7R4PLNKGOPF3E5PIJ6MUJ5PAOL","merge_version":"pith-open-graph-merge-v1","event_count":2,"valid_event_count":2,"invalid_event_count":0,"equivocation_count":0,"current":{"canonical_record":{"metadata":{"abstract_canon_sha256":"cafde4f98eb1e517a5bdd8b693c7a21f8b92bdea78320327b1ded662ff83b5d3","cross_cats_sorted":[],"license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.DC","submitted_at":"2024-11-08T02:45:30Z","title_canon_sha256":"d99ccdae091ed9566f0aa5b63e56ac872a5a2c868990e6fbdf45270fe4e07e3a"},"schema_version":"1.0","source":{"id":"2411.05288","kind":"arxiv","version":2}},"source_aliases":[{"alias_kind":"arxiv","alias_value":"2411.05288","created_at":"2026-07-05T10:58:20Z"},{"alias_kind":"arxiv_version","alias_value":"2411.05288v2","created_at":"2026-07-05T10:58:20Z"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.2411.05288","created_at":"2026-07-05T10:58:20Z"},{"alias_kind":"pith_short_12","alias_value":"7R4PLNKGOPF3","created_at":"2026-07-05T10:58:20Z"},{"alias_kind":"pith_short_16","alias_value":"7R4PLNKGOPF3E5PI","created_at":"2026-07-05T10:58:20Z"},{"alias_kind":"pith_short_8","alias_value":"7R4PLNKG","created_at":"2026-07-05T10:58:20Z"}],"graph_snapshots":[{"event_id":"sha256:d4cdf6911ad8022ba77fd242c7b52977ece19acc66a46f96e87c5c00fa314389","target":"graph","created_at":"2026-07-05T10:58:20Z","signer":{"key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signer_id":"pith.science","signer_type":"pith_registry"},"payload":{"graph_snapshot":{"author_claims":{"count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","strong_count":0},"builder_version":"pith-number-builder-2026-05-17-v1","claims":{"count":0,"items":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"integrity":{"available":true,"clean":true,"detectors_run":[],"endpoint":"/pith/2411.05288/integrity.json","findings":[],"snapshot_sha256":"c28c3603d3b5d939e8dc4c7e95fa8dfce3d595e45f758748cecf8e644a296938","summary":{"advisory":0,"by_detector":{},"critical":0,"informational":0}},"paper":{"abstract_excerpt":"Pipeline parallelism is widely used to scale the training of transformer-based large language models, various works have been done to improve its throughput and memory footprint. In this paper, we address a frequently overlooked issue: the vocabulary layers can cause imbalanced computation and memory usage across pipeline stages, worsening pipeline bubbles and the memory bottleneck. To tackle this, we partition the vocabulary layers evenly across pipeline devices and group the computation into pipeline passes. To reduce the activation memory overhead, we propose several algorithms to reduce co","authors_text":"Man Tsung Yeung, Min Lin, Penghui Qi, Xinyi Wan","cross_cats":[],"headline":"","license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.DC","submitted_at":"2024-11-08T02:45:30Z","title":"Balancing Pipeline Parallelism with Vocabulary Parallelism"},"references":{"count":0,"internal_anchors":0,"resolved_work":0,"sample":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"2411.05288","kind":"arxiv","version":2},"verdict":{"created_at":null,"id":null,"model_set":{},"one_line_summary":"","pipeline_version":null,"pith_extraction_headline":"","strongest_claim":"","weakest_assumption":""}},"verdict_id":null}}],"author_attestations":[],"timestamp_anchors":[],"storage_attestations":[],"citation_signatures":[],"replication_records":[],"corrections":[],"mirror_hints":[],"record_created":{"event_id":"sha256:653e38602741e4aabfdbf04ee6cce3e91b870c2fd0268bee113e9d35e8af03ee","target":"record","created_at":"2026-07-05T10:58:20Z","signer":{"key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signer_id":"pith.science","signer_type":"pith_registry"},"payload":{"attestation_state":"computed","canonical_record":{"metadata":{"abstract_canon_sha256":"cafde4f98eb1e517a5bdd8b693c7a21f8b92bdea78320327b1ded662ff83b5d3","cross_cats_sorted":[],"license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.DC","submitted_at":"2024-11-08T02:45:30Z","title_canon_sha256":"d99ccdae091ed9566f0aa5b63e56ac872a5a2c868990e6fbdf45270fe4e07e3a"},"schema_version":"1.0","source":{"id":"2411.05288","kind":"arxiv","version":2}},"canonical_sha256":"fc78f5b54673cbb275e84f9944f5e072d2487a4ec49dae11534a1f86fa56d95b","receipt":{"algorithm":"ed25519","builder_version":"pith-number-builder-2026-05-17-v1","canonical_sha256":"fc78f5b54673cbb275e84f9944f5e072d2487a4ec49dae11534a1f86fa56d95b","first_computed_at":"2026-07-05T10:58:20.114090Z","key_id":"pith-v1-2026-05","kind":"pith_receipt","last_reissued_at":"2026-07-05T10:58:20.114090Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","receipt_version":"0.3","signature_b64":"mdWLSda0P9ZfEXsFRmizto+WCF0aaFISDywsvyWpzZcAng1eW7+y3qdJlQ8SOP/IAHU5mC3FkgPxmCPxuJk+Dg==","signature_status":"signed_v1","signed_at":"2026-07-05T10:58:20.114629Z","signed_message":"canonical_sha256_bytes"},"source_id":"2411.05288","source_kind":"arxiv","source_version":2}}},"equivocations":[],"invalid_events":[],"applied_event_ids":["sha256:653e38602741e4aabfdbf04ee6cce3e91b870c2fd0268bee113e9d35e8af03ee","sha256:d4cdf6911ad8022ba77fd242c7b52977ece19acc66a46f96e87c5c00fa314389"],"state_sha256":"0d5743b37d067cac504cbee21c1cf024afdd21292736156f05ec76211971a501"},"bundle_signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"LEPza9kqdSGqzpbOpyx++NVZWnf//FqZndcSG5mrS+/LjN63ThdrCgPCpCuLNlLdRBW8dTfwnGk9dQ5hxx90DA==","signed_message":"bundle_sha256_bytes","signed_at":"2026-07-09T05:57:01.321598Z","bundle_sha256":"8164b3606212e2b61167482d4bc748a457476032556df6994d96571be7bce33e"}}