{"record_type":"pith_number_record","schema_url":"https://pith.science/schemas/pith-number/v1.json","pith_number":"pith:2024:7R4PLNKGOPF3E5PIJ6MUJ5PAOL","short_pith_number":"pith:7R4PLNKG","schema_version":"1.0","canonical_sha256":"fc78f5b54673cbb275e84f9944f5e072d2487a4ec49dae11534a1f86fa56d95b","source":{"kind":"arxiv","id":"2411.05288","version":2},"attestation_state":"computed","paper":{"title":"Balancing Pipeline Parallelism with Vocabulary Parallelism","license":"http://creativecommons.org/licenses/by/4.0/","headline":"","cross_cats":[],"primary_cat":"cs.DC","authors_text":"Man Tsung Yeung, Min Lin, Penghui Qi, Xinyi Wan","submitted_at":"2024-11-08T02:45:30Z","abstract_excerpt":"Pipeline parallelism is widely used to scale the training of transformer-based large language models, various works have been done to improve its throughput and memory footprint. In this paper, we address a frequently overlooked issue: the vocabulary layers can cause imbalanced computation and memory usage across pipeline stages, worsening pipeline bubbles and the memory bottleneck. To tackle this, we partition the vocabulary layers evenly across pipeline devices and group the computation into pipeline passes. To reduce the activation memory overhead, we propose several algorithms to reduce co"},"verification_status":{"content_addressed":true,"pith_receipt":true,"author_attested":false,"weak_author_claims":0,"strong_author_claims":0,"externally_anchored":false,"storage_verified":false,"citation_signatures":0,"replication_records":0,"graph_snapshot":true,"references_resolved":false,"formal_links_present":false},"canonical_record":{"source":{"id":"2411.05288","kind":"arxiv","version":2},"metadata":{"license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.DC","submitted_at":"2024-11-08T02:45:30Z","cross_cats_sorted":[],"title_canon_sha256":"d99ccdae091ed9566f0aa5b63e56ac872a5a2c868990e6fbdf45270fe4e07e3a","abstract_canon_sha256":"cafde4f98eb1e517a5bdd8b693c7a21f8b92bdea78320327b1ded662ff83b5d3"},"schema_version":"1.0"},"receipt":{"kind":"pith_receipt","key_id":"pith-v1-2026-05","algorithm":"ed25519","signed_at":"2026-07-05T10:58:20.114629Z","signature_b64":"mdWLSda0P9ZfEXsFRmizto+WCF0aaFISDywsvyWpzZcAng1eW7+y3qdJlQ8SOP/IAHU5mC3FkgPxmCPxuJk+Dg==","signed_message":"canonical_sha256_bytes","builder_version":"pith-number-builder-2026-05-17-v1","receipt_version":"0.3","canonical_sha256":"fc78f5b54673cbb275e84f9944f5e072d2487a4ec49dae11534a1f86fa56d95b","last_reissued_at":"2026-07-05T10:58:20.114090Z","signature_status":"signed_v1","first_computed_at":"2026-07-05T10:58:20.114090Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"graph_snapshot":{"paper":{"title":"Balancing Pipeline Parallelism with Vocabulary Parallelism","license":"http://creativecommons.org/licenses/by/4.0/","headline":"","cross_cats":[],"primary_cat":"cs.DC","authors_text":"Man Tsung Yeung, Min Lin, Penghui Qi, Xinyi Wan","submitted_at":"2024-11-08T02:45:30Z","abstract_excerpt":"Pipeline parallelism is widely used to scale the training of transformer-based large language models, various works have been done to improve its throughput and memory footprint. In this paper, we address a frequently overlooked issue: the vocabulary layers can cause imbalanced computation and memory usage across pipeline stages, worsening pipeline bubbles and the memory bottleneck. To tackle this, we partition the vocabulary layers evenly across pipeline devices and group the computation into pipeline passes. To reduce the activation memory overhead, we propose several algorithms to reduce co"},"claims":{"count":0,"items":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"2411.05288","kind":"arxiv","version":2},"verdict":{"id":null,"model_set":{},"created_at":null,"strongest_claim":"","one_line_summary":"","pipeline_version":null,"weakest_assumption":"","pith_extraction_headline":""},"integrity":{"clean":true,"summary":{"advisory":0,"critical":0,"by_detector":{},"informational":0},"endpoint":"/pith/2411.05288/integrity.json","findings":[],"available":true,"detectors_run":[],"snapshot_sha256":"c28c3603d3b5d939e8dc4c7e95fa8dfce3d595e45f758748cecf8e644a296938"},"references":{"count":0,"sample":[],"resolved_work":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","internal_anchors":0},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"author_claims":{"count":0,"strong_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"builder_version":"pith-number-builder-2026-05-17-v1"},"aliases":[{"alias_kind":"arxiv","alias_value":"2411.05288","created_at":"2026-07-05T10:58:20.114155+00:00"},{"alias_kind":"arxiv_version","alias_value":"2411.05288v2","created_at":"2026-07-05T10:58:20.114155+00:00"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.2411.05288","created_at":"2026-07-05T10:58:20.114155+00:00"},{"alias_kind":"pith_short_12","alias_value":"7R4PLNKGOPF3","created_at":"2026-07-05T10:58:20.114155+00:00"},{"alias_kind":"pith_short_16","alias_value":"7R4PLNKGOPF3E5PI","created_at":"2026-07-05T10:58:20.114155+00:00"},{"alias_kind":"pith_short_8","alias_value":"7R4PLNKG","created_at":"2026-07-05T10:58:20.114155+00:00"}],"events":[],"event_summary":{},"paper_claims":[],"inbound_citations":{"count":0,"internal_anchor_count":0,"sample":[]},"formal_canon":{"evidence_count":0,"sample":[],"anchors":[]},"links":{"html":"https://pith.science/pith/7R4PLNKGOPF3E5PIJ6MUJ5PAOL","json":"https://pith.science/pith/7R4PLNKGOPF3E5PIJ6MUJ5PAOL.json","graph_json":"https://pith.science/api/pith-number/7R4PLNKGOPF3E5PIJ6MUJ5PAOL/graph.json","events_json":"https://pith.science/api/pith-number/7R4PLNKGOPF3E5PIJ6MUJ5PAOL/events.json","paper":"https://pith.science/paper/7R4PLNKG"},"agent_actions":{"view_html":"https://pith.science/pith/7R4PLNKGOPF3E5PIJ6MUJ5PAOL","download_json":"https://pith.science/pith/7R4PLNKGOPF3E5PIJ6MUJ5PAOL.json","view_paper":"https://pith.science/paper/7R4PLNKG","resolve_alias":"https://pith.science/api/pith-number/resolve?arxiv=2411.05288&json=true","fetch_graph":"https://pith.science/api/pith-number/7R4PLNKGOPF3E5PIJ6MUJ5PAOL/graph.json","fetch_events":"https://pith.science/api/pith-number/7R4PLNKGOPF3E5PIJ6MUJ5PAOL/events.json","actions":{"anchor_timestamp":"https://pith.science/pith/7R4PLNKGOPF3E5PIJ6MUJ5PAOL/action/timestamp_anchor","attest_storage":"https://pith.science/pith/7R4PLNKGOPF3E5PIJ6MUJ5PAOL/action/storage_attestation","attest_author":"https://pith.science/pith/7R4PLNKGOPF3E5PIJ6MUJ5PAOL/action/author_attestation","sign_citation":"https://pith.science/pith/7R4PLNKGOPF3E5PIJ6MUJ5PAOL/action/citation_signature","submit_replication":"https://pith.science/pith/7R4PLNKGOPF3E5PIJ6MUJ5PAOL/action/replication_record"}},"created_at":"2026-07-05T10:58:20.114155+00:00","updated_at":"2026-07-05T10:58:20.114155+00:00"}