{"work":{"id":"277d0fdf-d9cc-4497-ad08-04aba6226c36","openalex_id":"https://openalex.org/W4397024360","doi":"10.48550/arxiv.2405.07719","arxiv_id":"2405.07719","raw_key":null,"title":"USP: A Unified Sequence Parallelism Approach for Long Context Generative AI","authors":null,"authors_text":"A Unified Sequence Parallelism Approach for Long Context Generative AI , author=","year":2024,"venue":"cs.LG","abstract":"Sequence parallelism (SP), which divides the sequence dimension of input tensors across multiple computational devices, is becoming key to unlocking the long-context capabilities of generative AI models. This paper investigates the state-of-the-art SP approaches, i.e. DeepSpeed-Ulysses and Ring-Attention, and proposes a unified SP approach, which is more robust to transformer model architectures and network hardware topology. This paper compares the communication and memory cost of SP and existing parallelism, including data/tensor/zero/pipeline parallelism, and discusses the best practices for designing hybrid 4D parallelism involving SP. We achieved 47% MFU on two 8xA800 nodes using SP for the LLAMA3-8B model training using sequence length 208K. Our code is publicly available at https://github.com/feifeibear/long-context-attention.","external_url":"https://arxiv.org/abs/2405.07719","cited_by_count":2,"metadata_source":"pith","metadata_fetched_at":"2026-08-05T02:28:24.338817+00:00","pith_arxiv_id":"2405.07719","created_at":"2026-05-08T18:28:57.545285+00:00","updated_at":"2026-08-05T02:28:24.338817+00:00","title_quality_ok":true,"display_title":"A unified sequence parallelism approach for long context generative ai","render_title":"A unified sequence parallelism approach for long context generative ai"},"hub":{"state":{"work_id":"277d0fdf-d9cc-4497-ad08-04aba6226c36","tier":"hub","tier_reason":"10+ Pith inbound or 1,000+ external citations","pith_inbound_count":19,"external_cited_by_count":2,"distinct_field_count":4,"first_pith_cited_at":"2024-05-23T11:00:07+00:00","last_pith_cited_at":"2026-07-07T12:41:19+00:00","author_build_status":"not_needed","summary_status":"needed","contexts_status":"needed","graph_status":"needed","ask_index_status":"not_needed","reader_status":"not_needed","recognition_status":"not_needed","updated_at":"2026-08-24T00:49:40.324974+00:00","tier_text":"hub"},"tier":"hub","role_counts":[{"context_role":"method","n":5},{"context_role":"background","n":1},{"context_role":"baseline","n":1}],"polarity_counts":[{"context_polarity":"use_method","n":5},{"context_polarity":"background","n":1},{"context_polarity":"baseline","n":1}],"runs":{},"summary":{},"graph":{},"authors":[]}}