{"work":{"id":"283d6d50-8219-44e7-94e7-eb6b8673dec7","openalex_id":"https://openalex.org/W4404649984","doi":"10.48550/arxiv.2411.13676","arxiv_id":"2411.13676","raw_key":null,"title":"Hymba: A Hybrid-head Architecture for Small Language Models","authors":null,"authors_text":"Xin Dong, Yonggan Fu, Shizhe Diao, Wonmin Byeon, Zijia Chen, Ameya Sunil Mahabaleshwarkar, Shih-Yang Liu, Matthijs Van Keirs- bilck, Min-Hung Chen, Yoshi Suhara, et al","year":2024,"venue":"cs.CL","abstract":"We propose Hymba, a family of small language models featuring a hybrid-head parallel architecture that integrates transformer attention mechanisms with state space models (SSMs) for enhanced efficiency. Attention heads provide high-resolution recall, while SSM heads enable efficient context summarization. Additionally, we introduce learnable meta tokens that are prepended to prompts, storing critical information and alleviating the \"forced-to-attend\" burden associated with attention mechanisms. This model is further optimized by incorporating cross-layer key-value (KV) sharing and partial sliding window attention, resulting in a compact cache size. During development, we conducted a controlled study comparing various architectures under identical settings and observed significant advantages of our proposed architecture. Notably, Hymba achieves state-of-the-art results for small LMs: Our Hymba-1.5B-Base model surpasses all sub-2B public models in performance and even outperforms Llama-3.2-3B with 1.32% higher average accuracy, an 11.67x cache size reduction, and 3.49x throughput.","external_url":"https://arxiv.org/abs/2411.13676","cited_by_count":2,"metadata_source":"pith","metadata_fetched_at":"2026-08-05T02:28:24.338817+00:00","pith_arxiv_id":"2411.13676","created_at":"2026-05-10T14:25:29.247682+00:00","updated_at":"2026-08-05T02:28:24.338817+00:00","title_quality_ok":true,"display_title":"Hymba: A hybrid-head architecture for small language models","render_title":"Hymba: A hybrid-head architecture for small language models"},"hub":{"state":{"work_id":"283d6d50-8219-44e7-94e7-eb6b8673dec7","tier":"hub","tier_reason":"10+ Pith inbound or 1,000+ external citations","pith_inbound_count":26,"external_cited_by_count":2,"distinct_field_count":5,"first_pith_cited_at":"2024-12-31T22:32:03+00:00","last_pith_cited_at":"2026-07-07T17:29:01+00:00","author_build_status":"not_needed","summary_status":"needed","contexts_status":"needed","graph_status":"needed","ask_index_status":"not_needed","reader_status":"not_needed","recognition_status":"not_needed","updated_at":"2026-08-22T07:49:51.291894+00:00","tier_text":"hub"},"tier":"hub","role_counts":[{"context_role":"background","n":5}],"polarity_counts":[{"context_polarity":"background","n":3},{"context_polarity":"unclear","n":2}],"runs":{},"summary":{},"graph":{},"authors":[]}}