{"work":{"id":"d2c14bfc-b868-4f6d-8b13-12bf89321d47","openalex_id":null,"doi":null,"arxiv_id":"2401.02385","raw_key":null,"title":"TinyLlama: An Open-Source Small Language Model","authors":null,"authors_text":"Peiyuan Zhang, Guangtao Zeng, Tianduo Wang, Wei Lu","year":2024,"venue":"cs.CL","abstract":"We present TinyLlama, a compact 1.1B language model pretrained on around 1 trillion tokens for approximately 3 epochs. Building on the architecture and tokenizer of Llama 2, TinyLlama leverages various advances contributed by the open-source community (e.g., FlashAttention and Lit-GPT), achieving better computational efficiency. Despite its relatively small size, TinyLlama demonstrates remarkable performance in a series of downstream tasks. It significantly outperforms existing open-source language models with comparable sizes. Our model checkpoints and code are publicly available on GitHub at https://github.com/jzhang38/TinyLlama.","external_url":"https://arxiv.org/abs/2401.02385","cited_by_count":null,"metadata_source":"pith","metadata_fetched_at":"2026-05-25T04:45:20.686957+00:00","pith_arxiv_id":"2401.02385","created_at":"2026-05-09T06:20:42.505095+00:00","updated_at":"2026-05-25T04:45:20.686957+00:00","title_quality_ok":true,"display_title":"TinyLlama: An Open-Source Small Language Model","render_title":"TinyLlama: An Open-Source Small Language Model"},"hub":{"state":{"work_id":"d2c14bfc-b868-4f6d-8b13-12bf89321d47","tier":"hub","tier_reason":"10+ Pith inbound or 1,000+ external citations","pith_inbound_count":47,"external_cited_by_count":null,"distinct_field_count":12,"first_pith_cited_at":"2024-01-26T18:59:01+00:00","last_pith_cited_at":"2026-05-22T17:16:35+00:00","author_build_status":"not_needed","summary_status":"needed","contexts_status":"needed","graph_status":"needed","ask_index_status":"not_needed","reader_status":"not_needed","recognition_status":"not_needed","updated_at":"2026-06-04T15:57:45.563489+00:00","tier_text":"hub"},"tier":"hub","role_counts":[{"context_role":"background","n":5}],"polarity_counts":[{"context_polarity":"background","n":5}],"runs":{},"summary":{},"graph":{},"authors":[]}}