{"work":{"id":"168a55d5-675d-49cf-be47-a17ee8cd742e","openalex_id":null,"doi":null,"arxiv_id":"2204.06745","raw_key":null,"title":"GPT-NeoX-20B: An Open-Source Autoregressive Language Model","authors":null,"authors_text":"Sid Black, Stella Biderman, Eric Hallahan, Quentin Anthony, Leo Gao, Laurence Golding, Horace He, Connor Leahy, Kyle McDonell, Jason Phang, et al","year":2022,"venue":"cs.CL","abstract":"We introduce GPT-NeoX-20B, a 20 billion parameter autoregressive language model trained on the Pile, whose weights will be made freely and openly available to the public through a permissive license. It is, to the best of our knowledge, the largest dense autoregressive model that has publicly available weights at the time of submission. In this work, we describe \\model{}'s architecture and training and evaluate its performance on a range of language-understanding, mathematics, and knowledge-based tasks. We find that GPT-NeoX-20B is a particularly powerful few-shot reasoner and gains far more in performance when evaluated five-shot than similarly sized GPT-3 and FairSeq models. We open-source the training and evaluation code, as well as the model weights, at https://github.com/EleutherAI/gpt-neox.","external_url":"https://arxiv.org/abs/2204.06745","cited_by_count":null,"metadata_source":"pith","metadata_fetched_at":"2026-05-24T08:29:11.384598+00:00","pith_arxiv_id":"2204.06745","created_at":"2026-05-09T05:55:30.187210+00:00","updated_at":"2026-05-24T08:29:11.384598+00:00","title_quality_ok":true,"display_title":"S., Purohit, S., Reynolds, L., Tow, J., Wang, B., and Weinbach, S","render_title":"S., Purohit, S., Reynolds, L., Tow, J., Wang, B., and Weinbach, S"},"hub":{"state":{"work_id":"168a55d5-675d-49cf-be47-a17ee8cd742e","tier":"hub","tier_reason":"10+ Pith inbound or 1,000+ external citations","pith_inbound_count":34,"external_cited_by_count":null,"distinct_field_count":7,"first_pith_cited_at":"2022-05-02T17:49:50+00:00","last_pith_cited_at":"2026-05-11T09:50:10+00:00","author_build_status":"not_needed","summary_status":"needed","contexts_status":"needed","graph_status":"needed","ask_index_status":"not_needed","reader_status":"not_needed","recognition_status":"not_needed","updated_at":"2026-05-31T13:32:42.994353+00:00","tier_text":"hub"},"tier":"hub","role_counts":[{"context_role":"background","n":11},{"context_role":"baseline","n":1},{"context_role":"dataset","n":1}],"polarity_counts":[{"context_polarity":"background","n":11},{"context_polarity":"baseline","n":1},{"context_polarity":"use_dataset","n":1}],"runs":{},"summary":{},"graph":{},"authors":[]}}