{"work":{"id":"b53b11e5-8613-439d-9d9d-0e2a9090d79f","openalex_id":null,"doi":null,"arxiv_id":"2302.01318","raw_key":null,"title":"Accelerating Large Language Model Decoding with Speculative Sampling","authors":null,"authors_text":"Charlie Chen, Sebastian Borgeaud, Geoffrey Irving, Jean-Baptiste Lespiau, Laurent Sifre, John Jumper","year":2023,"venue":"cs.CL","abstract":"We present speculative sampling, an algorithm for accelerating transformer decoding by enabling the generation of multiple tokens from each transformer call. Our algorithm relies on the observation that the latency of parallel scoring of short continuations, generated by a faster but less powerful draft model, is comparable to that of sampling a single token from the larger target model. This is combined with a novel modified rejection sampling scheme which preserves the distribution of the target model within hardware numerics. We benchmark speculative sampling with Chinchilla, a 70 billion parameter language model, achieving a 2-2.5x decoding speedup in a distributed setup, without compromising the sample quality or making modifications to the model itself.","external_url":"https://arxiv.org/abs/2302.01318","cited_by_count":null,"metadata_source":"pith","metadata_fetched_at":"2026-05-25T05:06:38.454845+00:00","pith_arxiv_id":"2302.01318","created_at":"2026-05-09T03:35:03.480377+00:00","updated_at":"2026-06-05T21:23:00.469572+00:00","title_quality_ok":true,"display_title":"Accelerating Large Language Model Decoding with Speculative Sampling","render_title":"Accelerating Large Language Model Decoding with Speculative Sampling"},"hub":{"state":{"work_id":"b53b11e5-8613-439d-9d9d-0e2a9090d79f","tier":"hub","tier_reason":"10+ Pith inbound or 1,000+ external citations","pith_inbound_count":95,"external_cited_by_count":null,"distinct_field_count":15,"first_pith_cited_at":"2022-11-30T17:33:28+00:00","last_pith_cited_at":"2026-05-22T02:31:32+00:00","author_build_status":"not_needed","summary_status":"needed","contexts_status":"needed","graph_status":"needed","ask_index_status":"not_needed","reader_status":"not_needed","recognition_status":"not_needed","updated_at":"2026-06-07T02:31:13.597353+00:00","tier_text":"hub"},"tier":"hub","role_counts":[{"context_role":"background","n":17},{"context_role":"method","n":5},{"context_role":"other","n":1}],"polarity_counts":[{"context_polarity":"background","n":17},{"context_polarity":"use_method","n":5},{"context_polarity":"unclear","n":1}],"runs":{"context_extract":{"job_type":"context_extract","status":"succeeded","result":{"enqueued_papers":25},"error":null,"updated_at":"2026-05-14T11:29:48.473255+00:00"},"graph_features":{"job_type":"graph_features","status":"succeeded","result":{"co_cited":[{"title":"Training Verifiers to Solve Math Word Problems","work_id":"acab1aa8-b4d6-40e0-a3ee-25341701dca2","shared_citers":25},{"title":"Qwen3 Technical Report","work_id":"25a4e30c-1232-48e7-9925-02fa12ba7c9e","shared_citers":16},{"title":"Evaluating Large Language Models Trained on Code","work_id":"042493e9-b26f-4b4e-bbde-382072ca9b08","shared_citers":14},{"title":"The Llama 3 Herd of Models","work_id":"1549a635-88af-4ac1-acfe-51ae7bb53345","shared_citers":12},{"title":"Medusa: Simple LLM Inference Acceleration Framework with Multiple Decoding Heads","work_id":"f6202cd1-1a78-4c19-8242-688acf4952b6","shared_citers":10},{"title":"DeepSeek-R1: Incentivizing Reasoning Capability in LLMs via Reinforcement Learning","work_id":"e6b75ad5-2877-4168-97c8-710407094d20","shared_citers":9},{"title":"Eagle-3: Scaling up inference acceleration of large language models via training-time test","work_id":"323c1b68-aec5-4444-944f-155a771bd8c6","shared_citers":9},{"title":"Eagle: Speculative sampling requires rethinking feature uncertainty","work_id":"9d4637dd-1cab-4f10-82d4-8c8d14bb96ed","shared_citers":7},{"title":"arXiv:2305.09781 , year=","work_id":"e08525de-cd5d-419d-8e75-7f51a8969588","shared_citers":6},{"title":"Fast inference from transformers via speculative decoding","work_id":"a17596db-fb15-4793-9ce6-fc620a92ae36","shared_citers":6},{"title":"GPT-4 Technical Report","work_id":"b928e041-6991-4c08-8c81-0359e4097c7b","shared_citers":6},{"title":"LLaMA: Open and Efficient Foundation Language Models","work_id":"c018fc23-6f3f-4035-9d02-28a2173b2b9d","shared_citers":6},{"title":"Scaling Laws for Neural Language Models","work_id":"b7dd8749-9c45-4977-ab9b-64478dce1ae8","shared_citers":6},{"title":"Zhuoming Chen, Avner May, Ruslan Svirschevski, Yuhsun Huang, Max Ryabinin, Zhihao Jia, and Beidi Chen","work_id":"860ff726-6feb-4917-a25f-95159b43dd49","shared_citers":6},{"title":"DeepSeek-V3 Technical Report","work_id":"57d2791d-2219-4c31-a077-afc04b12a75c","shared_citers":5},{"title":"Measuring Mathematical Problem Solving With the MATH Dataset","work_id":"50652ac6-fb7c-4675-a2c2-159c241feb17","shared_citers":5},{"title":"DeepSeekMath: Pushing the Limits of Mathematical Reasoning in Open Language Models","work_id":"c5006563-f3ec-438a-9e35-b7b484f34828","shared_citers":4},{"title":"Distilling the Knowledge in a Neural Network","work_id":"d927ab1f-17b8-4002-9d09-c3d55764fbad","shared_citers":4},{"title":"Distillspec: Improving speculative decoding via knowledge distillation","work_id":"5539a830-000f-41f0-9344-c030679cc16f","shared_citers":4},{"title":"FrugalGPT: How to Use Large Language Models While Reducing Cost and Improving Performance","work_id":"5c567414-35de-4373-ae7a-1790da594e8d","shared_citers":4},{"title":"GPQA: A Graduate-Level Google-Proof Q&A Benchmark","work_id":"9e2a976b-f5ad-4aee-af5c-243fe0fe75d2","shared_citers":4},{"title":"gpt-oss-120b & gpt-oss-20b Model Card","work_id":"178c1f7e-4f19-4392-a45d-45a6dfa88ead","shared_citers":4},{"title":"GPTQ: Accurate Post-Training Quantization for Generative Pre-trained Transformers","work_id":"19ed8c44-202a-48f6-8169-637d5a5f2408","shared_citers":4},{"title":"Gulavani, and Ramachandran Ramjee","work_id":"3dbdd757-ca01-436f-acfd-12ffcd6f64c6","shared_citers":4}],"time_series":[{"n":1,"year":2023},{"n":4,"year":2024},{"n":1,"year":2025},{"n":51,"year":2026}],"dependency_candidates":[]},"error":null,"updated_at":"2026-05-14T11:29:57.498023+00:00"},"identity_refresh":{"job_type":"identity_refresh","status":"succeeded","result":{"items":[{"title":"Qwen3 Technical Report","outcome":"unchanged","work_id":"25a4e30c-1232-48e7-9925-02fa12ba7c9e","resolver":"local_arxiv","confidence":0.98,"old_work_id":"25a4e30c-1232-48e7-9925-02fa12ba7c9e"}],"counts":{"fixed":0,"merged":0,"unchanged":1,"quarantined":0,"needs_external_resolution":0},"errors":[],"attempted":1},"error":null,"updated_at":"2026-05-14T11:29:53.069580+00:00"},"summary_claims":{"job_type":"summary_claims","status":"succeeded","result":{"title":"Accelerating Large Language Model Decoding with Speculative Sampling","claims":[{"claim_text":"We present speculative sampling, an algorithm for accelerating transformer decoding by enabling the generation of multiple tokens from each transformer call. Our algorithm relies on the observation that the latency of parallel scoring of short continuations, generated by a faster but less powerful draft model, is comparable to that of sampling a single token from the larger target model. This is combined with a novel modified rejection sampling scheme which preserves the distribution of the target model within hardware numerics. We benchmark speculative sampling with Chinchilla, a 70 billion p","claim_type":"abstract","evidence_strength":"source_metadata"}],"why_cited":"Pith tracks Accelerating Large Language Model Decoding with Speculative Sampling because it crossed a citation-hub threshold.","role_counts":[]},"error":null,"updated_at":"2026-05-14T11:29:46.501598+00:00"}},"summary":{"title":"Accelerating Large Language Model Decoding with Speculative Sampling","claims":[{"claim_text":"We present speculative sampling, an algorithm for accelerating transformer decoding by enabling the generation of multiple tokens from each transformer call. Our algorithm relies on the observation that the latency of parallel scoring of short continuations, generated by a faster but less powerful draft model, is comparable to that of sampling a single token from the larger target model. This is combined with a novel modified rejection sampling scheme which preserves the distribution of the target model within hardware numerics. We benchmark speculative sampling with Chinchilla, a 70 billion p","claim_type":"abstract","evidence_strength":"source_metadata"}],"why_cited":"Pith tracks Accelerating Large Language Model Decoding with Speculative Sampling because it crossed a citation-hub threshold.","role_counts":[]},"graph":{"co_cited":[{"title":"Training Verifiers to Solve Math Word Problems","work_id":"acab1aa8-b4d6-40e0-a3ee-25341701dca2","shared_citers":25},{"title":"Qwen3 Technical Report","work_id":"25a4e30c-1232-48e7-9925-02fa12ba7c9e","shared_citers":16},{"title":"Evaluating Large Language Models Trained on Code","work_id":"042493e9-b26f-4b4e-bbde-382072ca9b08","shared_citers":14},{"title":"The Llama 3 Herd of Models","work_id":"1549a635-88af-4ac1-acfe-51ae7bb53345","shared_citers":12},{"title":"Medusa: Simple LLM Inference Acceleration Framework with Multiple Decoding Heads","work_id":"f6202cd1-1a78-4c19-8242-688acf4952b6","shared_citers":10},{"title":"DeepSeek-R1: Incentivizing Reasoning Capability in LLMs via Reinforcement Learning","work_id":"e6b75ad5-2877-4168-97c8-710407094d20","shared_citers":9},{"title":"Eagle-3: Scaling up inference acceleration of large language models via training-time test","work_id":"323c1b68-aec5-4444-944f-155a771bd8c6","shared_citers":9},{"title":"Eagle: Speculative sampling requires rethinking feature uncertainty","work_id":"9d4637dd-1cab-4f10-82d4-8c8d14bb96ed","shared_citers":7},{"title":"arXiv:2305.09781 , year=","work_id":"e08525de-cd5d-419d-8e75-7f51a8969588","shared_citers":6},{"title":"Fast inference from transformers via speculative decoding","work_id":"a17596db-fb15-4793-9ce6-fc620a92ae36","shared_citers":6},{"title":"GPT-4 Technical Report","work_id":"b928e041-6991-4c08-8c81-0359e4097c7b","shared_citers":6},{"title":"LLaMA: Open and Efficient Foundation Language Models","work_id":"c018fc23-6f3f-4035-9d02-28a2173b2b9d","shared_citers":6},{"title":"Scaling Laws for Neural Language Models","work_id":"b7dd8749-9c45-4977-ab9b-64478dce1ae8","shared_citers":6},{"title":"Zhuoming Chen, Avner May, Ruslan Svirschevski, Yuhsun Huang, Max Ryabinin, Zhihao Jia, and Beidi Chen","work_id":"860ff726-6feb-4917-a25f-95159b43dd49","shared_citers":6},{"title":"DeepSeek-V3 Technical Report","work_id":"57d2791d-2219-4c31-a077-afc04b12a75c","shared_citers":5},{"title":"Measuring Mathematical Problem Solving With the MATH Dataset","work_id":"50652ac6-fb7c-4675-a2c2-159c241feb17","shared_citers":5},{"title":"DeepSeekMath: Pushing the Limits of Mathematical Reasoning in Open Language Models","work_id":"c5006563-f3ec-438a-9e35-b7b484f34828","shared_citers":4},{"title":"Distilling the Knowledge in a Neural Network","work_id":"d927ab1f-17b8-4002-9d09-c3d55764fbad","shared_citers":4},{"title":"Distillspec: Improving speculative decoding via knowledge distillation","work_id":"5539a830-000f-41f0-9344-c030679cc16f","shared_citers":4},{"title":"FrugalGPT: How to Use Large Language Models While Reducing Cost and Improving Performance","work_id":"5c567414-35de-4373-ae7a-1790da594e8d","shared_citers":4},{"title":"GPQA: A Graduate-Level Google-Proof Q&A Benchmark","work_id":"9e2a976b-f5ad-4aee-af5c-243fe0fe75d2","shared_citers":4},{"title":"gpt-oss-120b & gpt-oss-20b Model Card","work_id":"178c1f7e-4f19-4392-a45d-45a6dfa88ead","shared_citers":4},{"title":"GPTQ: Accurate Post-Training Quantization for Generative Pre-trained Transformers","work_id":"19ed8c44-202a-48f6-8169-637d5a5f2408","shared_citers":4},{"title":"Gulavani, and Ramachandran Ramjee","work_id":"3dbdd757-ca01-436f-acfd-12ffcd6f64c6","shared_citers":4}],"time_series":[{"n":1,"year":2023},{"n":4,"year":2024},{"n":1,"year":2025},{"n":51,"year":2026}],"dependency_candidates":[]},"authors":[]}}