{"work":{"id":"eb3dbae4-931f-40ab-a37b-507a35f42712","openalex_id":null,"doi":null,"arxiv_id":"2001.04451","raw_key":null,"title":"Reformer: The Efficient Transformer","authors":null,"authors_text":"Nikita Kitaev, {\\L}ukasz Kaiser, Anselm Levskaya","year":2020,"venue":"cs.LG","abstract":"Large Transformer models routinely achieve state-of-the-art results on a number of tasks but training these models can be prohibitively costly, especially on long sequences. We introduce two techniques to improve the efficiency of Transformers. For one, we replace dot-product attention by one that uses locality-sensitive hashing, changing its complexity from O($L^2$) to O($L\\log L$), where $L$ is the length of the sequence. Furthermore, we use reversible residual layers instead of the standard residuals, which allows storing activations only once in the training process instead of $N$ times, where $N$ is the number of layers. The resulting model, the Reformer, performs on par with Transformer models while being much more memory-efficient and much faster on long sequences.","external_url":"https://arxiv.org/abs/2001.04451","cited_by_count":null,"metadata_source":"pith","metadata_fetched_at":"2026-06-29T20:33:58.271320+00:00","pith_arxiv_id":"2001.04451","created_at":"2026-05-09T06:15:37.712922+00:00","updated_at":"2026-06-29T20:33:58.271320+00:00","title_quality_ok":false,"display_title":"Reformer: The Efficient Transformer","render_title":"Reformer: The Efficient Transformer"},"hub":{"state":{"work_id":"eb3dbae4-931f-40ab-a37b-507a35f42712","tier":"hub","tier_reason":"10+ Pith inbound or 1,000+ external citations","pith_inbound_count":49,"external_cited_by_count":null,"distinct_field_count":12,"first_pith_cited_at":"2019-10-09T03:23:22+00:00","last_pith_cited_at":"2026-05-27T15:19:41+00:00","author_build_status":"not_needed","summary_status":"needed","contexts_status":"needed","graph_status":"needed","ask_index_status":"not_needed","reader_status":"not_needed","recognition_status":"not_needed","updated_at":"2026-06-30T05:29:30.174529+00:00","tier_text":"hub"},"tier":"hub","role_counts":[{"context_role":"background","n":9},{"context_role":"baseline","n":1},{"context_role":"dataset","n":1},{"context_role":"method","n":1}],"polarity_counts":[{"context_polarity":"background","n":9},{"context_polarity":"baseline","n":1},{"context_polarity":"unclear","n":1},{"context_polarity":"use_dataset","n":1}],"runs":{},"summary":{},"graph":{},"authors":[]}}