{"record_type":"pith_number_record","schema_url":"https://pith.science/schemas/pith-number/v1.json","pith_number":"pith:2021:I55MIEJIXVQ3RV5UAJHDS4ILBA","short_pith_number":"pith:I55MIEJI","schema_version":"1.0","canonical_sha256":"477ac41128bd61b8d7b4024e39710b08113d2070338fa460ee7f9cc5ba6fa173","source":{"kind":"arxiv","id":"2105.08050","version":2},"attestation_state":"computed","paper":{"title":"Pay Attention to MLPs","license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","headline":"","cross_cats":["cs.CL","cs.CV"],"primary_cat":"cs.LG","authors_text":"David R. So, Hanxiao Liu, Quoc V. Le, Zihang Dai","submitted_at":"2021-05-17T17:55:04Z","abstract_excerpt":"Transformers have become one of the most important architectural innovations in deep learning and have enabled many breakthroughs over the past few years. Here we propose a simple network architecture, gMLP, based on MLPs with gating, and show that it can perform as well as Transformers in key language and vision applications. Our comparisons show that self-attention is not critical for Vision Transformers, as gMLP can achieve the same accuracy. For BERT, our model achieves parity with Transformers on pretraining perplexity and is better on some downstream NLP tasks. On finetuning tasks where "},"verification_status":{"content_addressed":true,"pith_receipt":true,"author_attested":false,"weak_author_claims":0,"strong_author_claims":0,"externally_anchored":false,"storage_verified":false,"citation_signatures":0,"replication_records":0,"graph_snapshot":true,"references_resolved":false,"formal_links_present":false},"canonical_record":{"source":{"id":"2105.08050","kind":"arxiv","version":2},"metadata":{"license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","primary_cat":"cs.LG","submitted_at":"2021-05-17T17:55:04Z","cross_cats_sorted":["cs.CL","cs.CV"],"title_canon_sha256":"a5486a413767c6326facb43f3865ea3d5e7bdecede38d8b9b34f4a85cd1bdb96","abstract_canon_sha256":"642577589ccb8250a5e7492749b99a312fee29d1c914bcb26d1f6865b8dc72a6"},"schema_version":"1.0"},"receipt":{"kind":"pith_receipt","key_id":"pith-v1-2026-05","algorithm":"ed25519","signed_at":"2026-07-05T02:45:38.157129Z","signature_b64":"bQkwIWDW3t+r6iSoYWTKIzzh4hw9uNk36m+oTFBht06g8EJ3NVlzHOBHW/YGs69bEoA6bFfbETGcjPYO5RiZDw==","signed_message":"canonical_sha256_bytes","builder_version":"pith-number-builder-2026-05-17-v1","receipt_version":"0.3","canonical_sha256":"477ac41128bd61b8d7b4024e39710b08113d2070338fa460ee7f9cc5ba6fa173","last_reissued_at":"2026-07-05T02:45:38.156662Z","signature_status":"signed_v1","first_computed_at":"2026-07-05T02:45:38.156662Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"graph_snapshot":{"paper":{"title":"Pay Attention to MLPs","license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","headline":"","cross_cats":["cs.CL","cs.CV"],"primary_cat":"cs.LG","authors_text":"David R. So, Hanxiao Liu, Quoc V. Le, Zihang Dai","submitted_at":"2021-05-17T17:55:04Z","abstract_excerpt":"Transformers have become one of the most important architectural innovations in deep learning and have enabled many breakthroughs over the past few years. Here we propose a simple network architecture, gMLP, based on MLPs with gating, and show that it can perform as well as Transformers in key language and vision applications. Our comparisons show that self-attention is not critical for Vision Transformers, as gMLP can achieve the same accuracy. For BERT, our model achieves parity with Transformers on pretraining perplexity and is better on some downstream NLP tasks. On finetuning tasks where "},"claims":{"count":0,"items":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"2105.08050","kind":"arxiv","version":2},"verdict":{"id":null,"model_set":{},"created_at":null,"strongest_claim":"","one_line_summary":"","pipeline_version":null,"weakest_assumption":"","pith_extraction_headline":""},"integrity":{"clean":true,"summary":{"advisory":0,"critical":0,"by_detector":{},"informational":0},"endpoint":"/pith/2105.08050/integrity.json","findings":[],"available":true,"detectors_run":[],"snapshot_sha256":"c28c3603d3b5d939e8dc4c7e95fa8dfce3d595e45f758748cecf8e644a296938"},"references":{"count":0,"sample":[],"resolved_work":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","internal_anchors":0},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"author_claims":{"count":0,"strong_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"builder_version":"pith-number-builder-2026-05-17-v1"},"aliases":[{"alias_kind":"arxiv","alias_value":"2105.08050","created_at":"2026-07-05T02:45:38.156729+00:00"},{"alias_kind":"arxiv_version","alias_value":"2105.08050v2","created_at":"2026-07-05T02:45:38.156729+00:00"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.2105.08050","created_at":"2026-07-05T02:45:38.156729+00:00"},{"alias_kind":"pith_short_12","alias_value":"I55MIEJIXVQ3","created_at":"2026-07-05T02:45:38.156729+00:00"},{"alias_kind":"pith_short_16","alias_value":"I55MIEJIXVQ3RV5U","created_at":"2026-07-05T02:45:38.156729+00:00"},{"alias_kind":"pith_short_8","alias_value":"I55MIEJI","created_at":"2026-07-05T02:45:38.156729+00:00"}],"events":[],"event_summary":{},"paper_claims":[],"inbound_citations":{"count":4,"internal_anchor_count":1,"sample":[{"citing_arxiv_id":"2607.07386","citing_title":"Sparse Delta Memory: Scaling the State of Linear RNNs through Sparsity","ref_index":100,"is_internal_anchor":true},{"citing_arxiv_id":"2606.25986","citing_title":"The Inference-Compute Frontier and a Latency-Efficient Architecture for Limit Order Book Prediction","ref_index":17,"is_internal_anchor":false},{"citing_arxiv_id":"2509.24552","citing_title":"Short window attention enables long-term memorization","ref_index":24,"is_internal_anchor":false},{"citing_arxiv_id":"2605.07232","citing_title":"Towards multi-modal forgery representation learning for AI-generated video detection and localization","ref_index":18,"is_internal_anchor":false}]},"formal_canon":{"evidence_count":0,"sample":[],"anchors":[]},"links":{"html":"https://pith.science/pith/I55MIEJIXVQ3RV5UAJHDS4ILBA","json":"https://pith.science/pith/I55MIEJIXVQ3RV5UAJHDS4ILBA.json","graph_json":"https://pith.science/api/pith-number/I55MIEJIXVQ3RV5UAJHDS4ILBA/graph.json","events_json":"https://pith.science/api/pith-number/I55MIEJIXVQ3RV5UAJHDS4ILBA/events.json","paper":"https://pith.science/paper/I55MIEJI"},"agent_actions":{"view_html":"https://pith.science/pith/I55MIEJIXVQ3RV5UAJHDS4ILBA","download_json":"https://pith.science/pith/I55MIEJIXVQ3RV5UAJHDS4ILBA.json","view_paper":"https://pith.science/paper/I55MIEJI","resolve_alias":"https://pith.science/api/pith-number/resolve?arxiv=2105.08050&json=true","fetch_graph":"https://pith.science/api/pith-number/I55MIEJIXVQ3RV5UAJHDS4ILBA/graph.json","fetch_events":"https://pith.science/api/pith-number/I55MIEJIXVQ3RV5UAJHDS4ILBA/events.json","actions":{"anchor_timestamp":"https://pith.science/pith/I55MIEJIXVQ3RV5UAJHDS4ILBA/action/timestamp_anchor","attest_storage":"https://pith.science/pith/I55MIEJIXVQ3RV5UAJHDS4ILBA/action/storage_attestation","attest_author":"https://pith.science/pith/I55MIEJIXVQ3RV5UAJHDS4ILBA/action/author_attestation","sign_citation":"https://pith.science/pith/I55MIEJIXVQ3RV5UAJHDS4ILBA/action/citation_signature","submit_replication":"https://pith.science/pith/I55MIEJIXVQ3RV5UAJHDS4ILBA/action/replication_record"}},"created_at":"2026-07-05T02:45:38.156729+00:00","updated_at":"2026-07-05T02:45:38.156729+00:00"}