{"record_type":"pith_number_record","schema_url":"https://pith.science/schemas/pith-number/v1.json","pith_number":"pith:2020:RN7U4VOPGSUBRHCM3XDWF2GCR5","short_pith_number":"pith:RN7U4VOP","schema_version":"1.0","canonical_sha256":"8b7f4e55cf34a8189c4cddc762e8c28f7a587360cad51cbc4a40adc9cb44c11b","source":{"kind":"arxiv","id":"2010.11859","version":2},"attestation_state":"computed","paper":{"title":"Not all parameters are born equal: Attention is mostly what you need","license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","headline":"","cross_cats":[],"primary_cat":"cs.CL","authors_text":"Nikolay Bogoychev","submitted_at":"2020-10-22T16:49:18Z","abstract_excerpt":"Transformers are widely used in state-of-the-art machine translation, but the key to their success is still unknown. To gain insight into this, we consider three groups of parameters: embeddings, attention, and feed forward neural network (FFN) layers. We examine the relative importance of each by performing an ablation study where we initialise them at random and freeze them, so that their weights do not change over the course of the training. Through this, we show that the attention and FFN are equally important and fulfil the same functionality in a model. We show that the decision about wh"},"verification_status":{"content_addressed":true,"pith_receipt":true,"author_attested":false,"weak_author_claims":0,"strong_author_claims":0,"externally_anchored":false,"storage_verified":false,"citation_signatures":0,"replication_records":0,"graph_snapshot":true,"references_resolved":false,"formal_links_present":false},"canonical_record":{"source":{"id":"2010.11859","kind":"arxiv","version":2},"metadata":{"license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","primary_cat":"cs.CL","submitted_at":"2020-10-22T16:49:18Z","cross_cats_sorted":[],"title_canon_sha256":"4eb0941cb7e838902545979a08cd872c313f6a8bb0a0726fb6a3c17874539ce8","abstract_canon_sha256":"4441eb757f314b39f0005d78460c91457122bb51de077bfb8cd6d40c7fa0293b"},"schema_version":"1.0"},"receipt":{"kind":"pith_receipt","key_id":"pith-v1-2026-05","algorithm":"ed25519","signed_at":"2026-07-05T03:16:06.174106Z","signature_b64":"Epik/2MpDa6yhskbN7DB5Xh2zyTgslcAVE5DQpmcRJio0SjTtkxWT9bQ4j9bPGDpg7I9m/fYoPqy6fELAd69DQ==","signed_message":"canonical_sha256_bytes","builder_version":"pith-number-builder-2026-05-17-v1","receipt_version":"0.3","canonical_sha256":"8b7f4e55cf34a8189c4cddc762e8c28f7a587360cad51cbc4a40adc9cb44c11b","last_reissued_at":"2026-07-05T03:16:06.173590Z","signature_status":"signed_v1","first_computed_at":"2026-07-05T03:16:06.173590Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"graph_snapshot":{"paper":{"title":"Not all parameters are born equal: Attention is mostly what you need","license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","headline":"","cross_cats":[],"primary_cat":"cs.CL","authors_text":"Nikolay Bogoychev","submitted_at":"2020-10-22T16:49:18Z","abstract_excerpt":"Transformers are widely used in state-of-the-art machine translation, but the key to their success is still unknown. To gain insight into this, we consider three groups of parameters: embeddings, attention, and feed forward neural network (FFN) layers. We examine the relative importance of each by performing an ablation study where we initialise them at random and freeze them, so that their weights do not change over the course of the training. Through this, we show that the attention and FFN are equally important and fulfil the same functionality in a model. We show that the decision about wh"},"claims":{"count":0,"items":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"2010.11859","kind":"arxiv","version":2},"verdict":{"id":null,"model_set":{},"created_at":null,"strongest_claim":"","one_line_summary":"","pipeline_version":null,"weakest_assumption":"","pith_extraction_headline":""},"integrity":{"clean":true,"summary":{"advisory":0,"critical":0,"by_detector":{},"informational":0},"endpoint":"/pith/2010.11859/integrity.json","findings":[],"available":true,"detectors_run":[],"snapshot_sha256":"c28c3603d3b5d939e8dc4c7e95fa8dfce3d595e45f758748cecf8e644a296938"},"references":{"count":0,"sample":[],"resolved_work":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","internal_anchors":0},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"author_claims":{"count":0,"strong_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"builder_version":"pith-number-builder-2026-05-17-v1"},"aliases":[{"alias_kind":"arxiv","alias_value":"2010.11859","created_at":"2026-07-05T03:16:06.173646+00:00"},{"alias_kind":"arxiv_version","alias_value":"2010.11859v2","created_at":"2026-07-05T03:16:06.173646+00:00"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.2010.11859","created_at":"2026-07-05T03:16:06.173646+00:00"},{"alias_kind":"pith_short_12","alias_value":"RN7U4VOPGSUB","created_at":"2026-07-05T03:16:06.173646+00:00"},{"alias_kind":"pith_short_16","alias_value":"RN7U4VOPGSUBRHCM","created_at":"2026-07-05T03:16:06.173646+00:00"},{"alias_kind":"pith_short_8","alias_value":"RN7U4VOP","created_at":"2026-07-05T03:16:06.173646+00:00"}],"events":[],"event_summary":{},"paper_claims":[],"inbound_citations":{"count":1,"internal_anchor_count":1,"sample":[{"citing_arxiv_id":"2501.00446","citing_title":"DEHYDRATOR: Enhancing Provenance Graph Storage via Hierarchical Encoding and Sequence Generation","ref_index":66,"is_internal_anchor":true}]},"formal_canon":{"evidence_count":0,"sample":[],"anchors":[]},"links":{"html":"https://pith.science/pith/RN7U4VOPGSUBRHCM3XDWF2GCR5","json":"https://pith.science/pith/RN7U4VOPGSUBRHCM3XDWF2GCR5.json","graph_json":"https://pith.science/api/pith-number/RN7U4VOPGSUBRHCM3XDWF2GCR5/graph.json","events_json":"https://pith.science/api/pith-number/RN7U4VOPGSUBRHCM3XDWF2GCR5/events.json","paper":"https://pith.science/paper/RN7U4VOP"},"agent_actions":{"view_html":"https://pith.science/pith/RN7U4VOPGSUBRHCM3XDWF2GCR5","download_json":"https://pith.science/pith/RN7U4VOPGSUBRHCM3XDWF2GCR5.json","view_paper":"https://pith.science/paper/RN7U4VOP","resolve_alias":"https://pith.science/api/pith-number/resolve?arxiv=2010.11859&json=true","fetch_graph":"https://pith.science/api/pith-number/RN7U4VOPGSUBRHCM3XDWF2GCR5/graph.json","fetch_events":"https://pith.science/api/pith-number/RN7U4VOPGSUBRHCM3XDWF2GCR5/events.json","actions":{"anchor_timestamp":"https://pith.science/pith/RN7U4VOPGSUBRHCM3XDWF2GCR5/action/timestamp_anchor","attest_storage":"https://pith.science/pith/RN7U4VOPGSUBRHCM3XDWF2GCR5/action/storage_attestation","attest_author":"https://pith.science/pith/RN7U4VOPGSUBRHCM3XDWF2GCR5/action/author_attestation","sign_citation":"https://pith.science/pith/RN7U4VOPGSUBRHCM3XDWF2GCR5/action/citation_signature","submit_replication":"https://pith.science/pith/RN7U4VOPGSUBRHCM3XDWF2GCR5/action/replication_record"}},"created_at":"2026-07-05T03:16:06.173646+00:00","updated_at":"2026-07-05T03:16:06.173646+00:00"}