{"state_type":"pith_open_graph_state","state_version":"1.0","pith_number":"pith:2023:JMZQR3UAQKTF2BQ4P2DXSLCFWL","merge_version":"pith-open-graph-merge-v1","event_count":2,"valid_event_count":2,"invalid_event_count":0,"equivocation_count":0,"current":{"canonical_record":{"metadata":{"abstract_canon_sha256":"75c263de82c96dc6e881253731155c4509a7e18fdc38b45f554b106adc7e5f40","cross_cats_sorted":["cs.AI","cs.LG"],"license":"http://creativecommons.org/licenses/by-nc-sa/4.0/","primary_cat":"cs.CL","submitted_at":"2023-02-13T15:41:20Z","title_canon_sha256":"261657ef833ee136ae75961c03d198554c6655a577d82d1e62aa5027ed84434f"},"schema_version":"1.0","source":{"id":"2302.06461","kind":"arxiv","version":1}},"source_aliases":[{"alias_kind":"arxiv","alias_value":"2302.06461","created_at":"2026-07-05T05:41:05Z"},{"alias_kind":"arxiv_version","alias_value":"2302.06461v1","created_at":"2026-07-05T05:41:05Z"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.2302.06461","created_at":"2026-07-05T05:41:05Z"},{"alias_kind":"pith_short_12","alias_value":"JMZQR3UAQKTF","created_at":"2026-07-05T05:41:05Z"},{"alias_kind":"pith_short_16","alias_value":"JMZQR3UAQKTF2BQ4","created_at":"2026-07-05T05:41:05Z"},{"alias_kind":"pith_short_8","alias_value":"JMZQR3UA","created_at":"2026-07-05T05:41:05Z"}],"graph_snapshots":[{"event_id":"sha256:4e562b80578a8ef717c5a06320854276fbd54f7dd804de0b371f64aa6aa9ddde","target":"graph","created_at":"2026-07-05T05:41:05Z","signer":{"key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signer_id":"pith.science","signer_type":"pith_registry"},"payload":{"graph_snapshot":{"author_claims":{"count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","strong_count":0},"builder_version":"pith-number-builder-2026-05-17-v1","claims":{"count":0,"items":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"integrity":{"available":true,"clean":true,"detectors_run":[],"endpoint":"/pith/2302.06461/integrity.json","findings":[],"snapshot_sha256":"c28c3603d3b5d939e8dc4c7e95fa8dfce3d595e45f758748cecf8e644a296938","summary":{"advisory":0,"by_detector":{},"critical":0,"informational":0}},"paper":{"abstract_excerpt":"The Transformer architecture consists of self-attention and feed-forward networks (FFNs) which can be viewed as key-value memories according to previous works. However, FFN and traditional memory utilize different activation functions (i.e., ReLU and Softmax respectively), which makes them not equivalent. In this paper, we first rebuild the connections between FFN and key-value memory by conducting extensive studies on ReLU and Softmax, and find they are equivalent when adding an additional layer normalization module on Softmax. In addition, ReLU outperforms Softmax on both FFN and key-value m","authors_text":"Jiang Bian, Junliang Guo, Kai Shen, Rui Wang, Siliang Tang, Xu Tan","cross_cats":["cs.AI","cs.LG"],"headline":"","license":"http://creativecommons.org/licenses/by-nc-sa/4.0/","primary_cat":"cs.CL","submitted_at":"2023-02-13T15:41:20Z","title":"A Study on ReLU and Softmax in Transformer"},"references":{"count":0,"internal_anchors":0,"resolved_work":0,"sample":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"2302.06461","kind":"arxiv","version":1},"verdict":{"created_at":null,"id":null,"model_set":{},"one_line_summary":"","pipeline_version":null,"pith_extraction_headline":"","strongest_claim":"","weakest_assumption":""}},"verdict_id":null}}],"author_attestations":[],"timestamp_anchors":[],"storage_attestations":[],"citation_signatures":[],"replication_records":[],"corrections":[],"mirror_hints":[],"record_created":{"event_id":"sha256:c23f781e2a76300e5fff05713ebea86a8fdc7cfe2cfcf86cfb57907f636ec1e4","target":"record","created_at":"2026-07-05T05:41:05Z","signer":{"key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signer_id":"pith.science","signer_type":"pith_registry"},"payload":{"attestation_state":"computed","canonical_record":{"metadata":{"abstract_canon_sha256":"75c263de82c96dc6e881253731155c4509a7e18fdc38b45f554b106adc7e5f40","cross_cats_sorted":["cs.AI","cs.LG"],"license":"http://creativecommons.org/licenses/by-nc-sa/4.0/","primary_cat":"cs.CL","submitted_at":"2023-02-13T15:41:20Z","title_canon_sha256":"261657ef833ee136ae75961c03d198554c6655a577d82d1e62aa5027ed84434f"},"schema_version":"1.0","source":{"id":"2302.06461","kind":"arxiv","version":1}},"canonical_sha256":"4b3308ee8082a65d061c7e87792c45b2c9d3f973c1e95769aec53e4a1c9936db","receipt":{"algorithm":"ed25519","builder_version":"pith-number-builder-2026-05-17-v1","canonical_sha256":"4b3308ee8082a65d061c7e87792c45b2c9d3f973c1e95769aec53e4a1c9936db","first_computed_at":"2026-07-05T05:41:05.502883Z","key_id":"pith-v1-2026-05","kind":"pith_receipt","last_reissued_at":"2026-07-05T05:41:05.502883Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","receipt_version":"0.3","signature_b64":"9XoQ0DJBbhWweg1EbweNDDijgOd5PsqYl1SzOZOPfFbucYObejNPMnv5AdU3yA0vRgnnaGS4fBXeQwGFgQ5VCQ==","signature_status":"signed_v1","signed_at":"2026-07-05T05:41:05.503321Z","signed_message":"canonical_sha256_bytes"},"source_id":"2302.06461","source_kind":"arxiv","source_version":1}}},"equivocations":[],"invalid_events":[],"applied_event_ids":["sha256:c23f781e2a76300e5fff05713ebea86a8fdc7cfe2cfcf86cfb57907f636ec1e4","sha256:4e562b80578a8ef717c5a06320854276fbd54f7dd804de0b371f64aa6aa9ddde"],"state_sha256":"8ed0531a74e92fb15cae71816863ba3faf7f2c0c32b17ee50fb820a32de2d449"}