{"record_type":"pith_number_record","schema_url":"https://pith.science/schemas/pith-number/v1.json","pith_number":"pith:2020:NAEMQC4BSIWDVE5PXS3ITLT4EZ","short_pith_number":"pith:NAEMQC4B","schema_version":"1.0","canonical_sha256":"6808c80b81922c3a93afbcb689ae7c267ca85805dd754ba5c853ffa9b5cacd04","source":{"kind":"arxiv","id":"2006.09471","version":2},"attestation_state":"computed","paper":{"title":"Untangling tradeoffs between recurrence and self-attention in neural networks","license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","headline":"","cross_cats":["stat.ML"],"primary_cat":"cs.LG","authors_text":"Anirudh Goyal, Bhargav Kanuparthi, Giancarlo Kerg, Guillaume Lajoie, Kyle Goyette, Yoshua Bengio","submitted_at":"2020-06-16T19:24:25Z","abstract_excerpt":"Attention and self-attention mechanisms, are now central to state-of-the-art deep learning on sequential tasks. However, most recent progress hinges on heuristic approaches with limited understanding of attention's role in model optimization and computation, and rely on considerable memory and computational resources that scale poorly. In this work, we present a formal analysis of how self-attention affects gradient propagation in recurrent networks, and prove that it mitigates the problem of vanishing gradients when trying to capture long-term dependencies by establishing concrete bounds for "},"verification_status":{"content_addressed":true,"pith_receipt":true,"author_attested":false,"weak_author_claims":0,"strong_author_claims":0,"externally_anchored":false,"storage_verified":false,"citation_signatures":0,"replication_records":0,"graph_snapshot":true,"references_resolved":false,"formal_links_present":false},"canonical_record":{"source":{"id":"2006.09471","kind":"arxiv","version":2},"metadata":{"license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","primary_cat":"cs.LG","submitted_at":"2020-06-16T19:24:25Z","cross_cats_sorted":["stat.ML"],"title_canon_sha256":"82bdc6a54e7c05697bd9618910266b328e20b6478e9543a151f41ad36765b160","abstract_canon_sha256":"6bf5117f5a0e771807862b4dd56c3dd2535baf2d317082d34519dbbe086e3ef3"},"schema_version":"1.0"},"receipt":{"kind":"pith_receipt","key_id":"pith-v1-2026-05","algorithm":"ed25519","signed_at":"2026-07-05T01:58:30.669036Z","signature_b64":"syrlD/B9hOmWsaOQdk92lBAbEANgP02wB/KJnAPjpEMzJG8uesdZZ4+NKRhgWDxHVPHUAVIw5acdyRa1P11rBA==","signed_message":"canonical_sha256_bytes","builder_version":"pith-number-builder-2026-05-17-v1","receipt_version":"0.3","canonical_sha256":"6808c80b81922c3a93afbcb689ae7c267ca85805dd754ba5c853ffa9b5cacd04","last_reissued_at":"2026-07-05T01:58:30.668473Z","signature_status":"signed_v1","first_computed_at":"2026-07-05T01:58:30.668473Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"graph_snapshot":{"paper":{"title":"Untangling tradeoffs between recurrence and self-attention in neural networks","license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","headline":"","cross_cats":["stat.ML"],"primary_cat":"cs.LG","authors_text":"Anirudh Goyal, Bhargav Kanuparthi, Giancarlo Kerg, Guillaume Lajoie, Kyle Goyette, Yoshua Bengio","submitted_at":"2020-06-16T19:24:25Z","abstract_excerpt":"Attention and self-attention mechanisms, are now central to state-of-the-art deep learning on sequential tasks. However, most recent progress hinges on heuristic approaches with limited understanding of attention's role in model optimization and computation, and rely on considerable memory and computational resources that scale poorly. In this work, we present a formal analysis of how self-attention affects gradient propagation in recurrent networks, and prove that it mitigates the problem of vanishing gradients when trying to capture long-term dependencies by establishing concrete bounds for "},"claims":{"count":0,"items":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"2006.09471","kind":"arxiv","version":2},"verdict":{"id":null,"model_set":{},"created_at":null,"strongest_claim":"","one_line_summary":"","pipeline_version":null,"weakest_assumption":"","pith_extraction_headline":""},"integrity":{"clean":true,"summary":{"advisory":0,"critical":0,"by_detector":{},"informational":0},"endpoint":"/pith/2006.09471/integrity.json","findings":[],"available":true,"detectors_run":[],"snapshot_sha256":"c28c3603d3b5d939e8dc4c7e95fa8dfce3d595e45f758748cecf8e644a296938"},"references":{"count":0,"sample":[],"resolved_work":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","internal_anchors":0},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"author_claims":{"count":0,"strong_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"builder_version":"pith-number-builder-2026-05-17-v1"},"aliases":[{"alias_kind":"arxiv","alias_value":"2006.09471","created_at":"2026-07-05T01:58:30.668550+00:00"},{"alias_kind":"arxiv_version","alias_value":"2006.09471v2","created_at":"2026-07-05T01:58:30.668550+00:00"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.2006.09471","created_at":"2026-07-05T01:58:30.668550+00:00"},{"alias_kind":"pith_short_12","alias_value":"NAEMQC4BSIWD","created_at":"2026-07-05T01:58:30.668550+00:00"},{"alias_kind":"pith_short_16","alias_value":"NAEMQC4BSIWDVE5P","created_at":"2026-07-05T01:58:30.668550+00:00"},{"alias_kind":"pith_short_8","alias_value":"NAEMQC4B","created_at":"2026-07-05T01:58:30.668550+00:00"}],"events":[],"event_summary":{},"paper_claims":[],"inbound_citations":{"count":0,"internal_anchor_count":0,"sample":[]},"formal_canon":{"evidence_count":0,"sample":[],"anchors":[]},"links":{"html":"https://pith.science/pith/NAEMQC4BSIWDVE5PXS3ITLT4EZ","json":"https://pith.science/pith/NAEMQC4BSIWDVE5PXS3ITLT4EZ.json","graph_json":"https://pith.science/api/pith-number/NAEMQC4BSIWDVE5PXS3ITLT4EZ/graph.json","events_json":"https://pith.science/api/pith-number/NAEMQC4BSIWDVE5PXS3ITLT4EZ/events.json","paper":"https://pith.science/paper/NAEMQC4B"},"agent_actions":{"view_html":"https://pith.science/pith/NAEMQC4BSIWDVE5PXS3ITLT4EZ","download_json":"https://pith.science/pith/NAEMQC4BSIWDVE5PXS3ITLT4EZ.json","view_paper":"https://pith.science/paper/NAEMQC4B","resolve_alias":"https://pith.science/api/pith-number/resolve?arxiv=2006.09471&json=true","fetch_graph":"https://pith.science/api/pith-number/NAEMQC4BSIWDVE5PXS3ITLT4EZ/graph.json","fetch_events":"https://pith.science/api/pith-number/NAEMQC4BSIWDVE5PXS3ITLT4EZ/events.json","actions":{"anchor_timestamp":"https://pith.science/pith/NAEMQC4BSIWDVE5PXS3ITLT4EZ/action/timestamp_anchor","attest_storage":"https://pith.science/pith/NAEMQC4BSIWDVE5PXS3ITLT4EZ/action/storage_attestation","attest_author":"https://pith.science/pith/NAEMQC4BSIWDVE5PXS3ITLT4EZ/action/author_attestation","sign_citation":"https://pith.science/pith/NAEMQC4BSIWDVE5PXS3ITLT4EZ/action/citation_signature","submit_replication":"https://pith.science/pith/NAEMQC4BSIWDVE5PXS3ITLT4EZ/action/replication_record"}},"created_at":"2026-07-05T01:58:30.668550+00:00","updated_at":"2026-07-05T01:58:30.668550+00:00"}