{"bundle_type":"pith_open_graph_bundle","bundle_version":"1.0","pith_number":"pith:2020:ZCLFEMMQWSCHSZTIW2RVTGQHKE","short_pith_number":"pith:ZCLFEMMQ","canonical_record":{"source":{"id":"2004.08483","kind":"arxiv","version":5},"metadata":{"license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","primary_cat":"cs.LG","submitted_at":"2020-04-17T23:10:18Z","cross_cats_sorted":["stat.ML"],"title_canon_sha256":"1357a89f61ce07a9fbd89f83f76b1b56053e09538c82ae694ee3a1395e53c398","abstract_canon_sha256":"3f7d47be4fdb14d683efb14c78e11252101cef847e4f4f11cdebe257a707a0d6"},"schema_version":"1.0"},"canonical_sha256":"c896523190b484796668b6a3599a07512523f96a4de1e0382daca052fb79f55a","source":{"kind":"arxiv","id":"2004.08483","version":5},"source_aliases":[{"alias_kind":"arxiv","alias_value":"2004.08483","created_at":"2026-07-05T01:46:18Z"},{"alias_kind":"arxiv_version","alias_value":"2004.08483v5","created_at":"2026-07-05T01:46:18Z"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.2004.08483","created_at":"2026-07-05T01:46:18Z"},{"alias_kind":"pith_short_12","alias_value":"ZCLFEMMQWSCH","created_at":"2026-07-05T01:46:18Z"},{"alias_kind":"pith_short_16","alias_value":"ZCLFEMMQWSCHSZTI","created_at":"2026-07-05T01:46:18Z"},{"alias_kind":"pith_short_8","alias_value":"ZCLFEMMQ","created_at":"2026-07-05T01:46:18Z"}],"events":[{"event_type":"record_created","subject_pith_number":"pith:2020:ZCLFEMMQWSCHSZTIW2RVTGQHKE","target":"record","payload":{"canonical_record":{"source":{"id":"2004.08483","kind":"arxiv","version":5},"metadata":{"license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","primary_cat":"cs.LG","submitted_at":"2020-04-17T23:10:18Z","cross_cats_sorted":["stat.ML"],"title_canon_sha256":"1357a89f61ce07a9fbd89f83f76b1b56053e09538c82ae694ee3a1395e53c398","abstract_canon_sha256":"3f7d47be4fdb14d683efb14c78e11252101cef847e4f4f11cdebe257a707a0d6"},"schema_version":"1.0"},"canonical_sha256":"c896523190b484796668b6a3599a07512523f96a4de1e0382daca052fb79f55a","receipt":{"kind":"pith_receipt","key_id":"pith-v1-2026-05","algorithm":"ed25519","signed_at":"2026-07-05T01:46:18.231422Z","signature_b64":"BOyYmW0cwEW4o1T1W6CCc5VvrI0F0Dv92qi6cAS8hUN4Y6gBL8HGUhIqs3Qeknr/1DcqXBEM7sVUZoE6OJGICQ==","signed_message":"canonical_sha256_bytes","builder_version":"pith-number-builder-2026-05-17-v1","receipt_version":"0.3","canonical_sha256":"c896523190b484796668b6a3599a07512523f96a4de1e0382daca052fb79f55a","last_reissued_at":"2026-07-05T01:46:18.230991Z","signature_status":"signed_v1","first_computed_at":"2026-07-05T01:46:18.230991Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"source_kind":"arxiv","source_id":"2004.08483","source_version":5,"attestation_state":"computed"},"signer":{"signer_id":"pith.science","signer_type":"pith_registry","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"created_at":"2026-07-05T01:46:18Z","supersedes":[],"prev_event":null,"signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"Csh99kDUexSxcjqTjbKeoKNFviSHTApJ0pwLoGToCtCuBwxPE5Ey+qG5h2IiVM84XUDHB6YGqIFREPcVzrHQAw==","signed_message":"open_graph_event_sha256_bytes","signed_at":"2026-07-06T18:38:05.630567Z"},"content_sha256":"4de4426041dabffb84bda4bfcf8f4384c8b38b3e74bbced6e30e6499585bd74c","schema_version":"1.0","event_id":"sha256:4de4426041dabffb84bda4bfcf8f4384c8b38b3e74bbced6e30e6499585bd74c"},{"event_type":"graph_snapshot","subject_pith_number":"pith:2020:ZCLFEMMQWSCHSZTIW2RVTGQHKE","target":"graph","payload":{"graph_snapshot":{"paper":{"title":"ETC: Encoding Long and Structured Inputs in Transformers","license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","headline":"","cross_cats":["stat.ML"],"primary_cat":"cs.LG","authors_text":"Anirudh Ravula, Chris Alberti, Joshua Ainslie, Li Yang, Philip Pham, Qifan Wang, Santiago Ontanon, Sumit Sanghai, Vaclav Cvicek, Zachary Fisher","submitted_at":"2020-04-17T23:10:18Z","abstract_excerpt":"Transformer models have advanced the state of the art in many Natural Language Processing (NLP) tasks. In this paper, we present a new Transformer architecture, Extended Transformer Construction (ETC), that addresses two key challenges of standard Transformer architectures, namely scaling input length and encoding structured inputs. To scale attention to longer inputs, we introduce a novel global-local attention mechanism between global tokens and regular input tokens. We also show that combining global-local attention with relative position encodings and a Contrastive Predictive Coding (CPC) "},"claims":{"count":0,"items":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"2004.08483","kind":"arxiv","version":5},"verdict":{"id":null,"model_set":{},"created_at":null,"strongest_claim":"","one_line_summary":"","pipeline_version":null,"weakest_assumption":"","pith_extraction_headline":""},"integrity":{"clean":true,"summary":{"advisory":0,"critical":0,"by_detector":{},"informational":0},"endpoint":"/pith/2004.08483/integrity.json","findings":[],"available":true,"detectors_run":[],"snapshot_sha256":"c28c3603d3b5d939e8dc4c7e95fa8dfce3d595e45f758748cecf8e644a296938"},"references":{"count":0,"sample":[],"resolved_work":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","internal_anchors":0},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"author_claims":{"count":0,"strong_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"builder_version":"pith-number-builder-2026-05-17-v1"},"verdict_id":null},"signer":{"signer_id":"pith.science","signer_type":"pith_registry","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"created_at":"2026-07-05T01:46:18Z","supersedes":[],"prev_event":null,"signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"p0yZ1ECndpdrokQozVhQeIPC7kHDTutgjOy7Q5cCm2wiRoZ0xvhFCu65cQZN36rvtbV11zpa9aUr6Olr0/o/AQ==","signed_message":"open_graph_event_sha256_bytes","signed_at":"2026-07-06T18:38:05.630948Z"},"content_sha256":"c4601f62cb45b723221b9b36b46706b1b4cdaf1f3d03b26795560db5f3033f13","schema_version":"1.0","event_id":"sha256:c4601f62cb45b723221b9b36b46706b1b4cdaf1f3d03b26795560db5f3033f13"}],"timestamp_proofs":[],"mirror_hints":[{"mirror_type":"https","name":"Pith Resolver","base_url":"https://pith.science","bundle_url":"https://pith.science/pith/ZCLFEMMQWSCHSZTIW2RVTGQHKE/bundle.json","state_url":"https://pith.science/pith/ZCLFEMMQWSCHSZTIW2RVTGQHKE/state.json","well_known_bundle_url":"https://pith.science/.well-known/pith/ZCLFEMMQWSCHSZTIW2RVTGQHKE/bundle.json","status":"primary"}],"public_keys":[{"key_id":"pith-v1-2026-05","algorithm":"ed25519","format":"raw","public_key_b64":"stVStoiQhXFxp4s2pdzPNoqVNBMojDU/fJ2db5S3CbM=","public_key_hex":"b2d552b68890857171a78b36a5dccf368a953413288c353f7c9d9d6f94b709b3","fingerprint_sha256_b32_first128bits":"RVFV5Z2OI2J3ZUO7ERDEBCYNKS","fingerprint_sha256_hex":"8d4b5ee74e4693bcd1df2446408b0d54","rotates_at":null,"url":"https://pith.science/pith-signing-key.json","notes":"Pith uses this Ed25519 key to sign canonical record SHA-256 digests. Verify with: ed25519_verify(public_key, message=canonical_sha256_bytes, signature=base64decode(signature_b64))."}],"merge_version":"pith-open-graph-merge-v1","built_at":"2026-07-06T18:38:05Z","links":{"resolver":"https://pith.science/pith/ZCLFEMMQWSCHSZTIW2RVTGQHKE","bundle":"https://pith.science/pith/ZCLFEMMQWSCHSZTIW2RVTGQHKE/bundle.json","state":"https://pith.science/pith/ZCLFEMMQWSCHSZTIW2RVTGQHKE/state.json","well_known_bundle":"https://pith.science/.well-known/pith/ZCLFEMMQWSCHSZTIW2RVTGQHKE/bundle.json"},"state":{"state_type":"pith_open_graph_state","state_version":"1.0","pith_number":"pith:2020:ZCLFEMMQWSCHSZTIW2RVTGQHKE","merge_version":"pith-open-graph-merge-v1","event_count":2,"valid_event_count":2,"invalid_event_count":0,"equivocation_count":0,"current":{"canonical_record":{"metadata":{"abstract_canon_sha256":"3f7d47be4fdb14d683efb14c78e11252101cef847e4f4f11cdebe257a707a0d6","cross_cats_sorted":["stat.ML"],"license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","primary_cat":"cs.LG","submitted_at":"2020-04-17T23:10:18Z","title_canon_sha256":"1357a89f61ce07a9fbd89f83f76b1b56053e09538c82ae694ee3a1395e53c398"},"schema_version":"1.0","source":{"id":"2004.08483","kind":"arxiv","version":5}},"source_aliases":[{"alias_kind":"arxiv","alias_value":"2004.08483","created_at":"2026-07-05T01:46:18Z"},{"alias_kind":"arxiv_version","alias_value":"2004.08483v5","created_at":"2026-07-05T01:46:18Z"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.2004.08483","created_at":"2026-07-05T01:46:18Z"},{"alias_kind":"pith_short_12","alias_value":"ZCLFEMMQWSCH","created_at":"2026-07-05T01:46:18Z"},{"alias_kind":"pith_short_16","alias_value":"ZCLFEMMQWSCHSZTI","created_at":"2026-07-05T01:46:18Z"},{"alias_kind":"pith_short_8","alias_value":"ZCLFEMMQ","created_at":"2026-07-05T01:46:18Z"}],"graph_snapshots":[{"event_id":"sha256:c4601f62cb45b723221b9b36b46706b1b4cdaf1f3d03b26795560db5f3033f13","target":"graph","created_at":"2026-07-05T01:46:18Z","signer":{"key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signer_id":"pith.science","signer_type":"pith_registry"},"payload":{"graph_snapshot":{"author_claims":{"count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","strong_count":0},"builder_version":"pith-number-builder-2026-05-17-v1","claims":{"count":0,"items":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"integrity":{"available":true,"clean":true,"detectors_run":[],"endpoint":"/pith/2004.08483/integrity.json","findings":[],"snapshot_sha256":"c28c3603d3b5d939e8dc4c7e95fa8dfce3d595e45f758748cecf8e644a296938","summary":{"advisory":0,"by_detector":{},"critical":0,"informational":0}},"paper":{"abstract_excerpt":"Transformer models have advanced the state of the art in many Natural Language Processing (NLP) tasks. In this paper, we present a new Transformer architecture, Extended Transformer Construction (ETC), that addresses two key challenges of standard Transformer architectures, namely scaling input length and encoding structured inputs. To scale attention to longer inputs, we introduce a novel global-local attention mechanism between global tokens and regular input tokens. We also show that combining global-local attention with relative position encodings and a Contrastive Predictive Coding (CPC) ","authors_text":"Anirudh Ravula, Chris Alberti, Joshua Ainslie, Li Yang, Philip Pham, Qifan Wang, Santiago Ontanon, Sumit Sanghai, Vaclav Cvicek, Zachary Fisher","cross_cats":["stat.ML"],"headline":"","license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","primary_cat":"cs.LG","submitted_at":"2020-04-17T23:10:18Z","title":"ETC: Encoding Long and Structured Inputs in Transformers"},"references":{"count":0,"internal_anchors":0,"resolved_work":0,"sample":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"2004.08483","kind":"arxiv","version":5},"verdict":{"created_at":null,"id":null,"model_set":{},"one_line_summary":"","pipeline_version":null,"pith_extraction_headline":"","strongest_claim":"","weakest_assumption":""}},"verdict_id":null}}],"author_attestations":[],"timestamp_anchors":[],"storage_attestations":[],"citation_signatures":[],"replication_records":[],"corrections":[],"mirror_hints":[],"record_created":{"event_id":"sha256:4de4426041dabffb84bda4bfcf8f4384c8b38b3e74bbced6e30e6499585bd74c","target":"record","created_at":"2026-07-05T01:46:18Z","signer":{"key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signer_id":"pith.science","signer_type":"pith_registry"},"payload":{"attestation_state":"computed","canonical_record":{"metadata":{"abstract_canon_sha256":"3f7d47be4fdb14d683efb14c78e11252101cef847e4f4f11cdebe257a707a0d6","cross_cats_sorted":["stat.ML"],"license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","primary_cat":"cs.LG","submitted_at":"2020-04-17T23:10:18Z","title_canon_sha256":"1357a89f61ce07a9fbd89f83f76b1b56053e09538c82ae694ee3a1395e53c398"},"schema_version":"1.0","source":{"id":"2004.08483","kind":"arxiv","version":5}},"canonical_sha256":"c896523190b484796668b6a3599a07512523f96a4de1e0382daca052fb79f55a","receipt":{"algorithm":"ed25519","builder_version":"pith-number-builder-2026-05-17-v1","canonical_sha256":"c896523190b484796668b6a3599a07512523f96a4de1e0382daca052fb79f55a","first_computed_at":"2026-07-05T01:46:18.230991Z","key_id":"pith-v1-2026-05","kind":"pith_receipt","last_reissued_at":"2026-07-05T01:46:18.230991Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","receipt_version":"0.3","signature_b64":"BOyYmW0cwEW4o1T1W6CCc5VvrI0F0Dv92qi6cAS8hUN4Y6gBL8HGUhIqs3Qeknr/1DcqXBEM7sVUZoE6OJGICQ==","signature_status":"signed_v1","signed_at":"2026-07-05T01:46:18.231422Z","signed_message":"canonical_sha256_bytes"},"source_id":"2004.08483","source_kind":"arxiv","source_version":5}}},"equivocations":[],"invalid_events":[],"applied_event_ids":["sha256:4de4426041dabffb84bda4bfcf8f4384c8b38b3e74bbced6e30e6499585bd74c","sha256:c4601f62cb45b723221b9b36b46706b1b4cdaf1f3d03b26795560db5f3033f13"],"state_sha256":"9e73902f2563c41b9815a25aa3bb97058f42d9bbb24f3c72b4865d06be5cbec8"},"bundle_signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"XJjb1Iot8g8vBhy9w6KQuY9yKrGtVyoWhBdmIcFrYFQD+8b1h6xJ8fvqlJsAi/CEmNBuy2MMayXui1+KuOrDAA==","signed_message":"bundle_sha256_bytes","signed_at":"2026-07-06T18:38:05.632950Z","bundle_sha256":"deb1323424ad7e121370bde871b67a02a89a31e38064d04d31a41de01419f3ba"}}