{"record_type":"pith_number_record","schema_url":"https://pith.science/schemas/pith-number/v1.json","pith_number":"pith:2022:H42ESIK7XQR2X3UPXGJMM6UDLQ","short_pith_number":"pith:H42ESIK7","schema_version":"1.0","canonical_sha256":"3f3449215fbc23abee8fb992c67a835c2042d764cbbee516de657fa95002200b","source":{"kind":"arxiv","id":"2212.10554","version":1},"attestation_state":"computed","paper":{"title":"A Length-Extrapolatable Transformer","license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","headline":"","cross_cats":[],"primary_cat":"cs.CL","authors_text":"Alon Benhaim, Barun Patra, Furu Wei, Li Dong, Shaohan Huang, Shuming Ma, Vishrav Chaudhary, Xia Song, Yutao Sun","submitted_at":"2022-12-20T18:56:20Z","abstract_excerpt":"Position modeling plays a critical role in Transformers. In this paper, we focus on length extrapolation, i.e., training on short texts while evaluating longer sequences. We define attention resolution as an indicator of extrapolation. Then we propose two designs to improve the above metric of Transformers. Specifically, we introduce a relative position embedding to explicitly maximize attention resolution. Moreover, we use blockwise causal attention during inference for better resolution. We evaluate different Transformer variants with language modeling. Experimental results show that our mod"},"verification_status":{"content_addressed":true,"pith_receipt":true,"author_attested":false,"weak_author_claims":0,"strong_author_claims":0,"externally_anchored":false,"storage_verified":false,"citation_signatures":0,"replication_records":0,"graph_snapshot":true,"references_resolved":false,"formal_links_present":false},"canonical_record":{"source":{"id":"2212.10554","kind":"arxiv","version":1},"metadata":{"license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","primary_cat":"cs.CL","submitted_at":"2022-12-20T18:56:20Z","cross_cats_sorted":[],"title_canon_sha256":"5ffdf21c4f82d8176f4b87348c660757e185b91b46bbf87447a70bc2e444e1c7","abstract_canon_sha256":"ec9f47d73ead426b4cdcd9ad77df43be3e680c19405aaa76425821b81066a330"},"schema_version":"1.0"},"receipt":{"kind":"pith_receipt","key_id":"pith-v1-2026-05","algorithm":"ed25519","signed_at":"2026-07-05T05:27:08.662156Z","signature_b64":"bvuw5/yMZILPozk6P3I3J+mxVLhrd4q8hyYAV7qU6PfCoLtGTTvzYq9gzL3mNUl3c6mgZc6JRJChkmzRGwhpCQ==","signed_message":"canonical_sha256_bytes","builder_version":"pith-number-builder-2026-05-17-v1","receipt_version":"0.3","canonical_sha256":"3f3449215fbc23abee8fb992c67a835c2042d764cbbee516de657fa95002200b","last_reissued_at":"2026-07-05T05:27:08.661708Z","signature_status":"signed_v1","first_computed_at":"2026-07-05T05:27:08.661708Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"graph_snapshot":{"paper":{"title":"A Length-Extrapolatable Transformer","license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","headline":"","cross_cats":[],"primary_cat":"cs.CL","authors_text":"Alon Benhaim, Barun Patra, Furu Wei, Li Dong, Shaohan Huang, Shuming Ma, Vishrav Chaudhary, Xia Song, Yutao Sun","submitted_at":"2022-12-20T18:56:20Z","abstract_excerpt":"Position modeling plays a critical role in Transformers. In this paper, we focus on length extrapolation, i.e., training on short texts while evaluating longer sequences. We define attention resolution as an indicator of extrapolation. Then we propose two designs to improve the above metric of Transformers. Specifically, we introduce a relative position embedding to explicitly maximize attention resolution. Moreover, we use blockwise causal attention during inference for better resolution. We evaluate different Transformer variants with language modeling. Experimental results show that our mod"},"claims":{"count":0,"items":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"2212.10554","kind":"arxiv","version":1},"verdict":{"id":null,"model_set":{},"created_at":null,"strongest_claim":"","one_line_summary":"","pipeline_version":null,"weakest_assumption":"","pith_extraction_headline":""},"integrity":{"clean":true,"summary":{"advisory":0,"critical":0,"by_detector":{},"informational":0},"endpoint":"/pith/2212.10554/integrity.json","findings":[],"available":true,"detectors_run":[],"snapshot_sha256":"c28c3603d3b5d939e8dc4c7e95fa8dfce3d595e45f758748cecf8e644a296938"},"references":{"count":0,"sample":[],"resolved_work":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","internal_anchors":0},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"author_claims":{"count":0,"strong_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"builder_version":"pith-number-builder-2026-05-17-v1"},"aliases":[{"alias_kind":"arxiv","alias_value":"2212.10554","created_at":"2026-07-05T05:27:08.661764+00:00"},{"alias_kind":"arxiv_version","alias_value":"2212.10554v1","created_at":"2026-07-05T05:27:08.661764+00:00"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.2212.10554","created_at":"2026-07-05T05:27:08.661764+00:00"},{"alias_kind":"pith_short_12","alias_value":"H42ESIK7XQR2","created_at":"2026-07-05T05:27:08.661764+00:00"},{"alias_kind":"pith_short_16","alias_value":"H42ESIK7XQR2X3UP","created_at":"2026-07-05T05:27:08.661764+00:00"},{"alias_kind":"pith_short_8","alias_value":"H42ESIK7","created_at":"2026-07-05T05:27:08.661764+00:00"}],"events":[],"event_summary":{},"paper_claims":[],"inbound_citations":{"count":12,"internal_anchor_count":0,"sample":[{"citing_arxiv_id":"2606.21300","citing_title":"SCOPE: Scale-Consistent One-Pass Estimation of 3D Geometry","ref_index":18,"is_internal_anchor":false},{"citing_arxiv_id":"2605.04217","citing_title":"Jordan-RoPE: Non-Semisimple Relative Positional Encoding via Complex Jordan Blocks","ref_index":26,"is_internal_anchor":false},{"citing_arxiv_id":"2509.04154","citing_title":"Robust Filter Attention: Self-Attention as Precision-Weighted State Estimation","ref_index":88,"is_internal_anchor":false},{"citing_arxiv_id":"2512.07805","citing_title":"Group Representational Position Encoding","ref_index":23,"is_internal_anchor":false},{"citing_arxiv_id":"2302.14045","citing_title":"Language Is Not All You Need: Aligning Perception with Language Models","ref_index":20,"is_internal_anchor":false},{"citing_arxiv_id":"2403.07815","citing_title":"Chronos: Learning the Language of Time Series","ref_index":81,"is_internal_anchor":false},{"citing_arxiv_id":"2605.11007","citing_title":"The Transformer as a Polar State Estimator","ref_index":191,"is_internal_anchor":false},{"citing_arxiv_id":"2308.14508","citing_title":"LongBench: A Bilingual, Multitask Benchmark for Long Context Understanding","ref_index":118,"is_internal_anchor":false},{"citing_arxiv_id":"2309.00071","citing_title":"YaRN: Efficient Context Window Extension of Large Language Models","ref_index":13,"is_internal_anchor":false},{"citing_arxiv_id":"2307.08621","citing_title":"Retentive Network: A Successor to Transformer for Large Language Models","ref_index":17,"is_internal_anchor":false},{"citing_arxiv_id":"2605.04217","citing_title":"Jordan-RoPE: Non-Semisimple Relative Positional Encoding via Complex Jordan Blocks","ref_index":6,"is_internal_anchor":false},{"citing_arxiv_id":"2604.18603","citing_title":"Dual Triangle Attention: Effective Bidirectional Attention Without Positional Embeddings","ref_index":31,"is_internal_anchor":false}]},"formal_canon":{"evidence_count":0,"sample":[],"anchors":[]},"links":{"html":"https://pith.science/pith/H42ESIK7XQR2X3UPXGJMM6UDLQ","json":"https://pith.science/pith/H42ESIK7XQR2X3UPXGJMM6UDLQ.json","graph_json":"https://pith.science/api/pith-number/H42ESIK7XQR2X3UPXGJMM6UDLQ/graph.json","events_json":"https://pith.science/api/pith-number/H42ESIK7XQR2X3UPXGJMM6UDLQ/events.json","paper":"https://pith.science/paper/H42ESIK7"},"agent_actions":{"view_html":"https://pith.science/pith/H42ESIK7XQR2X3UPXGJMM6UDLQ","download_json":"https://pith.science/pith/H42ESIK7XQR2X3UPXGJMM6UDLQ.json","view_paper":"https://pith.science/paper/H42ESIK7","resolve_alias":"https://pith.science/api/pith-number/resolve?arxiv=2212.10554&json=true","fetch_graph":"https://pith.science/api/pith-number/H42ESIK7XQR2X3UPXGJMM6UDLQ/graph.json","fetch_events":"https://pith.science/api/pith-number/H42ESIK7XQR2X3UPXGJMM6UDLQ/events.json","actions":{"anchor_timestamp":"https://pith.science/pith/H42ESIK7XQR2X3UPXGJMM6UDLQ/action/timestamp_anchor","attest_storage":"https://pith.science/pith/H42ESIK7XQR2X3UPXGJMM6UDLQ/action/storage_attestation","attest_author":"https://pith.science/pith/H42ESIK7XQR2X3UPXGJMM6UDLQ/action/author_attestation","sign_citation":"https://pith.science/pith/H42ESIK7XQR2X3UPXGJMM6UDLQ/action/citation_signature","submit_replication":"https://pith.science/pith/H42ESIK7XQR2X3UPXGJMM6UDLQ/action/replication_record"}},"created_at":"2026-07-05T05:27:08.661764+00:00","updated_at":"2026-07-05T05:27:08.661764+00:00"}