{"bundle_type":"pith_open_graph_bundle","bundle_version":"1.0","pith_number":"pith:2024:XF35BV7U2JIAANYZUYN66CNCMF","short_pith_number":"pith:XF35BV7U","canonical_record":{"source":{"id":"2404.12224","kind":"arxiv","version":2},"metadata":{"license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","primary_cat":"cs.CL","submitted_at":"2024-04-18T14:38:32Z","cross_cats_sorted":[],"title_canon_sha256":"9a494823dd6b09622a71bfc1fa9f4f403abd18896a84a8c9c773a9c26171fc71","abstract_canon_sha256":"ae3cabf0986a8b14020e34ded3a94aa75a581fcc1d92e53e50706e0319bcf6b1"},"schema_version":"1.0"},"canonical_sha256":"b977d0d7f4d250003719a61bef09a2615dafa6e6fa8827c35f36f725957a3a31","source":{"kind":"arxiv","id":"2404.12224","version":2},"source_aliases":[{"alias_kind":"arxiv","alias_value":"2404.12224","created_at":"2026-07-05T08:24:04Z"},{"alias_kind":"arxiv_version","alias_value":"2404.12224v2","created_at":"2026-07-05T08:24:04Z"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.2404.12224","created_at":"2026-07-05T08:24:04Z"},{"alias_kind":"pith_short_12","alias_value":"XF35BV7U2JIA","created_at":"2026-07-05T08:24:04Z"},{"alias_kind":"pith_short_16","alias_value":"XF35BV7U2JIAANYZ","created_at":"2026-07-05T08:24:04Z"},{"alias_kind":"pith_short_8","alias_value":"XF35BV7U","created_at":"2026-07-05T08:24:04Z"}],"events":[{"event_type":"record_created","subject_pith_number":"pith:2024:XF35BV7U2JIAANYZUYN66CNCMF","target":"record","payload":{"canonical_record":{"source":{"id":"2404.12224","kind":"arxiv","version":2},"metadata":{"license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","primary_cat":"cs.CL","submitted_at":"2024-04-18T14:38:32Z","cross_cats_sorted":[],"title_canon_sha256":"9a494823dd6b09622a71bfc1fa9f4f403abd18896a84a8c9c773a9c26171fc71","abstract_canon_sha256":"ae3cabf0986a8b14020e34ded3a94aa75a581fcc1d92e53e50706e0319bcf6b1"},"schema_version":"1.0"},"canonical_sha256":"b977d0d7f4d250003719a61bef09a2615dafa6e6fa8827c35f36f725957a3a31","receipt":{"kind":"pith_receipt","key_id":"pith-v1-2026-05","algorithm":"ed25519","signed_at":"2026-07-05T08:24:04.654155Z","signature_b64":"UlSE7Lt+kuK9RtOq8EqXKpd0XDsH/qYT3lNH2t4wzCT4wfW386nz+TH2Tr3AHs5D+r0V6spOx/mNiNUkSpm1Bg==","signed_message":"canonical_sha256_bytes","builder_version":"pith-number-builder-2026-05-17-v1","receipt_version":"0.3","canonical_sha256":"b977d0d7f4d250003719a61bef09a2615dafa6e6fa8827c35f36f725957a3a31","last_reissued_at":"2026-07-05T08:24:04.653708Z","signature_status":"signed_v1","first_computed_at":"2026-07-05T08:24:04.653708Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"source_kind":"arxiv","source_id":"2404.12224","source_version":2,"attestation_state":"computed"},"signer":{"signer_id":"pith.science","signer_type":"pith_registry","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"created_at":"2026-07-05T08:24:04Z","supersedes":[],"prev_event":null,"signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"ZRLuYE0bEJp7W+lu9r9UaiocEQO6g7UzTd7L+IJtvU9d9n2x4NFaw6b9pk/dOPVGZsBtqG2kMIwdEkNqJAqUAA==","signed_message":"open_graph_event_sha256_bytes","signed_at":"2026-08-08T23:53:29.047921Z"},"content_sha256":"9d8cb5af122213c62958a5aa0fab32144eccd396ea2907f76dd33e824e916452","schema_version":"1.0","event_id":"sha256:9d8cb5af122213c62958a5aa0fab32144eccd396ea2907f76dd33e824e916452"},{"event_type":"graph_snapshot","subject_pith_number":"pith:2024:XF35BV7U2JIAANYZUYN66CNCMF","target":"graph","payload":{"graph_snapshot":{"paper":{"title":"Length Generalization of Causal Transformers without Position Encoding","license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","headline":"","cross_cats":[],"primary_cat":"cs.CL","authors_text":"Hang Yan, Jie Wang, Qi Zhang, Tao Gui, Tao Ji, Xiaoling Wang, Xuanjing Huang, Yuanbin Wu","submitted_at":"2024-04-18T14:38:32Z","abstract_excerpt":"Generalizing to longer sentences is important for recent Transformer-based language models. Besides algorithms manipulating explicit position features, the success of Transformers without position encodings (NoPE) provides a new way to overcome the challenge. In this paper, we study the length generalization property of NoPE. We find that although NoPE can extend to longer sequences than the commonly used explicit position encodings, it still has a limited context length. We identify a connection between the failure of NoPE's generalization and the distraction of attention distributions. We pr"},"claims":{"count":0,"items":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"2404.12224","kind":"arxiv","version":2},"verdict":{"id":null,"model_set":{},"created_at":null,"strongest_claim":"","one_line_summary":"","pipeline_version":null,"weakest_assumption":"","pith_extraction_headline":""},"integrity":{"clean":true,"summary":{"advisory":0,"critical":0,"by_detector":{},"informational":0},"endpoint":"/pith/2404.12224/integrity.json","findings":[],"available":true,"detectors_run":[],"snapshot_sha256":"c28c3603d3b5d939e8dc4c7e95fa8dfce3d595e45f758748cecf8e644a296938"},"references":{"count":0,"sample":[],"resolved_work":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","internal_anchors":0},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"author_claims":{"count":0,"strong_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"builder_version":"pith-number-builder-2026-05-17-v1"},"verdict_id":null},"signer":{"signer_id":"pith.science","signer_type":"pith_registry","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"created_at":"2026-07-05T08:24:04Z","supersedes":[],"prev_event":null,"signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"ifcUwa2Jkdc31a0sIFMbeC9EHDz1iBbHGtA3vXjZ8FmIyAESYc7vv170HZc2ePM1dE08actYc5PNim38onn9Dw==","signed_message":"open_graph_event_sha256_bytes","signed_at":"2026-08-08T23:53:29.048542Z"},"content_sha256":"cb444fe0868dd20a5fc595f4a12bfb99f0c6acd344c9fc6c00f083007da09129","schema_version":"1.0","event_id":"sha256:cb444fe0868dd20a5fc595f4a12bfb99f0c6acd344c9fc6c00f083007da09129"}],"timestamp_proofs":[],"mirror_hints":[{"mirror_type":"https","name":"Pith Resolver","base_url":"https://pith.science","bundle_url":"https://pith.science/pith/XF35BV7U2JIAANYZUYN66CNCMF/bundle.json","state_url":"https://pith.science/pith/XF35BV7U2JIAANYZUYN66CNCMF/state.json","well_known_bundle_url":"https://pith.science/.well-known/pith/XF35BV7U2JIAANYZUYN66CNCMF/bundle.json","status":"primary"}],"public_keys":[{"key_id":"pith-v1-2026-05","algorithm":"ed25519","format":"raw","public_key_b64":"stVStoiQhXFxp4s2pdzPNoqVNBMojDU/fJ2db5S3CbM=","public_key_hex":"b2d552b68890857171a78b36a5dccf368a953413288c353f7c9d9d6f94b709b3","fingerprint_sha256_b32_first128bits":"RVFV5Z2OI2J3ZUO7ERDEBCYNKS","fingerprint_sha256_hex":"8d4b5ee74e4693bcd1df2446408b0d54","rotates_at":null,"url":"https://pith.science/pith-signing-key.json","notes":"Pith uses this Ed25519 key to sign canonical record SHA-256 digests. Verify with: ed25519_verify(public_key, message=canonical_sha256_bytes, signature=base64decode(signature_b64))."}],"merge_version":"pith-open-graph-merge-v1","built_at":"2026-08-08T23:53:29Z","links":{"resolver":"https://pith.science/pith/XF35BV7U2JIAANYZUYN66CNCMF","bundle":"https://pith.science/pith/XF35BV7U2JIAANYZUYN66CNCMF/bundle.json","state":"https://pith.science/pith/XF35BV7U2JIAANYZUYN66CNCMF/state.json","well_known_bundle":"https://pith.science/.well-known/pith/XF35BV7U2JIAANYZUYN66CNCMF/bundle.json"},"state":{"state_type":"pith_open_graph_state","state_version":"1.0","pith_number":"pith:2024:XF35BV7U2JIAANYZUYN66CNCMF","merge_version":"pith-open-graph-merge-v1","event_count":2,"valid_event_count":2,"invalid_event_count":0,"equivocation_count":0,"current":{"canonical_record":{"metadata":{"abstract_canon_sha256":"ae3cabf0986a8b14020e34ded3a94aa75a581fcc1d92e53e50706e0319bcf6b1","cross_cats_sorted":[],"license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","primary_cat":"cs.CL","submitted_at":"2024-04-18T14:38:32Z","title_canon_sha256":"9a494823dd6b09622a71bfc1fa9f4f403abd18896a84a8c9c773a9c26171fc71"},"schema_version":"1.0","source":{"id":"2404.12224","kind":"arxiv","version":2}},"source_aliases":[{"alias_kind":"arxiv","alias_value":"2404.12224","created_at":"2026-07-05T08:24:04Z"},{"alias_kind":"arxiv_version","alias_value":"2404.12224v2","created_at":"2026-07-05T08:24:04Z"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.2404.12224","created_at":"2026-07-05T08:24:04Z"},{"alias_kind":"pith_short_12","alias_value":"XF35BV7U2JIA","created_at":"2026-07-05T08:24:04Z"},{"alias_kind":"pith_short_16","alias_value":"XF35BV7U2JIAANYZ","created_at":"2026-07-05T08:24:04Z"},{"alias_kind":"pith_short_8","alias_value":"XF35BV7U","created_at":"2026-07-05T08:24:04Z"}],"graph_snapshots":[{"event_id":"sha256:cb444fe0868dd20a5fc595f4a12bfb99f0c6acd344c9fc6c00f083007da09129","target":"graph","created_at":"2026-07-05T08:24:04Z","signer":{"key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signer_id":"pith.science","signer_type":"pith_registry"},"payload":{"graph_snapshot":{"author_claims":{"count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","strong_count":0},"builder_version":"pith-number-builder-2026-05-17-v1","claims":{"count":0,"items":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"integrity":{"available":true,"clean":true,"detectors_run":[],"endpoint":"/pith/2404.12224/integrity.json","findings":[],"snapshot_sha256":"c28c3603d3b5d939e8dc4c7e95fa8dfce3d595e45f758748cecf8e644a296938","summary":{"advisory":0,"by_detector":{},"critical":0,"informational":0}},"paper":{"abstract_excerpt":"Generalizing to longer sentences is important for recent Transformer-based language models. Besides algorithms manipulating explicit position features, the success of Transformers without position encodings (NoPE) provides a new way to overcome the challenge. In this paper, we study the length generalization property of NoPE. We find that although NoPE can extend to longer sequences than the commonly used explicit position encodings, it still has a limited context length. We identify a connection between the failure of NoPE's generalization and the distraction of attention distributions. We pr","authors_text":"Hang Yan, Jie Wang, Qi Zhang, Tao Gui, Tao Ji, Xiaoling Wang, Xuanjing Huang, Yuanbin Wu","cross_cats":[],"headline":"","license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","primary_cat":"cs.CL","submitted_at":"2024-04-18T14:38:32Z","title":"Length Generalization of Causal Transformers without Position Encoding"},"references":{"count":0,"internal_anchors":0,"resolved_work":0,"sample":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"2404.12224","kind":"arxiv","version":2},"verdict":{"created_at":null,"id":null,"model_set":{},"one_line_summary":"","pipeline_version":null,"pith_extraction_headline":"","strongest_claim":"","weakest_assumption":""}},"verdict_id":null}}],"author_attestations":[],"timestamp_anchors":[],"storage_attestations":[],"citation_signatures":[],"replication_records":[],"corrections":[],"mirror_hints":[],"record_created":{"event_id":"sha256:9d8cb5af122213c62958a5aa0fab32144eccd396ea2907f76dd33e824e916452","target":"record","created_at":"2026-07-05T08:24:04Z","signer":{"key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signer_id":"pith.science","signer_type":"pith_registry"},"payload":{"attestation_state":"computed","canonical_record":{"metadata":{"abstract_canon_sha256":"ae3cabf0986a8b14020e34ded3a94aa75a581fcc1d92e53e50706e0319bcf6b1","cross_cats_sorted":[],"license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","primary_cat":"cs.CL","submitted_at":"2024-04-18T14:38:32Z","title_canon_sha256":"9a494823dd6b09622a71bfc1fa9f4f403abd18896a84a8c9c773a9c26171fc71"},"schema_version":"1.0","source":{"id":"2404.12224","kind":"arxiv","version":2}},"canonical_sha256":"b977d0d7f4d250003719a61bef09a2615dafa6e6fa8827c35f36f725957a3a31","receipt":{"algorithm":"ed25519","builder_version":"pith-number-builder-2026-05-17-v1","canonical_sha256":"b977d0d7f4d250003719a61bef09a2615dafa6e6fa8827c35f36f725957a3a31","first_computed_at":"2026-07-05T08:24:04.653708Z","key_id":"pith-v1-2026-05","kind":"pith_receipt","last_reissued_at":"2026-07-05T08:24:04.653708Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","receipt_version":"0.3","signature_b64":"UlSE7Lt+kuK9RtOq8EqXKpd0XDsH/qYT3lNH2t4wzCT4wfW386nz+TH2Tr3AHs5D+r0V6spOx/mNiNUkSpm1Bg==","signature_status":"signed_v1","signed_at":"2026-07-05T08:24:04.654155Z","signed_message":"canonical_sha256_bytes"},"source_id":"2404.12224","source_kind":"arxiv","source_version":2}}},"equivocations":[],"invalid_events":[],"applied_event_ids":["sha256:9d8cb5af122213c62958a5aa0fab32144eccd396ea2907f76dd33e824e916452","sha256:cb444fe0868dd20a5fc595f4a12bfb99f0c6acd344c9fc6c00f083007da09129"],"state_sha256":"631a1208be139c72ab52cdc71c999cb0d63c3efd149d8395479617328d18fb81"},"bundle_signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"OpDKG4Luk6A9y5HgniYNxKpH9041Ox6TMj6m+bH2kczmFSQ0xAwdwm8L0uiT02gN9WJ7Oui2Qxtr7agouZtrBQ==","signed_message":"bundle_sha256_bytes","signed_at":"2026-08-08T23:53:29.056055Z","bundle_sha256":"3e5fa3052cf7040c7dc985f87b35dd2611a3b0114fb15f9b037c9e485cdec8e4"}}