{"state_type":"pith_open_graph_state","state_version":"1.0","pith_number":"pith:2023:G6RJ74HP6VMADMSYIPOPDFJMWD","merge_version":"pith-open-graph-merge-v1","event_count":2,"valid_event_count":2,"invalid_event_count":0,"equivocation_count":0,"current":{"canonical_record":{"metadata":{"abstract_canon_sha256":"390de5df800adcb97386504310ddfb6d3e5b50594e7d9cb0f766efbcb227ed40","cross_cats_sorted":["cs.AI","cs.CL"],"license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","primary_cat":"cs.LG","submitted_at":"2023-09-15T09:36:51Z","title_canon_sha256":"3c5d92cf211eafd89fcdea69379c57eeb939abdad09258921ca35f7263ac1976"},"schema_version":"1.0","source":{"id":"2309.08646","kind":"arxiv","version":3}},"source_aliases":[{"alias_kind":"arxiv","alias_value":"2309.08646","created_at":"2026-07-05T07:50:03Z"},{"alias_kind":"arxiv_version","alias_value":"2309.08646v3","created_at":"2026-07-05T07:50:03Z"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.2309.08646","created_at":"2026-07-05T07:50:03Z"},{"alias_kind":"pith_short_12","alias_value":"G6RJ74HP6VMA","created_at":"2026-07-05T07:50:03Z"},{"alias_kind":"pith_short_16","alias_value":"G6RJ74HP6VMADMSY","created_at":"2026-07-05T07:50:03Z"},{"alias_kind":"pith_short_8","alias_value":"G6RJ74HP","created_at":"2026-07-05T07:50:03Z"}],"graph_snapshots":[{"event_id":"sha256:275ac989b88921912d37ce29b4827cade51c2904ff3bd2ce2fb4d2a5ef225bd6","target":"graph","created_at":"2026-07-05T07:50:03Z","signer":{"key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signer_id":"pith.science","signer_type":"pith_registry"},"payload":{"graph_snapshot":{"author_claims":{"count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","strong_count":0},"builder_version":"pith-number-builder-2026-05-17-v1","claims":{"count":0,"items":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"integrity":{"available":true,"clean":true,"detectors_run":[],"endpoint":"/pith/2309.08646/integrity.json","findings":[],"snapshot_sha256":"c28c3603d3b5d939e8dc4c7e95fa8dfce3d595e45f758748cecf8e644a296938","summary":{"advisory":0,"by_detector":{},"critical":0,"informational":0}},"paper":{"abstract_excerpt":"Self-attention and position embedding are two key modules in transformer-based Large Language Models (LLMs). However, the potential relationship between them is far from well studied, especially for long context window extending. In fact, anomalous behaviors harming long context extrapolation exist between Rotary Position Embedding (RoPE) and vanilla self-attention unveiled by our work. To address this issue, we propose a novel attention mechanism, CoCA (Collinear Constrained Attention). Specifically, we enforce a collinear constraint between $Q$ and $K$ to seamlessly integrate RoPE and self-a","authors_text":"Jianguo Li, Jing Ye, Qi Zhang, Shiyi Zhu, Siqiao Xue, Wei Jiang, Yifan Wu","cross_cats":["cs.AI","cs.CL"],"headline":"","license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","primary_cat":"cs.LG","submitted_at":"2023-09-15T09:36:51Z","title":"CoCA: Fusing Position Embedding with Collinear Constrained Attention in Transformers for Long Context Window Extending"},"references":{"count":0,"internal_anchors":0,"resolved_work":0,"sample":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"2309.08646","kind":"arxiv","version":3},"verdict":{"created_at":null,"id":null,"model_set":{},"one_line_summary":"","pipeline_version":null,"pith_extraction_headline":"","strongest_claim":"","weakest_assumption":""}},"verdict_id":null}}],"author_attestations":[],"timestamp_anchors":[],"storage_attestations":[],"citation_signatures":[],"replication_records":[],"corrections":[],"mirror_hints":[],"record_created":{"event_id":"sha256:d77e672d35572bb8849dc5c2f432278ec9e453e24336d9f0289d4082cf501d4d","target":"record","created_at":"2026-07-05T07:50:03Z","signer":{"key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signer_id":"pith.science","signer_type":"pith_registry"},"payload":{"attestation_state":"computed","canonical_record":{"metadata":{"abstract_canon_sha256":"390de5df800adcb97386504310ddfb6d3e5b50594e7d9cb0f766efbcb227ed40","cross_cats_sorted":["cs.AI","cs.CL"],"license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","primary_cat":"cs.LG","submitted_at":"2023-09-15T09:36:51Z","title_canon_sha256":"3c5d92cf211eafd89fcdea69379c57eeb939abdad09258921ca35f7263ac1976"},"schema_version":"1.0","source":{"id":"2309.08646","kind":"arxiv","version":3}},"canonical_sha256":"37a29ff0eff55801b25843dcf1952cb0f2e4dbde6c7b94173de90f9c7c5f6de2","receipt":{"algorithm":"ed25519","builder_version":"pith-number-builder-2026-05-17-v1","canonical_sha256":"37a29ff0eff55801b25843dcf1952cb0f2e4dbde6c7b94173de90f9c7c5f6de2","first_computed_at":"2026-07-05T07:50:03.417366Z","key_id":"pith-v1-2026-05","kind":"pith_receipt","last_reissued_at":"2026-07-05T07:50:03.417366Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","receipt_version":"0.3","signature_b64":"9sWlf6ypUElDU9QzLZjDObRzgBujGoHapgO3L6ZyrUYxPMgaPDRDUz5h8LNLz4y0vsfjgrkGBYzYHl/TYWNQDg==","signature_status":"signed_v1","signed_at":"2026-07-05T07:50:03.417837Z","signed_message":"canonical_sha256_bytes"},"source_id":"2309.08646","source_kind":"arxiv","source_version":3}}},"equivocations":[],"invalid_events":[],"applied_event_ids":["sha256:d77e672d35572bb8849dc5c2f432278ec9e453e24336d9f0289d4082cf501d4d","sha256:275ac989b88921912d37ce29b4827cade51c2904ff3bd2ce2fb4d2a5ef225bd6"],"state_sha256":"af7f40ea579e04713df5ddb2f07bc91812409741c0c0cd3bc93cd566d1b3f3fb"}