{"record_type":"pith_number_record","schema_url":"https://pith.science/schemas/pith-number/v1.json","pith_number":"pith:2024:5MGWGTP3QEEYVQPDKPVVFIIYVZ","short_pith_number":"pith:5MGWGTP3","schema_version":"1.0","canonical_sha256":"eb0d634dfb81098ac1e353eb52a118ae5b4d57043bbcb1e3f0231635994ea634","source":{"kind":"arxiv","id":"2410.06205","version":3},"attestation_state":"computed","paper":{"title":"Round and Round We Go! What makes Rotary Positional Encodings useful?","license":"http://creativecommons.org/licenses/by/4.0/","headline":"","cross_cats":["cs.LG"],"primary_cat":"cs.CL","authors_text":"Alex Vitvitskyi, Christos Perivolaropoulos, Federico Barbero, Petar Veli\\v{c}kovi\\'c, Razvan Pascanu","submitted_at":"2024-10-08T17:07:01Z","abstract_excerpt":"Positional Encodings (PEs) are a critical component of Transformer-based Large Language Models (LLMs), providing the attention mechanism with important sequence-position information. One of the most popular types of encoding used today in LLMs are Rotary Positional Encodings (RoPE), that rotate the queries and keys based on their relative distance. A common belief is that RoPE is useful because it helps to decay token dependency as relative distance increases. In this work, we argue that this is unlikely to be the core reason. We study the internals of a trained Gemma 7B model to understand ho"},"verification_status":{"content_addressed":true,"pith_receipt":true,"author_attested":false,"weak_author_claims":0,"strong_author_claims":0,"externally_anchored":false,"storage_verified":false,"citation_signatures":0,"replication_records":0,"graph_snapshot":true,"references_resolved":false,"formal_links_present":false},"canonical_record":{"source":{"id":"2410.06205","kind":"arxiv","version":3},"metadata":{"license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.CL","submitted_at":"2024-10-08T17:07:01Z","cross_cats_sorted":["cs.LG"],"title_canon_sha256":"909c0306a6c840a994b51be5c47f9fc06baa41aa67c6ae27933fe46f76e39677","abstract_canon_sha256":"8682bfa9e25efa4c8db150ba543b6778964d0fb6b53c3bdc663d3c64a81312a1"},"schema_version":"1.0"},"receipt":{"kind":"pith_receipt","key_id":"pith-v1-2026-05","algorithm":"ed25519","signed_at":"2026-07-05T11:02:13.175616Z","signature_b64":"qEK17mjirCwuY5qPdNZ2NYthyGQPFYegc/FsTdJRgLoXKTaCYCD/HtdOcOUjg6QWOeS9ZBf5q/qE7MxiaaUeAQ==","signed_message":"canonical_sha256_bytes","builder_version":"pith-number-builder-2026-05-17-v1","receipt_version":"0.3","canonical_sha256":"eb0d634dfb81098ac1e353eb52a118ae5b4d57043bbcb1e3f0231635994ea634","last_reissued_at":"2026-07-05T11:02:13.175016Z","signature_status":"signed_v1","first_computed_at":"2026-07-05T11:02:13.175016Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"graph_snapshot":{"paper":{"title":"Round and Round We Go! What makes Rotary Positional Encodings useful?","license":"http://creativecommons.org/licenses/by/4.0/","headline":"","cross_cats":["cs.LG"],"primary_cat":"cs.CL","authors_text":"Alex Vitvitskyi, Christos Perivolaropoulos, Federico Barbero, Petar Veli\\v{c}kovi\\'c, Razvan Pascanu","submitted_at":"2024-10-08T17:07:01Z","abstract_excerpt":"Positional Encodings (PEs) are a critical component of Transformer-based Large Language Models (LLMs), providing the attention mechanism with important sequence-position information. One of the most popular types of encoding used today in LLMs are Rotary Positional Encodings (RoPE), that rotate the queries and keys based on their relative distance. A common belief is that RoPE is useful because it helps to decay token dependency as relative distance increases. In this work, we argue that this is unlikely to be the core reason. We study the internals of a trained Gemma 7B model to understand ho"},"claims":{"count":0,"items":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"2410.06205","kind":"arxiv","version":3},"verdict":{"id":null,"model_set":{},"created_at":null,"strongest_claim":"","one_line_summary":"","pipeline_version":null,"weakest_assumption":"","pith_extraction_headline":""},"integrity":{"clean":true,"summary":{"advisory":0,"critical":0,"by_detector":{},"informational":0},"endpoint":"/pith/2410.06205/integrity.json","findings":[],"available":true,"detectors_run":[],"snapshot_sha256":"c28c3603d3b5d939e8dc4c7e95fa8dfce3d595e45f758748cecf8e644a296938"},"references":{"count":0,"sample":[],"resolved_work":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","internal_anchors":0},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"author_claims":{"count":0,"strong_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"builder_version":"pith-number-builder-2026-05-17-v1"},"aliases":[{"alias_kind":"arxiv","alias_value":"2410.06205","created_at":"2026-07-05T11:02:13.175083+00:00"},{"alias_kind":"arxiv_version","alias_value":"2410.06205v3","created_at":"2026-07-05T11:02:13.175083+00:00"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.2410.06205","created_at":"2026-07-05T11:02:13.175083+00:00"},{"alias_kind":"pith_short_12","alias_value":"5MGWGTP3QEEY","created_at":"2026-07-05T11:02:13.175083+00:00"},{"alias_kind":"pith_short_16","alias_value":"5MGWGTP3QEEYVQPD","created_at":"2026-07-05T11:02:13.175083+00:00"},{"alias_kind":"pith_short_8","alias_value":"5MGWGTP3","created_at":"2026-07-05T11:02:13.175083+00:00"}],"events":[],"event_summary":{},"paper_claims":[],"inbound_citations":{"count":17,"internal_anchor_count":1,"sample":[{"citing_arxiv_id":"2607.06621","citing_title":"Fingerprint, Not Blueprint: How Positional Schemes Set the Default Spectral Algebra of Attention","ref_index":1,"is_internal_anchor":true},{"citing_arxiv_id":"2606.12213","citing_title":"SHERPA: Seam-aware Harmonized ERP Adaptation for Open-Domain 360$^\\circ$ Panorama Generation","ref_index":36,"is_internal_anchor":false},{"citing_arxiv_id":"2606.11275","citing_title":"RoVE: Rotary Value Embeddings Attention for Relative Position-dependent Value Pathways","ref_index":2,"is_internal_anchor":false},{"citing_arxiv_id":"2606.04662","citing_title":"Why Muon Outperforms Adam: A Curvature Perspective","ref_index":1,"is_internal_anchor":false},{"citing_arxiv_id":"2505.12942","citing_title":"A3 : an Analytical Low-Rank Approximation Framework for Attention","ref_index":2,"is_internal_anchor":false},{"citing_arxiv_id":"2605.14457","citing_title":"Stateful Reasoning via Insight Replay","ref_index":20,"is_internal_anchor":false},{"citing_arxiv_id":"2502.05564","citing_title":"TabICL: A Tabular Foundation Model for In-Context Learning on Large Data","ref_index":24,"is_internal_anchor":false},{"citing_arxiv_id":"2605.19218","citing_title":"Rotation-Aligned Key Channel Pruning for Efficient Vision-Language Model Inference","ref_index":29,"is_internal_anchor":false},{"citing_arxiv_id":"2605.15514","citing_title":"RoPE Distinguishes Neither Positions Nor Tokens in Long Contexts, Provably","ref_index":37,"is_internal_anchor":false},{"citing_arxiv_id":"2512.07805","citing_title":"Group Representational Position Encoding","ref_index":2,"is_internal_anchor":false},{"citing_arxiv_id":"2605.14457","citing_title":"Stateful Reasoning via Insight Replay","ref_index":20,"is_internal_anchor":false},{"citing_arxiv_id":"2605.12922","citing_title":"When Attention Closes: How LLMs Lose the Thread in Multi-Turn Interaction","ref_index":32,"is_internal_anchor":false},{"citing_arxiv_id":"2605.10414","citing_title":"Remember to Forget: Gated Adaptive Positional Encoding","ref_index":1,"is_internal_anchor":false},{"citing_arxiv_id":"2605.00968","citing_title":"Adaptive 3D-RoPE: Physics-Aligned Rotary Positional Encoding for Wireless Foundation Models","ref_index":27,"is_internal_anchor":false},{"citing_arxiv_id":"2501.00103","citing_title":"LTX-Video: Realtime Video Latent Diffusion","ref_index":22,"is_internal_anchor":false},{"citing_arxiv_id":"2604.18603","citing_title":"Dual Triangle Attention: Effective Bidirectional Attention Without Positional Embeddings","ref_index":36,"is_internal_anchor":false},{"citing_arxiv_id":"2604.14922","citing_title":"LongAct: Harnessing Intrinsic Activation Patterns for Long-Context Reinforcement Learning","ref_index":4,"is_internal_anchor":false}]},"formal_canon":{"evidence_count":0,"sample":[],"anchors":[]},"links":{"html":"https://pith.science/pith/5MGWGTP3QEEYVQPDKPVVFIIYVZ","json":"https://pith.science/pith/5MGWGTP3QEEYVQPDKPVVFIIYVZ.json","graph_json":"https://pith.science/api/pith-number/5MGWGTP3QEEYVQPDKPVVFIIYVZ/graph.json","events_json":"https://pith.science/api/pith-number/5MGWGTP3QEEYVQPDKPVVFIIYVZ/events.json","paper":"https://pith.science/paper/5MGWGTP3"},"agent_actions":{"view_html":"https://pith.science/pith/5MGWGTP3QEEYVQPDKPVVFIIYVZ","download_json":"https://pith.science/pith/5MGWGTP3QEEYVQPDKPVVFIIYVZ.json","view_paper":"https://pith.science/paper/5MGWGTP3","resolve_alias":"https://pith.science/api/pith-number/resolve?arxiv=2410.06205&json=true","fetch_graph":"https://pith.science/api/pith-number/5MGWGTP3QEEYVQPDKPVVFIIYVZ/graph.json","fetch_events":"https://pith.science/api/pith-number/5MGWGTP3QEEYVQPDKPVVFIIYVZ/events.json","actions":{"anchor_timestamp":"https://pith.science/pith/5MGWGTP3QEEYVQPDKPVVFIIYVZ/action/timestamp_anchor","attest_storage":"https://pith.science/pith/5MGWGTP3QEEYVQPDKPVVFIIYVZ/action/storage_attestation","attest_author":"https://pith.science/pith/5MGWGTP3QEEYVQPDKPVVFIIYVZ/action/author_attestation","sign_citation":"https://pith.science/pith/5MGWGTP3QEEYVQPDKPVVFIIYVZ/action/citation_signature","submit_replication":"https://pith.science/pith/5MGWGTP3QEEYVQPDKPVVFIIYVZ/action/replication_record"}},"created_at":"2026-07-05T11:02:13.175083+00:00","updated_at":"2026-07-05T11:02:13.175083+00:00"}