{"record_type":"pith_number_record","schema_url":"https://pith.science/schemas/pith-number/v1.json","pith_number":"pith:2024:3FUXFNXKBUDEDVPOJG7WZ3XJ6Q","short_pith_number":"pith:3FUXFNXK","schema_version":"1.0","canonical_sha256":"d96972b6ea0d0641d5ee49bf6ceee9f40eb8a000cbd917c141634dd9f4c7cc92","source":{"kind":"arxiv","id":"2404.15574","version":1},"attestation_state":"computed","paper":{"title":"Retrieval Head Mechanistically Explains Long-Context Factuality","license":"http://creativecommons.org/licenses/by/4.0/","headline":"","cross_cats":[],"primary_cat":"cs.CL","authors_text":"Guangxuan Xiao, Hao Peng, Wenhao Wu, Yao Fu, Yizhong Wang","submitted_at":"2024-04-24T00:24:03Z","abstract_excerpt":"Despite the recent progress in long-context language models, it remains elusive how transformer-based models exhibit the capability to retrieve relevant information from arbitrary locations within the long context. This paper aims to address this question. Our systematic investigation across a wide spectrum of models reveals that a special type of attention heads are largely responsible for retrieving information, which we dub retrieval heads. We identify intriguing properties of retrieval heads:(1) universal: all the explored models with long-context capability have a set of retrieval heads; "},"verification_status":{"content_addressed":true,"pith_receipt":true,"author_attested":false,"weak_author_claims":0,"strong_author_claims":0,"externally_anchored":false,"storage_verified":false,"citation_signatures":0,"replication_records":0,"graph_snapshot":true,"references_resolved":false,"formal_links_present":false},"canonical_record":{"source":{"id":"2404.15574","kind":"arxiv","version":1},"metadata":{"license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.CL","submitted_at":"2024-04-24T00:24:03Z","cross_cats_sorted":[],"title_canon_sha256":"9b1779bc25cde7042663bd6f2197983361de70d1729798aa7bd0ce52d7732a86","abstract_canon_sha256":"41984b37edef9444afb80c7927bc61198c4b5481c01ea83c38a994ea08f98cce"},"schema_version":"1.0"},"receipt":{"kind":"pith_receipt","key_id":"pith-v1-2026-05","algorithm":"ed25519","signed_at":"2026-07-05T08:11:29.830859Z","signature_b64":"eTw3OpkP9vm4n7l/8sGZ9Si8FeZy7x21xn5lNvQbp88OlXz6Ki32a2VqXhdc7htdM4sxv2aFhevCQ9T+2z4xCg==","signed_message":"canonical_sha256_bytes","builder_version":"pith-number-builder-2026-05-17-v1","receipt_version":"0.3","canonical_sha256":"d96972b6ea0d0641d5ee49bf6ceee9f40eb8a000cbd917c141634dd9f4c7cc92","last_reissued_at":"2026-07-05T08:11:29.830376Z","signature_status":"signed_v1","first_computed_at":"2026-07-05T08:11:29.830376Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"graph_snapshot":{"paper":{"title":"Retrieval Head Mechanistically Explains Long-Context Factuality","license":"http://creativecommons.org/licenses/by/4.0/","headline":"","cross_cats":[],"primary_cat":"cs.CL","authors_text":"Guangxuan Xiao, Hao Peng, Wenhao Wu, Yao Fu, Yizhong Wang","submitted_at":"2024-04-24T00:24:03Z","abstract_excerpt":"Despite the recent progress in long-context language models, it remains elusive how transformer-based models exhibit the capability to retrieve relevant information from arbitrary locations within the long context. This paper aims to address this question. Our systematic investigation across a wide spectrum of models reveals that a special type of attention heads are largely responsible for retrieving information, which we dub retrieval heads. We identify intriguing properties of retrieval heads:(1) universal: all the explored models with long-context capability have a set of retrieval heads; "},"claims":{"count":0,"items":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"2404.15574","kind":"arxiv","version":1},"verdict":{"id":null,"model_set":{},"created_at":null,"strongest_claim":"","one_line_summary":"","pipeline_version":null,"weakest_assumption":"","pith_extraction_headline":""},"integrity":{"clean":true,"summary":{"advisory":0,"critical":0,"by_detector":{},"informational":0},"endpoint":"/pith/2404.15574/integrity.json","findings":[],"available":true,"detectors_run":[],"snapshot_sha256":"c28c3603d3b5d939e8dc4c7e95fa8dfce3d595e45f758748cecf8e644a296938"},"references":{"count":0,"sample":[],"resolved_work":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","internal_anchors":0},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"author_claims":{"count":0,"strong_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"builder_version":"pith-number-builder-2026-05-17-v1"},"aliases":[{"alias_kind":"arxiv","alias_value":"2404.15574","created_at":"2026-07-05T08:11:29.830437+00:00"},{"alias_kind":"arxiv_version","alias_value":"2404.15574v1","created_at":"2026-07-05T08:11:29.830437+00:00"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.2404.15574","created_at":"2026-07-05T08:11:29.830437+00:00"},{"alias_kind":"pith_short_12","alias_value":"3FUXFNXKBUDE","created_at":"2026-07-05T08:11:29.830437+00:00"},{"alias_kind":"pith_short_16","alias_value":"3FUXFNXKBUDEDVPO","created_at":"2026-07-05T08:11:29.830437+00:00"},{"alias_kind":"pith_short_8","alias_value":"3FUXFNXK","created_at":"2026-07-05T08:11:29.830437+00:00"}],"events":[],"event_summary":{},"paper_claims":[],"inbound_citations":{"count":23,"internal_anchor_count":0,"sample":[{"citing_arxiv_id":"2606.24937","citing_title":"The Hitchhiker's Guide to Agentic AI: From Foundations to Systems","ref_index":76,"is_internal_anchor":false},{"citing_arxiv_id":"2606.22953","citing_title":"Plans Don't Persist: Why Context Management Is Load Bearing for LLM Agents","ref_index":16,"is_internal_anchor":false},{"citing_arxiv_id":"2607.01538","citing_title":"Can Language Models Actually Retrieve In-Context? Drowning in Documents at Million Token Scale","ref_index":40,"is_internal_anchor":false},{"citing_arxiv_id":"2606.12412","citing_title":"Reroute, Don't Remove: Recoverable Visual Token Routing for Vision-Language Models","ref_index":68,"is_internal_anchor":false},{"citing_arxiv_id":"2606.03022","citing_title":"Hallucinations as Orthogonal Noise: Inference-Time Manifold Alignment via Dynamic Contextual Orthogonalization","ref_index":76,"is_internal_anchor":false},{"citing_arxiv_id":"2606.31519","citing_title":"RaBitQCache: Rotated Binary Quantization for KVCache in Long Context LLM Inference","ref_index":31,"is_internal_anchor":false},{"citing_arxiv_id":"2605.30022","citing_title":"Give it Space! Explicit Disentangling of Positional and Semantic Representations in Encoders","ref_index":34,"is_internal_anchor":false},{"citing_arxiv_id":"2606.06840","citing_title":"Characterize Then Distill: Mechanistic Reasoning in Large Output Spaces","ref_index":42,"is_internal_anchor":false},{"citing_arxiv_id":"2605.22870","citing_title":"The Readout Shortcut: Positional Number Copying Dominates Arithmetic CoT Readout in Small Language Models","ref_index":38,"is_internal_anchor":false},{"citing_arxiv_id":"2602.18196","citing_title":"RAT+: Train Dense, Infer Sparse -- Recurrence Augmented Attention for Dilated Inference","ref_index":30,"is_internal_anchor":false},{"citing_arxiv_id":"2507.20906","citing_title":"Soft Head Selection for Injecting ICL-Derived Task Embeddings","ref_index":21,"is_internal_anchor":false},{"citing_arxiv_id":"2512.12087","citing_title":"BLASST: Dynamic BLocked Attention Sparsity via Softmax Thresholding","ref_index":20,"is_internal_anchor":false},{"citing_arxiv_id":"2602.18196","citing_title":"RAT+: Train Dense, Infer Sparse -- Recurrence Augmented Attention for Dilated Inference","ref_index":30,"is_internal_anchor":false},{"citing_arxiv_id":"2604.01432","citing_title":"Are Finer Citations Always Better? Rethinking Granularity for Attributed Generation","ref_index":3,"is_internal_anchor":false},{"citing_arxiv_id":"2605.12110","citing_title":"AB-Sparse: Sparse Attention with Adaptive Block Size for Accurate and Efficient Long-Context Inference","ref_index":19,"is_internal_anchor":false},{"citing_arxiv_id":"2406.02069","citing_title":"PyramidKV: Dynamic KV Cache Compression based on Pyramidal Information Funneling","ref_index":22,"is_internal_anchor":false},{"citing_arxiv_id":"2604.19260","citing_title":"Understanding the Mechanism of Altruism in Large Language Models","ref_index":195,"is_internal_anchor":false},{"citing_arxiv_id":"2604.18487","citing_title":"Adversarial Humanities Benchmark: Results on Stylistic Robustness in Frontier Model Safety","ref_index":18,"is_internal_anchor":false},{"citing_arxiv_id":"2604.07394","citing_title":"Flux Attention: Context-Aware Hybrid Attention for Efficient LLMs Inference","ref_index":43,"is_internal_anchor":false},{"citing_arxiv_id":"2604.06393","citing_title":"ART: Attention Replacement Technique to Improve Factuality in LLMs","ref_index":10,"is_internal_anchor":false},{"citing_arxiv_id":"2604.14149","citing_title":"One Token per Highly Selective Frame: Towards Extreme Compression for Long Video Understanding","ref_index":66,"is_internal_anchor":false},{"citing_arxiv_id":"2604.15583","citing_title":"SAGE: Selective Attention-Guided Extraction for Token-Efficient Document Indexing","ref_index":46,"is_internal_anchor":false},{"citing_arxiv_id":"2605.04641","citing_title":"CAST: Mitigating Object Hallucination in Large Vision-Language Models via Caption-Guided Visual Attention Steering","ref_index":62,"is_internal_anchor":false}]},"formal_canon":{"evidence_count":0,"sample":[],"anchors":[]},"links":{"html":"https://pith.science/pith/3FUXFNXKBUDEDVPOJG7WZ3XJ6Q","json":"https://pith.science/pith/3FUXFNXKBUDEDVPOJG7WZ3XJ6Q.json","graph_json":"https://pith.science/api/pith-number/3FUXFNXKBUDEDVPOJG7WZ3XJ6Q/graph.json","events_json":"https://pith.science/api/pith-number/3FUXFNXKBUDEDVPOJG7WZ3XJ6Q/events.json","paper":"https://pith.science/paper/3FUXFNXK"},"agent_actions":{"view_html":"https://pith.science/pith/3FUXFNXKBUDEDVPOJG7WZ3XJ6Q","download_json":"https://pith.science/pith/3FUXFNXKBUDEDVPOJG7WZ3XJ6Q.json","view_paper":"https://pith.science/paper/3FUXFNXK","resolve_alias":"https://pith.science/api/pith-number/resolve?arxiv=2404.15574&json=true","fetch_graph":"https://pith.science/api/pith-number/3FUXFNXKBUDEDVPOJG7WZ3XJ6Q/graph.json","fetch_events":"https://pith.science/api/pith-number/3FUXFNXKBUDEDVPOJG7WZ3XJ6Q/events.json","actions":{"anchor_timestamp":"https://pith.science/pith/3FUXFNXKBUDEDVPOJG7WZ3XJ6Q/action/timestamp_anchor","attest_storage":"https://pith.science/pith/3FUXFNXKBUDEDVPOJG7WZ3XJ6Q/action/storage_attestation","attest_author":"https://pith.science/pith/3FUXFNXKBUDEDVPOJG7WZ3XJ6Q/action/author_attestation","sign_citation":"https://pith.science/pith/3FUXFNXKBUDEDVPOJG7WZ3XJ6Q/action/citation_signature","submit_replication":"https://pith.science/pith/3FUXFNXKBUDEDVPOJG7WZ3XJ6Q/action/replication_record"}},"created_at":"2026-07-05T08:11:29.830437+00:00","updated_at":"2026-07-05T08:11:29.830437+00:00"}