{"record_type":"pith_number_record","schema_url":"https://pith.science/schemas/pith-number/v1.json","pith_number":"pith:2024:2RNB4XOL3YVZASQJYUM64EOXCH","short_pith_number":"pith:2RNB4XOL","schema_version":"1.0","canonical_sha256":"d45a1e5dcbde2b904a09c519ee11d711c2eb035c50f19544e4f5ceef6411637a","source":{"kind":"arxiv","id":"2405.14880","version":4},"attestation_state":"computed","paper":{"title":"Dissecting Query-Key Interaction in Vision Transformers","license":"http://creativecommons.org/licenses/by-nc-sa/4.0/","headline":"","cross_cats":["cs.AI"],"primary_cat":"cs.CV","authors_text":"Aaron Philip, Odelia Schwartz, Xu Pan, Ziqian Xie","submitted_at":"2024-04-04T20:06:07Z","abstract_excerpt":"Self-attention in vision transformers is often thought to perform perceptual grouping where tokens attend to other tokens with similar embeddings, which could correspond to semantically similar features of an object. However, attending to dissimilar tokens can be beneficial by providing contextual information. We propose to analyze the query-key interaction by the singular value decomposition of the interaction matrix (i.e. ${\\textbf{W}_q}^\\top\\textbf{W}_k$). We find that in many ViTs, especially those with classification training objectives, early layers attend more to similar tokens, while l"},"verification_status":{"content_addressed":true,"pith_receipt":true,"author_attested":false,"weak_author_claims":0,"strong_author_claims":0,"externally_anchored":false,"storage_verified":false,"citation_signatures":0,"replication_records":0,"graph_snapshot":true,"references_resolved":false,"formal_links_present":false},"canonical_record":{"source":{"id":"2405.14880","kind":"arxiv","version":4},"metadata":{"license":"http://creativecommons.org/licenses/by-nc-sa/4.0/","primary_cat":"cs.CV","submitted_at":"2024-04-04T20:06:07Z","cross_cats_sorted":["cs.AI"],"title_canon_sha256":"122b1ef7ed2d22a9b22dabe161dcc4fac987683e1f83294c0913a117684dcb17","abstract_canon_sha256":"13d2c249f42a4083f8141e025fe318df265ddf44b1aa925515bbdc3a9e73ab29"},"schema_version":"1.0"},"receipt":{"kind":"pith_receipt","key_id":"pith-v1-2026-05","algorithm":"ed25519","signed_at":"2026-07-05T10:00:37.301882Z","signature_b64":"xpwY3uFfoIDiTkZh+/8Qri/mYSAkKiVaiHA8qhO7COyrUOITYLFPpoQj+zkeeoJmIQ0pemosymnCtk5EI9jZAg==","signed_message":"canonical_sha256_bytes","builder_version":"pith-number-builder-2026-05-17-v1","receipt_version":"0.3","canonical_sha256":"d45a1e5dcbde2b904a09c519ee11d711c2eb035c50f19544e4f5ceef6411637a","last_reissued_at":"2026-07-05T10:00:37.301412Z","signature_status":"signed_v1","first_computed_at":"2026-07-05T10:00:37.301412Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"graph_snapshot":{"paper":{"title":"Dissecting Query-Key Interaction in Vision Transformers","license":"http://creativecommons.org/licenses/by-nc-sa/4.0/","headline":"","cross_cats":["cs.AI"],"primary_cat":"cs.CV","authors_text":"Aaron Philip, Odelia Schwartz, Xu Pan, Ziqian Xie","submitted_at":"2024-04-04T20:06:07Z","abstract_excerpt":"Self-attention in vision transformers is often thought to perform perceptual grouping where tokens attend to other tokens with similar embeddings, which could correspond to semantically similar features of an object. However, attending to dissimilar tokens can be beneficial by providing contextual information. We propose to analyze the query-key interaction by the singular value decomposition of the interaction matrix (i.e. ${\\textbf{W}_q}^\\top\\textbf{W}_k$). We find that in many ViTs, especially those with classification training objectives, early layers attend more to similar tokens, while l"},"claims":{"count":0,"items":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"2405.14880","kind":"arxiv","version":4},"verdict":{"id":null,"model_set":{},"created_at":null,"strongest_claim":"","one_line_summary":"","pipeline_version":null,"weakest_assumption":"","pith_extraction_headline":""},"integrity":{"clean":true,"summary":{"advisory":0,"critical":0,"by_detector":{},"informational":0},"endpoint":"/pith/2405.14880/integrity.json","findings":[],"available":true,"detectors_run":[],"snapshot_sha256":"c28c3603d3b5d939e8dc4c7e95fa8dfce3d595e45f758748cecf8e644a296938"},"references":{"count":0,"sample":[],"resolved_work":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","internal_anchors":0},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"author_claims":{"count":0,"strong_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"builder_version":"pith-number-builder-2026-05-17-v1"},"aliases":[{"alias_kind":"arxiv","alias_value":"2405.14880","created_at":"2026-07-05T10:00:37.301470+00:00"},{"alias_kind":"arxiv_version","alias_value":"2405.14880v4","created_at":"2026-07-05T10:00:37.301470+00:00"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.2405.14880","created_at":"2026-07-05T10:00:37.301470+00:00"},{"alias_kind":"pith_short_12","alias_value":"2RNB4XOL3YVZ","created_at":"2026-07-05T10:00:37.301470+00:00"},{"alias_kind":"pith_short_16","alias_value":"2RNB4XOL3YVZASQJ","created_at":"2026-07-05T10:00:37.301470+00:00"},{"alias_kind":"pith_short_8","alias_value":"2RNB4XOL","created_at":"2026-07-05T10:00:37.301470+00:00"}],"events":[],"event_summary":{},"paper_claims":[],"inbound_citations":{"count":2,"internal_anchor_count":0,"sample":[{"citing_arxiv_id":"2606.22182","citing_title":"Dual-Stream EEG Decoding for 3D Visual Perception","ref_index":271,"is_internal_anchor":false},{"citing_arxiv_id":"2604.11005","citing_title":"Diffusion-CAM: Faithful Visual Explanations for dMLLMs","ref_index":2,"is_internal_anchor":false}]},"formal_canon":{"evidence_count":0,"sample":[],"anchors":[]},"links":{"html":"https://pith.science/pith/2RNB4XOL3YVZASQJYUM64EOXCH","json":"https://pith.science/pith/2RNB4XOL3YVZASQJYUM64EOXCH.json","graph_json":"https://pith.science/api/pith-number/2RNB4XOL3YVZASQJYUM64EOXCH/graph.json","events_json":"https://pith.science/api/pith-number/2RNB4XOL3YVZASQJYUM64EOXCH/events.json","paper":"https://pith.science/paper/2RNB4XOL"},"agent_actions":{"view_html":"https://pith.science/pith/2RNB4XOL3YVZASQJYUM64EOXCH","download_json":"https://pith.science/pith/2RNB4XOL3YVZASQJYUM64EOXCH.json","view_paper":"https://pith.science/paper/2RNB4XOL","resolve_alias":"https://pith.science/api/pith-number/resolve?arxiv=2405.14880&json=true","fetch_graph":"https://pith.science/api/pith-number/2RNB4XOL3YVZASQJYUM64EOXCH/graph.json","fetch_events":"https://pith.science/api/pith-number/2RNB4XOL3YVZASQJYUM64EOXCH/events.json","actions":{"anchor_timestamp":"https://pith.science/pith/2RNB4XOL3YVZASQJYUM64EOXCH/action/timestamp_anchor","attest_storage":"https://pith.science/pith/2RNB4XOL3YVZASQJYUM64EOXCH/action/storage_attestation","attest_author":"https://pith.science/pith/2RNB4XOL3YVZASQJYUM64EOXCH/action/author_attestation","sign_citation":"https://pith.science/pith/2RNB4XOL3YVZASQJYUM64EOXCH/action/citation_signature","submit_replication":"https://pith.science/pith/2RNB4XOL3YVZASQJYUM64EOXCH/action/replication_record"}},"created_at":"2026-07-05T10:00:37.301470+00:00","updated_at":"2026-07-05T10:00:37.301470+00:00"}