{"record_type":"pith_number_record","schema_url":"https://pith.science/schemas/pith-number/v1.json","pith_number":"pith:2025:3EUPNWJMCKOYRD54TKXBBR4TYT","short_pith_number":"pith:3EUPNWJM","schema_version":"1.0","canonical_sha256":"d928f6d92c129d888fbc9aae10c793c4c5ff0f8b102e070652a4f30456a76311","source":{"kind":"arxiv","id":"2503.03321","version":1},"attestation_state":"computed","paper":{"title":"See What You Are Told: Visual Attention Sink in Large Multimodal Models","license":"http://creativecommons.org/licenses/by/4.0/","headline":"","cross_cats":["cs.AI"],"primary_cat":"cs.CV","authors_text":"Jinyeong Kim, Junhyeok Kim, Seil Kang, Seong Jae Hwang","submitted_at":"2025-03-05T09:55:07Z","abstract_excerpt":"Large multimodal models (LMMs) \"see\" images by leveraging the attention mechanism between text and visual tokens in the transformer decoder. Ideally, these models should focus on key visual information relevant to the text token. However, recent findings indicate that LMMs have an extraordinary tendency to consistently allocate high attention weights to specific visual tokens, even when these tokens are irrelevant to the corresponding text. In this study, we investigate the property behind the appearance of these irrelevant visual tokens and examine their characteristics. Our findings show tha"},"verification_status":{"content_addressed":true,"pith_receipt":true,"author_attested":false,"weak_author_claims":0,"strong_author_claims":0,"externally_anchored":false,"storage_verified":false,"citation_signatures":0,"replication_records":0,"graph_snapshot":true,"references_resolved":false,"formal_links_present":false},"canonical_record":{"source":{"id":"2503.03321","kind":"arxiv","version":1},"metadata":{"license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.CV","submitted_at":"2025-03-05T09:55:07Z","cross_cats_sorted":["cs.AI"],"title_canon_sha256":"4d49c75759adac4e570122587e2dcb1d1a11a158aef8706afad3188409564f00","abstract_canon_sha256":"6cdc09a6dd47039c3441bd86870376b745b11c168ffa4f38ae9719e8be62d627"},"schema_version":"1.0"},"receipt":{"kind":"pith_receipt","key_id":"pith-v1-2026-05","algorithm":"ed25519","signed_at":"2026-07-05T10:24:43.335254Z","signature_b64":"1cwhZW3gjynjaKgw7OGmN9UgBERuji+8+TP9XnNyrDmvSgSfyTfVdVLEPs9it2Gmg1ZBZFvF6AwCm/OJy4u+CQ==","signed_message":"canonical_sha256_bytes","builder_version":"pith-number-builder-2026-05-17-v1","receipt_version":"0.3","canonical_sha256":"d928f6d92c129d888fbc9aae10c793c4c5ff0f8b102e070652a4f30456a76311","last_reissued_at":"2026-07-05T10:24:43.334728Z","signature_status":"signed_v1","first_computed_at":"2026-07-05T10:24:43.334728Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"graph_snapshot":{"paper":{"title":"See What You Are Told: Visual Attention Sink in Large Multimodal Models","license":"http://creativecommons.org/licenses/by/4.0/","headline":"","cross_cats":["cs.AI"],"primary_cat":"cs.CV","authors_text":"Jinyeong Kim, Junhyeok Kim, Seil Kang, Seong Jae Hwang","submitted_at":"2025-03-05T09:55:07Z","abstract_excerpt":"Large multimodal models (LMMs) \"see\" images by leveraging the attention mechanism between text and visual tokens in the transformer decoder. Ideally, these models should focus on key visual information relevant to the text token. However, recent findings indicate that LMMs have an extraordinary tendency to consistently allocate high attention weights to specific visual tokens, even when these tokens are irrelevant to the corresponding text. In this study, we investigate the property behind the appearance of these irrelevant visual tokens and examine their characteristics. Our findings show tha"},"claims":{"count":0,"items":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"2503.03321","kind":"arxiv","version":1},"verdict":{"id":null,"model_set":{},"created_at":null,"strongest_claim":"","one_line_summary":"","pipeline_version":null,"weakest_assumption":"","pith_extraction_headline":""},"integrity":{"clean":true,"summary":{"advisory":0,"critical":0,"by_detector":{},"informational":0},"endpoint":"/pith/2503.03321/integrity.json","findings":[],"available":true,"detectors_run":[],"snapshot_sha256":"c28c3603d3b5d939e8dc4c7e95fa8dfce3d595e45f758748cecf8e644a296938"},"references":{"count":0,"sample":[],"resolved_work":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","internal_anchors":0},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"author_claims":{"count":0,"strong_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"builder_version":"pith-number-builder-2026-05-17-v1"},"aliases":[{"alias_kind":"arxiv","alias_value":"2503.03321","created_at":"2026-07-05T10:24:43.334792+00:00"},{"alias_kind":"arxiv_version","alias_value":"2503.03321v1","created_at":"2026-07-05T10:24:43.334792+00:00"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.2503.03321","created_at":"2026-07-05T10:24:43.334792+00:00"},{"alias_kind":"pith_short_12","alias_value":"3EUPNWJMCKOY","created_at":"2026-07-05T10:24:43.334792+00:00"},{"alias_kind":"pith_short_16","alias_value":"3EUPNWJMCKOYRD54","created_at":"2026-07-05T10:24:43.334792+00:00"},{"alias_kind":"pith_short_8","alias_value":"3EUPNWJM","created_at":"2026-07-05T10:24:43.334792+00:00"}],"events":[],"event_summary":{},"paper_claims":[],"inbound_citations":{"count":26,"internal_anchor_count":0,"sample":[{"citing_arxiv_id":"2606.20077","citing_title":"The Hidden Evolution of Disguised Visual Context inside the VLM","ref_index":25,"is_internal_anchor":false},{"citing_arxiv_id":"2606.12412","citing_title":"Reroute, Don't Remove: Recoverable Visual Token Routing for Vision-Language Models","ref_index":30,"is_internal_anchor":false},{"citing_arxiv_id":"2606.14782","citing_title":"Last But Not Least: Boundary Attention CalibratiON for Multimodal KV Cache Compression","ref_index":38,"is_internal_anchor":false},{"citing_arxiv_id":"2606.10147","citing_title":"From Senses to Decisions: The Information Flow of Auditory and Visual Perception in Multimodal LLMs","ref_index":20,"is_internal_anchor":false},{"citing_arxiv_id":"2606.09303","citing_title":"Reason Twice: Segmentation via Candidate Discovery and Comparative Reasoning","ref_index":26,"is_internal_anchor":false},{"citing_arxiv_id":"2607.00434","citing_title":"Information-Regularized Attention for Visual-Centric Reasoning","ref_index":7,"is_internal_anchor":false},{"citing_arxiv_id":"2606.05843","citing_title":"Mechanistic Insights into Functional Sparsity in Multimodal LLMs via CoRe Heads","ref_index":7,"is_internal_anchor":false},{"citing_arxiv_id":"2606.03712","citing_title":"When Graph Tokens Sink: A Mechanistic Analysis of Graph Language Models","ref_index":14,"is_internal_anchor":false},{"citing_arxiv_id":"2605.18359","citing_title":"RAVE: Re-Allocating Visual Attention in Large Multimodal Models","ref_index":3,"is_internal_anchor":false},{"citing_arxiv_id":"2606.31054","citing_title":"ADAPT: Attention Dynamics Alignment with Preference Tuning for Faithful MLLMs","ref_index":10,"is_internal_anchor":false},{"citing_arxiv_id":"2605.25244","citing_title":"Inference Time Optimization with Confidence Dynamics","ref_index":7,"is_internal_anchor":false},{"citing_arxiv_id":"2606.30288","citing_title":"VisReflect: Latent Visual Reflection for Fine-Grained Perception in Long Visual Context","ref_index":14,"is_internal_anchor":false},{"citing_arxiv_id":"2605.25799","citing_title":"Addressing Exacerbated Attention Sink for Source-Free Cross-Domain Few-Shot Learning","ref_index":15,"is_internal_anchor":false},{"citing_arxiv_id":"2605.21954","citing_title":"MLLMs Know When Before Speaking: Revealing and Recovering Temporal Grounding via Attention Cues","ref_index":12,"is_internal_anchor":false},{"citing_arxiv_id":"2510.00054","citing_title":"HiDe: Rethinking The Zoom-IN method in High Resolution MLLMs via Hierarchical Decoupling","ref_index":4,"is_internal_anchor":false},{"citing_arxiv_id":"2511.14159","citing_title":"MVI-Bench: A Comprehensive Benchmark for Evaluating Robustness to Misleading Visual Inputs in LVLMs","ref_index":27,"is_internal_anchor":false},{"citing_arxiv_id":"2603.14184","citing_title":"Deeper Thought, Weaker Aim: Understanding and Mitigating Perceptual Impairment during Reasoning in Multimodal Large Language Models","ref_index":8,"is_internal_anchor":false},{"citing_arxiv_id":"2605.18359","citing_title":"RAVE: Re-Allocating Visual Attention in Large Multimodal Models","ref_index":3,"is_internal_anchor":false},{"citing_arxiv_id":"2511.17722","citing_title":"Can Vision-Language Models Count? A Synthetic Benchmark and Analysis of Attention-Based Interventions","ref_index":12,"is_internal_anchor":false},{"citing_arxiv_id":"2602.17419","citing_title":"EAGLE: Expert-Augmented Attention Guidance for Tuning-Free Industrial Anomaly Detection in Multimodal Large Language Models","ref_index":18,"is_internal_anchor":false},{"citing_arxiv_id":"2604.25273","citing_title":"Combating Visual Neglect and Semantic Drift in Large Multimodal Models for Enhanced Cross-Modal Retrieval","ref_index":17,"is_internal_anchor":false},{"citing_arxiv_id":"2605.05668","citing_title":"Large Vision-Language Models Get Lost in Attention","ref_index":27,"is_internal_anchor":false},{"citing_arxiv_id":"2604.10039","citing_title":"Counting to Four is still a Chore for VLMs","ref_index":17,"is_internal_anchor":false},{"citing_arxiv_id":"2604.14363","citing_title":"The Cost of Language: Centroid Erasure Exposes and Exploits Modal Competition in Multimodal Language Models","ref_index":8,"is_internal_anchor":false},{"citing_arxiv_id":"2604.20937","citing_title":"Sink-Token-Aware Pruning for Fine-Grained Video Understanding in Efficient Video LLMs","ref_index":17,"is_internal_anchor":false}]},"formal_canon":{"evidence_count":0,"sample":[],"anchors":[]},"links":{"html":"https://pith.science/pith/3EUPNWJMCKOYRD54TKXBBR4TYT","json":"https://pith.science/pith/3EUPNWJMCKOYRD54TKXBBR4TYT.json","graph_json":"https://pith.science/api/pith-number/3EUPNWJMCKOYRD54TKXBBR4TYT/graph.json","events_json":"https://pith.science/api/pith-number/3EUPNWJMCKOYRD54TKXBBR4TYT/events.json","paper":"https://pith.science/paper/3EUPNWJM"},"agent_actions":{"view_html":"https://pith.science/pith/3EUPNWJMCKOYRD54TKXBBR4TYT","download_json":"https://pith.science/pith/3EUPNWJMCKOYRD54TKXBBR4TYT.json","view_paper":"https://pith.science/paper/3EUPNWJM","resolve_alias":"https://pith.science/api/pith-number/resolve?arxiv=2503.03321&json=true","fetch_graph":"https://pith.science/api/pith-number/3EUPNWJMCKOYRD54TKXBBR4TYT/graph.json","fetch_events":"https://pith.science/api/pith-number/3EUPNWJMCKOYRD54TKXBBR4TYT/events.json","actions":{"anchor_timestamp":"https://pith.science/pith/3EUPNWJMCKOYRD54TKXBBR4TYT/action/timestamp_anchor","attest_storage":"https://pith.science/pith/3EUPNWJMCKOYRD54TKXBBR4TYT/action/storage_attestation","attest_author":"https://pith.science/pith/3EUPNWJMCKOYRD54TKXBBR4TYT/action/author_attestation","sign_citation":"https://pith.science/pith/3EUPNWJMCKOYRD54TKXBBR4TYT/action/citation_signature","submit_replication":"https://pith.science/pith/3EUPNWJMCKOYRD54TKXBBR4TYT/action/replication_record"}},"created_at":"2026-07-05T10:24:43.334792+00:00","updated_at":"2026-07-05T10:24:43.334792+00:00"}