{"record_type":"pith_number_record","schema_url":"https://pith.science/schemas/pith-number/v1.json","pith_number":"pith:2024:LE7ARFCNHEWEZEENCZ3Z4QA3V5","short_pith_number":"pith:LE7ARFCN","schema_version":"1.0","canonical_sha256":"593e08944d392c4c908d16779e401baf6845fa73cb450646cc58fec8f40735bd","source":{"kind":"arxiv","id":"2410.10594","version":2},"attestation_state":"computed","paper":{"title":"VisRAG: Vision-based Retrieval-augmented Generation on Multi-modality Documents","license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","headline":"VisRAG retrieves and generates from multi-modal documents by embedding them directly as images rather than parsing to text.","cross_cats":["cs.AI","cs.CL","cs.CV"],"primary_cat":"cs.IR","authors_text":"Bokai Xu, Chaoyue Tang, Junbo Cui, Junhao Ran, Maosong Sun, Shi Yu, Shuo Wang, Xu Han, Yukun Yan, Zhenghao Liu, Zhiyuan Liu","submitted_at":"2024-10-14T15:04:18Z","abstract_excerpt":"Retrieval-augmented generation (RAG) is an effective technique that enables large language models (LLMs) to utilize external knowledge sources for generation. However, current RAG systems are solely based on text, rendering it impossible to utilize vision information like layout and images that play crucial roles in real-world multi-modality documents. In this paper, we introduce VisRAG, which tackles this issue by establishing a vision-language model (VLM)-based RAG pipeline. In this pipeline, instead of first parsing the document to obtain text, the document is directly embedded using a VLM "},"verification_status":{"content_addressed":true,"pith_receipt":true,"author_attested":false,"weak_author_claims":0,"strong_author_claims":0,"externally_anchored":false,"storage_verified":false,"citation_signatures":0,"replication_records":0,"graph_snapshot":true,"references_resolved":true,"formal_links_present":true},"canonical_record":{"source":{"id":"2410.10594","kind":"arxiv","version":2},"metadata":{"license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","primary_cat":"cs.IR","submitted_at":"2024-10-14T15:04:18Z","cross_cats_sorted":["cs.AI","cs.CL","cs.CV"],"title_canon_sha256":"06d798e0973d1a421d2517422dcf3d932d2229c42b4b5b9dc66fd712adbdc73e","abstract_canon_sha256":"ef855c401c9db5f58828228443d2d54b7befe49e7a2d658a3c722ed3ecc37174"},"schema_version":"1.0"},"receipt":{"kind":"pith_receipt","key_id":"pith-v1-2026-05","algorithm":"ed25519","signed_at":"2026-05-17T23:38:47.418750Z","signature_b64":"95gO1Rxm2FcNIi18xdDitQ515HjdXfi38YDN1wap3iFrQ6s1RgWJtS3LTRTM7WLoqLd3tlv+OiuskEvTlKq7Cw==","signed_message":"canonical_sha256_bytes","builder_version":"pith-number-builder-2026-05-17-v1","receipt_version":"0.3","canonical_sha256":"593e08944d392c4c908d16779e401baf6845fa73cb450646cc58fec8f40735bd","last_reissued_at":"2026-05-17T23:38:47.418247Z","signature_status":"signed_v1","first_computed_at":"2026-05-17T23:38:47.418247Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"graph_snapshot":{"paper":{"title":"VisRAG: Vision-based Retrieval-augmented Generation on Multi-modality Documents","license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","headline":"VisRAG retrieves and generates from multi-modal documents by embedding them directly as images rather than parsing to text.","cross_cats":["cs.AI","cs.CL","cs.CV"],"primary_cat":"cs.IR","authors_text":"Bokai Xu, Chaoyue Tang, Junbo Cui, Junhao Ran, Maosong Sun, Shi Yu, Shuo Wang, Xu Han, Yukun Yan, Zhenghao Liu, Zhiyuan Liu","submitted_at":"2024-10-14T15:04:18Z","abstract_excerpt":"Retrieval-augmented generation (RAG) is an effective technique that enables large language models (LLMs) to utilize external knowledge sources for generation. However, current RAG systems are solely based on text, rendering it impossible to utilize vision information like layout and images that play crucial roles in real-world multi-modality documents. In this paper, we introduce VisRAG, which tackles this issue by establishing a vision-language model (VLM)-based RAG pipeline. In this pipeline, instead of first parsing the document to obtain text, the document is directly embedded using a VLM "},"claims":{"count":4,"items":[{"kind":"strongest_claim","text":"Experiments demonstrate that VisRAG outperforms traditional RAG in both the retrieval and generation stages, achieving a 20--40% end-to-end performance gain over traditional text-based RAG pipeline.","source":"verdict.strongest_claim","status":"machine_extracted","claim_id":"C1","attestation":"unclaimed"},{"kind":"weakest_assumption","text":"That vision-language models can reliably embed and retrieve relevant information directly from document images without text parsing, and that the collected open-source plus synthetic training data generalizes to unseen real-world multi-modality documents.","source":"verdict.weakest_assumption","status":"machine_extracted","claim_id":"C2","attestation":"unclaimed"},{"kind":"one_line_summary","text":"VisRAG achieves 20-40% better end-to-end performance than text-based RAG by directly embedding and retrieving document images with VLMs.","source":"verdict.one_line_summary","status":"machine_extracted","claim_id":"C3","attestation":"unclaimed"},{"kind":"headline","text":"VisRAG retrieves and generates from multi-modal documents by embedding them directly as images rather than parsing to text.","source":"verdict.pith_extraction.headline","status":"machine_extracted","claim_id":"C4","attestation":"unclaimed"}],"snapshot_sha256":"b322cd4defbaa2d37499b4c3fd6488d49179f019f5515278d81ce571cbe6333c"},"source":{"id":"2410.10594","kind":"arxiv","version":2},"verdict":{"id":"6c6e11ae-29c1-4f68-86c2-5cc11845d6ed","model_set":{"reader":"grok-4.3"},"created_at":"2026-05-16T15:33:33.637224Z","strongest_claim":"Experiments demonstrate that VisRAG outperforms traditional RAG in both the retrieval and generation stages, achieving a 20--40% end-to-end performance gain over traditional text-based RAG pipeline.","one_line_summary":"VisRAG achieves 20-40% better end-to-end performance than text-based RAG by directly embedding and retrieving document images with VLMs.","pipeline_version":"pith-pipeline@v0.9.0","weakest_assumption":"That vision-language models can reliably embed and retrieve relevant information directly from document images without text parsing, and that the collected open-source plus synthetic training data generalizes to unseen real-world multi-modality documents.","pith_extraction_headline":"VisRAG retrieves and generates from multi-modal documents by embedding them directly as images rather than parsing to text."},"references":{"count":43,"sample":[{"doi":"","year":null,"title":"GPT-4 Technical Report","work_id":"b928e041-6991-4c08-8c81-0359e4097c7b","ref_index":1,"cited_arxiv_id":"2303.08774","is_internal_anchor":true},{"doi":"","year":2023,"title":"A multitask, multilingual, multimodal evaluation of chatgpt on reasoning, hallucination, and interactivity","work_id":"1438d57e-4867-4699-8c77-b4fdb11e5d85","ref_index":2,"cited_arxiv_id":"","is_internal_anchor":false},{"doi":"","year":null,"title":"Allava: Harness- ing gpt4v-synthesized data for a lite vision-language model","work_id":"4cf5f4e3-c59a-4ccb-a655-563157a9ce74","ref_index":3,"cited_arxiv_id":"","is_internal_anchor":false},{"doi":"","year":2009,"title":"PP-OCR: A practical ultra lightweight OCR system.CoRR, abs/2009.09941","work_id":"555e4547-6e8f-49f5-acf4-0adae9b359e3","ref_index":4,"cited_arxiv_id":"","is_internal_anchor":false},{"doi":"","year":null,"title":"ColPali: Efficient Document Retrieval with Vision Language Models","work_id":"d2468d08-90dc-4690-887a-9b10a6d3574e","ref_index":5,"cited_arxiv_id":"2407.01449","is_internal_anchor":true}],"resolved_work":43,"snapshot_sha256":"7f5fed8f5fd4cda713d7c3cda7c3c4a24712e79509c6365858410fab1fa160c8","internal_anchors":11},"formal_canon":{"evidence_count":1,"snapshot_sha256":"06f901e2d13a8df9d09f1133efd5dab3dfdb4e98ddad94c2d634fa17de4a11bc"},"author_claims":{"count":0,"strong_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"builder_version":"pith-number-builder-2026-05-17-v1"},"aliases":[{"alias_kind":"arxiv","alias_value":"2410.10594","created_at":"2026-05-17T23:38:47.418321+00:00"},{"alias_kind":"arxiv_version","alias_value":"2410.10594v2","created_at":"2026-05-17T23:38:47.418321+00:00"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.2410.10594","created_at":"2026-05-17T23:38:47.418321+00:00"},{"alias_kind":"pith_short_12","alias_value":"LE7ARFCNHEWE","created_at":"2026-05-18T12:33:37.589309+00:00"},{"alias_kind":"pith_short_16","alias_value":"LE7ARFCNHEWEZEEN","created_at":"2026-05-18T12:33:37.589309+00:00"},{"alias_kind":"pith_short_8","alias_value":"LE7ARFCN","created_at":"2026-05-18T12:33:37.589309+00:00"}],"events":[],"event_summary":{},"paper_claims":[],"inbound_citations":{"count":25,"internal_anchor_count":25,"sample":[{"citing_arxiv_id":"2410.21169","citing_title":"Document Parsing Unveiled: Techniques, Challenges, and Prospects for Structured Information Extraction","ref_index":285,"is_internal_anchor":true},{"citing_arxiv_id":"2605.16671","citing_title":"Sustainable Intelligence for the Wild: Democratizing Ecological Monitoring via Knowledge-Adaptive Edge Expert Agents","ref_index":39,"is_internal_anchor":true},{"citing_arxiv_id":"2505.22095","citing_title":"Mixture-of-Retrieval Experts for Reasoning-Guided Multimodal Knowledge Exploitation","ref_index":42,"is_internal_anchor":true},{"citing_arxiv_id":"2507.04590","citing_title":"VLM2Vec-V2: Advancing Multimodal Embedding for Videos, Images, and Visual Documents","ref_index":29,"is_internal_anchor":true},{"citing_arxiv_id":"2506.20670","citing_title":"MMSearch-R1: Incentivizing LMMs to Search","ref_index":64,"is_internal_anchor":true},{"citing_arxiv_id":"2605.12882","citing_title":"CiteVQA: Benchmarking Evidence Attribution for Trustworthy Document Intelligence","ref_index":48,"is_internal_anchor":true},{"citing_arxiv_id":"2604.02073","citing_title":"PLUME: Latent Reasoning Based Universal Multimodal Embedding","ref_index":49,"is_internal_anchor":true},{"citing_arxiv_id":"2605.08133","citing_title":"VLADriver-RAG: Retrieval-Augmented Vision-Language-Action Models for Autonomous Driving","ref_index":19,"is_internal_anchor":true},{"citing_arxiv_id":"2605.08133","citing_title":"VLADriver-RAG: Retrieval-Augmented Vision-Language-Action Models for Autonomous Driving","ref_index":19,"is_internal_anchor":true},{"citing_arxiv_id":"2604.12812","citing_title":"DocSeeker: Structured Visual Reasoning with Evidence Grounding for Long Document Understanding","ref_index":17,"is_internal_anchor":true},{"citing_arxiv_id":"2605.09271","citing_title":"Shaping Schema via Language Representation as the Next Frontier for LLM Intelligence Expanding","ref_index":114,"is_internal_anchor":true},{"citing_arxiv_id":"2605.10120","citing_title":"MicroWorld: Empowering Multimodal Large Language Models to Bridge the Microscopic Domain Gap with Multimodal Attribute Graph","ref_index":24,"is_internal_anchor":true},{"citing_arxiv_id":"2604.13710","citing_title":"SLQ: Bridging Modalities via Shared Latent Queries for Retrieval with Frozen MLLMs","ref_index":41,"is_internal_anchor":true},{"citing_arxiv_id":"2604.22281","citing_title":"DocPrune:Efficient Document Question Answering via Background, Question, and Comprehension-aware Token Pruning","ref_index":37,"is_internal_anchor":true},{"citing_arxiv_id":"2604.22280","citing_title":"Beyond Chain-of-Thought: Rewrite as a Universal Interface for Generative Multimodal Embeddings","ref_index":49,"is_internal_anchor":true},{"citing_arxiv_id":"2604.11095","citing_title":"Bottleneck Tokens for Unified Multimodal Retrieval","ref_index":25,"is_internal_anchor":true},{"citing_arxiv_id":"2604.10167","citing_title":"Visual Late Chunking: An Empirical Study of Contextual Chunking for Efficient Visual Document Retrieval","ref_index":46,"is_internal_anchor":true},{"citing_arxiv_id":"2604.12812","citing_title":"DocSeeker: Structured Visual Reasoning with Evidence Grounding for Long Document Understanding","ref_index":17,"is_internal_anchor":true},{"citing_arxiv_id":"2604.07220","citing_title":"HIVE: Query, Hypothesize, Verify An LLM Framework for Multimodal Reasoning-Intensive Retrieval","ref_index":38,"is_internal_anchor":true},{"citing_arxiv_id":"2604.07201","citing_title":"BRIDGE: Multimodal-to-Text Retrieval via Reinforcement-Learned Query Alignment","ref_index":43,"is_internal_anchor":true},{"citing_arxiv_id":"2604.09508","citing_title":"VISOR: Agentic Visual Retrieval-Augmented Generation via Iterative Search and Over-horizon Reasoning","ref_index":31,"is_internal_anchor":true},{"citing_arxiv_id":"2604.07079","citing_title":"MARVEL: Multimodal Adaptive Reasoning-intensiVe Expand-rerank and retrievaL","ref_index":35,"is_internal_anchor":true},{"citing_arxiv_id":"2604.04901","citing_title":"FileGram: Grounding Agent Personalization in File-System Behavioral Traces","ref_index":26,"is_internal_anchor":true},{"citing_arxiv_id":"2604.13710","citing_title":"SLQ: Bridging Modalities via Shared Latent Queries for Retrieval with Frozen MLLMs","ref_index":41,"is_internal_anchor":true},{"citing_arxiv_id":"2604.14029","citing_title":"POINTS-Seeker: Towards Training a Multimodal Agentic Search Model from Scratch","ref_index":59,"is_internal_anchor":true}]},"formal_canon":{"evidence_count":1,"sample":[],"anchors":[]},"links":{"html":"https://pith.science/pith/LE7ARFCNHEWEZEENCZ3Z4QA3V5","json":"https://pith.science/pith/LE7ARFCNHEWEZEENCZ3Z4QA3V5.json","graph_json":"https://pith.science/api/pith-number/LE7ARFCNHEWEZEENCZ3Z4QA3V5/graph.json","events_json":"https://pith.science/api/pith-number/LE7ARFCNHEWEZEENCZ3Z4QA3V5/events.json","paper":"https://pith.science/paper/LE7ARFCN"},"agent_actions":{"view_html":"https://pith.science/pith/LE7ARFCNHEWEZEENCZ3Z4QA3V5","download_json":"https://pith.science/pith/LE7ARFCNHEWEZEENCZ3Z4QA3V5.json","view_paper":"https://pith.science/paper/LE7ARFCN","resolve_alias":"https://pith.science/api/pith-number/resolve?arxiv=2410.10594&json=true","fetch_graph":"https://pith.science/api/pith-number/LE7ARFCNHEWEZEENCZ3Z4QA3V5/graph.json","fetch_events":"https://pith.science/api/pith-number/LE7ARFCNHEWEZEENCZ3Z4QA3V5/events.json","actions":{"anchor_timestamp":"https://pith.science/pith/LE7ARFCNHEWEZEENCZ3Z4QA3V5/action/timestamp_anchor","attest_storage":"https://pith.science/pith/LE7ARFCNHEWEZEENCZ3Z4QA3V5/action/storage_attestation","attest_author":"https://pith.science/pith/LE7ARFCNHEWEZEENCZ3Z4QA3V5/action/author_attestation","sign_citation":"https://pith.science/pith/LE7ARFCNHEWEZEENCZ3Z4QA3V5/action/citation_signature","submit_replication":"https://pith.science/pith/LE7ARFCNHEWEZEENCZ3Z4QA3V5/action/replication_record"}},"created_at":"2026-05-17T23:38:47.418321+00:00","updated_at":"2026-05-17T23:38:47.418321+00:00"}