{"work":{"id":"ae2ef343-40c0-4828-9efe-e404f5c28b36","openalex_id":null,"doi":null,"arxiv_id":"2505.17015","raw_key":null,"title":"Multi-SpatialMLLM: Multi-Frame Spatial Understanding with Multi-Modal Large Language Models","authors":null,"authors_text":"Runsen Xu, Weiyao Wang, Hao Tang, Xingyu Chen, Xiaodong Wang, Fu-Jen Chu, Dahua Lin, Matt Feiszli, and Kevin J Liang","year":2025,"venue":"cs.CV","abstract":"Multi-modal large language models (MLLMs) have rapidly advanced in visual tasks, yet their spatial understanding remains limited to single images, leaving them ill-suited for physical-world applications that require multi-frame reasoning. In this paper, we propose a framework to equip MLLMs with multi-frame spatial understanding by integrating fundamental spatial skills, including depth perception, visual correspondence, and dynamic perception. We design a novel data pipeline and collect the MultiSPA dataset of more than 27 million samples spanning diverse 3D and 4D scenes to enable training. Alongside MultiSPA, we introduce a comprehensive benchmark that tests a wide spectrum of spatial tasks under uniform metrics. Our resulting model, Multi-SpatialMLLM, achieves significant gains over baselines and proprietary systems, demonstrating scalable and generalizable multi-frame perception. We further observe multi-task benefits and emergent spatial capabilities in challenging scenarios, and showcase how our model can serve as a multi-frame reward annotator for robotics.","external_url":"https://arxiv.org/abs/2505.17015","cited_by_count":null,"metadata_source":"pith","metadata_fetched_at":"2026-07-02T14:17:02.443466+00:00","pith_arxiv_id":"2505.17015","created_at":"2026-05-11T06:20:56.485737+00:00","updated_at":"2026-07-02T14:17:02.443466+00:00","title_quality_ok":true,"display_title":"Multi-spatialmllm: Multi-frame spatial understanding with multi-modal large language models","render_title":"Multi-spatialmllm: Multi-frame spatial understanding with multi-modal large language models"},"hub":{"state":{"work_id":"ae2ef343-40c0-4828-9efe-e404f5c28b36","tier":"hub","tier_reason":"10+ Pith inbound or 1,000+ external citations","pith_inbound_count":13,"external_cited_by_count":null,"distinct_field_count":2,"first_pith_cited_at":"2025-11-06T18:55:17+00:00","last_pith_cited_at":"2026-07-01T12:45:12+00:00","author_build_status":"not_needed","summary_status":"needed","contexts_status":"needed","graph_status":"needed","ask_index_status":"not_needed","reader_status":"not_needed","recognition_status":"not_needed","updated_at":"2026-07-03T04:33:22.323308+00:00","tier_text":"hub"},"tier":"hub","role_counts":[{"context_role":"background","n":4}],"polarity_counts":[{"context_polarity":"background","n":3},{"context_polarity":"unclear","n":1}],"runs":{},"summary":{},"graph":{},"authors":[]}}