{"work":{"id":"64030c49-afa6-4cf0-b64c-831d19d6674e","openalex_id":null,"doi":null,"arxiv_id":"2412.04454","raw_key":null,"title":"Aguvis: Unified Pure Vision Agents for Autonomous GUI Interaction","authors":null,"authors_text":"Yiheng Xu, Zekun Wang, Junli Wang, Dunjie Lu, Tianbao Xie, Amrita Saha","year":2024,"venue":"cs.CL","abstract":"Automating GUI tasks remains challenging due to reliance on textual representations, platform-specific action spaces, and limited reasoning capabilities. We introduce Aguvis, a unified vision-based framework for autonomous GUI agents that directly operates on screen images, standardizes cross-platform interactions and incorporates structured reasoning via inner monologue. To enable this, we construct Aguvis Data Collection, a large-scale dataset with multimodal grounding and reasoning annotations, and develop a two-stage training pipeline that separates GUI grounding from planning and reasoning. Experiments show that Aguvis achieves state-of-the-art performance across offline and real-world online benchmarks, marking the first fully autonomous vision-based GUI agent that operates without closed-source models. We open-source all datasets, models, and training recipes at https://aguvis-project.github.io to advance future research.","external_url":"https://arxiv.org/abs/2412.04454","cited_by_count":null,"metadata_source":"pith","metadata_fetched_at":"2026-07-04T10:39:44.899694+00:00","pith_arxiv_id":"2412.04454","created_at":"2026-05-09T22:39:15.064780+00:00","updated_at":"2026-07-04T10:39:44.899694+00:00","title_quality_ok":true,"display_title":"Aguvis: Unified Pure Vision Agents for Autonomous GUI Interaction","render_title":"Aguvis: Unified Pure Vision Agents for Autonomous GUI Interaction"},"hub":{"state":{"work_id":"64030c49-afa6-4cf0-b64c-831d19d6674e","tier":"hub","tier_reason":"10+ Pith inbound or 1,000+ external citations","pith_inbound_count":41,"external_cited_by_count":null,"distinct_field_count":6,"first_pith_cited_at":"2024-11-27T12:13:39+00:00","last_pith_cited_at":"2026-06-30T07:44:37+00:00","author_build_status":"not_needed","summary_status":"needed","contexts_status":"needed","graph_status":"needed","ask_index_status":"not_needed","reader_status":"not_needed","recognition_status":"not_needed","updated_at":"2026-08-23T07:09:41.632850+00:00","tier_text":"hub"},"tier":"hub","role_counts":[{"context_role":"background","n":4},{"context_role":"baseline","n":3}],"polarity_counts":[{"context_polarity":"background","n":4},{"context_polarity":"baseline","n":3}],"runs":{},"summary":{},"graph":{},"authors":[]}}