{"record_type":"pith_number_record","schema_url":"https://pith.science/schemas/pith-number/v1.json","pith_number":"pith:2025:7AUNQAIWAVUT53Q5YJZBY77SRM","short_pith_number":"pith:7AUNQAIW","schema_version":"1.0","canonical_sha256":"f828d8011605693eee1dc2721c7ff28b111cff090d69c09537251b68ef901d27","source":{"kind":"arxiv","id":"2504.11543","version":2},"attestation_state":"computed","paper":{"title":"REAL: Benchmarking Autonomous Agents on Deterministic Simulations of Real Websites","license":"http://creativecommons.org/licenses/by-nc-sa/4.0/","headline":"","cross_cats":[],"primary_cat":"cs.AI","authors_text":"Andis Draguns, Atharva Gundawar, Charles London, Christian Schroeder de Witt, Diego Caples, Divyansh Garg, Federico Lopez, James Liu, Jindong Gu, Michael Lara, Naman Garg, Nikil Ravi, Pranav Putta, Prannay Hebbar, Shaun VanWeelden, Sumeet Motwani, Tomas Abraham, Youngchul Joo","submitted_at":"2025-04-15T18:22:55Z","abstract_excerpt":"We introduce REAL, a benchmark and framework for multi-turn agent evaluations on deterministic simulations of real-world websites. REAL comprises high-fidelity, deterministic replicas of 11 widely-used websites across domains such as e-commerce, travel, communication, and professional networking. We also release a benchmark consisting of 112 practical tasks that mirror everyday complex user interactions requiring both accurate information retrieval and state-changing actions. All interactions occur within this fully controlled setting, eliminating safety risks and enabling robust, reproducible"},"verification_status":{"content_addressed":true,"pith_receipt":true,"author_attested":false,"weak_author_claims":0,"strong_author_claims":0,"externally_anchored":false,"storage_verified":false,"citation_signatures":0,"replication_records":0,"graph_snapshot":true,"references_resolved":false,"formal_links_present":false},"canonical_record":{"source":{"id":"2504.11543","kind":"arxiv","version":2},"metadata":{"license":"http://creativecommons.org/licenses/by-nc-sa/4.0/","primary_cat":"cs.AI","submitted_at":"2025-04-15T18:22:55Z","cross_cats_sorted":[],"title_canon_sha256":"ffeaa362abc98c87c94cf47957341aca1a1613c5b2692ccc0a6059e99c3e1197","abstract_canon_sha256":"6b3c8bb934525cb2b33ecf7f7cbd13b09b7f8cc44b200ea6d36c0335e546a76c"},"schema_version":"1.0"},"receipt":{"kind":"pith_receipt","key_id":"pith-v1-2026-05","algorithm":"ed25519","signed_at":"2026-07-05T10:50:33.621658Z","signature_b64":"mWkb5MZq2st6Kg9xUN9hPMl5/xD1ro6jVyyf1Gdz9UEpoGhDBRic4qoffa5jliDbEoj3x+Gh0GFtchdw4F00DA==","signed_message":"canonical_sha256_bytes","builder_version":"pith-number-builder-2026-05-17-v1","receipt_version":"0.3","canonical_sha256":"f828d8011605693eee1dc2721c7ff28b111cff090d69c09537251b68ef901d27","last_reissued_at":"2026-07-05T10:50:33.621162Z","signature_status":"signed_v1","first_computed_at":"2026-07-05T10:50:33.621162Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"graph_snapshot":{"paper":{"title":"REAL: Benchmarking Autonomous Agents on Deterministic Simulations of Real Websites","license":"http://creativecommons.org/licenses/by-nc-sa/4.0/","headline":"","cross_cats":[],"primary_cat":"cs.AI","authors_text":"Andis Draguns, Atharva Gundawar, Charles London, Christian Schroeder de Witt, Diego Caples, Divyansh Garg, Federico Lopez, James Liu, Jindong Gu, Michael Lara, Naman Garg, Nikil Ravi, Pranav Putta, Prannay Hebbar, Shaun VanWeelden, Sumeet Motwani, Tomas Abraham, Youngchul Joo","submitted_at":"2025-04-15T18:22:55Z","abstract_excerpt":"We introduce REAL, a benchmark and framework for multi-turn agent evaluations on deterministic simulations of real-world websites. REAL comprises high-fidelity, deterministic replicas of 11 widely-used websites across domains such as e-commerce, travel, communication, and professional networking. We also release a benchmark consisting of 112 practical tasks that mirror everyday complex user interactions requiring both accurate information retrieval and state-changing actions. All interactions occur within this fully controlled setting, eliminating safety risks and enabling robust, reproducible"},"claims":{"count":0,"items":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"2504.11543","kind":"arxiv","version":2},"verdict":{"id":null,"model_set":{},"created_at":null,"strongest_claim":"","one_line_summary":"","pipeline_version":null,"weakest_assumption":"","pith_extraction_headline":""},"integrity":{"clean":true,"summary":{"advisory":0,"critical":0,"by_detector":{},"informational":0},"endpoint":"/pith/2504.11543/integrity.json","findings":[],"available":true,"detectors_run":[],"snapshot_sha256":"c28c3603d3b5d939e8dc4c7e95fa8dfce3d595e45f758748cecf8e644a296938"},"references":{"count":0,"sample":[],"resolved_work":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","internal_anchors":0},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"author_claims":{"count":0,"strong_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"builder_version":"pith-number-builder-2026-05-17-v1"},"aliases":[{"alias_kind":"arxiv","alias_value":"2504.11543","created_at":"2026-07-05T10:50:33.621218+00:00"},{"alias_kind":"arxiv_version","alias_value":"2504.11543v2","created_at":"2026-07-05T10:50:33.621218+00:00"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.2504.11543","created_at":"2026-07-05T10:50:33.621218+00:00"},{"alias_kind":"pith_short_12","alias_value":"7AUNQAIWAVUT","created_at":"2026-07-05T10:50:33.621218+00:00"},{"alias_kind":"pith_short_16","alias_value":"7AUNQAIWAVUT53Q5","created_at":"2026-07-05T10:50:33.621218+00:00"},{"alias_kind":"pith_short_8","alias_value":"7AUNQAIW","created_at":"2026-07-05T10:50:33.621218+00:00"}],"events":[],"event_summary":{},"paper_claims":[],"inbound_citations":{"count":14,"internal_anchor_count":1,"sample":[{"citing_arxiv_id":"2607.06118","citing_title":"WebRetriever: A Large-Scale Comprehensive Benchmark for Efficient Web Agent Evaluation","ref_index":16,"is_internal_anchor":true},{"citing_arxiv_id":"2606.19559","citing_title":"Uncertainty Decomposition for Clarification Seeking in LLM Agents","ref_index":45,"is_internal_anchor":false},{"citing_arxiv_id":"2606.06708","citing_title":"Signal-Driven Observation for Long-Horizon Web Agents","ref_index":14,"is_internal_anchor":false},{"citing_arxiv_id":"2605.19149","citing_title":"Agent Meltdowns: The Road to Hell Is Paved with Helpful Agents","ref_index":11,"is_internal_anchor":false},{"citing_arxiv_id":"2508.13024","citing_title":"WebMall -- A Multi-Shop Benchmark for Evaluating Web Agents","ref_index":7,"is_internal_anchor":false},{"citing_arxiv_id":"2511.22074","citing_title":"Real-Time Procedural Learning From Experience for AI Agents","ref_index":6,"is_internal_anchor":false},{"citing_arxiv_id":"2511.23281","citing_title":"MCP vs RAG vs NLWeb vs HTML: A Comparison of the Effectiveness and Efficiency of Different Agent Interfaces to the Web (Technical Report)","ref_index":5,"is_internal_anchor":false},{"citing_arxiv_id":"2512.12634","citing_title":"MobiBench: Multi-Branch, Modular Benchmark for Mobile GUI Agents","ref_index":10,"is_internal_anchor":false},{"citing_arxiv_id":"2603.05044","citing_title":"WebFactory: Automated Compression of Foundational Language Intelligence into Grounded Web Agents","ref_index":9,"is_internal_anchor":false},{"citing_arxiv_id":"2605.08261","citing_title":"Computer Use at the Edge of the Statistical Precipice","ref_index":7,"is_internal_anchor":false},{"citing_arxiv_id":"2605.04165","citing_title":"FlowEval: Reference-based Evaluation of Generated User Interfaces","ref_index":6,"is_internal_anchor":false},{"citing_arxiv_id":"2605.00334","citing_title":"AgentFloor: How Far Up the tool use Ladder Can Small Open-Weight Models Go?","ref_index":18,"is_internal_anchor":false},{"citing_arxiv_id":"2604.09937","citing_title":"HealthAdminBench: Evaluating Computer-Use Agents on Healthcare Administration Tasks","ref_index":1,"is_internal_anchor":false},{"citing_arxiv_id":"2604.08523","citing_title":"ClawBench: Can AI Agents Complete Everyday Online Tasks?","ref_index":3,"is_internal_anchor":false}]},"formal_canon":{"evidence_count":0,"sample":[],"anchors":[]},"links":{"html":"https://pith.science/pith/7AUNQAIWAVUT53Q5YJZBY77SRM","json":"https://pith.science/pith/7AUNQAIWAVUT53Q5YJZBY77SRM.json","graph_json":"https://pith.science/api/pith-number/7AUNQAIWAVUT53Q5YJZBY77SRM/graph.json","events_json":"https://pith.science/api/pith-number/7AUNQAIWAVUT53Q5YJZBY77SRM/events.json","paper":"https://pith.science/paper/7AUNQAIW"},"agent_actions":{"view_html":"https://pith.science/pith/7AUNQAIWAVUT53Q5YJZBY77SRM","download_json":"https://pith.science/pith/7AUNQAIWAVUT53Q5YJZBY77SRM.json","view_paper":"https://pith.science/paper/7AUNQAIW","resolve_alias":"https://pith.science/api/pith-number/resolve?arxiv=2504.11543&json=true","fetch_graph":"https://pith.science/api/pith-number/7AUNQAIWAVUT53Q5YJZBY77SRM/graph.json","fetch_events":"https://pith.science/api/pith-number/7AUNQAIWAVUT53Q5YJZBY77SRM/events.json","actions":{"anchor_timestamp":"https://pith.science/pith/7AUNQAIWAVUT53Q5YJZBY77SRM/action/timestamp_anchor","attest_storage":"https://pith.science/pith/7AUNQAIWAVUT53Q5YJZBY77SRM/action/storage_attestation","attest_author":"https://pith.science/pith/7AUNQAIWAVUT53Q5YJZBY77SRM/action/author_attestation","sign_citation":"https://pith.science/pith/7AUNQAIWAVUT53Q5YJZBY77SRM/action/citation_signature","submit_replication":"https://pith.science/pith/7AUNQAIWAVUT53Q5YJZBY77SRM/action/replication_record"}},"created_at":"2026-07-05T10:50:33.621218+00:00","updated_at":"2026-07-05T10:50:33.621218+00:00"}