{"record_type":"pith_number_record","schema_url":"https://pith.science/schemas/pith-number/v1.json","pith_number":"pith:2024:RRV4GFQO3BSNG7R7ZASS5CJPUX","short_pith_number":"pith:RRV4GFQO","schema_version":"1.0","canonical_sha256":"8c6bc3160ed864d37e3fc8252e892fa5fefc98956feda7b04dfc1e388a6f7a0e","source":{"kind":"arxiv","id":"2405.07960","version":5},"attestation_state":"computed","paper":{"title":"AgentClinic: a multimodal agent benchmark to evaluate AI in simulated clinical environments","license":"http://creativecommons.org/licenses/by/4.0/","headline":"","cross_cats":["cs.CL"],"primary_cat":"cs.HC","authors_text":"Carl Harris, Eduardo Reis, Jeffrey Jopling, Michael Moor, Rojin Ziaei, Samuel Schmidgall","submitted_at":"2024-05-13T17:38:53Z","abstract_excerpt":"Evaluating large language models (LLM) in clinical scenarios is crucial to assessing their potential clinical utility. Existing benchmarks rely heavily on static question-answering, which does not accurately depict the complex, sequential nature of clinical decision-making. Here, we introduce AgentClinic, a multimodal agent benchmark for evaluating LLMs in simulated clinical environments that include patient interactions, multimodal data collection under incomplete information, and the usage of various tools, resulting in an in-depth evaluation across nine medical specialties and seven languag"},"verification_status":{"content_addressed":true,"pith_receipt":true,"author_attested":false,"weak_author_claims":0,"strong_author_claims":0,"externally_anchored":false,"storage_verified":false,"citation_signatures":0,"replication_records":0,"graph_snapshot":true,"references_resolved":false,"formal_links_present":false},"canonical_record":{"source":{"id":"2405.07960","kind":"arxiv","version":5},"metadata":{"license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.HC","submitted_at":"2024-05-13T17:38:53Z","cross_cats_sorted":["cs.CL"],"title_canon_sha256":"329298b668b7d8094011bd393da2ad3e586fdf5a58433485b44e89dc083faef2","abstract_canon_sha256":"6dab0129b3cd64aa7f6b361c04cad632a91ccee45f9bb199d24ecc97c73d5a18"},"schema_version":"1.0"},"receipt":{"kind":"pith_receipt","key_id":"pith-v1-2026-05","algorithm":"ed25519","signed_at":"2026-05-21T13:47:14.140102Z","signature_b64":"rQf3bBRBTZsz/4Vr4rxTme1qf+PMR+V9DMPuguzUiFFhoYMqzvhhK5bbEGxVpNBjlpsLFCJG2z1GO9FDLmHSBQ==","signed_message":"canonical_sha256_bytes","builder_version":"pith-number-builder-2026-05-17-v1","receipt_version":"0.3","canonical_sha256":"8c6bc3160ed864d37e3fc8252e892fa5fefc98956feda7b04dfc1e388a6f7a0e","last_reissued_at":"2026-05-21T13:47:14.138213Z","signature_status":"signed_v1","first_computed_at":"2026-05-21T13:47:14.138213Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"graph_snapshot":{"paper":{"title":"AgentClinic: a multimodal agent benchmark to evaluate AI in simulated clinical environments","license":"http://creativecommons.org/licenses/by/4.0/","headline":"","cross_cats":["cs.CL"],"primary_cat":"cs.HC","authors_text":"Carl Harris, Eduardo Reis, Jeffrey Jopling, Michael Moor, Rojin Ziaei, Samuel Schmidgall","submitted_at":"2024-05-13T17:38:53Z","abstract_excerpt":"Evaluating large language models (LLM) in clinical scenarios is crucial to assessing their potential clinical utility. Existing benchmarks rely heavily on static question-answering, which does not accurately depict the complex, sequential nature of clinical decision-making. Here, we introduce AgentClinic, a multimodal agent benchmark for evaluating LLMs in simulated clinical environments that include patient interactions, multimodal data collection under incomplete information, and the usage of various tools, resulting in an in-depth evaluation across nine medical specialties and seven languag"},"claims":{"count":0,"items":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"2405.07960","kind":"arxiv","version":5},"verdict":{"id":null,"model_set":{},"created_at":null,"strongest_claim":"","one_line_summary":"","pipeline_version":null,"weakest_assumption":"","pith_extraction_headline":""},"integrity":{"clean":true,"summary":{"advisory":0,"critical":0,"by_detector":{},"informational":0},"endpoint":"/pith/2405.07960/integrity.json","findings":[],"available":true,"detectors_run":[],"snapshot_sha256":"c28c3603d3b5d939e8dc4c7e95fa8dfce3d595e45f758748cecf8e644a296938"},"references":{"count":0,"sample":[],"resolved_work":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","internal_anchors":0},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"author_claims":{"count":0,"strong_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"builder_version":"pith-number-builder-2026-05-17-v1"},"aliases":[{"alias_kind":"arxiv","alias_value":"2405.07960","created_at":"2026-05-21T13:47:14.138300+00:00"},{"alias_kind":"arxiv_version","alias_value":"2405.07960v5","created_at":"2026-05-21T13:47:14.138300+00:00"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.2405.07960","created_at":"2026-05-21T13:47:14.138300+00:00"},{"alias_kind":"pith_short_12","alias_value":"RRV4GFQO3BSN","created_at":"2026-05-21T13:47:14.138300+00:00"},{"alias_kind":"pith_short_16","alias_value":"RRV4GFQO3BSNG7R7","created_at":"2026-05-21T13:47:14.138300+00:00"},{"alias_kind":"pith_short_8","alias_value":"RRV4GFQO","created_at":"2026-05-21T13:47:14.138300+00:00"}],"events":[],"event_summary":{},"paper_claims":[],"inbound_citations":{"count":24,"internal_anchor_count":24,"sample":[{"citing_arxiv_id":"2605.23559","citing_title":"PathNavigate: A Training-Free Pathology Agent with Surprise-Guided Scan and Shared Slide Memory for Whole-Slide Image VQA","ref_index":10,"is_internal_anchor":true},{"citing_arxiv_id":"2605.23204","citing_title":"AutoResearch AI: Towards AI-Powered Research Automation for Scientific Discovery","ref_index":168,"is_internal_anchor":true},{"citing_arxiv_id":"2605.23262","citing_title":"Design and Report Benchmarks for Knowledge Work","ref_index":71,"is_internal_anchor":true},{"citing_arxiv_id":"2605.20506","citing_title":"Reinforcing Human Behavior Simulation via Verbal Feedback","ref_index":32,"is_internal_anchor":true},{"citing_arxiv_id":"2605.16679","citing_title":"CHI-Bench: Can AI Agents Automate End-to-End, Long-Horizon, Policy-Rich Healthcare Workflows?","ref_index":48,"is_internal_anchor":true},{"citing_arxiv_id":"2605.17829","citing_title":"Interactive Evaluation Requires a Design Science","ref_index":50,"is_internal_anchor":true},{"citing_arxiv_id":"2605.20176","citing_title":"ClinSeekAgent: Automating Multimodal Evidence Seeking for Agentic Clinical Reasoning","ref_index":5,"is_internal_anchor":true},{"citing_arxiv_id":"2605.14892","citing_title":"Beyond Individual Intelligence: Surveying Collaboration, Failure Attribution, and Self-Evolution in LLM-based Multi-Agent Systems","ref_index":298,"is_internal_anchor":true},{"citing_arxiv_id":"2507.15867","citing_title":"RDMA: Cost Effective Agent-Driven Rare Disease Mining from Electronic Health Records","ref_index":50,"is_internal_anchor":true},{"citing_arxiv_id":"2509.20490","citing_title":"RadAgents: Multimodal Agentic Reasoning for Chest X-ray Interpretation with Radiologist-like Workflows","ref_index":13,"is_internal_anchor":true},{"citing_arxiv_id":"2501.04227","citing_title":"Agent Laboratory: Using LLM Agents as Research Assistants","ref_index":1,"is_internal_anchor":true},{"citing_arxiv_id":"2508.07407","citing_title":"A Comprehensive Survey of Self-Evolving AI Agents: A New Paradigm Bridging Foundation Models and Lifelong Agentic Systems","ref_index":84,"is_internal_anchor":true},{"citing_arxiv_id":"2503.12605","citing_title":"Multimodal Chain-of-Thought Reasoning: A Comprehensive Survey","ref_index":259,"is_internal_anchor":true},{"citing_arxiv_id":"2605.14892","citing_title":"Beyond Individual Intelligence: Surveying Collaboration, Failure Attribution, and Self-Evolution in LLM-based Multi-Agent Systems","ref_index":297,"is_internal_anchor":true},{"citing_arxiv_id":"2504.19678","citing_title":"From LLM Reasoning to Autonomous AI Agents: A Comprehensive Review","ref_index":33,"is_internal_anchor":true},{"citing_arxiv_id":"2605.13542","citing_title":"RealICU: Do LLM Agents Understand Long-Context ICU Data? A Benchmark Beyond Behavior Imitation","ref_index":28,"is_internal_anchor":true},{"citing_arxiv_id":"2605.09679","citing_title":"DeepTumorVQA: A Hierarchical 3D CT Benchmark for Stage-Wise Evaluation of Medical VLMs and Tool-Augmented Agents","ref_index":49,"is_internal_anchor":true},{"citing_arxiv_id":"2604.23802","citing_title":"EndoGov: A knowledge-governed multi-agent expert system for endometrial cancer risk stratification","ref_index":37,"is_internal_anchor":true},{"citing_arxiv_id":"2605.06177","citing_title":"BioMedArena: An Open-source Toolkit for Building and Evaluating Biomedical Deep Research Agents","ref_index":37,"is_internal_anchor":true},{"citing_arxiv_id":"2605.02240","citing_title":"PhysicianBench: Evaluating LLM Agents in Real-World EHR Environments","ref_index":30,"is_internal_anchor":true},{"citing_arxiv_id":"2604.14475","citing_title":"Evo-MedAgent: Beyond One-Shot Diagnosis with Agents That Remember, Reflect, and Improve","ref_index":18,"is_internal_anchor":true},{"citing_arxiv_id":"2604.12076","citing_title":"Narrative over Numbers: The Identifiable Victim Effect and its Amplification Under Alignment and Reasoning in Large Language Models","ref_index":13,"is_internal_anchor":true},{"citing_arxiv_id":"2604.06846","citing_title":"MedDialBench: Benchmarking LLM Diagnostic Robustness under Parametric Adversarial Patient Behaviors","ref_index":10,"is_internal_anchor":true},{"citing_arxiv_id":"2508.04325","citing_title":"Beyond the Leaderboard: Rethinking Medical Benchmarks for Large Language Models","ref_index":6,"is_internal_anchor":true}]},"formal_canon":{"evidence_count":0,"sample":[],"anchors":[]},"links":{"html":"https://pith.science/pith/RRV4GFQO3BSNG7R7ZASS5CJPUX","json":"https://pith.science/pith/RRV4GFQO3BSNG7R7ZASS5CJPUX.json","graph_json":"https://pith.science/api/pith-number/RRV4GFQO3BSNG7R7ZASS5CJPUX/graph.json","events_json":"https://pith.science/api/pith-number/RRV4GFQO3BSNG7R7ZASS5CJPUX/events.json","paper":"https://pith.science/paper/RRV4GFQO"},"agent_actions":{"view_html":"https://pith.science/pith/RRV4GFQO3BSNG7R7ZASS5CJPUX","download_json":"https://pith.science/pith/RRV4GFQO3BSNG7R7ZASS5CJPUX.json","view_paper":"https://pith.science/paper/RRV4GFQO","resolve_alias":"https://pith.science/api/pith-number/resolve?arxiv=2405.07960&json=true","fetch_graph":"https://pith.science/api/pith-number/RRV4GFQO3BSNG7R7ZASS5CJPUX/graph.json","fetch_events":"https://pith.science/api/pith-number/RRV4GFQO3BSNG7R7ZASS5CJPUX/events.json","actions":{"anchor_timestamp":"https://pith.science/pith/RRV4GFQO3BSNG7R7ZASS5CJPUX/action/timestamp_anchor","attest_storage":"https://pith.science/pith/RRV4GFQO3BSNG7R7ZASS5CJPUX/action/storage_attestation","attest_author":"https://pith.science/pith/RRV4GFQO3BSNG7R7ZASS5CJPUX/action/author_attestation","sign_citation":"https://pith.science/pith/RRV4GFQO3BSNG7R7ZASS5CJPUX/action/citation_signature","submit_replication":"https://pith.science/pith/RRV4GFQO3BSNG7R7ZASS5CJPUX/action/replication_record"}},"created_at":"2026-05-21T13:47:14.138300+00:00","updated_at":"2026-05-21T13:47:14.138300+00:00"}