{"record_type":"pith_number_record","schema_url":"https://pith.science/schemas/pith-number/v1.json","pith_number":"pith:2024:JEPNEK465VUXGQPQI7XGHKDNNP","short_pith_number":"pith:JEPNEK46","schema_version":"1.0","canonical_sha256":"491ed22b9eed697341f047ee63a86d6be4babac3d0410b09e2644925bc62b3bb","source":{"kind":"arxiv","id":"2405.20421","version":5},"attestation_state":"computed","paper":{"title":"Worse than Random? An Embarrassingly Simple Probing Evaluation of Large Multimodal Models in Medical VQA","license":"http://creativecommons.org/licenses/by/4.0/","headline":"","cross_cats":[],"primary_cat":"cs.AI","authors_text":"Qianqi Yan, Xiang Yue, Xin Eric Wang, Xuehai He","submitted_at":"2024-05-30T18:56:01Z","abstract_excerpt":"Large Multimodal Models (LMMs) have shown remarkable progress in medical Visual Question Answering (Med-VQA), achieving high accuracy on existing benchmarks. However, their reliability under robust evaluation is questionable. This study reveals that when subjected to simple probing evaluation, state-of-the-art models perform worse than random guessing on medical diagnosis questions. To address this critical evaluation problem, we introduce the Probing Evaluation for Medical Diagnosis (ProbMed) dataset to rigorously assess LMM performance in medical imaging through probing evaluation and proced"},"verification_status":{"content_addressed":true,"pith_receipt":true,"author_attested":false,"weak_author_claims":0,"strong_author_claims":0,"externally_anchored":false,"storage_verified":false,"citation_signatures":0,"replication_records":0,"graph_snapshot":true,"references_resolved":false,"formal_links_present":false},"canonical_record":{"source":{"id":"2405.20421","kind":"arxiv","version":5},"metadata":{"license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.AI","submitted_at":"2024-05-30T18:56:01Z","cross_cats_sorted":[],"title_canon_sha256":"dc291432b6d3599f26eb30f1ca8a32bd5c35191ac58315a40413a75f5b853f0b","abstract_canon_sha256":"0a5a7f4667fb9be4b4b036b15c641ed0e940cc595062b7f524d7e64594713989"},"schema_version":"1.0"},"receipt":{"kind":"pith_receipt","key_id":"pith-v1-2026-05","algorithm":"ed25519","signed_at":"2026-07-05T11:19:19.525746Z","signature_b64":"NoAJ2MqAxoa4WKHKYziHrJ4g4gI/iAyf8qGB7oKgd6wC1ceo9kmdrBcoWiTnKUt3ZUrvAoLblFLMSGTwjAIpAQ==","signed_message":"canonical_sha256_bytes","builder_version":"pith-number-builder-2026-05-17-v1","receipt_version":"0.3","canonical_sha256":"491ed22b9eed697341f047ee63a86d6be4babac3d0410b09e2644925bc62b3bb","last_reissued_at":"2026-07-05T11:19:19.525240Z","signature_status":"signed_v1","first_computed_at":"2026-07-05T11:19:19.525240Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"graph_snapshot":{"paper":{"title":"Worse than Random? An Embarrassingly Simple Probing Evaluation of Large Multimodal Models in Medical VQA","license":"http://creativecommons.org/licenses/by/4.0/","headline":"","cross_cats":[],"primary_cat":"cs.AI","authors_text":"Qianqi Yan, Xiang Yue, Xin Eric Wang, Xuehai He","submitted_at":"2024-05-30T18:56:01Z","abstract_excerpt":"Large Multimodal Models (LMMs) have shown remarkable progress in medical Visual Question Answering (Med-VQA), achieving high accuracy on existing benchmarks. However, their reliability under robust evaluation is questionable. This study reveals that when subjected to simple probing evaluation, state-of-the-art models perform worse than random guessing on medical diagnosis questions. To address this critical evaluation problem, we introduce the Probing Evaluation for Medical Diagnosis (ProbMed) dataset to rigorously assess LMM performance in medical imaging through probing evaluation and proced"},"claims":{"count":0,"items":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"2405.20421","kind":"arxiv","version":5},"verdict":{"id":null,"model_set":{},"created_at":null,"strongest_claim":"","one_line_summary":"","pipeline_version":null,"weakest_assumption":"","pith_extraction_headline":""},"integrity":{"clean":true,"summary":{"advisory":0,"critical":0,"by_detector":{},"informational":0},"endpoint":"/pith/2405.20421/integrity.json","findings":[],"available":true,"detectors_run":[],"snapshot_sha256":"c28c3603d3b5d939e8dc4c7e95fa8dfce3d595e45f758748cecf8e644a296938"},"references":{"count":0,"sample":[],"resolved_work":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","internal_anchors":0},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"author_claims":{"count":0,"strong_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"builder_version":"pith-number-builder-2026-05-17-v1"},"aliases":[{"alias_kind":"arxiv","alias_value":"2405.20421","created_at":"2026-07-05T11:19:19.525310+00:00"},{"alias_kind":"arxiv_version","alias_value":"2405.20421v5","created_at":"2026-07-05T11:19:19.525310+00:00"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.2405.20421","created_at":"2026-07-05T11:19:19.525310+00:00"},{"alias_kind":"pith_short_12","alias_value":"JEPNEK465VUX","created_at":"2026-07-05T11:19:19.525310+00:00"},{"alias_kind":"pith_short_16","alias_value":"JEPNEK465VUXGQPQ","created_at":"2026-07-05T11:19:19.525310+00:00"},{"alias_kind":"pith_short_8","alias_value":"JEPNEK46","created_at":"2026-07-05T11:19:19.525310+00:00"}],"events":[],"event_summary":{},"paper_claims":[],"inbound_citations":{"count":1,"internal_anchor_count":0,"sample":[{"citing_arxiv_id":"2605.18111","citing_title":"How Good LLMs Are at Answering Bangla Medical Visual Questions? Dataset and Benchmarking","ref_index":44,"is_internal_anchor":false}]},"formal_canon":{"evidence_count":0,"sample":[],"anchors":[]},"links":{"html":"https://pith.science/pith/JEPNEK465VUXGQPQI7XGHKDNNP","json":"https://pith.science/pith/JEPNEK465VUXGQPQI7XGHKDNNP.json","graph_json":"https://pith.science/api/pith-number/JEPNEK465VUXGQPQI7XGHKDNNP/graph.json","events_json":"https://pith.science/api/pith-number/JEPNEK465VUXGQPQI7XGHKDNNP/events.json","paper":"https://pith.science/paper/JEPNEK46"},"agent_actions":{"view_html":"https://pith.science/pith/JEPNEK465VUXGQPQI7XGHKDNNP","download_json":"https://pith.science/pith/JEPNEK465VUXGQPQI7XGHKDNNP.json","view_paper":"https://pith.science/paper/JEPNEK46","resolve_alias":"https://pith.science/api/pith-number/resolve?arxiv=2405.20421&json=true","fetch_graph":"https://pith.science/api/pith-number/JEPNEK465VUXGQPQI7XGHKDNNP/graph.json","fetch_events":"https://pith.science/api/pith-number/JEPNEK465VUXGQPQI7XGHKDNNP/events.json","actions":{"anchor_timestamp":"https://pith.science/pith/JEPNEK465VUXGQPQI7XGHKDNNP/action/timestamp_anchor","attest_storage":"https://pith.science/pith/JEPNEK465VUXGQPQI7XGHKDNNP/action/storage_attestation","attest_author":"https://pith.science/pith/JEPNEK465VUXGQPQI7XGHKDNNP/action/author_attestation","sign_citation":"https://pith.science/pith/JEPNEK465VUXGQPQI7XGHKDNNP/action/citation_signature","submit_replication":"https://pith.science/pith/JEPNEK465VUXGQPQI7XGHKDNNP/action/replication_record"}},"created_at":"2026-07-05T11:19:19.525310+00:00","updated_at":"2026-07-05T11:19:19.525310+00:00"}