{"record_type":"pith_number_record","schema_url":"https://pith.science/schemas/pith-number/v1.json","pith_number":"pith:2024:J7TFPIKQF4IIGHHRAJW546V22T","short_pith_number":"pith:J7TFPIKQ","schema_version":"1.0","canonical_sha256":"4fe657a1502f10831cf1026dde7abad4e0af0f44bdd8352a5a4f90cdae63f57d","source":{"kind":"arxiv","id":"2406.06573","version":2},"attestation_state":"computed","paper":{"title":"MedFuzz: Exploring the Robustness of Large Language Models in Medical Question Answering","license":"http://creativecommons.org/licenses/by/4.0/","headline":"","cross_cats":["cs.LG"],"primary_cat":"cs.CL","authors_text":"Carey E. Priebe, Eric Horvitz, Hayden Helm, Junaid Bajwa, Katie Matton, Robert Osazuwa Ness, Sheng Zhang","submitted_at":"2024-06-03T18:15:56Z","abstract_excerpt":"Large language models (LLM) have achieved impressive performance on medical question-answering benchmarks. However, high benchmark accuracy does not imply that the performance generalizes to real-world clinical settings. Medical question-answering benchmarks rely on assumptions consistent with quantifying LLM performance but that may not hold in the open world of the clinic. Yet LLMs learn broad knowledge that can help the LLM generalize to practical conditions regardless of unrealistic assumptions in celebrated benchmarks. We seek to quantify how well LLM medical question-answering benchmark "},"verification_status":{"content_addressed":true,"pith_receipt":true,"author_attested":false,"weak_author_claims":0,"strong_author_claims":0,"externally_anchored":false,"storage_verified":false,"citation_signatures":0,"replication_records":0,"graph_snapshot":true,"references_resolved":false,"formal_links_present":false},"canonical_record":{"source":{"id":"2406.06573","kind":"arxiv","version":2},"metadata":{"license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.CL","submitted_at":"2024-06-03T18:15:56Z","cross_cats_sorted":["cs.LG"],"title_canon_sha256":"0689c01bd6b9fe751d55661c96bfddd7b5a172264fbb8c9dfdf9c79d8398c46c","abstract_canon_sha256":"43b4e4a354f8590d7e598d9c2cb5497dddf9ff7019294c34f5f66da48052fddb"},"schema_version":"1.0"},"receipt":{"kind":"pith_receipt","key_id":"pith-v1-2026-05","algorithm":"ed25519","signed_at":"2026-07-05T09:01:35.224693Z","signature_b64":"HIjIZI4ZQMD2fuzM8jXJzr95geDWqkg/ZnAJT9gs0BMGiv5YatsWhwE2VuJyCaHufzBKaKWi4lmdPQyAoF9bDg==","signed_message":"canonical_sha256_bytes","builder_version":"pith-number-builder-2026-05-17-v1","receipt_version":"0.3","canonical_sha256":"4fe657a1502f10831cf1026dde7abad4e0af0f44bdd8352a5a4f90cdae63f57d","last_reissued_at":"2026-07-05T09:01:35.224264Z","signature_status":"signed_v1","first_computed_at":"2026-07-05T09:01:35.224264Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"graph_snapshot":{"paper":{"title":"MedFuzz: Exploring the Robustness of Large Language Models in Medical Question Answering","license":"http://creativecommons.org/licenses/by/4.0/","headline":"","cross_cats":["cs.LG"],"primary_cat":"cs.CL","authors_text":"Carey E. Priebe, Eric Horvitz, Hayden Helm, Junaid Bajwa, Katie Matton, Robert Osazuwa Ness, Sheng Zhang","submitted_at":"2024-06-03T18:15:56Z","abstract_excerpt":"Large language models (LLM) have achieved impressive performance on medical question-answering benchmarks. However, high benchmark accuracy does not imply that the performance generalizes to real-world clinical settings. Medical question-answering benchmarks rely on assumptions consistent with quantifying LLM performance but that may not hold in the open world of the clinic. Yet LLMs learn broad knowledge that can help the LLM generalize to practical conditions regardless of unrealistic assumptions in celebrated benchmarks. We seek to quantify how well LLM medical question-answering benchmark "},"claims":{"count":0,"items":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"2406.06573","kind":"arxiv","version":2},"verdict":{"id":null,"model_set":{},"created_at":null,"strongest_claim":"","one_line_summary":"","pipeline_version":null,"weakest_assumption":"","pith_extraction_headline":""},"integrity":{"clean":true,"summary":{"advisory":0,"critical":0,"by_detector":{},"informational":0},"endpoint":"/pith/2406.06573/integrity.json","findings":[],"available":true,"detectors_run":[],"snapshot_sha256":"c28c3603d3b5d939e8dc4c7e95fa8dfce3d595e45f758748cecf8e644a296938"},"references":{"count":0,"sample":[],"resolved_work":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","internal_anchors":0},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"author_claims":{"count":0,"strong_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"builder_version":"pith-number-builder-2026-05-17-v1"},"aliases":[{"alias_kind":"arxiv","alias_value":"2406.06573","created_at":"2026-07-05T09:01:35.224327+00:00"},{"alias_kind":"arxiv_version","alias_value":"2406.06573v2","created_at":"2026-07-05T09:01:35.224327+00:00"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.2406.06573","created_at":"2026-07-05T09:01:35.224327+00:00"},{"alias_kind":"pith_short_12","alias_value":"J7TFPIKQF4II","created_at":"2026-07-05T09:01:35.224327+00:00"},{"alias_kind":"pith_short_16","alias_value":"J7TFPIKQF4IIGHHR","created_at":"2026-07-05T09:01:35.224327+00:00"},{"alias_kind":"pith_short_8","alias_value":"J7TFPIKQ","created_at":"2026-07-05T09:01:35.224327+00:00"}],"events":[],"event_summary":{},"paper_claims":[],"inbound_citations":{"count":3,"internal_anchor_count":0,"sample":[{"citing_arxiv_id":"2606.07237","citing_title":"When Large Language Models Fail in Healthcare: Evaluating Sensitivity to Prompt Variations","ref_index":7,"is_internal_anchor":false},{"citing_arxiv_id":"2502.07143","citing_title":"Ask Patients with Patience: Enabling LLMs for Human-Centric Medical Dialogue with Grounded Reasoning","ref_index":1,"is_internal_anchor":false},{"citing_arxiv_id":"2604.06846","citing_title":"MedDialBench: Benchmarking LLM Diagnostic Robustness under Parametric Adversarial Patient Behaviors","ref_index":3,"is_internal_anchor":false}]},"formal_canon":{"evidence_count":0,"sample":[],"anchors":[]},"links":{"html":"https://pith.science/pith/J7TFPIKQF4IIGHHRAJW546V22T","json":"https://pith.science/pith/J7TFPIKQF4IIGHHRAJW546V22T.json","graph_json":"https://pith.science/api/pith-number/J7TFPIKQF4IIGHHRAJW546V22T/graph.json","events_json":"https://pith.science/api/pith-number/J7TFPIKQF4IIGHHRAJW546V22T/events.json","paper":"https://pith.science/paper/J7TFPIKQ"},"agent_actions":{"view_html":"https://pith.science/pith/J7TFPIKQF4IIGHHRAJW546V22T","download_json":"https://pith.science/pith/J7TFPIKQF4IIGHHRAJW546V22T.json","view_paper":"https://pith.science/paper/J7TFPIKQ","resolve_alias":"https://pith.science/api/pith-number/resolve?arxiv=2406.06573&json=true","fetch_graph":"https://pith.science/api/pith-number/J7TFPIKQF4IIGHHRAJW546V22T/graph.json","fetch_events":"https://pith.science/api/pith-number/J7TFPIKQF4IIGHHRAJW546V22T/events.json","actions":{"anchor_timestamp":"https://pith.science/pith/J7TFPIKQF4IIGHHRAJW546V22T/action/timestamp_anchor","attest_storage":"https://pith.science/pith/J7TFPIKQF4IIGHHRAJW546V22T/action/storage_attestation","attest_author":"https://pith.science/pith/J7TFPIKQF4IIGHHRAJW546V22T/action/author_attestation","sign_citation":"https://pith.science/pith/J7TFPIKQF4IIGHHRAJW546V22T/action/citation_signature","submit_replication":"https://pith.science/pith/J7TFPIKQF4IIGHHRAJW546V22T/action/replication_record"}},"created_at":"2026-07-05T09:01:35.224327+00:00","updated_at":"2026-07-05T09:01:35.224327+00:00"}