{"record_type":"pith_number_record","schema_url":"https://pith.science/schemas/pith-number/v1.json","pith_number":"pith:2025:5JRUKBBC5R6E6LXVRVCXH7PWW6","short_pith_number":"pith:5JRUKBBC","schema_version":"1.0","canonical_sha256":"ea63450422ec7c4f2ef58d4573fdf6b78c4621cc2ff752d7a705a160284ea579","source":{"kind":"arxiv","id":"2503.10694","version":1},"attestation_state":"computed","paper":{"title":"Medical Large Language Model Benchmarks Should Prioritize Construct Validity","license":"http://creativecommons.org/licenses/by/4.0/","headline":"","cross_cats":[],"primary_cat":"cs.CL","authors_text":"Ahmed Alaa, Frances Dean, Inioluwa Deborah Raji, Niloufar Golchini, Shiladitya Dutta, Thomas Hartvigsen, Travis Zack","submitted_at":"2025-03-12T05:08:02Z","abstract_excerpt":"Medical large language models (LLMs) research often makes bold claims, from encoding clinical knowledge to reasoning like a physician. These claims are usually backed by evaluation on competitive benchmarks; a tradition inherited from mainstream machine learning. But how do we separate real progress from a leaderboard flex? Medical LLM benchmarks, much like those in other fields, are arbitrarily constructed using medical licensing exam questions. For these benchmarks to truly measure progress, they must accurately capture the real-world tasks they aim to represent. In this position paper, we a"},"verification_status":{"content_addressed":true,"pith_receipt":true,"author_attested":false,"weak_author_claims":0,"strong_author_claims":0,"externally_anchored":false,"storage_verified":false,"citation_signatures":0,"replication_records":0,"graph_snapshot":true,"references_resolved":false,"formal_links_present":false},"canonical_record":{"source":{"id":"2503.10694","kind":"arxiv","version":1},"metadata":{"license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.CL","submitted_at":"2025-03-12T05:08:02Z","cross_cats_sorted":[],"title_canon_sha256":"b015ebec658297ccd4648d1da60ad81c3cb3a186b36777e865389542b264da5f","abstract_canon_sha256":"2bbf3136f8c2c093ade21956320fb64938e047594fbbde29826c7e4e82b0ab48"},"schema_version":"1.0"},"receipt":{"kind":"pith_receipt","key_id":"pith-v1-2026-05","algorithm":"ed25519","signed_at":"2026-07-05T10:31:03.839391Z","signature_b64":"fs9wXiZb7BgZiVR1L0SYSU5h7kRJAHb61G4AHd/XLtndHleQAS2HgTp1tzwPntbB8rAl9Nrb3eeIYuNpN6q3Cw==","signed_message":"canonical_sha256_bytes","builder_version":"pith-number-builder-2026-05-17-v1","receipt_version":"0.3","canonical_sha256":"ea63450422ec7c4f2ef58d4573fdf6b78c4621cc2ff752d7a705a160284ea579","last_reissued_at":"2026-07-05T10:31:03.838757Z","signature_status":"signed_v1","first_computed_at":"2026-07-05T10:31:03.838757Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"graph_snapshot":{"paper":{"title":"Medical Large Language Model Benchmarks Should Prioritize Construct Validity","license":"http://creativecommons.org/licenses/by/4.0/","headline":"","cross_cats":[],"primary_cat":"cs.CL","authors_text":"Ahmed Alaa, Frances Dean, Inioluwa Deborah Raji, Niloufar Golchini, Shiladitya Dutta, Thomas Hartvigsen, Travis Zack","submitted_at":"2025-03-12T05:08:02Z","abstract_excerpt":"Medical large language models (LLMs) research often makes bold claims, from encoding clinical knowledge to reasoning like a physician. These claims are usually backed by evaluation on competitive benchmarks; a tradition inherited from mainstream machine learning. But how do we separate real progress from a leaderboard flex? Medical LLM benchmarks, much like those in other fields, are arbitrarily constructed using medical licensing exam questions. For these benchmarks to truly measure progress, they must accurately capture the real-world tasks they aim to represent. In this position paper, we a"},"claims":{"count":0,"items":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"2503.10694","kind":"arxiv","version":1},"verdict":{"id":null,"model_set":{},"created_at":null,"strongest_claim":"","one_line_summary":"","pipeline_version":null,"weakest_assumption":"","pith_extraction_headline":""},"integrity":{"clean":true,"summary":{"advisory":0,"critical":0,"by_detector":{},"informational":0},"endpoint":"/pith/2503.10694/integrity.json","findings":[],"available":true,"detectors_run":[],"snapshot_sha256":"c28c3603d3b5d939e8dc4c7e95fa8dfce3d595e45f758748cecf8e644a296938"},"references":{"count":0,"sample":[],"resolved_work":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","internal_anchors":0},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"author_claims":{"count":0,"strong_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"builder_version":"pith-number-builder-2026-05-17-v1"},"aliases":[{"alias_kind":"arxiv","alias_value":"2503.10694","created_at":"2026-07-05T10:31:03.838838+00:00"},{"alias_kind":"arxiv_version","alias_value":"2503.10694v1","created_at":"2026-07-05T10:31:03.838838+00:00"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.2503.10694","created_at":"2026-07-05T10:31:03.838838+00:00"},{"alias_kind":"pith_short_12","alias_value":"5JRUKBBC5R6E","created_at":"2026-07-05T10:31:03.838838+00:00"},{"alias_kind":"pith_short_16","alias_value":"5JRUKBBC5R6E6LXV","created_at":"2026-07-05T10:31:03.838838+00:00"},{"alias_kind":"pith_short_8","alias_value":"5JRUKBBC","created_at":"2026-07-05T10:31:03.838838+00:00"}],"events":[],"event_summary":{},"paper_claims":[],"inbound_citations":{"count":6,"internal_anchor_count":1,"sample":[{"citing_arxiv_id":"2607.07761","citing_title":"Aligning Clinical Needs and AI Capabilities: A Survey on LLMs for Medical Reasoning","ref_index":172,"is_internal_anchor":true},{"citing_arxiv_id":"2606.26079","citing_title":"Same Evidence, Different Answer: Auditing Order Sensitivity in Multimodal Large Language Models","ref_index":69,"is_internal_anchor":false},{"citing_arxiv_id":"2605.30315","citing_title":"Resolution Diagnostics for Paired LLM Evaluation","ref_index":2,"is_internal_anchor":false},{"citing_arxiv_id":"2605.22612","citing_title":"Healthcare LLM Benchmarks Are Only as Good as Their Explicit Assumptions","ref_index":1,"is_internal_anchor":false},{"citing_arxiv_id":"2605.17281","citing_title":"ContractBench: Can LLM Agents Preserve Observation Contracts?","ref_index":19,"is_internal_anchor":false},{"citing_arxiv_id":"2510.15297","citing_title":"VERA-MH Concept Paper","ref_index":1,"is_internal_anchor":false}]},"formal_canon":{"evidence_count":0,"sample":[],"anchors":[]},"links":{"html":"https://pith.science/pith/5JRUKBBC5R6E6LXVRVCXH7PWW6","json":"https://pith.science/pith/5JRUKBBC5R6E6LXVRVCXH7PWW6.json","graph_json":"https://pith.science/api/pith-number/5JRUKBBC5R6E6LXVRVCXH7PWW6/graph.json","events_json":"https://pith.science/api/pith-number/5JRUKBBC5R6E6LXVRVCXH7PWW6/events.json","paper":"https://pith.science/paper/5JRUKBBC"},"agent_actions":{"view_html":"https://pith.science/pith/5JRUKBBC5R6E6LXVRVCXH7PWW6","download_json":"https://pith.science/pith/5JRUKBBC5R6E6LXVRVCXH7PWW6.json","view_paper":"https://pith.science/paper/5JRUKBBC","resolve_alias":"https://pith.science/api/pith-number/resolve?arxiv=2503.10694&json=true","fetch_graph":"https://pith.science/api/pith-number/5JRUKBBC5R6E6LXVRVCXH7PWW6/graph.json","fetch_events":"https://pith.science/api/pith-number/5JRUKBBC5R6E6LXVRVCXH7PWW6/events.json","actions":{"anchor_timestamp":"https://pith.science/pith/5JRUKBBC5R6E6LXVRVCXH7PWW6/action/timestamp_anchor","attest_storage":"https://pith.science/pith/5JRUKBBC5R6E6LXVRVCXH7PWW6/action/storage_attestation","attest_author":"https://pith.science/pith/5JRUKBBC5R6E6LXVRVCXH7PWW6/action/author_attestation","sign_citation":"https://pith.science/pith/5JRUKBBC5R6E6LXVRVCXH7PWW6/action/citation_signature","submit_replication":"https://pith.science/pith/5JRUKBBC5R6E6LXVRVCXH7PWW6/action/replication_record"}},"created_at":"2026-07-05T10:31:03.838838+00:00","updated_at":"2026-07-05T10:31:03.838838+00:00"}