{"record_type":"pith_number_record","schema_url":"https://pith.science/schemas/pith-number/v1.json","pith_number":"pith:2024:CXCY7WHFPIS7QQEVT7ZI3L2X2P","short_pith_number":"pith:CXCY7WHF","schema_version":"1.0","canonical_sha256":"15c58fd8e57a25f840959ff28daf57d3d428d903b680495d195e9d973cf23a4e","source":{"kind":"arxiv","id":"2405.00716","version":4},"attestation_state":"computed","paper":{"title":"Large Language Models in the Clinic: A Comprehensive Benchmark","license":"http://creativecommons.org/licenses/by/4.0/","headline":"","cross_cats":["cs.AI"],"primary_cat":"cs.CL","authors_text":"Anshul Thakur, Bing Yin, Chen Luo, David A. Clifton, Fenglin Liu, Haoming Jiang, Hongjian Zhou, Jingfeng Yang, Lei Clifton, Ming Zeng, Omid Rohanian, Priyanka Nigam, Qingyu Yin, Sreyashi Nag, Xianfeng Tang, Xuan Zhou, Yifan Gao, Yining Hua, Zheng Li","submitted_at":"2024-04-25T15:51:06Z","abstract_excerpt":"The adoption of large language models (LLMs) to assist clinicians has attracted remarkable attention. Existing works mainly adopt the close-ended question-answering (QA) task with answer options for evaluation. However, many clinical decisions involve answering open-ended questions without pre-set options. To better understand LLMs in the clinic, we construct a benchmark ClinicBench. We first collect eleven existing datasets covering diverse clinical language generation, understanding, and reasoning tasks. Furthermore, we construct six novel datasets and clinical tasks that are complex but com"},"verification_status":{"content_addressed":true,"pith_receipt":true,"author_attested":false,"weak_author_claims":0,"strong_author_claims":0,"externally_anchored":false,"storage_verified":false,"citation_signatures":0,"replication_records":0,"graph_snapshot":true,"references_resolved":false,"formal_links_present":false},"canonical_record":{"source":{"id":"2405.00716","kind":"arxiv","version":4},"metadata":{"license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.CL","submitted_at":"2024-04-25T15:51:06Z","cross_cats_sorted":["cs.AI"],"title_canon_sha256":"7fcaa275ebbb8fd47406a41393308de969763ac109c97f44882b257f308abbee","abstract_canon_sha256":"755c7a5f51f93e01167ee4882eabd56ed92555d4920bd305959fbc8984bf3f07"},"schema_version":"1.0"},"receipt":{"kind":"pith_receipt","key_id":"pith-v1-2026-05","algorithm":"ed25519","signed_at":"2026-07-05T09:21:04.098227Z","signature_b64":"ho2FpQaudhWS/gTQJUm8lEkfOxxkkJM0Pr/J6apLPCNJgqpNxmZ4c6EogMLjxv8yaCOEm5lmfqzWD01sm4lXCw==","signed_message":"canonical_sha256_bytes","builder_version":"pith-number-builder-2026-05-17-v1","receipt_version":"0.3","canonical_sha256":"15c58fd8e57a25f840959ff28daf57d3d428d903b680495d195e9d973cf23a4e","last_reissued_at":"2026-07-05T09:21:04.097705Z","signature_status":"signed_v1","first_computed_at":"2026-07-05T09:21:04.097705Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"graph_snapshot":{"paper":{"title":"Large Language Models in the Clinic: A Comprehensive Benchmark","license":"http://creativecommons.org/licenses/by/4.0/","headline":"","cross_cats":["cs.AI"],"primary_cat":"cs.CL","authors_text":"Anshul Thakur, Bing Yin, Chen Luo, David A. Clifton, Fenglin Liu, Haoming Jiang, Hongjian Zhou, Jingfeng Yang, Lei Clifton, Ming Zeng, Omid Rohanian, Priyanka Nigam, Qingyu Yin, Sreyashi Nag, Xianfeng Tang, Xuan Zhou, Yifan Gao, Yining Hua, Zheng Li","submitted_at":"2024-04-25T15:51:06Z","abstract_excerpt":"The adoption of large language models (LLMs) to assist clinicians has attracted remarkable attention. Existing works mainly adopt the close-ended question-answering (QA) task with answer options for evaluation. However, many clinical decisions involve answering open-ended questions without pre-set options. To better understand LLMs in the clinic, we construct a benchmark ClinicBench. We first collect eleven existing datasets covering diverse clinical language generation, understanding, and reasoning tasks. Furthermore, we construct six novel datasets and clinical tasks that are complex but com"},"claims":{"count":0,"items":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"2405.00716","kind":"arxiv","version":4},"verdict":{"id":null,"model_set":{},"created_at":null,"strongest_claim":"","one_line_summary":"","pipeline_version":null,"weakest_assumption":"","pith_extraction_headline":""},"integrity":{"clean":true,"summary":{"advisory":0,"critical":0,"by_detector":{},"informational":0},"endpoint":"/pith/2405.00716/integrity.json","findings":[],"available":true,"detectors_run":[],"snapshot_sha256":"c28c3603d3b5d939e8dc4c7e95fa8dfce3d595e45f758748cecf8e644a296938"},"references":{"count":0,"sample":[],"resolved_work":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","internal_anchors":0},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"author_claims":{"count":0,"strong_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"builder_version":"pith-number-builder-2026-05-17-v1"},"aliases":[{"alias_kind":"arxiv","alias_value":"2405.00716","created_at":"2026-07-05T09:21:04.097772+00:00"},{"alias_kind":"arxiv_version","alias_value":"2405.00716v4","created_at":"2026-07-05T09:21:04.097772+00:00"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.2405.00716","created_at":"2026-07-05T09:21:04.097772+00:00"},{"alias_kind":"pith_short_12","alias_value":"CXCY7WHFPIS7","created_at":"2026-07-05T09:21:04.097772+00:00"},{"alias_kind":"pith_short_16","alias_value":"CXCY7WHFPIS7QQEV","created_at":"2026-07-05T09:21:04.097772+00:00"},{"alias_kind":"pith_short_8","alias_value":"CXCY7WHF","created_at":"2026-07-05T09:21:04.097772+00:00"}],"events":[],"event_summary":{},"paper_claims":[],"inbound_citations":{"count":4,"internal_anchor_count":0,"sample":[{"citing_arxiv_id":"2606.18471","citing_title":"Possible or Definite? A Benchmark for Evaluating Diagnostic Uncertainty Preservation in Clinical Text","ref_index":32,"is_internal_anchor":false},{"citing_arxiv_id":"2606.28332","citing_title":"When Medical Safety Alignment Fails: A Benchmark for Evaluating LLMs on High-Risk Medical Queries","ref_index":56,"is_internal_anchor":false},{"citing_arxiv_id":"2605.30637","citing_title":"EHRBench: An Automated and Reliable EHR-based Benchmark for Clinical Decision Making with LLMs","ref_index":57,"is_internal_anchor":false},{"citing_arxiv_id":"2605.13936","citing_title":"Towards the Next Frontier of LLMs, Training on Private Data: A Cross-Domain Benchmark for Federated Fine-Tuning","ref_index":2,"is_internal_anchor":false}]},"formal_canon":{"evidence_count":0,"sample":[],"anchors":[]},"links":{"html":"https://pith.science/pith/CXCY7WHFPIS7QQEVT7ZI3L2X2P","json":"https://pith.science/pith/CXCY7WHFPIS7QQEVT7ZI3L2X2P.json","graph_json":"https://pith.science/api/pith-number/CXCY7WHFPIS7QQEVT7ZI3L2X2P/graph.json","events_json":"https://pith.science/api/pith-number/CXCY7WHFPIS7QQEVT7ZI3L2X2P/events.json","paper":"https://pith.science/paper/CXCY7WHF"},"agent_actions":{"view_html":"https://pith.science/pith/CXCY7WHFPIS7QQEVT7ZI3L2X2P","download_json":"https://pith.science/pith/CXCY7WHFPIS7QQEVT7ZI3L2X2P.json","view_paper":"https://pith.science/paper/CXCY7WHF","resolve_alias":"https://pith.science/api/pith-number/resolve?arxiv=2405.00716&json=true","fetch_graph":"https://pith.science/api/pith-number/CXCY7WHFPIS7QQEVT7ZI3L2X2P/graph.json","fetch_events":"https://pith.science/api/pith-number/CXCY7WHFPIS7QQEVT7ZI3L2X2P/events.json","actions":{"anchor_timestamp":"https://pith.science/pith/CXCY7WHFPIS7QQEVT7ZI3L2X2P/action/timestamp_anchor","attest_storage":"https://pith.science/pith/CXCY7WHFPIS7QQEVT7ZI3L2X2P/action/storage_attestation","attest_author":"https://pith.science/pith/CXCY7WHFPIS7QQEVT7ZI3L2X2P/action/author_attestation","sign_citation":"https://pith.science/pith/CXCY7WHFPIS7QQEVT7ZI3L2X2P/action/citation_signature","submit_replication":"https://pith.science/pith/CXCY7WHFPIS7QQEVT7ZI3L2X2P/action/replication_record"}},"created_at":"2026-07-05T09:21:04.097772+00:00","updated_at":"2026-07-05T09:21:04.097772+00:00"}