{"bundle_type":"pith_open_graph_bundle","bundle_version":"1.0","pith_number":"pith:2025:MSLRSR3MTSAHDW7QNMSIX2WXNG","short_pith_number":"pith:MSLRSR3M","canonical_record":{"source":{"id":"2505.23802","kind":"arxiv","version":2},"metadata":{"license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.CL","submitted_at":"2025-05-26T22:55:49Z","cross_cats_sorted":["cs.AI"],"title_canon_sha256":"a19084995afa31ab5ccde2a1e0a840b0f3f15eb6fe01051fa5cc3caaaad2d85e","abstract_canon_sha256":"56d5f3e9e7266a1d5fb605e308e95ae04d1c5f810305abf54c0d0f37e9d5c928"},"schema_version":"1.0"},"canonical_sha256":"649719476c9c8071dbf06b248bead769bebcbca10bf36aa7b7f30a82a1865720","source":{"kind":"arxiv","id":"2505.23802","version":2},"source_aliases":[{"alias_kind":"arxiv","alias_value":"2505.23802","created_at":"2026-07-05T11:14:08Z"},{"alias_kind":"arxiv_version","alias_value":"2505.23802v2","created_at":"2026-07-05T11:14:08Z"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.2505.23802","created_at":"2026-07-05T11:14:08Z"},{"alias_kind":"pith_short_12","alias_value":"MSLRSR3MTSAH","created_at":"2026-07-05T11:14:08Z"},{"alias_kind":"pith_short_16","alias_value":"MSLRSR3MTSAHDW7Q","created_at":"2026-07-05T11:14:08Z"},{"alias_kind":"pith_short_8","alias_value":"MSLRSR3M","created_at":"2026-07-05T11:14:08Z"}],"events":[{"event_type":"record_created","subject_pith_number":"pith:2025:MSLRSR3MTSAHDW7QNMSIX2WXNG","target":"record","payload":{"canonical_record":{"source":{"id":"2505.23802","kind":"arxiv","version":2},"metadata":{"license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.CL","submitted_at":"2025-05-26T22:55:49Z","cross_cats_sorted":["cs.AI"],"title_canon_sha256":"a19084995afa31ab5ccde2a1e0a840b0f3f15eb6fe01051fa5cc3caaaad2d85e","abstract_canon_sha256":"56d5f3e9e7266a1d5fb605e308e95ae04d1c5f810305abf54c0d0f37e9d5c928"},"schema_version":"1.0"},"canonical_sha256":"649719476c9c8071dbf06b248bead769bebcbca10bf36aa7b7f30a82a1865720","receipt":{"kind":"pith_receipt","key_id":"pith-v1-2026-05","algorithm":"ed25519","signed_at":"2026-07-05T11:14:08.832800Z","signature_b64":"M6p9/Yn3MAP8OvxnVhRYeOIEBR695lOH2OXh0azbqfyFGW5hfhSIRdfyNSyNsUJPpTEVVXjDnxZJZEd1bp78Bg==","signed_message":"canonical_sha256_bytes","builder_version":"pith-number-builder-2026-05-17-v1","receipt_version":"0.3","canonical_sha256":"649719476c9c8071dbf06b248bead769bebcbca10bf36aa7b7f30a82a1865720","last_reissued_at":"2026-07-05T11:14:08.832278Z","signature_status":"signed_v1","first_computed_at":"2026-07-05T11:14:08.832278Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"source_kind":"arxiv","source_id":"2505.23802","source_version":2,"attestation_state":"computed"},"signer":{"signer_id":"pith.science","signer_type":"pith_registry","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"created_at":"2026-07-05T11:14:08Z","supersedes":[],"prev_event":null,"signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"DSIcUiMi/1TTYrTx9X+ED5FGa+9z5rqwlz1Wmmf26b4ZqW38C34Bf84VWFkNHgqHMbTT05JgZV6wioJvPuyyCg==","signed_message":"open_graph_event_sha256_bytes","signed_at":"2026-07-06T23:09:51.466350Z"},"content_sha256":"980f846b03763b684877698a0777de1462eedf65678befd7a935fe5f45299376","schema_version":"1.0","event_id":"sha256:980f846b03763b684877698a0777de1462eedf65678befd7a935fe5f45299376"},{"event_type":"graph_snapshot","subject_pith_number":"pith:2025:MSLRSR3MTSAHDW7QNMSIX2WXNG","target":"graph","payload":{"graph_snapshot":{"paper":{"title":"MedHELM: Holistic Evaluation of Large Language Models for Medical Tasks","license":"http://creativecommons.org/licenses/by/4.0/","headline":"","cross_cats":["cs.AI"],"primary_cat":"cs.CL","authors_text":"Aditya Sharma, Akshay Chaudhari, Akshay Swaminathan, Albert S. Chiou, Alex Alekseyev, Alyssa Unell, Anurang Revri, Asad Aali, Ashwin Nayak, Aydin Zahedivash, Bethel Mieso, Bilal Mawji, Birju Patel, Brian Soetikno, Carlene Lugtu, Chia-Chun Chiang, Christy Hong, Danton Char, Dev Dash, Dong-han Yao, Eduardo Reis, Emily Alsentzer, Eric Horvitz, Ethan Goh, Faraz Ghoddusi, Fateme Nateghi, Francois Grolleau, Hannah Kirsch, Hao Qiu, Harshita Sharma, Hejie Cui, Hinesh Patel, Jarrod Helzer, Jason Alan Fries, Jenelle Jindal, Jennifer Lee, Jonathan Chen, Juan M. Banda, Kameron Black, Keith Morse, Kevin Schulman, Lance Downing, Leonardo Schettini, Matthew P. Lungren, Mehr Kashyap, Mert Oez, Michael F. Gensheimer, Michael Wornow, Miguel Fuentes, Mike Pfeffer, Mohana Roy, Nerissa Ambers, Nigam H. Shah, Nikesh Kotecha, Nima Aghaeepour, Nirmal Ravi, Oluseyi Fayanju, Percy Liang, Philip Chung, Rachna Saralkar, Robson Capasso, Roxana Daneshjou, Sanmi Koyejo, Sergios Gatidis, Shivam Vedak, Shreya Shah, Shrey Jain, Sneha S. Jain, Steven Lin, Suhana Bedi, Thomas Wang, Tho Pham, Timothy Keyes, Tony Lee, Vanessa Kennedy, Vasu Divi, Vicky Zhou, Vikas Kakkar, Wen-wai Yim, Yair Bannett, Yifan Mai","submitted_at":"2025-05-26T22:55:49Z","abstract_excerpt":"While large language models (LLMs) achieve near-perfect scores on medical licensing exams, these evaluations inadequately reflect the complexity and diversity of real-world clinical practice. We introduce MedHELM, an extensible evaluation framework for assessing LLM performance for medical tasks with three key contributions. First, a clinician-validated taxonomy spanning 5 categories, 22 subcategories, and 121 tasks developed with 29 clinicians. Second, a comprehensive benchmark suite comprising 35 benchmarks (17 existing, 18 newly formulated) providing complete coverage of all categories and "},"claims":{"count":0,"items":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"2505.23802","kind":"arxiv","version":2},"verdict":{"id":null,"model_set":{},"created_at":null,"strongest_claim":"","one_line_summary":"","pipeline_version":null,"weakest_assumption":"","pith_extraction_headline":""},"integrity":{"clean":true,"summary":{"advisory":0,"critical":0,"by_detector":{},"informational":0},"endpoint":"/pith/2505.23802/integrity.json","findings":[],"available":true,"detectors_run":[],"snapshot_sha256":"c28c3603d3b5d939e8dc4c7e95fa8dfce3d595e45f758748cecf8e644a296938"},"references":{"count":0,"sample":[],"resolved_work":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","internal_anchors":0},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"author_claims":{"count":0,"strong_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"builder_version":"pith-number-builder-2026-05-17-v1"},"verdict_id":null},"signer":{"signer_id":"pith.science","signer_type":"pith_registry","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"created_at":"2026-07-05T11:14:08Z","supersedes":[],"prev_event":null,"signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"W8Px69Fh3L33kd6lR1A3OGlFrbBoocDD9la2v46n7lcfWK4wXev2uKRgwpeHVO+kTGI+3IIEw5G2QmUnWEZyCQ==","signed_message":"open_graph_event_sha256_bytes","signed_at":"2026-07-06T23:09:51.466780Z"},"content_sha256":"b4cdb3454ac7254e5555f3d1212151010a16db6135569664ea5dc88a3d42ab31","schema_version":"1.0","event_id":"sha256:b4cdb3454ac7254e5555f3d1212151010a16db6135569664ea5dc88a3d42ab31"}],"timestamp_proofs":[],"mirror_hints":[{"mirror_type":"https","name":"Pith Resolver","base_url":"https://pith.science","bundle_url":"https://pith.science/pith/MSLRSR3MTSAHDW7QNMSIX2WXNG/bundle.json","state_url":"https://pith.science/pith/MSLRSR3MTSAHDW7QNMSIX2WXNG/state.json","well_known_bundle_url":"https://pith.science/.well-known/pith/MSLRSR3MTSAHDW7QNMSIX2WXNG/bundle.json","status":"primary"}],"public_keys":[{"key_id":"pith-v1-2026-05","algorithm":"ed25519","format":"raw","public_key_b64":"stVStoiQhXFxp4s2pdzPNoqVNBMojDU/fJ2db5S3CbM=","public_key_hex":"b2d552b68890857171a78b36a5dccf368a953413288c353f7c9d9d6f94b709b3","fingerprint_sha256_b32_first128bits":"RVFV5Z2OI2J3ZUO7ERDEBCYNKS","fingerprint_sha256_hex":"8d4b5ee74e4693bcd1df2446408b0d54","rotates_at":null,"url":"https://pith.science/pith-signing-key.json","notes":"Pith uses this Ed25519 key to sign canonical record SHA-256 digests. Verify with: ed25519_verify(public_key, message=canonical_sha256_bytes, signature=base64decode(signature_b64))."}],"merge_version":"pith-open-graph-merge-v1","built_at":"2026-07-06T23:09:51Z","links":{"resolver":"https://pith.science/pith/MSLRSR3MTSAHDW7QNMSIX2WXNG","bundle":"https://pith.science/pith/MSLRSR3MTSAHDW7QNMSIX2WXNG/bundle.json","state":"https://pith.science/pith/MSLRSR3MTSAHDW7QNMSIX2WXNG/state.json","well_known_bundle":"https://pith.science/.well-known/pith/MSLRSR3MTSAHDW7QNMSIX2WXNG/bundle.json"},"state":{"state_type":"pith_open_graph_state","state_version":"1.0","pith_number":"pith:2025:MSLRSR3MTSAHDW7QNMSIX2WXNG","merge_version":"pith-open-graph-merge-v1","event_count":2,"valid_event_count":2,"invalid_event_count":0,"equivocation_count":0,"current":{"canonical_record":{"metadata":{"abstract_canon_sha256":"56d5f3e9e7266a1d5fb605e308e95ae04d1c5f810305abf54c0d0f37e9d5c928","cross_cats_sorted":["cs.AI"],"license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.CL","submitted_at":"2025-05-26T22:55:49Z","title_canon_sha256":"a19084995afa31ab5ccde2a1e0a840b0f3f15eb6fe01051fa5cc3caaaad2d85e"},"schema_version":"1.0","source":{"id":"2505.23802","kind":"arxiv","version":2}},"source_aliases":[{"alias_kind":"arxiv","alias_value":"2505.23802","created_at":"2026-07-05T11:14:08Z"},{"alias_kind":"arxiv_version","alias_value":"2505.23802v2","created_at":"2026-07-05T11:14:08Z"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.2505.23802","created_at":"2026-07-05T11:14:08Z"},{"alias_kind":"pith_short_12","alias_value":"MSLRSR3MTSAH","created_at":"2026-07-05T11:14:08Z"},{"alias_kind":"pith_short_16","alias_value":"MSLRSR3MTSAHDW7Q","created_at":"2026-07-05T11:14:08Z"},{"alias_kind":"pith_short_8","alias_value":"MSLRSR3M","created_at":"2026-07-05T11:14:08Z"}],"graph_snapshots":[{"event_id":"sha256:b4cdb3454ac7254e5555f3d1212151010a16db6135569664ea5dc88a3d42ab31","target":"graph","created_at":"2026-07-05T11:14:08Z","signer":{"key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signer_id":"pith.science","signer_type":"pith_registry"},"payload":{"graph_snapshot":{"author_claims":{"count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","strong_count":0},"builder_version":"pith-number-builder-2026-05-17-v1","claims":{"count":0,"items":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"integrity":{"available":true,"clean":true,"detectors_run":[],"endpoint":"/pith/2505.23802/integrity.json","findings":[],"snapshot_sha256":"c28c3603d3b5d939e8dc4c7e95fa8dfce3d595e45f758748cecf8e644a296938","summary":{"advisory":0,"by_detector":{},"critical":0,"informational":0}},"paper":{"abstract_excerpt":"While large language models (LLMs) achieve near-perfect scores on medical licensing exams, these evaluations inadequately reflect the complexity and diversity of real-world clinical practice. We introduce MedHELM, an extensible evaluation framework for assessing LLM performance for medical tasks with three key contributions. First, a clinician-validated taxonomy spanning 5 categories, 22 subcategories, and 121 tasks developed with 29 clinicians. Second, a comprehensive benchmark suite comprising 35 benchmarks (17 existing, 18 newly formulated) providing complete coverage of all categories and ","authors_text":"Aditya Sharma, Akshay Chaudhari, Akshay Swaminathan, Albert S. Chiou, Alex Alekseyev, Alyssa Unell, Anurang Revri, Asad Aali, Ashwin Nayak, Aydin Zahedivash, Bethel Mieso, Bilal Mawji, Birju Patel, Brian Soetikno, Carlene Lugtu, Chia-Chun Chiang, Christy Hong, Danton Char, Dev Dash, Dong-han Yao, Eduardo Reis, Emily Alsentzer, Eric Horvitz, Ethan Goh, Faraz Ghoddusi, Fateme Nateghi, Francois Grolleau, Hannah Kirsch, Hao Qiu, Harshita Sharma, Hejie Cui, Hinesh Patel, Jarrod Helzer, Jason Alan Fries, Jenelle Jindal, Jennifer Lee, Jonathan Chen, Juan M. Banda, Kameron Black, Keith Morse, Kevin Schulman, Lance Downing, Leonardo Schettini, Matthew P. Lungren, Mehr Kashyap, Mert Oez, Michael F. Gensheimer, Michael Wornow, Miguel Fuentes, Mike Pfeffer, Mohana Roy, Nerissa Ambers, Nigam H. Shah, Nikesh Kotecha, Nima Aghaeepour, Nirmal Ravi, Oluseyi Fayanju, Percy Liang, Philip Chung, Rachna Saralkar, Robson Capasso, Roxana Daneshjou, Sanmi Koyejo, Sergios Gatidis, Shivam Vedak, Shreya Shah, Shrey Jain, Sneha S. Jain, Steven Lin, Suhana Bedi, Thomas Wang, Tho Pham, Timothy Keyes, Tony Lee, Vanessa Kennedy, Vasu Divi, Vicky Zhou, Vikas Kakkar, Wen-wai Yim, Yair Bannett, Yifan Mai","cross_cats":["cs.AI"],"headline":"","license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.CL","submitted_at":"2025-05-26T22:55:49Z","title":"MedHELM: Holistic Evaluation of Large Language Models for Medical Tasks"},"references":{"count":0,"internal_anchors":0,"resolved_work":0,"sample":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"2505.23802","kind":"arxiv","version":2},"verdict":{"created_at":null,"id":null,"model_set":{},"one_line_summary":"","pipeline_version":null,"pith_extraction_headline":"","strongest_claim":"","weakest_assumption":""}},"verdict_id":null}}],"author_attestations":[],"timestamp_anchors":[],"storage_attestations":[],"citation_signatures":[],"replication_records":[],"corrections":[],"mirror_hints":[],"record_created":{"event_id":"sha256:980f846b03763b684877698a0777de1462eedf65678befd7a935fe5f45299376","target":"record","created_at":"2026-07-05T11:14:08Z","signer":{"key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signer_id":"pith.science","signer_type":"pith_registry"},"payload":{"attestation_state":"computed","canonical_record":{"metadata":{"abstract_canon_sha256":"56d5f3e9e7266a1d5fb605e308e95ae04d1c5f810305abf54c0d0f37e9d5c928","cross_cats_sorted":["cs.AI"],"license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.CL","submitted_at":"2025-05-26T22:55:49Z","title_canon_sha256":"a19084995afa31ab5ccde2a1e0a840b0f3f15eb6fe01051fa5cc3caaaad2d85e"},"schema_version":"1.0","source":{"id":"2505.23802","kind":"arxiv","version":2}},"canonical_sha256":"649719476c9c8071dbf06b248bead769bebcbca10bf36aa7b7f30a82a1865720","receipt":{"algorithm":"ed25519","builder_version":"pith-number-builder-2026-05-17-v1","canonical_sha256":"649719476c9c8071dbf06b248bead769bebcbca10bf36aa7b7f30a82a1865720","first_computed_at":"2026-07-05T11:14:08.832278Z","key_id":"pith-v1-2026-05","kind":"pith_receipt","last_reissued_at":"2026-07-05T11:14:08.832278Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","receipt_version":"0.3","signature_b64":"M6p9/Yn3MAP8OvxnVhRYeOIEBR695lOH2OXh0azbqfyFGW5hfhSIRdfyNSyNsUJPpTEVVXjDnxZJZEd1bp78Bg==","signature_status":"signed_v1","signed_at":"2026-07-05T11:14:08.832800Z","signed_message":"canonical_sha256_bytes"},"source_id":"2505.23802","source_kind":"arxiv","source_version":2}}},"equivocations":[],"invalid_events":[],"applied_event_ids":["sha256:980f846b03763b684877698a0777de1462eedf65678befd7a935fe5f45299376","sha256:b4cdb3454ac7254e5555f3d1212151010a16db6135569664ea5dc88a3d42ab31"],"state_sha256":"9599cb230d64188fffd875c4d6705da956e99e47a1db926c978cc42e862b1f64"},"bundle_signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"JjvanLczsCNaQtqi6hqWVk+JaYroN1aojZYpZG5T3AOzcR2ycYHW+BKayZtVZcZQGOAC5T7m+XEOXfcXrxSKAg==","signed_message":"bundle_sha256_bytes","signed_at":"2026-07-06T23:09:51.468819Z","bundle_sha256":"d5e968dc3d91350974fae1729ef8c23592e564465f6b247307ccae4659a1720d"}}