{"bundle_type":"pith_open_graph_bundle","bundle_version":"1.0","pith_number":"pith:2025:5JRUKBBC5R6E6LXVRVCXH7PWW6","short_pith_number":"pith:5JRUKBBC","canonical_record":{"source":{"id":"2503.10694","kind":"arxiv","version":1},"metadata":{"license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.CL","submitted_at":"2025-03-12T05:08:02Z","cross_cats_sorted":[],"title_canon_sha256":"b015ebec658297ccd4648d1da60ad81c3cb3a186b36777e865389542b264da5f","abstract_canon_sha256":"2bbf3136f8c2c093ade21956320fb64938e047594fbbde29826c7e4e82b0ab48"},"schema_version":"1.0"},"canonical_sha256":"ea63450422ec7c4f2ef58d4573fdf6b78c4621cc2ff752d7a705a160284ea579","source":{"kind":"arxiv","id":"2503.10694","version":1},"source_aliases":[{"alias_kind":"arxiv","alias_value":"2503.10694","created_at":"2026-07-05T10:31:03Z"},{"alias_kind":"arxiv_version","alias_value":"2503.10694v1","created_at":"2026-07-05T10:31:03Z"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.2503.10694","created_at":"2026-07-05T10:31:03Z"},{"alias_kind":"pith_short_12","alias_value":"5JRUKBBC5R6E","created_at":"2026-07-05T10:31:03Z"},{"alias_kind":"pith_short_16","alias_value":"5JRUKBBC5R6E6LXV","created_at":"2026-07-05T10:31:03Z"},{"alias_kind":"pith_short_8","alias_value":"5JRUKBBC","created_at":"2026-07-05T10:31:03Z"}],"events":[{"event_type":"record_created","subject_pith_number":"pith:2025:5JRUKBBC5R6E6LXVRVCXH7PWW6","target":"record","payload":{"canonical_record":{"source":{"id":"2503.10694","kind":"arxiv","version":1},"metadata":{"license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.CL","submitted_at":"2025-03-12T05:08:02Z","cross_cats_sorted":[],"title_canon_sha256":"b015ebec658297ccd4648d1da60ad81c3cb3a186b36777e865389542b264da5f","abstract_canon_sha256":"2bbf3136f8c2c093ade21956320fb64938e047594fbbde29826c7e4e82b0ab48"},"schema_version":"1.0"},"canonical_sha256":"ea63450422ec7c4f2ef58d4573fdf6b78c4621cc2ff752d7a705a160284ea579","receipt":{"kind":"pith_receipt","key_id":"pith-v1-2026-05","algorithm":"ed25519","signed_at":"2026-07-05T10:31:03.839391Z","signature_b64":"fs9wXiZb7BgZiVR1L0SYSU5h7kRJAHb61G4AHd/XLtndHleQAS2HgTp1tzwPntbB8rAl9Nrb3eeIYuNpN6q3Cw==","signed_message":"canonical_sha256_bytes","builder_version":"pith-number-builder-2026-05-17-v1","receipt_version":"0.3","canonical_sha256":"ea63450422ec7c4f2ef58d4573fdf6b78c4621cc2ff752d7a705a160284ea579","last_reissued_at":"2026-07-05T10:31:03.838757Z","signature_status":"signed_v1","first_computed_at":"2026-07-05T10:31:03.838757Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"source_kind":"arxiv","source_id":"2503.10694","source_version":1,"attestation_state":"computed"},"signer":{"signer_id":"pith.science","signer_type":"pith_registry","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"created_at":"2026-07-05T10:31:03Z","supersedes":[],"prev_event":null,"signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"ZSVtihtcR0IsZo59J83BwyZiJbrr04Mq1HRxKFMqIl8Ijgh0G/XJW0nPwNalSyiP7KLOmc6w0FIs9cDu+Iz4BA==","signed_message":"open_graph_event_sha256_bytes","signed_at":"2026-08-04T22:59:52.408128Z"},"content_sha256":"a4fa0f19d7c2cedcff5ac2bbf316675377b17b771188ceaa9456b98ea8a9e065","schema_version":"1.0","event_id":"sha256:a4fa0f19d7c2cedcff5ac2bbf316675377b17b771188ceaa9456b98ea8a9e065"},{"event_type":"graph_snapshot","subject_pith_number":"pith:2025:5JRUKBBC5R6E6LXVRVCXH7PWW6","target":"graph","payload":{"graph_snapshot":{"paper":{"title":"Medical Large Language Model Benchmarks Should Prioritize Construct Validity","license":"http://creativecommons.org/licenses/by/4.0/","headline":"","cross_cats":[],"primary_cat":"cs.CL","authors_text":"Ahmed Alaa, Frances Dean, Inioluwa Deborah Raji, Niloufar Golchini, Shiladitya Dutta, Thomas Hartvigsen, Travis Zack","submitted_at":"2025-03-12T05:08:02Z","abstract_excerpt":"Medical large language models (LLMs) research often makes bold claims, from encoding clinical knowledge to reasoning like a physician. These claims are usually backed by evaluation on competitive benchmarks; a tradition inherited from mainstream machine learning. But how do we separate real progress from a leaderboard flex? Medical LLM benchmarks, much like those in other fields, are arbitrarily constructed using medical licensing exam questions. For these benchmarks to truly measure progress, they must accurately capture the real-world tasks they aim to represent. In this position paper, we a"},"claims":{"count":0,"items":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"2503.10694","kind":"arxiv","version":1},"verdict":{"id":null,"model_set":{},"created_at":null,"strongest_claim":"","one_line_summary":"","pipeline_version":null,"weakest_assumption":"","pith_extraction_headline":""},"integrity":{"clean":true,"summary":{"advisory":0,"critical":0,"by_detector":{},"informational":0},"endpoint":"/pith/2503.10694/integrity.json","findings":[],"available":true,"detectors_run":[],"snapshot_sha256":"c28c3603d3b5d939e8dc4c7e95fa8dfce3d595e45f758748cecf8e644a296938"},"references":{"count":0,"sample":[],"resolved_work":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","internal_anchors":0},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"author_claims":{"count":0,"strong_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"builder_version":"pith-number-builder-2026-05-17-v1"},"verdict_id":null},"signer":{"signer_id":"pith.science","signer_type":"pith_registry","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"created_at":"2026-07-05T10:31:03Z","supersedes":[],"prev_event":null,"signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"I9ZrebDDkMkpyBn6KbCr8j0+uuIvHrs25qUGJXDmasptXqexyYTVG1q8h+ti4sDEs8ZZIUBzCi3WtvhJP8sGAQ==","signed_message":"open_graph_event_sha256_bytes","signed_at":"2026-08-04T22:59:52.408797Z"},"content_sha256":"c9b13e78c6740ca772c1fb0c2bdf6404e0301a1de73577e9358496ebd987e6e6","schema_version":"1.0","event_id":"sha256:c9b13e78c6740ca772c1fb0c2bdf6404e0301a1de73577e9358496ebd987e6e6"}],"timestamp_proofs":[],"mirror_hints":[{"mirror_type":"https","name":"Pith Resolver","base_url":"https://pith.science","bundle_url":"https://pith.science/pith/5JRUKBBC5R6E6LXVRVCXH7PWW6/bundle.json","state_url":"https://pith.science/pith/5JRUKBBC5R6E6LXVRVCXH7PWW6/state.json","well_known_bundle_url":"https://pith.science/.well-known/pith/5JRUKBBC5R6E6LXVRVCXH7PWW6/bundle.json","status":"primary"}],"public_keys":[{"key_id":"pith-v1-2026-05","algorithm":"ed25519","format":"raw","public_key_b64":"stVStoiQhXFxp4s2pdzPNoqVNBMojDU/fJ2db5S3CbM=","public_key_hex":"b2d552b68890857171a78b36a5dccf368a953413288c353f7c9d9d6f94b709b3","fingerprint_sha256_b32_first128bits":"RVFV5Z2OI2J3ZUO7ERDEBCYNKS","fingerprint_sha256_hex":"8d4b5ee74e4693bcd1df2446408b0d54","rotates_at":null,"url":"https://pith.science/pith-signing-key.json","notes":"Pith uses this Ed25519 key to sign canonical record SHA-256 digests. Verify with: ed25519_verify(public_key, message=canonical_sha256_bytes, signature=base64decode(signature_b64))."}],"merge_version":"pith-open-graph-merge-v1","built_at":"2026-08-04T22:59:52Z","links":{"resolver":"https://pith.science/pith/5JRUKBBC5R6E6LXVRVCXH7PWW6","bundle":"https://pith.science/pith/5JRUKBBC5R6E6LXVRVCXH7PWW6/bundle.json","state":"https://pith.science/pith/5JRUKBBC5R6E6LXVRVCXH7PWW6/state.json","well_known_bundle":"https://pith.science/.well-known/pith/5JRUKBBC5R6E6LXVRVCXH7PWW6/bundle.json"},"state":{"state_type":"pith_open_graph_state","state_version":"1.0","pith_number":"pith:2025:5JRUKBBC5R6E6LXVRVCXH7PWW6","merge_version":"pith-open-graph-merge-v1","event_count":2,"valid_event_count":2,"invalid_event_count":0,"equivocation_count":0,"current":{"canonical_record":{"metadata":{"abstract_canon_sha256":"2bbf3136f8c2c093ade21956320fb64938e047594fbbde29826c7e4e82b0ab48","cross_cats_sorted":[],"license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.CL","submitted_at":"2025-03-12T05:08:02Z","title_canon_sha256":"b015ebec658297ccd4648d1da60ad81c3cb3a186b36777e865389542b264da5f"},"schema_version":"1.0","source":{"id":"2503.10694","kind":"arxiv","version":1}},"source_aliases":[{"alias_kind":"arxiv","alias_value":"2503.10694","created_at":"2026-07-05T10:31:03Z"},{"alias_kind":"arxiv_version","alias_value":"2503.10694v1","created_at":"2026-07-05T10:31:03Z"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.2503.10694","created_at":"2026-07-05T10:31:03Z"},{"alias_kind":"pith_short_12","alias_value":"5JRUKBBC5R6E","created_at":"2026-07-05T10:31:03Z"},{"alias_kind":"pith_short_16","alias_value":"5JRUKBBC5R6E6LXV","created_at":"2026-07-05T10:31:03Z"},{"alias_kind":"pith_short_8","alias_value":"5JRUKBBC","created_at":"2026-07-05T10:31:03Z"}],"graph_snapshots":[{"event_id":"sha256:c9b13e78c6740ca772c1fb0c2bdf6404e0301a1de73577e9358496ebd987e6e6","target":"graph","created_at":"2026-07-05T10:31:03Z","signer":{"key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signer_id":"pith.science","signer_type":"pith_registry"},"payload":{"graph_snapshot":{"author_claims":{"count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","strong_count":0},"builder_version":"pith-number-builder-2026-05-17-v1","claims":{"count":0,"items":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"integrity":{"available":true,"clean":true,"detectors_run":[],"endpoint":"/pith/2503.10694/integrity.json","findings":[],"snapshot_sha256":"c28c3603d3b5d939e8dc4c7e95fa8dfce3d595e45f758748cecf8e644a296938","summary":{"advisory":0,"by_detector":{},"critical":0,"informational":0}},"paper":{"abstract_excerpt":"Medical large language models (LLMs) research often makes bold claims, from encoding clinical knowledge to reasoning like a physician. These claims are usually backed by evaluation on competitive benchmarks; a tradition inherited from mainstream machine learning. But how do we separate real progress from a leaderboard flex? Medical LLM benchmarks, much like those in other fields, are arbitrarily constructed using medical licensing exam questions. For these benchmarks to truly measure progress, they must accurately capture the real-world tasks they aim to represent. In this position paper, we a","authors_text":"Ahmed Alaa, Frances Dean, Inioluwa Deborah Raji, Niloufar Golchini, Shiladitya Dutta, Thomas Hartvigsen, Travis Zack","cross_cats":[],"headline":"","license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.CL","submitted_at":"2025-03-12T05:08:02Z","title":"Medical Large Language Model Benchmarks Should Prioritize Construct Validity"},"references":{"count":0,"internal_anchors":0,"resolved_work":0,"sample":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"2503.10694","kind":"arxiv","version":1},"verdict":{"created_at":null,"id":null,"model_set":{},"one_line_summary":"","pipeline_version":null,"pith_extraction_headline":"","strongest_claim":"","weakest_assumption":""}},"verdict_id":null}}],"author_attestations":[],"timestamp_anchors":[],"storage_attestations":[],"citation_signatures":[],"replication_records":[],"corrections":[],"mirror_hints":[],"record_created":{"event_id":"sha256:a4fa0f19d7c2cedcff5ac2bbf316675377b17b771188ceaa9456b98ea8a9e065","target":"record","created_at":"2026-07-05T10:31:03Z","signer":{"key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signer_id":"pith.science","signer_type":"pith_registry"},"payload":{"attestation_state":"computed","canonical_record":{"metadata":{"abstract_canon_sha256":"2bbf3136f8c2c093ade21956320fb64938e047594fbbde29826c7e4e82b0ab48","cross_cats_sorted":[],"license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.CL","submitted_at":"2025-03-12T05:08:02Z","title_canon_sha256":"b015ebec658297ccd4648d1da60ad81c3cb3a186b36777e865389542b264da5f"},"schema_version":"1.0","source":{"id":"2503.10694","kind":"arxiv","version":1}},"canonical_sha256":"ea63450422ec7c4f2ef58d4573fdf6b78c4621cc2ff752d7a705a160284ea579","receipt":{"algorithm":"ed25519","builder_version":"pith-number-builder-2026-05-17-v1","canonical_sha256":"ea63450422ec7c4f2ef58d4573fdf6b78c4621cc2ff752d7a705a160284ea579","first_computed_at":"2026-07-05T10:31:03.838757Z","key_id":"pith-v1-2026-05","kind":"pith_receipt","last_reissued_at":"2026-07-05T10:31:03.838757Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","receipt_version":"0.3","signature_b64":"fs9wXiZb7BgZiVR1L0SYSU5h7kRJAHb61G4AHd/XLtndHleQAS2HgTp1tzwPntbB8rAl9Nrb3eeIYuNpN6q3Cw==","signature_status":"signed_v1","signed_at":"2026-07-05T10:31:03.839391Z","signed_message":"canonical_sha256_bytes"},"source_id":"2503.10694","source_kind":"arxiv","source_version":1}}},"equivocations":[],"invalid_events":[],"applied_event_ids":["sha256:a4fa0f19d7c2cedcff5ac2bbf316675377b17b771188ceaa9456b98ea8a9e065","sha256:c9b13e78c6740ca772c1fb0c2bdf6404e0301a1de73577e9358496ebd987e6e6"],"state_sha256":"20401b3b5663504eb9b386bc8ffd9628f85ed71ec90d2cdc752d6e88fdd899af"},"bundle_signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"JRb7xENn+I6Mmlf/cE0RKltsZFqZQ7gooKmLjmO56t8CdTdnWrPBVUm+iEVKLhRuLuJYTPkNq+N9wbc/+ApqAQ==","signed_message":"bundle_sha256_bytes","signed_at":"2026-08-04T22:59:52.414986Z","bundle_sha256":"d40d91562cf06ba2b5a13b1312b0392391ec65eb8510fc69e9c788fdedefb01d"}}