{"bundle_type":"pith_open_graph_bundle","bundle_version":"1.0","pith_number":"pith:2024:J7TFPIKQF4IIGHHRAJW546V22T","short_pith_number":"pith:J7TFPIKQ","canonical_record":{"source":{"id":"2406.06573","kind":"arxiv","version":2},"metadata":{"license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.CL","submitted_at":"2024-06-03T18:15:56Z","cross_cats_sorted":["cs.LG"],"title_canon_sha256":"0689c01bd6b9fe751d55661c96bfddd7b5a172264fbb8c9dfdf9c79d8398c46c","abstract_canon_sha256":"43b4e4a354f8590d7e598d9c2cb5497dddf9ff7019294c34f5f66da48052fddb"},"schema_version":"1.0"},"canonical_sha256":"4fe657a1502f10831cf1026dde7abad4e0af0f44bdd8352a5a4f90cdae63f57d","source":{"kind":"arxiv","id":"2406.06573","version":2},"source_aliases":[{"alias_kind":"arxiv","alias_value":"2406.06573","created_at":"2026-07-05T09:01:35Z"},{"alias_kind":"arxiv_version","alias_value":"2406.06573v2","created_at":"2026-07-05T09:01:35Z"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.2406.06573","created_at":"2026-07-05T09:01:35Z"},{"alias_kind":"pith_short_12","alias_value":"J7TFPIKQF4II","created_at":"2026-07-05T09:01:35Z"},{"alias_kind":"pith_short_16","alias_value":"J7TFPIKQF4IIGHHR","created_at":"2026-07-05T09:01:35Z"},{"alias_kind":"pith_short_8","alias_value":"J7TFPIKQ","created_at":"2026-07-05T09:01:35Z"}],"events":[{"event_type":"record_created","subject_pith_number":"pith:2024:J7TFPIKQF4IIGHHRAJW546V22T","target":"record","payload":{"canonical_record":{"source":{"id":"2406.06573","kind":"arxiv","version":2},"metadata":{"license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.CL","submitted_at":"2024-06-03T18:15:56Z","cross_cats_sorted":["cs.LG"],"title_canon_sha256":"0689c01bd6b9fe751d55661c96bfddd7b5a172264fbb8c9dfdf9c79d8398c46c","abstract_canon_sha256":"43b4e4a354f8590d7e598d9c2cb5497dddf9ff7019294c34f5f66da48052fddb"},"schema_version":"1.0"},"canonical_sha256":"4fe657a1502f10831cf1026dde7abad4e0af0f44bdd8352a5a4f90cdae63f57d","receipt":{"kind":"pith_receipt","key_id":"pith-v1-2026-05","algorithm":"ed25519","signed_at":"2026-07-05T09:01:35.224693Z","signature_b64":"HIjIZI4ZQMD2fuzM8jXJzr95geDWqkg/ZnAJT9gs0BMGiv5YatsWhwE2VuJyCaHufzBKaKWi4lmdPQyAoF9bDg==","signed_message":"canonical_sha256_bytes","builder_version":"pith-number-builder-2026-05-17-v1","receipt_version":"0.3","canonical_sha256":"4fe657a1502f10831cf1026dde7abad4e0af0f44bdd8352a5a4f90cdae63f57d","last_reissued_at":"2026-07-05T09:01:35.224264Z","signature_status":"signed_v1","first_computed_at":"2026-07-05T09:01:35.224264Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"source_kind":"arxiv","source_id":"2406.06573","source_version":2,"attestation_state":"computed"},"signer":{"signer_id":"pith.science","signer_type":"pith_registry","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"created_at":"2026-07-05T09:01:35Z","supersedes":[],"prev_event":null,"signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"nC88zQPamMcL0/mE5/jQlA8hDwLNTkswI4s5hONhONvxjGSNqcU08tddMhslBPCkvGJii1O6jReS/UD/VHkpBg==","signed_message":"open_graph_event_sha256_bytes","signed_at":"2026-08-07T08:07:59.006126Z"},"content_sha256":"69f371366b2f5925ad36589c6b030ed995c88c4818eb4d55466a669afb4db3c8","schema_version":"1.0","event_id":"sha256:69f371366b2f5925ad36589c6b030ed995c88c4818eb4d55466a669afb4db3c8"},{"event_type":"graph_snapshot","subject_pith_number":"pith:2024:J7TFPIKQF4IIGHHRAJW546V22T","target":"graph","payload":{"graph_snapshot":{"paper":{"title":"MedFuzz: Exploring the Robustness of Large Language Models in Medical Question Answering","license":"http://creativecommons.org/licenses/by/4.0/","headline":"","cross_cats":["cs.LG"],"primary_cat":"cs.CL","authors_text":"Carey E. Priebe, Eric Horvitz, Hayden Helm, Junaid Bajwa, Katie Matton, Robert Osazuwa Ness, Sheng Zhang","submitted_at":"2024-06-03T18:15:56Z","abstract_excerpt":"Large language models (LLM) have achieved impressive performance on medical question-answering benchmarks. However, high benchmark accuracy does not imply that the performance generalizes to real-world clinical settings. Medical question-answering benchmarks rely on assumptions consistent with quantifying LLM performance but that may not hold in the open world of the clinic. Yet LLMs learn broad knowledge that can help the LLM generalize to practical conditions regardless of unrealistic assumptions in celebrated benchmarks. We seek to quantify how well LLM medical question-answering benchmark "},"claims":{"count":0,"items":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"2406.06573","kind":"arxiv","version":2},"verdict":{"id":null,"model_set":{},"created_at":null,"strongest_claim":"","one_line_summary":"","pipeline_version":null,"weakest_assumption":"","pith_extraction_headline":""},"integrity":{"clean":true,"summary":{"advisory":0,"critical":0,"by_detector":{},"informational":0},"endpoint":"/pith/2406.06573/integrity.json","findings":[],"available":true,"detectors_run":[],"snapshot_sha256":"c28c3603d3b5d939e8dc4c7e95fa8dfce3d595e45f758748cecf8e644a296938"},"references":{"count":0,"sample":[],"resolved_work":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","internal_anchors":0},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"author_claims":{"count":0,"strong_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"builder_version":"pith-number-builder-2026-05-17-v1"},"verdict_id":null},"signer":{"signer_id":"pith.science","signer_type":"pith_registry","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"created_at":"2026-07-05T09:01:35Z","supersedes":[],"prev_event":null,"signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"IzeAShDHhg9wyLw2dAeBKNfIDoQ9Rkh0XLpfRfNvfiB18uENsprDtKGqpEzEV7ZuFJSKmuMiNZO4iLh2LjL2Bg==","signed_message":"open_graph_event_sha256_bytes","signed_at":"2026-08-07T08:07:59.007199Z"},"content_sha256":"6c623ffeb2bb2fca84b6344add87c18eeca215c4d9891808267345edfdb7d39b","schema_version":"1.0","event_id":"sha256:6c623ffeb2bb2fca84b6344add87c18eeca215c4d9891808267345edfdb7d39b"}],"timestamp_proofs":[],"mirror_hints":[{"mirror_type":"https","name":"Pith Resolver","base_url":"https://pith.science","bundle_url":"https://pith.science/pith/J7TFPIKQF4IIGHHRAJW546V22T/bundle.json","state_url":"https://pith.science/pith/J7TFPIKQF4IIGHHRAJW546V22T/state.json","well_known_bundle_url":"https://pith.science/.well-known/pith/J7TFPIKQF4IIGHHRAJW546V22T/bundle.json","status":"primary"}],"public_keys":[{"key_id":"pith-v1-2026-05","algorithm":"ed25519","format":"raw","public_key_b64":"stVStoiQhXFxp4s2pdzPNoqVNBMojDU/fJ2db5S3CbM=","public_key_hex":"b2d552b68890857171a78b36a5dccf368a953413288c353f7c9d9d6f94b709b3","fingerprint_sha256_b32_first128bits":"RVFV5Z2OI2J3ZUO7ERDEBCYNKS","fingerprint_sha256_hex":"8d4b5ee74e4693bcd1df2446408b0d54","rotates_at":null,"url":"https://pith.science/pith-signing-key.json","notes":"Pith uses this Ed25519 key to sign canonical record SHA-256 digests. Verify with: ed25519_verify(public_key, message=canonical_sha256_bytes, signature=base64decode(signature_b64))."}],"merge_version":"pith-open-graph-merge-v1","built_at":"2026-08-07T08:07:59Z","links":{"resolver":"https://pith.science/pith/J7TFPIKQF4IIGHHRAJW546V22T","bundle":"https://pith.science/pith/J7TFPIKQF4IIGHHRAJW546V22T/bundle.json","state":"https://pith.science/pith/J7TFPIKQF4IIGHHRAJW546V22T/state.json","well_known_bundle":"https://pith.science/.well-known/pith/J7TFPIKQF4IIGHHRAJW546V22T/bundle.json"},"state":{"state_type":"pith_open_graph_state","state_version":"1.0","pith_number":"pith:2024:J7TFPIKQF4IIGHHRAJW546V22T","merge_version":"pith-open-graph-merge-v1","event_count":2,"valid_event_count":2,"invalid_event_count":0,"equivocation_count":0,"current":{"canonical_record":{"metadata":{"abstract_canon_sha256":"43b4e4a354f8590d7e598d9c2cb5497dddf9ff7019294c34f5f66da48052fddb","cross_cats_sorted":["cs.LG"],"license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.CL","submitted_at":"2024-06-03T18:15:56Z","title_canon_sha256":"0689c01bd6b9fe751d55661c96bfddd7b5a172264fbb8c9dfdf9c79d8398c46c"},"schema_version":"1.0","source":{"id":"2406.06573","kind":"arxiv","version":2}},"source_aliases":[{"alias_kind":"arxiv","alias_value":"2406.06573","created_at":"2026-07-05T09:01:35Z"},{"alias_kind":"arxiv_version","alias_value":"2406.06573v2","created_at":"2026-07-05T09:01:35Z"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.2406.06573","created_at":"2026-07-05T09:01:35Z"},{"alias_kind":"pith_short_12","alias_value":"J7TFPIKQF4II","created_at":"2026-07-05T09:01:35Z"},{"alias_kind":"pith_short_16","alias_value":"J7TFPIKQF4IIGHHR","created_at":"2026-07-05T09:01:35Z"},{"alias_kind":"pith_short_8","alias_value":"J7TFPIKQ","created_at":"2026-07-05T09:01:35Z"}],"graph_snapshots":[{"event_id":"sha256:6c623ffeb2bb2fca84b6344add87c18eeca215c4d9891808267345edfdb7d39b","target":"graph","created_at":"2026-07-05T09:01:35Z","signer":{"key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signer_id":"pith.science","signer_type":"pith_registry"},"payload":{"graph_snapshot":{"author_claims":{"count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","strong_count":0},"builder_version":"pith-number-builder-2026-05-17-v1","claims":{"count":0,"items":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"integrity":{"available":true,"clean":true,"detectors_run":[],"endpoint":"/pith/2406.06573/integrity.json","findings":[],"snapshot_sha256":"c28c3603d3b5d939e8dc4c7e95fa8dfce3d595e45f758748cecf8e644a296938","summary":{"advisory":0,"by_detector":{},"critical":0,"informational":0}},"paper":{"abstract_excerpt":"Large language models (LLM) have achieved impressive performance on medical question-answering benchmarks. However, high benchmark accuracy does not imply that the performance generalizes to real-world clinical settings. Medical question-answering benchmarks rely on assumptions consistent with quantifying LLM performance but that may not hold in the open world of the clinic. Yet LLMs learn broad knowledge that can help the LLM generalize to practical conditions regardless of unrealistic assumptions in celebrated benchmarks. We seek to quantify how well LLM medical question-answering benchmark ","authors_text":"Carey E. Priebe, Eric Horvitz, Hayden Helm, Junaid Bajwa, Katie Matton, Robert Osazuwa Ness, Sheng Zhang","cross_cats":["cs.LG"],"headline":"","license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.CL","submitted_at":"2024-06-03T18:15:56Z","title":"MedFuzz: Exploring the Robustness of Large Language Models in Medical Question Answering"},"references":{"count":0,"internal_anchors":0,"resolved_work":0,"sample":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"2406.06573","kind":"arxiv","version":2},"verdict":{"created_at":null,"id":null,"model_set":{},"one_line_summary":"","pipeline_version":null,"pith_extraction_headline":"","strongest_claim":"","weakest_assumption":""}},"verdict_id":null}}],"author_attestations":[],"timestamp_anchors":[],"storage_attestations":[],"citation_signatures":[],"replication_records":[],"corrections":[],"mirror_hints":[],"record_created":{"event_id":"sha256:69f371366b2f5925ad36589c6b030ed995c88c4818eb4d55466a669afb4db3c8","target":"record","created_at":"2026-07-05T09:01:35Z","signer":{"key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signer_id":"pith.science","signer_type":"pith_registry"},"payload":{"attestation_state":"computed","canonical_record":{"metadata":{"abstract_canon_sha256":"43b4e4a354f8590d7e598d9c2cb5497dddf9ff7019294c34f5f66da48052fddb","cross_cats_sorted":["cs.LG"],"license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.CL","submitted_at":"2024-06-03T18:15:56Z","title_canon_sha256":"0689c01bd6b9fe751d55661c96bfddd7b5a172264fbb8c9dfdf9c79d8398c46c"},"schema_version":"1.0","source":{"id":"2406.06573","kind":"arxiv","version":2}},"canonical_sha256":"4fe657a1502f10831cf1026dde7abad4e0af0f44bdd8352a5a4f90cdae63f57d","receipt":{"algorithm":"ed25519","builder_version":"pith-number-builder-2026-05-17-v1","canonical_sha256":"4fe657a1502f10831cf1026dde7abad4e0af0f44bdd8352a5a4f90cdae63f57d","first_computed_at":"2026-07-05T09:01:35.224264Z","key_id":"pith-v1-2026-05","kind":"pith_receipt","last_reissued_at":"2026-07-05T09:01:35.224264Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","receipt_version":"0.3","signature_b64":"HIjIZI4ZQMD2fuzM8jXJzr95geDWqkg/ZnAJT9gs0BMGiv5YatsWhwE2VuJyCaHufzBKaKWi4lmdPQyAoF9bDg==","signature_status":"signed_v1","signed_at":"2026-07-05T09:01:35.224693Z","signed_message":"canonical_sha256_bytes"},"source_id":"2406.06573","source_kind":"arxiv","source_version":2}}},"equivocations":[],"invalid_events":[],"applied_event_ids":["sha256:69f371366b2f5925ad36589c6b030ed995c88c4818eb4d55466a669afb4db3c8","sha256:6c623ffeb2bb2fca84b6344add87c18eeca215c4d9891808267345edfdb7d39b"],"state_sha256":"89ee504f8bed7dc93c61294c81f2519deaa68bcde88c86adb2a55480d29de584"},"bundle_signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"lLctwRHVMVA0wiq4qX5EsJw03sb7spGYlhbH9Ml57LF0js/xUk+8gRBWgU//dJWyypEFum+TFRa2i41/fmqLAA==","signed_message":"bundle_sha256_bytes","signed_at":"2026-08-07T08:07:59.020087Z","bundle_sha256":"d99f4bff9d9f61d4b021a7f4ee8fd032cfe3c119c31a6e0fcf7cf7f783645514"}}