{"bundle_type":"pith_open_graph_bundle","bundle_version":"1.0","pith_number":"pith:2025:DXC2XHKQCSFHOOEP62JFO4ZKWE","short_pith_number":"pith:DXC2XHKQ","canonical_record":{"source":{"id":"2503.13508","kind":"arxiv","version":1},"metadata":{"license":"http://creativecommons.org/licenses/by-nc-sa/4.0/","primary_cat":"cs.CL","submitted_at":"2025-03-13T19:42:04Z","cross_cats_sorted":["cs.AI","cs.CY"],"title_canon_sha256":"dc7ac9b3b92327b448ab93680262d2999898813b22811103f853e0f50753adf7","abstract_canon_sha256":"1c29d2d0de14ad26ade0646dc7ef542d67d123140348384094bb989c96bf559b"},"schema_version":"1.0"},"canonical_sha256":"1dc5ab9d50148a77388ff69257732ab1275a88f279aba82a5afc2b879da28984","source":{"kind":"arxiv","id":"2503.13508","version":1},"source_aliases":[{"alias_kind":"arxiv","alias_value":"2503.13508","created_at":"2026-07-05T10:33:11Z"},{"alias_kind":"arxiv_version","alias_value":"2503.13508v1","created_at":"2026-07-05T10:33:11Z"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.2503.13508","created_at":"2026-07-05T10:33:11Z"},{"alias_kind":"pith_short_12","alias_value":"DXC2XHKQCSFH","created_at":"2026-07-05T10:33:11Z"},{"alias_kind":"pith_short_16","alias_value":"DXC2XHKQCSFHOOEP","created_at":"2026-07-05T10:33:11Z"},{"alias_kind":"pith_short_8","alias_value":"DXC2XHKQ","created_at":"2026-07-05T10:33:11Z"}],"events":[{"event_type":"record_created","subject_pith_number":"pith:2025:DXC2XHKQCSFHOOEP62JFO4ZKWE","target":"record","payload":{"canonical_record":{"source":{"id":"2503.13508","kind":"arxiv","version":1},"metadata":{"license":"http://creativecommons.org/licenses/by-nc-sa/4.0/","primary_cat":"cs.CL","submitted_at":"2025-03-13T19:42:04Z","cross_cats_sorted":["cs.AI","cs.CY"],"title_canon_sha256":"dc7ac9b3b92327b448ab93680262d2999898813b22811103f853e0f50753adf7","abstract_canon_sha256":"1c29d2d0de14ad26ade0646dc7ef542d67d123140348384094bb989c96bf559b"},"schema_version":"1.0"},"canonical_sha256":"1dc5ab9d50148a77388ff69257732ab1275a88f279aba82a5afc2b879da28984","receipt":{"kind":"pith_receipt","key_id":"pith-v1-2026-05","algorithm":"ed25519","signed_at":"2026-07-05T10:33:11.749678Z","signature_b64":"OHMI5JZWM2U0HNZIhGlA9/CxdLrP5TBMOd/kRiGXUOUrQXVjRjOPmt7rlv/6MfvfQJIfG0r3xzTmFsELPemuBQ==","signed_message":"canonical_sha256_bytes","builder_version":"pith-number-builder-2026-05-17-v1","receipt_version":"0.3","canonical_sha256":"1dc5ab9d50148a77388ff69257732ab1275a88f279aba82a5afc2b879da28984","last_reissued_at":"2026-07-05T10:33:11.749039Z","signature_status":"signed_v1","first_computed_at":"2026-07-05T10:33:11.749039Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"source_kind":"arxiv","source_id":"2503.13508","source_version":1,"attestation_state":"computed"},"signer":{"signer_id":"pith.science","signer_type":"pith_registry","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"created_at":"2026-07-05T10:33:11Z","supersedes":[],"prev_event":null,"signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"jTB0j1Ur2eoK5SULRxr+GWLspj3VXF63s6cf6LG/OAfsmVVi6mQbyrQby6fRMXf2vF8g7yZhGFLBQTYaeeh7BA==","signed_message":"open_graph_event_sha256_bytes","signed_at":"2026-07-07T08:47:12.423105Z"},"content_sha256":"4fd78ae79fbc0a35b3c6a2e72894d238e2dfc1b6d513ee1f09c6fc132472f8ee","schema_version":"1.0","event_id":"sha256:4fd78ae79fbc0a35b3c6a2e72894d238e2dfc1b6d513ee1f09c6fc132472f8ee"},{"event_type":"graph_snapshot","subject_pith_number":"pith:2025:DXC2XHKQCSFHOOEP62JFO4ZKWE","target":"graph","payload":{"graph_snapshot":{"paper":{"title":"It is Too Many Options: Pitfalls of Multiple-Choice Questions in Generative AI and Medical Education","license":"http://creativecommons.org/licenses/by-nc-sa/4.0/","headline":"","cross_cats":["cs.AI","cs.CY"],"primary_cat":"cs.CL","authors_text":"Ai Phuong S Tong, Anton Alyakin, Daniel Alexander Alber, Eric Claude Leuthardt, Eric Karl Oermann, Jaden Stryker, Karl Sangwon, Ki Yun Park, Mathew de la Paz, Miguel Hernandez-Rovira, Nicolas Goff, Shrutika Singh","submitted_at":"2025-03-13T19:42:04Z","abstract_excerpt":"The performance of Large Language Models (LLMs) on multiple-choice question (MCQ) benchmarks is frequently cited as proof of their medical capabilities. We hypothesized that LLM performance on medical MCQs may in part be illusory and driven by factors beyond medical content knowledge and reasoning capabilities. To assess this, we created a novel benchmark of free-response questions with paired MCQs (FreeMedQA). Using this benchmark, we evaluated three state-of-the-art LLMs (GPT-4o, GPT-3.5, and LLama-3-70B-instruct) and found an average absolute deterioration of 39.43% in performance on free-r"},"claims":{"count":0,"items":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"2503.13508","kind":"arxiv","version":1},"verdict":{"id":null,"model_set":{},"created_at":null,"strongest_claim":"","one_line_summary":"","pipeline_version":null,"weakest_assumption":"","pith_extraction_headline":""},"integrity":{"clean":true,"summary":{"advisory":0,"critical":0,"by_detector":{},"informational":0},"endpoint":"/pith/2503.13508/integrity.json","findings":[],"available":true,"detectors_run":[],"snapshot_sha256":"c28c3603d3b5d939e8dc4c7e95fa8dfce3d595e45f758748cecf8e644a296938"},"references":{"count":0,"sample":[],"resolved_work":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","internal_anchors":0},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"author_claims":{"count":0,"strong_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"builder_version":"pith-number-builder-2026-05-17-v1"},"verdict_id":null},"signer":{"signer_id":"pith.science","signer_type":"pith_registry","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"created_at":"2026-07-05T10:33:11Z","supersedes":[],"prev_event":null,"signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"Xuz0AxzL7HAwpYPUCHlh31/4xWtcRWHrA7UMCY1c7t3o87I+q+MFwMMXAlnz/GHqZpgNbTPKJEhSjx6pPOb5Cg==","signed_message":"open_graph_event_sha256_bytes","signed_at":"2026-07-07T08:47:12.423492Z"},"content_sha256":"ea5fcc78472e7f2578ca2a48660a0c03eb6c69e9b1795f5c37d4066fd2391a88","schema_version":"1.0","event_id":"sha256:ea5fcc78472e7f2578ca2a48660a0c03eb6c69e9b1795f5c37d4066fd2391a88"}],"timestamp_proofs":[],"mirror_hints":[{"mirror_type":"https","name":"Pith Resolver","base_url":"https://pith.science","bundle_url":"https://pith.science/pith/DXC2XHKQCSFHOOEP62JFO4ZKWE/bundle.json","state_url":"https://pith.science/pith/DXC2XHKQCSFHOOEP62JFO4ZKWE/state.json","well_known_bundle_url":"https://pith.science/.well-known/pith/DXC2XHKQCSFHOOEP62JFO4ZKWE/bundle.json","status":"primary"}],"public_keys":[{"key_id":"pith-v1-2026-05","algorithm":"ed25519","format":"raw","public_key_b64":"stVStoiQhXFxp4s2pdzPNoqVNBMojDU/fJ2db5S3CbM=","public_key_hex":"b2d552b68890857171a78b36a5dccf368a953413288c353f7c9d9d6f94b709b3","fingerprint_sha256_b32_first128bits":"RVFV5Z2OI2J3ZUO7ERDEBCYNKS","fingerprint_sha256_hex":"8d4b5ee74e4693bcd1df2446408b0d54","rotates_at":null,"url":"https://pith.science/pith-signing-key.json","notes":"Pith uses this Ed25519 key to sign canonical record SHA-256 digests. Verify with: ed25519_verify(public_key, message=canonical_sha256_bytes, signature=base64decode(signature_b64))."}],"merge_version":"pith-open-graph-merge-v1","built_at":"2026-07-07T08:47:12Z","links":{"resolver":"https://pith.science/pith/DXC2XHKQCSFHOOEP62JFO4ZKWE","bundle":"https://pith.science/pith/DXC2XHKQCSFHOOEP62JFO4ZKWE/bundle.json","state":"https://pith.science/pith/DXC2XHKQCSFHOOEP62JFO4ZKWE/state.json","well_known_bundle":"https://pith.science/.well-known/pith/DXC2XHKQCSFHOOEP62JFO4ZKWE/bundle.json"},"state":{"state_type":"pith_open_graph_state","state_version":"1.0","pith_number":"pith:2025:DXC2XHKQCSFHOOEP62JFO4ZKWE","merge_version":"pith-open-graph-merge-v1","event_count":2,"valid_event_count":2,"invalid_event_count":0,"equivocation_count":0,"current":{"canonical_record":{"metadata":{"abstract_canon_sha256":"1c29d2d0de14ad26ade0646dc7ef542d67d123140348384094bb989c96bf559b","cross_cats_sorted":["cs.AI","cs.CY"],"license":"http://creativecommons.org/licenses/by-nc-sa/4.0/","primary_cat":"cs.CL","submitted_at":"2025-03-13T19:42:04Z","title_canon_sha256":"dc7ac9b3b92327b448ab93680262d2999898813b22811103f853e0f50753adf7"},"schema_version":"1.0","source":{"id":"2503.13508","kind":"arxiv","version":1}},"source_aliases":[{"alias_kind":"arxiv","alias_value":"2503.13508","created_at":"2026-07-05T10:33:11Z"},{"alias_kind":"arxiv_version","alias_value":"2503.13508v1","created_at":"2026-07-05T10:33:11Z"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.2503.13508","created_at":"2026-07-05T10:33:11Z"},{"alias_kind":"pith_short_12","alias_value":"DXC2XHKQCSFH","created_at":"2026-07-05T10:33:11Z"},{"alias_kind":"pith_short_16","alias_value":"DXC2XHKQCSFHOOEP","created_at":"2026-07-05T10:33:11Z"},{"alias_kind":"pith_short_8","alias_value":"DXC2XHKQ","created_at":"2026-07-05T10:33:11Z"}],"graph_snapshots":[{"event_id":"sha256:ea5fcc78472e7f2578ca2a48660a0c03eb6c69e9b1795f5c37d4066fd2391a88","target":"graph","created_at":"2026-07-05T10:33:11Z","signer":{"key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signer_id":"pith.science","signer_type":"pith_registry"},"payload":{"graph_snapshot":{"author_claims":{"count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","strong_count":0},"builder_version":"pith-number-builder-2026-05-17-v1","claims":{"count":0,"items":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"integrity":{"available":true,"clean":true,"detectors_run":[],"endpoint":"/pith/2503.13508/integrity.json","findings":[],"snapshot_sha256":"c28c3603d3b5d939e8dc4c7e95fa8dfce3d595e45f758748cecf8e644a296938","summary":{"advisory":0,"by_detector":{},"critical":0,"informational":0}},"paper":{"abstract_excerpt":"The performance of Large Language Models (LLMs) on multiple-choice question (MCQ) benchmarks is frequently cited as proof of their medical capabilities. We hypothesized that LLM performance on medical MCQs may in part be illusory and driven by factors beyond medical content knowledge and reasoning capabilities. To assess this, we created a novel benchmark of free-response questions with paired MCQs (FreeMedQA). Using this benchmark, we evaluated three state-of-the-art LLMs (GPT-4o, GPT-3.5, and LLama-3-70B-instruct) and found an average absolute deterioration of 39.43% in performance on free-r","authors_text":"Ai Phuong S Tong, Anton Alyakin, Daniel Alexander Alber, Eric Claude Leuthardt, Eric Karl Oermann, Jaden Stryker, Karl Sangwon, Ki Yun Park, Mathew de la Paz, Miguel Hernandez-Rovira, Nicolas Goff, Shrutika Singh","cross_cats":["cs.AI","cs.CY"],"headline":"","license":"http://creativecommons.org/licenses/by-nc-sa/4.0/","primary_cat":"cs.CL","submitted_at":"2025-03-13T19:42:04Z","title":"It is Too Many Options: Pitfalls of Multiple-Choice Questions in Generative AI and Medical Education"},"references":{"count":0,"internal_anchors":0,"resolved_work":0,"sample":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"2503.13508","kind":"arxiv","version":1},"verdict":{"created_at":null,"id":null,"model_set":{},"one_line_summary":"","pipeline_version":null,"pith_extraction_headline":"","strongest_claim":"","weakest_assumption":""}},"verdict_id":null}}],"author_attestations":[],"timestamp_anchors":[],"storage_attestations":[],"citation_signatures":[],"replication_records":[],"corrections":[],"mirror_hints":[],"record_created":{"event_id":"sha256:4fd78ae79fbc0a35b3c6a2e72894d238e2dfc1b6d513ee1f09c6fc132472f8ee","target":"record","created_at":"2026-07-05T10:33:11Z","signer":{"key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signer_id":"pith.science","signer_type":"pith_registry"},"payload":{"attestation_state":"computed","canonical_record":{"metadata":{"abstract_canon_sha256":"1c29d2d0de14ad26ade0646dc7ef542d67d123140348384094bb989c96bf559b","cross_cats_sorted":["cs.AI","cs.CY"],"license":"http://creativecommons.org/licenses/by-nc-sa/4.0/","primary_cat":"cs.CL","submitted_at":"2025-03-13T19:42:04Z","title_canon_sha256":"dc7ac9b3b92327b448ab93680262d2999898813b22811103f853e0f50753adf7"},"schema_version":"1.0","source":{"id":"2503.13508","kind":"arxiv","version":1}},"canonical_sha256":"1dc5ab9d50148a77388ff69257732ab1275a88f279aba82a5afc2b879da28984","receipt":{"algorithm":"ed25519","builder_version":"pith-number-builder-2026-05-17-v1","canonical_sha256":"1dc5ab9d50148a77388ff69257732ab1275a88f279aba82a5afc2b879da28984","first_computed_at":"2026-07-05T10:33:11.749039Z","key_id":"pith-v1-2026-05","kind":"pith_receipt","last_reissued_at":"2026-07-05T10:33:11.749039Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","receipt_version":"0.3","signature_b64":"OHMI5JZWM2U0HNZIhGlA9/CxdLrP5TBMOd/kRiGXUOUrQXVjRjOPmt7rlv/6MfvfQJIfG0r3xzTmFsELPemuBQ==","signature_status":"signed_v1","signed_at":"2026-07-05T10:33:11.749678Z","signed_message":"canonical_sha256_bytes"},"source_id":"2503.13508","source_kind":"arxiv","source_version":1}}},"equivocations":[],"invalid_events":[],"applied_event_ids":["sha256:4fd78ae79fbc0a35b3c6a2e72894d238e2dfc1b6d513ee1f09c6fc132472f8ee","sha256:ea5fcc78472e7f2578ca2a48660a0c03eb6c69e9b1795f5c37d4066fd2391a88"],"state_sha256":"045a89baaf5ebf50d2197493f3e8d0f9052855145ef5ada301fcd394cfe5bced"},"bundle_signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"1wZz9zaI0LTBrlUASTLh1G5M74oFd9zFvA/k7xRzeMJe+4OX2QzfsZHMl4sPnQyvK+arIRX8hsvD/XQNUaveDg==","signed_message":"bundle_sha256_bytes","signed_at":"2026-07-07T08:47:12.425427Z","bundle_sha256":"5b9d8177dc14ce39cd3809369b51e38d1c350ed734b7460459c6fcc22f087d42"}}