{"as_of":"2026-08-12T14:28:00Z","caps":{"database_statements":6,"inbound":100,"outbound":100},"context_digest":"sha256:d9704d751aea0f98157a4f53705e73575290c30b6437da63b89a8e25d97ba3ba","coverage":[{"denominator":57,"lane":"reference_resolution","note":"Typed states for the displayed outbound observations.","records_observed":57,"source":"paper_references, paper_reference_links","source_observed_at":"2026-05-19T00:40:34.440305Z","state":"measured"},{"denominator":66,"lane":"standing_notices","note":"One-hop event checks from named stored sources.","records_observed":66,"source":"scholarly_work_events, retraction_status_cache","source_observed_at":"2026-08-12T06:34:41.77262+00:00","state":"measured"},{"denominator":9,"lane":"inbound_itemization","note":"Pith citing papers itemized under the disclosed page cap.","records_observed":9,"source":"paper_references, paper_reference_links","source_observed_at":"2026-08-05T17:57:40.893213Z","state":"measured"},{"denominator":1,"lane":"external_citation_measurements","note":"A source-named dated measurement, never combined with another source.","records_observed":1,"source":"pith","source_observed_at":"2026-08-05T02:28:24.338817Z","state":"measured"}],"external_citation_measurements":[{"count":3,"observed_at":"2026-08-05T02:28:24.338817Z","source":"pith"}],"inbound":[{"citation":{"cited_paper":{"arxiv_id":"2508.04325","last_updated":"2026-04-29T05:46:18Z","snapshot_observed_at":"2026-08-02T07:50:38.917887Z","submitted_at":"2025-08-06T11:11:40Z","title":"Beyond the Leaderboard: Rethinking Medical Benchmarks for Large Language Models","version":2},"cited_work":{"arxiv_id":"2508.04325","doi":"10.48550/arxiv.2508.04325","metadata_source":"pith","pith_arxiv_id":"2508.04325","snapshot_observed_at":"2026-08-05T02:49:54.815029Z","title":"Beyond the Leaderboard: Rethinking Medical Benchmarks for Large Language Models","venue":"cs.CL","work_id":"b3c260eb-e603-4976-a6db-d6397c4dbe83","year":2025},"citing_paper":{"arxiv_id":"2605.04221","last_updated":"2026-06-10T05:50:04Z","snapshot_observed_at":"2026-08-11T06:40:05.171554Z","submitted_at":"2026-05-05T19:03:40Z","title":"Self-Prompting Small Language Models for Privacy-Sensitive Clinical Information Extraction","version":1},"reference_index":36,"source":"pdf_text","source_observed_at":"2026-05-08T17:22:17.266351Z"},"links":{"cited_paper":"/paper/2508.04325","citing_paper":"/paper/2605.04221"},"observation_digest":"sha256:ec454a14b58cc91fba8e4656857f077d91fc3430ac35f4fccccf591550e06a51","observation_id":"b1117514-0ac2-434e-b5a5-fdffb5bf688e","resolution":{"observed_at":"2026-05-08T17:23:40.483260Z","resolver_source":"local_arxiv","status":"verified_exact"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-12T06:34:41.77262+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-12T06:34:41.77262+00:00","source":"crossref"},{"observed_at":"2026-08-12T06:34:36.333875+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":{"arxiv_id":"2508.04325","last_updated":"2026-04-29T05:46:18Z","snapshot_observed_at":"2026-08-02T07:50:38.917887Z","submitted_at":"2025-08-06T11:11:40Z","title":"Beyond the Leaderboard: Rethinking Medical Benchmarks for Large Language Models","version":2},"cited_work":{"arxiv_id":"2508.04325","doi":"10.48550/arxiv.2508.04325","metadata_source":"pith","pith_arxiv_id":"2508.04325","snapshot_observed_at":"2026-08-05T02:49:54.815029Z","title":"Beyond the Leaderboard: Rethinking Medical Benchmarks for Large Language Models","venue":"cs.CL","work_id":"b3c260eb-e603-4976-a6db-d6397c4dbe83","year":2025},"citing_paper":{"arxiv_id":"2605.04221","last_updated":"2026-06-10T05:50:04Z","snapshot_observed_at":"2026-08-11T06:40:05.171554Z","submitted_at":"2026-05-05T19:03:40Z","title":"Self-Prompting Small Language Models for Privacy-Sensitive Clinical Information Extraction","version":2},"reference_index":36,"source":"pdf_text","source_observed_at":"2026-06-30T23:55:22.768635Z"},"links":{"cited_paper":"/paper/2508.04325","citing_paper":"/paper/2605.04221"},"observation_digest":"sha256:d14e5738c47ef35609027777dd26a0fe34482cebdf7705e1445390b402a948d2","observation_id":"2d9fc485-8d1a-47c7-a81e-953ad57fe0a4","resolution":{"observed_at":"2026-07-01T00:05:09.474464Z","resolver_source":"local_arxiv","status":"verified_exact"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-12T06:34:41.77262+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-12T06:34:41.77262+00:00","source":"crossref"},{"observed_at":"2026-08-12T06:34:36.333875+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":{"arxiv_id":"2508.04325","last_updated":"2026-04-29T05:46:18Z","snapshot_observed_at":"2026-08-02T07:50:38.917887Z","submitted_at":"2025-08-06T11:11:40Z","title":"Beyond the Leaderboard: Rethinking Medical Benchmarks for Large Language Models","version":2},"cited_work":{"arxiv_id":"2508.04325","doi":"10.48550/arxiv.2508.04325","metadata_source":"pith","pith_arxiv_id":"2508.04325","snapshot_observed_at":"2026-08-05T02:49:54.815029Z","title":"Beyond the Leaderboard: Rethinking Medical Benchmarks for Large Language Models","venue":"cs.CL","work_id":"b3c260eb-e603-4976-a6db-d6397c4dbe83","year":2025},"citing_paper":{"arxiv_id":"2605.08445","last_updated":"2026-05-08T20:12:14Z","snapshot_observed_at":"2026-08-11T22:26:39.127469Z","submitted_at":"2026-05-08T20:12:14Z","title":"Measuring What Matters: Benchmarking Generative, Multimodal, and Agentic AI in Healthcare","version":1},"reference_index":2,"source":"pdf_text","source_observed_at":"2026-05-12T01:26:28.419570Z"},"links":{"cited_paper":"/paper/2508.04325","citing_paper":"/paper/2605.08445"},"observation_digest":"sha256:8da36ca93503c6c04ab3449a406d13700e80e1ddb6986831157a7779e321df4b","observation_id":"a063ba73-4ac4-4b98-a403-8fb3e350d976","resolution":{"observed_at":"2026-05-12T07:56:36.315888Z","resolver_source":"local_arxiv","status":"verified_exact"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-12T06:34:41.77262+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-12T06:34:41.77262+00:00","source":"crossref"},{"observed_at":"2026-08-12T06:34:36.333875+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":{"arxiv_id":"2508.04325","last_updated":"2026-04-29T05:46:18Z","snapshot_observed_at":"2026-08-02T07:50:38.917887Z","submitted_at":"2025-08-06T11:11:40Z","title":"Beyond the Leaderboard: Rethinking Medical Benchmarks for Large Language Models","version":2},"cited_work":{"arxiv_id":"2508.04325","doi":"10.48550/arxiv.2508.04325","metadata_source":"pith","pith_arxiv_id":"2508.04325","snapshot_observed_at":"2026-08-05T02:49:54.815029Z","title":"Beyond the Leaderboard: Rethinking Medical Benchmarks for Large Language Models","venue":"cs.CL","work_id":"b3c260eb-e603-4976-a6db-d6397c4dbe83","year":2025},"citing_paper":{"arxiv_id":"2606.12291","last_updated":"2026-06-10T16:27:26Z","snapshot_observed_at":"2026-08-02T15:04:58.080617Z","submitted_at":"2026-06-10T16:27:26Z","title":"Measuring Epistemic Resilience of LLMs Under Misleading Medical Context","version":1},"reference_index":24,"source":"pdf_text","source_observed_at":"2026-06-27T09:30:47.726480Z"},"links":{"cited_paper":"/paper/2508.04325","citing_paper":"/paper/2606.12291"},"observation_digest":"sha256:4f0951d7dbea7248d0a9ce20d002eee03603429b6d769fe8aa1b6b0e3508d63b","observation_id":"b64375e8-78c7-4159-b8ff-7db14240cf13","resolution":{"observed_at":"2026-06-27T09:40:47.697055Z","resolver_source":"local_arxiv","status":"verified_exact"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-12T06:34:41.77262+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-12T06:34:41.77262+00:00","source":"crossref"},{"observed_at":"2026-08-12T06:34:36.333875+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":{"arxiv_id":"2508.04325","last_updated":"2026-04-29T05:46:18Z","snapshot_observed_at":"2026-08-02T07:50:38.917887Z","submitted_at":"2025-08-06T11:11:40Z","title":"Beyond the Leaderboard: Rethinking Medical Benchmarks for Large Language Models","version":2},"cited_work":{"arxiv_id":"2508.04325","doi":"10.48550/arxiv.2508.04325","metadata_source":"pith","pith_arxiv_id":"2508.04325","snapshot_observed_at":"2026-08-05T02:49:54.815029Z","title":"Beyond the Leaderboard: Rethinking Medical Benchmarks for Large Language Models","venue":"cs.CL","work_id":"b3c260eb-e603-4976-a6db-d6397c4dbe83","year":2025},"citing_paper":{"arxiv_id":"2606.24155","last_updated":"2026-07-06T09:18:28Z","snapshot_observed_at":"2026-08-09T15:42:50.166972Z","submitted_at":"2026-06-23T05:23:46Z","title":"MedBench v5: A Dynamic, Process-Oriented, and Hallucination-Aware Benchmark for Clinical Multimodal Models","version":2},"reference_index":1,"source":"pdf_text","source_observed_at":"2026-06-26T00:40:26.700516Z"},"links":{"cited_paper":"/paper/2508.04325","citing_paper":"/paper/2606.24155"},"observation_digest":"sha256:7defd336f83c90b4239452f7d8903d01292660e5cb52fc986408210767498a48","observation_id":"f54b1aa8-b540-408b-abe8-e49fdb511d9f","resolution":{"observed_at":"2026-07-04T16:29:56.922691Z","resolver_source":"local_arxiv","status":"verified_exact"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-12T06:34:41.77262+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-12T06:34:41.77262+00:00","source":"crossref"},{"observed_at":"2026-08-12T06:34:36.333875+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":{"arxiv_id":"2508.04325","last_updated":"2026-04-29T05:46:18Z","snapshot_observed_at":"2026-08-02T07:50:38.917887Z","submitted_at":"2025-08-06T11:11:40Z","title":"Beyond the Leaderboard: Rethinking Medical Benchmarks for Large Language Models","version":2},"cited_work":{"arxiv_id":"2508.04325","doi":"10.48550/arxiv.2508.04325","metadata_source":"pith","pith_arxiv_id":"2508.04325","snapshot_observed_at":"2026-08-05T02:49:54.815029Z","title":"Beyond the Leaderboard: Rethinking Medical Benchmarks for Large Language Models","venue":"cs.CL","work_id":"b3c260eb-e603-4976-a6db-d6397c4dbe83","year":2025},"citing_paper":{"arxiv_id":"2606.24155","last_updated":"2026-07-06T09:18:28Z","snapshot_observed_at":"2026-08-09T15:42:50.166972Z","submitted_at":"2026-06-23T05:23:46Z","title":"MedBench v5: A Dynamic, Process-Oriented, and Hallucination-Aware Benchmark for Clinical Multimodal Models","version":3},"reference_index":1,"source":"pdf_text","source_observed_at":"2026-06-26T05:42:11.764973Z"},"links":{"cited_paper":"/paper/2508.04325","citing_paper":"/paper/2606.24155"},"observation_digest":"sha256:b1755051e6a0ab345590afad1230fa2f5a6ade90d86387b4fb0d76d604598d3d","observation_id":"2b6f817b-7124-4c0f-948e-1322b3e4f9e7","resolution":{"observed_at":"2026-07-04T12:59:52.284424Z","resolver_source":"local_arxiv","status":"verified_exact"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-12T06:34:41.77262+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-12T06:34:41.77262+00:00","source":"crossref"},{"observed_at":"2026-08-12T06:34:36.333875+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":{"arxiv_id":"2508.04325","last_updated":"2026-04-29T05:46:18Z","snapshot_observed_at":"2026-08-02T07:50:38.917887Z","submitted_at":"2025-08-06T11:11:40Z","title":"Beyond the Leaderboard: Rethinking Medical Benchmarks for Large Language Models","version":2},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2508.04325","snapshot_observed_at":"2026-07-12T12:36:00.189291Z","title":"Beyond the leaderboard: Rethinking medical benchmarks for large language models.arXiv preprint arXiv:2508.04325,","venue":null,"work_id":null,"year":null},"citing_paper":{"arxiv_id":"2606.24155","last_updated":"2026-07-06T09:18:28Z","snapshot_observed_at":"2026-08-09T15:42:50.166972Z","submitted_at":"2026-06-23T05:23:46Z","title":"MedBench v5: A Dynamic, Process-Oriented, and Hallucination-Aware Benchmark for Clinical Multimodal Models","version":4},"reference_index":1,"source":"pdf_text","source_observed_at":"2026-07-12T12:36:00.189291Z"},"links":{"cited_paper":"/paper/2508.04325","citing_paper":"/paper/2606.24155"},"observation_digest":"sha256:2de252e9dcad2aa83841bae6806277b6015e3b3a5917fd8c6e8d03c961027c40","observation_id":"4772a3c0-f726-4610-868e-a35c4e2e6e0f","resolution":{"observed_at":"2026-07-12T12:36:00.189291Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2508.04325","last_updated":"2026-04-29T05:46:18Z","snapshot_observed_at":"2026-08-02T07:50:38.917887Z","submitted_at":"2025-08-06T11:11:40Z","title":"Beyond the Leaderboard: Rethinking Medical Benchmarks for Large Language Models","version":2},"cited_work":{"arxiv_id":"2508.04325","doi":"10.48550/arxiv.2508.04325","metadata_source":"pith","pith_arxiv_id":"2508.04325","snapshot_observed_at":"2026-08-05T02:49:54.815029Z","title":"Beyond the Leaderboard: Rethinking Medical Benchmarks for Large Language Models","venue":"cs.CL","work_id":"b3c260eb-e603-4976-a6db-d6397c4dbe83","year":2025},"citing_paper":{"arxiv_id":"2606.28332","last_updated":"2026-05-26T14:39:18Z","snapshot_observed_at":"2026-08-12T05:40:29.957478Z","submitted_at":"2026-05-26T14:39:18Z","title":"When Medical Safety Alignment Fails: A Benchmark for Evaluating LLMs on High-Risk Medical Queries","version":1},"reference_index":17,"source":"arxiv_source","source_observed_at":"2026-06-30T11:03:52.230759Z"},"links":{"cited_paper":"/paper/2508.04325","citing_paper":"/paper/2606.28332"},"observation_digest":"sha256:084fd2fef48388e7d1130cf6967bd3935ebc2fc58253dfa926e761edcbbbf7ae","observation_id":"a0b7158f-dffb-4080-8b8c-4a42b6591ebf","resolution":{"observed_at":"2026-06-30T11:04:37.490635Z","resolver_source":"local_arxiv","status":"metadata_mismatch"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-12T06:34:41.77262+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-12T06:34:41.77262+00:00","source":"crossref"},{"observed_at":"2026-08-12T06:34:36.333875+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":{"arxiv_id":"2508.04325","last_updated":"2026-04-29T05:46:18Z","snapshot_observed_at":"2026-08-02T07:50:38.917887Z","submitted_at":"2025-08-06T11:11:40Z","title":"Beyond the Leaderboard: Rethinking Medical Benchmarks for Large Language Models","version":2},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2508.04325","snapshot_observed_at":"2026-08-05T17:57:40.893213Z","title":"Version 2 posted April 29, 2026; accepted by ACL","venue":null,"work_id":null,"year":2026},"citing_paper":{"arxiv_id":"2608.03500","last_updated":"2026-08-04T11:42:31Z","snapshot_observed_at":"2026-08-08T19:51:25.299904Z","submitted_at":"2026-08-04T11:42:31Z","title":"LLM-Assisted Review Prioritization for German Statutory Health Insurance Websites: A Multi-Stage Corpus Audit","version":1},"reference_index":3,"source":"pdf_text","source_observed_at":"2026-08-05T17:57:40.893213Z"},"links":{"cited_paper":"/paper/2508.04325","citing_paper":"/paper/2608.03500"},"observation_digest":"sha256:610c39110713dc862867e2add4dfd2c334dfa8ca988708b5f6d05abfd4280f0a","observation_id":"cce5bdbf-ff82-4873-b6f3-3f0aba30d6eb","resolution":{"observed_at":"2026-08-05T17:57:40.893213Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}}],"links":{"evidence":"/evidence","html":"/paper/2508.04325/citation-record","integrity":"/paper/2508.04325/integrity","json":"/paper/2508.04325/citation-record.json","paper":"/paper/2508.04325"},"outbound":[{"citation":{"cited_paper":{"arxiv_id":"2501.03465","last_updated":"2025-06-10T01:34:12Z","snapshot_observed_at":"2026-07-06T20:17:24.683086Z","submitted_at":"2025-01-07T01:47:49Z","title":"Extending Internet Access Over LoRa for Internet of Things and Critical Applications","version":3},"cited_work":{"arxiv_id":"2501.03465","doi":null,"metadata_source":"arxiv_reference","pith_arxiv_id":"2501.03465","snapshot_observed_at":"2026-06-05T21:23:00.469572Z","title":"Preprint","venue":null,"work_id":"d7d2abf8-548a-4122-ba3a-62d796dd714b","year":2025},"citing_paper":{"arxiv_id":"2508.04325","last_updated":"2026-04-29T05:46:18Z","snapshot_observed_at":"2026-08-02T07:50:38.917887Z","submitted_at":"2025-08-06T11:11:40Z","title":"Beyond the Leaderboard: Rethinking Medical Benchmarks for Large Language Models","version":2},"reference_index":1,"source":"pdf_text","source_observed_at":"2026-05-19T00:40:34.440305Z"},"links":{"cited_paper":"/paper/2501.03465","citing_paper":"/paper/2508.04325"},"observation_digest":"sha256:74dff223456221cee082bd6d6b904265c5989d8f9dce449dab0c72eccad31aec","observation_id":"1882affb-a6d0-438d-9f31-9e717c4cdad5","resolution":{"observed_at":"2026-05-09T07:21:27.027337Z","resolver_source":"arxiv_id","status":"verified_exact"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-12T06:34:41.77262+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-12T06:34:41.77262+00:00","source":"crossref"},{"observed_at":"2026-08-12T06:34:36.333875+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":{"arxiv_id":"2501.10711","last_updated":"2026-07-06T06:02:32Z","snapshot_observed_at":"2026-08-10T18:59:26.797441Z","submitted_at":"2025-01-18T09:51:57Z","title":"Code Benchmarks Should Prioritize Rigor, Reliability, and Reproducibility","version":5},"cited_work":{"arxiv_id":"2501.10711","doi":null,"metadata_source":"arxiv_reference","pith_arxiv_id":"2501.10711","snapshot_observed_at":"2026-07-07T03:18:46.014814Z","title":"How should we build a benchmark? revisiting 274 code-related benchmarks for llms","venue":null,"work_id":"40f0a07f-2759-4675-a8f7-ceaa0d3e65db","year":2025},"citing_paper":{"arxiv_id":"2508.04325","last_updated":"2026-04-29T05:46:18Z","snapshot_observed_at":"2026-08-02T07:50:38.917887Z","submitted_at":"2025-08-06T11:11:40Z","title":"Beyond the Leaderboard: Rethinking Medical Benchmarks for Large Language Models","version":2},"reference_index":2,"source":"pdf_text","source_observed_at":"2026-05-19T00:40:34.440305Z"},"links":{"cited_paper":"/paper/2501.10711","citing_paper":"/paper/2508.04325"},"observation_digest":"sha256:e60cd5fdc959b4cdd86bf3af0a2ed3db0387919692d1ea79be71a4ce40ee7218","observation_id":"6a12dde6-efe5-4dfd-a678-8026e8bc1c33","resolution":{"observed_at":"2026-07-07T03:18:46.014814Z","resolver_source":"arxiv_id","status":"metadata_mismatch"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-12T06:34:41.77262+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-12T06:34:41.77262+00:00","source":"crossref"},{"observed_at":"2026-08-12T06:34:36.333875+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":{"arxiv_id":"2502.14425","last_updated":"2025-06-05T00:49:05Z","snapshot_observed_at":"2026-08-10T16:06:55.462277Z","submitted_at":"2025-02-20T10:23:27Z","title":"A Survey on Data Contamination for Large Language Models","version":2},"cited_work":{"arxiv_id":"2502.14425","doi":"10.48550/arxiv.2502.14425","metadata_source":"arxiv_reference","pith_arxiv_id":"2502.14425","snapshot_observed_at":"2026-08-05T02:28:24.338817Z","title":"arXiv preprint arXiv:2502.14425 , year=","venue":"ArXiv.org","work_id":"93c324c7-c726-4b33-a231-2ee7f4fb0fab","year":2025},"citing_paper":{"arxiv_id":"2508.04325","last_updated":"2026-04-29T05:46:18Z","snapshot_observed_at":"2026-08-02T07:50:38.917887Z","submitted_at":"2025-08-06T11:11:40Z","title":"Beyond the Leaderboard: Rethinking Medical Benchmarks for Large Language Models","version":2},"reference_index":3,"source":"pdf_text","source_observed_at":"2026-05-19T00:40:34.440305Z"},"links":{"cited_paper":"/paper/2502.14425","citing_paper":"/paper/2508.04325"},"observation_digest":"sha256:e5305a6924c67b89b17cf709707fd849af84b9947889bb20988eb2a8a7d81ce6","observation_id":"c83eae94-12c4-46ae-9aec-cc52f427a514","resolution":{"observed_at":"2026-05-09T07:21:27.058821Z","resolver_source":"arxiv_id","status":"metadata_mismatch"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-12T06:34:41.77262+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-12T06:34:41.77262+00:00","source":"crossref"},{"observed_at":"2026-08-12T06:34:36.333875+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":{"arxiv_id":"2505.22919","last_updated":"2026-05-11T05:46:29Z","snapshot_observed_at":"2026-07-06T21:32:33.515702Z","submitted_at":"2025-05-28T22:43:44Z","title":"ER-Reason: A Benchmark Dataset for LLM Clinical Reasoning in the Emergency Room","version":3},"cited_work":{"arxiv_id":"2505.22919","doi":null,"metadata_source":"pith","pith_arxiv_id":"2505.22919","snapshot_observed_at":"2026-07-10T10:37:01.657285Z","title":"ER-Reason: A Benchmark Dataset for LLM Clinical Reasoning in the Emergency Room","venue":"cs.CL","work_id":"07d0e475-4eb6-47bf-a04d-cdef412707c8","year":2025},"citing_paper":{"arxiv_id":"2508.04325","last_updated":"2026-04-29T05:46:18Z","snapshot_observed_at":"2026-08-02T07:50:38.917887Z","submitted_at":"2025-08-06T11:11:40Z","title":"Beyond the Leaderboard: Rethinking Medical Benchmarks for Large Language Models","version":2},"reference_index":4,"source":"pdf_text","source_observed_at":"2026-05-19T00:40:34.440305Z"},"links":{"cited_paper":"/paper/2505.22919","citing_paper":"/paper/2508.04325"},"observation_digest":"sha256:4398b9b3f4a0a8d1ff2b75186da96758181cc6f87095c3583c2e770547301e83","observation_id":"967d7881-6b0a-438b-b28a-027b74ebfe1d","resolution":{"observed_at":"2026-05-12T02:45:57.533893Z","resolver_source":"arxiv_id","status":"metadata_mismatch"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-12T06:34:41.77262+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-12T06:34:41.77262+00:00","source":"crossref"},{"observed_at":"2026-08-12T06:34:36.333875+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":{"arxiv_id":"2405.10893","last_updated":"2024-05-17T16:31:56Z","snapshot_observed_at":"2026-08-11T16:20:08.734378Z","submitted_at":"2024-05-17T16:31:56Z","title":"COGNET-MD, an evaluation framework and dataset for Large Language Model benchmarks in the medical domain","version":1},"cited_work":{"arxiv_id":"2405.10893","doi":null,"metadata_source":"arxiv_reference","pith_arxiv_id":"2405.10893","snapshot_observed_at":"2026-06-05T21:23:00.469572Z","title":"In Proceedings of the 2024 Con- ference on Empirical Methods in Natural Language Processing, pages 8428–8438, Miami, Florida, USA","venue":null,"work_id":"e96e0686-4a5d-4242-97fd-ab988c77106f","year":2024},"citing_paper":{"arxiv_id":"2508.04325","last_updated":"2026-04-29T05:46:18Z","snapshot_observed_at":"2026-08-02T07:50:38.917887Z","submitted_at":"2025-08-06T11:11:40Z","title":"Beyond the Leaderboard: Rethinking Medical Benchmarks for Large Language Models","version":2},"reference_index":5,"source":"pdf_text","source_observed_at":"2026-05-19T00:40:34.440305Z"},"links":{"cited_paper":"/paper/2405.10893","citing_paper":"/paper/2508.04325"},"observation_digest":"sha256:b9fe5215f260f443d67029954c644e24a3d7c1632557d9369cba77b2488eb4cd","observation_id":"c475641c-7c69-4ecf-aa9a-02a0b10a969f","resolution":{"observed_at":"2026-05-09T07:21:27.071940Z","resolver_source":"arxiv_id","status":"verified_exact"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-12T06:34:41.77262+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-12T06:34:41.77262+00:00","source":"crossref"},{"observed_at":"2026-08-12T06:34:36.333875+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":{"arxiv_id":"2405.07960","last_updated":"2025-05-25T02:19:37Z","snapshot_observed_at":"2026-07-30T08:34:47.046912Z","submitted_at":"2024-05-13T17:38:53Z","title":"AgentClinic: a multimodal agent benchmark to evaluate AI in simulated clinical environments","version":5},"cited_work":{"arxiv_id":"2405.07960","doi":"10.48550/arxiv.2405.07960","metadata_source":"pith","pith_arxiv_id":"2405.07960","snapshot_observed_at":"2026-08-05T02:28:24.338817Z","title":"AgentClinic: a multimodal agent benchmark to evaluate AI in simulated clinical environments","venue":"cs.HC","work_id":"418a0992-6f06-45f9-958d-cfdfc51c72af","year":2024},"citing_paper":{"arxiv_id":"2508.04325","last_updated":"2026-04-29T05:46:18Z","snapshot_observed_at":"2026-08-02T07:50:38.917887Z","submitted_at":"2025-08-06T11:11:40Z","title":"Beyond the Leaderboard: Rethinking Medical Benchmarks for Large Language Models","version":2},"reference_index":6,"source":"pdf_text","source_observed_at":"2026-05-19T00:40:34.440305Z"},"links":{"cited_paper":"/paper/2405.07960","citing_paper":"/paper/2508.04325"},"observation_digest":"sha256:c682581dc32c30b0f721cedc118dab153691e05fa2e0856d824a4b2a04829f83","observation_id":"58f1b971-5115-4972-88cc-187979c2b949","resolution":{"observed_at":"2026-05-21T13:53:19.668674Z","resolver_source":"arxiv_id","status":"metadata_mismatch"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-12T06:34:41.77262+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-12T06:34:41.77262+00:00","source":"crossref"},{"observed_at":"2026-08-12T06:34:36.333875+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":{"arxiv_id":"2311.16502","last_updated":"2024-06-13T15:02:39Z","snapshot_observed_at":"2026-07-06T02:11:23.670680Z","submitted_at":"2023-11-27T17:33:21Z","title":"MMMU: A Massive Multi-discipline Multimodal Understanding and Reasoning Benchmark for Expert AGI","version":4},"cited_work":{"arxiv_id":"2311.16502","doi":"10.48550/arxiv.2311.16502","metadata_source":"pith","pith_arxiv_id":"2311.16502","snapshot_observed_at":"2026-08-05T02:28:24.338817Z","title":"MMMU: A Massive Multi-discipline Multimodal Understanding and Reasoning Benchmark for Expert AGI","venue":"cs.CL","work_id":"da087b16-ea05-4064-980e-ce1d6e281d49","year":2023},"citing_paper":{"arxiv_id":"2508.04325","last_updated":"2026-04-29T05:46:18Z","snapshot_observed_at":"2026-08-02T07:50:38.917887Z","submitted_at":"2025-08-06T11:11:40Z","title":"Beyond the Leaderboard: Rethinking Medical Benchmarks for Large Language Models","version":2},"reference_index":7,"source":"pdf_text","source_observed_at":"2026-05-19T00:40:34.440305Z"},"links":{"cited_paper":"/paper/2311.16502","citing_paper":"/paper/2508.04325"},"observation_digest":"sha256:3d8c45a7cd51b2ab5be9bc8f02bf199e4d3bfd3e47d1b97b34dde9d249666ce5","observation_id":"cc344051-3cd2-4e62-9f3f-9bff7da01e77","resolution":{"observed_at":"2026-05-15T05:37:42.029646Z","resolver_source":"arxiv_id","status":"metadata_mismatch"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-12T06:34:41.77262+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-12T06:34:41.77262+00:00","source":"crossref"},{"observed_at":"2026-08-10T17:38:12.261029+00:00","source":"crossref_status_cache"},{"observed_at":"2026-08-10T17:38:12.261029+00:00","source":"openalex_status_cache"},{"observed_at":"2026-08-12T06:34:36.333875+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-06-05T21:23:00.469572Z","title":"Ndepartment de- notes the number of medical departments, typi- cally referring to the medical specialties included in the model’s evaluation","venue":null,"work_id":"22ccf64b-5960-4191-960e-e61e599b55c8","year":null},"citing_paper":{"arxiv_id":"2508.04325","last_updated":"2026-04-29T05:46:18Z","snapshot_observed_at":"2026-08-02T07:50:38.917887Z","submitted_at":"2025-08-06T11:11:40Z","title":"Beyond the Leaderboard: Rethinking Medical Benchmarks for Large Language Models","version":2},"reference_index":8,"source":"pdf_text","source_observed_at":"2026-05-19T00:40:34.440305Z"},"links":{"citing_paper":"/paper/2508.04325"},"observation_digest":"sha256:e3c3f0386cadb7b2a530e41efad270c20be21b36f35b42c19f1b61720b737fe9","observation_id":"265a1e35-1630-4cbf-82af-bbea73f97d58","resolution":{"observed_at":"2026-05-09T07:21:27.098121Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-12T06:34:41.77262+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-12T06:34:41.77262+00:00","source":"crossref"},{"observed_at":"2026-08-12T06:34:36.333875+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-06-05T21:23:00.469572Z","title":null,"venue":null,"work_id":"e075df20-1fbb-464b-ac60-8717216a973a","year":null},"citing_paper":{"arxiv_id":"2508.04325","last_updated":"2026-04-29T05:46:18Z","snapshot_observed_at":"2026-08-02T07:50:38.917887Z","submitted_at":"2025-08-06T11:11:40Z","title":"Beyond the Leaderboard: Rethinking Medical Benchmarks for Large Language Models","version":2},"reference_index":9,"source":"pdf_text","source_observed_at":"2026-05-19T00:40:34.440305Z"},"links":{"citing_paper":"/paper/2508.04325"},"observation_digest":"sha256:30fb9fb40cb238f431de3d3d5ee86ecc5cb5267c98cd5d07dc56eff012dce5d6","observation_id":"254418af-93aa-4d7c-8828-d326e85dc3c7","resolution":{"observed_at":"2026-05-09T07:21:27.102979Z","resolver_source":"raw_fallback","status":"unresolved"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-12T06:34:41.77262+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-12T06:34:41.77262+00:00","source":"crossref"},{"observed_at":"2026-08-12T06:34:36.333875+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-06-05T21:23:00.469572Z","title":"The remaining 72% (38 out of 53) did not conduct in- ternal consistency assessments","venue":null,"work_id":"7e20b51f-5e35-4e28-9ed0-ca3d95b4238f","year":null},"citing_paper":{"arxiv_id":"2508.04325","last_updated":"2026-04-29T05:46:18Z","snapshot_observed_at":"2026-08-02T07:50:38.917887Z","submitted_at":"2025-08-06T11:11:40Z","title":"Beyond the Leaderboard: Rethinking Medical Benchmarks for Large Language Models","version":2},"reference_index":10,"source":"pdf_text","source_observed_at":"2026-05-19T00:40:34.440305Z"},"links":{"citing_paper":"/paper/2508.04325"},"observation_digest":"sha256:4419cd6ce330349459a98102112b409467e0cc9bfebc8c7aab40f78f15a6de33","observation_id":"46a46edf-669b-4cf1-8613-c7a638dc6533","resolution":{"observed_at":"2026-05-09T07:21:27.109994Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-12T06:34:41.77262+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-12T06:34:41.77262+00:00","source":"crossref"},{"observed_at":"2026-08-12T06:34:36.333875+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-06-05T21:23:00.469572Z","title":"These findings highlight a lack of rigorous statistical stan- dards in current benchmark design","venue":null,"work_id":"c6896fa9-648f-4607-8d3f-5d01b7cbd14c","year":null},"citing_paper":{"arxiv_id":"2508.04325","last_updated":"2026-04-29T05:46:18Z","snapshot_observed_at":"2026-08-02T07:50:38.917887Z","submitted_at":"2025-08-06T11:11:40Z","title":"Beyond the Leaderboard: Rethinking Medical Benchmarks for Large Language Models","version":2},"reference_index":11,"source":"pdf_text","source_observed_at":"2026-05-19T00:40:34.440305Z"},"links":{"citing_paper":"/paper/2508.04325"},"observation_digest":"sha256:47855acbd3279c72351ed2e7e161e63bebc63b04b251476fddbbdd03a3c436bc","observation_id":"2f15c379-6bad-4455-a54f-306ba260ab66","resolution":{"observed_at":"2026-05-09T07:21:27.117463Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-12T06:34:41.77262+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-12T06:34:41.77262+00:00","source":"crossref"},{"observed_at":"2026-08-12T06:34:36.333875+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-06-05T21:23:00.469572Z","title":"• Justification: Clearly defined evaluation objectives can avoid ambiguity, facili- tating subsequent data collection, task design, and metric selection","venue":null,"work_id":"2bb2e988-93a1-4945-b438-aaa9aff42856","year":null},"citing_paper":{"arxiv_id":"2508.04325","last_updated":"2026-04-29T05:46:18Z","snapshot_observed_at":"2026-08-02T07:50:38.917887Z","submitted_at":"2025-08-06T11:11:40Z","title":"Beyond the Leaderboard: Rethinking Medical Benchmarks for Large Language Models","version":2},"reference_index":12,"source":"pdf_text","source_observed_at":"2026-05-19T00:40:34.440305Z"},"links":{"citing_paper":"/paper/2508.04325"},"observation_digest":"sha256:8760c715e8869b3d41358c84aad2c560f43224ab31ff2edf14ee8103580b6b03","observation_id":"cd55380f-431d-4d8f-8da6-218be68a6ebd","resolution":{"observed_at":"2026-05-09T07:21:27.121027Z","resolver_source":"raw_fallback","status":"malformed_identifier"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-12T06:34:41.77262+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-12T06:34:41.77262+00:00","source":"crossref"},{"observed_at":"2026-08-12T06:34:36.333875+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-06-05T21:23:00.469572Z","title":"• Justification: Linking the benchmark to real-world application scenarios en- sures that the results are meaningful","venue":null,"work_id":"2e259b02-676c-48ae-a053-1f46b2a7bfbb","year":null},"citing_paper":{"arxiv_id":"2508.04325","last_updated":"2026-04-29T05:46:18Z","snapshot_observed_at":"2026-08-02T07:50:38.917887Z","submitted_at":"2025-08-06T11:11:40Z","title":"Beyond the Leaderboard: Rethinking Medical Benchmarks for Large Language Models","version":2},"reference_index":13,"source":"pdf_text","source_observed_at":"2026-05-19T00:40:34.440305Z"},"links":{"citing_paper":"/paper/2508.04325"},"observation_digest":"sha256:711dabd3ea140149774b420128d67ffc99e353b6e385e257c36e1862465c1463","observation_id":"6e578f29-ce02-4261-b9ba-924cee9406f3","resolution":{"observed_at":"2026-05-09T07:21:27.124399Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-12T06:34:41.77262+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-12T06:34:41.77262+00:00","source":"crossref"},{"observed_at":"2026-08-12T06:34:36.333875+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-06-05T21:23:00.469572Z","title":"• Justification: Demonstrating the unique- ness demonstrates the necessity and jus- tification of the new benchmark","venue":null,"work_id":"747557e5-7d12-43f3-9aeb-53d9f7fcb096","year":null},"citing_paper":{"arxiv_id":"2508.04325","last_updated":"2026-04-29T05:46:18Z","snapshot_observed_at":"2026-08-02T07:50:38.917887Z","submitted_at":"2025-08-06T11:11:40Z","title":"Beyond the Leaderboard: Rethinking Medical Benchmarks for Large Language Models","version":2},"reference_index":14,"source":"pdf_text","source_observed_at":"2026-05-19T00:40:34.440305Z"},"links":{"citing_paper":"/paper/2508.04325"},"observation_digest":"sha256:deb3db9f804bc65dbdd307c696f3bc46bb4ac65b2c8d9be36b2a0c4153a124ef","observation_id":"d9c5e1c6-3b52-4773-8332-3a9af8507bd7","resolution":{"observed_at":"2026-05-09T07:21:27.132540Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-12T06:34:41.77262+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-12T06:34:41.77262+00:00","source":"crossref"},{"observed_at":"2026-08-12T06:34:36.333875+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-06-05T21:23:00.469572Z","title":"• Scoring: – 0: Does not define the target LLM capability","venue":null,"work_id":"d6d79187-6f06-443b-9c4a-a43c18ce1485","year":null},"citing_paper":{"arxiv_id":"2508.04325","last_updated":"2026-04-29T05:46:18Z","snapshot_observed_at":"2026-08-02T07:50:38.917887Z","submitted_at":"2025-08-06T11:11:40Z","title":"Beyond the Leaderboard: Rethinking Medical Benchmarks for Large Language Models","version":2},"reference_index":15,"source":"pdf_text","source_observed_at":"2026-05-19T00:40:34.440305Z"},"links":{"citing_paper":"/paper/2508.04325"},"observation_digest":"sha256:c9433948c911f48a792c0894d51eb5bac955b6b466b96cbe6d1fc1198a0db4bd","observation_id":"8612c382-dbd2-4134-ab22-61234d9a245c","resolution":{"observed_at":"2026-05-09T07:21:27.135286Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-12T06:34:41.77262+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-12T06:34:41.77262+00:00","source":"crossref"},{"observed_at":"2026-08-12T06:34:36.333875+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-06-05T21:23:00.469572Z","title":"• Justification: By clearly defining the medical scope, it helps users better un- derstand the breath and depth of the cov- erage of the benchmark","venue":null,"work_id":"56df045e-c6b4-48b8-a4e4-ee6f13a0c364","year":null},"citing_paper":{"arxiv_id":"2508.04325","last_updated":"2026-04-29T05:46:18Z","snapshot_observed_at":"2026-08-02T07:50:38.917887Z","submitted_at":"2025-08-06T11:11:40Z","title":"Beyond the Leaderboard: Rethinking Medical Benchmarks for Large Language Models","version":2},"reference_index":16,"source":"pdf_text","source_observed_at":"2026-05-19T00:40:34.440305Z"},"links":{"citing_paper":"/paper/2508.04325"},"observation_digest":"sha256:a113bdf0c9aa99db66ff00c7684b1d1b641a7a6447ae28f76534d8dca73885d3","observation_id":"09c00e7b-7395-4424-b2e6-54d32ef3e37e","resolution":{"observed_at":"2026-05-09T07:21:27.138820Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-12T06:34:41.77262+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-12T06:34:41.77262+00:00","source":"crossref"},{"observed_at":"2026-08-12T06:34:36.333875+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-06-05T21:23:00.469572Z","title":"• Justification: An effective benchmark should serve the needs of users","venue":null,"work_id":"2b85039a-b5e2-43a0-ab1a-61be65bb288f","year":null},"citing_paper":{"arxiv_id":"2508.04325","last_updated":"2026-04-29T05:46:18Z","snapshot_observed_at":"2026-08-02T07:50:38.917887Z","submitted_at":"2025-08-06T11:11:40Z","title":"Beyond the Leaderboard: Rethinking Medical Benchmarks for Large Language Models","version":2},"reference_index":17,"source":"pdf_text","source_observed_at":"2026-05-19T00:40:34.440305Z"},"links":{"citing_paper":"/paper/2508.04325"},"observation_digest":"sha256:91156bdb7537ec2f91473ce38bacf307bebf5560bb390151eaf53d8854a51595","observation_id":"115ae008-ebe8-4610-ac17-0acbdd4448d7","resolution":{"observed_at":"2026-05-09T07:21:27.143822Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-12T06:34:41.77262+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-12T06:34:41.77262+00:00","source":"crossref"},{"observed_at":"2026-08-12T06:34:36.333875+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-06-05T21:23:00.469572Z","title":"• Justification: Due to the professionalism and rigor required in the medical field, the development of a benchmark must involve deep engagement from domain experts","venue":null,"work_id":"aaf30bbb-57d1-4de3-bbff-768a613d17db","year":null},"citing_paper":{"arxiv_id":"2508.04325","last_updated":"2026-04-29T05:46:18Z","snapshot_observed_at":"2026-08-02T07:50:38.917887Z","submitted_at":"2025-08-06T11:11:40Z","title":"Beyond the Leaderboard: Rethinking Medical Benchmarks for Large Language Models","version":2},"reference_index":18,"source":"pdf_text","source_observed_at":"2026-05-19T00:40:34.440305Z"},"links":{"citing_paper":"/paper/2508.04325"},"observation_digest":"sha256:4c45c9fb879d019551d911e3e05267799d5891c0f2097582ce55d18f52e4f837","observation_id":"afa28656-4f21-458a-9a2f-5661ac20abdd","resolution":{"observed_at":"2026-05-09T07:21:27.146898Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-12T06:34:41.77262+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-12T06:34:41.77262+00:00","source":"crossref"},{"observed_at":"2026-08-12T06:34:36.333875+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-06-05T21:23:00.469572Z","title":"• Justification: Benchmark content should adhere to recognized, evidence- based medical knowledge sources","venue":null,"work_id":"7808e20d-a988-45fa-8cf2-0c893cd095a0","year":null},"citing_paper":{"arxiv_id":"2508.04325","last_updated":"2026-04-29T05:46:18Z","snapshot_observed_at":"2026-08-02T07:50:38.917887Z","submitted_at":"2025-08-06T11:11:40Z","title":"Beyond the Leaderboard: Rethinking Medical Benchmarks for Large Language Models","version":2},"reference_index":19,"source":"pdf_text","source_observed_at":"2026-05-19T00:40:34.440305Z"},"links":{"citing_paper":"/paper/2508.04325"},"observation_digest":"sha256:b1707f1aa4ff0b34769e287fb8924acc130f7b8e4013973bcd1f165950f4af5b","observation_id":"af582f06-69b6-443b-bfa3-51f07b205f63","resolution":{"observed_at":"2026-05-09T07:21:27.149259Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-12T06:34:41.77262+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-12T06:34:41.77262+00:00","source":"crossref"},{"observed_at":"2026-08-12T06:34:36.333875+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-06-05T21:23:00.469572Z","title":"• Justification: Adherence to medical standards ensures the clinical relevance and consistency, facilitating integration and comparison in reflect real-world medical practice","venue":null,"work_id":"6fcb7c4a-2a7d-44c1-b935-56cff1c3cfb6","year":null},"citing_paper":{"arxiv_id":"2508.04325","last_updated":"2026-04-29T05:46:18Z","snapshot_observed_at":"2026-08-02T07:50:38.917887Z","submitted_at":"2025-08-06T11:11:40Z","title":"Beyond the Leaderboard: Rethinking Medical Benchmarks for Large Language Models","version":2},"reference_index":20,"source":"pdf_text","source_observed_at":"2026-05-19T00:40:34.440305Z"},"links":{"citing_paper":"/paper/2508.04325"},"observation_digest":"sha256:bf3c50222dcb4305792dab650dd95003ad554a515ca015dd41b2f108d447629f","observation_id":"54b06b0b-bfd1-43e8-9dfc-3a22d65cfb31","resolution":{"observed_at":"2026-05-09T07:21:27.154387Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-12T06:34:41.77262+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-12T06:34:41.77262+00:00","source":"crossref"},{"observed_at":"2026-08-12T06:34:36.333875+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-06-05T21:23:00.469572Z","title":"• Justification: Evaluation metrics di- rectly shape the interpretation of results","venue":null,"work_id":"f9be051c-e5f1-4840-8ec7-056efb3e2185","year":null},"citing_paper":{"arxiv_id":"2508.04325","last_updated":"2026-04-29T05:46:18Z","snapshot_observed_at":"2026-08-02T07:50:38.917887Z","submitted_at":"2025-08-06T11:11:40Z","title":"Beyond the Leaderboard: Rethinking Medical Benchmarks for Large Language Models","version":2},"reference_index":21,"source":"pdf_text","source_observed_at":"2026-05-19T00:40:34.440305Z"},"links":{"citing_paper":"/paper/2508.04325"},"observation_digest":"sha256:905835e6e701720a2e22373528e6879a64cbc8c9461505edaa57ac91e42992af","observation_id":"2a0d267c-a790-4a4e-8775-45d0295c4742","resolution":{"observed_at":"2026-05-09T07:21:27.159888Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-12T06:34:41.77262+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-12T06:34:41.77262+00:00","source":"crossref"},{"observed_at":"2026-08-12T06:34:36.333875+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-06-05T21:23:00.469572Z","title":"• Justification: In high-stakes medical do- main, going beyond correctness is vi- tal","venue":null,"work_id":"9b16a395-071e-4536-984c-469c018e802e","year":null},"citing_paper":{"arxiv_id":"2508.04325","last_updated":"2026-04-29T05:46:18Z","snapshot_observed_at":"2026-08-02T07:50:38.917887Z","submitted_at":"2025-08-06T11:11:40Z","title":"Beyond the Leaderboard: Rethinking Medical Benchmarks for Large Language Models","version":2},"reference_index":22,"source":"pdf_text","source_observed_at":"2026-05-19T00:40:34.440305Z"},"links":{"citing_paper":"/paper/2508.04325"},"observation_digest":"sha256:db9332b2ec5b38c1a61e78ce032cc8f59ee698e88a379979c1bbd625f2ea88b5","observation_id":"6d837b2a-c9ec-41d3-9b41-993b9ea59a8c","resolution":{"observed_at":"2026-05-09T07:21:27.162167Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-12T06:34:41.77262+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-12T06:34:41.77262+00:00","source":"crossref"},{"observed_at":"2026-08-12T06:34:36.333875+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-06-05T21:23:00.469572Z","title":null,"venue":null,"work_id":"2699eb33-8da9-4e82-a6bd-fb8f3551d48c","year":null},"citing_paper":{"arxiv_id":"2508.04325","last_updated":"2026-04-29T05:46:18Z","snapshot_observed_at":"2026-08-02T07:50:38.917887Z","submitted_at":"2025-08-06T11:11:40Z","title":"Beyond the Leaderboard: Rethinking Medical Benchmarks for Large Language Models","version":2},"reference_index":23,"source":"pdf_text","source_observed_at":"2026-05-19T00:40:34.440305Z"},"links":{"citing_paper":"/paper/2508.04325"},"observation_digest":"sha256:eef093036a6448ae8e4688f674dd728a751ddf8303451173181325af96b33604","observation_id":"90ff61c2-0a41-4b02-ae99-c208046edd83","resolution":{"observed_at":"2026-05-09T07:21:27.164379Z","resolver_source":"raw_fallback","status":"unresolved"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-12T06:34:41.77262+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-12T06:34:41.77262+00:00","source":"crossref"},{"observed_at":"2026-08-12T06:34:36.333875+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-06-05T21:23:00.469572Z","title":"• Justification: Clear and traceable data sources are critical to ensure trans- parency and ethical data usage, which is especially important when sensitive data is involved","venue":null,"work_id":"c09bfcce-3aeb-4c90-97c5-deab5c8bae0e","year":null},"citing_paper":{"arxiv_id":"2508.04325","last_updated":"2026-04-29T05:46:18Z","snapshot_observed_at":"2026-08-02T07:50:38.917887Z","submitted_at":"2025-08-06T11:11:40Z","title":"Beyond the Leaderboard: Rethinking Medical Benchmarks for Large Language Models","version":2},"reference_index":24,"source":"pdf_text","source_observed_at":"2026-05-19T00:40:34.440305Z"},"links":{"citing_paper":"/paper/2508.04325"},"observation_digest":"sha256:80271196b8eaf887a1816efb5656686f64f37ae8b0bae9a720983b15a936b781","observation_id":"a4f054ea-2641-43f2-ad86-1ad89a34ad89","resolution":{"observed_at":"2026-05-09T07:21:27.169014Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-12T06:34:41.77262+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-12T06:34:41.77262+00:00","source":"crossref"},{"observed_at":"2026-08-12T06:34:36.333875+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-06-05T21:23:00.469572Z","title":"• Justification: Collecting data from unre- liable sources may lead to invalid results for medical applications","venue":null,"work_id":"56703b01-c726-4f73-997e-6c1a1c92e389","year":null},"citing_paper":{"arxiv_id":"2508.04325","last_updated":"2026-04-29T05:46:18Z","snapshot_observed_at":"2026-08-02T07:50:38.917887Z","submitted_at":"2025-08-06T11:11:40Z","title":"Beyond the Leaderboard: Rethinking Medical Benchmarks for Large Language Models","version":2},"reference_index":25,"source":"pdf_text","source_observed_at":"2026-05-19T00:40:34.440305Z"},"links":{"citing_paper":"/paper/2508.04325"},"observation_digest":"sha256:6f447c52afe32cbdd76cff7ef0453f9f62567ad2eccec77819c4cbd13bd378df","observation_id":"2cd13a52-7c2e-44bf-8c0e-b6a43ef6218b","resolution":{"observed_at":"2026-05-09T07:21:27.171308Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-12T06:34:41.77262+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-12T06:34:41.77262+00:00","source":"crossref"},{"observed_at":"2026-08-12T06:34:36.333875+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-06-05T21:23:00.469572Z","title":"For synthetically generated data, the con- struction process and verification for its authenticity (e.g., expert review) should also be described","venue":null,"work_id":"1592907a-64a9-4460-a651-3fee421c60f2","year":null},"citing_paper":{"arxiv_id":"2508.04325","last_updated":"2026-04-29T05:46:18Z","snapshot_observed_at":"2026-08-02T07:50:38.917887Z","submitted_at":"2025-08-06T11:11:40Z","title":"Beyond the Leaderboard: Rethinking Medical Benchmarks for Large Language Models","version":2},"reference_index":26,"source":"pdf_text","source_observed_at":"2026-05-19T00:40:34.440305Z"},"links":{"citing_paper":"/paper/2508.04325"},"observation_digest":"sha256:2708e7eb53db82b4be0a578ea5bfe1430046b4e6fa4e5fe165929c46b8ebce2f","observation_id":"26248f49-8d28-4ac9-83b7-388909d07d43","resolution":{"observed_at":"2026-05-09T07:21:27.173632Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-12T06:34:41.77262+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-12T06:34:41.77262+00:00","source":"crossref"},{"observed_at":"2026-08-12T06:34:36.333875+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-06-05T21:23:00.469572Z","title":"• Justification: A benchmark that lacks representativeness may lead to bias in evaluation, reducing the clinical rele- vance, generalizability and fairness of the results","venue":null,"work_id":"08120959-56c1-4a86-8d7f-28905d50b005","year":null},"citing_paper":{"arxiv_id":"2508.04325","last_updated":"2026-04-29T05:46:18Z","snapshot_observed_at":"2026-08-02T07:50:38.917887Z","submitted_at":"2025-08-06T11:11:40Z","title":"Beyond the Leaderboard: Rethinking Medical Benchmarks for Large Language Models","version":2},"reference_index":27,"source":"pdf_text","source_observed_at":"2026-05-19T00:40:34.440305Z"},"links":{"citing_paper":"/paper/2508.04325"},"observation_digest":"sha256:91e19e31ec04fb561c630085167f1353f485a031a59a8f037c1cb9c033f6de2f","observation_id":"da50aa52-0560-44e8-8d6f-a1ac0c9946f0","resolution":{"observed_at":"2026-05-09T07:21:27.178837Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-12T06:34:41.77262+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-12T06:34:41.77262+00:00","source":"crossref"},{"observed_at":"2026-08-12T06:34:36.333875+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-06-05T21:23:00.469572Z","title":"• Justification: Ensuring the dataset cov- ers a variety helps comprehensively eval- uate the model’s generalization ability, reducing bias in the evaluation results","venue":null,"work_id":"2411b73c-c8ae-460b-95c9-7d3576791c77","year":null},"citing_paper":{"arxiv_id":"2508.04325","last_updated":"2026-04-29T05:46:18Z","snapshot_observed_at":"2026-08-02T07:50:38.917887Z","submitted_at":"2025-08-06T11:11:40Z","title":"Beyond the Leaderboard: Rethinking Medical Benchmarks for Large Language Models","version":2},"reference_index":28,"source":"pdf_text","source_observed_at":"2026-05-19T00:40:34.440305Z"},"links":{"citing_paper":"/paper/2508.04325"},"observation_digest":"sha256:82eec1bb36a5ca84ccdceaf38a5fc884358dd12e43fcfd9acaf07492d2df648b","observation_id":"5994d96b-e7f1-4614-928e-423073eec76f","resolution":{"observed_at":"2026-05-09T07:21:27.182725Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-12T06:34:41.77262+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-12T06:34:41.77262+00:00","source":"crossref"},{"observed_at":"2026-08-12T06:34:36.333875+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-06-05T21:23:00.469572Z","title":"• Justification: It ensures that the final dataset is well-structured, enhancing re- liability","venue":null,"work_id":"c8a34253-2595-42b7-91ee-95d496f25f16","year":null},"citing_paper":{"arxiv_id":"2508.04325","last_updated":"2026-04-29T05:46:18Z","snapshot_observed_at":"2026-08-02T07:50:38.917887Z","submitted_at":"2025-08-06T11:11:40Z","title":"Beyond the Leaderboard: Rethinking Medical Benchmarks for Large Language Models","version":2},"reference_index":29,"source":"pdf_text","source_observed_at":"2026-05-19T00:40:34.440305Z"},"links":{"citing_paper":"/paper/2508.04325"},"observation_digest":"sha256:8b9a6fd696f28bc394b42d645f7ee098107b6e5a988993c543e411f7347b4973","observation_id":"e42c3370-851a-4a00-b9b6-417e3ba13e45","resolution":{"observed_at":"2026-05-09T07:21:27.184919Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-12T06:34:41.77262+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-12T06:34:41.77262+00:00","source":"crossref"},{"observed_at":"2026-08-12T06:34:36.333875+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-06-05T21:23:00.469572Z","title":"Methods of de-identification should be described and compliance with relevant regulations (e.g., HIPAA) should be clearly stated","venue":null,"work_id":"ee83a1e8-e089-47c6-b69b-22953af99b3b","year":null},"citing_paper":{"arxiv_id":"2508.04325","last_updated":"2026-04-29T05:46:18Z","snapshot_observed_at":"2026-08-02T07:50:38.917887Z","submitted_at":"2025-08-06T11:11:40Z","title":"Beyond the Leaderboard: Rethinking Medical Benchmarks for Large Language Models","version":2},"reference_index":30,"source":"pdf_text","source_observed_at":"2026-05-19T00:40:34.440305Z"},"links":{"citing_paper":"/paper/2508.04325"},"observation_digest":"sha256:f6943cdea2152932807940d23ba2848c6f5b76ad2b9d6717f55d856229dce280","observation_id":"aa223e34-7247-4c85-95a6-cfdd5525fc8c","resolution":{"observed_at":"2026-05-09T07:21:27.203571Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-12T06:34:41.77262+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-12T06:34:41.77262+00:00","source":"crossref"},{"observed_at":"2026-08-12T06:34:36.333875+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-06-05T21:23:00.469572Z","title":"• Justification: A clear and consistent for- mat is essential for standardized evalua- tion","venue":null,"work_id":"cf64c8cf-e1de-450f-a42a-63abcd077e73","year":null},"citing_paper":{"arxiv_id":"2508.04325","last_updated":"2026-04-29T05:46:18Z","snapshot_observed_at":"2026-08-02T07:50:38.917887Z","submitted_at":"2025-08-06T11:11:40Z","title":"Beyond the Leaderboard: Rethinking Medical Benchmarks for Large Language Models","version":2},"reference_index":31,"source":"pdf_text","source_observed_at":"2026-05-19T00:40:34.440305Z"},"links":{"citing_paper":"/paper/2508.04325"},"observation_digest":"sha256:5b2312057441e0414e242e75901966edcc2b179495eb811511de1bfe5d0ba9c2","observation_id":"50fc5e83-57f2-4993-9827-6357b3053bb2","resolution":{"observed_at":"2026-05-09T07:21:27.217994Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-12T06:34:41.77262+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-12T06:34:41.77262+00:00","source":"crossref"},{"observed_at":"2026-08-12T06:34:36.333875+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-06-05T21:23:00.469572Z","title":"• Justification: A dataset construction pro- cess without review mechanism is prone to errors","venue":null,"work_id":"36181980-7cb8-444b-8a57-60255e8d90a2","year":null},"citing_paper":{"arxiv_id":"2508.04325","last_updated":"2026-04-29T05:46:18Z","snapshot_observed_at":"2026-08-02T07:50:38.917887Z","submitted_at":"2025-08-06T11:11:40Z","title":"Beyond the Leaderboard: Rethinking Medical Benchmarks for Large Language Models","version":2},"reference_index":32,"source":"pdf_text","source_observed_at":"2026-05-19T00:40:34.440305Z"},"links":{"citing_paper":"/paper/2508.04325"},"observation_digest":"sha256:22110eb2b980a8f95e7c1baf609b14155282fcffd1b9a09a0407e2655f8eeeff","observation_id":"4505880c-2f99-471d-812f-24e410f96e33","resolution":{"observed_at":"2026-05-09T07:21:27.220262Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-12T06:34:41.77262+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-12T06:34:41.77262+00:00","source":"crossref"},{"observed_at":"2026-08-12T06:34:36.333875+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-06-05T21:23:00.469572Z","title":"• Justification: Clear reference answers or scoring guidelines ensures transpar- ent and accurate evaluation","venue":null,"work_id":"21041119-a4d1-4b23-84d8-72b8812ba9bc","year":null},"citing_paper":{"arxiv_id":"2508.04325","last_updated":"2026-04-29T05:46:18Z","snapshot_observed_at":"2026-08-02T07:50:38.917887Z","submitted_at":"2025-08-06T11:11:40Z","title":"Beyond the Leaderboard: Rethinking Medical Benchmarks for Large Language Models","version":2},"reference_index":33,"source":"pdf_text","source_observed_at":"2026-05-19T00:40:34.440305Z"},"links":{"citing_paper":"/paper/2508.04325"},"observation_digest":"sha256:4d5e259e9d0e6f86413a8abf24c27993a6c10dbcf9eac07e06e106d94ae63c97","observation_id":"bea2e980-b546-4663-b09a-922cebbd5a73","resolution":{"observed_at":"2026-05-09T07:21:27.222414Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-12T06:34:41.77262+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-12T06:34:41.77262+00:00","source":"crossref"},{"observed_at":"2026-08-12T06:34:36.333875+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-06-05T21:23:00.469572Z","title":"• Justification: Data contamination may lead to inflated performance, which only reflect memorization instead of medical capability from the models","venue":null,"work_id":"e7910845-eb7c-407f-b801-103ceb1b9d2c","year":null},"citing_paper":{"arxiv_id":"2508.04325","last_updated":"2026-04-29T05:46:18Z","snapshot_observed_at":"2026-08-02T07:50:38.917887Z","submitted_at":"2025-08-06T11:11:40Z","title":"Beyond the Leaderboard: Rethinking Medical Benchmarks for Large Language Models","version":2},"reference_index":34,"source":"pdf_text","source_observed_at":"2026-05-19T00:40:34.440305Z"},"links":{"citing_paper":"/paper/2508.04325"},"observation_digest":"sha256:6a442c456821e884a93e2902239092c250793f3fa674d8de9d9562211e98466f","observation_id":"a75d24a2-b9f0-46d3-a2a4-eea584efc0c5","resolution":{"observed_at":"2026-05-09T07:21:27.224666Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-12T06:34:41.77262+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-12T06:34:41.77262+00:00","source":"crossref"},{"observed_at":"2026-08-12T06:34:36.333875+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-06-05T21:23:00.469572Z","title":"• Justification: It ensures that users can use the benchmark conveniently, promot- ing benchmark adoption and ensuring fair, transparent, and consistent evalua- tion","venue":null,"work_id":"7624fd07-d7c4-41cb-8a49-d20233358921","year":null},"citing_paper":{"arxiv_id":"2508.04325","last_updated":"2026-04-29T05:46:18Z","snapshot_observed_at":"2026-08-02T07:50:38.917887Z","submitted_at":"2025-08-06T11:11:40Z","title":"Beyond the Leaderboard: Rethinking Medical Benchmarks for Large Language Models","version":2},"reference_index":35,"source":"pdf_text","source_observed_at":"2026-05-19T00:40:34.440305Z"},"links":{"citing_paper":"/paper/2508.04325"},"observation_digest":"sha256:48081cc46f4778bcb1e4ffce99200468c4437e0d02047520ccfb87c68320065e","observation_id":"b31b3c6d-4ae9-4238-bf66-8ce3282ff151","resolution":{"observed_at":"2026-05-09T07:21:27.248332Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-12T06:34:41.77262+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-12T06:34:41.77262+00:00","source":"crossref"},{"observed_at":"2026-08-12T06:34:36.333875+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-06-05T21:23:00.469572Z","title":null,"venue":null,"work_id":"4f5da9e7-11b8-4e06-9b33-fc9670078bf4","year":null},"citing_paper":{"arxiv_id":"2508.04325","last_updated":"2026-04-29T05:46:18Z","snapshot_observed_at":"2026-08-02T07:50:38.917887Z","submitted_at":"2025-08-06T11:11:40Z","title":"Beyond the Leaderboard: Rethinking Medical Benchmarks for Large Language Models","version":2},"reference_index":36,"source":"pdf_text","source_observed_at":"2026-05-19T00:40:34.440305Z"},"links":{"citing_paper":"/paper/2508.04325"},"observation_digest":"sha256:24a93e067a3314b62aec77fcca3a09fb4642fa8318a89fe4fc81d4beba696810","observation_id":"9e5e6522-2d9a-4b6f-928b-c0510cbf89a5","resolution":{"observed_at":"2026-05-09T07:21:27.253645Z","resolver_source":"raw_fallback","status":"unresolved"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-12T06:34:41.77262+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-12T06:34:41.77262+00:00","source":"crossref"},{"observed_at":"2026-08-12T06:34:36.333875+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-06-05T21:23:00.469572Z","title":"• Justification: Providing different per- formance baselines allows comparison against the model’s performance, en- abling a deeper understanding and better interpretability","venue":null,"work_id":"959853c2-c718-473d-8263-bfe697bacd6e","year":null},"citing_paper":{"arxiv_id":"2508.04325","last_updated":"2026-04-29T05:46:18Z","snapshot_observed_at":"2026-08-02T07:50:38.917887Z","submitted_at":"2025-08-06T11:11:40Z","title":"Beyond the Leaderboard: Rethinking Medical Benchmarks for Large Language Models","version":2},"reference_index":37,"source":"pdf_text","source_observed_at":"2026-05-19T00:40:34.440305Z"},"links":{"citing_paper":"/paper/2508.04325"},"observation_digest":"sha256:af270c80d42d01b212382a8a1f897d64820bb0c5d5c753828fe652585b685262","observation_id":"90d60789-305f-4f8a-8114-8bcee9f2df84","resolution":{"observed_at":"2026-05-09T07:21:27.258693Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-12T06:34:41.77262+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-12T06:34:41.77262+00:00","source":"crossref"},{"observed_at":"2026-08-12T06:34:36.333875+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-06-05T21:23:00.469572Z","title":"• Justification: In medical domain, un- derstanding the model’s decision-making process is just as important as the final answer","venue":null,"work_id":"0f1d6cfe-f6d0-4c67-826c-63e740084bfb","year":null},"citing_paper":{"arxiv_id":"2508.04325","last_updated":"2026-04-29T05:46:18Z","snapshot_observed_at":"2026-08-02T07:50:38.917887Z","submitted_at":"2025-08-06T11:11:40Z","title":"Beyond the Leaderboard: Rethinking Medical Benchmarks for Large Language Models","version":2},"reference_index":38,"source":"pdf_text","source_observed_at":"2026-05-19T00:40:34.440305Z"},"links":{"citing_paper":"/paper/2508.04325"},"observation_digest":"sha256:5014921d2605c3adb13ca606334d78aa28f37e715a9092dc00d772733c39d180","observation_id":"0b27be8a-1277-44db-9437-4516e2592798","resolution":{"observed_at":"2026-05-09T07:21:27.260885Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-12T06:34:41.77262+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-12T06:34:41.77262+00:00","source":"crossref"},{"observed_at":"2026-08-12T06:34:36.333875+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-06-05T21:23:00.469572Z","title":"Robustness testing en- sures model’s output is consistent and reliable under different conditions","venue":null,"work_id":"fc99778d-9e84-4dba-8e02-b4327fd8c71e","year":null},"citing_paper":{"arxiv_id":"2508.04325","last_updated":"2026-04-29T05:46:18Z","snapshot_observed_at":"2026-08-02T07:50:38.917887Z","submitted_at":"2025-08-06T11:11:40Z","title":"Beyond the Leaderboard: Rethinking Medical Benchmarks for Large Language Models","version":2},"reference_index":39,"source":"pdf_text","source_observed_at":"2026-05-19T00:40:34.440305Z"},"links":{"citing_paper":"/paper/2508.04325"},"observation_digest":"sha256:9a91f84d37dd6e4e387c1b4bfd0748972d5cbb1159bc538ed3c36c36aaf48917","observation_id":"8c4670c0-ee98-4236-a665-adff6ff18ef4","resolution":{"observed_at":"2026-05-09T07:21:27.262979Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-12T06:34:41.77262+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-12T06:34:41.77262+00:00","source":"crossref"},{"observed_at":"2026-08-12T06:34:36.333875+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-06-05T21:23:00.469572Z","title":null,"venue":null,"work_id":"bcbf3e42-e049-454b-ae8a-0880a86dd8f2","year":null},"citing_paper":{"arxiv_id":"2508.04325","last_updated":"2026-04-29T05:46:18Z","snapshot_observed_at":"2026-08-02T07:50:38.917887Z","submitted_at":"2025-08-06T11:11:40Z","title":"Beyond the Leaderboard: Rethinking Medical Benchmarks for Large Language Models","version":2},"reference_index":40,"source":"pdf_text","source_observed_at":"2026-05-19T00:40:34.440305Z"},"links":{"citing_paper":"/paper/2508.04325"},"observation_digest":"sha256:b3ed838c2023242765b4fe1f449e566aea5d3f6217683b3c1ce2f3f8af6e9991","observation_id":"4e559e94-892d-4108-9972-9c703ea4c069","resolution":{"observed_at":"2026-05-09T07:21:27.264831Z","resolver_source":"raw_fallback","status":"unresolved"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-12T06:34:41.77262+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-12T06:34:41.77262+00:00","source":"crossref"},{"observed_at":"2026-08-12T06:34:36.333875+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-06-05T21:23:00.469572Z","title":"I don’t know","venue":null,"work_id":"1983b092-d04c-46fa-992b-b3a56ba7a6cf","year":null},"citing_paper":{"arxiv_id":"2508.04325","last_updated":"2026-04-29T05:46:18Z","snapshot_observed_at":"2026-08-02T07:50:38.917887Z","submitted_at":"2025-08-06T11:11:40Z","title":"Beyond the Leaderboard: Rethinking Medical Benchmarks for Large Language Models","version":2},"reference_index":41,"source":"pdf_text","source_observed_at":"2026-05-19T00:40:34.440305Z"},"links":{"citing_paper":"/paper/2508.04325"},"observation_digest":"sha256:095b0ee8f1f57d0e1d8700b43bb4c8dbd23fbacda73f8a9eefcce50335d37eb8","observation_id":"a87bf755-a425-4f44-bc89-28e74f1ad7f0","resolution":{"observed_at":"2026-05-09T07:21:27.266632Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-12T06:34:41.77262+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-12T06:34:41.77262+00:00","source":"crossref"},{"observed_at":"2026-08-12T06:34:36.333875+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-06-05T21:23:00.469572Z","title":"• Justification: It ensures that different types of models can be tested under the same interface","venue":null,"work_id":"f40ac056-4f9d-4923-9c27-889d28c2addd","year":null},"citing_paper":{"arxiv_id":"2508.04325","last_updated":"2026-04-29T05:46:18Z","snapshot_observed_at":"2026-08-02T07:50:38.917887Z","submitted_at":"2025-08-06T11:11:40Z","title":"Beyond the Leaderboard: Rethinking Medical Benchmarks for Large Language Models","version":2},"reference_index":42,"source":"pdf_text","source_observed_at":"2026-05-19T00:40:34.440305Z"},"links":{"citing_paper":"/paper/2508.04325"},"observation_digest":"sha256:14ad3ddf17baf08d0b05e461a8c215e0ec43f5610473287132c579fbaddd0d37","observation_id":"75f581da-0c51-41ed-9e84-e3cb2fb9927b","resolution":{"observed_at":"2026-05-09T07:21:27.268513Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-12T06:34:41.77262+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-12T06:34:41.77262+00:00","source":"crossref"},{"observed_at":"2026-08-12T06:34:36.333875+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-06-05T21:23:00.469572Z","title":"• Justification: Sufficient coverage of the core medical competencies the bench- mark aims to measure is the prerequisite for establishing content validity","venue":null,"work_id":"227d5c09-d704-42df-aab8-cd7f3838712b","year":null},"citing_paper":{"arxiv_id":"2508.04325","last_updated":"2026-04-29T05:46:18Z","snapshot_observed_at":"2026-08-02T07:50:38.917887Z","submitted_at":"2025-08-06T11:11:40Z","title":"Beyond the Leaderboard: Rethinking Medical Benchmarks for Large Language Models","version":2},"reference_index":43,"source":"pdf_text","source_observed_at":"2026-05-19T00:40:34.440305Z"},"links":{"citing_paper":"/paper/2508.04325"},"observation_digest":"sha256:bbb2f26c1c386d10faf9ac4a27254abec6f088ec864afff92d69f8c7f9b204f0","observation_id":"6938e0c3-ce32-4bac-9b7d-17a2803e2ef7","resolution":{"observed_at":"2026-05-09T07:21:27.274950Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-12T06:34:41.77262+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-12T06:34:41.77262+00:00","source":"crossref"},{"observed_at":"2026-08-12T06:34:36.333875+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-06-05T21:23:00.469572Z","title":"• Justification: Ensuring that the evalu- ation task closely mirrors the targeted clinical practice in real-world scenarios enhance the relevance of the benchmark","venue":null,"work_id":"62e9c303-c2db-40ce-9ddb-cc1b54a199fe","year":null},"citing_paper":{"arxiv_id":"2508.04325","last_updated":"2026-04-29T05:46:18Z","snapshot_observed_at":"2026-08-02T07:50:38.917887Z","submitted_at":"2025-08-06T11:11:40Z","title":"Beyond the Leaderboard: Rethinking Medical Benchmarks for Large Language Models","version":2},"reference_index":44,"source":"pdf_text","source_observed_at":"2026-05-19T00:40:34.440305Z"},"links":{"citing_paper":"/paper/2508.04325"},"observation_digest":"sha256:61d94a46c20b01c565709ed1ec74f456a8d66ddd7870133c818119b4a8010747","observation_id":"fe874795-7e6f-4106-919b-ae7a965300c3","resolution":{"observed_at":"2026-05-09T07:21:27.284769Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-12T06:34:41.77262+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-12T06:34:41.77262+00:00","source":"crossref"},{"observed_at":"2026-08-12T06:34:36.333875+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-06-05T21:23:00.469572Z","title":"• Justification: An effective benchmark should be capable of differentiating and distinguishing models of varying capabil- ities","venue":null,"work_id":"2b5d381f-edfb-4fa0-b477-ac2c474aeae3","year":null},"citing_paper":{"arxiv_id":"2508.04325","last_updated":"2026-04-29T05:46:18Z","snapshot_observed_at":"2026-08-02T07:50:38.917887Z","submitted_at":"2025-08-06T11:11:40Z","title":"Beyond the Leaderboard: Rethinking Medical Benchmarks for Large Language Models","version":2},"reference_index":45,"source":"pdf_text","source_observed_at":"2026-05-19T00:40:34.440305Z"},"links":{"citing_paper":"/paper/2508.04325"},"observation_digest":"sha256:c0fbe9bf0b1a5f7a4d76bc1408de2fcdeb28a55e505097649ff4b66b9af7b743","observation_id":"237b8ce0-284e-4b9a-86ec-b3ae24123eb9","resolution":{"observed_at":"2026-05-09T07:21:27.287397Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-12T06:34:41.77262+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-12T06:34:41.77262+00:00","source":"crossref"},{"observed_at":"2026-08-12T06:34:36.333875+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-06-05T21:23:00.469572Z","title":null,"venue":null,"work_id":"108eb5c7-4f43-4baf-b1d0-4d68b3e2f357","year":null},"citing_paper":{"arxiv_id":"2508.04325","last_updated":"2026-04-29T05:46:18Z","snapshot_observed_at":"2026-08-02T07:50:38.917887Z","submitted_at":"2025-08-06T11:11:40Z","title":"Beyond the Leaderboard: Rethinking Medical Benchmarks for Large Language Models","version":2},"reference_index":46,"source":"pdf_text","source_observed_at":"2026-05-19T00:40:34.440305Z"},"links":{"citing_paper":"/paper/2508.04325"},"observation_digest":"sha256:86bc120154c8f4ec7fedc6f95f46397845e272d6dc81baffa35c4ae65bd38d32","observation_id":"448593d2-abb6-4985-9034-af5d5e4f029b","resolution":{"observed_at":"2026-05-09T07:21:27.289251Z","resolver_source":"raw_fallback","status":"unresolved"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-12T06:34:41.77262+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-12T06:34:41.77262+00:00","source":"crossref"},{"observed_at":"2026-08-12T06:34:36.333875+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-06-05T21:23:00.469572Z","title":"• Scoring: – 0: Does not mention or conduct any internal consistency measurement","venue":null,"work_id":"0571ccfd-0029-421a-8b07-d776cae169ed","year":null},"citing_paper":{"arxiv_id":"2508.04325","last_updated":"2026-04-29T05:46:18Z","snapshot_observed_at":"2026-08-02T07:50:38.917887Z","submitted_at":"2025-08-06T11:11:40Z","title":"Beyond the Leaderboard: Rethinking Medical Benchmarks for Large Language Models","version":2},"reference_index":47,"source":"pdf_text","source_observed_at":"2026-05-19T00:40:34.440305Z"},"links":{"citing_paper":"/paper/2508.04325"},"observation_digest":"sha256:2441abdacf7e859b9d0a64062807b6988a64e3df8370809e600b381c64d126f4","observation_id":"07a93eb3-cf09-4d01-bfea-f096a83fd057","resolution":{"observed_at":"2026-05-09T07:21:27.291391Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-12T06:34:41.77262+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-12T06:34:41.77262+00:00","source":"crossref"},{"observed_at":"2026-08-12T06:34:36.333875+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-06-05T21:23:00.469572Z","title":null,"venue":null,"work_id":"bb1b70d0-0ef3-49cb-8dcb-a6796934b371","year":null},"citing_paper":{"arxiv_id":"2508.04325","last_updated":"2026-04-29T05:46:18Z","snapshot_observed_at":"2026-08-02T07:50:38.917887Z","submitted_at":"2025-08-06T11:11:40Z","title":"Beyond the Leaderboard: Rethinking Medical Benchmarks for Large Language Models","version":2},"reference_index":48,"source":"pdf_text","source_observed_at":"2026-05-19T00:40:34.440305Z"},"links":{"citing_paper":"/paper/2508.04325"},"observation_digest":"sha256:d4b3a2441ff06de7740127a4c991ea514040724cc152ebe168011cf2142d8aeb","observation_id":"57553101-f584-42f9-a6cb-227bb695d59f","resolution":{"observed_at":"2026-05-09T07:21:27.293226Z","resolver_source":"raw_fallback","status":"unresolved"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-12T06:34:41.77262+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-12T06:34:41.77262+00:00","source":"crossref"},{"observed_at":"2026-08-12T06:34:36.333875+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-06-05T21:23:00.469572Z","title":"• Justification: A complete and clear doc- umentation help users understand and use the benchmark properly, enhancing the usability, reproducibility and trans- parency","venue":null,"work_id":"3399dc07-74ef-4444-9fea-710b1a10fc20","year":null},"citing_paper":{"arxiv_id":"2508.04325","last_updated":"2026-04-29T05:46:18Z","snapshot_observed_at":"2026-08-02T07:50:38.917887Z","submitted_at":"2025-08-06T11:11:40Z","title":"Beyond the Leaderboard: Rethinking Medical Benchmarks for Large Language Models","version":2},"reference_index":49,"source":"pdf_text","source_observed_at":"2026-05-19T00:40:34.440305Z"},"links":{"citing_paper":"/paper/2508.04325"},"observation_digest":"sha256:c9a580fcf4416964f77d99bb7aa02f91e56fa93b1b2228f80cdbd0eaef087ebf","observation_id":"1f8582eb-a03b-49b5-9826-2d23b583ec2a","resolution":{"observed_at":"2026-05-09T07:21:27.295162Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-12T06:34:41.77262+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-12T06:34:41.77262+00:00","source":"crossref"},{"observed_at":"2026-08-12T06:34:36.333875+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-06-05T21:23:00.469572Z","title":"• Justification: Clear evaluation guide- lines help users better understand how model performance is quantified, ensur- ing a shared interpretation and under- standing","venue":null,"work_id":"e12b2165-229c-4b21-8f1a-e1559539605c","year":null},"citing_paper":{"arxiv_id":"2508.04325","last_updated":"2026-04-29T05:46:18Z","snapshot_observed_at":"2026-08-02T07:50:38.917887Z","submitted_at":"2025-08-06T11:11:40Z","title":"Beyond the Leaderboard: Rethinking Medical Benchmarks for Large Language Models","version":2},"reference_index":50,"source":"pdf_text","source_observed_at":"2026-05-19T00:40:34.440305Z"},"links":{"citing_paper":"/paper/2508.04325"},"observation_digest":"sha256:092a5e92287869c1799b794817497c4ed730c68703498a6a1c5b4b4544136666","observation_id":"e086260e-9abc-4c22-bbba-40f112dbed87","resolution":{"observed_at":"2026-05-09T07:21:27.308599Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-12T06:34:41.77262+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-12T06:34:41.77262+00:00","source":"crossref"},{"observed_at":"2026-08-12T06:34:36.333875+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-06-05T21:23:00.469572Z","title":"• Justification: Disclosing limitations and risks demonstrates scientific rigor and responsibility","venue":null,"work_id":"146ab2ab-3247-4cd2-8558-90d0a61ea841","year":null},"citing_paper":{"arxiv_id":"2508.04325","last_updated":"2026-04-29T05:46:18Z","snapshot_observed_at":"2026-08-02T07:50:38.917887Z","submitted_at":"2025-08-06T11:11:40Z","title":"Beyond the Leaderboard: Rethinking Medical Benchmarks for Large Language Models","version":2},"reference_index":51,"source":"pdf_text","source_observed_at":"2026-05-19T00:40:34.440305Z"},"links":{"citing_paper":"/paper/2508.04325"},"observation_digest":"sha256:cc392b134cc92229146c3d5bec4d36ff041b9b862686c04830af87c3b868b35c","observation_id":"c6186465-86d9-4fe7-8d99-e82efc9804e6","resolution":{"observed_at":"2026-05-09T07:21:27.313469Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-12T06:34:41.77262+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-12T06:34:41.77262+00:00","source":"crossref"},{"observed_at":"2026-08-12T06:34:36.333875+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-06-05T21:23:00.469572Z","title":"• Justification: Going through peer review process means that the design, validity and results of a benchmark has been rig- orous evaluated","venue":null,"work_id":"7582b4bd-289c-4e0c-887a-f326d613fee2","year":null},"citing_paper":{"arxiv_id":"2508.04325","last_updated":"2026-04-29T05:46:18Z","snapshot_observed_at":"2026-08-02T07:50:38.917887Z","submitted_at":"2025-08-06T11:11:40Z","title":"Beyond the Leaderboard: Rethinking Medical Benchmarks for Large Language Models","version":2},"reference_index":52,"source":"pdf_text","source_observed_at":"2026-05-19T00:40:34.440305Z"},"links":{"citing_paper":"/paper/2508.04325"},"observation_digest":"sha256:61e637e2769a3fec8436619db38d2c4fed7af951b0addcd6d7e12a50b1bc6987","observation_id":"7322dbd1-2315-4f61-981c-143649b8df95","resolution":{"observed_at":"2026-05-09T07:21:27.315887Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-12T06:34:41.77262+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-12T06:34:41.77262+00:00","source":"crossref"},{"observed_at":"2026-08-12T06:34:36.333875+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-06-05T21:23:00.469572Z","title":"on platforms like GitHub or Hugging Face) along with the applica- ble license","venue":null,"work_id":"8ed3d643-3dc0-4da8-99d7-e5f81ceda745","year":null},"citing_paper":{"arxiv_id":"2508.04325","last_updated":"2026-04-29T05:46:18Z","snapshot_observed_at":"2026-08-02T07:50:38.917887Z","submitted_at":"2025-08-06T11:11:40Z","title":"Beyond the Leaderboard: Rethinking Medical Benchmarks for Large Language Models","version":2},"reference_index":53,"source":"pdf_text","source_observed_at":"2026-05-19T00:40:34.440305Z"},"links":{"citing_paper":"/paper/2508.04325"},"observation_digest":"sha256:e6a811f148eb7a477b7884e4ce12c4b1eb6dd780de32564406ea74dc4f3342e8","observation_id":"6d37a8ac-628e-4a4c-8b71-b39960abd0b8","resolution":{"observed_at":"2026-05-09T07:21:27.317975Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-12T06:34:41.77262+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-12T06:34:41.77262+00:00","source":"crossref"},{"observed_at":"2026-08-12T06:34:36.333875+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-06-05T21:23:00.469572Z","title":"• Justification: Proper usage and citation guidelines help maintain academic in- tegrity","venue":null,"work_id":"239ba75a-36e0-4fbd-b2cb-3b68b2162bac","year":null},"citing_paper":{"arxiv_id":"2508.04325","last_updated":"2026-04-29T05:46:18Z","snapshot_observed_at":"2026-08-02T07:50:38.917887Z","submitted_at":"2025-08-06T11:11:40Z","title":"Beyond the Leaderboard: Rethinking Medical Benchmarks for Large Language Models","version":2},"reference_index":54,"source":"pdf_text","source_observed_at":"2026-05-19T00:40:34.440305Z"},"links":{"citing_paper":"/paper/2508.04325"},"observation_digest":"sha256:0c6542677ae566831e906eb6f04508fbea49a84a4ba6696baed52935ebbb7409","observation_id":"b1a4caca-eddf-4e7d-9e5e-3b183afb2fb7","resolution":{"observed_at":"2026-05-09T07:21:27.319871Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-12T06:34:41.77262+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-12T06:34:41.77262+00:00","source":"crossref"},{"observed_at":"2026-08-12T06:34:36.333875+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-06-05T21:23:00.469572Z","title":"• Justification: Maintaining an effective feedback channel allows users to provide feedback when issues with the bench- mark are discovered","venue":null,"work_id":"80bd3ec4-ab9f-4eff-8a7f-9b0eba0bcf6a","year":null},"citing_paper":{"arxiv_id":"2508.04325","last_updated":"2026-04-29T05:46:18Z","snapshot_observed_at":"2026-08-02T07:50:38.917887Z","submitted_at":"2025-08-06T11:11:40Z","title":"Beyond the Leaderboard: Rethinking Medical Benchmarks for Large Language Models","version":2},"reference_index":55,"source":"pdf_text","source_observed_at":"2026-05-19T00:40:34.440305Z"},"links":{"citing_paper":"/paper/2508.04325"},"observation_digest":"sha256:e3a41925b175c1264d63eb66c62d3d5ae8d46783f4e345064b87d1094ecea78e","observation_id":"9e4bcdd5-65eb-4968-8231-a82ffecb67bf","resolution":{"observed_at":"2026-05-09T07:21:27.321986Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-12T06:34:41.77262+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-12T06:34:41.77262+00:00","source":"crossref"},{"observed_at":"2026-08-12T06:34:36.333875+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-06-05T21:23:00.469572Z","title":"• Justification: Maintaining an effective feedback channel allows users to provide feedback when issues with the bench- mark are discovered","venue":null,"work_id":"57938be1-b1e6-4406-80e7-d65a319c2bc7","year":null},"citing_paper":{"arxiv_id":"2508.04325","last_updated":"2026-04-29T05:46:18Z","snapshot_observed_at":"2026-08-02T07:50:38.917887Z","submitted_at":"2025-08-06T11:11:40Z","title":"Beyond the Leaderboard: Rethinking Medical Benchmarks for Large Language Models","version":2},"reference_index":56,"source":"pdf_text","source_observed_at":"2026-05-19T00:40:34.440305Z"},"links":{"citing_paper":"/paper/2508.04325"},"observation_digest":"sha256:0f2495cb509e6416c6936af647b185de44e8a2a55d103e4309938e56aeefcc5b","observation_id":"42ffb1e8-6d51-4cbe-beb4-5696a4053aa4","resolution":{"observed_at":"2026-05-09T07:21:27.324007Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-12T06:34:41.77262+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-12T06:34:41.77262+00:00","source":"crossref"},{"observed_at":"2026-08-12T06:34:36.333875+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-06-05T21:23:00.469572Z","title":"• Justification: Clarifying who holds long- term responsibility reassures the commu- nity that it will be actively supported and improved, ensuring usability and credi- bility","venue":null,"work_id":"1fa27104-af43-423e-a0bc-242412a687c6","year":null},"citing_paper":{"arxiv_id":"2508.04325","last_updated":"2026-04-29T05:46:18Z","snapshot_observed_at":"2026-08-02T07:50:38.917887Z","submitted_at":"2025-08-06T11:11:40Z","title":"Beyond the Leaderboard: Rethinking Medical Benchmarks for Large Language Models","version":2},"reference_index":57,"source":"pdf_text","source_observed_at":"2026-05-19T00:40:34.440305Z"},"links":{"citing_paper":"/paper/2508.04325"},"observation_digest":"sha256:6490216af6ad71ca0710ea91055f5eb92860ce103cca89d7185d66cd6a6c9dd1","observation_id":"c8c03501-839a-4f87-beea-c2fbe5820cc7","resolution":{"observed_at":"2026-05-09T07:21:27.328464Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-12T06:34:41.77262+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-12T06:34:41.77262+00:00","source":"crossref"},{"observed_at":"2026-08-12T06:34:36.333875+00:00","source":"retraction_watch"}],"state":"measured"}}],"paper":{"arxiv_id":"2508.04325","last_updated":"2026-04-29T05:46:18Z","latest_version":2,"primary_category":"cs.CL","snapshot_observed_at":"2026-08-02T07:50:38.917887Z","submitted_at":"2025-08-06T11:11:40Z","title":"Beyond the Leaderboard: Rethinking Medical Benchmarks for Large Language Models"},"reference_resolution":{"displayed":57,"state_counts":{"malformed_identifier":1,"metadata_mismatch":5,"parse_uncertain":0,"unresolved":6,"verified_exact":2,"verified_fuzzy":43},"total_outbound_references":57},"refusal":"A citation records a reference. It does not transfer a finding from one paper to another.","schema":"pith.paper-citation-record.v1","standing_sources":[{"observed_at":"2026-08-12T06:34:41.77262+00:00","source":"crossref"},{"observed_at":"2026-08-12T06:34:36.333875+00:00","source":"retraction_watch"}],"thesis":"As of 12 August 2026, this Paper Citation Record lists 57 of 57 outbound references and 9 inbound Pith citation observations for arXiv:2508.04325."}