{"bundle_type":"pith_open_graph_bundle","bundle_version":"1.0","pith_number":"pith:2025:4OEIIU5WIHJJ2P63PYZUKD73I5","short_pith_number":"pith:4OEIIU5W","canonical_record":{"source":{"id":"2507.15717","kind":"arxiv","version":1},"metadata":{"license":"http://creativecommons.org/licenses/by-nc-nd/4.0/","primary_cat":"cs.CL","submitted_at":"2025-07-21T15:27:32Z","cross_cats_sorted":["cs.AI"],"title_canon_sha256":"51e83ac8a6516654ed1ef1c0c5b36ad6bd2fd3374c96f2ca1c07a9ab7a2f3f4f","abstract_canon_sha256":"af18afab720bb66c99f21a7940d01912eb985aaf25735dbb8c087c11e415685f"},"schema_version":"1.0"},"canonical_sha256":"e3888453b641d29d3fdb7e33450ffb4768e5ea79575bd2170b1ec896d2735940","source":{"kind":"arxiv","id":"2507.15717","version":1},"source_aliases":[{"alias_kind":"arxiv","alias_value":"2507.15717","created_at":"2026-07-05T11:40:42Z"},{"alias_kind":"arxiv_version","alias_value":"2507.15717v1","created_at":"2026-07-05T11:40:42Z"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.2507.15717","created_at":"2026-07-05T11:40:42Z"},{"alias_kind":"pith_short_12","alias_value":"4OEIIU5WIHJJ","created_at":"2026-07-05T11:40:42Z"},{"alias_kind":"pith_short_16","alias_value":"4OEIIU5WIHJJ2P63","created_at":"2026-07-05T11:40:42Z"},{"alias_kind":"pith_short_8","alias_value":"4OEIIU5W","created_at":"2026-07-05T11:40:42Z"}],"events":[{"event_type":"record_created","subject_pith_number":"pith:2025:4OEIIU5WIHJJ2P63PYZUKD73I5","target":"record","payload":{"canonical_record":{"source":{"id":"2507.15717","kind":"arxiv","version":1},"metadata":{"license":"http://creativecommons.org/licenses/by-nc-nd/4.0/","primary_cat":"cs.CL","submitted_at":"2025-07-21T15:27:32Z","cross_cats_sorted":["cs.AI"],"title_canon_sha256":"51e83ac8a6516654ed1ef1c0c5b36ad6bd2fd3374c96f2ca1c07a9ab7a2f3f4f","abstract_canon_sha256":"af18afab720bb66c99f21a7940d01912eb985aaf25735dbb8c087c11e415685f"},"schema_version":"1.0"},"canonical_sha256":"e3888453b641d29d3fdb7e33450ffb4768e5ea79575bd2170b1ec896d2735940","receipt":{"kind":"pith_receipt","key_id":"pith-v1-2026-05","algorithm":"ed25519","signed_at":"2026-07-05T11:40:42.657124Z","signature_b64":"t4sKfT7s6ToDsPZBTEVorVRZzllQS9XWTpBhRvEi6a5mj7o/8lC2Mu91bkx/wW8laJupvT3sXSnb8zczFQ78DQ==","signed_message":"canonical_sha256_bytes","builder_version":"pith-number-builder-2026-05-17-v1","receipt_version":"0.3","canonical_sha256":"e3888453b641d29d3fdb7e33450ffb4768e5ea79575bd2170b1ec896d2735940","last_reissued_at":"2026-07-05T11:40:42.656718Z","signature_status":"signed_v1","first_computed_at":"2026-07-05T11:40:42.656718Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"source_kind":"arxiv","source_id":"2507.15717","source_version":1,"attestation_state":"computed"},"signer":{"signer_id":"pith.science","signer_type":"pith_registry","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"created_at":"2026-07-05T11:40:42Z","supersedes":[],"prev_event":null,"signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"t+7Jom8qarNxzHiOWQDMwDWqo56sTfcdzgqEroBagVspBO3LrRqcecoYYefdGbYaTiv2iV0nf1L/Kzhq3JRIBA==","signed_message":"open_graph_event_sha256_bytes","signed_at":"2026-08-03T14:32:51.606572Z"},"content_sha256":"9a7913fc00d8dbf1fb670a809bf3a794cac50bfea1ab4e5677f52a2e02a66299","schema_version":"1.0","event_id":"sha256:9a7913fc00d8dbf1fb670a809bf3a794cac50bfea1ab4e5677f52a2e02a66299"},{"event_type":"graph_snapshot","subject_pith_number":"pith:2025:4OEIIU5WIHJJ2P63PYZUKD73I5","target":"graph","payload":{"graph_snapshot":{"paper":{"title":"BEnchmarking LLMs for Ophthalmology (BELO) for Ophthalmological Knowledge and Reasoning","license":"http://creativecommons.org/licenses/by-nc-nd/4.0/","headline":"","cross_cats":["cs.AI"],"primary_cat":"cs.CL","authors_text":"Aidan Gilson, Amisha Dave, Chen-Hsin Sun, David Ziyou Chen, Dianbo Liu, Emily Yuelei Fu, Fares Antaki, Gabriel Dawei Yang, Hyunjae Kim, Jocelyn Goh, Jost Jonas, Kelvin Zhenghao Li, Ke Zou, Kristen Nwanyanwu, Krithi Pushpanathan, Lucian V Del Priore, Mark Chia, Maxwell Singer, Michelle Ongyong Hui, Mingjia Yang, Minjie Zou, Qingyu Chen, Ron Adelman, Sahana Srinivasan, Samantha Yew, Thaddaeus Wai Soon Lo, Wan Ting Loke, Wendy Meihua Wong, Xuguang Ai, Yibing Chen, Yih-Chung Tham, Yiming Kong","submitted_at":"2025-07-21T15:27:32Z","abstract_excerpt":"Current benchmarks evaluating large language models (LLMs) in ophthalmology are limited in scope and disproportionately prioritise accuracy. We introduce BELO (BEnchmarking LLMs for Ophthalmology), a standardized and comprehensive evaluation benchmark developed through multiple rounds of expert checking by 13 ophthalmologists. BELO assesses ophthalmology-related clinical accuracy and reasoning quality. Using keyword matching and a fine-tuned PubMedBERT model, we curated ophthalmology-specific multiple-choice-questions (MCQs) from diverse medical datasets (BCSC, MedMCQA, MedQA, BioASQ, and PubM"},"claims":{"count":0,"items":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"2507.15717","kind":"arxiv","version":1},"verdict":{"id":null,"model_set":{},"created_at":null,"strongest_claim":"","one_line_summary":"","pipeline_version":null,"weakest_assumption":"","pith_extraction_headline":""},"integrity":{"clean":true,"summary":{"advisory":0,"critical":0,"by_detector":{},"informational":0},"endpoint":"/pith/2507.15717/integrity.json","findings":[],"available":true,"detectors_run":[],"snapshot_sha256":"c28c3603d3b5d939e8dc4c7e95fa8dfce3d595e45f758748cecf8e644a296938"},"references":{"count":0,"sample":[],"resolved_work":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","internal_anchors":0},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"author_claims":{"count":0,"strong_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"builder_version":"pith-number-builder-2026-05-17-v1"},"verdict_id":null},"signer":{"signer_id":"pith.science","signer_type":"pith_registry","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"created_at":"2026-07-05T11:40:42Z","supersedes":[],"prev_event":null,"signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"s2GgbXdZk9FAkDKJhXkPW2vvvhd8zkn1uXtszfwJQud8BhjejKzHJjJfV2KR/HAp5T3tmT1O3lSii52BR/1HDA==","signed_message":"open_graph_event_sha256_bytes","signed_at":"2026-08-03T14:32:51.607088Z"},"content_sha256":"2b48008253bc0524368cb2f1391c1a87df5d6399a8cf0bcac6b2ee8727f089c8","schema_version":"1.0","event_id":"sha256:2b48008253bc0524368cb2f1391c1a87df5d6399a8cf0bcac6b2ee8727f089c8"}],"timestamp_proofs":[],"mirror_hints":[{"mirror_type":"https","name":"Pith Resolver","base_url":"https://pith.science","bundle_url":"https://pith.science/pith/4OEIIU5WIHJJ2P63PYZUKD73I5/bundle.json","state_url":"https://pith.science/pith/4OEIIU5WIHJJ2P63PYZUKD73I5/state.json","well_known_bundle_url":"https://pith.science/.well-known/pith/4OEIIU5WIHJJ2P63PYZUKD73I5/bundle.json","status":"primary"}],"public_keys":[{"key_id":"pith-v1-2026-05","algorithm":"ed25519","format":"raw","public_key_b64":"stVStoiQhXFxp4s2pdzPNoqVNBMojDU/fJ2db5S3CbM=","public_key_hex":"b2d552b68890857171a78b36a5dccf368a953413288c353f7c9d9d6f94b709b3","fingerprint_sha256_b32_first128bits":"RVFV5Z2OI2J3ZUO7ERDEBCYNKS","fingerprint_sha256_hex":"8d4b5ee74e4693bcd1df2446408b0d54","rotates_at":null,"url":"https://pith.science/pith-signing-key.json","notes":"Pith uses this Ed25519 key to sign canonical record SHA-256 digests. Verify with: ed25519_verify(public_key, message=canonical_sha256_bytes, signature=base64decode(signature_b64))."}],"merge_version":"pith-open-graph-merge-v1","built_at":"2026-08-03T14:32:51Z","links":{"resolver":"https://pith.science/pith/4OEIIU5WIHJJ2P63PYZUKD73I5","bundle":"https://pith.science/pith/4OEIIU5WIHJJ2P63PYZUKD73I5/bundle.json","state":"https://pith.science/pith/4OEIIU5WIHJJ2P63PYZUKD73I5/state.json","well_known_bundle":"https://pith.science/.well-known/pith/4OEIIU5WIHJJ2P63PYZUKD73I5/bundle.json"},"state":{"state_type":"pith_open_graph_state","state_version":"1.0","pith_number":"pith:2025:4OEIIU5WIHJJ2P63PYZUKD73I5","merge_version":"pith-open-graph-merge-v1","event_count":2,"valid_event_count":2,"invalid_event_count":0,"equivocation_count":0,"current":{"canonical_record":{"metadata":{"abstract_canon_sha256":"af18afab720bb66c99f21a7940d01912eb985aaf25735dbb8c087c11e415685f","cross_cats_sorted":["cs.AI"],"license":"http://creativecommons.org/licenses/by-nc-nd/4.0/","primary_cat":"cs.CL","submitted_at":"2025-07-21T15:27:32Z","title_canon_sha256":"51e83ac8a6516654ed1ef1c0c5b36ad6bd2fd3374c96f2ca1c07a9ab7a2f3f4f"},"schema_version":"1.0","source":{"id":"2507.15717","kind":"arxiv","version":1}},"source_aliases":[{"alias_kind":"arxiv","alias_value":"2507.15717","created_at":"2026-07-05T11:40:42Z"},{"alias_kind":"arxiv_version","alias_value":"2507.15717v1","created_at":"2026-07-05T11:40:42Z"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.2507.15717","created_at":"2026-07-05T11:40:42Z"},{"alias_kind":"pith_short_12","alias_value":"4OEIIU5WIHJJ","created_at":"2026-07-05T11:40:42Z"},{"alias_kind":"pith_short_16","alias_value":"4OEIIU5WIHJJ2P63","created_at":"2026-07-05T11:40:42Z"},{"alias_kind":"pith_short_8","alias_value":"4OEIIU5W","created_at":"2026-07-05T11:40:42Z"}],"graph_snapshots":[{"event_id":"sha256:2b48008253bc0524368cb2f1391c1a87df5d6399a8cf0bcac6b2ee8727f089c8","target":"graph","created_at":"2026-07-05T11:40:42Z","signer":{"key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signer_id":"pith.science","signer_type":"pith_registry"},"payload":{"graph_snapshot":{"author_claims":{"count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","strong_count":0},"builder_version":"pith-number-builder-2026-05-17-v1","claims":{"count":0,"items":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"integrity":{"available":true,"clean":true,"detectors_run":[],"endpoint":"/pith/2507.15717/integrity.json","findings":[],"snapshot_sha256":"c28c3603d3b5d939e8dc4c7e95fa8dfce3d595e45f758748cecf8e644a296938","summary":{"advisory":0,"by_detector":{},"critical":0,"informational":0}},"paper":{"abstract_excerpt":"Current benchmarks evaluating large language models (LLMs) in ophthalmology are limited in scope and disproportionately prioritise accuracy. We introduce BELO (BEnchmarking LLMs for Ophthalmology), a standardized and comprehensive evaluation benchmark developed through multiple rounds of expert checking by 13 ophthalmologists. BELO assesses ophthalmology-related clinical accuracy and reasoning quality. Using keyword matching and a fine-tuned PubMedBERT model, we curated ophthalmology-specific multiple-choice-questions (MCQs) from diverse medical datasets (BCSC, MedMCQA, MedQA, BioASQ, and PubM","authors_text":"Aidan Gilson, Amisha Dave, Chen-Hsin Sun, David Ziyou Chen, Dianbo Liu, Emily Yuelei Fu, Fares Antaki, Gabriel Dawei Yang, Hyunjae Kim, Jocelyn Goh, Jost Jonas, Kelvin Zhenghao Li, Ke Zou, Kristen Nwanyanwu, Krithi Pushpanathan, Lucian V Del Priore, Mark Chia, Maxwell Singer, Michelle Ongyong Hui, Mingjia Yang, Minjie Zou, Qingyu Chen, Ron Adelman, Sahana Srinivasan, Samantha Yew, Thaddaeus Wai Soon Lo, Wan Ting Loke, Wendy Meihua Wong, Xuguang Ai, Yibing Chen, Yih-Chung Tham, Yiming Kong","cross_cats":["cs.AI"],"headline":"","license":"http://creativecommons.org/licenses/by-nc-nd/4.0/","primary_cat":"cs.CL","submitted_at":"2025-07-21T15:27:32Z","title":"BEnchmarking LLMs for Ophthalmology (BELO) for Ophthalmological Knowledge and Reasoning"},"references":{"count":0,"internal_anchors":0,"resolved_work":0,"sample":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"2507.15717","kind":"arxiv","version":1},"verdict":{"created_at":null,"id":null,"model_set":{},"one_line_summary":"","pipeline_version":null,"pith_extraction_headline":"","strongest_claim":"","weakest_assumption":""}},"verdict_id":null}}],"author_attestations":[],"timestamp_anchors":[],"storage_attestations":[],"citation_signatures":[],"replication_records":[],"corrections":[],"mirror_hints":[],"record_created":{"event_id":"sha256:9a7913fc00d8dbf1fb670a809bf3a794cac50bfea1ab4e5677f52a2e02a66299","target":"record","created_at":"2026-07-05T11:40:42Z","signer":{"key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signer_id":"pith.science","signer_type":"pith_registry"},"payload":{"attestation_state":"computed","canonical_record":{"metadata":{"abstract_canon_sha256":"af18afab720bb66c99f21a7940d01912eb985aaf25735dbb8c087c11e415685f","cross_cats_sorted":["cs.AI"],"license":"http://creativecommons.org/licenses/by-nc-nd/4.0/","primary_cat":"cs.CL","submitted_at":"2025-07-21T15:27:32Z","title_canon_sha256":"51e83ac8a6516654ed1ef1c0c5b36ad6bd2fd3374c96f2ca1c07a9ab7a2f3f4f"},"schema_version":"1.0","source":{"id":"2507.15717","kind":"arxiv","version":1}},"canonical_sha256":"e3888453b641d29d3fdb7e33450ffb4768e5ea79575bd2170b1ec896d2735940","receipt":{"algorithm":"ed25519","builder_version":"pith-number-builder-2026-05-17-v1","canonical_sha256":"e3888453b641d29d3fdb7e33450ffb4768e5ea79575bd2170b1ec896d2735940","first_computed_at":"2026-07-05T11:40:42.656718Z","key_id":"pith-v1-2026-05","kind":"pith_receipt","last_reissued_at":"2026-07-05T11:40:42.656718Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","receipt_version":"0.3","signature_b64":"t4sKfT7s6ToDsPZBTEVorVRZzllQS9XWTpBhRvEi6a5mj7o/8lC2Mu91bkx/wW8laJupvT3sXSnb8zczFQ78DQ==","signature_status":"signed_v1","signed_at":"2026-07-05T11:40:42.657124Z","signed_message":"canonical_sha256_bytes"},"source_id":"2507.15717","source_kind":"arxiv","source_version":1}}},"equivocations":[],"invalid_events":[],"applied_event_ids":["sha256:9a7913fc00d8dbf1fb670a809bf3a794cac50bfea1ab4e5677f52a2e02a66299","sha256:2b48008253bc0524368cb2f1391c1a87df5d6399a8cf0bcac6b2ee8727f089c8"],"state_sha256":"1120418f80020eb0069377bbb538005592c1b66cc0e065faa415c19863ca5b9b"},"bundle_signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"L/B3JH0eCmru0vFx3aPhgG4NlnDHKzQ7HX8rI8HUPVB8g1Ajg5zc5FmyY4eJZaD7Z7fQ44qjsknmGWxGwiNaBA==","signed_message":"bundle_sha256_bytes","signed_at":"2026-08-03T14:32:51.611976Z","bundle_sha256":"e29936832852b7cdc9656e87e56d666a9102cb8cef3494f91fea2de121990936"}}