{"bundle_type":"pith_open_graph_bundle","bundle_version":"1.0","pith_number":"pith:2023:R6ATVCKHPEQ3Y5QHVGCHKFTBKP","short_pith_number":"pith:R6ATVCKH","canonical_record":{"source":{"id":"2307.02191","kind":"arxiv","version":2},"metadata":{"license":"http://creativecommons.org/licenses/by-nc-nd/4.0/","primary_cat":"cs.LG","submitted_at":"2023-07-05T10:33:45Z","cross_cats_sorted":["cs.CV","stat.ME","stat.ML"],"title_canon_sha256":"8aabe3d51e6ca7c28955c5e92755f7835d4d910e80ff5c73bad4fbffa3be1b7f","abstract_canon_sha256":"9838f8d6f19edbcf9cc5d4b43f688bad34cc5e662279b7c65393d32143ee3c12"},"schema_version":"1.0"},"canonical_sha256":"8f813a89477921bc7607a98475166153fb167b9a6349e4ada5dd2b16157b72ce","source":{"kind":"arxiv","id":"2307.02191","version":2},"source_aliases":[{"alias_kind":"arxiv","alias_value":"2307.02191","created_at":"2026-07-05T10:48:12Z"},{"alias_kind":"arxiv_version","alias_value":"2307.02191v2","created_at":"2026-07-05T10:48:12Z"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.2307.02191","created_at":"2026-07-05T10:48:12Z"},{"alias_kind":"pith_short_12","alias_value":"R6ATVCKHPEQ3","created_at":"2026-07-05T10:48:12Z"},{"alias_kind":"pith_short_16","alias_value":"R6ATVCKHPEQ3Y5QH","created_at":"2026-07-05T10:48:12Z"},{"alias_kind":"pith_short_8","alias_value":"R6ATVCKH","created_at":"2026-07-05T10:48:12Z"}],"events":[{"event_type":"record_created","subject_pith_number":"pith:2023:R6ATVCKHPEQ3Y5QHVGCHKFTBKP","target":"record","payload":{"canonical_record":{"source":{"id":"2307.02191","kind":"arxiv","version":2},"metadata":{"license":"http://creativecommons.org/licenses/by-nc-nd/4.0/","primary_cat":"cs.LG","submitted_at":"2023-07-05T10:33:45Z","cross_cats_sorted":["cs.CV","stat.ME","stat.ML"],"title_canon_sha256":"8aabe3d51e6ca7c28955c5e92755f7835d4d910e80ff5c73bad4fbffa3be1b7f","abstract_canon_sha256":"9838f8d6f19edbcf9cc5d4b43f688bad34cc5e662279b7c65393d32143ee3c12"},"schema_version":"1.0"},"canonical_sha256":"8f813a89477921bc7607a98475166153fb167b9a6349e4ada5dd2b16157b72ce","receipt":{"kind":"pith_receipt","key_id":"pith-v1-2026-05","algorithm":"ed25519","signed_at":"2026-07-05T10:48:12.515143Z","signature_b64":"1pD5nvCxRcWzcqSDC2nJoimD4aOA4Iz3v53q0d3A9yezFugBtNwPqpEGx7e9w1SRIyiUMwaTlhFCzPnH3jDkAg==","signed_message":"canonical_sha256_bytes","builder_version":"pith-number-builder-2026-05-17-v1","receipt_version":"0.3","canonical_sha256":"8f813a89477921bc7607a98475166153fb167b9a6349e4ada5dd2b16157b72ce","last_reissued_at":"2026-07-05T10:48:12.514646Z","signature_status":"signed_v1","first_computed_at":"2026-07-05T10:48:12.514646Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"source_kind":"arxiv","source_id":"2307.02191","source_version":2,"attestation_state":"computed"},"signer":{"signer_id":"pith.science","signer_type":"pith_registry","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"created_at":"2026-07-05T10:48:12Z","supersedes":[],"prev_event":null,"signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"vTA/mUrPN8LDv5VCV/b8pQ+HMa4Y7aFf+s1Q9spyMAxiG13/y2p0wYZAnSvXw7yRG+SBpHGdlIzQm9nkkhI/Dw==","signed_message":"open_graph_event_sha256_bytes","signed_at":"2026-08-24T00:17:07.207747Z"},"content_sha256":"7caa3ab9cda5fe53b2f0309575edd20f541c9a1cef9f2be8ba60fb98de32b19c","schema_version":"1.0","event_id":"sha256:7caa3ab9cda5fe53b2f0309575edd20f541c9a1cef9f2be8ba60fb98de32b19c"},{"event_type":"graph_snapshot","subject_pith_number":"pith:2023:R6ATVCKHPEQ3Y5QHVGCHKFTBKP","target":"graph","payload":{"graph_snapshot":{"paper":{"title":"Evaluating AI systems under uncertain ground truth: a case study in dermatology","license":"http://creativecommons.org/licenses/by-nc-nd/4.0/","headline":"","cross_cats":["cs.CV","stat.ME","stat.ML"],"primary_cat":"cs.LG","authors_text":"Abhijit Guha Roy, Alan Karthikesalingam, Ali Taylan Cemgil, Arnaud Doucet, Beverly Freeman, Dale R. Webster, David Stutz, Greg S. Corrado, Jan Freyberg, Javier Perez Matos, Melih Barsbey, Mike Schaekermann, Patricia Strachan, Pushmeet Kohli, Rajeev Rikhye, Tatiana Matejovicova, Umesh Telang, Yossi Matias, Yuan Liu, Yun Liu","submitted_at":"2023-07-05T10:33:45Z","abstract_excerpt":"For safety, medical AI systems undergo thorough evaluations before deployment, validating their predictions against a ground truth which is assumed to be fixed and certain. However, this ground truth is often curated in the form of differential diagnoses. While a single differential diagnosis reflects the uncertainty in one expert assessment, multiple experts introduce another layer of uncertainty through disagreement. Both forms of uncertainty are ignored in standard evaluation which aggregates these differential diagnoses to a single label. In this paper, we show that ignoring uncertainty le"},"claims":{"count":0,"items":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"2307.02191","kind":"arxiv","version":2},"verdict":{"id":null,"model_set":{},"created_at":null,"strongest_claim":"","one_line_summary":"","pipeline_version":null,"weakest_assumption":"","pith_extraction_headline":""},"integrity":{"clean":true,"summary":{"advisory":0,"critical":0,"by_detector":{},"informational":0},"endpoint":"/pith/2307.02191/integrity.json","findings":[],"available":true,"detectors_run":[],"snapshot_sha256":"c28c3603d3b5d939e8dc4c7e95fa8dfce3d595e45f758748cecf8e644a296938"},"references":{"count":0,"sample":[],"resolved_work":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","internal_anchors":0},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"author_claims":{"count":0,"strong_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"builder_version":"pith-number-builder-2026-05-17-v1"},"verdict_id":null},"signer":{"signer_id":"pith.science","signer_type":"pith_registry","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"created_at":"2026-07-05T10:48:12Z","supersedes":[],"prev_event":null,"signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"0FSGtCprXeeDC+YdN2DbvPmwIXOoC6wMYRI2XF4m+arkJEcgOUp6+zmnCP6zPYCTUQrvubsvUYFa4/J3+yqVDw==","signed_message":"open_graph_event_sha256_bytes","signed_at":"2026-08-24T00:17:07.208265Z"},"content_sha256":"8b9a83fe0314eea881e6f9b3a9458f28e0aad0d1b42537c5aa85b3e00bf8ebf8","schema_version":"1.0","event_id":"sha256:8b9a83fe0314eea881e6f9b3a9458f28e0aad0d1b42537c5aa85b3e00bf8ebf8"}],"timestamp_proofs":[],"mirror_hints":[{"mirror_type":"https","name":"Pith Resolver","base_url":"https://pith.science","bundle_url":"https://pith.science/pith/R6ATVCKHPEQ3Y5QHVGCHKFTBKP/bundle.json","state_url":"https://pith.science/pith/R6ATVCKHPEQ3Y5QHVGCHKFTBKP/state.json","well_known_bundle_url":"https://pith.science/.well-known/pith/R6ATVCKHPEQ3Y5QHVGCHKFTBKP/bundle.json","status":"primary"}],"public_keys":[{"key_id":"pith-v1-2026-05","algorithm":"ed25519","format":"raw","public_key_b64":"stVStoiQhXFxp4s2pdzPNoqVNBMojDU/fJ2db5S3CbM=","public_key_hex":"b2d552b68890857171a78b36a5dccf368a953413288c353f7c9d9d6f94b709b3","fingerprint_sha256_b32_first128bits":"RVFV5Z2OI2J3ZUO7ERDEBCYNKS","fingerprint_sha256_hex":"8d4b5ee74e4693bcd1df2446408b0d54","rotates_at":null,"url":"https://pith.science/pith-signing-key.json","notes":"Pith uses this Ed25519 key to sign canonical record SHA-256 digests. Verify with: ed25519_verify(public_key, message=canonical_sha256_bytes, signature=base64decode(signature_b64))."}],"merge_version":"pith-open-graph-merge-v1","built_at":"2026-08-24T00:17:07Z","links":{"resolver":"https://pith.science/pith/R6ATVCKHPEQ3Y5QHVGCHKFTBKP","bundle":"https://pith.science/pith/R6ATVCKHPEQ3Y5QHVGCHKFTBKP/bundle.json","state":"https://pith.science/pith/R6ATVCKHPEQ3Y5QHVGCHKFTBKP/state.json","well_known_bundle":"https://pith.science/.well-known/pith/R6ATVCKHPEQ3Y5QHVGCHKFTBKP/bundle.json"},"state":{"state_type":"pith_open_graph_state","state_version":"1.0","pith_number":"pith:2023:R6ATVCKHPEQ3Y5QHVGCHKFTBKP","merge_version":"pith-open-graph-merge-v1","event_count":2,"valid_event_count":2,"invalid_event_count":0,"equivocation_count":0,"current":{"canonical_record":{"metadata":{"abstract_canon_sha256":"9838f8d6f19edbcf9cc5d4b43f688bad34cc5e662279b7c65393d32143ee3c12","cross_cats_sorted":["cs.CV","stat.ME","stat.ML"],"license":"http://creativecommons.org/licenses/by-nc-nd/4.0/","primary_cat":"cs.LG","submitted_at":"2023-07-05T10:33:45Z","title_canon_sha256":"8aabe3d51e6ca7c28955c5e92755f7835d4d910e80ff5c73bad4fbffa3be1b7f"},"schema_version":"1.0","source":{"id":"2307.02191","kind":"arxiv","version":2}},"source_aliases":[{"alias_kind":"arxiv","alias_value":"2307.02191","created_at":"2026-07-05T10:48:12Z"},{"alias_kind":"arxiv_version","alias_value":"2307.02191v2","created_at":"2026-07-05T10:48:12Z"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.2307.02191","created_at":"2026-07-05T10:48:12Z"},{"alias_kind":"pith_short_12","alias_value":"R6ATVCKHPEQ3","created_at":"2026-07-05T10:48:12Z"},{"alias_kind":"pith_short_16","alias_value":"R6ATVCKHPEQ3Y5QH","created_at":"2026-07-05T10:48:12Z"},{"alias_kind":"pith_short_8","alias_value":"R6ATVCKH","created_at":"2026-07-05T10:48:12Z"}],"graph_snapshots":[{"event_id":"sha256:8b9a83fe0314eea881e6f9b3a9458f28e0aad0d1b42537c5aa85b3e00bf8ebf8","target":"graph","created_at":"2026-07-05T10:48:12Z","signer":{"key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signer_id":"pith.science","signer_type":"pith_registry"},"payload":{"graph_snapshot":{"author_claims":{"count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","strong_count":0},"builder_version":"pith-number-builder-2026-05-17-v1","claims":{"count":0,"items":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"integrity":{"available":true,"clean":true,"detectors_run":[],"endpoint":"/pith/2307.02191/integrity.json","findings":[],"snapshot_sha256":"c28c3603d3b5d939e8dc4c7e95fa8dfce3d595e45f758748cecf8e644a296938","summary":{"advisory":0,"by_detector":{},"critical":0,"informational":0}},"paper":{"abstract_excerpt":"For safety, medical AI systems undergo thorough evaluations before deployment, validating their predictions against a ground truth which is assumed to be fixed and certain. However, this ground truth is often curated in the form of differential diagnoses. While a single differential diagnosis reflects the uncertainty in one expert assessment, multiple experts introduce another layer of uncertainty through disagreement. Both forms of uncertainty are ignored in standard evaluation which aggregates these differential diagnoses to a single label. In this paper, we show that ignoring uncertainty le","authors_text":"Abhijit Guha Roy, Alan Karthikesalingam, Ali Taylan Cemgil, Arnaud Doucet, Beverly Freeman, Dale R. Webster, David Stutz, Greg S. Corrado, Jan Freyberg, Javier Perez Matos, Melih Barsbey, Mike Schaekermann, Patricia Strachan, Pushmeet Kohli, Rajeev Rikhye, Tatiana Matejovicova, Umesh Telang, Yossi Matias, Yuan Liu, Yun Liu","cross_cats":["cs.CV","stat.ME","stat.ML"],"headline":"","license":"http://creativecommons.org/licenses/by-nc-nd/4.0/","primary_cat":"cs.LG","submitted_at":"2023-07-05T10:33:45Z","title":"Evaluating AI systems under uncertain ground truth: a case study in dermatology"},"references":{"count":0,"internal_anchors":0,"resolved_work":0,"sample":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"2307.02191","kind":"arxiv","version":2},"verdict":{"created_at":null,"id":null,"model_set":{},"one_line_summary":"","pipeline_version":null,"pith_extraction_headline":"","strongest_claim":"","weakest_assumption":""}},"verdict_id":null}}],"author_attestations":[],"timestamp_anchors":[],"storage_attestations":[],"citation_signatures":[],"replication_records":[],"corrections":[],"mirror_hints":[],"record_created":{"event_id":"sha256:7caa3ab9cda5fe53b2f0309575edd20f541c9a1cef9f2be8ba60fb98de32b19c","target":"record","created_at":"2026-07-05T10:48:12Z","signer":{"key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signer_id":"pith.science","signer_type":"pith_registry"},"payload":{"attestation_state":"computed","canonical_record":{"metadata":{"abstract_canon_sha256":"9838f8d6f19edbcf9cc5d4b43f688bad34cc5e662279b7c65393d32143ee3c12","cross_cats_sorted":["cs.CV","stat.ME","stat.ML"],"license":"http://creativecommons.org/licenses/by-nc-nd/4.0/","primary_cat":"cs.LG","submitted_at":"2023-07-05T10:33:45Z","title_canon_sha256":"8aabe3d51e6ca7c28955c5e92755f7835d4d910e80ff5c73bad4fbffa3be1b7f"},"schema_version":"1.0","source":{"id":"2307.02191","kind":"arxiv","version":2}},"canonical_sha256":"8f813a89477921bc7607a98475166153fb167b9a6349e4ada5dd2b16157b72ce","receipt":{"algorithm":"ed25519","builder_version":"pith-number-builder-2026-05-17-v1","canonical_sha256":"8f813a89477921bc7607a98475166153fb167b9a6349e4ada5dd2b16157b72ce","first_computed_at":"2026-07-05T10:48:12.514646Z","key_id":"pith-v1-2026-05","kind":"pith_receipt","last_reissued_at":"2026-07-05T10:48:12.514646Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","receipt_version":"0.3","signature_b64":"1pD5nvCxRcWzcqSDC2nJoimD4aOA4Iz3v53q0d3A9yezFugBtNwPqpEGx7e9w1SRIyiUMwaTlhFCzPnH3jDkAg==","signature_status":"signed_v1","signed_at":"2026-07-05T10:48:12.515143Z","signed_message":"canonical_sha256_bytes"},"source_id":"2307.02191","source_kind":"arxiv","source_version":2}}},"equivocations":[],"invalid_events":[],"applied_event_ids":["sha256:7caa3ab9cda5fe53b2f0309575edd20f541c9a1cef9f2be8ba60fb98de32b19c","sha256:8b9a83fe0314eea881e6f9b3a9458f28e0aad0d1b42537c5aa85b3e00bf8ebf8"],"state_sha256":"48babe6e6c27c6c04f8a56dab3825ccc461b34701d16c249486e7739ae6af59d"},"bundle_signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"ruEPgGjdzZN7tbqDBy535t+vPgHFRXEm59CFG8HEygDCkJZ4i2oPAPzwV+hMCq+PsTmXpXB64IqMwPSl6+unCw==","signed_message":"bundle_sha256_bytes","signed_at":"2026-08-24T00:17:07.213282Z","bundle_sha256":"2d04a59ee72583bf0171c47ca0af3537bfb4cd909f72cee6d960c6be1ea33eef"}}