{"bundle_type":"pith_open_graph_bundle","bundle_version":"1.0","pith_number":"pith:2023:W6XX3WIZCKD3FV6347P7OEZDL5","short_pith_number":"pith:W6XX3WIZ","canonical_record":{"source":{"id":"2312.03733","kind":"arxiv","version":2},"metadata":{"license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.CL","submitted_at":"2023-11-28T05:44:06Z","cross_cats_sorted":["cs.AI"],"title_canon_sha256":"76d46c63ca8945d8d8e22b5707d3dd1b89c0608a31ffe1efe3db8bafc0b609d7","abstract_canon_sha256":"61dffad41c66ff1893c77a708dd744c6e83dfde0a68a3c93a42cb63d8eacccae"},"schema_version":"1.0"},"canonical_sha256":"b7af7dd9191287b2d7dbe7dff713235f7da92e4db0dff61cef411db6800b5733","source":{"kind":"arxiv","id":"2312.03733","version":2},"source_aliases":[{"alias_kind":"arxiv","alias_value":"2312.03733","created_at":"2026-07-05T07:21:47Z"},{"alias_kind":"arxiv_version","alias_value":"2312.03733v2","created_at":"2026-07-05T07:21:47Z"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.2312.03733","created_at":"2026-07-05T07:21:47Z"},{"alias_kind":"pith_short_12","alias_value":"W6XX3WIZCKD3","created_at":"2026-07-05T07:21:47Z"},{"alias_kind":"pith_short_16","alias_value":"W6XX3WIZCKD3FV63","created_at":"2026-07-05T07:21:47Z"},{"alias_kind":"pith_short_8","alias_value":"W6XX3WIZ","created_at":"2026-07-05T07:21:47Z"}],"events":[{"event_type":"record_created","subject_pith_number":"pith:2023:W6XX3WIZCKD3FV6347P7OEZDL5","target":"record","payload":{"canonical_record":{"source":{"id":"2312.03733","kind":"arxiv","version":2},"metadata":{"license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.CL","submitted_at":"2023-11-28T05:44:06Z","cross_cats_sorted":["cs.AI"],"title_canon_sha256":"76d46c63ca8945d8d8e22b5707d3dd1b89c0608a31ffe1efe3db8bafc0b609d7","abstract_canon_sha256":"61dffad41c66ff1893c77a708dd744c6e83dfde0a68a3c93a42cb63d8eacccae"},"schema_version":"1.0"},"canonical_sha256":"b7af7dd9191287b2d7dbe7dff713235f7da92e4db0dff61cef411db6800b5733","receipt":{"kind":"pith_receipt","key_id":"pith-v1-2026-05","algorithm":"ed25519","signed_at":"2026-07-05T07:21:47.372540Z","signature_b64":"H1tVAFb4h88mK8hXmDtDMO17GUQSaYoSQ4kOPlFvMuHjnIg7JEqkXChU92T22q5s/4/Xv4/YmQaF+L9yxZdqAA==","signed_message":"canonical_sha256_bytes","builder_version":"pith-number-builder-2026-05-17-v1","receipt_version":"0.3","canonical_sha256":"b7af7dd9191287b2d7dbe7dff713235f7da92e4db0dff61cef411db6800b5733","last_reissued_at":"2026-07-05T07:21:47.371969Z","signature_status":"signed_v1","first_computed_at":"2026-07-05T07:21:47.371969Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"source_kind":"arxiv","source_id":"2312.03733","source_version":2,"attestation_state":"computed"},"signer":{"signer_id":"pith.science","signer_type":"pith_registry","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"created_at":"2026-07-05T07:21:47Z","supersedes":[],"prev_event":null,"signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"Su2tCsgQ5YhnTLS8JkvfpKa/Z9L31Rfg7XQhFaS/NXwEV1lLGGAjHRtFm6v8bBfMmykiKyH5+rzsL+htASoTDQ==","signed_message":"open_graph_event_sha256_bytes","signed_at":"2026-08-16T12:48:21.131794Z"},"content_sha256":"1921bb225e345c3316057ae9def8f50071da431892249628379e7e7076916d0b","schema_version":"1.0","event_id":"sha256:1921bb225e345c3316057ae9def8f50071da431892249628379e7e7076916d0b"},{"event_type":"graph_snapshot","subject_pith_number":"pith:2023:W6XX3WIZCKD3FV6347P7OEZDL5","target":"graph","payload":{"graph_snapshot":{"paper":{"title":"Methods to Estimate Large Language Model Confidence","license":"http://creativecommons.org/licenses/by/4.0/","headline":"","cross_cats":["cs.AI"],"primary_cat":"cs.CL","authors_text":"Ashwin Nayak, Maia Kotelanski, Robert Gallo, Thomas Savage","submitted_at":"2023-11-28T05:44:06Z","abstract_excerpt":"Large Language Models have difficulty communicating uncertainty, which is a significant obstacle to applying LLMs to complex medical tasks. This study evaluates methods to measure LLM confidence when suggesting a diagnosis for challenging clinical vignettes. GPT4 was asked a series of challenging case questions using Chain of Thought and Self Consistency prompting. Multiple methods were investigated to assess model confidence and evaluated on their ability to predict the models observed accuracy. The methods evaluated were Intrinsic Confidence, SC Agreement Frequency and CoT Response Length. S"},"claims":{"count":0,"items":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"2312.03733","kind":"arxiv","version":2},"verdict":{"id":null,"model_set":{},"created_at":null,"strongest_claim":"","one_line_summary":"","pipeline_version":null,"weakest_assumption":"","pith_extraction_headline":""},"integrity":{"clean":true,"summary":{"advisory":0,"critical":0,"by_detector":{},"informational":0},"endpoint":"/pith/2312.03733/integrity.json","findings":[],"available":true,"detectors_run":[],"snapshot_sha256":"c28c3603d3b5d939e8dc4c7e95fa8dfce3d595e45f758748cecf8e644a296938"},"references":{"count":0,"sample":[],"resolved_work":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","internal_anchors":0},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"author_claims":{"count":0,"strong_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"builder_version":"pith-number-builder-2026-05-17-v1"},"verdict_id":null},"signer":{"signer_id":"pith.science","signer_type":"pith_registry","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"created_at":"2026-07-05T07:21:47Z","supersedes":[],"prev_event":null,"signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"p0o5a3VI6R0DAZThT8Oe6/gGu8EF6FwgUbdC3xYoVds4dS82i7mUsRv32/9H5aghQu7E1CkR//yAsEDN8OaLDQ==","signed_message":"open_graph_event_sha256_bytes","signed_at":"2026-08-16T12:48:21.132708Z"},"content_sha256":"4dd35f34078017a3cfee08038d9dc26a6bc060420a5d24aaf08c2903dee53ba3","schema_version":"1.0","event_id":"sha256:4dd35f34078017a3cfee08038d9dc26a6bc060420a5d24aaf08c2903dee53ba3"}],"timestamp_proofs":[],"mirror_hints":[{"mirror_type":"https","name":"Pith Resolver","base_url":"https://pith.science","bundle_url":"https://pith.science/pith/W6XX3WIZCKD3FV6347P7OEZDL5/bundle.json","state_url":"https://pith.science/pith/W6XX3WIZCKD3FV6347P7OEZDL5/state.json","well_known_bundle_url":"https://pith.science/.well-known/pith/W6XX3WIZCKD3FV6347P7OEZDL5/bundle.json","status":"primary"}],"public_keys":[{"key_id":"pith-v1-2026-05","algorithm":"ed25519","format":"raw","public_key_b64":"stVStoiQhXFxp4s2pdzPNoqVNBMojDU/fJ2db5S3CbM=","public_key_hex":"b2d552b68890857171a78b36a5dccf368a953413288c353f7c9d9d6f94b709b3","fingerprint_sha256_b32_first128bits":"RVFV5Z2OI2J3ZUO7ERDEBCYNKS","fingerprint_sha256_hex":"8d4b5ee74e4693bcd1df2446408b0d54","rotates_at":null,"url":"https://pith.science/pith-signing-key.json","notes":"Pith uses this Ed25519 key to sign canonical record SHA-256 digests. Verify with: ed25519_verify(public_key, message=canonical_sha256_bytes, signature=base64decode(signature_b64))."}],"merge_version":"pith-open-graph-merge-v1","built_at":"2026-08-16T12:48:21Z","links":{"resolver":"https://pith.science/pith/W6XX3WIZCKD3FV6347P7OEZDL5","bundle":"https://pith.science/pith/W6XX3WIZCKD3FV6347P7OEZDL5/bundle.json","state":"https://pith.science/pith/W6XX3WIZCKD3FV6347P7OEZDL5/state.json","well_known_bundle":"https://pith.science/.well-known/pith/W6XX3WIZCKD3FV6347P7OEZDL5/bundle.json"},"state":{"state_type":"pith_open_graph_state","state_version":"1.0","pith_number":"pith:2023:W6XX3WIZCKD3FV6347P7OEZDL5","merge_version":"pith-open-graph-merge-v1","event_count":2,"valid_event_count":2,"invalid_event_count":0,"equivocation_count":0,"current":{"canonical_record":{"metadata":{"abstract_canon_sha256":"61dffad41c66ff1893c77a708dd744c6e83dfde0a68a3c93a42cb63d8eacccae","cross_cats_sorted":["cs.AI"],"license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.CL","submitted_at":"2023-11-28T05:44:06Z","title_canon_sha256":"76d46c63ca8945d8d8e22b5707d3dd1b89c0608a31ffe1efe3db8bafc0b609d7"},"schema_version":"1.0","source":{"id":"2312.03733","kind":"arxiv","version":2}},"source_aliases":[{"alias_kind":"arxiv","alias_value":"2312.03733","created_at":"2026-07-05T07:21:47Z"},{"alias_kind":"arxiv_version","alias_value":"2312.03733v2","created_at":"2026-07-05T07:21:47Z"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.2312.03733","created_at":"2026-07-05T07:21:47Z"},{"alias_kind":"pith_short_12","alias_value":"W6XX3WIZCKD3","created_at":"2026-07-05T07:21:47Z"},{"alias_kind":"pith_short_16","alias_value":"W6XX3WIZCKD3FV63","created_at":"2026-07-05T07:21:47Z"},{"alias_kind":"pith_short_8","alias_value":"W6XX3WIZ","created_at":"2026-07-05T07:21:47Z"}],"graph_snapshots":[{"event_id":"sha256:4dd35f34078017a3cfee08038d9dc26a6bc060420a5d24aaf08c2903dee53ba3","target":"graph","created_at":"2026-07-05T07:21:47Z","signer":{"key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signer_id":"pith.science","signer_type":"pith_registry"},"payload":{"graph_snapshot":{"author_claims":{"count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","strong_count":0},"builder_version":"pith-number-builder-2026-05-17-v1","claims":{"count":0,"items":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"integrity":{"available":true,"clean":true,"detectors_run":[],"endpoint":"/pith/2312.03733/integrity.json","findings":[],"snapshot_sha256":"c28c3603d3b5d939e8dc4c7e95fa8dfce3d595e45f758748cecf8e644a296938","summary":{"advisory":0,"by_detector":{},"critical":0,"informational":0}},"paper":{"abstract_excerpt":"Large Language Models have difficulty communicating uncertainty, which is a significant obstacle to applying LLMs to complex medical tasks. This study evaluates methods to measure LLM confidence when suggesting a diagnosis for challenging clinical vignettes. GPT4 was asked a series of challenging case questions using Chain of Thought and Self Consistency prompting. Multiple methods were investigated to assess model confidence and evaluated on their ability to predict the models observed accuracy. The methods evaluated were Intrinsic Confidence, SC Agreement Frequency and CoT Response Length. S","authors_text":"Ashwin Nayak, Maia Kotelanski, Robert Gallo, Thomas Savage","cross_cats":["cs.AI"],"headline":"","license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.CL","submitted_at":"2023-11-28T05:44:06Z","title":"Methods to Estimate Large Language Model Confidence"},"references":{"count":0,"internal_anchors":0,"resolved_work":0,"sample":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"2312.03733","kind":"arxiv","version":2},"verdict":{"created_at":null,"id":null,"model_set":{},"one_line_summary":"","pipeline_version":null,"pith_extraction_headline":"","strongest_claim":"","weakest_assumption":""}},"verdict_id":null}}],"author_attestations":[],"timestamp_anchors":[],"storage_attestations":[],"citation_signatures":[],"replication_records":[],"corrections":[],"mirror_hints":[],"record_created":{"event_id":"sha256:1921bb225e345c3316057ae9def8f50071da431892249628379e7e7076916d0b","target":"record","created_at":"2026-07-05T07:21:47Z","signer":{"key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signer_id":"pith.science","signer_type":"pith_registry"},"payload":{"attestation_state":"computed","canonical_record":{"metadata":{"abstract_canon_sha256":"61dffad41c66ff1893c77a708dd744c6e83dfde0a68a3c93a42cb63d8eacccae","cross_cats_sorted":["cs.AI"],"license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.CL","submitted_at":"2023-11-28T05:44:06Z","title_canon_sha256":"76d46c63ca8945d8d8e22b5707d3dd1b89c0608a31ffe1efe3db8bafc0b609d7"},"schema_version":"1.0","source":{"id":"2312.03733","kind":"arxiv","version":2}},"canonical_sha256":"b7af7dd9191287b2d7dbe7dff713235f7da92e4db0dff61cef411db6800b5733","receipt":{"algorithm":"ed25519","builder_version":"pith-number-builder-2026-05-17-v1","canonical_sha256":"b7af7dd9191287b2d7dbe7dff713235f7da92e4db0dff61cef411db6800b5733","first_computed_at":"2026-07-05T07:21:47.371969Z","key_id":"pith-v1-2026-05","kind":"pith_receipt","last_reissued_at":"2026-07-05T07:21:47.371969Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","receipt_version":"0.3","signature_b64":"H1tVAFb4h88mK8hXmDtDMO17GUQSaYoSQ4kOPlFvMuHjnIg7JEqkXChU92T22q5s/4/Xv4/YmQaF+L9yxZdqAA==","signature_status":"signed_v1","signed_at":"2026-07-05T07:21:47.372540Z","signed_message":"canonical_sha256_bytes"},"source_id":"2312.03733","source_kind":"arxiv","source_version":2}}},"equivocations":[],"invalid_events":[],"applied_event_ids":["sha256:1921bb225e345c3316057ae9def8f50071da431892249628379e7e7076916d0b","sha256:4dd35f34078017a3cfee08038d9dc26a6bc060420a5d24aaf08c2903dee53ba3"],"state_sha256":"cdb61ecd33e244f93f68dbc9bd2ee7080e9d11a6336b8bd7c7703e011ba56ca6"},"bundle_signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"sEZh+SToV/JV1RU27c+wlyNIgZnTtVBdh2FoJ4Hup33sfTmHjoTtQUOg+2YOqi1aEhJaeuJGyPLbji45rB3GBg==","signed_message":"bundle_sha256_bytes","signed_at":"2026-08-16T12:48:21.139692Z","bundle_sha256":"3b8df2e6f1f8bb02a8d669abd51e9e5eabfc43870ef51f3fc9f3ba543ca6a828"}}