{"bundle_type":"pith_open_graph_bundle","bundle_version":"1.0","pith_number":"pith:2024:EFQMBPIHHMALI5UWYWQTAS6EXZ","short_pith_number":"pith:EFQMBPIH","canonical_record":{"source":{"id":"2406.08391","kind":"arxiv","version":3},"metadata":{"license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","primary_cat":"cs.LG","submitted_at":"2024-06-12T16:41:31Z","cross_cats_sorted":["cs.AI","cs.CL","stat.ML"],"title_canon_sha256":"aef5451ff38246c4cda6971358dabcecb34b3bddf074969356d7e289ce0821ee","abstract_canon_sha256":"e82e4a279e056da8e72ce74e9177c7d404ef06af80bdc3460e67c27834f03782"},"schema_version":"1.0"},"canonical_sha256":"2160c0bd073b00b47696c5a1304bc4be7c87aa6586162ab9c81a2434c49b340a","source":{"kind":"arxiv","id":"2406.08391","version":3},"source_aliases":[{"alias_kind":"arxiv","alias_value":"2406.08391","created_at":"2026-07-05T11:54:40Z"},{"alias_kind":"arxiv_version","alias_value":"2406.08391v3","created_at":"2026-07-05T11:54:40Z"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.2406.08391","created_at":"2026-07-05T11:54:40Z"},{"alias_kind":"pith_short_12","alias_value":"EFQMBPIHHMAL","created_at":"2026-07-05T11:54:40Z"},{"alias_kind":"pith_short_16","alias_value":"EFQMBPIHHMALI5UW","created_at":"2026-07-05T11:54:40Z"},{"alias_kind":"pith_short_8","alias_value":"EFQMBPIH","created_at":"2026-07-05T11:54:40Z"}],"events":[{"event_type":"record_created","subject_pith_number":"pith:2024:EFQMBPIHHMALI5UWYWQTAS6EXZ","target":"record","payload":{"canonical_record":{"source":{"id":"2406.08391","kind":"arxiv","version":3},"metadata":{"license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","primary_cat":"cs.LG","submitted_at":"2024-06-12T16:41:31Z","cross_cats_sorted":["cs.AI","cs.CL","stat.ML"],"title_canon_sha256":"aef5451ff38246c4cda6971358dabcecb34b3bddf074969356d7e289ce0821ee","abstract_canon_sha256":"e82e4a279e056da8e72ce74e9177c7d404ef06af80bdc3460e67c27834f03782"},"schema_version":"1.0"},"canonical_sha256":"2160c0bd073b00b47696c5a1304bc4be7c87aa6586162ab9c81a2434c49b340a","receipt":{"kind":"pith_receipt","key_id":"pith-v1-2026-05","algorithm":"ed25519","signed_at":"2026-07-05T11:54:40.057471Z","signature_b64":"dpf4UciZ2NsVdYKhJDnL3FKafXiCWUUQb8Mc00YupymNfEaqyuQpSjF/bHt/f6HEeykfZFiyVbBE7Oh5bDfiCA==","signed_message":"canonical_sha256_bytes","builder_version":"pith-number-builder-2026-05-17-v1","receipt_version":"0.3","canonical_sha256":"2160c0bd073b00b47696c5a1304bc4be7c87aa6586162ab9c81a2434c49b340a","last_reissued_at":"2026-07-05T11:54:40.056939Z","signature_status":"signed_v1","first_computed_at":"2026-07-05T11:54:40.056939Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"source_kind":"arxiv","source_id":"2406.08391","source_version":3,"attestation_state":"computed"},"signer":{"signer_id":"pith.science","signer_type":"pith_registry","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"created_at":"2026-07-05T11:54:40Z","supersedes":[],"prev_event":null,"signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"7ML0ru3mWRdZcsk1h83vX0YRpUSVpGk67NBAvX0g1fsJzGJbe/05AchvnjQA3oiRqvPZgIDqDpiqrIeQaVH5BQ==","signed_message":"open_graph_event_sha256_bytes","signed_at":"2026-08-07T17:50:35.053513Z"},"content_sha256":"769db2c3b84efe5b84468864b96b94046792a759dbfe3e2e41151fd9c355fe60","schema_version":"1.0","event_id":"sha256:769db2c3b84efe5b84468864b96b94046792a759dbfe3e2e41151fd9c355fe60"},{"event_type":"graph_snapshot","subject_pith_number":"pith:2024:EFQMBPIHHMALI5UWYWQTAS6EXZ","target":"graph","payload":{"graph_snapshot":{"paper":{"title":"Large Language Models Must Be Taught to Know What They Don't Know","license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","headline":"","cross_cats":["cs.AI","cs.CL","stat.ML"],"primary_cat":"cs.LG","authors_text":"Adrian Weller, Andrew Gordon Wilson, Arka Pal, Katherine Collins, Manley Roberts, Micah Goldblum, Nate Gruver, Samuel Dooley, Sanyam Kapoor, Umang Bhatt","submitted_at":"2024-06-12T16:41:31Z","abstract_excerpt":"When using large language models (LLMs) in high-stakes applications, we need to know when we can trust their predictions. Some works argue that prompting high-performance LLMs is sufficient to produce calibrated uncertainties, while others introduce sampling methods that can be prohibitively expensive. In this work, we first argue that prompting on its own is insufficient to achieve good calibration and then show that fine-tuning on a small dataset of correct and incorrect answers can create an uncertainty estimate with good generalization and small computational overhead. We show that a thous"},"claims":{"count":0,"items":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"2406.08391","kind":"arxiv","version":3},"verdict":{"id":null,"model_set":{},"created_at":null,"strongest_claim":"","one_line_summary":"","pipeline_version":null,"weakest_assumption":"","pith_extraction_headline":""},"integrity":{"clean":true,"summary":{"advisory":0,"critical":0,"by_detector":{},"informational":0},"endpoint":"/pith/2406.08391/integrity.json","findings":[],"available":true,"detectors_run":[],"snapshot_sha256":"c28c3603d3b5d939e8dc4c7e95fa8dfce3d595e45f758748cecf8e644a296938"},"references":{"count":0,"sample":[],"resolved_work":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","internal_anchors":0},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"author_claims":{"count":0,"strong_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"builder_version":"pith-number-builder-2026-05-17-v1"},"verdict_id":null},"signer":{"signer_id":"pith.science","signer_type":"pith_registry","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"created_at":"2026-07-05T11:54:40Z","supersedes":[],"prev_event":null,"signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"tXRChQ1bDjtqh0Bu/0nbn03QvEsHJwkA44OYlYyxZjXQwILanPvGWCoVTTa5sWpsFeCl3i6jH7eh6mPxyFwYDw==","signed_message":"open_graph_event_sha256_bytes","signed_at":"2026-08-07T17:50:35.054031Z"},"content_sha256":"e1f9f12e3c9b1c485886c37fcb0996e7eed3474154d74187e879cc93782eb876","schema_version":"1.0","event_id":"sha256:e1f9f12e3c9b1c485886c37fcb0996e7eed3474154d74187e879cc93782eb876"}],"timestamp_proofs":[],"mirror_hints":[{"mirror_type":"https","name":"Pith Resolver","base_url":"https://pith.science","bundle_url":"https://pith.science/pith/EFQMBPIHHMALI5UWYWQTAS6EXZ/bundle.json","state_url":"https://pith.science/pith/EFQMBPIHHMALI5UWYWQTAS6EXZ/state.json","well_known_bundle_url":"https://pith.science/.well-known/pith/EFQMBPIHHMALI5UWYWQTAS6EXZ/bundle.json","status":"primary"}],"public_keys":[{"key_id":"pith-v1-2026-05","algorithm":"ed25519","format":"raw","public_key_b64":"stVStoiQhXFxp4s2pdzPNoqVNBMojDU/fJ2db5S3CbM=","public_key_hex":"b2d552b68890857171a78b36a5dccf368a953413288c353f7c9d9d6f94b709b3","fingerprint_sha256_b32_first128bits":"RVFV5Z2OI2J3ZUO7ERDEBCYNKS","fingerprint_sha256_hex":"8d4b5ee74e4693bcd1df2446408b0d54","rotates_at":null,"url":"https://pith.science/pith-signing-key.json","notes":"Pith uses this Ed25519 key to sign canonical record SHA-256 digests. Verify with: ed25519_verify(public_key, message=canonical_sha256_bytes, signature=base64decode(signature_b64))."}],"merge_version":"pith-open-graph-merge-v1","built_at":"2026-08-07T17:50:35Z","links":{"resolver":"https://pith.science/pith/EFQMBPIHHMALI5UWYWQTAS6EXZ","bundle":"https://pith.science/pith/EFQMBPIHHMALI5UWYWQTAS6EXZ/bundle.json","state":"https://pith.science/pith/EFQMBPIHHMALI5UWYWQTAS6EXZ/state.json","well_known_bundle":"https://pith.science/.well-known/pith/EFQMBPIHHMALI5UWYWQTAS6EXZ/bundle.json"},"state":{"state_type":"pith_open_graph_state","state_version":"1.0","pith_number":"pith:2024:EFQMBPIHHMALI5UWYWQTAS6EXZ","merge_version":"pith-open-graph-merge-v1","event_count":2,"valid_event_count":2,"invalid_event_count":0,"equivocation_count":0,"current":{"canonical_record":{"metadata":{"abstract_canon_sha256":"e82e4a279e056da8e72ce74e9177c7d404ef06af80bdc3460e67c27834f03782","cross_cats_sorted":["cs.AI","cs.CL","stat.ML"],"license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","primary_cat":"cs.LG","submitted_at":"2024-06-12T16:41:31Z","title_canon_sha256":"aef5451ff38246c4cda6971358dabcecb34b3bddf074969356d7e289ce0821ee"},"schema_version":"1.0","source":{"id":"2406.08391","kind":"arxiv","version":3}},"source_aliases":[{"alias_kind":"arxiv","alias_value":"2406.08391","created_at":"2026-07-05T11:54:40Z"},{"alias_kind":"arxiv_version","alias_value":"2406.08391v3","created_at":"2026-07-05T11:54:40Z"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.2406.08391","created_at":"2026-07-05T11:54:40Z"},{"alias_kind":"pith_short_12","alias_value":"EFQMBPIHHMAL","created_at":"2026-07-05T11:54:40Z"},{"alias_kind":"pith_short_16","alias_value":"EFQMBPIHHMALI5UW","created_at":"2026-07-05T11:54:40Z"},{"alias_kind":"pith_short_8","alias_value":"EFQMBPIH","created_at":"2026-07-05T11:54:40Z"}],"graph_snapshots":[{"event_id":"sha256:e1f9f12e3c9b1c485886c37fcb0996e7eed3474154d74187e879cc93782eb876","target":"graph","created_at":"2026-07-05T11:54:40Z","signer":{"key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signer_id":"pith.science","signer_type":"pith_registry"},"payload":{"graph_snapshot":{"author_claims":{"count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","strong_count":0},"builder_version":"pith-number-builder-2026-05-17-v1","claims":{"count":0,"items":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"integrity":{"available":true,"clean":true,"detectors_run":[],"endpoint":"/pith/2406.08391/integrity.json","findings":[],"snapshot_sha256":"c28c3603d3b5d939e8dc4c7e95fa8dfce3d595e45f758748cecf8e644a296938","summary":{"advisory":0,"by_detector":{},"critical":0,"informational":0}},"paper":{"abstract_excerpt":"When using large language models (LLMs) in high-stakes applications, we need to know when we can trust their predictions. Some works argue that prompting high-performance LLMs is sufficient to produce calibrated uncertainties, while others introduce sampling methods that can be prohibitively expensive. In this work, we first argue that prompting on its own is insufficient to achieve good calibration and then show that fine-tuning on a small dataset of correct and incorrect answers can create an uncertainty estimate with good generalization and small computational overhead. We show that a thous","authors_text":"Adrian Weller, Andrew Gordon Wilson, Arka Pal, Katherine Collins, Manley Roberts, Micah Goldblum, Nate Gruver, Samuel Dooley, Sanyam Kapoor, Umang Bhatt","cross_cats":["cs.AI","cs.CL","stat.ML"],"headline":"","license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","primary_cat":"cs.LG","submitted_at":"2024-06-12T16:41:31Z","title":"Large Language Models Must Be Taught to Know What They Don't Know"},"references":{"count":0,"internal_anchors":0,"resolved_work":0,"sample":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"2406.08391","kind":"arxiv","version":3},"verdict":{"created_at":null,"id":null,"model_set":{},"one_line_summary":"","pipeline_version":null,"pith_extraction_headline":"","strongest_claim":"","weakest_assumption":""}},"verdict_id":null}}],"author_attestations":[],"timestamp_anchors":[],"storage_attestations":[],"citation_signatures":[],"replication_records":[],"corrections":[],"mirror_hints":[],"record_created":{"event_id":"sha256:769db2c3b84efe5b84468864b96b94046792a759dbfe3e2e41151fd9c355fe60","target":"record","created_at":"2026-07-05T11:54:40Z","signer":{"key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signer_id":"pith.science","signer_type":"pith_registry"},"payload":{"attestation_state":"computed","canonical_record":{"metadata":{"abstract_canon_sha256":"e82e4a279e056da8e72ce74e9177c7d404ef06af80bdc3460e67c27834f03782","cross_cats_sorted":["cs.AI","cs.CL","stat.ML"],"license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","primary_cat":"cs.LG","submitted_at":"2024-06-12T16:41:31Z","title_canon_sha256":"aef5451ff38246c4cda6971358dabcecb34b3bddf074969356d7e289ce0821ee"},"schema_version":"1.0","source":{"id":"2406.08391","kind":"arxiv","version":3}},"canonical_sha256":"2160c0bd073b00b47696c5a1304bc4be7c87aa6586162ab9c81a2434c49b340a","receipt":{"algorithm":"ed25519","builder_version":"pith-number-builder-2026-05-17-v1","canonical_sha256":"2160c0bd073b00b47696c5a1304bc4be7c87aa6586162ab9c81a2434c49b340a","first_computed_at":"2026-07-05T11:54:40.056939Z","key_id":"pith-v1-2026-05","kind":"pith_receipt","last_reissued_at":"2026-07-05T11:54:40.056939Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","receipt_version":"0.3","signature_b64":"dpf4UciZ2NsVdYKhJDnL3FKafXiCWUUQb8Mc00YupymNfEaqyuQpSjF/bHt/f6HEeykfZFiyVbBE7Oh5bDfiCA==","signature_status":"signed_v1","signed_at":"2026-07-05T11:54:40.057471Z","signed_message":"canonical_sha256_bytes"},"source_id":"2406.08391","source_kind":"arxiv","source_version":3}}},"equivocations":[],"invalid_events":[],"applied_event_ids":["sha256:769db2c3b84efe5b84468864b96b94046792a759dbfe3e2e41151fd9c355fe60","sha256:e1f9f12e3c9b1c485886c37fcb0996e7eed3474154d74187e879cc93782eb876"],"state_sha256":"cf86bfa286ad82cde65329fb33baab34a2b8454d1d6d5e23f6d79167b69ef985"},"bundle_signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"1QBvNLbUhAVB7bQWS4fNmTyooj5lKLpeLsSXRbWDt282zyRj4hfoDtGngPU8Pf9aylt+0ZSYC+XX2Rlf8ev4BQ==","signed_message":"bundle_sha256_bytes","signed_at":"2026-08-07T17:50:35.059064Z","bundle_sha256":"95b4da02f37098fa29941d7d6a759b6251c3080c5d6217977e2e8c0b1431361e"}}