{"record_type":"pith_number_record","schema_url":"https://pith.science/schemas/pith-number/v1.json","pith_number":"pith:2024:6UCW3VT3K4Q3ECGWNZHN44CNOL","short_pith_number":"pith:6UCW3VT3","schema_version":"1.0","canonical_sha256":"f5056dd67b5721b208d66e4ede704d72cf313989c208af2c5ac45e5f30e14867","source":{"kind":"arxiv","id":"2406.02543","version":2},"attestation_state":"computed","paper":{"title":"To Believe or Not to Believe Your LLM","license":"http://creativecommons.org/licenses/by/4.0/","headline":"","cross_cats":["cs.AI","cs.CL"],"primary_cat":"cs.LG","authors_text":"Andr\\'as Gy\\\"orgy, Csaba Szepesv\\'ari, Ilja Kuzborskij, Yasin Abbasi Yadkori","submitted_at":"2024-06-04T17:58:18Z","abstract_excerpt":"We explore uncertainty quantification in large language models (LLMs), with the goal to identify when uncertainty in responses given a query is large. We simultaneously consider both epistemic and aleatoric uncertainties, where the former comes from the lack of knowledge about the ground truth (such as about facts or the language), and the latter comes from irreducible randomness (such as multiple possible answers). In particular, we derive an information-theoretic metric that allows to reliably detect when only epistemic uncertainty is large, in which case the output of the model is unreliabl"},"verification_status":{"content_addressed":true,"pith_receipt":true,"author_attested":false,"weak_author_claims":0,"strong_author_claims":0,"externally_anchored":false,"storage_verified":false,"citation_signatures":0,"replication_records":0,"graph_snapshot":true,"references_resolved":false,"formal_links_present":false},"canonical_record":{"source":{"id":"2406.02543","kind":"arxiv","version":2},"metadata":{"license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.LG","submitted_at":"2024-06-04T17:58:18Z","cross_cats_sorted":["cs.AI","cs.CL"],"title_canon_sha256":"9d55607cdba93f312baab520737fe212f8bd24756ba732e9e90136947038597f","abstract_canon_sha256":"4f1cb435e52a776221c6eeb33f74451cee5fd184398f1f0bc74035cc53fe1c5a"},"schema_version":"1.0"},"receipt":{"kind":"pith_receipt","key_id":"pith-v1-2026-05","algorithm":"ed25519","signed_at":"2026-07-05T08:44:54.991360Z","signature_b64":"66aGg98M7X20U1ErAWj2UoTawrFYOHLF/au4tCUoeLGbtXEtM1nxTLtIv6VSQn+4a3FczBzGi+S2FC6wgVPrCg==","signed_message":"canonical_sha256_bytes","builder_version":"pith-number-builder-2026-05-17-v1","receipt_version":"0.3","canonical_sha256":"f5056dd67b5721b208d66e4ede704d72cf313989c208af2c5ac45e5f30e14867","last_reissued_at":"2026-07-05T08:44:54.990929Z","signature_status":"signed_v1","first_computed_at":"2026-07-05T08:44:54.990929Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"graph_snapshot":{"paper":{"title":"To Believe or Not to Believe Your LLM","license":"http://creativecommons.org/licenses/by/4.0/","headline":"","cross_cats":["cs.AI","cs.CL"],"primary_cat":"cs.LG","authors_text":"Andr\\'as Gy\\\"orgy, Csaba Szepesv\\'ari, Ilja Kuzborskij, Yasin Abbasi Yadkori","submitted_at":"2024-06-04T17:58:18Z","abstract_excerpt":"We explore uncertainty quantification in large language models (LLMs), with the goal to identify when uncertainty in responses given a query is large. We simultaneously consider both epistemic and aleatoric uncertainties, where the former comes from the lack of knowledge about the ground truth (such as about facts or the language), and the latter comes from irreducible randomness (such as multiple possible answers). In particular, we derive an information-theoretic metric that allows to reliably detect when only epistemic uncertainty is large, in which case the output of the model is unreliabl"},"claims":{"count":0,"items":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"2406.02543","kind":"arxiv","version":2},"verdict":{"id":null,"model_set":{},"created_at":null,"strongest_claim":"","one_line_summary":"","pipeline_version":null,"weakest_assumption":"","pith_extraction_headline":""},"integrity":{"clean":true,"summary":{"advisory":0,"critical":0,"by_detector":{},"informational":0},"endpoint":"/pith/2406.02543/integrity.json","findings":[],"available":true,"detectors_run":[],"snapshot_sha256":"c28c3603d3b5d939e8dc4c7e95fa8dfce3d595e45f758748cecf8e644a296938"},"references":{"count":0,"sample":[],"resolved_work":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","internal_anchors":0},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"author_claims":{"count":0,"strong_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"builder_version":"pith-number-builder-2026-05-17-v1"},"aliases":[{"alias_kind":"arxiv","alias_value":"2406.02543","created_at":"2026-07-05T08:44:54.990983+00:00"},{"alias_kind":"arxiv_version","alias_value":"2406.02543v2","created_at":"2026-07-05T08:44:54.990983+00:00"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.2406.02543","created_at":"2026-07-05T08:44:54.990983+00:00"},{"alias_kind":"pith_short_12","alias_value":"6UCW3VT3K4Q3","created_at":"2026-07-05T08:44:54.990983+00:00"},{"alias_kind":"pith_short_16","alias_value":"6UCW3VT3K4Q3ECGW","created_at":"2026-07-05T08:44:54.990983+00:00"},{"alias_kind":"pith_short_8","alias_value":"6UCW3VT3","created_at":"2026-07-05T08:44:54.990983+00:00"}],"events":[],"event_summary":{},"paper_claims":[],"inbound_citations":{"count":13,"internal_anchor_count":1,"sample":[{"citing_arxiv_id":"2607.05721","citing_title":"SpanUQ: Span-Level Uncertainty Quantification for Large Language Model Generation","ref_index":11,"is_internal_anchor":true},{"citing_arxiv_id":"2606.12587","citing_title":"Strategic Decision Support for AI Agents","ref_index":94,"is_internal_anchor":false},{"citing_arxiv_id":"2606.02289","citing_title":"DECK: A Consistency x Confidence Taxonomy of LLM Hallucinations","ref_index":42,"is_internal_anchor":false},{"citing_arxiv_id":"2605.04295","citing_title":"LLMs Uncertainty Quantification via Adaptive Conformal Semantic Entropy","ref_index":39,"is_internal_anchor":false},{"citing_arxiv_id":"2606.32032","citing_title":"Reinforcement Learning with Metacognitive Feedback Elicits Faithful Uncertainty Expression in LLMs","ref_index":112,"is_internal_anchor":false},{"citing_arxiv_id":"2605.28778","citing_title":"Can LLMs Use Linguistic Uncertainty Markers to Reliably Reflect Intrinsic Confidence?","ref_index":90,"is_internal_anchor":false},{"citing_arxiv_id":"2505.11737","citing_title":"TokUR: Token-Level Uncertainty Estimation for Large Language Model Reasoning","ref_index":42,"is_internal_anchor":false},{"citing_arxiv_id":"2602.06239","citing_title":"Provably avoiding over-optimization in Direct Preference Optimization without knowing the data distribution","ref_index":35,"is_internal_anchor":false},{"citing_arxiv_id":"2506.14387","citing_title":"SEAT: Sparse Entity-Aware Tuning for Knowledge Adaptation while Preserving Epistemic Abstention","ref_index":14,"is_internal_anchor":false},{"citing_arxiv_id":"2602.06239","citing_title":"Provably avoiding over-optimization in Direct Preference Optimization without knowing the data distribution","ref_index":35,"is_internal_anchor":false},{"citing_arxiv_id":"2604.02543","citing_title":"Overconfidence and Calibration in Medical VQA: Empirical Findings and Hallucination-Aware Mitigation","ref_index":28,"is_internal_anchor":false},{"citing_arxiv_id":"2605.11328","citing_title":"Epistemic Uncertainty for Test-Time Discovery","ref_index":28,"is_internal_anchor":false},{"citing_arxiv_id":"2605.04295","citing_title":"LLMs Uncertainty Quantification via Adaptive Conformal Semantic Entropy","ref_index":15,"is_internal_anchor":false}]},"formal_canon":{"evidence_count":0,"sample":[],"anchors":[]},"links":{"html":"https://pith.science/pith/6UCW3VT3K4Q3ECGWNZHN44CNOL","json":"https://pith.science/pith/6UCW3VT3K4Q3ECGWNZHN44CNOL.json","graph_json":"https://pith.science/api/pith-number/6UCW3VT3K4Q3ECGWNZHN44CNOL/graph.json","events_json":"https://pith.science/api/pith-number/6UCW3VT3K4Q3ECGWNZHN44CNOL/events.json","paper":"https://pith.science/paper/6UCW3VT3"},"agent_actions":{"view_html":"https://pith.science/pith/6UCW3VT3K4Q3ECGWNZHN44CNOL","download_json":"https://pith.science/pith/6UCW3VT3K4Q3ECGWNZHN44CNOL.json","view_paper":"https://pith.science/paper/6UCW3VT3","resolve_alias":"https://pith.science/api/pith-number/resolve?arxiv=2406.02543&json=true","fetch_graph":"https://pith.science/api/pith-number/6UCW3VT3K4Q3ECGWNZHN44CNOL/graph.json","fetch_events":"https://pith.science/api/pith-number/6UCW3VT3K4Q3ECGWNZHN44CNOL/events.json","actions":{"anchor_timestamp":"https://pith.science/pith/6UCW3VT3K4Q3ECGWNZHN44CNOL/action/timestamp_anchor","attest_storage":"https://pith.science/pith/6UCW3VT3K4Q3ECGWNZHN44CNOL/action/storage_attestation","attest_author":"https://pith.science/pith/6UCW3VT3K4Q3ECGWNZHN44CNOL/action/author_attestation","sign_citation":"https://pith.science/pith/6UCW3VT3K4Q3ECGWNZHN44CNOL/action/citation_signature","submit_replication":"https://pith.science/pith/6UCW3VT3K4Q3ECGWNZHN44CNOL/action/replication_record"}},"created_at":"2026-07-05T08:44:54.990983+00:00","updated_at":"2026-07-05T08:44:54.990983+00:00"}