{"record_type":"pith_number_record","schema_url":"https://pith.science/schemas/pith-number/v1.json","pith_number":"pith:2024:Q57JLP7EKBSBYKZKBWP7ANMAM4","short_pith_number":"pith:Q57JLP7E","schema_version":"1.0","canonical_sha256":"877e95bfe450641c2b2a0d9ff035806732542746e85ea6e31313bbd36c0dc07c","source":{"kind":"arxiv","id":"2402.13213","version":4},"attestation_state":"computed","paper":{"title":"Probabilities of Chat LLMs Are Miscalibrated but Still Predict Correctness on Multiple-Choice Q&A","license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","headline":"","cross_cats":["cs.AI","cs.LG"],"primary_cat":"cs.CL","authors_text":"Benjamin Plaut, Nguyen X. Khanh, Tu Trinh","submitted_at":"2024-02-20T18:24:47Z","abstract_excerpt":"We study 15 large language models (LLMs) fine-tuned for chat and find that their maximum softmax probabilities (MSPs) are consistently miscalibrated on multiple-choice Q&A. However, those MSPs might still encode useful uncertainty information. Specifically, we hypothesized that wrong answers would be associated with smaller MSPs compared to correct answers. Via rigorous statistical testing, we show that this hypothesis holds for models which perform well on the underlying Q&A task. We also find a strong direction correlation between Q&A accuracy and MSP correctness prediction, while finding no"},"verification_status":{"content_addressed":true,"pith_receipt":true,"author_attested":false,"weak_author_claims":0,"strong_author_claims":0,"externally_anchored":false,"storage_verified":false,"citation_signatures":0,"replication_records":0,"graph_snapshot":true,"references_resolved":false,"formal_links_present":false},"canonical_record":{"source":{"id":"2402.13213","kind":"arxiv","version":4},"metadata":{"license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","primary_cat":"cs.CL","submitted_at":"2024-02-20T18:24:47Z","cross_cats_sorted":["cs.AI","cs.LG"],"title_canon_sha256":"a2d422f860b006bddd838cb265bb1717daa887c1c7a67cf74b9ba303ef676275","abstract_canon_sha256":"c01b43d6776eba0485cc1632daedb78e483fb058afa384c775c716956f492e76"},"schema_version":"1.0"},"receipt":{"kind":"pith_receipt","key_id":"pith-v1-2026-05","algorithm":"ed25519","signed_at":"2026-07-05T11:49:45.109285Z","signature_b64":"JL1LtM1kuDI75I28YHj/uk2LPy70mbRwsBHD9LKGilR8ZKFPuinry+Yxi8ZVxeCIo5dLUWwi85RnJnD+NSibCQ==","signed_message":"canonical_sha256_bytes","builder_version":"pith-number-builder-2026-05-17-v1","receipt_version":"0.3","canonical_sha256":"877e95bfe450641c2b2a0d9ff035806732542746e85ea6e31313bbd36c0dc07c","last_reissued_at":"2026-07-05T11:49:45.108874Z","signature_status":"signed_v1","first_computed_at":"2026-07-05T11:49:45.108874Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"graph_snapshot":{"paper":{"title":"Probabilities of Chat LLMs Are Miscalibrated but Still Predict Correctness on Multiple-Choice Q&A","license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","headline":"","cross_cats":["cs.AI","cs.LG"],"primary_cat":"cs.CL","authors_text":"Benjamin Plaut, Nguyen X. Khanh, Tu Trinh","submitted_at":"2024-02-20T18:24:47Z","abstract_excerpt":"We study 15 large language models (LLMs) fine-tuned for chat and find that their maximum softmax probabilities (MSPs) are consistently miscalibrated on multiple-choice Q&A. However, those MSPs might still encode useful uncertainty information. Specifically, we hypothesized that wrong answers would be associated with smaller MSPs compared to correct answers. Via rigorous statistical testing, we show that this hypothesis holds for models which perform well on the underlying Q&A task. We also find a strong direction correlation between Q&A accuracy and MSP correctness prediction, while finding no"},"claims":{"count":0,"items":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"2402.13213","kind":"arxiv","version":4},"verdict":{"id":null,"model_set":{},"created_at":null,"strongest_claim":"","one_line_summary":"","pipeline_version":null,"weakest_assumption":"","pith_extraction_headline":""},"integrity":{"clean":true,"summary":{"advisory":0,"critical":0,"by_detector":{},"informational":0},"endpoint":"/pith/2402.13213/integrity.json","findings":[],"available":true,"detectors_run":[],"snapshot_sha256":"c28c3603d3b5d939e8dc4c7e95fa8dfce3d595e45f758748cecf8e644a296938"},"references":{"count":0,"sample":[],"resolved_work":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","internal_anchors":0},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"author_claims":{"count":0,"strong_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"builder_version":"pith-number-builder-2026-05-17-v1"},"aliases":[{"alias_kind":"arxiv","alias_value":"2402.13213","created_at":"2026-07-05T11:49:45.108928+00:00"},{"alias_kind":"arxiv_version","alias_value":"2402.13213v4","created_at":"2026-07-05T11:49:45.108928+00:00"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.2402.13213","created_at":"2026-07-05T11:49:45.108928+00:00"},{"alias_kind":"pith_short_12","alias_value":"Q57JLP7EKBSB","created_at":"2026-07-05T11:49:45.108928+00:00"},{"alias_kind":"pith_short_16","alias_value":"Q57JLP7EKBSBYKZK","created_at":"2026-07-05T11:49:45.108928+00:00"},{"alias_kind":"pith_short_8","alias_value":"Q57JLP7E","created_at":"2026-07-05T11:49:45.108928+00:00"}],"events":[],"event_summary":{},"paper_claims":[],"inbound_citations":{"count":9,"internal_anchor_count":0,"sample":[{"citing_arxiv_id":"2606.08088","citing_title":"ConSteer-RL: Steering Reasoning Capabilities in Large Language Models via Confidence-Aware Reinforcement Learning","ref_index":10,"is_internal_anchor":false},{"citing_arxiv_id":"2412.02904","citing_title":"Enhancing Trust in Large Language Models via Uncertainty-Calibrated Fine-Tuning","ref_index":53,"is_internal_anchor":false},{"citing_arxiv_id":"2412.15176","citing_title":"Rethinking Uncertainty Estimation in LLMs: A Principled Single-Sequence Measure","ref_index":14,"is_internal_anchor":false},{"citing_arxiv_id":"2605.09273","citing_title":"Instance-Adaptive Online Multicalibration","ref_index":30,"is_internal_anchor":false},{"citing_arxiv_id":"2605.21776","citing_title":"PromptNCE: Pointwise Mutual Information Predictions Using Only LLMs and Contrastive Estimation Prompts","ref_index":27,"is_internal_anchor":false},{"citing_arxiv_id":"2603.22161","citing_title":"Causal Evidence that Language Models use Confidence to Drive Behavior","ref_index":13,"is_internal_anchor":false},{"citing_arxiv_id":"2605.09273","citing_title":"Instance-Adaptive Online Multicalibration","ref_index":30,"is_internal_anchor":false},{"citing_arxiv_id":"2605.10202","citing_title":"Task-Aware Calibration: Provably Optimal Decoding in LLMs","ref_index":39,"is_internal_anchor":false},{"citing_arxiv_id":"2604.17200","citing_title":"Calibrating Model-Based Evaluation Metrics for Summarization","ref_index":67,"is_internal_anchor":false}]},"formal_canon":{"evidence_count":0,"sample":[],"anchors":[]},"links":{"html":"https://pith.science/pith/Q57JLP7EKBSBYKZKBWP7ANMAM4","json":"https://pith.science/pith/Q57JLP7EKBSBYKZKBWP7ANMAM4.json","graph_json":"https://pith.science/api/pith-number/Q57JLP7EKBSBYKZKBWP7ANMAM4/graph.json","events_json":"https://pith.science/api/pith-number/Q57JLP7EKBSBYKZKBWP7ANMAM4/events.json","paper":"https://pith.science/paper/Q57JLP7E"},"agent_actions":{"view_html":"https://pith.science/pith/Q57JLP7EKBSBYKZKBWP7ANMAM4","download_json":"https://pith.science/pith/Q57JLP7EKBSBYKZKBWP7ANMAM4.json","view_paper":"https://pith.science/paper/Q57JLP7E","resolve_alias":"https://pith.science/api/pith-number/resolve?arxiv=2402.13213&json=true","fetch_graph":"https://pith.science/api/pith-number/Q57JLP7EKBSBYKZKBWP7ANMAM4/graph.json","fetch_events":"https://pith.science/api/pith-number/Q57JLP7EKBSBYKZKBWP7ANMAM4/events.json","actions":{"anchor_timestamp":"https://pith.science/pith/Q57JLP7EKBSBYKZKBWP7ANMAM4/action/timestamp_anchor","attest_storage":"https://pith.science/pith/Q57JLP7EKBSBYKZKBWP7ANMAM4/action/storage_attestation","attest_author":"https://pith.science/pith/Q57JLP7EKBSBYKZKBWP7ANMAM4/action/author_attestation","sign_citation":"https://pith.science/pith/Q57JLP7EKBSBYKZKBWP7ANMAM4/action/citation_signature","submit_replication":"https://pith.science/pith/Q57JLP7EKBSBYKZKBWP7ANMAM4/action/replication_record"}},"created_at":"2026-07-05T11:49:45.108928+00:00","updated_at":"2026-07-05T11:49:45.108928+00:00"}