{"record_type":"pith_number_record","schema_url":"https://pith.science/schemas/pith-number/v1.json","pith_number":"pith:2020:QCAXLNMOABZIDNI5YRPFBIP4OF","short_pith_number":"pith:QCAXLNMO","schema_version":"1.0","canonical_sha256":"808175b58e007281b51dc45e50a1fc716704bff81c89e98fa669239819cc5fdf","source":{"kind":"arxiv","id":"2003.07892","version":3},"attestation_state":"computed","paper":{"title":"Calibration of Pre-trained Transformers","license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","headline":"","cross_cats":["cs.LG"],"primary_cat":"cs.CL","authors_text":"Greg Durrett, Shrey Desai","submitted_at":"2020-03-17T18:58:44Z","abstract_excerpt":"Pre-trained Transformers are now ubiquitous in natural language processing, but despite their high end-task performance, little is known empirically about whether they are calibrated. Specifically, do these models' posterior probabilities provide an accurate empirical measure of how likely the model is to be correct on a given example? We focus on BERT and RoBERTa in this work, and analyze their calibration across three tasks: natural language inference, paraphrase detection, and commonsense reasoning. For each task, we consider in-domain as well as challenging out-of-domain settings, where mo"},"verification_status":{"content_addressed":true,"pith_receipt":true,"author_attested":false,"weak_author_claims":0,"strong_author_claims":0,"externally_anchored":false,"storage_verified":false,"citation_signatures":0,"replication_records":0,"graph_snapshot":true,"references_resolved":false,"formal_links_present":false},"canonical_record":{"source":{"id":"2003.07892","kind":"arxiv","version":3},"metadata":{"license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","primary_cat":"cs.CL","submitted_at":"2020-03-17T18:58:44Z","cross_cats_sorted":["cs.LG"],"title_canon_sha256":"8bc2b539cb085dbb1d4cde776ce2f69155ef7320bd24584dc9cbc2c48be7d70b","abstract_canon_sha256":"cb758587801e7ae8889be271905d13a8c28e6ffff02a8109c1c5dacf9c062ba8"},"schema_version":"1.0"},"receipt":{"kind":"pith_receipt","key_id":"pith-v1-2026-05","algorithm":"ed25519","signed_at":"2026-07-05T01:43:14.432308Z","signature_b64":"RrLKlIhgU9XArwItCNdQkjJdrG5OMpIEV47zHPbsQ6s5D6IuTQPYK6Kq9CTSVNEjEbG9c1C/Npl5qqQDF2F1BA==","signed_message":"canonical_sha256_bytes","builder_version":"pith-number-builder-2026-05-17-v1","receipt_version":"0.3","canonical_sha256":"808175b58e007281b51dc45e50a1fc716704bff81c89e98fa669239819cc5fdf","last_reissued_at":"2026-07-05T01:43:14.431847Z","signature_status":"signed_v1","first_computed_at":"2026-07-05T01:43:14.431847Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"graph_snapshot":{"paper":{"title":"Calibration of Pre-trained Transformers","license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","headline":"","cross_cats":["cs.LG"],"primary_cat":"cs.CL","authors_text":"Greg Durrett, Shrey Desai","submitted_at":"2020-03-17T18:58:44Z","abstract_excerpt":"Pre-trained Transformers are now ubiquitous in natural language processing, but despite their high end-task performance, little is known empirically about whether they are calibrated. Specifically, do these models' posterior probabilities provide an accurate empirical measure of how likely the model is to be correct on a given example? We focus on BERT and RoBERTa in this work, and analyze their calibration across three tasks: natural language inference, paraphrase detection, and commonsense reasoning. For each task, we consider in-domain as well as challenging out-of-domain settings, where mo"},"claims":{"count":0,"items":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"2003.07892","kind":"arxiv","version":3},"verdict":{"id":null,"model_set":{},"created_at":null,"strongest_claim":"","one_line_summary":"","pipeline_version":null,"weakest_assumption":"","pith_extraction_headline":""},"integrity":{"clean":true,"summary":{"advisory":0,"critical":0,"by_detector":{},"informational":0},"endpoint":"/pith/2003.07892/integrity.json","findings":[],"available":true,"detectors_run":[],"snapshot_sha256":"c28c3603d3b5d939e8dc4c7e95fa8dfce3d595e45f758748cecf8e644a296938"},"references":{"count":0,"sample":[],"resolved_work":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","internal_anchors":0},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"author_claims":{"count":0,"strong_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"builder_version":"pith-number-builder-2026-05-17-v1"},"aliases":[{"alias_kind":"arxiv","alias_value":"2003.07892","created_at":"2026-07-05T01:43:14.431905+00:00"},{"alias_kind":"arxiv_version","alias_value":"2003.07892v3","created_at":"2026-07-05T01:43:14.431905+00:00"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.2003.07892","created_at":"2026-07-05T01:43:14.431905+00:00"},{"alias_kind":"pith_short_12","alias_value":"QCAXLNMOABZI","created_at":"2026-07-05T01:43:14.431905+00:00"},{"alias_kind":"pith_short_16","alias_value":"QCAXLNMOABZIDNI5","created_at":"2026-07-05T01:43:14.431905+00:00"},{"alias_kind":"pith_short_8","alias_value":"QCAXLNMO","created_at":"2026-07-05T01:43:14.431905+00:00"}],"events":[],"event_summary":{},"paper_claims":[],"inbound_citations":{"count":6,"internal_anchor_count":0,"sample":[{"citing_arxiv_id":"2606.03535","citing_title":"Can LLM Rerankers Predict Their Own Ranking Performance?","ref_index":39,"is_internal_anchor":false},{"citing_arxiv_id":"2308.05374","citing_title":"Trustworthy LLMs: a Survey and Guideline for Evaluating Large Language Models' Alignment","ref_index":98,"is_internal_anchor":false},{"citing_arxiv_id":"2605.05710","citing_title":"On the Blessing of Pre-training in Weak-to-Strong Generalization","ref_index":69,"is_internal_anchor":false},{"citing_arxiv_id":"2207.05221","citing_title":"Language Models (Mostly) Know What They Know","ref_index":215,"is_internal_anchor":false},{"citing_arxiv_id":"2604.17200","citing_title":"Calibrating Model-Based Evaluation Metrics for Summarization","ref_index":85,"is_internal_anchor":false},{"citing_arxiv_id":"2604.19895","citing_title":"Learning When Not to Decide: A Framework for Overcoming Factual Presumptuousness in AI Adjudication","ref_index":8,"is_internal_anchor":false}]},"formal_canon":{"evidence_count":0,"sample":[],"anchors":[]},"links":{"html":"https://pith.science/pith/QCAXLNMOABZIDNI5YRPFBIP4OF","json":"https://pith.science/pith/QCAXLNMOABZIDNI5YRPFBIP4OF.json","graph_json":"https://pith.science/api/pith-number/QCAXLNMOABZIDNI5YRPFBIP4OF/graph.json","events_json":"https://pith.science/api/pith-number/QCAXLNMOABZIDNI5YRPFBIP4OF/events.json","paper":"https://pith.science/paper/QCAXLNMO"},"agent_actions":{"view_html":"https://pith.science/pith/QCAXLNMOABZIDNI5YRPFBIP4OF","download_json":"https://pith.science/pith/QCAXLNMOABZIDNI5YRPFBIP4OF.json","view_paper":"https://pith.science/paper/QCAXLNMO","resolve_alias":"https://pith.science/api/pith-number/resolve?arxiv=2003.07892&json=true","fetch_graph":"https://pith.science/api/pith-number/QCAXLNMOABZIDNI5YRPFBIP4OF/graph.json","fetch_events":"https://pith.science/api/pith-number/QCAXLNMOABZIDNI5YRPFBIP4OF/events.json","actions":{"anchor_timestamp":"https://pith.science/pith/QCAXLNMOABZIDNI5YRPFBIP4OF/action/timestamp_anchor","attest_storage":"https://pith.science/pith/QCAXLNMOABZIDNI5YRPFBIP4OF/action/storage_attestation","attest_author":"https://pith.science/pith/QCAXLNMOABZIDNI5YRPFBIP4OF/action/author_attestation","sign_citation":"https://pith.science/pith/QCAXLNMOABZIDNI5YRPFBIP4OF/action/citation_signature","submit_replication":"https://pith.science/pith/QCAXLNMOABZIDNI5YRPFBIP4OF/action/replication_record"}},"created_at":"2026-07-05T01:43:14.431905+00:00","updated_at":"2026-07-05T01:43:14.431905+00:00"}