{"record_type":"pith_number_record","schema_url":"https://pith.science/schemas/pith-number/v1.json","pith_number":"pith:2024:UFKAHJWLAX7OLPJGXVZYB55TCY","short_pith_number":"pith:UFKAHJWL","schema_version":"1.0","canonical_sha256":"a15403a6cb05fee5bd26bd7380f7b316278cb38a1bd08b9a4da1231c577b53ff","source":{"kind":"arxiv","id":"2404.03344","version":2},"attestation_state":"computed","paper":{"title":"Schroedinger's Threshold: When the AUC doesn't predict Accuracy","license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","headline":"","cross_cats":[],"primary_cat":"cs.CL","authors_text":"Juri Opitz","submitted_at":"2024-04-04T10:18:03Z","abstract_excerpt":"The Area Under Curve measure (AUC) seems apt to evaluate and compare diverse models, possibly without calibration. An important example of AUC application is the evaluation and benchmarking of models that predict faithfulness of generated text. But we show that the AUC yields an academic and optimistic notion of accuracy that can misalign with the actual accuracy observed in application, yielding significant changes in benchmark rankings. To paint a more realistic picture of downstream model performance (and prepare a model for actual application), we explore different calibration modes, testi"},"verification_status":{"content_addressed":true,"pith_receipt":true,"author_attested":false,"weak_author_claims":0,"strong_author_claims":0,"externally_anchored":false,"storage_verified":false,"citation_signatures":0,"replication_records":0,"graph_snapshot":true,"references_resolved":false,"formal_links_present":false},"canonical_record":{"source":{"id":"2404.03344","kind":"arxiv","version":2},"metadata":{"license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","primary_cat":"cs.CL","submitted_at":"2024-04-04T10:18:03Z","cross_cats_sorted":[],"title_canon_sha256":"0644fb1cdd5657e853cd5b587d671ed5c39d739385c4f68a35a3d2586651d2ce","abstract_canon_sha256":"3eb0b6f1cc6654d96e4126cc240efe61ac1dfcfde54b0b6cd542270c2dd18243"},"schema_version":"1.0"},"receipt":{"kind":"pith_receipt","key_id":"pith-v1-2026-05","algorithm":"ed25519","signed_at":"2026-07-05T08:23:20.944272Z","signature_b64":"OWeMJJBd/qbAE7iE3LnaoiZatFgQ0lNvIIQ/MGyK474WeHTIuf0A8PywXU2Cv2NbgJoXw2vRa8ceb4PToB26Ag==","signed_message":"canonical_sha256_bytes","builder_version":"pith-number-builder-2026-05-17-v1","receipt_version":"0.3","canonical_sha256":"a15403a6cb05fee5bd26bd7380f7b316278cb38a1bd08b9a4da1231c577b53ff","last_reissued_at":"2026-07-05T08:23:20.943801Z","signature_status":"signed_v1","first_computed_at":"2026-07-05T08:23:20.943801Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"graph_snapshot":{"paper":{"title":"Schroedinger's Threshold: When the AUC doesn't predict Accuracy","license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","headline":"","cross_cats":[],"primary_cat":"cs.CL","authors_text":"Juri Opitz","submitted_at":"2024-04-04T10:18:03Z","abstract_excerpt":"The Area Under Curve measure (AUC) seems apt to evaluate and compare diverse models, possibly without calibration. An important example of AUC application is the evaluation and benchmarking of models that predict faithfulness of generated text. But we show that the AUC yields an academic and optimistic notion of accuracy that can misalign with the actual accuracy observed in application, yielding significant changes in benchmark rankings. To paint a more realistic picture of downstream model performance (and prepare a model for actual application), we explore different calibration modes, testi"},"claims":{"count":0,"items":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"2404.03344","kind":"arxiv","version":2},"verdict":{"id":null,"model_set":{},"created_at":null,"strongest_claim":"","one_line_summary":"","pipeline_version":null,"weakest_assumption":"","pith_extraction_headline":""},"integrity":{"clean":true,"summary":{"advisory":0,"critical":0,"by_detector":{},"informational":0},"endpoint":"/pith/2404.03344/integrity.json","findings":[],"available":true,"detectors_run":[],"snapshot_sha256":"c28c3603d3b5d939e8dc4c7e95fa8dfce3d595e45f758748cecf8e644a296938"},"references":{"count":0,"sample":[],"resolved_work":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","internal_anchors":0},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"author_claims":{"count":0,"strong_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"builder_version":"pith-number-builder-2026-05-17-v1"},"aliases":[{"alias_kind":"arxiv","alias_value":"2404.03344","created_at":"2026-07-05T08:23:20.943865+00:00"},{"alias_kind":"arxiv_version","alias_value":"2404.03344v2","created_at":"2026-07-05T08:23:20.943865+00:00"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.2404.03344","created_at":"2026-07-05T08:23:20.943865+00:00"},{"alias_kind":"pith_short_12","alias_value":"UFKAHJWLAX7O","created_at":"2026-07-05T08:23:20.943865+00:00"},{"alias_kind":"pith_short_16","alias_value":"UFKAHJWLAX7OLPJG","created_at":"2026-07-05T08:23:20.943865+00:00"},{"alias_kind":"pith_short_8","alias_value":"UFKAHJWL","created_at":"2026-07-05T08:23:20.943865+00:00"}],"events":[],"event_summary":{},"paper_claims":[],"inbound_citations":{"count":1,"internal_anchor_count":1,"sample":[{"citing_arxiv_id":"2412.12148","citing_title":"How to Choose a Threshold for an Evaluation Metric for Large Language Models","ref_index":12,"is_internal_anchor":true}]},"formal_canon":{"evidence_count":0,"sample":[],"anchors":[]},"links":{"html":"https://pith.science/pith/UFKAHJWLAX7OLPJGXVZYB55TCY","json":"https://pith.science/pith/UFKAHJWLAX7OLPJGXVZYB55TCY.json","graph_json":"https://pith.science/api/pith-number/UFKAHJWLAX7OLPJGXVZYB55TCY/graph.json","events_json":"https://pith.science/api/pith-number/UFKAHJWLAX7OLPJGXVZYB55TCY/events.json","paper":"https://pith.science/paper/UFKAHJWL"},"agent_actions":{"view_html":"https://pith.science/pith/UFKAHJWLAX7OLPJGXVZYB55TCY","download_json":"https://pith.science/pith/UFKAHJWLAX7OLPJGXVZYB55TCY.json","view_paper":"https://pith.science/paper/UFKAHJWL","resolve_alias":"https://pith.science/api/pith-number/resolve?arxiv=2404.03344&json=true","fetch_graph":"https://pith.science/api/pith-number/UFKAHJWLAX7OLPJGXVZYB55TCY/graph.json","fetch_events":"https://pith.science/api/pith-number/UFKAHJWLAX7OLPJGXVZYB55TCY/events.json","actions":{"anchor_timestamp":"https://pith.science/pith/UFKAHJWLAX7OLPJGXVZYB55TCY/action/timestamp_anchor","attest_storage":"https://pith.science/pith/UFKAHJWLAX7OLPJGXVZYB55TCY/action/storage_attestation","attest_author":"https://pith.science/pith/UFKAHJWLAX7OLPJGXVZYB55TCY/action/author_attestation","sign_citation":"https://pith.science/pith/UFKAHJWLAX7OLPJGXVZYB55TCY/action/citation_signature","submit_replication":"https://pith.science/pith/UFKAHJWLAX7OLPJGXVZYB55TCY/action/replication_record"}},"created_at":"2026-07-05T08:23:20.943865+00:00","updated_at":"2026-07-05T08:23:20.943865+00:00"}