{"record_type":"pith_number_record","schema_url":"https://pith.science/schemas/pith-number/v1.json","pith_number":"pith:2024:RBTMX6SKNM7Z5XDQMPOBXKNS3T","short_pith_number":"pith:RBTMX6SK","schema_version":"1.0","canonical_sha256":"8866cbfa4a6b3f9edc7063dc1ba9b2dccefc14ebc676b45aee5a92074dc0e9de","source":{"kind":"arxiv","id":"2411.02603","version":3},"attestation_state":"computed","paper":{"title":"FactTest: Factuality Testing in Large Language Models with Finite-Sample and Distribution-Free Guarantees","license":"http://creativecommons.org/licenses/by-nc-nd/4.0/","headline":"","cross_cats":["cs.AI","stat.ML"],"primary_cat":"cs.CL","authors_text":"Fan Nie, Huaxiu Yao, James Zou, Linjun Zhang, Shuhang Lin, Xiaotian Hou","submitted_at":"2024-11-04T20:53:04Z","abstract_excerpt":"The propensity of Large Language Models (LLMs) to generate hallucinations and non-factual content undermines their reliability in high-stakes domains, where rigorous control over Type I errors (the conditional probability of incorrectly classifying hallucinations as truthful content) is essential. Despite its importance, formal verification of LLM factuality with such guarantees remains largely unexplored. In this paper, we introduce FactTest, a novel framework that statistically assesses whether a LLM can confidently provide correct answers to given questions with high-probability correctness"},"verification_status":{"content_addressed":true,"pith_receipt":true,"author_attested":false,"weak_author_claims":0,"strong_author_claims":0,"externally_anchored":false,"storage_verified":false,"citation_signatures":0,"replication_records":0,"graph_snapshot":true,"references_resolved":false,"formal_links_present":false},"canonical_record":{"source":{"id":"2411.02603","kind":"arxiv","version":3},"metadata":{"license":"http://creativecommons.org/licenses/by-nc-nd/4.0/","primary_cat":"cs.CL","submitted_at":"2024-11-04T20:53:04Z","cross_cats_sorted":["cs.AI","stat.ML"],"title_canon_sha256":"272678510611b634548bac6cf0989f171948668849c680f76d728ee46d0c9c61","abstract_canon_sha256":"ea88f55e6c6b854f3fa49406b86b69e4afb291edfe97611c952c0dcd26fa5766"},"schema_version":"1.0"},"receipt":{"kind":"pith_receipt","key_id":"pith-v1-2026-05","algorithm":"ed25519","signed_at":"2026-07-05T09:32:13.619683Z","signature_b64":"6XuO0XlJ9cP42nyqVzjZonDLICMVC2BsjDcBDlVmGslfQ5T8eQakF0lWawDdANgDoL/ZeEzlceyaTquLKdtHCw==","signed_message":"canonical_sha256_bytes","builder_version":"pith-number-builder-2026-05-17-v1","receipt_version":"0.3","canonical_sha256":"8866cbfa4a6b3f9edc7063dc1ba9b2dccefc14ebc676b45aee5a92074dc0e9de","last_reissued_at":"2026-07-05T09:32:13.619192Z","signature_status":"signed_v1","first_computed_at":"2026-07-05T09:32:13.619192Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"graph_snapshot":{"paper":{"title":"FactTest: Factuality Testing in Large Language Models with Finite-Sample and Distribution-Free Guarantees","license":"http://creativecommons.org/licenses/by-nc-nd/4.0/","headline":"","cross_cats":["cs.AI","stat.ML"],"primary_cat":"cs.CL","authors_text":"Fan Nie, Huaxiu Yao, James Zou, Linjun Zhang, Shuhang Lin, Xiaotian Hou","submitted_at":"2024-11-04T20:53:04Z","abstract_excerpt":"The propensity of Large Language Models (LLMs) to generate hallucinations and non-factual content undermines their reliability in high-stakes domains, where rigorous control over Type I errors (the conditional probability of incorrectly classifying hallucinations as truthful content) is essential. Despite its importance, formal verification of LLM factuality with such guarantees remains largely unexplored. In this paper, we introduce FactTest, a novel framework that statistically assesses whether a LLM can confidently provide correct answers to given questions with high-probability correctness"},"claims":{"count":0,"items":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"2411.02603","kind":"arxiv","version":3},"verdict":{"id":null,"model_set":{},"created_at":null,"strongest_claim":"","one_line_summary":"","pipeline_version":null,"weakest_assumption":"","pith_extraction_headline":""},"integrity":{"clean":true,"summary":{"advisory":0,"critical":0,"by_detector":{},"informational":0},"endpoint":"/pith/2411.02603/integrity.json","findings":[],"available":true,"detectors_run":[],"snapshot_sha256":"c28c3603d3b5d939e8dc4c7e95fa8dfce3d595e45f758748cecf8e644a296938"},"references":{"count":0,"sample":[],"resolved_work":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","internal_anchors":0},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"author_claims":{"count":0,"strong_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"builder_version":"pith-number-builder-2026-05-17-v1"},"aliases":[{"alias_kind":"arxiv","alias_value":"2411.02603","created_at":"2026-07-05T09:32:13.619250+00:00"},{"alias_kind":"arxiv_version","alias_value":"2411.02603v3","created_at":"2026-07-05T09:32:13.619250+00:00"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.2411.02603","created_at":"2026-07-05T09:32:13.619250+00:00"},{"alias_kind":"pith_short_12","alias_value":"RBTMX6SKNM7Z","created_at":"2026-07-05T09:32:13.619250+00:00"},{"alias_kind":"pith_short_16","alias_value":"RBTMX6SKNM7Z5XDQ","created_at":"2026-07-05T09:32:13.619250+00:00"},{"alias_kind":"pith_short_8","alias_value":"RBTMX6SK","created_at":"2026-07-05T09:32:13.619250+00:00"}],"events":[],"event_summary":{},"paper_claims":[],"inbound_citations":{"count":1,"internal_anchor_count":0,"sample":[{"citing_arxiv_id":"2604.16686","citing_title":"No-Worse Context-Aware Decoding: Preventing Neutral Regression in Context-Conditioned Generation","ref_index":17,"is_internal_anchor":false}]},"formal_canon":{"evidence_count":0,"sample":[],"anchors":[]},"links":{"html":"https://pith.science/pith/RBTMX6SKNM7Z5XDQMPOBXKNS3T","json":"https://pith.science/pith/RBTMX6SKNM7Z5XDQMPOBXKNS3T.json","graph_json":"https://pith.science/api/pith-number/RBTMX6SKNM7Z5XDQMPOBXKNS3T/graph.json","events_json":"https://pith.science/api/pith-number/RBTMX6SKNM7Z5XDQMPOBXKNS3T/events.json","paper":"https://pith.science/paper/RBTMX6SK"},"agent_actions":{"view_html":"https://pith.science/pith/RBTMX6SKNM7Z5XDQMPOBXKNS3T","download_json":"https://pith.science/pith/RBTMX6SKNM7Z5XDQMPOBXKNS3T.json","view_paper":"https://pith.science/paper/RBTMX6SK","resolve_alias":"https://pith.science/api/pith-number/resolve?arxiv=2411.02603&json=true","fetch_graph":"https://pith.science/api/pith-number/RBTMX6SKNM7Z5XDQMPOBXKNS3T/graph.json","fetch_events":"https://pith.science/api/pith-number/RBTMX6SKNM7Z5XDQMPOBXKNS3T/events.json","actions":{"anchor_timestamp":"https://pith.science/pith/RBTMX6SKNM7Z5XDQMPOBXKNS3T/action/timestamp_anchor","attest_storage":"https://pith.science/pith/RBTMX6SKNM7Z5XDQMPOBXKNS3T/action/storage_attestation","attest_author":"https://pith.science/pith/RBTMX6SKNM7Z5XDQMPOBXKNS3T/action/author_attestation","sign_citation":"https://pith.science/pith/RBTMX6SKNM7Z5XDQMPOBXKNS3T/action/citation_signature","submit_replication":"https://pith.science/pith/RBTMX6SKNM7Z5XDQMPOBXKNS3T/action/replication_record"}},"created_at":"2026-07-05T09:32:13.619250+00:00","updated_at":"2026-07-05T09:32:13.619250+00:00"}