{"bundle_type":"pith_open_graph_bundle","bundle_version":"1.0","pith_number":"pith:2021:IFPYM52XEP34DMTCINOO2PGOIV","short_pith_number":"pith:IFPYM52X","canonical_record":{"source":{"id":"2109.07958","kind":"arxiv","version":2},"metadata":{"license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","primary_cat":"cs.CL","submitted_at":"2021-09-08T17:15:27Z","cross_cats_sorted":["cs.AI","cs.CY","cs.LG"],"title_canon_sha256":"2d11b4bac2c042daee54b5f544ebd32f698aef6baff5a03635cd35a7dc8c1516","abstract_canon_sha256":"d02dd95148184519ea9626e9d10f61710729a05170315c500810c89935071097"},"schema_version":"1.0"},"canonical_sha256":"415f86775723f7c1b262435ced3cce45520faf3c6b360f778301d450786f6caf","source":{"kind":"arxiv","id":"2109.07958","version":2},"source_aliases":[{"alias_kind":"arxiv","alias_value":"2109.07958","created_at":"2026-07-05T04:21:05Z"},{"alias_kind":"arxiv_version","alias_value":"2109.07958v2","created_at":"2026-07-05T04:21:05Z"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.2109.07958","created_at":"2026-07-05T04:21:05Z"},{"alias_kind":"pith_short_12","alias_value":"IFPYM52XEP34","created_at":"2026-07-05T04:21:05Z"},{"alias_kind":"pith_short_16","alias_value":"IFPYM52XEP34DMTC","created_at":"2026-07-05T04:21:05Z"},{"alias_kind":"pith_short_8","alias_value":"IFPYM52X","created_at":"2026-07-05T04:21:05Z"}],"events":[{"event_type":"record_created","subject_pith_number":"pith:2021:IFPYM52XEP34DMTCINOO2PGOIV","target":"record","payload":{"canonical_record":{"source":{"id":"2109.07958","kind":"arxiv","version":2},"metadata":{"license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","primary_cat":"cs.CL","submitted_at":"2021-09-08T17:15:27Z","cross_cats_sorted":["cs.AI","cs.CY","cs.LG"],"title_canon_sha256":"2d11b4bac2c042daee54b5f544ebd32f698aef6baff5a03635cd35a7dc8c1516","abstract_canon_sha256":"d02dd95148184519ea9626e9d10f61710729a05170315c500810c89935071097"},"schema_version":"1.0"},"canonical_sha256":"415f86775723f7c1b262435ced3cce45520faf3c6b360f778301d450786f6caf","receipt":{"kind":"pith_receipt","key_id":"pith-v1-2026-05","algorithm":"ed25519","signed_at":"2026-07-05T04:21:05.806044Z","signature_b64":"f2cVhGQHmROrxhFzXkP3GnApX/4Pg7lDNbi6398bMzg072iWriwfwvZopbP22FWLzcQrM+Zc1t97Cw4Ln1fBDg==","signed_message":"canonical_sha256_bytes","builder_version":"pith-number-builder-2026-05-17-v1","receipt_version":"0.3","canonical_sha256":"415f86775723f7c1b262435ced3cce45520faf3c6b360f778301d450786f6caf","last_reissued_at":"2026-07-05T04:21:05.805534Z","signature_status":"signed_v1","first_computed_at":"2026-07-05T04:21:05.805534Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"source_kind":"arxiv","source_id":"2109.07958","source_version":2,"attestation_state":"computed"},"signer":{"signer_id":"pith.science","signer_type":"pith_registry","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"created_at":"2026-07-05T04:21:05Z","supersedes":[],"prev_event":null,"signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"1Ges7GbuNQHgVVqxbgaIDE/1WFFtLLCrgZalxG22G4Ufy+2eneWdbrgTXzHK55TMcIbLAizuJKi3pPop5/KnBQ==","signed_message":"open_graph_event_sha256_bytes","signed_at":"2026-07-09T03:56:30.048437Z"},"content_sha256":"23bf463852a6cbe1e4bde5cf679261b75ad18d32fad7c24e6b0368e7025f530e","schema_version":"1.0","event_id":"sha256:23bf463852a6cbe1e4bde5cf679261b75ad18d32fad7c24e6b0368e7025f530e"},{"event_type":"graph_snapshot","subject_pith_number":"pith:2021:IFPYM52XEP34DMTCINOO2PGOIV","target":"graph","payload":{"graph_snapshot":{"paper":{"title":"TruthfulQA: Measuring How Models Mimic Human Falsehoods","license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","headline":"Language models repeat human misconceptions more as they get larger, according to a new benchmark of 817 questions.","cross_cats":["cs.AI","cs.CY","cs.LG"],"primary_cat":"cs.CL","authors_text":"Jacob Hilton, Owain Evans, Stephanie Lin","submitted_at":"2021-09-08T17:15:27Z","abstract_excerpt":"We propose a benchmark to measure whether a language model is truthful in generating answers to questions. The benchmark comprises 817 questions that span 38 categories, including health, law, finance and politics. We crafted questions that some humans would answer falsely due to a false belief or misconception. To perform well, models must avoid generating false answers learned from imitating human texts. We tested GPT-3, GPT-Neo/J, GPT-2 and a T5-based model. The best model was truthful on 58% of questions, while human performance was 94%. Models generated many false answers that mimic popul"},"claims":{"count":4,"items":[{"kind":"strongest_claim","text":"The largest models were generally the least truthful. This contrasts with other NLP tasks, where performance improves with model size. However, this result is expected if false answers are learned from the training distribution.","source":"verdict.strongest_claim","status":"machine_extracted","claim_id":"C1","attestation":"unclaimed"},{"kind":"weakest_assumption","text":"That the 817 questions accurately capture misconceptions that models learn from training data rather than other factors, and that avoiding these specific false answers measures general truthfulness.","source":"verdict.weakest_assumption","status":"machine_extracted","claim_id":"C2","attestation":"unclaimed"},{"kind":"one_line_summary","text":"A new benchmark reveals that language models including GPT-3 are truthful on only 58% of questions designed to elicit popular misconceptions, far below human performance of 94%, with larger models performing worse.","source":"verdict.one_line_summary","status":"machine_extracted","claim_id":"C3","attestation":"unclaimed"},{"kind":"headline","text":"Language models repeat human misconceptions more as they get larger, according to a new benchmark of 817 questions.","source":"verdict.pith_extraction.headline","status":"machine_extracted","claim_id":"C4","attestation":"unclaimed"}],"snapshot_sha256":"44c3daac860fc63f14fd075fa42f59abe2c8112143195c7d66688e69251d4043"},"source":{"id":"2109.07958","kind":"arxiv","version":2},"verdict":{"id":"1d13e202-a3d0-40b4-a310-18592e0aa91d","model_set":{"reader":"grok-4.3"},"created_at":"2026-05-11T21:44:11.773355Z","strongest_claim":"The largest models were generally the least truthful. This contrasts with other NLP tasks, where performance improves with model size. However, this result is expected if false answers are learned from the training distribution.","one_line_summary":"A new benchmark reveals that language models including GPT-3 are truthful on only 58% of questions designed to elicit popular misconceptions, far below human performance of 94%, with larger models performing worse.","pipeline_version":"pith-pipeline@v0.9.0","weakest_assumption":"That the 817 questions accurately capture misconceptions that models learn from training data rather than other factors, and that avoiding these specific false answers measures general truthfulness.","pith_extraction_headline":"Language models repeat human misconceptions more as they get larger, according to a new benchmark of 817 questions."},"integrity":{"clean":true,"summary":{"advisory":0,"critical":0,"by_detector":{},"informational":0},"endpoint":"/pith/2109.07958/integrity.json","findings":[],"available":true,"detectors_run":[],"snapshot_sha256":"c28c3603d3b5d939e8dc4c7e95fa8dfce3d595e45f758748cecf8e644a296938"},"references":{"count":16,"sample":[{"doi":"","year":2018,"title":"A General Language Assistant as a Laboratory for Alignment","work_id":"a43f9ea0-01be-47d5-b8ee-a1a9f73381c5","ref_index":1,"cited_arxiv_id":"2112.00861","is_internal_anchor":true},{"doi":"","year":2018,"title":"Evaluating Large Language Models Trained on Code","work_id":"042493e9-b26f-4b4e-bbde-382072ca9b08","ref_index":2,"cited_arxiv_id":"2107.03374","is_internal_anchor":true},{"doi":"","year":2021,"title":"Retrieval-Augmented Generation for Knowledge-Intensive NLP Tasks","work_id":"27eaec54-c105-4969-8188-da5f0fca3688","ref_index":3,"cited_arxiv_id":"2005.11401","is_internal_anchor":true},{"doi":"","year":2019,"title":"arXiv preprint arXiv:2105.11447 , year=","work_id":"4b0122d4-63a1-4e63-8f4b-e70d000c195c","ref_index":4,"cited_arxiv_id":"","is_internal_anchor":false},{"doi":"","year":2021,"title":"Retrieval augmentation reduces hallucination in conversation","work_id":"764f4422-9533-439b-857b-07eab22d3719","ref_index":5,"cited_arxiv_id":"","is_internal_anchor":false}],"resolved_work":16,"snapshot_sha256":"cab272164c1387d5866cf3ebe96b7e7e83746d422db67f5ed6a337f3cdbdf45f","internal_anchors":3},"formal_canon":{"evidence_count":2,"snapshot_sha256":"2351ebcf543ca2ee775a538c259060b93769847ed4fc11e41b67d852b26b44d7"},"author_claims":{"count":0,"strong_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"builder_version":"pith-number-builder-2026-05-17-v1"},"verdict_id":"1d13e202-a3d0-40b4-a310-18592e0aa91d"},"signer":{"signer_id":"pith.science","signer_type":"pith_registry","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"created_at":"2026-07-05T04:21:05Z","supersedes":[],"prev_event":null,"signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"D2ldS4FEVrpeDJParVnQwKyY9Fw9TH6TWEdNz7NmG+Lh0Dt3+7KgCqk3Dq306DCE7B97mY44243QfI3X8vWQDg==","signed_message":"open_graph_event_sha256_bytes","signed_at":"2026-07-09T03:56:30.048973Z"},"content_sha256":"adbe3bd29e362860d0c93a6cbcef2356ce6d6222cce77d2a86832991e0d360ac","schema_version":"1.0","event_id":"sha256:adbe3bd29e362860d0c93a6cbcef2356ce6d6222cce77d2a86832991e0d360ac"}],"timestamp_proofs":[],"mirror_hints":[{"mirror_type":"https","name":"Pith Resolver","base_url":"https://pith.science","bundle_url":"https://pith.science/pith/IFPYM52XEP34DMTCINOO2PGOIV/bundle.json","state_url":"https://pith.science/pith/IFPYM52XEP34DMTCINOO2PGOIV/state.json","well_known_bundle_url":"https://pith.science/.well-known/pith/IFPYM52XEP34DMTCINOO2PGOIV/bundle.json","status":"primary"}],"public_keys":[{"key_id":"pith-v1-2026-05","algorithm":"ed25519","format":"raw","public_key_b64":"stVStoiQhXFxp4s2pdzPNoqVNBMojDU/fJ2db5S3CbM=","public_key_hex":"b2d552b68890857171a78b36a5dccf368a953413288c353f7c9d9d6f94b709b3","fingerprint_sha256_b32_first128bits":"RVFV5Z2OI2J3ZUO7ERDEBCYNKS","fingerprint_sha256_hex":"8d4b5ee74e4693bcd1df2446408b0d54","rotates_at":null,"url":"https://pith.science/pith-signing-key.json","notes":"Pith uses this Ed25519 key to sign canonical record SHA-256 digests. Verify with: ed25519_verify(public_key, message=canonical_sha256_bytes, signature=base64decode(signature_b64))."}],"merge_version":"pith-open-graph-merge-v1","built_at":"2026-07-09T03:56:30Z","links":{"resolver":"https://pith.science/pith/IFPYM52XEP34DMTCINOO2PGOIV","bundle":"https://pith.science/pith/IFPYM52XEP34DMTCINOO2PGOIV/bundle.json","state":"https://pith.science/pith/IFPYM52XEP34DMTCINOO2PGOIV/state.json","well_known_bundle":"https://pith.science/.well-known/pith/IFPYM52XEP34DMTCINOO2PGOIV/bundle.json"},"state":{"state_type":"pith_open_graph_state","state_version":"1.0","pith_number":"pith:2021:IFPYM52XEP34DMTCINOO2PGOIV","merge_version":"pith-open-graph-merge-v1","event_count":2,"valid_event_count":2,"invalid_event_count":0,"equivocation_count":0,"current":{"canonical_record":{"metadata":{"abstract_canon_sha256":"d02dd95148184519ea9626e9d10f61710729a05170315c500810c89935071097","cross_cats_sorted":["cs.AI","cs.CY","cs.LG"],"license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","primary_cat":"cs.CL","submitted_at":"2021-09-08T17:15:27Z","title_canon_sha256":"2d11b4bac2c042daee54b5f544ebd32f698aef6baff5a03635cd35a7dc8c1516"},"schema_version":"1.0","source":{"id":"2109.07958","kind":"arxiv","version":2}},"source_aliases":[{"alias_kind":"arxiv","alias_value":"2109.07958","created_at":"2026-07-05T04:21:05Z"},{"alias_kind":"arxiv_version","alias_value":"2109.07958v2","created_at":"2026-07-05T04:21:05Z"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.2109.07958","created_at":"2026-07-05T04:21:05Z"},{"alias_kind":"pith_short_12","alias_value":"IFPYM52XEP34","created_at":"2026-07-05T04:21:05Z"},{"alias_kind":"pith_short_16","alias_value":"IFPYM52XEP34DMTC","created_at":"2026-07-05T04:21:05Z"},{"alias_kind":"pith_short_8","alias_value":"IFPYM52X","created_at":"2026-07-05T04:21:05Z"}],"graph_snapshots":[{"event_id":"sha256:adbe3bd29e362860d0c93a6cbcef2356ce6d6222cce77d2a86832991e0d360ac","target":"graph","created_at":"2026-07-05T04:21:05Z","signer":{"key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signer_id":"pith.science","signer_type":"pith_registry"},"payload":{"graph_snapshot":{"author_claims":{"count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","strong_count":0},"builder_version":"pith-number-builder-2026-05-17-v1","claims":{"count":4,"items":[{"attestation":"unclaimed","claim_id":"C1","kind":"strongest_claim","source":"verdict.strongest_claim","status":"machine_extracted","text":"The largest models were generally the least truthful. This contrasts with other NLP tasks, where performance improves with model size. However, this result is expected if false answers are learned from the training distribution."},{"attestation":"unclaimed","claim_id":"C2","kind":"weakest_assumption","source":"verdict.weakest_assumption","status":"machine_extracted","text":"That the 817 questions accurately capture misconceptions that models learn from training data rather than other factors, and that avoiding these specific false answers measures general truthfulness."},{"attestation":"unclaimed","claim_id":"C3","kind":"one_line_summary","source":"verdict.one_line_summary","status":"machine_extracted","text":"A new benchmark reveals that language models including GPT-3 are truthful on only 58% of questions designed to elicit popular misconceptions, far below human performance of 94%, with larger models performing worse."},{"attestation":"unclaimed","claim_id":"C4","kind":"headline","source":"verdict.pith_extraction.headline","status":"machine_extracted","text":"Language models repeat human misconceptions more as they get larger, according to a new benchmark of 817 questions."}],"snapshot_sha256":"44c3daac860fc63f14fd075fa42f59abe2c8112143195c7d66688e69251d4043"},"formal_canon":{"evidence_count":2,"snapshot_sha256":"2351ebcf543ca2ee775a538c259060b93769847ed4fc11e41b67d852b26b44d7"},"integrity":{"available":true,"clean":true,"detectors_run":[],"endpoint":"/pith/2109.07958/integrity.json","findings":[],"snapshot_sha256":"c28c3603d3b5d939e8dc4c7e95fa8dfce3d595e45f758748cecf8e644a296938","summary":{"advisory":0,"by_detector":{},"critical":0,"informational":0}},"paper":{"abstract_excerpt":"We propose a benchmark to measure whether a language model is truthful in generating answers to questions. The benchmark comprises 817 questions that span 38 categories, including health, law, finance and politics. We crafted questions that some humans would answer falsely due to a false belief or misconception. To perform well, models must avoid generating false answers learned from imitating human texts. We tested GPT-3, GPT-Neo/J, GPT-2 and a T5-based model. The best model was truthful on 58% of questions, while human performance was 94%. Models generated many false answers that mimic popul","authors_text":"Jacob Hilton, Owain Evans, Stephanie Lin","cross_cats":["cs.AI","cs.CY","cs.LG"],"headline":"Language models repeat human misconceptions more as they get larger, according to a new benchmark of 817 questions.","license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","primary_cat":"cs.CL","submitted_at":"2021-09-08T17:15:27Z","title":"TruthfulQA: Measuring How Models Mimic Human Falsehoods"},"references":{"count":16,"internal_anchors":3,"resolved_work":16,"sample":[{"cited_arxiv_id":"2112.00861","doi":"","is_internal_anchor":true,"ref_index":1,"title":"A General Language Assistant as a Laboratory for Alignment","work_id":"a43f9ea0-01be-47d5-b8ee-a1a9f73381c5","year":2018},{"cited_arxiv_id":"2107.03374","doi":"","is_internal_anchor":true,"ref_index":2,"title":"Evaluating Large Language Models Trained on Code","work_id":"042493e9-b26f-4b4e-bbde-382072ca9b08","year":2018},{"cited_arxiv_id":"2005.11401","doi":"","is_internal_anchor":true,"ref_index":3,"title":"Retrieval-Augmented Generation for Knowledge-Intensive NLP Tasks","work_id":"27eaec54-c105-4969-8188-da5f0fca3688","year":2021},{"cited_arxiv_id":"","doi":"","is_internal_anchor":false,"ref_index":4,"title":"arXiv preprint arXiv:2105.11447 , year=","work_id":"4b0122d4-63a1-4e63-8f4b-e70d000c195c","year":2019},{"cited_arxiv_id":"","doi":"","is_internal_anchor":false,"ref_index":5,"title":"Retrieval augmentation reduces hallucination in conversation","work_id":"764f4422-9533-439b-857b-07eab22d3719","year":2021}],"snapshot_sha256":"cab272164c1387d5866cf3ebe96b7e7e83746d422db67f5ed6a337f3cdbdf45f"},"source":{"id":"2109.07958","kind":"arxiv","version":2},"verdict":{"created_at":"2026-05-11T21:44:11.773355Z","id":"1d13e202-a3d0-40b4-a310-18592e0aa91d","model_set":{"reader":"grok-4.3"},"one_line_summary":"A new benchmark reveals that language models including GPT-3 are truthful on only 58% of questions designed to elicit popular misconceptions, far below human performance of 94%, with larger models performing worse.","pipeline_version":"pith-pipeline@v0.9.0","pith_extraction_headline":"Language models repeat human misconceptions more as they get larger, according to a new benchmark of 817 questions.","strongest_claim":"The largest models were generally the least truthful. This contrasts with other NLP tasks, where performance improves with model size. However, this result is expected if false answers are learned from the training distribution.","weakest_assumption":"That the 817 questions accurately capture misconceptions that models learn from training data rather than other factors, and that avoiding these specific false answers measures general truthfulness."}},"verdict_id":"1d13e202-a3d0-40b4-a310-18592e0aa91d"}}],"author_attestations":[],"timestamp_anchors":[],"storage_attestations":[],"citation_signatures":[],"replication_records":[],"corrections":[],"mirror_hints":[],"record_created":{"event_id":"sha256:23bf463852a6cbe1e4bde5cf679261b75ad18d32fad7c24e6b0368e7025f530e","target":"record","created_at":"2026-07-05T04:21:05Z","signer":{"key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signer_id":"pith.science","signer_type":"pith_registry"},"payload":{"attestation_state":"computed","canonical_record":{"metadata":{"abstract_canon_sha256":"d02dd95148184519ea9626e9d10f61710729a05170315c500810c89935071097","cross_cats_sorted":["cs.AI","cs.CY","cs.LG"],"license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","primary_cat":"cs.CL","submitted_at":"2021-09-08T17:15:27Z","title_canon_sha256":"2d11b4bac2c042daee54b5f544ebd32f698aef6baff5a03635cd35a7dc8c1516"},"schema_version":"1.0","source":{"id":"2109.07958","kind":"arxiv","version":2}},"canonical_sha256":"415f86775723f7c1b262435ced3cce45520faf3c6b360f778301d450786f6caf","receipt":{"algorithm":"ed25519","builder_version":"pith-number-builder-2026-05-17-v1","canonical_sha256":"415f86775723f7c1b262435ced3cce45520faf3c6b360f778301d450786f6caf","first_computed_at":"2026-07-05T04:21:05.805534Z","key_id":"pith-v1-2026-05","kind":"pith_receipt","last_reissued_at":"2026-07-05T04:21:05.805534Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","receipt_version":"0.3","signature_b64":"f2cVhGQHmROrxhFzXkP3GnApX/4Pg7lDNbi6398bMzg072iWriwfwvZopbP22FWLzcQrM+Zc1t97Cw4Ln1fBDg==","signature_status":"signed_v1","signed_at":"2026-07-05T04:21:05.806044Z","signed_message":"canonical_sha256_bytes"},"source_id":"2109.07958","source_kind":"arxiv","source_version":2}}},"equivocations":[],"invalid_events":[],"applied_event_ids":["sha256:23bf463852a6cbe1e4bde5cf679261b75ad18d32fad7c24e6b0368e7025f530e","sha256:adbe3bd29e362860d0c93a6cbcef2356ce6d6222cce77d2a86832991e0d360ac"],"state_sha256":"716e2372f2785f843c30ea27b8ae12403c0268c7e4191c5979e0f8ff7d3e4de5"},"bundle_signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"sIJFa2aAfUVgj7yY+atTXWm+S5rbuS7k+Shy64cLkmWwP3fJ4P+19OdcKiSTxXObAK4/u7YN0YUDUSe4vCIfAA==","signed_message":"bundle_sha256_bytes","signed_at":"2026-07-09T03:56:30.051347Z","bundle_sha256":"571d8baf51ee2ab6387a2df5793cadab2c4a90a304542d97d591f7887b224593"}}