{"record_type":"pith_number_record","schema_url":"https://pith.science/schemas/pith-number/v1.json","pith_number":"pith:2022:ZQQQKQOYYL4DX7GDGGRTCW4QC7","short_pith_number":"pith:ZQQQKQOY","schema_version":"1.0","canonical_sha256":"cc210541d8c2f83bfcc331a3315b9017db870ba3a06b90c69be000461d6c191d","source":{"kind":"arxiv","id":"2202.06935","version":1},"attestation_state":"computed","paper":{"title":"Repairing the Cracked Foundation: A Survey of Obstacles in Evaluation Practices for Generated Text","license":"http://creativecommons.org/licenses/by-sa/4.0/","headline":"","cross_cats":["cs.AI","cs.LG"],"primary_cat":"cs.CL","authors_text":"Elizabeth Clark, Sebastian Gehrmann, Thibault Sellam","submitted_at":"2022-02-14T18:51:07Z","abstract_excerpt":"Evaluation practices in natural language generation (NLG) have many known flaws, but improved evaluation approaches are rarely widely adopted. This issue has become more urgent, since neural NLG models have improved to the point where they can often no longer be distinguished based on the surface-level features that older metrics rely on. This paper surveys the issues with human and automatic model evaluations and with commonly used datasets in NLG that have been pointed out over the past 20 years. We summarize, categorize, and discuss how researchers have been addressing these issues and what"},"verification_status":{"content_addressed":true,"pith_receipt":true,"author_attested":false,"weak_author_claims":0,"strong_author_claims":0,"externally_anchored":false,"storage_verified":false,"citation_signatures":0,"replication_records":0,"graph_snapshot":true,"references_resolved":false,"formal_links_present":false},"canonical_record":{"source":{"id":"2202.06935","kind":"arxiv","version":1},"metadata":{"license":"http://creativecommons.org/licenses/by-sa/4.0/","primary_cat":"cs.CL","submitted_at":"2022-02-14T18:51:07Z","cross_cats_sorted":["cs.AI","cs.LG"],"title_canon_sha256":"bf3fa504df68bc80cda87d5fab0875097898d2d8a7a8401718b231b0c9b44d1c","abstract_canon_sha256":"0c9d2dc4498b9f91b66a92d6fbc9d8781607f7885c8e6713c0cf0a5cd80f7797"},"schema_version":"1.0"},"receipt":{"kind":"pith_receipt","key_id":"pith-v1-2026-05","algorithm":"ed25519","signed_at":"2026-07-05T03:56:42.771976Z","signature_b64":"+ESSMzqzlDX1AKL2sA9CG/6xKX2sEb3RQLYunWL8pSmC7NS8lwSKb/Iy7OxR2HiqgII4qB1Di7RAmb5e2PIZAw==","signed_message":"canonical_sha256_bytes","builder_version":"pith-number-builder-2026-05-17-v1","receipt_version":"0.3","canonical_sha256":"cc210541d8c2f83bfcc331a3315b9017db870ba3a06b90c69be000461d6c191d","last_reissued_at":"2026-07-05T03:56:42.771575Z","signature_status":"signed_v1","first_computed_at":"2026-07-05T03:56:42.771575Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"graph_snapshot":{"paper":{"title":"Repairing the Cracked Foundation: A Survey of Obstacles in Evaluation Practices for Generated Text","license":"http://creativecommons.org/licenses/by-sa/4.0/","headline":"","cross_cats":["cs.AI","cs.LG"],"primary_cat":"cs.CL","authors_text":"Elizabeth Clark, Sebastian Gehrmann, Thibault Sellam","submitted_at":"2022-02-14T18:51:07Z","abstract_excerpt":"Evaluation practices in natural language generation (NLG) have many known flaws, but improved evaluation approaches are rarely widely adopted. This issue has become more urgent, since neural NLG models have improved to the point where they can often no longer be distinguished based on the surface-level features that older metrics rely on. This paper surveys the issues with human and automatic model evaluations and with commonly used datasets in NLG that have been pointed out over the past 20 years. We summarize, categorize, and discuss how researchers have been addressing these issues and what"},"claims":{"count":0,"items":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"2202.06935","kind":"arxiv","version":1},"verdict":{"id":null,"model_set":{},"created_at":null,"strongest_claim":"","one_line_summary":"","pipeline_version":null,"weakest_assumption":"","pith_extraction_headline":""},"integrity":{"clean":true,"summary":{"advisory":0,"critical":0,"by_detector":{},"informational":0},"endpoint":"/pith/2202.06935/integrity.json","findings":[],"available":true,"detectors_run":[],"snapshot_sha256":"c28c3603d3b5d939e8dc4c7e95fa8dfce3d595e45f758748cecf8e644a296938"},"references":{"count":0,"sample":[],"resolved_work":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","internal_anchors":0},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"author_claims":{"count":0,"strong_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"builder_version":"pith-number-builder-2026-05-17-v1"},"aliases":[{"alias_kind":"arxiv","alias_value":"2202.06935","created_at":"2026-07-05T03:56:42.771633+00:00"},{"alias_kind":"arxiv_version","alias_value":"2202.06935v1","created_at":"2026-07-05T03:56:42.771633+00:00"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.2202.06935","created_at":"2026-07-05T03:56:42.771633+00:00"},{"alias_kind":"pith_short_12","alias_value":"ZQQQKQOYYL4D","created_at":"2026-07-05T03:56:42.771633+00:00"},{"alias_kind":"pith_short_16","alias_value":"ZQQQKQOYYL4DX7GD","created_at":"2026-07-05T03:56:42.771633+00:00"},{"alias_kind":"pith_short_8","alias_value":"ZQQQKQOY","created_at":"2026-07-05T03:56:42.771633+00:00"}],"events":[],"event_summary":{},"paper_claims":[],"inbound_citations":{"count":5,"internal_anchor_count":0,"sample":[{"citing_arxiv_id":"2606.24894","citing_title":"RWGBench: Evaluating Scholarly Positioning in Related Work Generation","ref_index":13,"is_internal_anchor":false},{"citing_arxiv_id":"2303.17564","citing_title":"BloombergGPT: A Large Language Model for Finance","ref_index":34,"is_internal_anchor":false},{"citing_arxiv_id":"2211.05100","citing_title":"BLOOM: A 176B-Parameter Open-Access Multilingual Language Model","ref_index":240,"is_internal_anchor":false},{"citing_arxiv_id":"2211.05100","citing_title":"BLOOM: A 176B-Parameter Open-Access Multilingual Language Model","ref_index":55,"is_internal_anchor":false},{"citing_arxiv_id":"2204.02311","citing_title":"PaLM: Scaling Language Modeling with Pathways","ref_index":53,"is_internal_anchor":false}]},"formal_canon":{"evidence_count":0,"sample":[],"anchors":[]},"links":{"html":"https://pith.science/pith/ZQQQKQOYYL4DX7GDGGRTCW4QC7","json":"https://pith.science/pith/ZQQQKQOYYL4DX7GDGGRTCW4QC7.json","graph_json":"https://pith.science/api/pith-number/ZQQQKQOYYL4DX7GDGGRTCW4QC7/graph.json","events_json":"https://pith.science/api/pith-number/ZQQQKQOYYL4DX7GDGGRTCW4QC7/events.json","paper":"https://pith.science/paper/ZQQQKQOY"},"agent_actions":{"view_html":"https://pith.science/pith/ZQQQKQOYYL4DX7GDGGRTCW4QC7","download_json":"https://pith.science/pith/ZQQQKQOYYL4DX7GDGGRTCW4QC7.json","view_paper":"https://pith.science/paper/ZQQQKQOY","resolve_alias":"https://pith.science/api/pith-number/resolve?arxiv=2202.06935&json=true","fetch_graph":"https://pith.science/api/pith-number/ZQQQKQOYYL4DX7GDGGRTCW4QC7/graph.json","fetch_events":"https://pith.science/api/pith-number/ZQQQKQOYYL4DX7GDGGRTCW4QC7/events.json","actions":{"anchor_timestamp":"https://pith.science/pith/ZQQQKQOYYL4DX7GDGGRTCW4QC7/action/timestamp_anchor","attest_storage":"https://pith.science/pith/ZQQQKQOYYL4DX7GDGGRTCW4QC7/action/storage_attestation","attest_author":"https://pith.science/pith/ZQQQKQOYYL4DX7GDGGRTCW4QC7/action/author_attestation","sign_citation":"https://pith.science/pith/ZQQQKQOYYL4DX7GDGGRTCW4QC7/action/citation_signature","submit_replication":"https://pith.science/pith/ZQQQKQOYYL4DX7GDGGRTCW4QC7/action/replication_record"}},"created_at":"2026-07-05T03:56:42.771633+00:00","updated_at":"2026-07-05T03:56:42.771633+00:00"}