{"record_type":"pith_number_record","schema_url":"https://pith.science/schemas/pith-number/v1.json","pith_number":"pith:2023:YT4J2VFQACPFACWAI3MVOBTK7F","short_pith_number":"pith:YT4J2VFQ","schema_version":"1.0","canonical_sha256":"c4f89d54b0009e500ac046d957066af966258485bc64b9bb7828c6fb47e368d7","source":{"kind":"arxiv","id":"2307.08153","version":4},"attestation_state":"computed","paper":{"title":"Analyzing Dataset Annotation Quality Management in the Wild","license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","headline":"","cross_cats":[],"primary_cat":"cs.CL","authors_text":"Iryna Gurevych, Jan-Christoph Klie, Richard Eckart de Castilho","submitted_at":"2023-07-16T21:22:40Z","abstract_excerpt":"Data quality is crucial for training accurate, unbiased, and trustworthy machine learning models as well as for their correct evaluation. Recent works, however, have shown that even popular datasets used to train and evaluate state-of-the-art models contain a non-negligible amount of erroneous annotations, biases, or artifacts. While practices and guidelines regarding dataset creation projects exist, to our knowledge, large-scale analysis has yet to be performed on how quality management is conducted when creating natural language datasets and whether these recommendations are followed. Theref"},"verification_status":{"content_addressed":true,"pith_receipt":true,"author_attested":false,"weak_author_claims":0,"strong_author_claims":0,"externally_anchored":false,"storage_verified":false,"citation_signatures":0,"replication_records":0,"graph_snapshot":true,"references_resolved":false,"formal_links_present":false},"canonical_record":{"source":{"id":"2307.08153","kind":"arxiv","version":4},"metadata":{"license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","primary_cat":"cs.CL","submitted_at":"2023-07-16T21:22:40Z","cross_cats_sorted":[],"title_canon_sha256":"f24774bd8d5c06269cd5fff96eabf959c436b4a4ce1c908039ee0b640d8f37b6","abstract_canon_sha256":"2ba3e4625391ec8a9135e7eed488b07e621ccdcd20aa6582cbcb54cd88b81018"},"schema_version":"1.0"},"receipt":{"kind":"pith_receipt","key_id":"pith-v1-2026-05","algorithm":"ed25519","signed_at":"2026-07-05T07:54:02.235904Z","signature_b64":"TGWkzyWIbmA/0Ep2gM6DBl32T5TBB3MjizUvidUNCHiMCXZGkmEImL18AYLcTVGgJzZGOk4TzPoJpS5p31+7Cw==","signed_message":"canonical_sha256_bytes","builder_version":"pith-number-builder-2026-05-17-v1","receipt_version":"0.3","canonical_sha256":"c4f89d54b0009e500ac046d957066af966258485bc64b9bb7828c6fb47e368d7","last_reissued_at":"2026-07-05T07:54:02.235557Z","signature_status":"signed_v1","first_computed_at":"2026-07-05T07:54:02.235557Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"graph_snapshot":{"paper":{"title":"Analyzing Dataset Annotation Quality Management in the Wild","license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","headline":"","cross_cats":[],"primary_cat":"cs.CL","authors_text":"Iryna Gurevych, Jan-Christoph Klie, Richard Eckart de Castilho","submitted_at":"2023-07-16T21:22:40Z","abstract_excerpt":"Data quality is crucial for training accurate, unbiased, and trustworthy machine learning models as well as for their correct evaluation. Recent works, however, have shown that even popular datasets used to train and evaluate state-of-the-art models contain a non-negligible amount of erroneous annotations, biases, or artifacts. While practices and guidelines regarding dataset creation projects exist, to our knowledge, large-scale analysis has yet to be performed on how quality management is conducted when creating natural language datasets and whether these recommendations are followed. Theref"},"claims":{"count":0,"items":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"2307.08153","kind":"arxiv","version":4},"verdict":{"id":null,"model_set":{},"created_at":null,"strongest_claim":"","one_line_summary":"","pipeline_version":null,"weakest_assumption":"","pith_extraction_headline":""},"integrity":{"clean":true,"summary":{"advisory":0,"critical":0,"by_detector":{},"informational":0},"endpoint":"/pith/2307.08153/integrity.json","findings":[],"available":true,"detectors_run":[],"snapshot_sha256":"c28c3603d3b5d939e8dc4c7e95fa8dfce3d595e45f758748cecf8e644a296938"},"references":{"count":0,"sample":[],"resolved_work":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","internal_anchors":0},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"author_claims":{"count":0,"strong_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"builder_version":"pith-number-builder-2026-05-17-v1"},"aliases":[{"alias_kind":"arxiv","alias_value":"2307.08153","created_at":"2026-07-05T07:54:02.235614+00:00"},{"alias_kind":"arxiv_version","alias_value":"2307.08153v4","created_at":"2026-07-05T07:54:02.235614+00:00"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.2307.08153","created_at":"2026-07-05T07:54:02.235614+00:00"},{"alias_kind":"pith_short_12","alias_value":"YT4J2VFQACPF","created_at":"2026-07-05T07:54:02.235614+00:00"},{"alias_kind":"pith_short_16","alias_value":"YT4J2VFQACPFACWA","created_at":"2026-07-05T07:54:02.235614+00:00"},{"alias_kind":"pith_short_8","alias_value":"YT4J2VFQ","created_at":"2026-07-05T07:54:02.235614+00:00"}],"events":[],"event_summary":{},"paper_claims":[],"inbound_citations":{"count":1,"internal_anchor_count":1,"sample":[{"citing_arxiv_id":"2411.11081","citing_title":"The Promises and Pitfalls of LLM Annotations in Dataset Labeling: a Case Study on Media Bias Detection","ref_index":21,"is_internal_anchor":true}]},"formal_canon":{"evidence_count":0,"sample":[],"anchors":[]},"links":{"html":"https://pith.science/pith/YT4J2VFQACPFACWAI3MVOBTK7F","json":"https://pith.science/pith/YT4J2VFQACPFACWAI3MVOBTK7F.json","graph_json":"https://pith.science/api/pith-number/YT4J2VFQACPFACWAI3MVOBTK7F/graph.json","events_json":"https://pith.science/api/pith-number/YT4J2VFQACPFACWAI3MVOBTK7F/events.json","paper":"https://pith.science/paper/YT4J2VFQ"},"agent_actions":{"view_html":"https://pith.science/pith/YT4J2VFQACPFACWAI3MVOBTK7F","download_json":"https://pith.science/pith/YT4J2VFQACPFACWAI3MVOBTK7F.json","view_paper":"https://pith.science/paper/YT4J2VFQ","resolve_alias":"https://pith.science/api/pith-number/resolve?arxiv=2307.08153&json=true","fetch_graph":"https://pith.science/api/pith-number/YT4J2VFQACPFACWAI3MVOBTK7F/graph.json","fetch_events":"https://pith.science/api/pith-number/YT4J2VFQACPFACWAI3MVOBTK7F/events.json","actions":{"anchor_timestamp":"https://pith.science/pith/YT4J2VFQACPFACWAI3MVOBTK7F/action/timestamp_anchor","attest_storage":"https://pith.science/pith/YT4J2VFQACPFACWAI3MVOBTK7F/action/storage_attestation","attest_author":"https://pith.science/pith/YT4J2VFQACPFACWAI3MVOBTK7F/action/author_attestation","sign_citation":"https://pith.science/pith/YT4J2VFQACPFACWAI3MVOBTK7F/action/citation_signature","submit_replication":"https://pith.science/pith/YT4J2VFQACPFACWAI3MVOBTK7F/action/replication_record"}},"created_at":"2026-07-05T07:54:02.235614+00:00","updated_at":"2026-07-05T07:54:02.235614+00:00"}