{"state_type":"pith_open_graph_state","state_version":"1.0","pith_number":"pith:2021:5LZPZHSVHXBKWI5FWWZXGXCA7O","merge_version":"pith-open-graph-merge-v1","event_count":2,"valid_event_count":2,"invalid_event_count":0,"equivocation_count":0,"current":{"canonical_record":{"metadata":{"abstract_canon_sha256":"0f8f89fb2776d6a62926c97f953bf638cb6077eac93b4d70acf051c1c8c31444","cross_cats_sorted":[],"license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","primary_cat":"cs.CL","submitted_at":"2021-04-05T20:36:11Z","title_canon_sha256":"26ef90efc21489307c3471cd7281ed3e8dd1c41dd229742e2a8ca03f6106a117"},"schema_version":"1.0","source":{"id":"2104.02145","kind":"arxiv","version":3}},"source_aliases":[{"alias_kind":"arxiv","alias_value":"2104.02145","created_at":"2026-07-05T03:23:09Z"},{"alias_kind":"arxiv_version","alias_value":"2104.02145v3","created_at":"2026-07-05T03:23:09Z"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.2104.02145","created_at":"2026-07-05T03:23:09Z"},{"alias_kind":"pith_short_12","alias_value":"5LZPZHSVHXBK","created_at":"2026-07-05T03:23:09Z"},{"alias_kind":"pith_short_16","alias_value":"5LZPZHSVHXBKWI5F","created_at":"2026-07-05T03:23:09Z"},{"alias_kind":"pith_short_8","alias_value":"5LZPZHSV","created_at":"2026-07-05T03:23:09Z"}],"graph_snapshots":[{"event_id":"sha256:3de670fb83d186a8e3bafe096f274edfe1faf28aab31e5eb20338848f65df05a","target":"graph","created_at":"2026-07-05T03:23:09Z","signer":{"key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signer_id":"pith.science","signer_type":"pith_registry"},"payload":{"graph_snapshot":{"author_claims":{"count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","strong_count":0},"builder_version":"pith-number-builder-2026-05-17-v1","claims":{"count":0,"items":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"integrity":{"available":true,"clean":true,"detectors_run":[],"endpoint":"/pith/2104.02145/integrity.json","findings":[],"snapshot_sha256":"c28c3603d3b5d939e8dc4c7e95fa8dfce3d595e45f758748cecf8e644a296938","summary":{"advisory":0,"by_detector":{},"critical":0,"informational":0}},"paper":{"abstract_excerpt":"Evaluation for many natural language understanding (NLU) tasks is broken: Unreliable and biased systems score so highly on standard benchmarks that there is little room for researchers who develop better systems to demonstrate their improvements. The recent trend to abandon IID benchmarks in favor of adversarially-constructed, out-of-distribution test sets ensures that current models will perform poorly, but ultimately only obscures the abilities that we want our benchmarks to measure. In this position paper, we lay out four criteria that we argue NLU benchmarks should meet. We argue most curr","authors_text":"George E. Dahl, Samuel R. Bowman","cross_cats":[],"headline":"","license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","primary_cat":"cs.CL","submitted_at":"2021-04-05T20:36:11Z","title":"What Will it Take to Fix Benchmarking in Natural Language Understanding?"},"references":{"count":0,"internal_anchors":0,"resolved_work":0,"sample":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"2104.02145","kind":"arxiv","version":3},"verdict":{"created_at":null,"id":null,"model_set":{},"one_line_summary":"","pipeline_version":null,"pith_extraction_headline":"","strongest_claim":"","weakest_assumption":""}},"verdict_id":null}}],"author_attestations":[],"timestamp_anchors":[],"storage_attestations":[],"citation_signatures":[],"replication_records":[],"corrections":[],"mirror_hints":[],"record_created":{"event_id":"sha256:2ee16b596a7866a3f4f4039090262e157803a562b309645df34486d01f189a9b","target":"record","created_at":"2026-07-05T03:23:09Z","signer":{"key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signer_id":"pith.science","signer_type":"pith_registry"},"payload":{"attestation_state":"computed","canonical_record":{"metadata":{"abstract_canon_sha256":"0f8f89fb2776d6a62926c97f953bf638cb6077eac93b4d70acf051c1c8c31444","cross_cats_sorted":[],"license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","primary_cat":"cs.CL","submitted_at":"2021-04-05T20:36:11Z","title_canon_sha256":"26ef90efc21489307c3471cd7281ed3e8dd1c41dd229742e2a8ca03f6106a117"},"schema_version":"1.0","source":{"id":"2104.02145","kind":"arxiv","version":3}},"canonical_sha256":"eaf2fc9e553dc2ab23a5b5b3735c40fba312f271715110d696bb5b367a984069","receipt":{"algorithm":"ed25519","builder_version":"pith-number-builder-2026-05-17-v1","canonical_sha256":"eaf2fc9e553dc2ab23a5b5b3735c40fba312f271715110d696bb5b367a984069","first_computed_at":"2026-07-05T03:23:09.094668Z","key_id":"pith-v1-2026-05","kind":"pith_receipt","last_reissued_at":"2026-07-05T03:23:09.094668Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","receipt_version":"0.3","signature_b64":"/vgxSEx5v+Me+l3WgZx8KNwZqOdQ/YtlpYiGAZycCwW+aQZrYGfSitPWR4j/mDXkttYTaD+zkJKQiCrPMbr/CA==","signature_status":"signed_v1","signed_at":"2026-07-05T03:23:09.095123Z","signed_message":"canonical_sha256_bytes"},"source_id":"2104.02145","source_kind":"arxiv","source_version":3}}},"equivocations":[],"invalid_events":[],"applied_event_ids":["sha256:2ee16b596a7866a3f4f4039090262e157803a562b309645df34486d01f189a9b","sha256:3de670fb83d186a8e3bafe096f274edfe1faf28aab31e5eb20338848f65df05a"],"state_sha256":"8f64ae991f32e655b34472274f30920955197d46420aef95528c5e4de0033d2b"}