{"state_type":"pith_open_graph_state","state_version":"1.0","pith_number":"pith:2020:AMJ37KAYRMDXB4W5U3NTU6UJGD","merge_version":"pith-open-graph-merge-v1","event_count":2,"valid_event_count":2,"invalid_event_count":0,"equivocation_count":0,"current":{"canonical_record":{"metadata":{"abstract_canon_sha256":"5cbb5295e76de7ee55556cea6316f696fbedd79380f3e7608c7123eb6c670d72","cross_cats_sorted":["cs.LG"],"license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","primary_cat":"cs.CL","submitted_at":"2020-05-08T15:48:31Z","title_canon_sha256":"11fec14a977e09eda3e96c43356cb5f06cdef062c70bc7eb7476736c2c742948"},"schema_version":"1.0","source":{"id":"2005.04118","kind":"arxiv","version":1}},"source_aliases":[{"alias_kind":"arxiv","alias_value":"2005.04118","created_at":"2026-07-05T01:01:29Z"},{"alias_kind":"arxiv_version","alias_value":"2005.04118v1","created_at":"2026-07-05T01:01:29Z"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.2005.04118","created_at":"2026-07-05T01:01:29Z"},{"alias_kind":"pith_short_12","alias_value":"AMJ37KAYRMDX","created_at":"2026-07-05T01:01:29Z"},{"alias_kind":"pith_short_16","alias_value":"AMJ37KAYRMDXB4W5","created_at":"2026-07-05T01:01:29Z"},{"alias_kind":"pith_short_8","alias_value":"AMJ37KAY","created_at":"2026-07-05T01:01:29Z"}],"graph_snapshots":[{"event_id":"sha256:52cba22c62537ed8d56901063467cd673cb6def527f025c47e8c51accab24d13","target":"graph","created_at":"2026-07-05T01:01:29Z","signer":{"key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signer_id":"pith.science","signer_type":"pith_registry"},"payload":{"graph_snapshot":{"author_claims":{"count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","strong_count":0},"builder_version":"pith-number-builder-2026-05-17-v1","claims":{"count":0,"items":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"integrity":{"available":true,"clean":true,"detectors_run":[],"endpoint":"/pith/2005.04118/integrity.json","findings":[],"snapshot_sha256":"c28c3603d3b5d939e8dc4c7e95fa8dfce3d595e45f758748cecf8e644a296938","summary":{"advisory":0,"by_detector":{},"critical":0,"informational":0}},"paper":{"abstract_excerpt":"Although measuring held-out accuracy has been the primary approach to evaluate generalization, it often overestimates the performance of NLP models, while alternative approaches for evaluating models either focus on individual tasks or on specific behaviors. Inspired by principles of behavioral testing in software engineering, we introduce CheckList, a task-agnostic methodology for testing NLP models. CheckList includes a matrix of general linguistic capabilities and test types that facilitate comprehensive test ideation, as well as a software tool to generate a large and diverse number of tes","authors_text":"Carlos Guestrin, Marco Tulio Ribeiro, Sameer Singh, Tongshuang Wu","cross_cats":["cs.LG"],"headline":"","license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","primary_cat":"cs.CL","submitted_at":"2020-05-08T15:48:31Z","title":"Beyond Accuracy: Behavioral Testing of NLP models with CheckList"},"references":{"count":0,"internal_anchors":0,"resolved_work":0,"sample":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"2005.04118","kind":"arxiv","version":1},"verdict":{"created_at":null,"id":null,"model_set":{},"one_line_summary":"","pipeline_version":null,"pith_extraction_headline":"","strongest_claim":"","weakest_assumption":""}},"verdict_id":null}}],"author_attestations":[],"timestamp_anchors":[],"storage_attestations":[],"citation_signatures":[],"replication_records":[],"corrections":[],"mirror_hints":[],"record_created":{"event_id":"sha256:230f29de935ff42aea30f0683242a1c7c38e0f6f15dd9b22010246ed8650f452","target":"record","created_at":"2026-07-05T01:01:29Z","signer":{"key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signer_id":"pith.science","signer_type":"pith_registry"},"payload":{"attestation_state":"computed","canonical_record":{"metadata":{"abstract_canon_sha256":"5cbb5295e76de7ee55556cea6316f696fbedd79380f3e7608c7123eb6c670d72","cross_cats_sorted":["cs.LG"],"license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","primary_cat":"cs.CL","submitted_at":"2020-05-08T15:48:31Z","title_canon_sha256":"11fec14a977e09eda3e96c43356cb5f06cdef062c70bc7eb7476736c2c742948"},"schema_version":"1.0","source":{"id":"2005.04118","kind":"arxiv","version":1}},"canonical_sha256":"0313bfa8188b0770f2dda6db3a7a8930d601138de1f2703ffba05a5877d03dde","receipt":{"algorithm":"ed25519","builder_version":"pith-number-builder-2026-05-17-v1","canonical_sha256":"0313bfa8188b0770f2dda6db3a7a8930d601138de1f2703ffba05a5877d03dde","first_computed_at":"2026-07-05T01:01:29.719808Z","key_id":"pith-v1-2026-05","kind":"pith_receipt","last_reissued_at":"2026-07-05T01:01:29.719808Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","receipt_version":"0.3","signature_b64":"X7kyfgGokSK4nlmMLYG6FQo6zkEPqUPmt9wNkWIPqFpIksnHLMXb005teNfu7LbYEWwUb6TmMC6CHCd0fEDCBw==","signature_status":"signed_v1","signed_at":"2026-07-05T01:01:29.720289Z","signed_message":"canonical_sha256_bytes"},"source_id":"2005.04118","source_kind":"arxiv","source_version":1}}},"equivocations":[],"invalid_events":[],"applied_event_ids":["sha256:230f29de935ff42aea30f0683242a1c7c38e0f6f15dd9b22010246ed8650f452","sha256:52cba22c62537ed8d56901063467cd673cb6def527f025c47e8c51accab24d13"],"state_sha256":"08ab227dffd64ea1e4113c5b2d342f98d1b78c92cbf04ca0170964848c0774d7"}