{"record_type":"pith_number_record","schema_url":"https://pith.science/schemas/pith-number/v1.json","pith_number":"pith:2020:5NKRPZ7GRRLXTQKVZOERAHKLGC","short_pith_number":"pith:5NKRPZ7G","schema_version":"1.0","canonical_sha256":"eb5517e7e68c5779c155cb89101d4b30a00e0fbd7ec33f109c3e4a1d9586a27e","source":{"kind":"arxiv","id":"2004.02709","version":2},"attestation_state":"computed","paper":{"title":"Evaluating Models' Local Decision Boundaries via Contrast Sets","license":"http://creativecommons.org/licenses/by/4.0/","headline":"","cross_cats":[],"primary_cat":"cs.CL","authors_text":"Ally Zhang, Ananth Gottumukkala, Ben Bogin, Ben Zhou, Daniel Khashabi, Dheeru Dua, Eric Wallace, Gabriel Ilharco, Hanna Hajishirzi, Jiangming Liu, Jonathan Berant, Kevin Lin, Matt Gardner, Nelson F. Liu, Nitish Gupta, Noah A. Smith, Phoebe Mulcaire, Pradeep Dasigi, Qiang Ning, Reut Tsarfaty, Sameer Singh, Sanjay Subramanian, Sihao Chen, Victoria Basmova, Yanai Elazar, Yoav Artzi","submitted_at":"2020-04-06T14:47:18Z","abstract_excerpt":"Standard test sets for supervised learning evaluate in-distribution generalization. Unfortunately, when a dataset has systematic gaps (e.g., annotation artifacts), these evaluations are misleading: a model can learn simple decision rules that perform well on the test set but do not capture a dataset's intended capabilities. We propose a new annotation paradigm for NLP that helps to close systematic gaps in the test data. In particular, after a dataset is constructed, we recommend that the dataset authors manually perturb the test instances in small but meaningful ways that (typically) change t"},"verification_status":{"content_addressed":true,"pith_receipt":true,"author_attested":false,"weak_author_claims":0,"strong_author_claims":0,"externally_anchored":false,"storage_verified":false,"citation_signatures":0,"replication_records":0,"graph_snapshot":true,"references_resolved":false,"formal_links_present":false},"canonical_record":{"source":{"id":"2004.02709","kind":"arxiv","version":2},"metadata":{"license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.CL","submitted_at":"2020-04-06T14:47:18Z","cross_cats_sorted":[],"title_canon_sha256":"0c0e1adb285c9da4efac50d642ca68d0e7b2f2bf3af81a5419d56cff6c668ab1","abstract_canon_sha256":"aa400139877a36db4d70da057f96adfca4316fcc5279237d2f41e630d9d6d7c9"},"schema_version":"1.0"},"receipt":{"kind":"pith_receipt","key_id":"pith-v1-2026-05","algorithm":"ed25519","signed_at":"2026-07-05T01:39:43.040973Z","signature_b64":"zZeR2XK73L9A4j5VjbH+LRfDU6wqS2fuzuex4LLRDOzg7dIEWrI11VZqA5ywtNV7251+v1jpnL3EfO6iZWRLDw==","signed_message":"canonical_sha256_bytes","builder_version":"pith-number-builder-2026-05-17-v1","receipt_version":"0.3","canonical_sha256":"eb5517e7e68c5779c155cb89101d4b30a00e0fbd7ec33f109c3e4a1d9586a27e","last_reissued_at":"2026-07-05T01:39:43.040475Z","signature_status":"signed_v1","first_computed_at":"2026-07-05T01:39:43.040475Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"graph_snapshot":{"paper":{"title":"Evaluating Models' Local Decision Boundaries via Contrast Sets","license":"http://creativecommons.org/licenses/by/4.0/","headline":"","cross_cats":[],"primary_cat":"cs.CL","authors_text":"Ally Zhang, Ananth Gottumukkala, Ben Bogin, Ben Zhou, Daniel Khashabi, Dheeru Dua, Eric Wallace, Gabriel Ilharco, Hanna Hajishirzi, Jiangming Liu, Jonathan Berant, Kevin Lin, Matt Gardner, Nelson F. Liu, Nitish Gupta, Noah A. Smith, Phoebe Mulcaire, Pradeep Dasigi, Qiang Ning, Reut Tsarfaty, Sameer Singh, Sanjay Subramanian, Sihao Chen, Victoria Basmova, Yanai Elazar, Yoav Artzi","submitted_at":"2020-04-06T14:47:18Z","abstract_excerpt":"Standard test sets for supervised learning evaluate in-distribution generalization. Unfortunately, when a dataset has systematic gaps (e.g., annotation artifacts), these evaluations are misleading: a model can learn simple decision rules that perform well on the test set but do not capture a dataset's intended capabilities. We propose a new annotation paradigm for NLP that helps to close systematic gaps in the test data. In particular, after a dataset is constructed, we recommend that the dataset authors manually perturb the test instances in small but meaningful ways that (typically) change t"},"claims":{"count":0,"items":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"2004.02709","kind":"arxiv","version":2},"verdict":{"id":null,"model_set":{},"created_at":null,"strongest_claim":"","one_line_summary":"","pipeline_version":null,"weakest_assumption":"","pith_extraction_headline":""},"integrity":{"clean":true,"summary":{"advisory":0,"critical":0,"by_detector":{},"informational":0},"endpoint":"/pith/2004.02709/integrity.json","findings":[],"available":true,"detectors_run":[],"snapshot_sha256":"c28c3603d3b5d939e8dc4c7e95fa8dfce3d595e45f758748cecf8e644a296938"},"references":{"count":0,"sample":[],"resolved_work":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","internal_anchors":0},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"author_claims":{"count":0,"strong_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"builder_version":"pith-number-builder-2026-05-17-v1"},"aliases":[{"alias_kind":"arxiv","alias_value":"2004.02709","created_at":"2026-07-05T01:39:43.040553+00:00"},{"alias_kind":"arxiv_version","alias_value":"2004.02709v2","created_at":"2026-07-05T01:39:43.040553+00:00"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.2004.02709","created_at":"2026-07-05T01:39:43.040553+00:00"},{"alias_kind":"pith_short_12","alias_value":"5NKRPZ7GRRLX","created_at":"2026-07-05T01:39:43.040553+00:00"},{"alias_kind":"pith_short_16","alias_value":"5NKRPZ7GRRLXTQKV","created_at":"2026-07-05T01:39:43.040553+00:00"},{"alias_kind":"pith_short_8","alias_value":"5NKRPZ7G","created_at":"2026-07-05T01:39:43.040553+00:00"}],"events":[],"event_summary":{},"paper_claims":[],"inbound_citations":{"count":2,"internal_anchor_count":0,"sample":[{"citing_arxiv_id":"2008.02275","citing_title":"Aligning AI With Shared Human Values","ref_index":6,"is_internal_anchor":false},{"citing_arxiv_id":"2303.09014","citing_title":"ART: Automatic multi-step reasoning and tool-use for large language models","ref_index":117,"is_internal_anchor":false}]},"formal_canon":{"evidence_count":0,"sample":[],"anchors":[]},"links":{"html":"https://pith.science/pith/5NKRPZ7GRRLXTQKVZOERAHKLGC","json":"https://pith.science/pith/5NKRPZ7GRRLXTQKVZOERAHKLGC.json","graph_json":"https://pith.science/api/pith-number/5NKRPZ7GRRLXTQKVZOERAHKLGC/graph.json","events_json":"https://pith.science/api/pith-number/5NKRPZ7GRRLXTQKVZOERAHKLGC/events.json","paper":"https://pith.science/paper/5NKRPZ7G"},"agent_actions":{"view_html":"https://pith.science/pith/5NKRPZ7GRRLXTQKVZOERAHKLGC","download_json":"https://pith.science/pith/5NKRPZ7GRRLXTQKVZOERAHKLGC.json","view_paper":"https://pith.science/paper/5NKRPZ7G","resolve_alias":"https://pith.science/api/pith-number/resolve?arxiv=2004.02709&json=true","fetch_graph":"https://pith.science/api/pith-number/5NKRPZ7GRRLXTQKVZOERAHKLGC/graph.json","fetch_events":"https://pith.science/api/pith-number/5NKRPZ7GRRLXTQKVZOERAHKLGC/events.json","actions":{"anchor_timestamp":"https://pith.science/pith/5NKRPZ7GRRLXTQKVZOERAHKLGC/action/timestamp_anchor","attest_storage":"https://pith.science/pith/5NKRPZ7GRRLXTQKVZOERAHKLGC/action/storage_attestation","attest_author":"https://pith.science/pith/5NKRPZ7GRRLXTQKVZOERAHKLGC/action/author_attestation","sign_citation":"https://pith.science/pith/5NKRPZ7GRRLXTQKVZOERAHKLGC/action/citation_signature","submit_replication":"https://pith.science/pith/5NKRPZ7GRRLXTQKVZOERAHKLGC/action/replication_record"}},"created_at":"2026-07-05T01:39:43.040553+00:00","updated_at":"2026-07-05T01:39:43.040553+00:00"}