{"bundle_type":"pith_open_graph_bundle","bundle_version":"1.0","pith_number":"pith:2020:5NKRPZ7GRRLXTQKVZOERAHKLGC","short_pith_number":"pith:5NKRPZ7G","canonical_record":{"source":{"id":"2004.02709","kind":"arxiv","version":2},"metadata":{"license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.CL","submitted_at":"2020-04-06T14:47:18Z","cross_cats_sorted":[],"title_canon_sha256":"0c0e1adb285c9da4efac50d642ca68d0e7b2f2bf3af81a5419d56cff6c668ab1","abstract_canon_sha256":"aa400139877a36db4d70da057f96adfca4316fcc5279237d2f41e630d9d6d7c9"},"schema_version":"1.0"},"canonical_sha256":"eb5517e7e68c5779c155cb89101d4b30a00e0fbd7ec33f109c3e4a1d9586a27e","source":{"kind":"arxiv","id":"2004.02709","version":2},"source_aliases":[{"alias_kind":"arxiv","alias_value":"2004.02709","created_at":"2026-07-05T01:39:43Z"},{"alias_kind":"arxiv_version","alias_value":"2004.02709v2","created_at":"2026-07-05T01:39:43Z"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.2004.02709","created_at":"2026-07-05T01:39:43Z"},{"alias_kind":"pith_short_12","alias_value":"5NKRPZ7GRRLX","created_at":"2026-07-05T01:39:43Z"},{"alias_kind":"pith_short_16","alias_value":"5NKRPZ7GRRLXTQKV","created_at":"2026-07-05T01:39:43Z"},{"alias_kind":"pith_short_8","alias_value":"5NKRPZ7G","created_at":"2026-07-05T01:39:43Z"}],"events":[{"event_type":"record_created","subject_pith_number":"pith:2020:5NKRPZ7GRRLXTQKVZOERAHKLGC","target":"record","payload":{"canonical_record":{"source":{"id":"2004.02709","kind":"arxiv","version":2},"metadata":{"license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.CL","submitted_at":"2020-04-06T14:47:18Z","cross_cats_sorted":[],"title_canon_sha256":"0c0e1adb285c9da4efac50d642ca68d0e7b2f2bf3af81a5419d56cff6c668ab1","abstract_canon_sha256":"aa400139877a36db4d70da057f96adfca4316fcc5279237d2f41e630d9d6d7c9"},"schema_version":"1.0"},"canonical_sha256":"eb5517e7e68c5779c155cb89101d4b30a00e0fbd7ec33f109c3e4a1d9586a27e","receipt":{"kind":"pith_receipt","key_id":"pith-v1-2026-05","algorithm":"ed25519","signed_at":"2026-07-05T01:39:43.040973Z","signature_b64":"zZeR2XK73L9A4j5VjbH+LRfDU6wqS2fuzuex4LLRDOzg7dIEWrI11VZqA5ywtNV7251+v1jpnL3EfO6iZWRLDw==","signed_message":"canonical_sha256_bytes","builder_version":"pith-number-builder-2026-05-17-v1","receipt_version":"0.3","canonical_sha256":"eb5517e7e68c5779c155cb89101d4b30a00e0fbd7ec33f109c3e4a1d9586a27e","last_reissued_at":"2026-07-05T01:39:43.040475Z","signature_status":"signed_v1","first_computed_at":"2026-07-05T01:39:43.040475Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"source_kind":"arxiv","source_id":"2004.02709","source_version":2,"attestation_state":"computed"},"signer":{"signer_id":"pith.science","signer_type":"pith_registry","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"created_at":"2026-07-05T01:39:43Z","supersedes":[],"prev_event":null,"signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"txelU6rm5zuOH3vldxrxEkJbyUuTnZMR0DrUKLK1SwQ3VDR1oeXCU+vXWvXXT80nCZazoOM40Y1eYKgc8vDXDg==","signed_message":"open_graph_event_sha256_bytes","signed_at":"2026-07-06T19:07:29.992568Z"},"content_sha256":"7757b8d870b6a957f93ee1f995d400cbb84b3df7e7ac81b23f5d06cbc53bbf91","schema_version":"1.0","event_id":"sha256:7757b8d870b6a957f93ee1f995d400cbb84b3df7e7ac81b23f5d06cbc53bbf91"},{"event_type":"graph_snapshot","subject_pith_number":"pith:2020:5NKRPZ7GRRLXTQKVZOERAHKLGC","target":"graph","payload":{"graph_snapshot":{"paper":{"title":"Evaluating Models' Local Decision Boundaries via Contrast Sets","license":"http://creativecommons.org/licenses/by/4.0/","headline":"","cross_cats":[],"primary_cat":"cs.CL","authors_text":"Ally Zhang, Ananth Gottumukkala, Ben Bogin, Ben Zhou, Daniel Khashabi, Dheeru Dua, Eric Wallace, Gabriel Ilharco, Hanna Hajishirzi, Jiangming Liu, Jonathan Berant, Kevin Lin, Matt Gardner, Nelson F. Liu, Nitish Gupta, Noah A. Smith, Phoebe Mulcaire, Pradeep Dasigi, Qiang Ning, Reut Tsarfaty, Sameer Singh, Sanjay Subramanian, Sihao Chen, Victoria Basmova, Yanai Elazar, Yoav Artzi","submitted_at":"2020-04-06T14:47:18Z","abstract_excerpt":"Standard test sets for supervised learning evaluate in-distribution generalization. Unfortunately, when a dataset has systematic gaps (e.g., annotation artifacts), these evaluations are misleading: a model can learn simple decision rules that perform well on the test set but do not capture a dataset's intended capabilities. We propose a new annotation paradigm for NLP that helps to close systematic gaps in the test data. In particular, after a dataset is constructed, we recommend that the dataset authors manually perturb the test instances in small but meaningful ways that (typically) change t"},"claims":{"count":0,"items":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"2004.02709","kind":"arxiv","version":2},"verdict":{"id":null,"model_set":{},"created_at":null,"strongest_claim":"","one_line_summary":"","pipeline_version":null,"weakest_assumption":"","pith_extraction_headline":""},"integrity":{"clean":true,"summary":{"advisory":0,"critical":0,"by_detector":{},"informational":0},"endpoint":"/pith/2004.02709/integrity.json","findings":[],"available":true,"detectors_run":[],"snapshot_sha256":"c28c3603d3b5d939e8dc4c7e95fa8dfce3d595e45f758748cecf8e644a296938"},"references":{"count":0,"sample":[],"resolved_work":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","internal_anchors":0},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"author_claims":{"count":0,"strong_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"builder_version":"pith-number-builder-2026-05-17-v1"},"verdict_id":null},"signer":{"signer_id":"pith.science","signer_type":"pith_registry","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"created_at":"2026-07-05T01:39:43Z","supersedes":[],"prev_event":null,"signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"+J93enY3v5HRubQbnvJr9ILzsYISJc9W0boXpyQl2L4EipvmA9n+Cxmqcac0SGnRMGmFh2y1uS/l8cZAXO9GDw==","signed_message":"open_graph_event_sha256_bytes","signed_at":"2026-07-06T19:07:29.992945Z"},"content_sha256":"95feb30db19297bfa84c8d83096752f9d4acbb51d05768a40e8f1eda627a414a","schema_version":"1.0","event_id":"sha256:95feb30db19297bfa84c8d83096752f9d4acbb51d05768a40e8f1eda627a414a"}],"timestamp_proofs":[],"mirror_hints":[{"mirror_type":"https","name":"Pith Resolver","base_url":"https://pith.science","bundle_url":"https://pith.science/pith/5NKRPZ7GRRLXTQKVZOERAHKLGC/bundle.json","state_url":"https://pith.science/pith/5NKRPZ7GRRLXTQKVZOERAHKLGC/state.json","well_known_bundle_url":"https://pith.science/.well-known/pith/5NKRPZ7GRRLXTQKVZOERAHKLGC/bundle.json","status":"primary"}],"public_keys":[{"key_id":"pith-v1-2026-05","algorithm":"ed25519","format":"raw","public_key_b64":"stVStoiQhXFxp4s2pdzPNoqVNBMojDU/fJ2db5S3CbM=","public_key_hex":"b2d552b68890857171a78b36a5dccf368a953413288c353f7c9d9d6f94b709b3","fingerprint_sha256_b32_first128bits":"RVFV5Z2OI2J3ZUO7ERDEBCYNKS","fingerprint_sha256_hex":"8d4b5ee74e4693bcd1df2446408b0d54","rotates_at":null,"url":"https://pith.science/pith-signing-key.json","notes":"Pith uses this Ed25519 key to sign canonical record SHA-256 digests. Verify with: ed25519_verify(public_key, message=canonical_sha256_bytes, signature=base64decode(signature_b64))."}],"merge_version":"pith-open-graph-merge-v1","built_at":"2026-07-06T19:07:29Z","links":{"resolver":"https://pith.science/pith/5NKRPZ7GRRLXTQKVZOERAHKLGC","bundle":"https://pith.science/pith/5NKRPZ7GRRLXTQKVZOERAHKLGC/bundle.json","state":"https://pith.science/pith/5NKRPZ7GRRLXTQKVZOERAHKLGC/state.json","well_known_bundle":"https://pith.science/.well-known/pith/5NKRPZ7GRRLXTQKVZOERAHKLGC/bundle.json"},"state":{"state_type":"pith_open_graph_state","state_version":"1.0","pith_number":"pith:2020:5NKRPZ7GRRLXTQKVZOERAHKLGC","merge_version":"pith-open-graph-merge-v1","event_count":2,"valid_event_count":2,"invalid_event_count":0,"equivocation_count":0,"current":{"canonical_record":{"metadata":{"abstract_canon_sha256":"aa400139877a36db4d70da057f96adfca4316fcc5279237d2f41e630d9d6d7c9","cross_cats_sorted":[],"license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.CL","submitted_at":"2020-04-06T14:47:18Z","title_canon_sha256":"0c0e1adb285c9da4efac50d642ca68d0e7b2f2bf3af81a5419d56cff6c668ab1"},"schema_version":"1.0","source":{"id":"2004.02709","kind":"arxiv","version":2}},"source_aliases":[{"alias_kind":"arxiv","alias_value":"2004.02709","created_at":"2026-07-05T01:39:43Z"},{"alias_kind":"arxiv_version","alias_value":"2004.02709v2","created_at":"2026-07-05T01:39:43Z"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.2004.02709","created_at":"2026-07-05T01:39:43Z"},{"alias_kind":"pith_short_12","alias_value":"5NKRPZ7GRRLX","created_at":"2026-07-05T01:39:43Z"},{"alias_kind":"pith_short_16","alias_value":"5NKRPZ7GRRLXTQKV","created_at":"2026-07-05T01:39:43Z"},{"alias_kind":"pith_short_8","alias_value":"5NKRPZ7G","created_at":"2026-07-05T01:39:43Z"}],"graph_snapshots":[{"event_id":"sha256:95feb30db19297bfa84c8d83096752f9d4acbb51d05768a40e8f1eda627a414a","target":"graph","created_at":"2026-07-05T01:39:43Z","signer":{"key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signer_id":"pith.science","signer_type":"pith_registry"},"payload":{"graph_snapshot":{"author_claims":{"count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","strong_count":0},"builder_version":"pith-number-builder-2026-05-17-v1","claims":{"count":0,"items":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"integrity":{"available":true,"clean":true,"detectors_run":[],"endpoint":"/pith/2004.02709/integrity.json","findings":[],"snapshot_sha256":"c28c3603d3b5d939e8dc4c7e95fa8dfce3d595e45f758748cecf8e644a296938","summary":{"advisory":0,"by_detector":{},"critical":0,"informational":0}},"paper":{"abstract_excerpt":"Standard test sets for supervised learning evaluate in-distribution generalization. Unfortunately, when a dataset has systematic gaps (e.g., annotation artifacts), these evaluations are misleading: a model can learn simple decision rules that perform well on the test set but do not capture a dataset's intended capabilities. We propose a new annotation paradigm for NLP that helps to close systematic gaps in the test data. In particular, after a dataset is constructed, we recommend that the dataset authors manually perturb the test instances in small but meaningful ways that (typically) change t","authors_text":"Ally Zhang, Ananth Gottumukkala, Ben Bogin, Ben Zhou, Daniel Khashabi, Dheeru Dua, Eric Wallace, Gabriel Ilharco, Hanna Hajishirzi, Jiangming Liu, Jonathan Berant, Kevin Lin, Matt Gardner, Nelson F. Liu, Nitish Gupta, Noah A. Smith, Phoebe Mulcaire, Pradeep Dasigi, Qiang Ning, Reut Tsarfaty, Sameer Singh, Sanjay Subramanian, Sihao Chen, Victoria Basmova, Yanai Elazar, Yoav Artzi","cross_cats":[],"headline":"","license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.CL","submitted_at":"2020-04-06T14:47:18Z","title":"Evaluating Models' Local Decision Boundaries via Contrast Sets"},"references":{"count":0,"internal_anchors":0,"resolved_work":0,"sample":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"2004.02709","kind":"arxiv","version":2},"verdict":{"created_at":null,"id":null,"model_set":{},"one_line_summary":"","pipeline_version":null,"pith_extraction_headline":"","strongest_claim":"","weakest_assumption":""}},"verdict_id":null}}],"author_attestations":[],"timestamp_anchors":[],"storage_attestations":[],"citation_signatures":[],"replication_records":[],"corrections":[],"mirror_hints":[],"record_created":{"event_id":"sha256:7757b8d870b6a957f93ee1f995d400cbb84b3df7e7ac81b23f5d06cbc53bbf91","target":"record","created_at":"2026-07-05T01:39:43Z","signer":{"key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signer_id":"pith.science","signer_type":"pith_registry"},"payload":{"attestation_state":"computed","canonical_record":{"metadata":{"abstract_canon_sha256":"aa400139877a36db4d70da057f96adfca4316fcc5279237d2f41e630d9d6d7c9","cross_cats_sorted":[],"license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.CL","submitted_at":"2020-04-06T14:47:18Z","title_canon_sha256":"0c0e1adb285c9da4efac50d642ca68d0e7b2f2bf3af81a5419d56cff6c668ab1"},"schema_version":"1.0","source":{"id":"2004.02709","kind":"arxiv","version":2}},"canonical_sha256":"eb5517e7e68c5779c155cb89101d4b30a00e0fbd7ec33f109c3e4a1d9586a27e","receipt":{"algorithm":"ed25519","builder_version":"pith-number-builder-2026-05-17-v1","canonical_sha256":"eb5517e7e68c5779c155cb89101d4b30a00e0fbd7ec33f109c3e4a1d9586a27e","first_computed_at":"2026-07-05T01:39:43.040475Z","key_id":"pith-v1-2026-05","kind":"pith_receipt","last_reissued_at":"2026-07-05T01:39:43.040475Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","receipt_version":"0.3","signature_b64":"zZeR2XK73L9A4j5VjbH+LRfDU6wqS2fuzuex4LLRDOzg7dIEWrI11VZqA5ywtNV7251+v1jpnL3EfO6iZWRLDw==","signature_status":"signed_v1","signed_at":"2026-07-05T01:39:43.040973Z","signed_message":"canonical_sha256_bytes"},"source_id":"2004.02709","source_kind":"arxiv","source_version":2}}},"equivocations":[],"invalid_events":[],"applied_event_ids":["sha256:7757b8d870b6a957f93ee1f995d400cbb84b3df7e7ac81b23f5d06cbc53bbf91","sha256:95feb30db19297bfa84c8d83096752f9d4acbb51d05768a40e8f1eda627a414a"],"state_sha256":"f9879d32c8fb1b38b0cb08fb99440390a1ff4be6888102e4072b368831da3377"},"bundle_signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"4s5+sFL2YXlwjIJQ+00x6MAUnv2k7vpbhupFlBVmP/uJfBPwh7iVgdCe8DGIo1B+SBSieAWm0cEDECzHDWkcDA==","signed_message":"bundle_sha256_bytes","signed_at":"2026-07-06T19:07:29.995016Z","bundle_sha256":"8a8f950cec4a549374a3b0a18564fdbbc9918fed02967e37a5ff9ed8964d9404"}}