{"bundle_type":"pith_open_graph_bundle","bundle_version":"1.0","pith_number":"pith:2022:I4WV4LDUXEIH5YTO6RUZJMJ4XH","short_pith_number":"pith:I4WV4LDU","canonical_record":{"source":{"id":"2210.08097","kind":"arxiv","version":1},"metadata":{"license":"http://creativecommons.org/licenses/by-nc-nd/4.0/","primary_cat":"cs.SE","submitted_at":"2022-10-14T20:42:16Z","cross_cats_sorted":["cs.AI","cs.CL","cs.LG"],"title_canon_sha256":"6340df6fe8cbfe7aee4705bf5eccbfe4c73e8e4987c8aed0afefe3f54acad021","abstract_canon_sha256":"4707e0decc54752457f5013fe00efe4e4e0a24a745d0914669a1110720c16b68"},"schema_version":"1.0"},"canonical_sha256":"472d5e2c74b9107ee26ef46994b13cb9c3b946954ce11945fc6546f31c5ce2ab","source":{"kind":"arxiv","id":"2210.08097","version":1},"source_aliases":[{"alias_kind":"arxiv","alias_value":"2210.08097","created_at":"2026-07-05T05:07:06Z"},{"alias_kind":"arxiv_version","alias_value":"2210.08097v1","created_at":"2026-07-05T05:07:06Z"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.2210.08097","created_at":"2026-07-05T05:07:06Z"},{"alias_kind":"pith_short_12","alias_value":"I4WV4LDUXEIH","created_at":"2026-07-05T05:07:06Z"},{"alias_kind":"pith_short_16","alias_value":"I4WV4LDUXEIH5YTO","created_at":"2026-07-05T05:07:06Z"},{"alias_kind":"pith_short_8","alias_value":"I4WV4LDU","created_at":"2026-07-05T05:07:06Z"}],"events":[{"event_type":"record_created","subject_pith_number":"pith:2022:I4WV4LDUXEIH5YTO6RUZJMJ4XH","target":"record","payload":{"canonical_record":{"source":{"id":"2210.08097","kind":"arxiv","version":1},"metadata":{"license":"http://creativecommons.org/licenses/by-nc-nd/4.0/","primary_cat":"cs.SE","submitted_at":"2022-10-14T20:42:16Z","cross_cats_sorted":["cs.AI","cs.CL","cs.LG"],"title_canon_sha256":"6340df6fe8cbfe7aee4705bf5eccbfe4c73e8e4987c8aed0afefe3f54acad021","abstract_canon_sha256":"4707e0decc54752457f5013fe00efe4e4e0a24a745d0914669a1110720c16b68"},"schema_version":"1.0"},"canonical_sha256":"472d5e2c74b9107ee26ef46994b13cb9c3b946954ce11945fc6546f31c5ce2ab","receipt":{"kind":"pith_receipt","key_id":"pith-v1-2026-05","algorithm":"ed25519","signed_at":"2026-07-05T05:07:06.959597Z","signature_b64":"Nb76G1RoOJsobiOUvCOt3xVEek2IDNW47nU2M0QK9nKKuD1FN/3Ht+Xe7aXnQnnhsaKGZxy3eIWIGi6kCXaRBQ==","signed_message":"canonical_sha256_bytes","builder_version":"pith-number-builder-2026-05-17-v1","receipt_version":"0.3","canonical_sha256":"472d5e2c74b9107ee26ef46994b13cb9c3b946954ce11945fc6546f31c5ce2ab","last_reissued_at":"2026-07-05T05:07:06.959173Z","signature_status":"signed_v1","first_computed_at":"2026-07-05T05:07:06.959173Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"source_kind":"arxiv","source_id":"2210.08097","source_version":1,"attestation_state":"computed"},"signer":{"signer_id":"pith.science","signer_type":"pith_registry","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"created_at":"2026-07-05T05:07:06Z","supersedes":[],"prev_event":null,"signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"6oibaBofQde2X2Ztko9DSdHTUkot4meq/1OcJHosIgwKQJDOBssZVarW2aJckTXIjMgia+Vc+97JTPz6JmhaDQ==","signed_message":"open_graph_event_sha256_bytes","signed_at":"2026-08-14T06:54:07.294731Z"},"content_sha256":"a9da0add56287dbd693a45441bb0a15cab97623d029d607938ab772b7f57ce2c","schema_version":"1.0","event_id":"sha256:a9da0add56287dbd693a45441bb0a15cab97623d029d607938ab772b7f57ce2c"},{"event_type":"graph_snapshot","subject_pith_number":"pith:2022:I4WV4LDUXEIH5YTO6RUZJMJ4XH","target":"graph","payload":{"graph_snapshot":{"paper":{"title":"TestAug: A Framework for Augmenting Capability-based NLP Tests","license":"http://creativecommons.org/licenses/by-nc-nd/4.0/","headline":"","cross_cats":["cs.AI","cs.CL","cs.LG"],"primary_cat":"cs.SE","authors_text":"Guanqun Yang, Mirazul Haque, Qiaochu Song, Wei Yang, Xueqing Liu","submitted_at":"2022-10-14T20:42:16Z","abstract_excerpt":"The recently proposed capability-based NLP testing allows model developers to test the functional capabilities of NLP models, revealing functional failures that cannot be detected by the traditional heldout mechanism. However, existing work on capability-based testing requires extensive manual efforts and domain expertise in creating the test cases. In this paper, we investigate a low-cost approach for the test case generation by leveraging the GPT-3 engine. We further propose to use a classifier to remove the invalid outputs from GPT-3 and expand the outputs into templates to generate more te"},"claims":{"count":0,"items":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"2210.08097","kind":"arxiv","version":1},"verdict":{"id":null,"model_set":{},"created_at":null,"strongest_claim":"","one_line_summary":"","pipeline_version":null,"weakest_assumption":"","pith_extraction_headline":""},"integrity":{"clean":true,"summary":{"advisory":0,"critical":0,"by_detector":{},"informational":0},"endpoint":"/pith/2210.08097/integrity.json","findings":[],"available":true,"detectors_run":[],"snapshot_sha256":"c28c3603d3b5d939e8dc4c7e95fa8dfce3d595e45f758748cecf8e644a296938"},"references":{"count":0,"sample":[],"resolved_work":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","internal_anchors":0},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"author_claims":{"count":0,"strong_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"builder_version":"pith-number-builder-2026-05-17-v1"},"verdict_id":null},"signer":{"signer_id":"pith.science","signer_type":"pith_registry","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"created_at":"2026-07-05T05:07:06Z","supersedes":[],"prev_event":null,"signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"qsnHBdh2gLUwf5WYmPSmCGUvnRtgzmG7/HloORdtVskVGNjax6cj1LcYdA9RO+TsaV0UG/+mCDqsfFUbwvppDA==","signed_message":"open_graph_event_sha256_bytes","signed_at":"2026-08-14T06:54:07.295883Z"},"content_sha256":"811c6e8df3304046d115038577c2c5829d1578b2000e40b118b9fc1b3bbd11c6","schema_version":"1.0","event_id":"sha256:811c6e8df3304046d115038577c2c5829d1578b2000e40b118b9fc1b3bbd11c6"}],"timestamp_proofs":[],"mirror_hints":[{"mirror_type":"https","name":"Pith Resolver","base_url":"https://pith.science","bundle_url":"https://pith.science/pith/I4WV4LDUXEIH5YTO6RUZJMJ4XH/bundle.json","state_url":"https://pith.science/pith/I4WV4LDUXEIH5YTO6RUZJMJ4XH/state.json","well_known_bundle_url":"https://pith.science/.well-known/pith/I4WV4LDUXEIH5YTO6RUZJMJ4XH/bundle.json","status":"primary"}],"public_keys":[{"key_id":"pith-v1-2026-05","algorithm":"ed25519","format":"raw","public_key_b64":"stVStoiQhXFxp4s2pdzPNoqVNBMojDU/fJ2db5S3CbM=","public_key_hex":"b2d552b68890857171a78b36a5dccf368a953413288c353f7c9d9d6f94b709b3","fingerprint_sha256_b32_first128bits":"RVFV5Z2OI2J3ZUO7ERDEBCYNKS","fingerprint_sha256_hex":"8d4b5ee74e4693bcd1df2446408b0d54","rotates_at":null,"url":"https://pith.science/pith-signing-key.json","notes":"Pith uses this Ed25519 key to sign canonical record SHA-256 digests. Verify with: ed25519_verify(public_key, message=canonical_sha256_bytes, signature=base64decode(signature_b64))."}],"merge_version":"pith-open-graph-merge-v1","built_at":"2026-08-14T06:54:07Z","links":{"resolver":"https://pith.science/pith/I4WV4LDUXEIH5YTO6RUZJMJ4XH","bundle":"https://pith.science/pith/I4WV4LDUXEIH5YTO6RUZJMJ4XH/bundle.json","state":"https://pith.science/pith/I4WV4LDUXEIH5YTO6RUZJMJ4XH/state.json","well_known_bundle":"https://pith.science/.well-known/pith/I4WV4LDUXEIH5YTO6RUZJMJ4XH/bundle.json"},"state":{"state_type":"pith_open_graph_state","state_version":"1.0","pith_number":"pith:2022:I4WV4LDUXEIH5YTO6RUZJMJ4XH","merge_version":"pith-open-graph-merge-v1","event_count":2,"valid_event_count":2,"invalid_event_count":0,"equivocation_count":0,"current":{"canonical_record":{"metadata":{"abstract_canon_sha256":"4707e0decc54752457f5013fe00efe4e4e0a24a745d0914669a1110720c16b68","cross_cats_sorted":["cs.AI","cs.CL","cs.LG"],"license":"http://creativecommons.org/licenses/by-nc-nd/4.0/","primary_cat":"cs.SE","submitted_at":"2022-10-14T20:42:16Z","title_canon_sha256":"6340df6fe8cbfe7aee4705bf5eccbfe4c73e8e4987c8aed0afefe3f54acad021"},"schema_version":"1.0","source":{"id":"2210.08097","kind":"arxiv","version":1}},"source_aliases":[{"alias_kind":"arxiv","alias_value":"2210.08097","created_at":"2026-07-05T05:07:06Z"},{"alias_kind":"arxiv_version","alias_value":"2210.08097v1","created_at":"2026-07-05T05:07:06Z"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.2210.08097","created_at":"2026-07-05T05:07:06Z"},{"alias_kind":"pith_short_12","alias_value":"I4WV4LDUXEIH","created_at":"2026-07-05T05:07:06Z"},{"alias_kind":"pith_short_16","alias_value":"I4WV4LDUXEIH5YTO","created_at":"2026-07-05T05:07:06Z"},{"alias_kind":"pith_short_8","alias_value":"I4WV4LDU","created_at":"2026-07-05T05:07:06Z"}],"graph_snapshots":[{"event_id":"sha256:811c6e8df3304046d115038577c2c5829d1578b2000e40b118b9fc1b3bbd11c6","target":"graph","created_at":"2026-07-05T05:07:06Z","signer":{"key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signer_id":"pith.science","signer_type":"pith_registry"},"payload":{"graph_snapshot":{"author_claims":{"count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","strong_count":0},"builder_version":"pith-number-builder-2026-05-17-v1","claims":{"count":0,"items":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"integrity":{"available":true,"clean":true,"detectors_run":[],"endpoint":"/pith/2210.08097/integrity.json","findings":[],"snapshot_sha256":"c28c3603d3b5d939e8dc4c7e95fa8dfce3d595e45f758748cecf8e644a296938","summary":{"advisory":0,"by_detector":{},"critical":0,"informational":0}},"paper":{"abstract_excerpt":"The recently proposed capability-based NLP testing allows model developers to test the functional capabilities of NLP models, revealing functional failures that cannot be detected by the traditional heldout mechanism. However, existing work on capability-based testing requires extensive manual efforts and domain expertise in creating the test cases. In this paper, we investigate a low-cost approach for the test case generation by leveraging the GPT-3 engine. We further propose to use a classifier to remove the invalid outputs from GPT-3 and expand the outputs into templates to generate more te","authors_text":"Guanqun Yang, Mirazul Haque, Qiaochu Song, Wei Yang, Xueqing Liu","cross_cats":["cs.AI","cs.CL","cs.LG"],"headline":"","license":"http://creativecommons.org/licenses/by-nc-nd/4.0/","primary_cat":"cs.SE","submitted_at":"2022-10-14T20:42:16Z","title":"TestAug: A Framework for Augmenting Capability-based NLP Tests"},"references":{"count":0,"internal_anchors":0,"resolved_work":0,"sample":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"2210.08097","kind":"arxiv","version":1},"verdict":{"created_at":null,"id":null,"model_set":{},"one_line_summary":"","pipeline_version":null,"pith_extraction_headline":"","strongest_claim":"","weakest_assumption":""}},"verdict_id":null}}],"author_attestations":[],"timestamp_anchors":[],"storage_attestations":[],"citation_signatures":[],"replication_records":[],"corrections":[],"mirror_hints":[],"record_created":{"event_id":"sha256:a9da0add56287dbd693a45441bb0a15cab97623d029d607938ab772b7f57ce2c","target":"record","created_at":"2026-07-05T05:07:06Z","signer":{"key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signer_id":"pith.science","signer_type":"pith_registry"},"payload":{"attestation_state":"computed","canonical_record":{"metadata":{"abstract_canon_sha256":"4707e0decc54752457f5013fe00efe4e4e0a24a745d0914669a1110720c16b68","cross_cats_sorted":["cs.AI","cs.CL","cs.LG"],"license":"http://creativecommons.org/licenses/by-nc-nd/4.0/","primary_cat":"cs.SE","submitted_at":"2022-10-14T20:42:16Z","title_canon_sha256":"6340df6fe8cbfe7aee4705bf5eccbfe4c73e8e4987c8aed0afefe3f54acad021"},"schema_version":"1.0","source":{"id":"2210.08097","kind":"arxiv","version":1}},"canonical_sha256":"472d5e2c74b9107ee26ef46994b13cb9c3b946954ce11945fc6546f31c5ce2ab","receipt":{"algorithm":"ed25519","builder_version":"pith-number-builder-2026-05-17-v1","canonical_sha256":"472d5e2c74b9107ee26ef46994b13cb9c3b946954ce11945fc6546f31c5ce2ab","first_computed_at":"2026-07-05T05:07:06.959173Z","key_id":"pith-v1-2026-05","kind":"pith_receipt","last_reissued_at":"2026-07-05T05:07:06.959173Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","receipt_version":"0.3","signature_b64":"Nb76G1RoOJsobiOUvCOt3xVEek2IDNW47nU2M0QK9nKKuD1FN/3Ht+Xe7aXnQnnhsaKGZxy3eIWIGi6kCXaRBQ==","signature_status":"signed_v1","signed_at":"2026-07-05T05:07:06.959597Z","signed_message":"canonical_sha256_bytes"},"source_id":"2210.08097","source_kind":"arxiv","source_version":1}}},"equivocations":[],"invalid_events":[],"applied_event_ids":["sha256:a9da0add56287dbd693a45441bb0a15cab97623d029d607938ab772b7f57ce2c","sha256:811c6e8df3304046d115038577c2c5829d1578b2000e40b118b9fc1b3bbd11c6"],"state_sha256":"76f6a737add62d82433854b21264c6120804a607ebcb5e0e14c1b5453974b1e8"},"bundle_signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"ww0oybGHVxF0L7gKEySQkG9mehyFlQ4NPfDwIWKpemzWy4MVyewABbtmfz4jMJ5TEbMI0pzbWChLvNnA75beBQ==","signed_message":"bundle_sha256_bytes","signed_at":"2026-08-14T06:54:07.302739Z","bundle_sha256":"4048286e387ef7773fa1683db763742aedf72896edcbc02ad8272e534025fa6b"}}