{"bundle_type":"pith_open_graph_bundle","bundle_version":"1.0","pith_number":"pith:2024:5OPWKVKJQD5B3MYGIIYINE3GZM","short_pith_number":"pith:5OPWKVKJ","canonical_record":{"source":{"id":"2408.00161","kind":"arxiv","version":2},"metadata":{"license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","primary_cat":"cs.CL","submitted_at":"2024-07-31T21:12:21Z","cross_cats_sorted":["cs.AI","cs.ET","cs.LG"],"title_canon_sha256":"87e0f613a165e96775666e8b46369ffcd44a73d7af2e1df67984ccda1f94b9bb","abstract_canon_sha256":"618262aea00f6377a7ed51832927e593d78676b50e45d1bdfa11daec1d793e7d"},"schema_version":"1.0"},"canonical_sha256":"eb9f65554980fa1db3064230869366cb22ba69c9aa657aed893c7b6c46d5efd3","source":{"kind":"arxiv","id":"2408.00161","version":2},"source_aliases":[{"alias_kind":"arxiv","alias_value":"2408.00161","created_at":"2026-07-05T08:53:24Z"},{"alias_kind":"arxiv_version","alias_value":"2408.00161v2","created_at":"2026-07-05T08:53:24Z"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.2408.00161","created_at":"2026-07-05T08:53:24Z"},{"alias_kind":"pith_short_12","alias_value":"5OPWKVKJQD5B","created_at":"2026-07-05T08:53:24Z"},{"alias_kind":"pith_short_16","alias_value":"5OPWKVKJQD5B3MYG","created_at":"2026-07-05T08:53:24Z"},{"alias_kind":"pith_short_8","alias_value":"5OPWKVKJ","created_at":"2026-07-05T08:53:24Z"}],"events":[{"event_type":"record_created","subject_pith_number":"pith:2024:5OPWKVKJQD5B3MYGIIYINE3GZM","target":"record","payload":{"canonical_record":{"source":{"id":"2408.00161","kind":"arxiv","version":2},"metadata":{"license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","primary_cat":"cs.CL","submitted_at":"2024-07-31T21:12:21Z","cross_cats_sorted":["cs.AI","cs.ET","cs.LG"],"title_canon_sha256":"87e0f613a165e96775666e8b46369ffcd44a73d7af2e1df67984ccda1f94b9bb","abstract_canon_sha256":"618262aea00f6377a7ed51832927e593d78676b50e45d1bdfa11daec1d793e7d"},"schema_version":"1.0"},"canonical_sha256":"eb9f65554980fa1db3064230869366cb22ba69c9aa657aed893c7b6c46d5efd3","receipt":{"kind":"pith_receipt","key_id":"pith-v1-2026-05","algorithm":"ed25519","signed_at":"2026-07-05T08:53:24.758427Z","signature_b64":"fpGWWPjMeXYUujjXQnpkMdSpidnuKac32yaNR0xp13bLqxb3aJE+KgUmJhyKQui9hDE2BxouYWhGetWbYpX4AA==","signed_message":"canonical_sha256_bytes","builder_version":"pith-number-builder-2026-05-17-v1","receipt_version":"0.3","canonical_sha256":"eb9f65554980fa1db3064230869366cb22ba69c9aa657aed893c7b6c46d5efd3","last_reissued_at":"2026-07-05T08:53:24.757935Z","signature_status":"signed_v1","first_computed_at":"2026-07-05T08:53:24.757935Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"source_kind":"arxiv","source_id":"2408.00161","source_version":2,"attestation_state":"computed"},"signer":{"signer_id":"pith.science","signer_type":"pith_registry","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"created_at":"2026-07-05T08:53:24Z","supersedes":[],"prev_event":null,"signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"RBhpKSF6ySEMOnsFCUWLBIn+PYDsp1ulh+QkFVn3vclICk90yjc08ZKTHWE827g4oDU5DQBLbZeaYwI/XT1nBg==","signed_message":"open_graph_event_sha256_bytes","signed_at":"2026-07-07T15:59:44.610283Z"},"content_sha256":"9635154da64feb2c2beaa4a0805c44cd16691bd17e9b5f5ff41a90c40b81e836","schema_version":"1.0","event_id":"sha256:9635154da64feb2c2beaa4a0805c44cd16691bd17e9b5f5ff41a90c40b81e836"},{"event_type":"graph_snapshot","subject_pith_number":"pith:2024:5OPWKVKJQD5B3MYGIIYINE3GZM","target":"graph","payload":{"graph_snapshot":{"paper":{"title":"Automatic Generation of Behavioral Test Cases For Natural Language Processing Using Clustering and Prompting","license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","headline":"","cross_cats":["cs.AI","cs.ET","cs.LG"],"primary_cat":"cs.CL","authors_text":"Agus Sudjianto, Rahul Singh, Tarun Joshi, Ying Li","submitted_at":"2024-07-31T21:12:21Z","abstract_excerpt":"Recent work in behavioral testing for natural language processing (NLP) models, such as Checklist, is inspired by related paradigms in software engineering testing. They allow evaluation of general linguistic capabilities and domain understanding, hence can help evaluate conceptual soundness and identify model weaknesses. However, a major challenge is the creation of test cases. The current packages rely on semi-automated approach using manual development which requires domain expertise and can be time consuming. This paper introduces an automated approach to develop test cases by exploiting t"},"claims":{"count":0,"items":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"2408.00161","kind":"arxiv","version":2},"verdict":{"id":null,"model_set":{},"created_at":null,"strongest_claim":"","one_line_summary":"","pipeline_version":null,"weakest_assumption":"","pith_extraction_headline":""},"integrity":{"clean":true,"summary":{"advisory":0,"critical":0,"by_detector":{},"informational":0},"endpoint":"/pith/2408.00161/integrity.json","findings":[],"available":true,"detectors_run":[],"snapshot_sha256":"c28c3603d3b5d939e8dc4c7e95fa8dfce3d595e45f758748cecf8e644a296938"},"references":{"count":0,"sample":[],"resolved_work":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","internal_anchors":0},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"author_claims":{"count":0,"strong_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"builder_version":"pith-number-builder-2026-05-17-v1"},"verdict_id":null},"signer":{"signer_id":"pith.science","signer_type":"pith_registry","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"created_at":"2026-07-05T08:53:24Z","supersedes":[],"prev_event":null,"signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"nP8NZKazjWFMjM1CQGpLKT82l9ZVg+O4uEPaoffffQosJupskGJct0OeSMQfdcUl6Y4lD6aIOiJfjc5mOeElAw==","signed_message":"open_graph_event_sha256_bytes","signed_at":"2026-07-07T15:59:44.610678Z"},"content_sha256":"f370107507f20a67dc59559bab3d8ef13ef968a8f7bcc7abe57a1e6242ec9968","schema_version":"1.0","event_id":"sha256:f370107507f20a67dc59559bab3d8ef13ef968a8f7bcc7abe57a1e6242ec9968"}],"timestamp_proofs":[],"mirror_hints":[{"mirror_type":"https","name":"Pith Resolver","base_url":"https://pith.science","bundle_url":"https://pith.science/pith/5OPWKVKJQD5B3MYGIIYINE3GZM/bundle.json","state_url":"https://pith.science/pith/5OPWKVKJQD5B3MYGIIYINE3GZM/state.json","well_known_bundle_url":"https://pith.science/.well-known/pith/5OPWKVKJQD5B3MYGIIYINE3GZM/bundle.json","status":"primary"}],"public_keys":[{"key_id":"pith-v1-2026-05","algorithm":"ed25519","format":"raw","public_key_b64":"stVStoiQhXFxp4s2pdzPNoqVNBMojDU/fJ2db5S3CbM=","public_key_hex":"b2d552b68890857171a78b36a5dccf368a953413288c353f7c9d9d6f94b709b3","fingerprint_sha256_b32_first128bits":"RVFV5Z2OI2J3ZUO7ERDEBCYNKS","fingerprint_sha256_hex":"8d4b5ee74e4693bcd1df2446408b0d54","rotates_at":null,"url":"https://pith.science/pith-signing-key.json","notes":"Pith uses this Ed25519 key to sign canonical record SHA-256 digests. Verify with: ed25519_verify(public_key, message=canonical_sha256_bytes, signature=base64decode(signature_b64))."}],"merge_version":"pith-open-graph-merge-v1","built_at":"2026-07-07T15:59:44Z","links":{"resolver":"https://pith.science/pith/5OPWKVKJQD5B3MYGIIYINE3GZM","bundle":"https://pith.science/pith/5OPWKVKJQD5B3MYGIIYINE3GZM/bundle.json","state":"https://pith.science/pith/5OPWKVKJQD5B3MYGIIYINE3GZM/state.json","well_known_bundle":"https://pith.science/.well-known/pith/5OPWKVKJQD5B3MYGIIYINE3GZM/bundle.json"},"state":{"state_type":"pith_open_graph_state","state_version":"1.0","pith_number":"pith:2024:5OPWKVKJQD5B3MYGIIYINE3GZM","merge_version":"pith-open-graph-merge-v1","event_count":2,"valid_event_count":2,"invalid_event_count":0,"equivocation_count":0,"current":{"canonical_record":{"metadata":{"abstract_canon_sha256":"618262aea00f6377a7ed51832927e593d78676b50e45d1bdfa11daec1d793e7d","cross_cats_sorted":["cs.AI","cs.ET","cs.LG"],"license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","primary_cat":"cs.CL","submitted_at":"2024-07-31T21:12:21Z","title_canon_sha256":"87e0f613a165e96775666e8b46369ffcd44a73d7af2e1df67984ccda1f94b9bb"},"schema_version":"1.0","source":{"id":"2408.00161","kind":"arxiv","version":2}},"source_aliases":[{"alias_kind":"arxiv","alias_value":"2408.00161","created_at":"2026-07-05T08:53:24Z"},{"alias_kind":"arxiv_version","alias_value":"2408.00161v2","created_at":"2026-07-05T08:53:24Z"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.2408.00161","created_at":"2026-07-05T08:53:24Z"},{"alias_kind":"pith_short_12","alias_value":"5OPWKVKJQD5B","created_at":"2026-07-05T08:53:24Z"},{"alias_kind":"pith_short_16","alias_value":"5OPWKVKJQD5B3MYG","created_at":"2026-07-05T08:53:24Z"},{"alias_kind":"pith_short_8","alias_value":"5OPWKVKJ","created_at":"2026-07-05T08:53:24Z"}],"graph_snapshots":[{"event_id":"sha256:f370107507f20a67dc59559bab3d8ef13ef968a8f7bcc7abe57a1e6242ec9968","target":"graph","created_at":"2026-07-05T08:53:24Z","signer":{"key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signer_id":"pith.science","signer_type":"pith_registry"},"payload":{"graph_snapshot":{"author_claims":{"count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","strong_count":0},"builder_version":"pith-number-builder-2026-05-17-v1","claims":{"count":0,"items":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"integrity":{"available":true,"clean":true,"detectors_run":[],"endpoint":"/pith/2408.00161/integrity.json","findings":[],"snapshot_sha256":"c28c3603d3b5d939e8dc4c7e95fa8dfce3d595e45f758748cecf8e644a296938","summary":{"advisory":0,"by_detector":{},"critical":0,"informational":0}},"paper":{"abstract_excerpt":"Recent work in behavioral testing for natural language processing (NLP) models, such as Checklist, is inspired by related paradigms in software engineering testing. They allow evaluation of general linguistic capabilities and domain understanding, hence can help evaluate conceptual soundness and identify model weaknesses. However, a major challenge is the creation of test cases. The current packages rely on semi-automated approach using manual development which requires domain expertise and can be time consuming. This paper introduces an automated approach to develop test cases by exploiting t","authors_text":"Agus Sudjianto, Rahul Singh, Tarun Joshi, Ying Li","cross_cats":["cs.AI","cs.ET","cs.LG"],"headline":"","license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","primary_cat":"cs.CL","submitted_at":"2024-07-31T21:12:21Z","title":"Automatic Generation of Behavioral Test Cases For Natural Language Processing Using Clustering and Prompting"},"references":{"count":0,"internal_anchors":0,"resolved_work":0,"sample":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"2408.00161","kind":"arxiv","version":2},"verdict":{"created_at":null,"id":null,"model_set":{},"one_line_summary":"","pipeline_version":null,"pith_extraction_headline":"","strongest_claim":"","weakest_assumption":""}},"verdict_id":null}}],"author_attestations":[],"timestamp_anchors":[],"storage_attestations":[],"citation_signatures":[],"replication_records":[],"corrections":[],"mirror_hints":[],"record_created":{"event_id":"sha256:9635154da64feb2c2beaa4a0805c44cd16691bd17e9b5f5ff41a90c40b81e836","target":"record","created_at":"2026-07-05T08:53:24Z","signer":{"key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signer_id":"pith.science","signer_type":"pith_registry"},"payload":{"attestation_state":"computed","canonical_record":{"metadata":{"abstract_canon_sha256":"618262aea00f6377a7ed51832927e593d78676b50e45d1bdfa11daec1d793e7d","cross_cats_sorted":["cs.AI","cs.ET","cs.LG"],"license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","primary_cat":"cs.CL","submitted_at":"2024-07-31T21:12:21Z","title_canon_sha256":"87e0f613a165e96775666e8b46369ffcd44a73d7af2e1df67984ccda1f94b9bb"},"schema_version":"1.0","source":{"id":"2408.00161","kind":"arxiv","version":2}},"canonical_sha256":"eb9f65554980fa1db3064230869366cb22ba69c9aa657aed893c7b6c46d5efd3","receipt":{"algorithm":"ed25519","builder_version":"pith-number-builder-2026-05-17-v1","canonical_sha256":"eb9f65554980fa1db3064230869366cb22ba69c9aa657aed893c7b6c46d5efd3","first_computed_at":"2026-07-05T08:53:24.757935Z","key_id":"pith-v1-2026-05","kind":"pith_receipt","last_reissued_at":"2026-07-05T08:53:24.757935Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","receipt_version":"0.3","signature_b64":"fpGWWPjMeXYUujjXQnpkMdSpidnuKac32yaNR0xp13bLqxb3aJE+KgUmJhyKQui9hDE2BxouYWhGetWbYpX4AA==","signature_status":"signed_v1","signed_at":"2026-07-05T08:53:24.758427Z","signed_message":"canonical_sha256_bytes"},"source_id":"2408.00161","source_kind":"arxiv","source_version":2}}},"equivocations":[],"invalid_events":[],"applied_event_ids":["sha256:9635154da64feb2c2beaa4a0805c44cd16691bd17e9b5f5ff41a90c40b81e836","sha256:f370107507f20a67dc59559bab3d8ef13ef968a8f7bcc7abe57a1e6242ec9968"],"state_sha256":"eabc449d4d97e2c438430a902069a410f1b4f861c68ebf49102315aa98123d89"},"bundle_signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"IG45RwsP1rO/DJcj4KabNYqMMR8GGiUkPMgQ0r9ZWv1ExAFAKJ/ffUJWV0CDG7BsL1YWJRs1PfiVBWxVdn1pBg==","signed_message":"bundle_sha256_bytes","signed_at":"2026-07-07T15:59:44.612942Z","bundle_sha256":"bb78a9d1c9ba3498e3c3a3f6f303d251f2b2581252d1b4f01b81978f57d2d26e"}}