{"bundle_type":"pith_open_graph_bundle","bundle_version":"1.0","pith_number":"pith:2024:APGIE3RXEUCS7VY2YXJKWJOLNU","short_pith_number":"pith:APGIE3RX","canonical_record":{"source":{"id":"2406.04531","kind":"arxiv","version":2},"metadata":{"license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.SE","submitted_at":"2024-06-06T22:07:50Z","cross_cats_sorted":[],"title_canon_sha256":"36a039ee539c0e23d3584e30cc3b6ecb24963c6b93f5a1b2374ef55d4f89a89c","abstract_canon_sha256":"1df7ad2d4f077b0f7425e41356cabf766d6a5e23d56c17825399df13ac3bb5c7"},"schema_version":"1.0"},"canonical_sha256":"03cc826e3725052fd71ac5d2ab25cb6d0a3b5ab45743fe97e1540a485046204b","source":{"kind":"arxiv","id":"2406.04531","version":2},"source_aliases":[{"alias_kind":"arxiv","alias_value":"2406.04531","created_at":"2026-07-05T10:08:17Z"},{"alias_kind":"arxiv_version","alias_value":"2406.04531v2","created_at":"2026-07-05T10:08:17Z"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.2406.04531","created_at":"2026-07-05T10:08:17Z"},{"alias_kind":"pith_short_12","alias_value":"APGIE3RXEUCS","created_at":"2026-07-05T10:08:17Z"},{"alias_kind":"pith_short_16","alias_value":"APGIE3RXEUCS7VY2","created_at":"2026-07-05T10:08:17Z"},{"alias_kind":"pith_short_8","alias_value":"APGIE3RX","created_at":"2026-07-05T10:08:17Z"}],"events":[{"event_type":"record_created","subject_pith_number":"pith:2024:APGIE3RXEUCS7VY2YXJKWJOLNU","target":"record","payload":{"canonical_record":{"source":{"id":"2406.04531","kind":"arxiv","version":2},"metadata":{"license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.SE","submitted_at":"2024-06-06T22:07:50Z","cross_cats_sorted":[],"title_canon_sha256":"36a039ee539c0e23d3584e30cc3b6ecb24963c6b93f5a1b2374ef55d4f89a89c","abstract_canon_sha256":"1df7ad2d4f077b0f7425e41356cabf766d6a5e23d56c17825399df13ac3bb5c7"},"schema_version":"1.0"},"canonical_sha256":"03cc826e3725052fd71ac5d2ab25cb6d0a3b5ab45743fe97e1540a485046204b","receipt":{"kind":"pith_receipt","key_id":"pith-v1-2026-05","algorithm":"ed25519","signed_at":"2026-07-05T10:08:17.733960Z","signature_b64":"O3PbRrPrA6ckl4b/6edYs7V2Vow5qo9XgcI7LbJ+6TPj0Xu7p0g8THhE6yMabqsb3nht+H95dQY5Sg1rYtdjBg==","signed_message":"canonical_sha256_bytes","builder_version":"pith-number-builder-2026-05-17-v1","receipt_version":"0.3","canonical_sha256":"03cc826e3725052fd71ac5d2ab25cb6d0a3b5ab45743fe97e1540a485046204b","last_reissued_at":"2026-07-05T10:08:17.733382Z","signature_status":"signed_v1","first_computed_at":"2026-07-05T10:08:17.733382Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"source_kind":"arxiv","source_id":"2406.04531","source_version":2,"attestation_state":"computed"},"signer":{"signer_id":"pith.science","signer_type":"pith_registry","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"created_at":"2026-07-05T10:08:17Z","supersedes":[],"prev_event":null,"signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"/bGILF428DPxhnwEGDbjNMbfWJD0UBOz5upC9s1SMpEr8UEpQYT5Mck/5tk1mYw0jrznX53f2QDJjTd3rN9eDg==","signed_message":"open_graph_event_sha256_bytes","signed_at":"2026-08-07T17:08:20.218069Z"},"content_sha256":"80df3bdcb8b53bd04dec33a664a2d5ecf39f00595c41363a2209cc114955fe32","schema_version":"1.0","event_id":"sha256:80df3bdcb8b53bd04dec33a664a2d5ecf39f00595c41363a2209cc114955fe32"},{"event_type":"graph_snapshot","subject_pith_number":"pith:2024:APGIE3RXEUCS7VY2YXJKWJOLNU","target":"graph","payload":{"graph_snapshot":{"paper":{"title":"TESTEVAL: Benchmarking Large Language Models for Test Case Generation","license":"http://creativecommons.org/licenses/by/4.0/","headline":"","cross_cats":[],"primary_cat":"cs.SE","authors_text":"An Ran Chen, Chenyuan Yang, Da Song, Lei Ma, Lingming Zhang, Wenhan Wang, Yuheng Huang, Zhaoyang Chu, Zhijie Wang","submitted_at":"2024-06-06T22:07:50Z","abstract_excerpt":"Testing plays a crucial role in the software development cycle, enabling the detection of bugs, vulnerabilities, and other undesirable behaviors. To perform software testing, testers need to write code snippets that execute the program under test. Recently, researchers have recognized the potential of large language models (LLMs) in software testing. However, there remains a lack of fair comparisons between different LLMs in terms of test case generation capabilities.\n  In this paper, we propose TESTEVAL, a novel benchmark for test case generation with LLMs. We collect 210 Python programs from"},"claims":{"count":0,"items":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"2406.04531","kind":"arxiv","version":2},"verdict":{"id":null,"model_set":{},"created_at":null,"strongest_claim":"","one_line_summary":"","pipeline_version":null,"weakest_assumption":"","pith_extraction_headline":""},"integrity":{"clean":true,"summary":{"advisory":0,"critical":0,"by_detector":{},"informational":0},"endpoint":"/pith/2406.04531/integrity.json","findings":[],"available":true,"detectors_run":[],"snapshot_sha256":"c28c3603d3b5d939e8dc4c7e95fa8dfce3d595e45f758748cecf8e644a296938"},"references":{"count":0,"sample":[],"resolved_work":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","internal_anchors":0},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"author_claims":{"count":0,"strong_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"builder_version":"pith-number-builder-2026-05-17-v1"},"verdict_id":null},"signer":{"signer_id":"pith.science","signer_type":"pith_registry","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"created_at":"2026-07-05T10:08:17Z","supersedes":[],"prev_event":null,"signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"qpVYQt0VgJ3aqlP0PtwDlw4qHP7e0twbE6n6I3eNDbGZtg9sTlkstwHo6TQ3ouRofEjfpFZX9aIracfB7lhZCQ==","signed_message":"open_graph_event_sha256_bytes","signed_at":"2026-08-07T17:08:20.218579Z"},"content_sha256":"79869628d00af0566c4328415e882e389b3ecc05782a3fef3302e13256016387","schema_version":"1.0","event_id":"sha256:79869628d00af0566c4328415e882e389b3ecc05782a3fef3302e13256016387"}],"timestamp_proofs":[],"mirror_hints":[{"mirror_type":"https","name":"Pith Resolver","base_url":"https://pith.science","bundle_url":"https://pith.science/pith/APGIE3RXEUCS7VY2YXJKWJOLNU/bundle.json","state_url":"https://pith.science/pith/APGIE3RXEUCS7VY2YXJKWJOLNU/state.json","well_known_bundle_url":"https://pith.science/.well-known/pith/APGIE3RXEUCS7VY2YXJKWJOLNU/bundle.json","status":"primary"}],"public_keys":[{"key_id":"pith-v1-2026-05","algorithm":"ed25519","format":"raw","public_key_b64":"stVStoiQhXFxp4s2pdzPNoqVNBMojDU/fJ2db5S3CbM=","public_key_hex":"b2d552b68890857171a78b36a5dccf368a953413288c353f7c9d9d6f94b709b3","fingerprint_sha256_b32_first128bits":"RVFV5Z2OI2J3ZUO7ERDEBCYNKS","fingerprint_sha256_hex":"8d4b5ee74e4693bcd1df2446408b0d54","rotates_at":null,"url":"https://pith.science/pith-signing-key.json","notes":"Pith uses this Ed25519 key to sign canonical record SHA-256 digests. Verify with: ed25519_verify(public_key, message=canonical_sha256_bytes, signature=base64decode(signature_b64))."}],"merge_version":"pith-open-graph-merge-v1","built_at":"2026-08-07T17:08:20Z","links":{"resolver":"https://pith.science/pith/APGIE3RXEUCS7VY2YXJKWJOLNU","bundle":"https://pith.science/pith/APGIE3RXEUCS7VY2YXJKWJOLNU/bundle.json","state":"https://pith.science/pith/APGIE3RXEUCS7VY2YXJKWJOLNU/state.json","well_known_bundle":"https://pith.science/.well-known/pith/APGIE3RXEUCS7VY2YXJKWJOLNU/bundle.json"},"state":{"state_type":"pith_open_graph_state","state_version":"1.0","pith_number":"pith:2024:APGIE3RXEUCS7VY2YXJKWJOLNU","merge_version":"pith-open-graph-merge-v1","event_count":2,"valid_event_count":2,"invalid_event_count":0,"equivocation_count":0,"current":{"canonical_record":{"metadata":{"abstract_canon_sha256":"1df7ad2d4f077b0f7425e41356cabf766d6a5e23d56c17825399df13ac3bb5c7","cross_cats_sorted":[],"license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.SE","submitted_at":"2024-06-06T22:07:50Z","title_canon_sha256":"36a039ee539c0e23d3584e30cc3b6ecb24963c6b93f5a1b2374ef55d4f89a89c"},"schema_version":"1.0","source":{"id":"2406.04531","kind":"arxiv","version":2}},"source_aliases":[{"alias_kind":"arxiv","alias_value":"2406.04531","created_at":"2026-07-05T10:08:17Z"},{"alias_kind":"arxiv_version","alias_value":"2406.04531v2","created_at":"2026-07-05T10:08:17Z"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.2406.04531","created_at":"2026-07-05T10:08:17Z"},{"alias_kind":"pith_short_12","alias_value":"APGIE3RXEUCS","created_at":"2026-07-05T10:08:17Z"},{"alias_kind":"pith_short_16","alias_value":"APGIE3RXEUCS7VY2","created_at":"2026-07-05T10:08:17Z"},{"alias_kind":"pith_short_8","alias_value":"APGIE3RX","created_at":"2026-07-05T10:08:17Z"}],"graph_snapshots":[{"event_id":"sha256:79869628d00af0566c4328415e882e389b3ecc05782a3fef3302e13256016387","target":"graph","created_at":"2026-07-05T10:08:17Z","signer":{"key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signer_id":"pith.science","signer_type":"pith_registry"},"payload":{"graph_snapshot":{"author_claims":{"count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","strong_count":0},"builder_version":"pith-number-builder-2026-05-17-v1","claims":{"count":0,"items":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"integrity":{"available":true,"clean":true,"detectors_run":[],"endpoint":"/pith/2406.04531/integrity.json","findings":[],"snapshot_sha256":"c28c3603d3b5d939e8dc4c7e95fa8dfce3d595e45f758748cecf8e644a296938","summary":{"advisory":0,"by_detector":{},"critical":0,"informational":0}},"paper":{"abstract_excerpt":"Testing plays a crucial role in the software development cycle, enabling the detection of bugs, vulnerabilities, and other undesirable behaviors. To perform software testing, testers need to write code snippets that execute the program under test. Recently, researchers have recognized the potential of large language models (LLMs) in software testing. However, there remains a lack of fair comparisons between different LLMs in terms of test case generation capabilities.\n  In this paper, we propose TESTEVAL, a novel benchmark for test case generation with LLMs. We collect 210 Python programs from","authors_text":"An Ran Chen, Chenyuan Yang, Da Song, Lei Ma, Lingming Zhang, Wenhan Wang, Yuheng Huang, Zhaoyang Chu, Zhijie Wang","cross_cats":[],"headline":"","license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.SE","submitted_at":"2024-06-06T22:07:50Z","title":"TESTEVAL: Benchmarking Large Language Models for Test Case Generation"},"references":{"count":0,"internal_anchors":0,"resolved_work":0,"sample":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"2406.04531","kind":"arxiv","version":2},"verdict":{"created_at":null,"id":null,"model_set":{},"one_line_summary":"","pipeline_version":null,"pith_extraction_headline":"","strongest_claim":"","weakest_assumption":""}},"verdict_id":null}}],"author_attestations":[],"timestamp_anchors":[],"storage_attestations":[],"citation_signatures":[],"replication_records":[],"corrections":[],"mirror_hints":[],"record_created":{"event_id":"sha256:80df3bdcb8b53bd04dec33a664a2d5ecf39f00595c41363a2209cc114955fe32","target":"record","created_at":"2026-07-05T10:08:17Z","signer":{"key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signer_id":"pith.science","signer_type":"pith_registry"},"payload":{"attestation_state":"computed","canonical_record":{"metadata":{"abstract_canon_sha256":"1df7ad2d4f077b0f7425e41356cabf766d6a5e23d56c17825399df13ac3bb5c7","cross_cats_sorted":[],"license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.SE","submitted_at":"2024-06-06T22:07:50Z","title_canon_sha256":"36a039ee539c0e23d3584e30cc3b6ecb24963c6b93f5a1b2374ef55d4f89a89c"},"schema_version":"1.0","source":{"id":"2406.04531","kind":"arxiv","version":2}},"canonical_sha256":"03cc826e3725052fd71ac5d2ab25cb6d0a3b5ab45743fe97e1540a485046204b","receipt":{"algorithm":"ed25519","builder_version":"pith-number-builder-2026-05-17-v1","canonical_sha256":"03cc826e3725052fd71ac5d2ab25cb6d0a3b5ab45743fe97e1540a485046204b","first_computed_at":"2026-07-05T10:08:17.733382Z","key_id":"pith-v1-2026-05","kind":"pith_receipt","last_reissued_at":"2026-07-05T10:08:17.733382Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","receipt_version":"0.3","signature_b64":"O3PbRrPrA6ckl4b/6edYs7V2Vow5qo9XgcI7LbJ+6TPj0Xu7p0g8THhE6yMabqsb3nht+H95dQY5Sg1rYtdjBg==","signature_status":"signed_v1","signed_at":"2026-07-05T10:08:17.733960Z","signed_message":"canonical_sha256_bytes"},"source_id":"2406.04531","source_kind":"arxiv","source_version":2}}},"equivocations":[],"invalid_events":[],"applied_event_ids":["sha256:80df3bdcb8b53bd04dec33a664a2d5ecf39f00595c41363a2209cc114955fe32","sha256:79869628d00af0566c4328415e882e389b3ecc05782a3fef3302e13256016387"],"state_sha256":"0587160fcb732a4a4b3cc8e72fa880a0a1209dcf6aa72a8fd36b2c911a0054a7"},"bundle_signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"DrrWi3U0tYFm4iaGK9/TJZYp5+0WPywJJu9phoJ+PjjZM6cRcD3PoMeV6dPudmO9UPspSLV5bE8QQW7n1BXTBQ==","signed_message":"bundle_sha256_bytes","signed_at":"2026-08-07T17:08:20.222552Z","bundle_sha256":"0366065266ff901babbfd809fecb604148d0d0b7b5c3fdb48cc3e1d3041ace39"}}