{"bundle_type":"pith_open_graph_bundle","bundle_version":"1.0","pith_number":"pith:2024:EW27QSKLMHWA6JJHS6QDKBDXN3","short_pith_number":"pith:EW27QSKL","canonical_record":{"source":{"id":"2406.08216","kind":"arxiv","version":1},"metadata":{"license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.SE","submitted_at":"2024-06-12T13:45:45Z","cross_cats_sorted":[],"title_canon_sha256":"9e247cfea7fb5c43fffdb4417d084c4cfd8c98ef133590ef24316b9cc6392c8c","abstract_canon_sha256":"1e0fc20050ab6234dd5c8cfef36d6604430fe455e6db48e9dacd49eff1bea469"},"schema_version":"1.0"},"canonical_sha256":"25b5f8494b61ec0f252797a03504776efce66505c12b0bd28af5b942eefbe54f","source":{"kind":"arxiv","id":"2406.08216","version":1},"source_aliases":[{"alias_kind":"arxiv","alias_value":"2406.08216","created_at":"2026-07-05T08:30:56Z"},{"alias_kind":"arxiv_version","alias_value":"2406.08216v1","created_at":"2026-07-05T08:30:56Z"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.2406.08216","created_at":"2026-07-05T08:30:56Z"},{"alias_kind":"pith_short_12","alias_value":"EW27QSKLMHWA","created_at":"2026-07-05T08:30:56Z"},{"alias_kind":"pith_short_16","alias_value":"EW27QSKLMHWA6JJH","created_at":"2026-07-05T08:30:56Z"},{"alias_kind":"pith_short_8","alias_value":"EW27QSKL","created_at":"2026-07-05T08:30:56Z"}],"events":[{"event_type":"record_created","subject_pith_number":"pith:2024:EW27QSKLMHWA6JJHS6QDKBDXN3","target":"record","payload":{"canonical_record":{"source":{"id":"2406.08216","kind":"arxiv","version":1},"metadata":{"license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.SE","submitted_at":"2024-06-12T13:45:45Z","cross_cats_sorted":[],"title_canon_sha256":"9e247cfea7fb5c43fffdb4417d084c4cfd8c98ef133590ef24316b9cc6392c8c","abstract_canon_sha256":"1e0fc20050ab6234dd5c8cfef36d6604430fe455e6db48e9dacd49eff1bea469"},"schema_version":"1.0"},"canonical_sha256":"25b5f8494b61ec0f252797a03504776efce66505c12b0bd28af5b942eefbe54f","receipt":{"kind":"pith_receipt","key_id":"pith-v1-2026-05","algorithm":"ed25519","signed_at":"2026-07-05T08:30:56.787789Z","signature_b64":"A11+5Kj8BPqUrl6kCcYHVFzf7qurCVDokICcOv8yhG3MlMHnRqKdIkBNsX2RcTPj+mA/gIkjp9xAOaXQ75u9Dg==","signed_message":"canonical_sha256_bytes","builder_version":"pith-number-builder-2026-05-17-v1","receipt_version":"0.3","canonical_sha256":"25b5f8494b61ec0f252797a03504776efce66505c12b0bd28af5b942eefbe54f","last_reissued_at":"2026-07-05T08:30:56.787330Z","signature_status":"signed_v1","first_computed_at":"2026-07-05T08:30:56.787330Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"source_kind":"arxiv","source_id":"2406.08216","source_version":1,"attestation_state":"computed"},"signer":{"signer_id":"pith.science","signer_type":"pith_registry","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"created_at":"2026-07-05T08:30:56Z","supersedes":[],"prev_event":null,"signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"3emQcgI1/UE7Y6+0K7V7HpWaWpM7ohfLR5v5vw5DlJshnmZrwfTRSyA0kU4p615MlU2wD/mkI1SMis8++kIcBw==","signed_message":"open_graph_event_sha256_bytes","signed_at":"2026-08-18T03:06:00.370695Z"},"content_sha256":"50dcc3dca9e70dfc797b682dc7e789eb64c205d2888a092616d2c83857ff1aa5","schema_version":"1.0","event_id":"sha256:50dcc3dca9e70dfc797b682dc7e789eb64c205d2888a092616d2c83857ff1aa5"},{"event_type":"graph_snapshot","subject_pith_number":"pith:2024:EW27QSKLMHWA6JJHS6QDKBDXN3","target":"graph","payload":{"graph_snapshot":{"paper":{"title":"A Software Engineering Perspective on Testing Large Language Models: Research, Practice, Tools and Benchmarks","license":"http://creativecommons.org/licenses/by/4.0/","headline":"","cross_cats":[],"primary_cat":"cs.SE","authors_text":"Boyue Caroline Hu, Marsha Chechik, Sinclair Hudson, Sophia Jit","submitted_at":"2024-06-12T13:45:45Z","abstract_excerpt":"Large Language Models (LLMs) are rapidly becoming ubiquitous both as stand-alone tools and as components of current and future software systems. To enable usage of LLMs in the high-stake or safety-critical systems of 2030, they need to undergo rigorous testing. Software Engineering (SE) research on testing Machine Learning (ML) components and ML-based systems has systematically explored many topics such as test input generation and robustness. We believe knowledge about tools, benchmarks, research and practitioner views related to LLM testing needs to be similarly organized. To this end, we pr"},"claims":{"count":0,"items":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"2406.08216","kind":"arxiv","version":1},"verdict":{"id":null,"model_set":{},"created_at":null,"strongest_claim":"","one_line_summary":"","pipeline_version":null,"weakest_assumption":"","pith_extraction_headline":""},"integrity":{"clean":true,"summary":{"advisory":0,"critical":0,"by_detector":{},"informational":0},"endpoint":"/pith/2406.08216/integrity.json","findings":[],"available":true,"detectors_run":[],"snapshot_sha256":"c28c3603d3b5d939e8dc4c7e95fa8dfce3d595e45f758748cecf8e644a296938"},"references":{"count":0,"sample":[],"resolved_work":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","internal_anchors":0},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"author_claims":{"count":0,"strong_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"builder_version":"pith-number-builder-2026-05-17-v1"},"verdict_id":null},"signer":{"signer_id":"pith.science","signer_type":"pith_registry","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"created_at":"2026-07-05T08:30:56Z","supersedes":[],"prev_event":null,"signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"+5U8b/qGkjR5eWQsGwMD7ZSwCB4lsbmx9EtAL1pTedzXC0A67afdfLeaqG8QPUAWEYbenyiBFrPK9S7gx7t+Aw==","signed_message":"open_graph_event_sha256_bytes","signed_at":"2026-08-18T03:06:00.371643Z"},"content_sha256":"b5ea1b81bf53141d0379a100fa35c8b4269a336fe0bd10a04e39382b09d2591c","schema_version":"1.0","event_id":"sha256:b5ea1b81bf53141d0379a100fa35c8b4269a336fe0bd10a04e39382b09d2591c"}],"timestamp_proofs":[],"mirror_hints":[{"mirror_type":"https","name":"Pith Resolver","base_url":"https://pith.science","bundle_url":"https://pith.science/pith/EW27QSKLMHWA6JJHS6QDKBDXN3/bundle.json","state_url":"https://pith.science/pith/EW27QSKLMHWA6JJHS6QDKBDXN3/state.json","well_known_bundle_url":"https://pith.science/.well-known/pith/EW27QSKLMHWA6JJHS6QDKBDXN3/bundle.json","status":"primary"}],"public_keys":[{"key_id":"pith-v1-2026-05","algorithm":"ed25519","format":"raw","public_key_b64":"stVStoiQhXFxp4s2pdzPNoqVNBMojDU/fJ2db5S3CbM=","public_key_hex":"b2d552b68890857171a78b36a5dccf368a953413288c353f7c9d9d6f94b709b3","fingerprint_sha256_b32_first128bits":"RVFV5Z2OI2J3ZUO7ERDEBCYNKS","fingerprint_sha256_hex":"8d4b5ee74e4693bcd1df2446408b0d54","rotates_at":null,"url":"https://pith.science/pith-signing-key.json","notes":"Pith uses this Ed25519 key to sign canonical record SHA-256 digests. Verify with: ed25519_verify(public_key, message=canonical_sha256_bytes, signature=base64decode(signature_b64))."}],"merge_version":"pith-open-graph-merge-v1","built_at":"2026-08-18T03:06:00Z","links":{"resolver":"https://pith.science/pith/EW27QSKLMHWA6JJHS6QDKBDXN3","bundle":"https://pith.science/pith/EW27QSKLMHWA6JJHS6QDKBDXN3/bundle.json","state":"https://pith.science/pith/EW27QSKLMHWA6JJHS6QDKBDXN3/state.json","well_known_bundle":"https://pith.science/.well-known/pith/EW27QSKLMHWA6JJHS6QDKBDXN3/bundle.json"},"state":{"state_type":"pith_open_graph_state","state_version":"1.0","pith_number":"pith:2024:EW27QSKLMHWA6JJHS6QDKBDXN3","merge_version":"pith-open-graph-merge-v1","event_count":2,"valid_event_count":2,"invalid_event_count":0,"equivocation_count":0,"current":{"canonical_record":{"metadata":{"abstract_canon_sha256":"1e0fc20050ab6234dd5c8cfef36d6604430fe455e6db48e9dacd49eff1bea469","cross_cats_sorted":[],"license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.SE","submitted_at":"2024-06-12T13:45:45Z","title_canon_sha256":"9e247cfea7fb5c43fffdb4417d084c4cfd8c98ef133590ef24316b9cc6392c8c"},"schema_version":"1.0","source":{"id":"2406.08216","kind":"arxiv","version":1}},"source_aliases":[{"alias_kind":"arxiv","alias_value":"2406.08216","created_at":"2026-07-05T08:30:56Z"},{"alias_kind":"arxiv_version","alias_value":"2406.08216v1","created_at":"2026-07-05T08:30:56Z"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.2406.08216","created_at":"2026-07-05T08:30:56Z"},{"alias_kind":"pith_short_12","alias_value":"EW27QSKLMHWA","created_at":"2026-07-05T08:30:56Z"},{"alias_kind":"pith_short_16","alias_value":"EW27QSKLMHWA6JJH","created_at":"2026-07-05T08:30:56Z"},{"alias_kind":"pith_short_8","alias_value":"EW27QSKL","created_at":"2026-07-05T08:30:56Z"}],"graph_snapshots":[{"event_id":"sha256:b5ea1b81bf53141d0379a100fa35c8b4269a336fe0bd10a04e39382b09d2591c","target":"graph","created_at":"2026-07-05T08:30:56Z","signer":{"key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signer_id":"pith.science","signer_type":"pith_registry"},"payload":{"graph_snapshot":{"author_claims":{"count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","strong_count":0},"builder_version":"pith-number-builder-2026-05-17-v1","claims":{"count":0,"items":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"integrity":{"available":true,"clean":true,"detectors_run":[],"endpoint":"/pith/2406.08216/integrity.json","findings":[],"snapshot_sha256":"c28c3603d3b5d939e8dc4c7e95fa8dfce3d595e45f758748cecf8e644a296938","summary":{"advisory":0,"by_detector":{},"critical":0,"informational":0}},"paper":{"abstract_excerpt":"Large Language Models (LLMs) are rapidly becoming ubiquitous both as stand-alone tools and as components of current and future software systems. To enable usage of LLMs in the high-stake or safety-critical systems of 2030, they need to undergo rigorous testing. Software Engineering (SE) research on testing Machine Learning (ML) components and ML-based systems has systematically explored many topics such as test input generation and robustness. We believe knowledge about tools, benchmarks, research and practitioner views related to LLM testing needs to be similarly organized. To this end, we pr","authors_text":"Boyue Caroline Hu, Marsha Chechik, Sinclair Hudson, Sophia Jit","cross_cats":[],"headline":"","license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.SE","submitted_at":"2024-06-12T13:45:45Z","title":"A Software Engineering Perspective on Testing Large Language Models: Research, Practice, Tools and Benchmarks"},"references":{"count":0,"internal_anchors":0,"resolved_work":0,"sample":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"2406.08216","kind":"arxiv","version":1},"verdict":{"created_at":null,"id":null,"model_set":{},"one_line_summary":"","pipeline_version":null,"pith_extraction_headline":"","strongest_claim":"","weakest_assumption":""}},"verdict_id":null}}],"author_attestations":[],"timestamp_anchors":[],"storage_attestations":[],"citation_signatures":[],"replication_records":[],"corrections":[],"mirror_hints":[],"record_created":{"event_id":"sha256:50dcc3dca9e70dfc797b682dc7e789eb64c205d2888a092616d2c83857ff1aa5","target":"record","created_at":"2026-07-05T08:30:56Z","signer":{"key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signer_id":"pith.science","signer_type":"pith_registry"},"payload":{"attestation_state":"computed","canonical_record":{"metadata":{"abstract_canon_sha256":"1e0fc20050ab6234dd5c8cfef36d6604430fe455e6db48e9dacd49eff1bea469","cross_cats_sorted":[],"license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.SE","submitted_at":"2024-06-12T13:45:45Z","title_canon_sha256":"9e247cfea7fb5c43fffdb4417d084c4cfd8c98ef133590ef24316b9cc6392c8c"},"schema_version":"1.0","source":{"id":"2406.08216","kind":"arxiv","version":1}},"canonical_sha256":"25b5f8494b61ec0f252797a03504776efce66505c12b0bd28af5b942eefbe54f","receipt":{"algorithm":"ed25519","builder_version":"pith-number-builder-2026-05-17-v1","canonical_sha256":"25b5f8494b61ec0f252797a03504776efce66505c12b0bd28af5b942eefbe54f","first_computed_at":"2026-07-05T08:30:56.787330Z","key_id":"pith-v1-2026-05","kind":"pith_receipt","last_reissued_at":"2026-07-05T08:30:56.787330Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","receipt_version":"0.3","signature_b64":"A11+5Kj8BPqUrl6kCcYHVFzf7qurCVDokICcOv8yhG3MlMHnRqKdIkBNsX2RcTPj+mA/gIkjp9xAOaXQ75u9Dg==","signature_status":"signed_v1","signed_at":"2026-07-05T08:30:56.787789Z","signed_message":"canonical_sha256_bytes"},"source_id":"2406.08216","source_kind":"arxiv","source_version":1}}},"equivocations":[],"invalid_events":[],"applied_event_ids":["sha256:50dcc3dca9e70dfc797b682dc7e789eb64c205d2888a092616d2c83857ff1aa5","sha256:b5ea1b81bf53141d0379a100fa35c8b4269a336fe0bd10a04e39382b09d2591c"],"state_sha256":"e9d66fce17da608008fcca4a3828750c6bf3ff8cdc9440a373e9c7821e2a95c9"},"bundle_signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"MDNFuLW6Wl18PEKPI9IRktNgIXKln4IvS6jPuaBsNEfVbqwrttl/p8ucRG6n8EuGWkZ3+g3QkqJKkSjjRkXjAQ==","signed_message":"bundle_sha256_bytes","signed_at":"2026-08-18T03:06:00.378478Z","bundle_sha256":"6ef35e6b2a90aec06328921e500420f768af6c26704bb627e5d01f92e090a600"}}