{"bundle_type":"pith_open_graph_bundle","bundle_version":"1.0","pith_number":"pith:2024:P5UPY46Q3G2XQ3RUWAVEH5XYY7","short_pith_number":"pith:P5UPY46Q","canonical_record":{"source":{"id":"2410.17141","kind":"arxiv","version":4},"metadata":{"license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.CR","submitted_at":"2024-10-22T16:18:41Z","cross_cats_sorted":["cs.AI"],"title_canon_sha256":"ad837f4f1af681b8e1054993be1078d1d73cebfbd4f8c8c5d015c1efe4dedc95","abstract_canon_sha256":"c257f1c4a1283718c1129841048e27fd65898c7119dda962c98ae6c51ce88366"},"schema_version":"1.0"},"canonical_sha256":"7f68fc73d0d9b5786e34b02a43f6f8c7f481b36bdb6cece60e1740d09306f7ca","source":{"kind":"arxiv","id":"2410.17141","version":4},"source_aliases":[{"alias_kind":"arxiv","alias_value":"2410.17141","created_at":"2026-07-05T10:17:50Z"},{"alias_kind":"arxiv_version","alias_value":"2410.17141v4","created_at":"2026-07-05T10:17:50Z"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.2410.17141","created_at":"2026-07-05T10:17:50Z"},{"alias_kind":"pith_short_12","alias_value":"P5UPY46Q3G2X","created_at":"2026-07-05T10:17:50Z"},{"alias_kind":"pith_short_16","alias_value":"P5UPY46Q3G2XQ3RU","created_at":"2026-07-05T10:17:50Z"},{"alias_kind":"pith_short_8","alias_value":"P5UPY46Q","created_at":"2026-07-05T10:17:50Z"}],"events":[{"event_type":"record_created","subject_pith_number":"pith:2024:P5UPY46Q3G2XQ3RUWAVEH5XYY7","target":"record","payload":{"canonical_record":{"source":{"id":"2410.17141","kind":"arxiv","version":4},"metadata":{"license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.CR","submitted_at":"2024-10-22T16:18:41Z","cross_cats_sorted":["cs.AI"],"title_canon_sha256":"ad837f4f1af681b8e1054993be1078d1d73cebfbd4f8c8c5d015c1efe4dedc95","abstract_canon_sha256":"c257f1c4a1283718c1129841048e27fd65898c7119dda962c98ae6c51ce88366"},"schema_version":"1.0"},"canonical_sha256":"7f68fc73d0d9b5786e34b02a43f6f8c7f481b36bdb6cece60e1740d09306f7ca","receipt":{"kind":"pith_receipt","key_id":"pith-v1-2026-05","algorithm":"ed25519","signed_at":"2026-07-05T10:17:50.976351Z","signature_b64":"SlR88TJc4Z8i5TZe1+UTeNatgIUjg4KvsyADcFRIkkWxRKpgc/SFOZdkADgpgWndAKFVP6LQASvrWMj8XcysDA==","signed_message":"canonical_sha256_bytes","builder_version":"pith-number-builder-2026-05-17-v1","receipt_version":"0.3","canonical_sha256":"7f68fc73d0d9b5786e34b02a43f6f8c7f481b36bdb6cece60e1740d09306f7ca","last_reissued_at":"2026-07-05T10:17:50.975871Z","signature_status":"signed_v1","first_computed_at":"2026-07-05T10:17:50.975871Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"source_kind":"arxiv","source_id":"2410.17141","source_version":4,"attestation_state":"computed"},"signer":{"signer_id":"pith.science","signer_type":"pith_registry","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"created_at":"2026-07-05T10:17:50Z","supersedes":[],"prev_event":null,"signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"CtQLi6PCUWb99RACEfndtMq21EH3nd9EcHkdet8NlcaeHlkLk7zmBI8CPYBSPyxYKAGZwoMdHCewlubq+BgKBQ==","signed_message":"open_graph_event_sha256_bytes","signed_at":"2026-08-16T12:33:21.508564Z"},"content_sha256":"3c011a3efdca37f1aab681ce7f81c5d1371534cc5e00008d6b74f79879092bad","schema_version":"1.0","event_id":"sha256:3c011a3efdca37f1aab681ce7f81c5d1371534cc5e00008d6b74f79879092bad"},{"event_type":"graph_snapshot","subject_pith_number":"pith:2024:P5UPY46Q3G2XQ3RUWAVEH5XYY7","target":"graph","payload":{"graph_snapshot":{"paper":{"title":"Towards Automated Penetration Testing: Introducing LLM Benchmark, Analysis, and Improvements","license":"http://creativecommons.org/licenses/by/4.0/","headline":"","cross_cats":["cs.AI"],"primary_cat":"cs.CR","authors_text":"Edward Kim, Isamu Isozaki, Manil Shrestha, Rick Console","submitted_at":"2024-10-22T16:18:41Z","abstract_excerpt":"Hacking poses a significant threat to cybersecurity, inflicting billions of dollars in damages annually. To mitigate these risks, ethical hacking, or penetration testing, is employed to identify vulnerabilities in systems and networks. Recent advancements in large language models (LLMs) have shown potential across various domains, including cybersecurity. However, there is currently no comprehensive, open, automated, end-to-end penetration testing benchmark to drive progress and evaluate the capabilities of these models in security contexts. This paper introduces a novel open benchmark for LLM"},"claims":{"count":0,"items":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"2410.17141","kind":"arxiv","version":4},"verdict":{"id":null,"model_set":{},"created_at":null,"strongest_claim":"","one_line_summary":"","pipeline_version":null,"weakest_assumption":"","pith_extraction_headline":""},"integrity":{"clean":true,"summary":{"advisory":0,"critical":0,"by_detector":{},"informational":0},"endpoint":"/pith/2410.17141/integrity.json","findings":[],"available":true,"detectors_run":[],"snapshot_sha256":"c28c3603d3b5d939e8dc4c7e95fa8dfce3d595e45f758748cecf8e644a296938"},"references":{"count":0,"sample":[],"resolved_work":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","internal_anchors":0},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"author_claims":{"count":0,"strong_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"builder_version":"pith-number-builder-2026-05-17-v1"},"verdict_id":null},"signer":{"signer_id":"pith.science","signer_type":"pith_registry","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"created_at":"2026-07-05T10:17:50Z","supersedes":[],"prev_event":null,"signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"DHfRr9rld1eMg+bFlNQbH4H9SIqfD1uM8TMSbHex2RnK6LWzpMTtr36ie7ItdtRUGapX7j+ecDc7Gc3rpFPKBQ==","signed_message":"open_graph_event_sha256_bytes","signed_at":"2026-08-16T12:33:21.509278Z"},"content_sha256":"e9fa1ba5be2d069aed7e0e43c1521844a33fccdf7b42b8ab299b3bb12650db35","schema_version":"1.0","event_id":"sha256:e9fa1ba5be2d069aed7e0e43c1521844a33fccdf7b42b8ab299b3bb12650db35"}],"timestamp_proofs":[],"mirror_hints":[{"mirror_type":"https","name":"Pith Resolver","base_url":"https://pith.science","bundle_url":"https://pith.science/pith/P5UPY46Q3G2XQ3RUWAVEH5XYY7/bundle.json","state_url":"https://pith.science/pith/P5UPY46Q3G2XQ3RUWAVEH5XYY7/state.json","well_known_bundle_url":"https://pith.science/.well-known/pith/P5UPY46Q3G2XQ3RUWAVEH5XYY7/bundle.json","status":"primary"}],"public_keys":[{"key_id":"pith-v1-2026-05","algorithm":"ed25519","format":"raw","public_key_b64":"stVStoiQhXFxp4s2pdzPNoqVNBMojDU/fJ2db5S3CbM=","public_key_hex":"b2d552b68890857171a78b36a5dccf368a953413288c353f7c9d9d6f94b709b3","fingerprint_sha256_b32_first128bits":"RVFV5Z2OI2J3ZUO7ERDEBCYNKS","fingerprint_sha256_hex":"8d4b5ee74e4693bcd1df2446408b0d54","rotates_at":null,"url":"https://pith.science/pith-signing-key.json","notes":"Pith uses this Ed25519 key to sign canonical record SHA-256 digests. Verify with: ed25519_verify(public_key, message=canonical_sha256_bytes, signature=base64decode(signature_b64))."}],"merge_version":"pith-open-graph-merge-v1","built_at":"2026-08-16T12:33:21Z","links":{"resolver":"https://pith.science/pith/P5UPY46Q3G2XQ3RUWAVEH5XYY7","bundle":"https://pith.science/pith/P5UPY46Q3G2XQ3RUWAVEH5XYY7/bundle.json","state":"https://pith.science/pith/P5UPY46Q3G2XQ3RUWAVEH5XYY7/state.json","well_known_bundle":"https://pith.science/.well-known/pith/P5UPY46Q3G2XQ3RUWAVEH5XYY7/bundle.json"},"state":{"state_type":"pith_open_graph_state","state_version":"1.0","pith_number":"pith:2024:P5UPY46Q3G2XQ3RUWAVEH5XYY7","merge_version":"pith-open-graph-merge-v1","event_count":2,"valid_event_count":2,"invalid_event_count":0,"equivocation_count":0,"current":{"canonical_record":{"metadata":{"abstract_canon_sha256":"c257f1c4a1283718c1129841048e27fd65898c7119dda962c98ae6c51ce88366","cross_cats_sorted":["cs.AI"],"license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.CR","submitted_at":"2024-10-22T16:18:41Z","title_canon_sha256":"ad837f4f1af681b8e1054993be1078d1d73cebfbd4f8c8c5d015c1efe4dedc95"},"schema_version":"1.0","source":{"id":"2410.17141","kind":"arxiv","version":4}},"source_aliases":[{"alias_kind":"arxiv","alias_value":"2410.17141","created_at":"2026-07-05T10:17:50Z"},{"alias_kind":"arxiv_version","alias_value":"2410.17141v4","created_at":"2026-07-05T10:17:50Z"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.2410.17141","created_at":"2026-07-05T10:17:50Z"},{"alias_kind":"pith_short_12","alias_value":"P5UPY46Q3G2X","created_at":"2026-07-05T10:17:50Z"},{"alias_kind":"pith_short_16","alias_value":"P5UPY46Q3G2XQ3RU","created_at":"2026-07-05T10:17:50Z"},{"alias_kind":"pith_short_8","alias_value":"P5UPY46Q","created_at":"2026-07-05T10:17:50Z"}],"graph_snapshots":[{"event_id":"sha256:e9fa1ba5be2d069aed7e0e43c1521844a33fccdf7b42b8ab299b3bb12650db35","target":"graph","created_at":"2026-07-05T10:17:50Z","signer":{"key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signer_id":"pith.science","signer_type":"pith_registry"},"payload":{"graph_snapshot":{"author_claims":{"count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","strong_count":0},"builder_version":"pith-number-builder-2026-05-17-v1","claims":{"count":0,"items":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"integrity":{"available":true,"clean":true,"detectors_run":[],"endpoint":"/pith/2410.17141/integrity.json","findings":[],"snapshot_sha256":"c28c3603d3b5d939e8dc4c7e95fa8dfce3d595e45f758748cecf8e644a296938","summary":{"advisory":0,"by_detector":{},"critical":0,"informational":0}},"paper":{"abstract_excerpt":"Hacking poses a significant threat to cybersecurity, inflicting billions of dollars in damages annually. To mitigate these risks, ethical hacking, or penetration testing, is employed to identify vulnerabilities in systems and networks. Recent advancements in large language models (LLMs) have shown potential across various domains, including cybersecurity. However, there is currently no comprehensive, open, automated, end-to-end penetration testing benchmark to drive progress and evaluate the capabilities of these models in security contexts. This paper introduces a novel open benchmark for LLM","authors_text":"Edward Kim, Isamu Isozaki, Manil Shrestha, Rick Console","cross_cats":["cs.AI"],"headline":"","license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.CR","submitted_at":"2024-10-22T16:18:41Z","title":"Towards Automated Penetration Testing: Introducing LLM Benchmark, Analysis, and Improvements"},"references":{"count":0,"internal_anchors":0,"resolved_work":0,"sample":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"2410.17141","kind":"arxiv","version":4},"verdict":{"created_at":null,"id":null,"model_set":{},"one_line_summary":"","pipeline_version":null,"pith_extraction_headline":"","strongest_claim":"","weakest_assumption":""}},"verdict_id":null}}],"author_attestations":[],"timestamp_anchors":[],"storage_attestations":[],"citation_signatures":[],"replication_records":[],"corrections":[],"mirror_hints":[],"record_created":{"event_id":"sha256:3c011a3efdca37f1aab681ce7f81c5d1371534cc5e00008d6b74f79879092bad","target":"record","created_at":"2026-07-05T10:17:50Z","signer":{"key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signer_id":"pith.science","signer_type":"pith_registry"},"payload":{"attestation_state":"computed","canonical_record":{"metadata":{"abstract_canon_sha256":"c257f1c4a1283718c1129841048e27fd65898c7119dda962c98ae6c51ce88366","cross_cats_sorted":["cs.AI"],"license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.CR","submitted_at":"2024-10-22T16:18:41Z","title_canon_sha256":"ad837f4f1af681b8e1054993be1078d1d73cebfbd4f8c8c5d015c1efe4dedc95"},"schema_version":"1.0","source":{"id":"2410.17141","kind":"arxiv","version":4}},"canonical_sha256":"7f68fc73d0d9b5786e34b02a43f6f8c7f481b36bdb6cece60e1740d09306f7ca","receipt":{"algorithm":"ed25519","builder_version":"pith-number-builder-2026-05-17-v1","canonical_sha256":"7f68fc73d0d9b5786e34b02a43f6f8c7f481b36bdb6cece60e1740d09306f7ca","first_computed_at":"2026-07-05T10:17:50.975871Z","key_id":"pith-v1-2026-05","kind":"pith_receipt","last_reissued_at":"2026-07-05T10:17:50.975871Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","receipt_version":"0.3","signature_b64":"SlR88TJc4Z8i5TZe1+UTeNatgIUjg4KvsyADcFRIkkWxRKpgc/SFOZdkADgpgWndAKFVP6LQASvrWMj8XcysDA==","signature_status":"signed_v1","signed_at":"2026-07-05T10:17:50.976351Z","signed_message":"canonical_sha256_bytes"},"source_id":"2410.17141","source_kind":"arxiv","source_version":4}}},"equivocations":[],"invalid_events":[],"applied_event_ids":["sha256:3c011a3efdca37f1aab681ce7f81c5d1371534cc5e00008d6b74f79879092bad","sha256:e9fa1ba5be2d069aed7e0e43c1521844a33fccdf7b42b8ab299b3bb12650db35"],"state_sha256":"89cb9794a4044debce3a6cc02678514f6c2de92c7c8d599b1c349ae8e1c0b20e"},"bundle_signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"HPqZhrg86HSiwQNg3e/r5X5ICmtmfxlseyvlCCMBIn01zhnvmFL9TcuOqMWCT1smE3mC8wLGoFb8F6sYLd5bAg==","signed_message":"bundle_sha256_bytes","signed_at":"2026-08-16T12:33:21.519954Z","bundle_sha256":"ace012e65aa68bfe6cc9dbab7c233dfefec5b91f788bd11fefa38ab177eeaaf9"}}