{"state_type":"pith_open_graph_state","state_version":"1.0","pith_number":"pith:2024:GT2GC4BNUZ3VJCWLBAMRW7SJS5","merge_version":"pith-open-graph-merge-v1","event_count":2,"valid_event_count":2,"invalid_event_count":0,"equivocation_count":0,"current":{"canonical_record":{"metadata":{"abstract_canon_sha256":"3eb6c838a7f139fa5cac7582ab28b01de23441b2a54de26e16628665437f1242","cross_cats_sorted":["cs.AI","cs.CY","cs.LG"],"license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","primary_cat":"cs.CR","submitted_at":"2024-06-08T22:21:42Z","title_canon_sha256":"3a3fe51f63959ddead9a1e3edf16a9b6ddcf446c88f9ecafabe609c39929c8ee"},"schema_version":"1.0","source":{"id":"2406.05590","kind":"arxiv","version":3}},"source_aliases":[{"alias_kind":"arxiv","alias_value":"2406.05590","created_at":"2026-07-05T10:16:10Z"},{"alias_kind":"arxiv_version","alias_value":"2406.05590v3","created_at":"2026-07-05T10:16:10Z"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.2406.05590","created_at":"2026-07-05T10:16:10Z"},{"alias_kind":"pith_short_12","alias_value":"GT2GC4BNUZ3V","created_at":"2026-07-05T10:16:10Z"},{"alias_kind":"pith_short_16","alias_value":"GT2GC4BNUZ3VJCWL","created_at":"2026-07-05T10:16:10Z"},{"alias_kind":"pith_short_8","alias_value":"GT2GC4BN","created_at":"2026-07-05T10:16:10Z"}],"graph_snapshots":[{"event_id":"sha256:459d9d5c85252ca8dc49e2962a8a7471aca7d774ed5bea4560baabf7e19e92e9","target":"graph","created_at":"2026-07-05T10:16:10Z","signer":{"key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signer_id":"pith.science","signer_type":"pith_registry"},"payload":{"graph_snapshot":{"author_claims":{"count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","strong_count":0},"builder_version":"pith-number-builder-2026-05-17-v1","claims":{"count":0,"items":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"integrity":{"available":true,"clean":true,"detectors_run":[],"endpoint":"/pith/2406.05590/integrity.json","findings":[],"snapshot_sha256":"c28c3603d3b5d939e8dc4c7e95fa8dfce3d595e45f758748cecf8e644a296938","summary":{"advisory":0,"by_detector":{},"critical":0,"informational":0}},"paper":{"abstract_excerpt":"Large Language Models (LLMs) are being deployed across various domains today. However, their capacity to solve Capture the Flag (CTF) challenges in cybersecurity has not been thoroughly evaluated. To address this, we develop a novel method to assess LLMs in solving CTF challenges by creating a scalable, open-source benchmark database specifically designed for these applications. This database includes metadata for LLM testing and adaptive learning, compiling a diverse range of CTF challenges from popular competitions. Utilizing the advanced function calling capabilities of LLMs, we build a ful","authors_text":"Boyuan Chen, Brendan Dolan-Gavitt, Farshad Khorrami, Haoran Xi, Kimberly Milner, Max Yin, Meet Udeshi, Minghao Shao, Muhammad Shafique, Prashanth Krishnamurthy, Ramesh Karri, Siddharth Garg, Sofija Jancheska","cross_cats":["cs.AI","cs.CY","cs.LG"],"headline":"","license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","primary_cat":"cs.CR","submitted_at":"2024-06-08T22:21:42Z","title":"NYU CTF Bench: A Scalable Open-Source Benchmark Dataset for Evaluating LLMs in Offensive Security"},"references":{"count":0,"internal_anchors":0,"resolved_work":0,"sample":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"2406.05590","kind":"arxiv","version":3},"verdict":{"created_at":null,"id":null,"model_set":{},"one_line_summary":"","pipeline_version":null,"pith_extraction_headline":"","strongest_claim":"","weakest_assumption":""}},"verdict_id":null}}],"author_attestations":[],"timestamp_anchors":[],"storage_attestations":[],"citation_signatures":[],"replication_records":[],"corrections":[],"mirror_hints":[],"record_created":{"event_id":"sha256:9996fd3ecf78b12dc18b4a856cd630582e48b4a94fe174d991f797d04c3f8571","target":"record","created_at":"2026-07-05T10:16:10Z","signer":{"key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signer_id":"pith.science","signer_type":"pith_registry"},"payload":{"attestation_state":"computed","canonical_record":{"metadata":{"abstract_canon_sha256":"3eb6c838a7f139fa5cac7582ab28b01de23441b2a54de26e16628665437f1242","cross_cats_sorted":["cs.AI","cs.CY","cs.LG"],"license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","primary_cat":"cs.CR","submitted_at":"2024-06-08T22:21:42Z","title_canon_sha256":"3a3fe51f63959ddead9a1e3edf16a9b6ddcf446c88f9ecafabe609c39929c8ee"},"schema_version":"1.0","source":{"id":"2406.05590","kind":"arxiv","version":3}},"canonical_sha256":"34f461702da677548acb08191b7e499769069ff9884ca8110a7542fce70d09ba","receipt":{"algorithm":"ed25519","builder_version":"pith-number-builder-2026-05-17-v1","canonical_sha256":"34f461702da677548acb08191b7e499769069ff9884ca8110a7542fce70d09ba","first_computed_at":"2026-07-05T10:16:10.411906Z","key_id":"pith-v1-2026-05","kind":"pith_receipt","last_reissued_at":"2026-07-05T10:16:10.411906Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","receipt_version":"0.3","signature_b64":"htFtNCwEBS8PkTaAV4D6+I1W3FDKixdR0a1nKzsF1bpDa0XEsxbVOBgKmoVNLhGHtDzIU/Mq6nAQA0V7bEuvDg==","signature_status":"signed_v1","signed_at":"2026-07-05T10:16:10.412476Z","signed_message":"canonical_sha256_bytes"},"source_id":"2406.05590","source_kind":"arxiv","source_version":3}}},"equivocations":[],"invalid_events":[],"applied_event_ids":["sha256:9996fd3ecf78b12dc18b4a856cd630582e48b4a94fe174d991f797d04c3f8571","sha256:459d9d5c85252ca8dc49e2962a8a7471aca7d774ed5bea4560baabf7e19e92e9"],"state_sha256":"0875d0f009650a7ff6d63ed46d1d1d8be4304f26d4a5d1d7668a540697507806"}