{"state_type":"pith_open_graph_state","state_version":"1.0","pith_number":"pith:2026:X6JDETBD4HGH7BX7UJTOKPLJKZ","merge_version":"pith-open-graph-merge-v1","event_count":2,"valid_event_count":2,"invalid_event_count":0,"equivocation_count":0,"current":{"canonical_record":{"metadata":{"abstract_canon_sha256":"76a5c6b177d8821b79c4c5d9e4494d086e5460bd95646760f7c125fa70532cca","cross_cats_sorted":["cs.CR"],"license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","primary_cat":"cs.LG","submitted_at":"2026-05-12T04:23:42Z","title_canon_sha256":"bd882ff0c994f80eecba4c978baed905e74c14858a0a28144e6e71af767b86aa"},"schema_version":"1.0","source":{"id":"2605.11504","kind":"arxiv","version":2}},"source_aliases":[{"alias_kind":"arxiv","alias_value":"2605.11504","created_at":"2026-07-14T01:20:03Z"},{"alias_kind":"arxiv_version","alias_value":"2605.11504v2","created_at":"2026-07-14T01:20:03Z"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.2605.11504","created_at":"2026-07-14T01:20:03Z"},{"alias_kind":"pith_short_12","alias_value":"X6JDETBD4HGH","created_at":"2026-07-14T01:20:03Z"},{"alias_kind":"pith_short_16","alias_value":"X6JDETBD4HGH7BX7","created_at":"2026-07-14T01:20:03Z"},{"alias_kind":"pith_short_8","alias_value":"X6JDETBD","created_at":"2026-07-14T01:20:03Z"}],"graph_snapshots":[{"event_id":"sha256:d428dff14ac64a0a2135b18b83d7cf6abaca7ae9536b2db8152ace04d6b2cf70","target":"graph","created_at":"2026-07-14T01:20:03Z","signer":{"key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signer_id":"pith.science","signer_type":"pith_registry"},"payload":{"graph_snapshot":{"author_claims":{"count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","strong_count":0},"builder_version":"pith-number-builder-2026-05-17-v1","claims":{"count":4,"items":[{"attestation":"unclaimed","claim_id":"C1","kind":"strongest_claim","source":"verdict.strongest_claim","status":"machine_extracted","text":"existing CTF benchmarks can be unreliable in assessing LLM-based agents, while CTFusion can serve as a robust solution for evaluating cybersecurity agents."},{"attestation":"unclaimed","claim_id":"C2","kind":"weakest_assumption","source":"verdict.weakest_assumption","status":"machine_extracted","text":"That live CTF events remain uncontaminated and that the per-agent independence plus first-flag forwarding fully eliminates the contamination and competition-impact problems demonstrated with web-search tools."},{"attestation":"unclaimed","claim_id":"C3","kind":"one_line_summary","source":"verdict.one_line_summary","status":"machine_extracted","text":"CTFusion is a live-CTF streaming benchmark that prevents data contamination by forwarding only the first correct flag per challenge under a shared team account."},{"attestation":"unclaimed","claim_id":"C4","kind":"headline","source":"verdict.pith_extraction.headline","status":"machine_extracted","text":"Reused CTF challenges allow data contamination that inflates LLM agent scores, which CTFusion fixes by streaming evaluations from live events."}],"snapshot_sha256":"1fe892b9d84b8bc28fe98a66b577c919ddd98e59ac1c6e813a5424e945aac632"},"formal_canon":{"evidence_count":1,"snapshot_sha256":"3a24f1dee57e1f71f53a9097ee69bb3595ab7487d4fa48626f552c2966fd40fb"},"integrity":{"available":true,"clean":true,"detectors_run":[{"findings_count":0,"name":"claim_evidence","ran_at":"2026-05-20T04:02:00.704837Z","status":"completed","version":"1.0.0"},{"findings_count":0,"name":"ai_meta_artifact","ran_at":"2026-05-19T12:34:54.349487Z","status":"completed","version":"1.0.0"},{"findings_count":0,"name":"doi_title_agreement","ran_at":"2026-05-19T09:31:19.106952Z","status":"completed","version":"1.0.0"},{"findings_count":0,"name":"doi_compliance","ran_at":"2026-05-19T08:20:57.289989Z","status":"completed","version":"1.0.0"}],"endpoint":"/pith/2605.11504/integrity.json","findings":[],"snapshot_sha256":"c6e3a7bf7fbf0d9f0f065f377e15c141ab1d90951248af6412c92059876cb89b","summary":{"advisory":0,"by_detector":{},"critical":0,"informational":0}},"paper":{"abstract_excerpt":"Recent advances in Large Language Models (LLMs) have enabled agentic systems for complex, multi-step tasks; cybersecurity is emerging as a prominent application. To evaluate such agents, researchers widely adopt Capture The Flag (CTF) benchmarks. However, current CTF benchmarks reuse existing challenges, which exposes them to data contamination and potential cheating. Notably, we confirmed these issues in practice by integrating web search tools into an existing agent. To address these limitations, we present CTFusion, a streaming evaluation framework built on Live CTFs. To achieve this, CTFus","authors_text":"Dongjun Lee, Ga-eun Bae, Insu Yun","cross_cats":["cs.CR"],"headline":"Reused CTF challenges allow data contamination that inflates LLM agent scores, which CTFusion fixes by streaming evaluations from live events.","license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","primary_cat":"cs.LG","submitted_at":"2026-05-12T04:23:42Z","title":"CTFusion: A CTF-based Benchmark for LLM Agent Evaluation"},"references":{"count":0,"internal_anchors":0,"resolved_work":0,"sample":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"2605.11504","kind":"arxiv","version":2},"verdict":{"created_at":"2026-05-13T01:33:21.599669Z","id":"3dc7618e-7e16-4d92-9d47-1a96c996e63c","model_set":{"reader":"grok-4.3"},"one_line_summary":"CTFusion is a live-CTF streaming benchmark that prevents data contamination by forwarding only the first correct flag per challenge under a shared team account.","pipeline_version":"pith-pipeline@v0.9.0","pith_extraction_headline":"Reused CTF challenges allow data contamination that inflates LLM agent scores, which CTFusion fixes by streaming evaluations from live events.","strongest_claim":"existing CTF benchmarks can be unreliable in assessing LLM-based agents, while CTFusion can serve as a robust solution for evaluating cybersecurity agents.","weakest_assumption":"That live CTF events remain uncontaminated and that the per-agent independence plus first-flag forwarding fully eliminates the contamination and competition-impact problems demonstrated with web-search tools."}},"verdict_id":"3dc7618e-7e16-4d92-9d47-1a96c996e63c"}}],"author_attestations":[],"timestamp_anchors":[],"storage_attestations":[],"citation_signatures":[],"replication_records":[],"corrections":[],"mirror_hints":[],"record_created":{"event_id":"sha256:4eda39a32551262ac6074ee2e94f1d35381d88bd7a8dc6da241aa381556e1ae3","target":"record","created_at":"2026-07-14T01:20:03Z","signer":{"key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signer_id":"pith.science","signer_type":"pith_registry"},"payload":{"attestation_state":"computed","canonical_record":{"metadata":{"abstract_canon_sha256":"76a5c6b177d8821b79c4c5d9e4494d086e5460bd95646760f7c125fa70532cca","cross_cats_sorted":["cs.CR"],"license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","primary_cat":"cs.LG","submitted_at":"2026-05-12T04:23:42Z","title_canon_sha256":"bd882ff0c994f80eecba4c978baed905e74c14858a0a28144e6e71af767b86aa"},"schema_version":"1.0","source":{"id":"2605.11504","kind":"arxiv","version":2}},"canonical_sha256":"bf92324c23e1cc7f86ffa266e53d695660b3e72b5c4107beeeaea3765aa0a583","receipt":{"algorithm":"ed25519","builder_version":"pith-number-builder-2026-05-17-v1","canonical_sha256":"bf92324c23e1cc7f86ffa266e53d695660b3e72b5c4107beeeaea3765aa0a583","first_computed_at":"2026-07-14T01:20:03.316488Z","key_id":"pith-v1-2026-05","kind":"pith_receipt","last_reissued_at":"2026-07-14T01:20:03.316488Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","receipt_version":"0.3","signature_b64":"ZFKuDyudiGjVk5Eimox1kWM4oflV31q2HKNZYSSFOQKGbqsEjXZ1ZkcZBN5FzSjidPEy758qw/nAq1n6bpVIDw==","signature_status":"signed_v1","signed_at":"2026-07-14T01:20:03.317609Z","signed_message":"canonical_sha256_bytes"},"source_id":"2605.11504","source_kind":"arxiv","source_version":2}}},"equivocations":[],"invalid_events":[],"applied_event_ids":["sha256:4eda39a32551262ac6074ee2e94f1d35381d88bd7a8dc6da241aa381556e1ae3","sha256:d428dff14ac64a0a2135b18b83d7cf6abaca7ae9536b2db8152ace04d6b2cf70"],"state_sha256":"2efd456f40f4fa7035cb78bcd117094bc742f3f8b4b259a54fa9f3c98c5e18eb"}