{"work":{"id":"0a06199e-9108-413c-9992-d7c4809340ea","openalex_id":"https://openalex.org/W4403585381","doi":"10.48550/arxiv.2408.01605","arxiv_id":"2408.01605","raw_key":null,"title":"CYBERSECEVAL 3: Advancing the Evaluation of Cybersecurity Risks and Capabilities in Large Language Models","authors":null,"authors_text":"S","year":2024,"venue":"cs.CR","abstract":"We are releasing a new suite of security benchmarks for LLMs, CYBERSECEVAL 3, to continue the conversation on empirically measuring LLM cybersecurity risks and capabilities. CYBERSECEVAL 3 assesses 8 different risks across two broad categories: risk to third parties, and risk to application developers and end users. Compared to previous work, we add new areas focused on offensive security capabilities: automated social engineering, scaling manual offensive cyber operations, and autonomous offensive cyber operations. In this paper we discuss applying these benchmarks to the Llama 3 models and a suite of contemporaneous state-of-the-art LLMs, enabling us to contextualize risks both with and without mitigations in place.","external_url":"https://arxiv.org/abs/2408.01605","cited_by_count":8,"metadata_source":"pith","metadata_fetched_at":"2026-08-05T02:28:24.338817+00:00","pith_arxiv_id":"2408.01605","created_at":"2026-05-09T06:45:39.713681+00:00","updated_at":"2026-08-05T02:28:24.338817+00:00","title_quality_ok":true,"display_title":"CyberSecEval 3: Advancing the evaluation of cybersecurity risks and capabilities in large language models","render_title":"CyberSecEval 3: Advancing the evaluation of cybersecurity risks and capabilities in large language models"},"hub":{"state":{"work_id":"0a06199e-9108-413c-9992-d7c4809340ea","tier":"hub","tier_reason":"10+ Pith inbound or 1,000+ external citations","pith_inbound_count":15,"external_cited_by_count":8,"distinct_field_count":3,"first_pith_cited_at":"2026-02-05T10:30:57+00:00","last_pith_cited_at":"2026-07-08T07:50:49+00:00","author_build_status":"not_needed","summary_status":"needed","contexts_status":"needed","graph_status":"needed","ask_index_status":"not_needed","reader_status":"not_needed","recognition_status":"not_needed","updated_at":"2026-08-23T09:10:01.707434+00:00","tier_text":"hub"},"tier":"hub","role_counts":[{"context_role":"background","n":4}],"polarity_counts":[{"context_polarity":"background","n":4}],"runs":{},"summary":{},"graph":{},"authors":[]}}