{"record_type":"pith_number_record","schema_url":"https://pith.science/schemas/pith-number/v1.json","pith_number":"pith:2024:YE4KOCWECH4NGRZSF2NT6U6WEN","short_pith_number":"pith:YE4KOCWE","schema_version":"1.0","canonical_sha256":"c138a70ac411f8d347322e9b3f53d62360b352dc664cdf225b3ecd174e23ead8","source":{"kind":"arxiv","id":"2402.07688","version":2},"attestation_state":"computed","paper":{"title":"CyberMetric: A Benchmark Dataset based on Retrieval-Augmented Generation for Evaluating LLMs in Cybersecurity Knowledge","license":"http://creativecommons.org/licenses/by/4.0/","headline":"","cross_cats":["cs.CR"],"primary_cat":"cs.AI","authors_text":"Merouane Debbah, Mohamed Amine Ferrag, Norbert Tihanyi, Ridhi Jain, Tamas Bisztray","submitted_at":"2024-02-12T14:53:28Z","abstract_excerpt":"Large Language Models (LLMs) are increasingly used across various domains, from software development to cyber threat intelligence. Understanding all the different fields of cybersecurity, which includes topics such as cryptography, reverse engineering, and risk assessment, poses a challenge even for human experts. To accurately test the general knowledge of LLMs in cybersecurity, the research community needs a diverse, accurate, and up-to-date dataset. To address this gap, we present CyberMetric-80, CyberMetric-500, CyberMetric-2000, and CyberMetric-10000, which are multiple-choice Q&A benchma"},"verification_status":{"content_addressed":true,"pith_receipt":true,"author_attested":false,"weak_author_claims":0,"strong_author_claims":0,"externally_anchored":false,"storage_verified":false,"citation_signatures":0,"replication_records":0,"graph_snapshot":true,"references_resolved":false,"formal_links_present":false},"canonical_record":{"source":{"id":"2402.07688","kind":"arxiv","version":2},"metadata":{"license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.AI","submitted_at":"2024-02-12T14:53:28Z","cross_cats_sorted":["cs.CR"],"title_canon_sha256":"e8cee2427740b907d7227926dadac5c98a47042d8e41afb4a2956dec5e0620c4","abstract_canon_sha256":"f8dde44e72200984c4bf8d03fe718b2b7cd6ed0bdf80038121cc2fc33f52b30f"},"schema_version":"1.0"},"receipt":{"kind":"pith_receipt","key_id":"pith-v1-2026-05","algorithm":"ed25519","signed_at":"2026-07-05T08:26:19.255575Z","signature_b64":"9KqXBElXUnk1aBgoHHcAQhALkio8v56Z38seBFh1oVSbiu7xz9RgE2bJZNzzrTgWHwiRJass8KdGCvMQcwKeAA==","signed_message":"canonical_sha256_bytes","builder_version":"pith-number-builder-2026-05-17-v1","receipt_version":"0.3","canonical_sha256":"c138a70ac411f8d347322e9b3f53d62360b352dc664cdf225b3ecd174e23ead8","last_reissued_at":"2026-07-05T08:26:19.255079Z","signature_status":"signed_v1","first_computed_at":"2026-07-05T08:26:19.255079Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"graph_snapshot":{"paper":{"title":"CyberMetric: A Benchmark Dataset based on Retrieval-Augmented Generation for Evaluating LLMs in Cybersecurity Knowledge","license":"http://creativecommons.org/licenses/by/4.0/","headline":"","cross_cats":["cs.CR"],"primary_cat":"cs.AI","authors_text":"Merouane Debbah, Mohamed Amine Ferrag, Norbert Tihanyi, Ridhi Jain, Tamas Bisztray","submitted_at":"2024-02-12T14:53:28Z","abstract_excerpt":"Large Language Models (LLMs) are increasingly used across various domains, from software development to cyber threat intelligence. Understanding all the different fields of cybersecurity, which includes topics such as cryptography, reverse engineering, and risk assessment, poses a challenge even for human experts. To accurately test the general knowledge of LLMs in cybersecurity, the research community needs a diverse, accurate, and up-to-date dataset. To address this gap, we present CyberMetric-80, CyberMetric-500, CyberMetric-2000, and CyberMetric-10000, which are multiple-choice Q&A benchma"},"claims":{"count":0,"items":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"2402.07688","kind":"arxiv","version":2},"verdict":{"id":null,"model_set":{},"created_at":null,"strongest_claim":"","one_line_summary":"","pipeline_version":null,"weakest_assumption":"","pith_extraction_headline":""},"integrity":{"clean":true,"summary":{"advisory":0,"critical":0,"by_detector":{},"informational":0},"endpoint":"/pith/2402.07688/integrity.json","findings":[],"available":true,"detectors_run":[],"snapshot_sha256":"c28c3603d3b5d939e8dc4c7e95fa8dfce3d595e45f758748cecf8e644a296938"},"references":{"count":0,"sample":[],"resolved_work":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","internal_anchors":0},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"author_claims":{"count":0,"strong_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"builder_version":"pith-number-builder-2026-05-17-v1"},"aliases":[{"alias_kind":"arxiv","alias_value":"2402.07688","created_at":"2026-07-05T08:26:19.255133+00:00"},{"alias_kind":"arxiv_version","alias_value":"2402.07688v2","created_at":"2026-07-05T08:26:19.255133+00:00"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.2402.07688","created_at":"2026-07-05T08:26:19.255133+00:00"},{"alias_kind":"pith_short_12","alias_value":"YE4KOCWECH4N","created_at":"2026-07-05T08:26:19.255133+00:00"},{"alias_kind":"pith_short_16","alias_value":"YE4KOCWECH4NGRZS","created_at":"2026-07-05T08:26:19.255133+00:00"},{"alias_kind":"pith_short_8","alias_value":"YE4KOCWE","created_at":"2026-07-05T08:26:19.255133+00:00"}],"events":[],"event_summary":{},"paper_claims":[],"inbound_citations":{"count":3,"internal_anchor_count":0,"sample":[{"citing_arxiv_id":"2605.28146","citing_title":"Cybersecurity AI (CAI) Dataset","ref_index":13,"is_internal_anchor":false},{"citing_arxiv_id":"2604.19533","citing_title":"Cyber Defense Benchmark: Agentic Threat Hunting Evaluation for LLMs in SecOps","ref_index":3,"is_internal_anchor":false},{"citing_arxiv_id":"2604.05440","citing_title":"LanG -- A Governance-Aware Agentic AI Platform for Unified Security Operations","ref_index":62,"is_internal_anchor":false}]},"formal_canon":{"evidence_count":0,"sample":[],"anchors":[]},"links":{"html":"https://pith.science/pith/YE4KOCWECH4NGRZSF2NT6U6WEN","json":"https://pith.science/pith/YE4KOCWECH4NGRZSF2NT6U6WEN.json","graph_json":"https://pith.science/api/pith-number/YE4KOCWECH4NGRZSF2NT6U6WEN/graph.json","events_json":"https://pith.science/api/pith-number/YE4KOCWECH4NGRZSF2NT6U6WEN/events.json","paper":"https://pith.science/paper/YE4KOCWE"},"agent_actions":{"view_html":"https://pith.science/pith/YE4KOCWECH4NGRZSF2NT6U6WEN","download_json":"https://pith.science/pith/YE4KOCWECH4NGRZSF2NT6U6WEN.json","view_paper":"https://pith.science/paper/YE4KOCWE","resolve_alias":"https://pith.science/api/pith-number/resolve?arxiv=2402.07688&json=true","fetch_graph":"https://pith.science/api/pith-number/YE4KOCWECH4NGRZSF2NT6U6WEN/graph.json","fetch_events":"https://pith.science/api/pith-number/YE4KOCWECH4NGRZSF2NT6U6WEN/events.json","actions":{"anchor_timestamp":"https://pith.science/pith/YE4KOCWECH4NGRZSF2NT6U6WEN/action/timestamp_anchor","attest_storage":"https://pith.science/pith/YE4KOCWECH4NGRZSF2NT6U6WEN/action/storage_attestation","attest_author":"https://pith.science/pith/YE4KOCWECH4NGRZSF2NT6U6WEN/action/author_attestation","sign_citation":"https://pith.science/pith/YE4KOCWECH4NGRZSF2NT6U6WEN/action/citation_signature","submit_replication":"https://pith.science/pith/YE4KOCWECH4NGRZSF2NT6U6WEN/action/replication_record"}},"created_at":"2026-07-05T08:26:19.255133+00:00","updated_at":"2026-07-05T08:26:19.255133+00:00"}