{"record_type":"pith_number_record","schema_url":"https://pith.science/schemas/pith-number/v1.json","pith_number":"pith:2025:ZZLEI73GAA6GZWNXDEID7MCBTG","short_pith_number":"pith:ZZLEI73G","schema_version":"1.0","canonical_sha256":"ce56447f66003c6cd9b719103fb04199a7a64c8630b2f768b824a7b12ecc9a12","source":{"kind":"arxiv","id":"2506.14074","version":1},"attestation_state":"computed","paper":{"title":"Comprehensive Verilog Design Problems: A Next-Generation Benchmark Dataset for Evaluating Large Language Models and Agents on RTL Design and Verification","license":"http://creativecommons.org/licenses/by/4.0/","headline":"","cross_cats":["cs.AR"],"primary_cat":"cs.LG","authors_text":"Brucek Khailany, Chenhui Deng, Chia-Tung Ho, Haoxing Ren, Mingjie Liu, Nathaniel Pinckney, Wenfei Zhou, Yun-Da Tsai","submitted_at":"2025-06-17T00:11:13Z","abstract_excerpt":"We present the Comprehensive Verilog Design Problems (CVDP) benchmark, a new dataset and infrastructure to advance LLM and agent research in hardware design and verification. CVDP includes 783 problems across 13 task categories, covering RTL generation, verification, debugging, specification alignment, and technical Q&A authored by experienced hardware engineers. Problems are offered in both non-agentic and agentic formats. The benchmark introduces more realistic and challenging contexts than prior work, with state-of-the-art models achieving no more than 34% pass@1 on code generation. Agentic"},"verification_status":{"content_addressed":true,"pith_receipt":true,"author_attested":false,"weak_author_claims":0,"strong_author_claims":0,"externally_anchored":false,"storage_verified":false,"citation_signatures":0,"replication_records":0,"graph_snapshot":true,"references_resolved":false,"formal_links_present":false},"canonical_record":{"source":{"id":"2506.14074","kind":"arxiv","version":1},"metadata":{"license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.LG","submitted_at":"2025-06-17T00:11:13Z","cross_cats_sorted":["cs.AR"],"title_canon_sha256":"0c95abb8df20b0dad05bafd23f5337fbf0aa780423d574a2762ed1186ab6bd0d","abstract_canon_sha256":"48863103aae1e2ccc82acaa70f455b914d1afffa2e26770b7166ca4201961120"},"schema_version":"1.0"},"receipt":{"kind":"pith_receipt","key_id":"pith-v1-2026-05","algorithm":"ed25519","signed_at":"2026-07-05T11:22:48.650114Z","signature_b64":"EIGZ7rfc7gScW+a8AIaRfgnmcsDFmJcJz8+qPneFDWdX93uiH2q0OM+FwOFJjeiGaJpekGj+HUsWdqEC/lGFAA==","signed_message":"canonical_sha256_bytes","builder_version":"pith-number-builder-2026-05-17-v1","receipt_version":"0.3","canonical_sha256":"ce56447f66003c6cd9b719103fb04199a7a64c8630b2f768b824a7b12ecc9a12","last_reissued_at":"2026-07-05T11:22:48.649606Z","signature_status":"signed_v1","first_computed_at":"2026-07-05T11:22:48.649606Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"graph_snapshot":{"paper":{"title":"Comprehensive Verilog Design Problems: A Next-Generation Benchmark Dataset for Evaluating Large Language Models and Agents on RTL Design and Verification","license":"http://creativecommons.org/licenses/by/4.0/","headline":"","cross_cats":["cs.AR"],"primary_cat":"cs.LG","authors_text":"Brucek Khailany, Chenhui Deng, Chia-Tung Ho, Haoxing Ren, Mingjie Liu, Nathaniel Pinckney, Wenfei Zhou, Yun-Da Tsai","submitted_at":"2025-06-17T00:11:13Z","abstract_excerpt":"We present the Comprehensive Verilog Design Problems (CVDP) benchmark, a new dataset and infrastructure to advance LLM and agent research in hardware design and verification. CVDP includes 783 problems across 13 task categories, covering RTL generation, verification, debugging, specification alignment, and technical Q&A authored by experienced hardware engineers. Problems are offered in both non-agentic and agentic formats. The benchmark introduces more realistic and challenging contexts than prior work, with state-of-the-art models achieving no more than 34% pass@1 on code generation. Agentic"},"claims":{"count":0,"items":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"2506.14074","kind":"arxiv","version":1},"verdict":{"id":null,"model_set":{},"created_at":null,"strongest_claim":"","one_line_summary":"","pipeline_version":null,"weakest_assumption":"","pith_extraction_headline":""},"integrity":{"clean":true,"summary":{"advisory":0,"critical":0,"by_detector":{},"informational":0},"endpoint":"/pith/2506.14074/integrity.json","findings":[],"available":true,"detectors_run":[],"snapshot_sha256":"c28c3603d3b5d939e8dc4c7e95fa8dfce3d595e45f758748cecf8e644a296938"},"references":{"count":0,"sample":[],"resolved_work":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","internal_anchors":0},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"author_claims":{"count":0,"strong_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"builder_version":"pith-number-builder-2026-05-17-v1"},"aliases":[{"alias_kind":"arxiv","alias_value":"2506.14074","created_at":"2026-07-05T11:22:48.649665+00:00"},{"alias_kind":"arxiv_version","alias_value":"2506.14074v1","created_at":"2026-07-05T11:22:48.649665+00:00"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.2506.14074","created_at":"2026-07-05T11:22:48.649665+00:00"},{"alias_kind":"pith_short_12","alias_value":"ZZLEI73GAA6G","created_at":"2026-07-05T11:22:48.649665+00:00"},{"alias_kind":"pith_short_16","alias_value":"ZZLEI73GAA6GZWNX","created_at":"2026-07-05T11:22:48.649665+00:00"},{"alias_kind":"pith_short_8","alias_value":"ZZLEI73G","created_at":"2026-07-05T11:22:48.649665+00:00"}],"events":[],"event_summary":{},"paper_claims":[],"inbound_citations":{"count":18,"internal_anchor_count":0,"sample":[{"citing_arxiv_id":"2606.27350","citing_title":"CHIA: An open-source framework for principled, agentic AI-driven hardware/software co-design research","ref_index":83,"is_internal_anchor":false},{"citing_arxiv_id":"2606.19387","citing_title":"Interpretable and Verifiable Hardware Generation with LLM-Driven Stepwise Refinement","ref_index":35,"is_internal_anchor":false},{"citing_arxiv_id":"2606.12983","citing_title":"Structured Testbench Generation for LLM-Driven HDL Design and Verification-Oriented Data Curation","ref_index":15,"is_internal_anchor":false},{"citing_arxiv_id":"2606.08976","citing_title":"RTL-BenchLS: A Large-Scale Benchmark for RTL Reasoning and Generation with Large Language Models","ref_index":29,"is_internal_anchor":false},{"citing_arxiv_id":"2606.05680","citing_title":"CASS-RTL: Correctness-Aware Subspace Steering for RTL Generation with LLMs","ref_index":34,"is_internal_anchor":false},{"citing_arxiv_id":"2606.28279","citing_title":"Agentic Hardware Design as Repository-Level Code Evolution","ref_index":4,"is_internal_anchor":false},{"citing_arxiv_id":"2605.27472","citing_title":"AssertLLM2: A Comprehensive LLM Benchmark for Assertion Generation from Design Specifications","ref_index":17,"is_internal_anchor":false},{"citing_arxiv_id":"2605.12857","citing_title":"ChipMATE: Multi-Agent Training via Reinforcement Learning for Enhanced RTL Generation","ref_index":20,"is_internal_anchor":false},{"citing_arxiv_id":"2606.27350","citing_title":"CHIA: An open-source framework for principled, agentic AI-driven hardware/software co-design research","ref_index":84,"is_internal_anchor":false},{"citing_arxiv_id":"2604.15606","citing_title":"Spec2Cov: An Agentic Framework for Code Coverage Closure of Digital Hardware Designs","ref_index":10,"is_internal_anchor":false},{"citing_arxiv_id":"2605.15537","citing_title":"RTL-BenchMT: Dynamic Maintenance of RTL Generation Benchmark Through Agent-Assisted Analysis and Revision","ref_index":15,"is_internal_anchor":false},{"citing_arxiv_id":"2604.27780","citing_title":"RuC: HDL-Agnostic Rule Completion Benchmark Generation","ref_index":9,"is_internal_anchor":false},{"citing_arxiv_id":"2604.27238","citing_title":"SafeTune: Mitigating Data Poisoning in LLM Fine-Tuning for RTL Code Generation","ref_index":16,"is_internal_anchor":false},{"citing_arxiv_id":"2604.19856","citing_title":"ChipCraftBrain: Validation-First RTL Generation via Multi-Agent Orchestration","ref_index":2,"is_internal_anchor":false},{"citing_arxiv_id":"2604.14709","citing_title":"HWE-Bench: Benchmarking LLM Agents on Real-World Hardware Bug Repair Tasks","ref_index":21,"is_internal_anchor":false},{"citing_arxiv_id":"2604.14989","citing_title":"Dr. RTL: Autonomous Agentic RTL Optimization through Tool-Grounded Self-Improvement","ref_index":29,"is_internal_anchor":false},{"citing_arxiv_id":"2604.15606","citing_title":"Spec2Cov: An Agentic Framework for Code Coverage Closure of Digital Hardware Designs","ref_index":10,"is_internal_anchor":false},{"citing_arxiv_id":"2604.15657","citing_title":"Understanding Inference-Time Token Allocation and Coverage Limits in Agentic Hardware Verification","ref_index":27,"is_internal_anchor":false}]},"formal_canon":{"evidence_count":0,"sample":[],"anchors":[]},"links":{"html":"https://pith.science/pith/ZZLEI73GAA6GZWNXDEID7MCBTG","json":"https://pith.science/pith/ZZLEI73GAA6GZWNXDEID7MCBTG.json","graph_json":"https://pith.science/api/pith-number/ZZLEI73GAA6GZWNXDEID7MCBTG/graph.json","events_json":"https://pith.science/api/pith-number/ZZLEI73GAA6GZWNXDEID7MCBTG/events.json","paper":"https://pith.science/paper/ZZLEI73G"},"agent_actions":{"view_html":"https://pith.science/pith/ZZLEI73GAA6GZWNXDEID7MCBTG","download_json":"https://pith.science/pith/ZZLEI73GAA6GZWNXDEID7MCBTG.json","view_paper":"https://pith.science/paper/ZZLEI73G","resolve_alias":"https://pith.science/api/pith-number/resolve?arxiv=2506.14074&json=true","fetch_graph":"https://pith.science/api/pith-number/ZZLEI73GAA6GZWNXDEID7MCBTG/graph.json","fetch_events":"https://pith.science/api/pith-number/ZZLEI73GAA6GZWNXDEID7MCBTG/events.json","actions":{"anchor_timestamp":"https://pith.science/pith/ZZLEI73GAA6GZWNXDEID7MCBTG/action/timestamp_anchor","attest_storage":"https://pith.science/pith/ZZLEI73GAA6GZWNXDEID7MCBTG/action/storage_attestation","attest_author":"https://pith.science/pith/ZZLEI73GAA6GZWNXDEID7MCBTG/action/author_attestation","sign_citation":"https://pith.science/pith/ZZLEI73GAA6GZWNXDEID7MCBTG/action/citation_signature","submit_replication":"https://pith.science/pith/ZZLEI73GAA6GZWNXDEID7MCBTG/action/replication_record"}},"created_at":"2026-07-05T11:22:48.649665+00:00","updated_at":"2026-07-05T11:22:48.649665+00:00"}