{"record_type":"pith_number_record","schema_url":"https://pith.science/schemas/pith-number/v1.json","pith_number":"pith:2024:7GGXCLG656PNK2ZCQQV5MLHKBS","short_pith_number":"pith:7GGXCLG6","schema_version":"1.0","canonical_sha256":"f98d712cdeef9ed56b22842bd62cea0cbd75ac82172fca2b7e08d3750db1febb","source":{"kind":"arxiv","id":"2408.16498","version":2},"attestation_state":"computed","paper":{"title":"A Survey on Evaluating Large Language Models in Code Generation Tasks","license":"http://creativecommons.org/licenses/by/4.0/","headline":"","cross_cats":[],"primary_cat":"cs.SE","authors_text":"Hongrui Jia, Jian Wu, Jindong Wang, Liguo Chen, Qi Guo, Qing Gao, Shikun Zhang, Wei Ye, Xin Wang, Yidong Wang, Yijiang Xu, Zhengran Zeng","submitted_at":"2024-08-29T12:56:06Z","abstract_excerpt":"This paper provides a comprehensive review of the current methods and metrics used to evaluate the performance of Large Language Models (LLMs) in code generation tasks. With the rapid growth in demand for automated software development, LLMs have demonstrated significant potential in the field of code generation. The paper begins by reviewing the historical development of LLMs and their applications in code generation. Next, it details various methods and metrics for assessing the code generation capabilities of LLMs, including code correctness, efficiency, readability, and evaluation methods "},"verification_status":{"content_addressed":true,"pith_receipt":true,"author_attested":false,"weak_author_claims":0,"strong_author_claims":0,"externally_anchored":false,"storage_verified":false,"citation_signatures":0,"replication_records":0,"graph_snapshot":true,"references_resolved":false,"formal_links_present":false},"canonical_record":{"source":{"id":"2408.16498","kind":"arxiv","version":2},"metadata":{"license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.SE","submitted_at":"2024-08-29T12:56:06Z","cross_cats_sorted":[],"title_canon_sha256":"8880bd11746b90e31b024875bd0c182dbd3fbce27a8ede35affcdfbaa85e31c4","abstract_canon_sha256":"a44c2a5379137a91bf97fe608fe981760a2b8586f1ffa5813a3b55ef4d40b657"},"schema_version":"1.0"},"receipt":{"kind":"pith_receipt","key_id":"pith-v1-2026-05","algorithm":"ed25519","signed_at":"2026-07-05T10:23:55.714537Z","signature_b64":"P5/h6+tbyQSE5fHcNIfRt52aOxfsJuN9eyImDZNa0+QvkmQt6xMn52ui0XjSI9VMtFFCikIwFW4p2vIE8taGAg==","signed_message":"canonical_sha256_bytes","builder_version":"pith-number-builder-2026-05-17-v1","receipt_version":"0.3","canonical_sha256":"f98d712cdeef9ed56b22842bd62cea0cbd75ac82172fca2b7e08d3750db1febb","last_reissued_at":"2026-07-05T10:23:55.713345Z","signature_status":"signed_v1","first_computed_at":"2026-07-05T10:23:55.713345Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"graph_snapshot":{"paper":{"title":"A Survey on Evaluating Large Language Models in Code Generation Tasks","license":"http://creativecommons.org/licenses/by/4.0/","headline":"","cross_cats":[],"primary_cat":"cs.SE","authors_text":"Hongrui Jia, Jian Wu, Jindong Wang, Liguo Chen, Qi Guo, Qing Gao, Shikun Zhang, Wei Ye, Xin Wang, Yidong Wang, Yijiang Xu, Zhengran Zeng","submitted_at":"2024-08-29T12:56:06Z","abstract_excerpt":"This paper provides a comprehensive review of the current methods and metrics used to evaluate the performance of Large Language Models (LLMs) in code generation tasks. With the rapid growth in demand for automated software development, LLMs have demonstrated significant potential in the field of code generation. The paper begins by reviewing the historical development of LLMs and their applications in code generation. Next, it details various methods and metrics for assessing the code generation capabilities of LLMs, including code correctness, efficiency, readability, and evaluation methods "},"claims":{"count":0,"items":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"2408.16498","kind":"arxiv","version":2},"verdict":{"id":null,"model_set":{},"created_at":null,"strongest_claim":"","one_line_summary":"","pipeline_version":null,"weakest_assumption":"","pith_extraction_headline":""},"integrity":{"clean":true,"summary":{"advisory":0,"critical":0,"by_detector":{},"informational":0},"endpoint":"/pith/2408.16498/integrity.json","findings":[],"available":true,"detectors_run":[],"snapshot_sha256":"c28c3603d3b5d939e8dc4c7e95fa8dfce3d595e45f758748cecf8e644a296938"},"references":{"count":0,"sample":[],"resolved_work":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","internal_anchors":0},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"author_claims":{"count":0,"strong_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"builder_version":"pith-number-builder-2026-05-17-v1"},"aliases":[{"alias_kind":"arxiv","alias_value":"2408.16498","created_at":"2026-07-05T10:23:55.713499+00:00"},{"alias_kind":"arxiv_version","alias_value":"2408.16498v2","created_at":"2026-07-05T10:23:55.713499+00:00"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.2408.16498","created_at":"2026-07-05T10:23:55.713499+00:00"},{"alias_kind":"pith_short_12","alias_value":"7GGXCLG656PN","created_at":"2026-07-05T10:23:55.713499+00:00"},{"alias_kind":"pith_short_16","alias_value":"7GGXCLG656PNK2ZC","created_at":"2026-07-05T10:23:55.713499+00:00"},{"alias_kind":"pith_short_8","alias_value":"7GGXCLG6","created_at":"2026-07-05T10:23:55.713499+00:00"}],"events":[],"event_summary":{},"paper_claims":[],"inbound_citations":{"count":14,"internal_anchor_count":1,"sample":[{"citing_arxiv_id":"2607.08009","citing_title":"From Execution to Education: A Bloom-Aligned Framework for Measuring Educational Control in LLMs","ref_index":32,"is_internal_anchor":true},{"citing_arxiv_id":"2606.20146","citing_title":"BIM-Edit: Benchmarking Large Language Models for IFC-Based Building Information Modeling","ref_index":6,"is_internal_anchor":false},{"citing_arxiv_id":"2606.09852","citing_title":"LLM-Based Code Documentation Generation and Multi-Judge Evaluation","ref_index":5,"is_internal_anchor":false},{"citing_arxiv_id":"2605.29822","citing_title":"Inferring Code Correctness from Specification","ref_index":7,"is_internal_anchor":false},{"citing_arxiv_id":"2503.17181","citing_title":"A Study of LLMs' Preferences for Libraries and Programming Languages","ref_index":8,"is_internal_anchor":false},{"citing_arxiv_id":"2509.22202","citing_title":"Library Hallucinations in LLM-Generated Code: A Risk Analysis Grounded in Developer Queries","ref_index":4,"is_internal_anchor":false},{"citing_arxiv_id":"2605.17957","citing_title":"Contextualized Code Pretraining for Code Generation","ref_index":8,"is_internal_anchor":false},{"citing_arxiv_id":"2603.00989","citing_title":"Sustainable Code Generation Using Large Language Models: A Systematic Literature Review","ref_index":59,"is_internal_anchor":false},{"citing_arxiv_id":"2604.27001","citing_title":"An Empirical Security Evaluation of LLM-Generated Cryptographic Rust Code","ref_index":4,"is_internal_anchor":false},{"citing_arxiv_id":"2604.24678","citing_title":"Leveraging LLMs for Multi-File DSL Code Generation: An Industrial Case Study","ref_index":4,"is_internal_anchor":false},{"citing_arxiv_id":"2604.24621","citing_title":"Evaluation of LLM-Based Software Engineering Tools: Practices, Challenges, and Future Directions","ref_index":9,"is_internal_anchor":false},{"citing_arxiv_id":"2605.05267","citing_title":"Bridging Generation and Training: A Systematic Review of Quality Issues in LLMs for Code","ref_index":13,"is_internal_anchor":false},{"citing_arxiv_id":"2604.09515","citing_title":"When LLMs Lag Behind: Knowledge Conflicts from Evolving APIs in Code Generation","ref_index":4,"is_internal_anchor":false},{"citing_arxiv_id":"2604.06742","citing_title":"Evaluating LLM-Based 0-to-1 Software Generation in End-to-End CLI Tool Scenarios","ref_index":5,"is_internal_anchor":false}]},"formal_canon":{"evidence_count":0,"sample":[],"anchors":[]},"links":{"html":"https://pith.science/pith/7GGXCLG656PNK2ZCQQV5MLHKBS","json":"https://pith.science/pith/7GGXCLG656PNK2ZCQQV5MLHKBS.json","graph_json":"https://pith.science/api/pith-number/7GGXCLG656PNK2ZCQQV5MLHKBS/graph.json","events_json":"https://pith.science/api/pith-number/7GGXCLG656PNK2ZCQQV5MLHKBS/events.json","paper":"https://pith.science/paper/7GGXCLG6"},"agent_actions":{"view_html":"https://pith.science/pith/7GGXCLG656PNK2ZCQQV5MLHKBS","download_json":"https://pith.science/pith/7GGXCLG656PNK2ZCQQV5MLHKBS.json","view_paper":"https://pith.science/paper/7GGXCLG6","resolve_alias":"https://pith.science/api/pith-number/resolve?arxiv=2408.16498&json=true","fetch_graph":"https://pith.science/api/pith-number/7GGXCLG656PNK2ZCQQV5MLHKBS/graph.json","fetch_events":"https://pith.science/api/pith-number/7GGXCLG656PNK2ZCQQV5MLHKBS/events.json","actions":{"anchor_timestamp":"https://pith.science/pith/7GGXCLG656PNK2ZCQQV5MLHKBS/action/timestamp_anchor","attest_storage":"https://pith.science/pith/7GGXCLG656PNK2ZCQQV5MLHKBS/action/storage_attestation","attest_author":"https://pith.science/pith/7GGXCLG656PNK2ZCQQV5MLHKBS/action/author_attestation","sign_citation":"https://pith.science/pith/7GGXCLG656PNK2ZCQQV5MLHKBS/action/citation_signature","submit_replication":"https://pith.science/pith/7GGXCLG656PNK2ZCQQV5MLHKBS/action/replication_record"}},"created_at":"2026-07-05T10:23:55.713499+00:00","updated_at":"2026-07-05T10:23:55.713499+00:00"}