{"record_type":"pith_number_record","schema_url":"https://pith.science/schemas/pith-number/v1.json","pith_number":"pith:2024:KNP6RNYOKWQD2L3IAGKB4FYLP6","short_pith_number":"pith:KNP6RNYO","schema_version":"1.0","canonical_sha256":"535fe8b70e55a03d2f6801941e170b7fb87f82a1d255d1661b43ce7d023e9c3e","source":{"kind":"arxiv","id":"2410.02184","version":1},"attestation_state":"computed","paper":{"title":"CodeJudge: Evaluating Code Generation with Large Language Models","license":"http://creativecommons.org/licenses/by/4.0/","headline":"","cross_cats":["cs.CL","cs.SE"],"primary_cat":"cs.LG","authors_text":"Tianyi Zhang, Weixi Tong","submitted_at":"2024-10-03T03:58:03Z","abstract_excerpt":"Large Language Models (LLMs) have shown promising performance in code generation. However, how to reliably evaluate code generated by LLMs remains an unresolved problem. This paper presents CodeJudge, a code evaluation framework that leverages LLMs to evaluate the semantic correctness of generated code without the need for test cases. We investigate different ways to guide the LLM in performing \"slow thinking\" to arrive at an in-depth and reliable evaluation. We experimented with four LLMs as evaluators on four code generation datasets and five programming languages. The results show that Code"},"verification_status":{"content_addressed":true,"pith_receipt":true,"author_attested":false,"weak_author_claims":0,"strong_author_claims":0,"externally_anchored":false,"storage_verified":false,"citation_signatures":0,"replication_records":0,"graph_snapshot":true,"references_resolved":false,"formal_links_present":false},"canonical_record":{"source":{"id":"2410.02184","kind":"arxiv","version":1},"metadata":{"license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.LG","submitted_at":"2024-10-03T03:58:03Z","cross_cats_sorted":["cs.CL","cs.SE"],"title_canon_sha256":"85387f02502c8f65c3e6b8c17c4cc54fcb97098fbd8a25dcec2d7e68580e3212","abstract_canon_sha256":"5566ba347bd25207371bb3bd4f1de2355e11a3e7555caec023d048dbcff8da6f"},"schema_version":"1.0"},"receipt":{"kind":"pith_receipt","key_id":"pith-v1-2026-05","algorithm":"ed25519","signed_at":"2026-07-05T09:15:17.393744Z","signature_b64":"eBGKk5VAVozl75MgnaKTL4Sy92u1T2CSqg32g38MHrk64Dt00s8IMUj02uJ7evW3ODHGLHxzXNNVEgaUL1JdAA==","signed_message":"canonical_sha256_bytes","builder_version":"pith-number-builder-2026-05-17-v1","receipt_version":"0.3","canonical_sha256":"535fe8b70e55a03d2f6801941e170b7fb87f82a1d255d1661b43ce7d023e9c3e","last_reissued_at":"2026-07-05T09:15:17.393272Z","signature_status":"signed_v1","first_computed_at":"2026-07-05T09:15:17.393272Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"graph_snapshot":{"paper":{"title":"CodeJudge: Evaluating Code Generation with Large Language Models","license":"http://creativecommons.org/licenses/by/4.0/","headline":"","cross_cats":["cs.CL","cs.SE"],"primary_cat":"cs.LG","authors_text":"Tianyi Zhang, Weixi Tong","submitted_at":"2024-10-03T03:58:03Z","abstract_excerpt":"Large Language Models (LLMs) have shown promising performance in code generation. However, how to reliably evaluate code generated by LLMs remains an unresolved problem. This paper presents CodeJudge, a code evaluation framework that leverages LLMs to evaluate the semantic correctness of generated code without the need for test cases. We investigate different ways to guide the LLM in performing \"slow thinking\" to arrive at an in-depth and reliable evaluation. We experimented with four LLMs as evaluators on four code generation datasets and five programming languages. The results show that Code"},"claims":{"count":0,"items":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"2410.02184","kind":"arxiv","version":1},"verdict":{"id":null,"model_set":{},"created_at":null,"strongest_claim":"","one_line_summary":"","pipeline_version":null,"weakest_assumption":"","pith_extraction_headline":""},"integrity":{"clean":true,"summary":{"advisory":0,"critical":0,"by_detector":{},"informational":0},"endpoint":"/pith/2410.02184/integrity.json","findings":[],"available":true,"detectors_run":[],"snapshot_sha256":"c28c3603d3b5d939e8dc4c7e95fa8dfce3d595e45f758748cecf8e644a296938"},"references":{"count":0,"sample":[],"resolved_work":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","internal_anchors":0},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"author_claims":{"count":0,"strong_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"builder_version":"pith-number-builder-2026-05-17-v1"},"aliases":[{"alias_kind":"arxiv","alias_value":"2410.02184","created_at":"2026-07-05T09:15:17.393332+00:00"},{"alias_kind":"arxiv_version","alias_value":"2410.02184v1","created_at":"2026-07-05T09:15:17.393332+00:00"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.2410.02184","created_at":"2026-07-05T09:15:17.393332+00:00"},{"alias_kind":"pith_short_12","alias_value":"KNP6RNYOKWQD","created_at":"2026-07-05T09:15:17.393332+00:00"},{"alias_kind":"pith_short_16","alias_value":"KNP6RNYOKWQD2L3I","created_at":"2026-07-05T09:15:17.393332+00:00"},{"alias_kind":"pith_short_8","alias_value":"KNP6RNYO","created_at":"2026-07-05T09:15:17.393332+00:00"}],"events":[],"event_summary":{},"paper_claims":[],"inbound_citations":{"count":3,"internal_anchor_count":0,"sample":[{"citing_arxiv_id":"2606.00118","citing_title":"An Empirical Study on Logging Evolution On Stack Overflow: Trends, Topics, and Challenges","ref_index":26,"is_internal_anchor":false},{"citing_arxiv_id":"2605.05267","citing_title":"Bridging Generation and Training: A Systematic Review of Quality Issues in LLMs for Code","ref_index":120,"is_internal_anchor":false},{"citing_arxiv_id":"2604.16790","citing_title":"Bias in the Loop: Auditing LLM-as-a-Judge for Software Engineering","ref_index":29,"is_internal_anchor":false}]},"formal_canon":{"evidence_count":0,"sample":[],"anchors":[]},"links":{"html":"https://pith.science/pith/KNP6RNYOKWQD2L3IAGKB4FYLP6","json":"https://pith.science/pith/KNP6RNYOKWQD2L3IAGKB4FYLP6.json","graph_json":"https://pith.science/api/pith-number/KNP6RNYOKWQD2L3IAGKB4FYLP6/graph.json","events_json":"https://pith.science/api/pith-number/KNP6RNYOKWQD2L3IAGKB4FYLP6/events.json","paper":"https://pith.science/paper/KNP6RNYO"},"agent_actions":{"view_html":"https://pith.science/pith/KNP6RNYOKWQD2L3IAGKB4FYLP6","download_json":"https://pith.science/pith/KNP6RNYOKWQD2L3IAGKB4FYLP6.json","view_paper":"https://pith.science/paper/KNP6RNYO","resolve_alias":"https://pith.science/api/pith-number/resolve?arxiv=2410.02184&json=true","fetch_graph":"https://pith.science/api/pith-number/KNP6RNYOKWQD2L3IAGKB4FYLP6/graph.json","fetch_events":"https://pith.science/api/pith-number/KNP6RNYOKWQD2L3IAGKB4FYLP6/events.json","actions":{"anchor_timestamp":"https://pith.science/pith/KNP6RNYOKWQD2L3IAGKB4FYLP6/action/timestamp_anchor","attest_storage":"https://pith.science/pith/KNP6RNYOKWQD2L3IAGKB4FYLP6/action/storage_attestation","attest_author":"https://pith.science/pith/KNP6RNYOKWQD2L3IAGKB4FYLP6/action/author_attestation","sign_citation":"https://pith.science/pith/KNP6RNYOKWQD2L3IAGKB4FYLP6/action/citation_signature","submit_replication":"https://pith.science/pith/KNP6RNYOKWQD2L3IAGKB4FYLP6/action/replication_record"}},"created_at":"2026-07-05T09:15:17.393332+00:00","updated_at":"2026-07-05T09:15:17.393332+00:00"}