{"record_type":"pith_number_record","schema_url":"https://pith.science/schemas/pith-number/v1.json","pith_number":"pith:2024:FICSFSHMTA3OBM3XTZ6RXWBW3S","short_pith_number":"pith:FICSFSHM","schema_version":"1.0","canonical_sha256":"2a0522c8ec9836e0b3779e7d1bd836dcaca3b0378a356ed3b09b7aa53f499b81","source":{"kind":"arxiv","id":"2403.04811","version":1},"attestation_state":"computed","paper":{"title":"Quantifying Contamination in Evaluating Code Generation Capabilities of Language Models","license":"http://creativecommons.org/licenses/by-sa/4.0/","headline":"","cross_cats":["cs.CL","cs.LG"],"primary_cat":"cs.SE","authors_text":"Ansong Ni, Arman Cohan, Martin Riddell","submitted_at":"2024-03-06T21:45:35Z","abstract_excerpt":"While large language models have achieved remarkable performance on various code generation benchmarks, there have been growing concerns regarding potential contamination of these benchmarks as they may be leaked into pretraining and finetuning data. While recent work has investigated contamination in natural language generation and understanding tasks, there has been less extensive research into how data contamination impacts the evaluation of code generation, which is critical for understanding the robustness and reliability of LLMs in programming contexts. In this work, we perform a compreh"},"verification_status":{"content_addressed":true,"pith_receipt":true,"author_attested":false,"weak_author_claims":0,"strong_author_claims":0,"externally_anchored":false,"storage_verified":false,"citation_signatures":0,"replication_records":0,"graph_snapshot":true,"references_resolved":false,"formal_links_present":false},"canonical_record":{"source":{"id":"2403.04811","kind":"arxiv","version":1},"metadata":{"license":"http://creativecommons.org/licenses/by-sa/4.0/","primary_cat":"cs.SE","submitted_at":"2024-03-06T21:45:35Z","cross_cats_sorted":["cs.CL","cs.LG"],"title_canon_sha256":"956bb06dd0639398e9c4e492862f9a2fdd5468e45955ad77b86739ed04011a26","abstract_canon_sha256":"ab0e42ab350ff22bdd22f914bb129d45b2c49e45731c5ef4212812018396e4bd"},"schema_version":"1.0"},"receipt":{"kind":"pith_receipt","key_id":"pith-v1-2026-05","algorithm":"ed25519","signed_at":"2026-07-05T07:53:34.731805Z","signature_b64":"3oAhyBG+kjPLqZeCVOFhjgTwO38VQAsmSjk/wXq+ZLyhFs+PZw7K65VR9YIdhHCQin/tZ6STFgdUyz6UsP9SDQ==","signed_message":"canonical_sha256_bytes","builder_version":"pith-number-builder-2026-05-17-v1","receipt_version":"0.3","canonical_sha256":"2a0522c8ec9836e0b3779e7d1bd836dcaca3b0378a356ed3b09b7aa53f499b81","last_reissued_at":"2026-07-05T07:53:34.731398Z","signature_status":"signed_v1","first_computed_at":"2026-07-05T07:53:34.731398Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"graph_snapshot":{"paper":{"title":"Quantifying Contamination in Evaluating Code Generation Capabilities of Language Models","license":"http://creativecommons.org/licenses/by-sa/4.0/","headline":"","cross_cats":["cs.CL","cs.LG"],"primary_cat":"cs.SE","authors_text":"Ansong Ni, Arman Cohan, Martin Riddell","submitted_at":"2024-03-06T21:45:35Z","abstract_excerpt":"While large language models have achieved remarkable performance on various code generation benchmarks, there have been growing concerns regarding potential contamination of these benchmarks as they may be leaked into pretraining and finetuning data. While recent work has investigated contamination in natural language generation and understanding tasks, there has been less extensive research into how data contamination impacts the evaluation of code generation, which is critical for understanding the robustness and reliability of LLMs in programming contexts. In this work, we perform a compreh"},"claims":{"count":0,"items":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"2403.04811","kind":"arxiv","version":1},"verdict":{"id":null,"model_set":{},"created_at":null,"strongest_claim":"","one_line_summary":"","pipeline_version":null,"weakest_assumption":"","pith_extraction_headline":""},"integrity":{"clean":true,"summary":{"advisory":0,"critical":0,"by_detector":{},"informational":0},"endpoint":"/pith/2403.04811/integrity.json","findings":[],"available":true,"detectors_run":[],"snapshot_sha256":"c28c3603d3b5d939e8dc4c7e95fa8dfce3d595e45f758748cecf8e644a296938"},"references":{"count":0,"sample":[],"resolved_work":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","internal_anchors":0},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"author_claims":{"count":0,"strong_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"builder_version":"pith-number-builder-2026-05-17-v1"},"aliases":[{"alias_kind":"arxiv","alias_value":"2403.04811","created_at":"2026-07-05T07:53:34.731455+00:00"},{"alias_kind":"arxiv_version","alias_value":"2403.04811v1","created_at":"2026-07-05T07:53:34.731455+00:00"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.2403.04811","created_at":"2026-07-05T07:53:34.731455+00:00"},{"alias_kind":"pith_short_12","alias_value":"FICSFSHMTA3O","created_at":"2026-07-05T07:53:34.731455+00:00"},{"alias_kind":"pith_short_16","alias_value":"FICSFSHMTA3OBM3X","created_at":"2026-07-05T07:53:34.731455+00:00"},{"alias_kind":"pith_short_8","alias_value":"FICSFSHM","created_at":"2026-07-05T07:53:34.731455+00:00"}],"events":[],"event_summary":{},"paper_claims":[],"inbound_citations":{"count":4,"internal_anchor_count":0,"sample":[{"citing_arxiv_id":"2402.01411","citing_title":"CodePori: Large-Scale System for Autonomous Software Development Using Multi-Agent Technology","ref_index":72,"is_internal_anchor":false},{"citing_arxiv_id":"2406.04244","citing_title":"Benchmark Data Contamination of Large Language Models: A Survey","ref_index":124,"is_internal_anchor":false},{"citing_arxiv_id":"2605.21615","citing_title":"ASSEMBLAGE-DEEPHISTORY: A Cross-Build Binary Dataset with Temporal Coverage","ref_index":66,"is_internal_anchor":false},{"citing_arxiv_id":"2403.07974","citing_title":"LiveCodeBench: Holistic and Contamination Free Evaluation of Large Language Models for Code","ref_index":296,"is_internal_anchor":false}]},"formal_canon":{"evidence_count":0,"sample":[],"anchors":[]},"links":{"html":"https://pith.science/pith/FICSFSHMTA3OBM3XTZ6RXWBW3S","json":"https://pith.science/pith/FICSFSHMTA3OBM3XTZ6RXWBW3S.json","graph_json":"https://pith.science/api/pith-number/FICSFSHMTA3OBM3XTZ6RXWBW3S/graph.json","events_json":"https://pith.science/api/pith-number/FICSFSHMTA3OBM3XTZ6RXWBW3S/events.json","paper":"https://pith.science/paper/FICSFSHM"},"agent_actions":{"view_html":"https://pith.science/pith/FICSFSHMTA3OBM3XTZ6RXWBW3S","download_json":"https://pith.science/pith/FICSFSHMTA3OBM3XTZ6RXWBW3S.json","view_paper":"https://pith.science/paper/FICSFSHM","resolve_alias":"https://pith.science/api/pith-number/resolve?arxiv=2403.04811&json=true","fetch_graph":"https://pith.science/api/pith-number/FICSFSHMTA3OBM3XTZ6RXWBW3S/graph.json","fetch_events":"https://pith.science/api/pith-number/FICSFSHMTA3OBM3XTZ6RXWBW3S/events.json","actions":{"anchor_timestamp":"https://pith.science/pith/FICSFSHMTA3OBM3XTZ6RXWBW3S/action/timestamp_anchor","attest_storage":"https://pith.science/pith/FICSFSHMTA3OBM3XTZ6RXWBW3S/action/storage_attestation","attest_author":"https://pith.science/pith/FICSFSHMTA3OBM3XTZ6RXWBW3S/action/author_attestation","sign_citation":"https://pith.science/pith/FICSFSHMTA3OBM3XTZ6RXWBW3S/action/citation_signature","submit_replication":"https://pith.science/pith/FICSFSHMTA3OBM3XTZ6RXWBW3S/action/replication_record"}},"created_at":"2026-07-05T07:53:34.731455+00:00","updated_at":"2026-07-05T07:53:34.731455+00:00"}