{"record_type":"pith_number_record","schema_url":"https://pith.science/schemas/pith-number/v1.json","pith_number":"pith:2024:UBXHA6HCOLGEQFA7MCTV4M7K42","short_pith_number":"pith:UBXHA6HC","schema_version":"1.0","canonical_sha256":"a06e7078e272cc48141f60a75e33eae69cf4ceb424821aad0991ccbf8fc51291","source":{"kind":"arxiv","id":"2405.16281","version":1},"attestation_state":"computed","paper":{"title":"ConStat: Performance-Based Contamination Detection in Large Language Models","license":"http://creativecommons.org/licenses/by/4.0/","headline":"","cross_cats":[],"primary_cat":"cs.CL","authors_text":"Jasper Dekoninck, Mark Niklas M\\\"uller, Martin Vechev","submitted_at":"2024-05-25T15:36:37Z","abstract_excerpt":"Public benchmarks play an essential role in the evaluation of large language models. However, data contamination can lead to inflated performance, rendering them unreliable for model comparison. It is therefore crucial to detect contamination and estimate its impact on measured performance. Unfortunately, existing detection methods can be easily evaded and fail to quantify contamination. To overcome these limitations, we propose a novel definition of contamination as artificially inflated and non-generalizing benchmark performance instead of the inclusion of benchmark samples in the training d"},"verification_status":{"content_addressed":true,"pith_receipt":true,"author_attested":false,"weak_author_claims":0,"strong_author_claims":0,"externally_anchored":false,"storage_verified":false,"citation_signatures":0,"replication_records":0,"graph_snapshot":true,"references_resolved":false,"formal_links_present":false},"canonical_record":{"source":{"id":"2405.16281","kind":"arxiv","version":1},"metadata":{"license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.CL","submitted_at":"2024-05-25T15:36:37Z","cross_cats_sorted":[],"title_canon_sha256":"804025135ca614cce748b048e87a864ce27eb541485b553cd192675e6e8e09bd","abstract_canon_sha256":"6b687ff630ebc8c977e8b877b7c05831475655bb67477e9abf270d757b4e15e3"},"schema_version":"1.0"},"receipt":{"kind":"pith_receipt","key_id":"pith-v1-2026-05","algorithm":"ed25519","signed_at":"2026-07-05T08:23:28.118898Z","signature_b64":"c8FXm1eEEkmhYh0U1jt1tZhtKT1DiChlajkthtv+ZiCk9aYHRONxKqOUOkxke1L57hj7kZoXX7xQr1T8bvx/Cw==","signed_message":"canonical_sha256_bytes","builder_version":"pith-number-builder-2026-05-17-v1","receipt_version":"0.3","canonical_sha256":"a06e7078e272cc48141f60a75e33eae69cf4ceb424821aad0991ccbf8fc51291","last_reissued_at":"2026-07-05T08:23:28.118386Z","signature_status":"signed_v1","first_computed_at":"2026-07-05T08:23:28.118386Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"graph_snapshot":{"paper":{"title":"ConStat: Performance-Based Contamination Detection in Large Language Models","license":"http://creativecommons.org/licenses/by/4.0/","headline":"","cross_cats":[],"primary_cat":"cs.CL","authors_text":"Jasper Dekoninck, Mark Niklas M\\\"uller, Martin Vechev","submitted_at":"2024-05-25T15:36:37Z","abstract_excerpt":"Public benchmarks play an essential role in the evaluation of large language models. However, data contamination can lead to inflated performance, rendering them unreliable for model comparison. It is therefore crucial to detect contamination and estimate its impact on measured performance. Unfortunately, existing detection methods can be easily evaded and fail to quantify contamination. To overcome these limitations, we propose a novel definition of contamination as artificially inflated and non-generalizing benchmark performance instead of the inclusion of benchmark samples in the training d"},"claims":{"count":0,"items":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"2405.16281","kind":"arxiv","version":1},"verdict":{"id":null,"model_set":{},"created_at":null,"strongest_claim":"","one_line_summary":"","pipeline_version":null,"weakest_assumption":"","pith_extraction_headline":""},"integrity":{"clean":true,"summary":{"advisory":0,"critical":0,"by_detector":{},"informational":0},"endpoint":"/pith/2405.16281/integrity.json","findings":[],"available":true,"detectors_run":[],"snapshot_sha256":"c28c3603d3b5d939e8dc4c7e95fa8dfce3d595e45f758748cecf8e644a296938"},"references":{"count":0,"sample":[],"resolved_work":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","internal_anchors":0},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"author_claims":{"count":0,"strong_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"builder_version":"pith-number-builder-2026-05-17-v1"},"aliases":[{"alias_kind":"arxiv","alias_value":"2405.16281","created_at":"2026-07-05T08:23:28.118457+00:00"},{"alias_kind":"arxiv_version","alias_value":"2405.16281v1","created_at":"2026-07-05T08:23:28.118457+00:00"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.2405.16281","created_at":"2026-07-05T08:23:28.118457+00:00"},{"alias_kind":"pith_short_12","alias_value":"UBXHA6HCOLGE","created_at":"2026-07-05T08:23:28.118457+00:00"},{"alias_kind":"pith_short_16","alias_value":"UBXHA6HCOLGEQFA7","created_at":"2026-07-05T08:23:28.118457+00:00"},{"alias_kind":"pith_short_8","alias_value":"UBXHA6HC","created_at":"2026-07-05T08:23:28.118457+00:00"}],"events":[],"event_summary":{},"paper_claims":[],"inbound_citations":{"count":5,"internal_anchor_count":0,"sample":[{"citing_arxiv_id":"2606.03305","citing_title":"The Reliability Gap in Benchmark Auditing: Distribution Shift and Scale as Failure Modes of Contamination Detection","ref_index":7,"is_internal_anchor":false},{"citing_arxiv_id":"2606.03305","citing_title":"The Reliability Gap in Benchmark Auditing: Distribution Shift and Scale as Failure Modes of Contamination Detection","ref_index":7,"is_internal_anchor":false},{"citing_arxiv_id":"2410.07095","citing_title":"MLE-bench: Evaluating Machine Learning Agents on Machine Learning Engineering","ref_index":7,"is_internal_anchor":false},{"citing_arxiv_id":"2605.07453","citing_title":"Data Contamination in Neural Hieroglyphic Translation: A Reproducibility Study","ref_index":23,"is_internal_anchor":false},{"citing_arxiv_id":"2605.06327","citing_title":"Measuring Evaluation-Context Divergence in Open-Weight LLMs: A Paired-Prompt Protocol with Pilot Evidence of Alignment-Pipeline-Specific Heterogeneity","ref_index":27,"is_internal_anchor":false}]},"formal_canon":{"evidence_count":0,"sample":[],"anchors":[]},"links":{"html":"https://pith.science/pith/UBXHA6HCOLGEQFA7MCTV4M7K42","json":"https://pith.science/pith/UBXHA6HCOLGEQFA7MCTV4M7K42.json","graph_json":"https://pith.science/api/pith-number/UBXHA6HCOLGEQFA7MCTV4M7K42/graph.json","events_json":"https://pith.science/api/pith-number/UBXHA6HCOLGEQFA7MCTV4M7K42/events.json","paper":"https://pith.science/paper/UBXHA6HC"},"agent_actions":{"view_html":"https://pith.science/pith/UBXHA6HCOLGEQFA7MCTV4M7K42","download_json":"https://pith.science/pith/UBXHA6HCOLGEQFA7MCTV4M7K42.json","view_paper":"https://pith.science/paper/UBXHA6HC","resolve_alias":"https://pith.science/api/pith-number/resolve?arxiv=2405.16281&json=true","fetch_graph":"https://pith.science/api/pith-number/UBXHA6HCOLGEQFA7MCTV4M7K42/graph.json","fetch_events":"https://pith.science/api/pith-number/UBXHA6HCOLGEQFA7MCTV4M7K42/events.json","actions":{"anchor_timestamp":"https://pith.science/pith/UBXHA6HCOLGEQFA7MCTV4M7K42/action/timestamp_anchor","attest_storage":"https://pith.science/pith/UBXHA6HCOLGEQFA7MCTV4M7K42/action/storage_attestation","attest_author":"https://pith.science/pith/UBXHA6HCOLGEQFA7MCTV4M7K42/action/author_attestation","sign_citation":"https://pith.science/pith/UBXHA6HCOLGEQFA7MCTV4M7K42/action/citation_signature","submit_replication":"https://pith.science/pith/UBXHA6HCOLGEQFA7MCTV4M7K42/action/replication_record"}},"created_at":"2026-07-05T08:23:28.118457+00:00","updated_at":"2026-07-05T08:23:28.118457+00:00"}