{"state_type":"pith_open_graph_state","state_version":"1.0","pith_number":"pith:2024:UBXHA6HCOLGEQFA7MCTV4M7K42","merge_version":"pith-open-graph-merge-v1","event_count":2,"valid_event_count":2,"invalid_event_count":0,"equivocation_count":0,"current":{"canonical_record":{"metadata":{"abstract_canon_sha256":"6b687ff630ebc8c977e8b877b7c05831475655bb67477e9abf270d757b4e15e3","cross_cats_sorted":[],"license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.CL","submitted_at":"2024-05-25T15:36:37Z","title_canon_sha256":"804025135ca614cce748b048e87a864ce27eb541485b553cd192675e6e8e09bd"},"schema_version":"1.0","source":{"id":"2405.16281","kind":"arxiv","version":1}},"source_aliases":[{"alias_kind":"arxiv","alias_value":"2405.16281","created_at":"2026-07-05T08:23:28Z"},{"alias_kind":"arxiv_version","alias_value":"2405.16281v1","created_at":"2026-07-05T08:23:28Z"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.2405.16281","created_at":"2026-07-05T08:23:28Z"},{"alias_kind":"pith_short_12","alias_value":"UBXHA6HCOLGE","created_at":"2026-07-05T08:23:28Z"},{"alias_kind":"pith_short_16","alias_value":"UBXHA6HCOLGEQFA7","created_at":"2026-07-05T08:23:28Z"},{"alias_kind":"pith_short_8","alias_value":"UBXHA6HC","created_at":"2026-07-05T08:23:28Z"}],"graph_snapshots":[{"event_id":"sha256:469d77fa2f1f4cbfd0071e130bfcecd4dde81f6930a8e86a42111923cc26132b","target":"graph","created_at":"2026-07-05T08:23:28Z","signer":{"key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signer_id":"pith.science","signer_type":"pith_registry"},"payload":{"graph_snapshot":{"author_claims":{"count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","strong_count":0},"builder_version":"pith-number-builder-2026-05-17-v1","claims":{"count":0,"items":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"integrity":{"available":true,"clean":true,"detectors_run":[],"endpoint":"/pith/2405.16281/integrity.json","findings":[],"snapshot_sha256":"c28c3603d3b5d939e8dc4c7e95fa8dfce3d595e45f758748cecf8e644a296938","summary":{"advisory":0,"by_detector":{},"critical":0,"informational":0}},"paper":{"abstract_excerpt":"Public benchmarks play an essential role in the evaluation of large language models. However, data contamination can lead to inflated performance, rendering them unreliable for model comparison. It is therefore crucial to detect contamination and estimate its impact on measured performance. Unfortunately, existing detection methods can be easily evaded and fail to quantify contamination. To overcome these limitations, we propose a novel definition of contamination as artificially inflated and non-generalizing benchmark performance instead of the inclusion of benchmark samples in the training d","authors_text":"Jasper Dekoninck, Mark Niklas M\\\"uller, Martin Vechev","cross_cats":[],"headline":"","license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.CL","submitted_at":"2024-05-25T15:36:37Z","title":"ConStat: Performance-Based Contamination Detection in Large Language Models"},"references":{"count":0,"internal_anchors":0,"resolved_work":0,"sample":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"2405.16281","kind":"arxiv","version":1},"verdict":{"created_at":null,"id":null,"model_set":{},"one_line_summary":"","pipeline_version":null,"pith_extraction_headline":"","strongest_claim":"","weakest_assumption":""}},"verdict_id":null}}],"author_attestations":[],"timestamp_anchors":[],"storage_attestations":[],"citation_signatures":[],"replication_records":[],"corrections":[],"mirror_hints":[],"record_created":{"event_id":"sha256:796e2a6bdf0b7fd6eaf57a3a84c8415fb228cf151eccbcda8e68a9a08f104632","target":"record","created_at":"2026-07-05T08:23:28Z","signer":{"key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signer_id":"pith.science","signer_type":"pith_registry"},"payload":{"attestation_state":"computed","canonical_record":{"metadata":{"abstract_canon_sha256":"6b687ff630ebc8c977e8b877b7c05831475655bb67477e9abf270d757b4e15e3","cross_cats_sorted":[],"license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.CL","submitted_at":"2024-05-25T15:36:37Z","title_canon_sha256":"804025135ca614cce748b048e87a864ce27eb541485b553cd192675e6e8e09bd"},"schema_version":"1.0","source":{"id":"2405.16281","kind":"arxiv","version":1}},"canonical_sha256":"a06e7078e272cc48141f60a75e33eae69cf4ceb424821aad0991ccbf8fc51291","receipt":{"algorithm":"ed25519","builder_version":"pith-number-builder-2026-05-17-v1","canonical_sha256":"a06e7078e272cc48141f60a75e33eae69cf4ceb424821aad0991ccbf8fc51291","first_computed_at":"2026-07-05T08:23:28.118386Z","key_id":"pith-v1-2026-05","kind":"pith_receipt","last_reissued_at":"2026-07-05T08:23:28.118386Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","receipt_version":"0.3","signature_b64":"c8FXm1eEEkmhYh0U1jt1tZhtKT1DiChlajkthtv+ZiCk9aYHRONxKqOUOkxke1L57hj7kZoXX7xQr1T8bvx/Cw==","signature_status":"signed_v1","signed_at":"2026-07-05T08:23:28.118898Z","signed_message":"canonical_sha256_bytes"},"source_id":"2405.16281","source_kind":"arxiv","source_version":1}}},"equivocations":[],"invalid_events":[],"applied_event_ids":["sha256:796e2a6bdf0b7fd6eaf57a3a84c8415fb228cf151eccbcda8e68a9a08f104632","sha256:469d77fa2f1f4cbfd0071e130bfcecd4dde81f6930a8e86a42111923cc26132b"],"state_sha256":"e9358ec4a68c048d67b39b560c27e84f2abd5aa60487e49e47b31ccc690e34e9"}