{"state_type":"pith_open_graph_state","state_version":"1.0","pith_number":"pith:2024:WIDIQ5T5MPNKPXTWZYKLQSMSLL","merge_version":"pith-open-graph-merge-v1","event_count":2,"valid_event_count":2,"invalid_event_count":0,"equivocation_count":0,"current":{"canonical_record":{"metadata":{"abstract_canon_sha256":"bffd0af25f0ca46d84bed88893932c9704faf9d1c49e1432fac4db5368f2e354","cross_cats_sorted":[],"license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.CL","submitted_at":"2024-11-06T13:54:08Z","title_canon_sha256":"ddd1a9fbc02ff2f16b988a434a178242b4450ff825a653c19e23ab4b542861bf"},"schema_version":"1.0","source":{"id":"2411.03923","kind":"arxiv","version":1}},"source_aliases":[{"alias_kind":"arxiv","alias_value":"2411.03923","created_at":"2026-07-05T09:31:58Z"},{"alias_kind":"arxiv_version","alias_value":"2411.03923v1","created_at":"2026-07-05T09:31:58Z"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.2411.03923","created_at":"2026-07-05T09:31:58Z"},{"alias_kind":"pith_short_12","alias_value":"WIDIQ5T5MPNK","created_at":"2026-07-05T09:31:58Z"},{"alias_kind":"pith_short_16","alias_value":"WIDIQ5T5MPNKPXTW","created_at":"2026-07-05T09:31:58Z"},{"alias_kind":"pith_short_8","alias_value":"WIDIQ5T5","created_at":"2026-07-05T09:31:58Z"}],"graph_snapshots":[{"event_id":"sha256:b87e40415c7249a80b5b9f7c16220b4b633e51b43fd6b7fb43b852b9a185e94a","target":"graph","created_at":"2026-07-05T09:31:58Z","signer":{"key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signer_id":"pith.science","signer_type":"pith_registry"},"payload":{"graph_snapshot":{"author_claims":{"count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","strong_count":0},"builder_version":"pith-number-builder-2026-05-17-v1","claims":{"count":0,"items":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"integrity":{"available":true,"clean":true,"detectors_run":[],"endpoint":"/pith/2411.03923/integrity.json","findings":[],"snapshot_sha256":"c28c3603d3b5d939e8dc4c7e95fa8dfce3d595e45f758748cecf8e644a296938","summary":{"advisory":0,"by_detector":{},"critical":0,"informational":0}},"paper":{"abstract_excerpt":"Hampering the interpretation of benchmark scores, evaluation data contamination has become a growing concern in the evaluation of LLMs, and an active area of research studies its effects. While evaluation data contamination is easily understood intuitively, it is surprisingly difficult to define precisely which samples should be considered contaminated and, consequently, how it impacts benchmark scores. We propose that these questions should be addressed together and that contamination metrics can be assessed based on whether models benefit from the examples they mark contaminated. We propose ","authors_text":"Aaditya K. Singh, Andrew Poulton, David Esiobu, Dieuwke Hupkes, Gergely Szilvasy, Maria Lomeli, Muhammed Yusuf Kocyigit","cross_cats":[],"headline":"","license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.CL","submitted_at":"2024-11-06T13:54:08Z","title":"Evaluation data contamination in LLMs: how do we measure it and (when) does it matter?"},"references":{"count":0,"internal_anchors":0,"resolved_work":0,"sample":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"2411.03923","kind":"arxiv","version":1},"verdict":{"created_at":null,"id":null,"model_set":{},"one_line_summary":"","pipeline_version":null,"pith_extraction_headline":"","strongest_claim":"","weakest_assumption":""}},"verdict_id":null}}],"author_attestations":[],"timestamp_anchors":[],"storage_attestations":[],"citation_signatures":[],"replication_records":[],"corrections":[],"mirror_hints":[],"record_created":{"event_id":"sha256:a9faf306a8bcb99915c4bbdaf1a6ade4a2d2809770337e14bebbd2e3e8752759","target":"record","created_at":"2026-07-05T09:31:58Z","signer":{"key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signer_id":"pith.science","signer_type":"pith_registry"},"payload":{"attestation_state":"computed","canonical_record":{"metadata":{"abstract_canon_sha256":"bffd0af25f0ca46d84bed88893932c9704faf9d1c49e1432fac4db5368f2e354","cross_cats_sorted":[],"license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.CL","submitted_at":"2024-11-06T13:54:08Z","title_canon_sha256":"ddd1a9fbc02ff2f16b988a434a178242b4450ff825a653c19e23ab4b542861bf"},"schema_version":"1.0","source":{"id":"2411.03923","kind":"arxiv","version":1}},"canonical_sha256":"b20688767d63daa7de76ce14b849925afb726822e8ea2cecc15c37b29a4b22f7","receipt":{"algorithm":"ed25519","builder_version":"pith-number-builder-2026-05-17-v1","canonical_sha256":"b20688767d63daa7de76ce14b849925afb726822e8ea2cecc15c37b29a4b22f7","first_computed_at":"2026-07-05T09:31:58.226521Z","key_id":"pith-v1-2026-05","kind":"pith_receipt","last_reissued_at":"2026-07-05T09:31:58.226521Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","receipt_version":"0.3","signature_b64":"+eG4ME4NKH5PXvZyREShQPcs3EUrdK1a1ieyG+/MwjePHdwonl1tMqKrYUYBjgmAeBTaI75I8ANXs0s4iKl+Bg==","signature_status":"signed_v1","signed_at":"2026-07-05T09:31:58.227009Z","signed_message":"canonical_sha256_bytes"},"source_id":"2411.03923","source_kind":"arxiv","source_version":1}}},"equivocations":[],"invalid_events":[],"applied_event_ids":["sha256:a9faf306a8bcb99915c4bbdaf1a6ade4a2d2809770337e14bebbd2e3e8752759","sha256:b87e40415c7249a80b5b9f7c16220b4b633e51b43fd6b7fb43b852b9a185e94a"],"state_sha256":"aa57f8b1483e8b27a73b8fb8acade36e9d7e197a11e6d6d7b23a215e38648da9"}