{"bundle_type":"pith_open_graph_bundle","bundle_version":"1.0","pith_number":"pith:2025:V6G5IIYUEPBDGE57QCFCEMVV5C","short_pith_number":"pith:V6G5IIYU","canonical_record":{"source":{"id":"2501.18771","kind":"arxiv","version":1},"metadata":{"license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.CL","submitted_at":"2025-01-30T21:51:18Z","cross_cats_sorted":["cs.AI"],"title_canon_sha256":"20f1fa3bdd09de57092c4c57d8e5383ef1d09ab81afaf41689781bd15614ae20","abstract_canon_sha256":"fd0741f88451a1651edf583ab7e4ba84b010f67d28cb8654be7459f165ce81f8"},"schema_version":"1.0"},"canonical_sha256":"af8dd4231423c23313bf808a2232b5e88f1fbfb34b360175c56841255efb448c","source":{"kind":"arxiv","id":"2501.18771","version":1},"source_aliases":[{"alias_kind":"arxiv","alias_value":"2501.18771","created_at":"2026-07-05T10:07:51Z"},{"alias_kind":"arxiv_version","alias_value":"2501.18771v1","created_at":"2026-07-05T10:07:51Z"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.2501.18771","created_at":"2026-07-05T10:07:51Z"},{"alias_kind":"pith_short_12","alias_value":"V6G5IIYUEPBD","created_at":"2026-07-05T10:07:51Z"},{"alias_kind":"pith_short_16","alias_value":"V6G5IIYUEPBDGE57","created_at":"2026-07-05T10:07:51Z"},{"alias_kind":"pith_short_8","alias_value":"V6G5IIYU","created_at":"2026-07-05T10:07:51Z"}],"events":[{"event_type":"record_created","subject_pith_number":"pith:2025:V6G5IIYUEPBDGE57QCFCEMVV5C","target":"record","payload":{"canonical_record":{"source":{"id":"2501.18771","kind":"arxiv","version":1},"metadata":{"license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.CL","submitted_at":"2025-01-30T21:51:18Z","cross_cats_sorted":["cs.AI"],"title_canon_sha256":"20f1fa3bdd09de57092c4c57d8e5383ef1d09ab81afaf41689781bd15614ae20","abstract_canon_sha256":"fd0741f88451a1651edf583ab7e4ba84b010f67d28cb8654be7459f165ce81f8"},"schema_version":"1.0"},"canonical_sha256":"af8dd4231423c23313bf808a2232b5e88f1fbfb34b360175c56841255efb448c","receipt":{"kind":"pith_receipt","key_id":"pith-v1-2026-05","algorithm":"ed25519","signed_at":"2026-07-05T10:07:51.343390Z","signature_b64":"izP4ekSZhwoRZQ6n90OsdXf0/4V7W4eUrWT/eVFsL8y7jClFMlYS6WT0+KjjIZCV9gG0KxUJGy+m1GeUGXLvBA==","signed_message":"canonical_sha256_bytes","builder_version":"pith-number-builder-2026-05-17-v1","receipt_version":"0.3","canonical_sha256":"af8dd4231423c23313bf808a2232b5e88f1fbfb34b360175c56841255efb448c","last_reissued_at":"2026-07-05T10:07:51.342977Z","signature_status":"signed_v1","first_computed_at":"2026-07-05T10:07:51.342977Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"source_kind":"arxiv","source_id":"2501.18771","source_version":1,"attestation_state":"computed"},"signer":{"signer_id":"pith.science","signer_type":"pith_registry","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"created_at":"2026-07-05T10:07:51Z","supersedes":[],"prev_event":null,"signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"9rsBsi1r4nmbcouMU2wu9BxQcfa6bMplUXHNdi+ke1dN9AMoAFq3fNZPZAU/sjuGvY1SrbJQuGVDHTsFcMD5DQ==","signed_message":"open_graph_event_sha256_bytes","signed_at":"2026-08-05T06:12:18.208549Z"},"content_sha256":"6a4b8b176bbb8627a84cc81fa59730da46422cb2ed0fd72512508885cd027ed7","schema_version":"1.0","event_id":"sha256:6a4b8b176bbb8627a84cc81fa59730da46422cb2ed0fd72512508885cd027ed7"},{"event_type":"graph_snapshot","subject_pith_number":"pith:2025:V6G5IIYUEPBDGE57QCFCEMVV5C","target":"graph","payload":{"graph_snapshot":{"paper":{"title":"Overestimation in LLM Evaluation: A Controlled Large-Scale Study on Data Contamination's Impact on Machine Translation","license":"http://creativecommons.org/licenses/by/4.0/","headline":"","cross_cats":["cs.AI"],"primary_cat":"cs.CL","authors_text":"Colin Cherry, Daniel Deutsch, Eleftheria Briakou, Jiaming Luo, Markus Freitag, Muhammed Yusuf Kocyigit","submitted_at":"2025-01-30T21:51:18Z","abstract_excerpt":"Data contamination -- the accidental consumption of evaluation examples within the pre-training data -- can undermine the validity of evaluation benchmarks. In this paper, we present a rigorous analysis of the effects of contamination on language models at 1B and 8B scales on the machine translation task. Starting from a carefully decontaminated train-test split, we systematically introduce contamination at various stages, scales, and data formats to isolate its effect and measure its impact on performance metrics. Our experiments reveal that contamination with both source and target substanti"},"claims":{"count":0,"items":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"2501.18771","kind":"arxiv","version":1},"verdict":{"id":null,"model_set":{},"created_at":null,"strongest_claim":"","one_line_summary":"","pipeline_version":null,"weakest_assumption":"","pith_extraction_headline":""},"integrity":{"clean":true,"summary":{"advisory":0,"critical":0,"by_detector":{},"informational":0},"endpoint":"/pith/2501.18771/integrity.json","findings":[],"available":true,"detectors_run":[],"snapshot_sha256":"c28c3603d3b5d939e8dc4c7e95fa8dfce3d595e45f758748cecf8e644a296938"},"references":{"count":0,"sample":[],"resolved_work":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","internal_anchors":0},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"author_claims":{"count":0,"strong_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"builder_version":"pith-number-builder-2026-05-17-v1"},"verdict_id":null},"signer":{"signer_id":"pith.science","signer_type":"pith_registry","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"created_at":"2026-07-05T10:07:51Z","supersedes":[],"prev_event":null,"signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"uEpfYglrCyzFNX5hmWBDdCzx+ov0NbEp9qvhmhQ1ALrSxiYTFPgM2eIH/VmS48NrutCkeXBZn7lh9m5/lWFtDw==","signed_message":"open_graph_event_sha256_bytes","signed_at":"2026-08-05T06:12:18.209066Z"},"content_sha256":"6c218a30300600f0d7737994cdbc430e4d4cf71d81d9a21193b0096934b9ea99","schema_version":"1.0","event_id":"sha256:6c218a30300600f0d7737994cdbc430e4d4cf71d81d9a21193b0096934b9ea99"}],"timestamp_proofs":[],"mirror_hints":[{"mirror_type":"https","name":"Pith Resolver","base_url":"https://pith.science","bundle_url":"https://pith.science/pith/V6G5IIYUEPBDGE57QCFCEMVV5C/bundle.json","state_url":"https://pith.science/pith/V6G5IIYUEPBDGE57QCFCEMVV5C/state.json","well_known_bundle_url":"https://pith.science/.well-known/pith/V6G5IIYUEPBDGE57QCFCEMVV5C/bundle.json","status":"primary"}],"public_keys":[{"key_id":"pith-v1-2026-05","algorithm":"ed25519","format":"raw","public_key_b64":"stVStoiQhXFxp4s2pdzPNoqVNBMojDU/fJ2db5S3CbM=","public_key_hex":"b2d552b68890857171a78b36a5dccf368a953413288c353f7c9d9d6f94b709b3","fingerprint_sha256_b32_first128bits":"RVFV5Z2OI2J3ZUO7ERDEBCYNKS","fingerprint_sha256_hex":"8d4b5ee74e4693bcd1df2446408b0d54","rotates_at":null,"url":"https://pith.science/pith-signing-key.json","notes":"Pith uses this Ed25519 key to sign canonical record SHA-256 digests. Verify with: ed25519_verify(public_key, message=canonical_sha256_bytes, signature=base64decode(signature_b64))."}],"merge_version":"pith-open-graph-merge-v1","built_at":"2026-08-05T06:12:18Z","links":{"resolver":"https://pith.science/pith/V6G5IIYUEPBDGE57QCFCEMVV5C","bundle":"https://pith.science/pith/V6G5IIYUEPBDGE57QCFCEMVV5C/bundle.json","state":"https://pith.science/pith/V6G5IIYUEPBDGE57QCFCEMVV5C/state.json","well_known_bundle":"https://pith.science/.well-known/pith/V6G5IIYUEPBDGE57QCFCEMVV5C/bundle.json"},"state":{"state_type":"pith_open_graph_state","state_version":"1.0","pith_number":"pith:2025:V6G5IIYUEPBDGE57QCFCEMVV5C","merge_version":"pith-open-graph-merge-v1","event_count":2,"valid_event_count":2,"invalid_event_count":0,"equivocation_count":0,"current":{"canonical_record":{"metadata":{"abstract_canon_sha256":"fd0741f88451a1651edf583ab7e4ba84b010f67d28cb8654be7459f165ce81f8","cross_cats_sorted":["cs.AI"],"license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.CL","submitted_at":"2025-01-30T21:51:18Z","title_canon_sha256":"20f1fa3bdd09de57092c4c57d8e5383ef1d09ab81afaf41689781bd15614ae20"},"schema_version":"1.0","source":{"id":"2501.18771","kind":"arxiv","version":1}},"source_aliases":[{"alias_kind":"arxiv","alias_value":"2501.18771","created_at":"2026-07-05T10:07:51Z"},{"alias_kind":"arxiv_version","alias_value":"2501.18771v1","created_at":"2026-07-05T10:07:51Z"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.2501.18771","created_at":"2026-07-05T10:07:51Z"},{"alias_kind":"pith_short_12","alias_value":"V6G5IIYUEPBD","created_at":"2026-07-05T10:07:51Z"},{"alias_kind":"pith_short_16","alias_value":"V6G5IIYUEPBDGE57","created_at":"2026-07-05T10:07:51Z"},{"alias_kind":"pith_short_8","alias_value":"V6G5IIYU","created_at":"2026-07-05T10:07:51Z"}],"graph_snapshots":[{"event_id":"sha256:6c218a30300600f0d7737994cdbc430e4d4cf71d81d9a21193b0096934b9ea99","target":"graph","created_at":"2026-07-05T10:07:51Z","signer":{"key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signer_id":"pith.science","signer_type":"pith_registry"},"payload":{"graph_snapshot":{"author_claims":{"count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","strong_count":0},"builder_version":"pith-number-builder-2026-05-17-v1","claims":{"count":0,"items":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"integrity":{"available":true,"clean":true,"detectors_run":[],"endpoint":"/pith/2501.18771/integrity.json","findings":[],"snapshot_sha256":"c28c3603d3b5d939e8dc4c7e95fa8dfce3d595e45f758748cecf8e644a296938","summary":{"advisory":0,"by_detector":{},"critical":0,"informational":0}},"paper":{"abstract_excerpt":"Data contamination -- the accidental consumption of evaluation examples within the pre-training data -- can undermine the validity of evaluation benchmarks. In this paper, we present a rigorous analysis of the effects of contamination on language models at 1B and 8B scales on the machine translation task. Starting from a carefully decontaminated train-test split, we systematically introduce contamination at various stages, scales, and data formats to isolate its effect and measure its impact on performance metrics. Our experiments reveal that contamination with both source and target substanti","authors_text":"Colin Cherry, Daniel Deutsch, Eleftheria Briakou, Jiaming Luo, Markus Freitag, Muhammed Yusuf Kocyigit","cross_cats":["cs.AI"],"headline":"","license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.CL","submitted_at":"2025-01-30T21:51:18Z","title":"Overestimation in LLM Evaluation: A Controlled Large-Scale Study on Data Contamination's Impact on Machine Translation"},"references":{"count":0,"internal_anchors":0,"resolved_work":0,"sample":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"2501.18771","kind":"arxiv","version":1},"verdict":{"created_at":null,"id":null,"model_set":{},"one_line_summary":"","pipeline_version":null,"pith_extraction_headline":"","strongest_claim":"","weakest_assumption":""}},"verdict_id":null}}],"author_attestations":[],"timestamp_anchors":[],"storage_attestations":[],"citation_signatures":[],"replication_records":[],"corrections":[],"mirror_hints":[],"record_created":{"event_id":"sha256:6a4b8b176bbb8627a84cc81fa59730da46422cb2ed0fd72512508885cd027ed7","target":"record","created_at":"2026-07-05T10:07:51Z","signer":{"key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signer_id":"pith.science","signer_type":"pith_registry"},"payload":{"attestation_state":"computed","canonical_record":{"metadata":{"abstract_canon_sha256":"fd0741f88451a1651edf583ab7e4ba84b010f67d28cb8654be7459f165ce81f8","cross_cats_sorted":["cs.AI"],"license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.CL","submitted_at":"2025-01-30T21:51:18Z","title_canon_sha256":"20f1fa3bdd09de57092c4c57d8e5383ef1d09ab81afaf41689781bd15614ae20"},"schema_version":"1.0","source":{"id":"2501.18771","kind":"arxiv","version":1}},"canonical_sha256":"af8dd4231423c23313bf808a2232b5e88f1fbfb34b360175c56841255efb448c","receipt":{"algorithm":"ed25519","builder_version":"pith-number-builder-2026-05-17-v1","canonical_sha256":"af8dd4231423c23313bf808a2232b5e88f1fbfb34b360175c56841255efb448c","first_computed_at":"2026-07-05T10:07:51.342977Z","key_id":"pith-v1-2026-05","kind":"pith_receipt","last_reissued_at":"2026-07-05T10:07:51.342977Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","receipt_version":"0.3","signature_b64":"izP4ekSZhwoRZQ6n90OsdXf0/4V7W4eUrWT/eVFsL8y7jClFMlYS6WT0+KjjIZCV9gG0KxUJGy+m1GeUGXLvBA==","signature_status":"signed_v1","signed_at":"2026-07-05T10:07:51.343390Z","signed_message":"canonical_sha256_bytes"},"source_id":"2501.18771","source_kind":"arxiv","source_version":1}}},"equivocations":[],"invalid_events":[],"applied_event_ids":["sha256:6a4b8b176bbb8627a84cc81fa59730da46422cb2ed0fd72512508885cd027ed7","sha256:6c218a30300600f0d7737994cdbc430e4d4cf71d81d9a21193b0096934b9ea99"],"state_sha256":"8a796f1046cdd3c2d88daaa54415938bc761c0698a51ed6463c2b0a34fd1436e"},"bundle_signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"3YMfaUh2O2BH3cGUbIC0iNdCPo0DHCz0hYX2LbF1PCiHG1PAHBtL15Krg1E1UPliAqmf/K2ygNYkSBmsRjvYAQ==","signed_message":"bundle_sha256_bytes","signed_at":"2026-08-05T06:12:18.212800Z","bundle_sha256":"78ad5fd033e18178ab05be7d3fc1ac3f145ca067427fbb3b2861d53cc27a2d61"}}