{"bundle_type":"pith_open_graph_bundle","bundle_version":"1.0","pith_number":"pith:2024:R6JS5JHEWHBXONCYL3376AF4IP","short_pith_number":"pith:R6JS5JHE","canonical_record":{"source":{"id":"2410.16186","kind":"arxiv","version":1},"metadata":{"license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.CL","submitted_at":"2024-10-21T16:49:35Z","cross_cats_sorted":[],"title_canon_sha256":"f021106cef4948a924e1192e8158d4095add79580a43ef70baa668398c22bb37","abstract_canon_sha256":"8e4ad931e4a42f325bef4a3eacc871135f2f8427e9f89469b19e814d99d0bcf6"},"schema_version":"1.0"},"canonical_sha256":"8f932ea4e4b1c37734585ef7ff00bc43e3e195689a0c69698bd8eae7b6d7c312","source":{"kind":"arxiv","id":"2410.16186","version":1},"source_aliases":[{"alias_kind":"arxiv","alias_value":"2410.16186","created_at":"2026-07-05T09:23:31Z"},{"alias_kind":"arxiv_version","alias_value":"2410.16186v1","created_at":"2026-07-05T09:23:31Z"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.2410.16186","created_at":"2026-07-05T09:23:31Z"},{"alias_kind":"pith_short_12","alias_value":"R6JS5JHEWHBX","created_at":"2026-07-05T09:23:31Z"},{"alias_kind":"pith_short_16","alias_value":"R6JS5JHEWHBXONCY","created_at":"2026-07-05T09:23:31Z"},{"alias_kind":"pith_short_8","alias_value":"R6JS5JHE","created_at":"2026-07-05T09:23:31Z"}],"events":[{"event_type":"record_created","subject_pith_number":"pith:2024:R6JS5JHEWHBXONCYL3376AF4IP","target":"record","payload":{"canonical_record":{"source":{"id":"2410.16186","kind":"arxiv","version":1},"metadata":{"license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.CL","submitted_at":"2024-10-21T16:49:35Z","cross_cats_sorted":[],"title_canon_sha256":"f021106cef4948a924e1192e8158d4095add79580a43ef70baa668398c22bb37","abstract_canon_sha256":"8e4ad931e4a42f325bef4a3eacc871135f2f8427e9f89469b19e814d99d0bcf6"},"schema_version":"1.0"},"canonical_sha256":"8f932ea4e4b1c37734585ef7ff00bc43e3e195689a0c69698bd8eae7b6d7c312","receipt":{"kind":"pith_receipt","key_id":"pith-v1-2026-05","algorithm":"ed25519","signed_at":"2026-07-05T09:23:31.026774Z","signature_b64":"kGEItekho1DC7Bn9HnjBNNXwy15yzU+it6wDKbU+WAblEYd9KZNLdsoVq57kmFrAmN5sadgectkkVB3YGZpxAA==","signed_message":"canonical_sha256_bytes","builder_version":"pith-number-builder-2026-05-17-v1","receipt_version":"0.3","canonical_sha256":"8f932ea4e4b1c37734585ef7ff00bc43e3e195689a0c69698bd8eae7b6d7c312","last_reissued_at":"2026-07-05T09:23:31.026290Z","signature_status":"signed_v1","first_computed_at":"2026-07-05T09:23:31.026290Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"source_kind":"arxiv","source_id":"2410.16186","source_version":1,"attestation_state":"computed"},"signer":{"signer_id":"pith.science","signer_type":"pith_registry","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"created_at":"2026-07-05T09:23:31Z","supersedes":[],"prev_event":null,"signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"6gxnbCJnbav8eYs5Mg7yyT8BxKnl7Wr0yhN9c3vjIXj9GEDJfyeFY2MuXsWWhC1R3Hz7/B64F/McvD6wzjO+BA==","signed_message":"open_graph_event_sha256_bytes","signed_at":"2026-07-08T20:28:16.799834Z"},"content_sha256":"40a55937ad4a1eedc971dcee0c05c3d09e03a1cec5f175ac5b93ea9f73fd8354","schema_version":"1.0","event_id":"sha256:40a55937ad4a1eedc971dcee0c05c3d09e03a1cec5f175ac5b93ea9f73fd8354"},{"event_type":"graph_snapshot","subject_pith_number":"pith:2024:R6JS5JHEWHBXONCYL3376AF4IP","target":"graph","payload":{"graph_snapshot":{"paper":{"title":"Contamination Report for Multilingual Benchmarks","license":"http://creativecommons.org/licenses/by/4.0/","headline":"","cross_cats":[],"primary_cat":"cs.CL","authors_text":"Sanchit Ahuja, Sunayana Sitaram, Varun Gumma","submitted_at":"2024-10-21T16:49:35Z","abstract_excerpt":"Benchmark contamination refers to the presence of test datasets in Large Language Model (LLM) pre-training or post-training data. Contamination can lead to inflated scores on benchmarks, compromising evaluation results and making it difficult to determine the capabilities of models. In this work, we study the contamination of popular multilingual benchmarks in LLMs that support multiple languages. We use the Black Box test to determine whether $7$ frequently used multilingual benchmarks are contaminated in $7$ popular open and closed LLMs and find that almost all models show signs of being con"},"claims":{"count":0,"items":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"2410.16186","kind":"arxiv","version":1},"verdict":{"id":null,"model_set":{},"created_at":null,"strongest_claim":"","one_line_summary":"","pipeline_version":null,"weakest_assumption":"","pith_extraction_headline":""},"integrity":{"clean":true,"summary":{"advisory":0,"critical":0,"by_detector":{},"informational":0},"endpoint":"/pith/2410.16186/integrity.json","findings":[],"available":true,"detectors_run":[],"snapshot_sha256":"c28c3603d3b5d939e8dc4c7e95fa8dfce3d595e45f758748cecf8e644a296938"},"references":{"count":0,"sample":[],"resolved_work":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","internal_anchors":0},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"author_claims":{"count":0,"strong_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"builder_version":"pith-number-builder-2026-05-17-v1"},"verdict_id":null},"signer":{"signer_id":"pith.science","signer_type":"pith_registry","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"created_at":"2026-07-05T09:23:31Z","supersedes":[],"prev_event":null,"signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"gqHVMIQV3PYTPOGXCkEdSt4uirsnO8IoMqeyC8FpTQeOA17O4WbSFx73iy4163o5Uq9tl5S3W7qIrKiLCJ/rAg==","signed_message":"open_graph_event_sha256_bytes","signed_at":"2026-07-08T20:28:16.800203Z"},"content_sha256":"b716d8901e19db9172783d24477d84e13b5d1478f8bc013b1c0ca4f6aecff37c","schema_version":"1.0","event_id":"sha256:b716d8901e19db9172783d24477d84e13b5d1478f8bc013b1c0ca4f6aecff37c"}],"timestamp_proofs":[],"mirror_hints":[{"mirror_type":"https","name":"Pith Resolver","base_url":"https://pith.science","bundle_url":"https://pith.science/pith/R6JS5JHEWHBXONCYL3376AF4IP/bundle.json","state_url":"https://pith.science/pith/R6JS5JHEWHBXONCYL3376AF4IP/state.json","well_known_bundle_url":"https://pith.science/.well-known/pith/R6JS5JHEWHBXONCYL3376AF4IP/bundle.json","status":"primary"}],"public_keys":[{"key_id":"pith-v1-2026-05","algorithm":"ed25519","format":"raw","public_key_b64":"stVStoiQhXFxp4s2pdzPNoqVNBMojDU/fJ2db5S3CbM=","public_key_hex":"b2d552b68890857171a78b36a5dccf368a953413288c353f7c9d9d6f94b709b3","fingerprint_sha256_b32_first128bits":"RVFV5Z2OI2J3ZUO7ERDEBCYNKS","fingerprint_sha256_hex":"8d4b5ee74e4693bcd1df2446408b0d54","rotates_at":null,"url":"https://pith.science/pith-signing-key.json","notes":"Pith uses this Ed25519 key to sign canonical record SHA-256 digests. Verify with: ed25519_verify(public_key, message=canonical_sha256_bytes, signature=base64decode(signature_b64))."}],"merge_version":"pith-open-graph-merge-v1","built_at":"2026-07-08T20:28:16Z","links":{"resolver":"https://pith.science/pith/R6JS5JHEWHBXONCYL3376AF4IP","bundle":"https://pith.science/pith/R6JS5JHEWHBXONCYL3376AF4IP/bundle.json","state":"https://pith.science/pith/R6JS5JHEWHBXONCYL3376AF4IP/state.json","well_known_bundle":"https://pith.science/.well-known/pith/R6JS5JHEWHBXONCYL3376AF4IP/bundle.json"},"state":{"state_type":"pith_open_graph_state","state_version":"1.0","pith_number":"pith:2024:R6JS5JHEWHBXONCYL3376AF4IP","merge_version":"pith-open-graph-merge-v1","event_count":2,"valid_event_count":2,"invalid_event_count":0,"equivocation_count":0,"current":{"canonical_record":{"metadata":{"abstract_canon_sha256":"8e4ad931e4a42f325bef4a3eacc871135f2f8427e9f89469b19e814d99d0bcf6","cross_cats_sorted":[],"license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.CL","submitted_at":"2024-10-21T16:49:35Z","title_canon_sha256":"f021106cef4948a924e1192e8158d4095add79580a43ef70baa668398c22bb37"},"schema_version":"1.0","source":{"id":"2410.16186","kind":"arxiv","version":1}},"source_aliases":[{"alias_kind":"arxiv","alias_value":"2410.16186","created_at":"2026-07-05T09:23:31Z"},{"alias_kind":"arxiv_version","alias_value":"2410.16186v1","created_at":"2026-07-05T09:23:31Z"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.2410.16186","created_at":"2026-07-05T09:23:31Z"},{"alias_kind":"pith_short_12","alias_value":"R6JS5JHEWHBX","created_at":"2026-07-05T09:23:31Z"},{"alias_kind":"pith_short_16","alias_value":"R6JS5JHEWHBXONCY","created_at":"2026-07-05T09:23:31Z"},{"alias_kind":"pith_short_8","alias_value":"R6JS5JHE","created_at":"2026-07-05T09:23:31Z"}],"graph_snapshots":[{"event_id":"sha256:b716d8901e19db9172783d24477d84e13b5d1478f8bc013b1c0ca4f6aecff37c","target":"graph","created_at":"2026-07-05T09:23:31Z","signer":{"key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signer_id":"pith.science","signer_type":"pith_registry"},"payload":{"graph_snapshot":{"author_claims":{"count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","strong_count":0},"builder_version":"pith-number-builder-2026-05-17-v1","claims":{"count":0,"items":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"integrity":{"available":true,"clean":true,"detectors_run":[],"endpoint":"/pith/2410.16186/integrity.json","findings":[],"snapshot_sha256":"c28c3603d3b5d939e8dc4c7e95fa8dfce3d595e45f758748cecf8e644a296938","summary":{"advisory":0,"by_detector":{},"critical":0,"informational":0}},"paper":{"abstract_excerpt":"Benchmark contamination refers to the presence of test datasets in Large Language Model (LLM) pre-training or post-training data. Contamination can lead to inflated scores on benchmarks, compromising evaluation results and making it difficult to determine the capabilities of models. In this work, we study the contamination of popular multilingual benchmarks in LLMs that support multiple languages. We use the Black Box test to determine whether $7$ frequently used multilingual benchmarks are contaminated in $7$ popular open and closed LLMs and find that almost all models show signs of being con","authors_text":"Sanchit Ahuja, Sunayana Sitaram, Varun Gumma","cross_cats":[],"headline":"","license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.CL","submitted_at":"2024-10-21T16:49:35Z","title":"Contamination Report for Multilingual Benchmarks"},"references":{"count":0,"internal_anchors":0,"resolved_work":0,"sample":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"2410.16186","kind":"arxiv","version":1},"verdict":{"created_at":null,"id":null,"model_set":{},"one_line_summary":"","pipeline_version":null,"pith_extraction_headline":"","strongest_claim":"","weakest_assumption":""}},"verdict_id":null}}],"author_attestations":[],"timestamp_anchors":[],"storage_attestations":[],"citation_signatures":[],"replication_records":[],"corrections":[],"mirror_hints":[],"record_created":{"event_id":"sha256:40a55937ad4a1eedc971dcee0c05c3d09e03a1cec5f175ac5b93ea9f73fd8354","target":"record","created_at":"2026-07-05T09:23:31Z","signer":{"key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signer_id":"pith.science","signer_type":"pith_registry"},"payload":{"attestation_state":"computed","canonical_record":{"metadata":{"abstract_canon_sha256":"8e4ad931e4a42f325bef4a3eacc871135f2f8427e9f89469b19e814d99d0bcf6","cross_cats_sorted":[],"license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.CL","submitted_at":"2024-10-21T16:49:35Z","title_canon_sha256":"f021106cef4948a924e1192e8158d4095add79580a43ef70baa668398c22bb37"},"schema_version":"1.0","source":{"id":"2410.16186","kind":"arxiv","version":1}},"canonical_sha256":"8f932ea4e4b1c37734585ef7ff00bc43e3e195689a0c69698bd8eae7b6d7c312","receipt":{"algorithm":"ed25519","builder_version":"pith-number-builder-2026-05-17-v1","canonical_sha256":"8f932ea4e4b1c37734585ef7ff00bc43e3e195689a0c69698bd8eae7b6d7c312","first_computed_at":"2026-07-05T09:23:31.026290Z","key_id":"pith-v1-2026-05","kind":"pith_receipt","last_reissued_at":"2026-07-05T09:23:31.026290Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","receipt_version":"0.3","signature_b64":"kGEItekho1DC7Bn9HnjBNNXwy15yzU+it6wDKbU+WAblEYd9KZNLdsoVq57kmFrAmN5sadgectkkVB3YGZpxAA==","signature_status":"signed_v1","signed_at":"2026-07-05T09:23:31.026774Z","signed_message":"canonical_sha256_bytes"},"source_id":"2410.16186","source_kind":"arxiv","source_version":1}}},"equivocations":[],"invalid_events":[],"applied_event_ids":["sha256:40a55937ad4a1eedc971dcee0c05c3d09e03a1cec5f175ac5b93ea9f73fd8354","sha256:b716d8901e19db9172783d24477d84e13b5d1478f8bc013b1c0ca4f6aecff37c"],"state_sha256":"f7cedbee547dfd6ed12fee0ec1d6639315d523dbedc292475d123f1c51efb10d"},"bundle_signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"WkknJ+zjstoXVchmC/CX+UQdjBeB+nZRtHQ3TOLCeHcAbqvdnNQtq81t09ilvkasrSRqAtzc1qxnrhuqtAtiBg==","signed_message":"bundle_sha256_bytes","signed_at":"2026-07-08T20:28:16.802215Z","bundle_sha256":"60a07312d9d1fac38704433b8461ec0bc88cd223965b1b80fbdf3399ebc6777c"}}