{"record_type":"pith_number_record","schema_url":"https://pith.science/schemas/pith-number/v1.json","pith_number":"pith:2024:R6JS5JHEWHBXONCYL3376AF4IP","short_pith_number":"pith:R6JS5JHE","schema_version":"1.0","canonical_sha256":"8f932ea4e4b1c37734585ef7ff00bc43e3e195689a0c69698bd8eae7b6d7c312","source":{"kind":"arxiv","id":"2410.16186","version":1},"attestation_state":"computed","paper":{"title":"Contamination Report for Multilingual Benchmarks","license":"http://creativecommons.org/licenses/by/4.0/","headline":"","cross_cats":[],"primary_cat":"cs.CL","authors_text":"Sanchit Ahuja, Sunayana Sitaram, Varun Gumma","submitted_at":"2024-10-21T16:49:35Z","abstract_excerpt":"Benchmark contamination refers to the presence of test datasets in Large Language Model (LLM) pre-training or post-training data. Contamination can lead to inflated scores on benchmarks, compromising evaluation results and making it difficult to determine the capabilities of models. In this work, we study the contamination of popular multilingual benchmarks in LLMs that support multiple languages. We use the Black Box test to determine whether $7$ frequently used multilingual benchmarks are contaminated in $7$ popular open and closed LLMs and find that almost all models show signs of being con"},"verification_status":{"content_addressed":true,"pith_receipt":true,"author_attested":false,"weak_author_claims":0,"strong_author_claims":0,"externally_anchored":false,"storage_verified":false,"citation_signatures":0,"replication_records":0,"graph_snapshot":true,"references_resolved":false,"formal_links_present":false},"canonical_record":{"source":{"id":"2410.16186","kind":"arxiv","version":1},"metadata":{"license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.CL","submitted_at":"2024-10-21T16:49:35Z","cross_cats_sorted":[],"title_canon_sha256":"f021106cef4948a924e1192e8158d4095add79580a43ef70baa668398c22bb37","abstract_canon_sha256":"8e4ad931e4a42f325bef4a3eacc871135f2f8427e9f89469b19e814d99d0bcf6"},"schema_version":"1.0"},"receipt":{"kind":"pith_receipt","key_id":"pith-v1-2026-05","algorithm":"ed25519","signed_at":"2026-07-05T09:23:31.026774Z","signature_b64":"kGEItekho1DC7Bn9HnjBNNXwy15yzU+it6wDKbU+WAblEYd9KZNLdsoVq57kmFrAmN5sadgectkkVB3YGZpxAA==","signed_message":"canonical_sha256_bytes","builder_version":"pith-number-builder-2026-05-17-v1","receipt_version":"0.3","canonical_sha256":"8f932ea4e4b1c37734585ef7ff00bc43e3e195689a0c69698bd8eae7b6d7c312","last_reissued_at":"2026-07-05T09:23:31.026290Z","signature_status":"signed_v1","first_computed_at":"2026-07-05T09:23:31.026290Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"graph_snapshot":{"paper":{"title":"Contamination Report for Multilingual Benchmarks","license":"http://creativecommons.org/licenses/by/4.0/","headline":"","cross_cats":[],"primary_cat":"cs.CL","authors_text":"Sanchit Ahuja, Sunayana Sitaram, Varun Gumma","submitted_at":"2024-10-21T16:49:35Z","abstract_excerpt":"Benchmark contamination refers to the presence of test datasets in Large Language Model (LLM) pre-training or post-training data. Contamination can lead to inflated scores on benchmarks, compromising evaluation results and making it difficult to determine the capabilities of models. In this work, we study the contamination of popular multilingual benchmarks in LLMs that support multiple languages. We use the Black Box test to determine whether $7$ frequently used multilingual benchmarks are contaminated in $7$ popular open and closed LLMs and find that almost all models show signs of being con"},"claims":{"count":0,"items":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"2410.16186","kind":"arxiv","version":1},"verdict":{"id":null,"model_set":{},"created_at":null,"strongest_claim":"","one_line_summary":"","pipeline_version":null,"weakest_assumption":"","pith_extraction_headline":""},"integrity":{"clean":true,"summary":{"advisory":0,"critical":0,"by_detector":{},"informational":0},"endpoint":"/pith/2410.16186/integrity.json","findings":[],"available":true,"detectors_run":[],"snapshot_sha256":"c28c3603d3b5d939e8dc4c7e95fa8dfce3d595e45f758748cecf8e644a296938"},"references":{"count":0,"sample":[],"resolved_work":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","internal_anchors":0},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"author_claims":{"count":0,"strong_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"builder_version":"pith-number-builder-2026-05-17-v1"},"aliases":[{"alias_kind":"arxiv","alias_value":"2410.16186","created_at":"2026-07-05T09:23:31.026357+00:00"},{"alias_kind":"arxiv_version","alias_value":"2410.16186v1","created_at":"2026-07-05T09:23:31.026357+00:00"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.2410.16186","created_at":"2026-07-05T09:23:31.026357+00:00"},{"alias_kind":"pith_short_12","alias_value":"R6JS5JHEWHBX","created_at":"2026-07-05T09:23:31.026357+00:00"},{"alias_kind":"pith_short_16","alias_value":"R6JS5JHEWHBXONCY","created_at":"2026-07-05T09:23:31.026357+00:00"},{"alias_kind":"pith_short_8","alias_value":"R6JS5JHE","created_at":"2026-07-05T09:23:31.026357+00:00"}],"events":[],"event_summary":{},"paper_claims":[],"inbound_citations":{"count":4,"internal_anchor_count":0,"sample":[{"citing_arxiv_id":"2508.15503","citing_title":"Guidelines for Empirical Studies in Software Engineering involving Large Language Models","ref_index":2,"is_internal_anchor":false},{"citing_arxiv_id":"2605.19999","citing_title":"LLM Benchmark Datasets Should Be Contamination-Resistant","ref_index":77,"is_internal_anchor":false},{"citing_arxiv_id":"2508.15503","citing_title":"Guidelines for Empirical Studies in Software Engineering involving Large Language Models","ref_index":2,"is_internal_anchor":false},{"citing_arxiv_id":"2511.14774","citing_title":"LiveCLKTBench: Towards Reliable Evaluation of Cross-Lingual Knowledge Transfer in Multilingual LLMs","ref_index":2,"is_internal_anchor":false}]},"formal_canon":{"evidence_count":0,"sample":[],"anchors":[]},"links":{"html":"https://pith.science/pith/R6JS5JHEWHBXONCYL3376AF4IP","json":"https://pith.science/pith/R6JS5JHEWHBXONCYL3376AF4IP.json","graph_json":"https://pith.science/api/pith-number/R6JS5JHEWHBXONCYL3376AF4IP/graph.json","events_json":"https://pith.science/api/pith-number/R6JS5JHEWHBXONCYL3376AF4IP/events.json","paper":"https://pith.science/paper/R6JS5JHE"},"agent_actions":{"view_html":"https://pith.science/pith/R6JS5JHEWHBXONCYL3376AF4IP","download_json":"https://pith.science/pith/R6JS5JHEWHBXONCYL3376AF4IP.json","view_paper":"https://pith.science/paper/R6JS5JHE","resolve_alias":"https://pith.science/api/pith-number/resolve?arxiv=2410.16186&json=true","fetch_graph":"https://pith.science/api/pith-number/R6JS5JHEWHBXONCYL3376AF4IP/graph.json","fetch_events":"https://pith.science/api/pith-number/R6JS5JHEWHBXONCYL3376AF4IP/events.json","actions":{"anchor_timestamp":"https://pith.science/pith/R6JS5JHEWHBXONCYL3376AF4IP/action/timestamp_anchor","attest_storage":"https://pith.science/pith/R6JS5JHEWHBXONCYL3376AF4IP/action/storage_attestation","attest_author":"https://pith.science/pith/R6JS5JHEWHBXONCYL3376AF4IP/action/author_attestation","sign_citation":"https://pith.science/pith/R6JS5JHEWHBXONCYL3376AF4IP/action/citation_signature","submit_replication":"https://pith.science/pith/R6JS5JHEWHBXONCYL3376AF4IP/action/replication_record"}},"created_at":"2026-07-05T09:23:31.026357+00:00","updated_at":"2026-07-05T09:23:31.026357+00:00"}