{"record_type":"pith_number_record","schema_url":"https://pith.science/schemas/pith-number/v1.json","pith_number":"pith:2024:5GDS2KFJCMPQTHTH77CECN74GL","short_pith_number":"pith:5GDS2KFJ","schema_version":"1.0","canonical_sha256":"e9872d28a9131f099e67ffc44137fc32edf9f7000095cc072f159f51f8e01372","source":{"kind":"arxiv","id":"2412.03304","version":2},"attestation_state":"computed","paper":{"title":"Global MMLU: Understanding and Addressing Cultural and Linguistic Biases in Multilingual Evaluation","license":"http://creativecommons.org/licenses/by/4.0/","headline":"","cross_cats":[],"primary_cat":"cs.CL","authors_text":"Alice Oh, Andre F. T. Martins, Angelika Romanou, Antoine Bosselut, Beyza Ermis, Cl\\'ementine Fourrier, Daniel Vila-Suero, Daphne Ippolito, David I. Adelani, Enzo Ferrante, Jian Gang Ngui, Kelly Marchisio, Leshem Choshen, Madeline Smith, Marzieh Fadaee, Peerat Limkonchotiwat, Raymond Ng, Sara Hooker, Sebastian Ruder, Shayne Longpre, Shivalika Singh, Wei Qi Leong, Wei-Yin Ko, Yosephine Susanto","submitted_at":"2024-12-04T13:27:09Z","abstract_excerpt":"Cultural biases in multilingual datasets pose significant challenges for their effectiveness as global benchmarks. These biases stem not only from differences in language but also from the cultural knowledge required to interpret questions, reducing the practical utility of translated datasets like MMLU. Furthermore, translation often introduces artefacts that can distort the meaning or clarity of questions in the target language. A common practice in multilingual evaluation is to rely on machine-translated evaluation sets, but simply translating a dataset is insufficient to address these chal"},"verification_status":{"content_addressed":true,"pith_receipt":true,"author_attested":false,"weak_author_claims":0,"strong_author_claims":0,"externally_anchored":false,"storage_verified":false,"citation_signatures":0,"replication_records":0,"graph_snapshot":true,"references_resolved":false,"formal_links_present":false},"canonical_record":{"source":{"id":"2412.03304","kind":"arxiv","version":2},"metadata":{"license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.CL","submitted_at":"2024-12-04T13:27:09Z","cross_cats_sorted":[],"title_canon_sha256":"a80f5478de9a966b86b324ddc0c24d9f9daa956b38a8afc0e961139a7837788e","abstract_canon_sha256":"1696edc59f38e6f209087be5c2d11c03690759b7653b8d48a1db9bb01ab8a9ca"},"schema_version":"1.0"},"receipt":{"kind":"pith_receipt","key_id":"pith-v1-2026-05","algorithm":"ed25519","signed_at":"2026-07-05T10:16:39.959051Z","signature_b64":"81daTC9xT20oK3DY2Kc2PZzzHiIIpXynMo8YJcqc3b4K6nmOcPaNlf3E9T/YfeqtQ6rJOiBGVEZN2IYj4gQ/Ag==","signed_message":"canonical_sha256_bytes","builder_version":"pith-number-builder-2026-05-17-v1","receipt_version":"0.3","canonical_sha256":"e9872d28a9131f099e67ffc44137fc32edf9f7000095cc072f159f51f8e01372","last_reissued_at":"2026-07-05T10:16:39.958543Z","signature_status":"signed_v1","first_computed_at":"2026-07-05T10:16:39.958543Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"graph_snapshot":{"paper":{"title":"Global MMLU: Understanding and Addressing Cultural and Linguistic Biases in Multilingual Evaluation","license":"http://creativecommons.org/licenses/by/4.0/","headline":"","cross_cats":[],"primary_cat":"cs.CL","authors_text":"Alice Oh, Andre F. T. Martins, Angelika Romanou, Antoine Bosselut, Beyza Ermis, Cl\\'ementine Fourrier, Daniel Vila-Suero, Daphne Ippolito, David I. Adelani, Enzo Ferrante, Jian Gang Ngui, Kelly Marchisio, Leshem Choshen, Madeline Smith, Marzieh Fadaee, Peerat Limkonchotiwat, Raymond Ng, Sara Hooker, Sebastian Ruder, Shayne Longpre, Shivalika Singh, Wei Qi Leong, Wei-Yin Ko, Yosephine Susanto","submitted_at":"2024-12-04T13:27:09Z","abstract_excerpt":"Cultural biases in multilingual datasets pose significant challenges for their effectiveness as global benchmarks. These biases stem not only from differences in language but also from the cultural knowledge required to interpret questions, reducing the practical utility of translated datasets like MMLU. Furthermore, translation often introduces artefacts that can distort the meaning or clarity of questions in the target language. A common practice in multilingual evaluation is to rely on machine-translated evaluation sets, but simply translating a dataset is insufficient to address these chal"},"claims":{"count":0,"items":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"2412.03304","kind":"arxiv","version":2},"verdict":{"id":null,"model_set":{},"created_at":null,"strongest_claim":"","one_line_summary":"","pipeline_version":null,"weakest_assumption":"","pith_extraction_headline":""},"integrity":{"clean":true,"summary":{"advisory":0,"critical":0,"by_detector":{},"informational":0},"endpoint":"/pith/2412.03304/integrity.json","findings":[],"available":true,"detectors_run":[],"snapshot_sha256":"c28c3603d3b5d939e8dc4c7e95fa8dfce3d595e45f758748cecf8e644a296938"},"references":{"count":0,"sample":[],"resolved_work":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","internal_anchors":0},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"author_claims":{"count":0,"strong_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"builder_version":"pith-number-builder-2026-05-17-v1"},"aliases":[{"alias_kind":"arxiv","alias_value":"2412.03304","created_at":"2026-07-05T10:16:39.958606+00:00"},{"alias_kind":"arxiv_version","alias_value":"2412.03304v2","created_at":"2026-07-05T10:16:39.958606+00:00"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.2412.03304","created_at":"2026-07-05T10:16:39.958606+00:00"},{"alias_kind":"pith_short_12","alias_value":"5GDS2KFJCMPQ","created_at":"2026-07-05T10:16:39.958606+00:00"},{"alias_kind":"pith_short_16","alias_value":"5GDS2KFJCMPQTHTH","created_at":"2026-07-05T10:16:39.958606+00:00"},{"alias_kind":"pith_short_8","alias_value":"5GDS2KFJ","created_at":"2026-07-05T10:16:39.958606+00:00"}],"events":[],"event_summary":{},"paper_claims":[],"inbound_citations":{"count":15,"internal_anchor_count":0,"sample":[{"citing_arxiv_id":"2606.27306","citing_title":"Multilingual Reasoning Cascades Need More Context","ref_index":2,"is_internal_anchor":false},{"citing_arxiv_id":"2605.22137","citing_title":"Cross-Lingual Consensus: Aligning Multilingual Cultural Knowledge via Multilingual Self-Consistency","ref_index":25,"is_internal_anchor":false},{"citing_arxiv_id":"2605.24904","citing_title":"Quantifying the Impact of Translation Errors on Multilingual LLM Evaluation","ref_index":6,"is_internal_anchor":false},{"citing_arxiv_id":"2606.28345","citing_title":"Auditing LLM-Governed Social Robots with Culture-Specific Moral Gradients","ref_index":49,"is_internal_anchor":false},{"citing_arxiv_id":"2605.26735","citing_title":"Rethinking the Multilingual Reasoning Gap with Layer Swap","ref_index":5,"is_internal_anchor":false},{"citing_arxiv_id":"2605.29128","citing_title":"Apertus LLM Family Expansion via Distillation and Quantization","ref_index":17,"is_internal_anchor":false},{"citing_arxiv_id":"2605.31220","citing_title":"Shared Doubt: Zero-Shot Cross-Lingual Confidence Estimation for Language Models","ref_index":4,"is_internal_anchor":false},{"citing_arxiv_id":"2605.22137","citing_title":"Cross-Lingual Consensus: Aligning Multilingual Cultural Knowledge via Multilingual Self-Consistency","ref_index":7,"is_internal_anchor":false},{"citing_arxiv_id":"2507.06261","citing_title":"Gemini 2.5: Pushing the Frontier with Advanced Reasoning, Multimodality, Long Context, and Next Generation Agentic Capabilities","ref_index":78,"is_internal_anchor":false},{"citing_arxiv_id":"2507.23009","citing_title":"Position: Stop Evaluating AI with Human Tests, Develop Principled, AI-specific Tests instead","ref_index":70,"is_internal_anchor":false},{"citing_arxiv_id":"2512.20856","citing_title":"NVIDIA Nemotron 3: Efficient and Open Intelligence","ref_index":143,"is_internal_anchor":false},{"citing_arxiv_id":"2511.02135","citing_title":"Graph-Based Alternatives to LLMs for Human Simulation","ref_index":68,"is_internal_anchor":false},{"citing_arxiv_id":"2604.24544","citing_title":"STELLAR-E: a Synthetic, Tailored, End-to-end LLM Application Rigorous Evaluator","ref_index":28,"is_internal_anchor":false},{"citing_arxiv_id":"2604.20720","citing_title":"COMPASS: COntinual Multilingual PEFT with Adaptive Semantic Sampling","ref_index":63,"is_internal_anchor":false},{"citing_arxiv_id":"2604.18914","citing_title":"MORPHOGEN: A Multilingual Benchmark for Evaluating Gender-Aware Morphological Generation","ref_index":34,"is_internal_anchor":false}]},"formal_canon":{"evidence_count":0,"sample":[],"anchors":[]},"links":{"html":"https://pith.science/pith/5GDS2KFJCMPQTHTH77CECN74GL","json":"https://pith.science/pith/5GDS2KFJCMPQTHTH77CECN74GL.json","graph_json":"https://pith.science/api/pith-number/5GDS2KFJCMPQTHTH77CECN74GL/graph.json","events_json":"https://pith.science/api/pith-number/5GDS2KFJCMPQTHTH77CECN74GL/events.json","paper":"https://pith.science/paper/5GDS2KFJ"},"agent_actions":{"view_html":"https://pith.science/pith/5GDS2KFJCMPQTHTH77CECN74GL","download_json":"https://pith.science/pith/5GDS2KFJCMPQTHTH77CECN74GL.json","view_paper":"https://pith.science/paper/5GDS2KFJ","resolve_alias":"https://pith.science/api/pith-number/resolve?arxiv=2412.03304&json=true","fetch_graph":"https://pith.science/api/pith-number/5GDS2KFJCMPQTHTH77CECN74GL/graph.json","fetch_events":"https://pith.science/api/pith-number/5GDS2KFJCMPQTHTH77CECN74GL/events.json","actions":{"anchor_timestamp":"https://pith.science/pith/5GDS2KFJCMPQTHTH77CECN74GL/action/timestamp_anchor","attest_storage":"https://pith.science/pith/5GDS2KFJCMPQTHTH77CECN74GL/action/storage_attestation","attest_author":"https://pith.science/pith/5GDS2KFJCMPQTHTH77CECN74GL/action/author_attestation","sign_citation":"https://pith.science/pith/5GDS2KFJCMPQTHTH77CECN74GL/action/citation_signature","submit_replication":"https://pith.science/pith/5GDS2KFJCMPQTHTH77CECN74GL/action/replication_record"}},"created_at":"2026-07-05T10:16:39.958606+00:00","updated_at":"2026-07-05T10:16:39.958606+00:00"}