{"record_type":"pith_number_record","schema_url":"https://pith.science/schemas/pith-number/v1.json","pith_number":"pith:2024:GDS7MHYH7VWQAE32ACL2RSMBMT","short_pith_number":"pith:GDS7MHYH","schema_version":"1.0","canonical_sha256":"30e5f61f07fd6d00137a0097a8c98164c31549318ce8be9191383e268f7a5d91","source":{"kind":"arxiv","id":"2406.04127","version":3},"attestation_state":"computed","paper":{"title":"Are We Done with MMLU?","license":"http://creativecommons.org/licenses/by/4.0/","headline":"","cross_cats":["cs.AI"],"primary_cat":"cs.CL","authors_text":"Alberto Carlo Maria Mancino, Alessio Devoto, Aryo Pradipta Gema, Claire Barale, Emile van Krieken, Giwon Hong, Jean Kaddour, Joshua Harris, Joshua Ong Jun Leang, Mohammad Reza Ghasemi Madani, Pasquale Minervini, Robert McHardy, Rohit Saxena, Xiaotang Du, Xuanli He, Yu Zhao","submitted_at":"2024-06-06T14:49:06Z","abstract_excerpt":"Maybe not. We identify and analyse errors in the popular Massive Multitask Language Understanding (MMLU) benchmark. Even though MMLU is widely adopted, our analysis demonstrates numerous ground truth errors that obscure the true capabilities of LLMs. For example, we find that 57% of the analysed questions in the Virology subset contain errors. To address this issue, we introduce a comprehensive framework for identifying dataset errors using a novel error annotation protocol. Then, we create MMLU-Redux, which is a subset of 5,700 manually re-annotated questions across all 57 MMLU subjects. We e"},"verification_status":{"content_addressed":true,"pith_receipt":true,"author_attested":false,"weak_author_claims":0,"strong_author_claims":0,"externally_anchored":false,"storage_verified":false,"citation_signatures":0,"replication_records":0,"graph_snapshot":true,"references_resolved":false,"formal_links_present":false},"canonical_record":{"source":{"id":"2406.04127","kind":"arxiv","version":3},"metadata":{"license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.CL","submitted_at":"2024-06-06T14:49:06Z","cross_cats_sorted":["cs.AI"],"title_canon_sha256":"943a1c434b5b4a487de0e9c3e56d98db7b929f9aa4058d84e6e153a7237e5d4c","abstract_canon_sha256":"47d71667955c77b30f75cfbfdce825653d66e87629a2771eb9bca5345ad71f03"},"schema_version":"1.0"},"receipt":{"kind":"pith_receipt","key_id":"pith-v1-2026-05","algorithm":"ed25519","signed_at":"2026-07-05T09:59:16.124435Z","signature_b64":"ZqHFwGFdXTeIao48xidFGTSKhl4V3oZQuisRqS8dSBVfqvQIqcdNa6xfx/nbybtNxVkA8ki5rAHFoiDvjf+dDQ==","signed_message":"canonical_sha256_bytes","builder_version":"pith-number-builder-2026-05-17-v1","receipt_version":"0.3","canonical_sha256":"30e5f61f07fd6d00137a0097a8c98164c31549318ce8be9191383e268f7a5d91","last_reissued_at":"2026-07-05T09:59:16.123997Z","signature_status":"signed_v1","first_computed_at":"2026-07-05T09:59:16.123997Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"graph_snapshot":{"paper":{"title":"Are We Done with MMLU?","license":"http://creativecommons.org/licenses/by/4.0/","headline":"","cross_cats":["cs.AI"],"primary_cat":"cs.CL","authors_text":"Alberto Carlo Maria Mancino, Alessio Devoto, Aryo Pradipta Gema, Claire Barale, Emile van Krieken, Giwon Hong, Jean Kaddour, Joshua Harris, Joshua Ong Jun Leang, Mohammad Reza Ghasemi Madani, Pasquale Minervini, Robert McHardy, Rohit Saxena, Xiaotang Du, Xuanli He, Yu Zhao","submitted_at":"2024-06-06T14:49:06Z","abstract_excerpt":"Maybe not. We identify and analyse errors in the popular Massive Multitask Language Understanding (MMLU) benchmark. Even though MMLU is widely adopted, our analysis demonstrates numerous ground truth errors that obscure the true capabilities of LLMs. For example, we find that 57% of the analysed questions in the Virology subset contain errors. To address this issue, we introduce a comprehensive framework for identifying dataset errors using a novel error annotation protocol. Then, we create MMLU-Redux, which is a subset of 5,700 manually re-annotated questions across all 57 MMLU subjects. We e"},"claims":{"count":0,"items":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"2406.04127","kind":"arxiv","version":3},"verdict":{"id":null,"model_set":{},"created_at":null,"strongest_claim":"","one_line_summary":"","pipeline_version":null,"weakest_assumption":"","pith_extraction_headline":""},"integrity":{"clean":true,"summary":{"advisory":0,"critical":0,"by_detector":{},"informational":0},"endpoint":"/pith/2406.04127/integrity.json","findings":[],"available":true,"detectors_run":[],"snapshot_sha256":"c28c3603d3b5d939e8dc4c7e95fa8dfce3d595e45f758748cecf8e644a296938"},"references":{"count":0,"sample":[],"resolved_work":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","internal_anchors":0},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"author_claims":{"count":0,"strong_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"builder_version":"pith-number-builder-2026-05-17-v1"},"aliases":[{"alias_kind":"arxiv","alias_value":"2406.04127","created_at":"2026-07-05T09:59:16.124051+00:00"},{"alias_kind":"arxiv_version","alias_value":"2406.04127v3","created_at":"2026-07-05T09:59:16.124051+00:00"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.2406.04127","created_at":"2026-07-05T09:59:16.124051+00:00"},{"alias_kind":"pith_short_12","alias_value":"GDS7MHYH7VWQ","created_at":"2026-07-05T09:59:16.124051+00:00"},{"alias_kind":"pith_short_16","alias_value":"GDS7MHYH7VWQAE32","created_at":"2026-07-05T09:59:16.124051+00:00"},{"alias_kind":"pith_short_8","alias_value":"GDS7MHYH","created_at":"2026-07-05T09:59:16.124051+00:00"}],"events":[],"event_summary":{},"paper_claims":[],"inbound_citations":{"count":23,"internal_anchor_count":0,"sample":[{"citing_arxiv_id":"2606.25331","citing_title":"Improved Large Language Diffusion Models","ref_index":45,"is_internal_anchor":false},{"citing_arxiv_id":"2606.18216","citing_title":"Zone of Proximal Policy Optimization: Teacher in Prompts, Not Gradients","ref_index":149,"is_internal_anchor":false},{"citing_arxiv_id":"2606.05104","citing_title":"Knowledge Index of Noah's Ark","ref_index":10,"is_internal_anchor":false},{"citing_arxiv_id":"2606.27632","citing_title":"Yuvion LLM: An Adversarially-Aware Large Language Model for Content And AI Safety","ref_index":13,"is_internal_anchor":false},{"citing_arxiv_id":"2605.31268","citing_title":"Mellum2 Technical Report","ref_index":20,"is_internal_anchor":false},{"citing_arxiv_id":"2502.13923","citing_title":"Qwen2.5-VL Technical Report","ref_index":11,"is_internal_anchor":false},{"citing_arxiv_id":"2506.12119","citing_title":"Mixture-of-Experts Can Surpass Dense LLMs Under Strictly Equal Resource","ref_index":11,"is_internal_anchor":false},{"citing_arxiv_id":"2605.08738","citing_title":"SlimQwen: Exploring the Pruning and Distillation in Large MoE Model Pre-training","ref_index":31,"is_internal_anchor":false},{"citing_arxiv_id":"2605.18904","citing_title":"Dynamic Model Merging Made Slim","ref_index":30,"is_internal_anchor":false},{"citing_arxiv_id":"2507.23009","citing_title":"Position: Stop Evaluating AI with Human Tests, Develop Principled, AI-specific Tests instead","ref_index":27,"is_internal_anchor":false},{"citing_arxiv_id":"2406.08464","citing_title":"Magpie: Alignment Data Synthesis from Scratch by Prompting Aligned LLMs with Nothing","ref_index":110,"is_internal_anchor":false},{"citing_arxiv_id":"2602.10718","citing_title":"SnapMLA: Efficient Long-Context MLA Decoding via Hardware-Aware FP8 Quantized Pipelining","ref_index":10,"is_internal_anchor":false},{"citing_arxiv_id":"2501.15383","citing_title":"Qwen2.5-1M Technical Report","ref_index":10,"is_internal_anchor":false},{"citing_arxiv_id":"2601.08584","citing_title":"Ministral 3","ref_index":20,"is_internal_anchor":false},{"citing_arxiv_id":"2510.26692","citing_title":"Kimi Linear: An Expressive, Efficient Attention Architecture","ref_index":26,"is_internal_anchor":false},{"citing_arxiv_id":"2601.02780","citing_title":"MiMo-V2-Flash Technical Report","ref_index":17,"is_internal_anchor":false},{"citing_arxiv_id":"2605.08738","citing_title":"SlimQwen: Exploring the Pruning and Distillation in Large MoE Model Pre-training","ref_index":31,"is_internal_anchor":false},{"citing_arxiv_id":"2509.17765","citing_title":"Qwen3-Omni Technical Report","ref_index":11,"is_internal_anchor":false},{"citing_arxiv_id":"2503.20215","citing_title":"Qwen2.5-Omni Technical Report","ref_index":16,"is_internal_anchor":false},{"citing_arxiv_id":"2507.20534","citing_title":"Kimi K2: Open Agentic Intelligence","ref_index":18,"is_internal_anchor":false},{"citing_arxiv_id":"2604.15804","citing_title":"Qwen3.5-Omni Technical Report","ref_index":13,"is_internal_anchor":false},{"citing_arxiv_id":"2505.09388","citing_title":"Qwen3 Technical Report","ref_index":14,"is_internal_anchor":false},{"citing_arxiv_id":"2605.02442","citing_title":"Measuring AI Reasoning: A Guide for Researchers","ref_index":147,"is_internal_anchor":false}]},"formal_canon":{"evidence_count":0,"sample":[],"anchors":[]},"links":{"html":"https://pith.science/pith/GDS7MHYH7VWQAE32ACL2RSMBMT","json":"https://pith.science/pith/GDS7MHYH7VWQAE32ACL2RSMBMT.json","graph_json":"https://pith.science/api/pith-number/GDS7MHYH7VWQAE32ACL2RSMBMT/graph.json","events_json":"https://pith.science/api/pith-number/GDS7MHYH7VWQAE32ACL2RSMBMT/events.json","paper":"https://pith.science/paper/GDS7MHYH"},"agent_actions":{"view_html":"https://pith.science/pith/GDS7MHYH7VWQAE32ACL2RSMBMT","download_json":"https://pith.science/pith/GDS7MHYH7VWQAE32ACL2RSMBMT.json","view_paper":"https://pith.science/paper/GDS7MHYH","resolve_alias":"https://pith.science/api/pith-number/resolve?arxiv=2406.04127&json=true","fetch_graph":"https://pith.science/api/pith-number/GDS7MHYH7VWQAE32ACL2RSMBMT/graph.json","fetch_events":"https://pith.science/api/pith-number/GDS7MHYH7VWQAE32ACL2RSMBMT/events.json","actions":{"anchor_timestamp":"https://pith.science/pith/GDS7MHYH7VWQAE32ACL2RSMBMT/action/timestamp_anchor","attest_storage":"https://pith.science/pith/GDS7MHYH7VWQAE32ACL2RSMBMT/action/storage_attestation","attest_author":"https://pith.science/pith/GDS7MHYH7VWQAE32ACL2RSMBMT/action/author_attestation","sign_citation":"https://pith.science/pith/GDS7MHYH7VWQAE32ACL2RSMBMT/action/citation_signature","submit_replication":"https://pith.science/pith/GDS7MHYH7VWQAE32ACL2RSMBMT/action/replication_record"}},"created_at":"2026-07-05T09:59:16.124051+00:00","updated_at":"2026-07-05T09:59:16.124051+00:00"}