{"bundle_type":"pith_open_graph_bundle","bundle_version":"1.0","pith_number":"pith:2024:GDS7MHYH7VWQAE32ACL2RSMBMT","short_pith_number":"pith:GDS7MHYH","canonical_record":{"source":{"id":"2406.04127","kind":"arxiv","version":3},"metadata":{"license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.CL","submitted_at":"2024-06-06T14:49:06Z","cross_cats_sorted":["cs.AI"],"title_canon_sha256":"943a1c434b5b4a487de0e9c3e56d98db7b929f9aa4058d84e6e153a7237e5d4c","abstract_canon_sha256":"47d71667955c77b30f75cfbfdce825653d66e87629a2771eb9bca5345ad71f03"},"schema_version":"1.0"},"canonical_sha256":"30e5f61f07fd6d00137a0097a8c98164c31549318ce8be9191383e268f7a5d91","source":{"kind":"arxiv","id":"2406.04127","version":3},"source_aliases":[{"alias_kind":"arxiv","alias_value":"2406.04127","created_at":"2026-07-05T09:59:16Z"},{"alias_kind":"arxiv_version","alias_value":"2406.04127v3","created_at":"2026-07-05T09:59:16Z"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.2406.04127","created_at":"2026-07-05T09:59:16Z"},{"alias_kind":"pith_short_12","alias_value":"GDS7MHYH7VWQ","created_at":"2026-07-05T09:59:16Z"},{"alias_kind":"pith_short_16","alias_value":"GDS7MHYH7VWQAE32","created_at":"2026-07-05T09:59:16Z"},{"alias_kind":"pith_short_8","alias_value":"GDS7MHYH","created_at":"2026-07-05T09:59:16Z"}],"events":[{"event_type":"record_created","subject_pith_number":"pith:2024:GDS7MHYH7VWQAE32ACL2RSMBMT","target":"record","payload":{"canonical_record":{"source":{"id":"2406.04127","kind":"arxiv","version":3},"metadata":{"license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.CL","submitted_at":"2024-06-06T14:49:06Z","cross_cats_sorted":["cs.AI"],"title_canon_sha256":"943a1c434b5b4a487de0e9c3e56d98db7b929f9aa4058d84e6e153a7237e5d4c","abstract_canon_sha256":"47d71667955c77b30f75cfbfdce825653d66e87629a2771eb9bca5345ad71f03"},"schema_version":"1.0"},"canonical_sha256":"30e5f61f07fd6d00137a0097a8c98164c31549318ce8be9191383e268f7a5d91","receipt":{"kind":"pith_receipt","key_id":"pith-v1-2026-05","algorithm":"ed25519","signed_at":"2026-07-05T09:59:16.124435Z","signature_b64":"ZqHFwGFdXTeIao48xidFGTSKhl4V3oZQuisRqS8dSBVfqvQIqcdNa6xfx/nbybtNxVkA8ki5rAHFoiDvjf+dDQ==","signed_message":"canonical_sha256_bytes","builder_version":"pith-number-builder-2026-05-17-v1","receipt_version":"0.3","canonical_sha256":"30e5f61f07fd6d00137a0097a8c98164c31549318ce8be9191383e268f7a5d91","last_reissued_at":"2026-07-05T09:59:16.123997Z","signature_status":"signed_v1","first_computed_at":"2026-07-05T09:59:16.123997Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"source_kind":"arxiv","source_id":"2406.04127","source_version":3,"attestation_state":"computed"},"signer":{"signer_id":"pith.science","signer_type":"pith_registry","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"created_at":"2026-07-05T09:59:16Z","supersedes":[],"prev_event":null,"signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"1tbzTCuUQZLauU4nhzxeFdz264C0KhkTAu1KI19hIeGaOZNjtbaJqE+rBTR/Q1M0gRhPkNCiZVnOpEUdaHP6Dg==","signed_message":"open_graph_event_sha256_bytes","signed_at":"2026-07-09T03:32:46.707363Z"},"content_sha256":"f4bd852f0fc84c0a482aaaf1cd284622a4c34db52f84c28885c5f11822cbbd2a","schema_version":"1.0","event_id":"sha256:f4bd852f0fc84c0a482aaaf1cd284622a4c34db52f84c28885c5f11822cbbd2a"},{"event_type":"graph_snapshot","subject_pith_number":"pith:2024:GDS7MHYH7VWQAE32ACL2RSMBMT","target":"graph","payload":{"graph_snapshot":{"paper":{"title":"Are We Done with MMLU?","license":"http://creativecommons.org/licenses/by/4.0/","headline":"","cross_cats":["cs.AI"],"primary_cat":"cs.CL","authors_text":"Alberto Carlo Maria Mancino, Alessio Devoto, Aryo Pradipta Gema, Claire Barale, Emile van Krieken, Giwon Hong, Jean Kaddour, Joshua Harris, Joshua Ong Jun Leang, Mohammad Reza Ghasemi Madani, Pasquale Minervini, Robert McHardy, Rohit Saxena, Xiaotang Du, Xuanli He, Yu Zhao","submitted_at":"2024-06-06T14:49:06Z","abstract_excerpt":"Maybe not. We identify and analyse errors in the popular Massive Multitask Language Understanding (MMLU) benchmark. Even though MMLU is widely adopted, our analysis demonstrates numerous ground truth errors that obscure the true capabilities of LLMs. For example, we find that 57% of the analysed questions in the Virology subset contain errors. To address this issue, we introduce a comprehensive framework for identifying dataset errors using a novel error annotation protocol. Then, we create MMLU-Redux, which is a subset of 5,700 manually re-annotated questions across all 57 MMLU subjects. We e"},"claims":{"count":0,"items":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"2406.04127","kind":"arxiv","version":3},"verdict":{"id":null,"model_set":{},"created_at":null,"strongest_claim":"","one_line_summary":"","pipeline_version":null,"weakest_assumption":"","pith_extraction_headline":""},"integrity":{"clean":true,"summary":{"advisory":0,"critical":0,"by_detector":{},"informational":0},"endpoint":"/pith/2406.04127/integrity.json","findings":[],"available":true,"detectors_run":[],"snapshot_sha256":"c28c3603d3b5d939e8dc4c7e95fa8dfce3d595e45f758748cecf8e644a296938"},"references":{"count":0,"sample":[],"resolved_work":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","internal_anchors":0},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"author_claims":{"count":0,"strong_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"builder_version":"pith-number-builder-2026-05-17-v1"},"verdict_id":null},"signer":{"signer_id":"pith.science","signer_type":"pith_registry","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"created_at":"2026-07-05T09:59:16Z","supersedes":[],"prev_event":null,"signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"euHvX0bGfmy0tMYX80ecB9wtJdE/cWrl2iJbFfuf50crhCq18bttezWVWl5YHlLYDSZtwPZwC7AP2F1vR2t3AA==","signed_message":"open_graph_event_sha256_bytes","signed_at":"2026-07-09T03:32:46.707744Z"},"content_sha256":"ab5d83feb80bd68a948f652b5e99b02b1dc4425048990497867e6b3a32bb9eea","schema_version":"1.0","event_id":"sha256:ab5d83feb80bd68a948f652b5e99b02b1dc4425048990497867e6b3a32bb9eea"}],"timestamp_proofs":[],"mirror_hints":[{"mirror_type":"https","name":"Pith Resolver","base_url":"https://pith.science","bundle_url":"https://pith.science/pith/GDS7MHYH7VWQAE32ACL2RSMBMT/bundle.json","state_url":"https://pith.science/pith/GDS7MHYH7VWQAE32ACL2RSMBMT/state.json","well_known_bundle_url":"https://pith.science/.well-known/pith/GDS7MHYH7VWQAE32ACL2RSMBMT/bundle.json","status":"primary"}],"public_keys":[{"key_id":"pith-v1-2026-05","algorithm":"ed25519","format":"raw","public_key_b64":"stVStoiQhXFxp4s2pdzPNoqVNBMojDU/fJ2db5S3CbM=","public_key_hex":"b2d552b68890857171a78b36a5dccf368a953413288c353f7c9d9d6f94b709b3","fingerprint_sha256_b32_first128bits":"RVFV5Z2OI2J3ZUO7ERDEBCYNKS","fingerprint_sha256_hex":"8d4b5ee74e4693bcd1df2446408b0d54","rotates_at":null,"url":"https://pith.science/pith-signing-key.json","notes":"Pith uses this Ed25519 key to sign canonical record SHA-256 digests. Verify with: ed25519_verify(public_key, message=canonical_sha256_bytes, signature=base64decode(signature_b64))."}],"merge_version":"pith-open-graph-merge-v1","built_at":"2026-07-09T03:32:46Z","links":{"resolver":"https://pith.science/pith/GDS7MHYH7VWQAE32ACL2RSMBMT","bundle":"https://pith.science/pith/GDS7MHYH7VWQAE32ACL2RSMBMT/bundle.json","state":"https://pith.science/pith/GDS7MHYH7VWQAE32ACL2RSMBMT/state.json","well_known_bundle":"https://pith.science/.well-known/pith/GDS7MHYH7VWQAE32ACL2RSMBMT/bundle.json"},"state":{"state_type":"pith_open_graph_state","state_version":"1.0","pith_number":"pith:2024:GDS7MHYH7VWQAE32ACL2RSMBMT","merge_version":"pith-open-graph-merge-v1","event_count":2,"valid_event_count":2,"invalid_event_count":0,"equivocation_count":0,"current":{"canonical_record":{"metadata":{"abstract_canon_sha256":"47d71667955c77b30f75cfbfdce825653d66e87629a2771eb9bca5345ad71f03","cross_cats_sorted":["cs.AI"],"license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.CL","submitted_at":"2024-06-06T14:49:06Z","title_canon_sha256":"943a1c434b5b4a487de0e9c3e56d98db7b929f9aa4058d84e6e153a7237e5d4c"},"schema_version":"1.0","source":{"id":"2406.04127","kind":"arxiv","version":3}},"source_aliases":[{"alias_kind":"arxiv","alias_value":"2406.04127","created_at":"2026-07-05T09:59:16Z"},{"alias_kind":"arxiv_version","alias_value":"2406.04127v3","created_at":"2026-07-05T09:59:16Z"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.2406.04127","created_at":"2026-07-05T09:59:16Z"},{"alias_kind":"pith_short_12","alias_value":"GDS7MHYH7VWQ","created_at":"2026-07-05T09:59:16Z"},{"alias_kind":"pith_short_16","alias_value":"GDS7MHYH7VWQAE32","created_at":"2026-07-05T09:59:16Z"},{"alias_kind":"pith_short_8","alias_value":"GDS7MHYH","created_at":"2026-07-05T09:59:16Z"}],"graph_snapshots":[{"event_id":"sha256:ab5d83feb80bd68a948f652b5e99b02b1dc4425048990497867e6b3a32bb9eea","target":"graph","created_at":"2026-07-05T09:59:16Z","signer":{"key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signer_id":"pith.science","signer_type":"pith_registry"},"payload":{"graph_snapshot":{"author_claims":{"count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","strong_count":0},"builder_version":"pith-number-builder-2026-05-17-v1","claims":{"count":0,"items":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"integrity":{"available":true,"clean":true,"detectors_run":[],"endpoint":"/pith/2406.04127/integrity.json","findings":[],"snapshot_sha256":"c28c3603d3b5d939e8dc4c7e95fa8dfce3d595e45f758748cecf8e644a296938","summary":{"advisory":0,"by_detector":{},"critical":0,"informational":0}},"paper":{"abstract_excerpt":"Maybe not. We identify and analyse errors in the popular Massive Multitask Language Understanding (MMLU) benchmark. Even though MMLU is widely adopted, our analysis demonstrates numerous ground truth errors that obscure the true capabilities of LLMs. For example, we find that 57% of the analysed questions in the Virology subset contain errors. To address this issue, we introduce a comprehensive framework for identifying dataset errors using a novel error annotation protocol. Then, we create MMLU-Redux, which is a subset of 5,700 manually re-annotated questions across all 57 MMLU subjects. We e","authors_text":"Alberto Carlo Maria Mancino, Alessio Devoto, Aryo Pradipta Gema, Claire Barale, Emile van Krieken, Giwon Hong, Jean Kaddour, Joshua Harris, Joshua Ong Jun Leang, Mohammad Reza Ghasemi Madani, Pasquale Minervini, Robert McHardy, Rohit Saxena, Xiaotang Du, Xuanli He, Yu Zhao","cross_cats":["cs.AI"],"headline":"","license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.CL","submitted_at":"2024-06-06T14:49:06Z","title":"Are We Done with MMLU?"},"references":{"count":0,"internal_anchors":0,"resolved_work":0,"sample":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"2406.04127","kind":"arxiv","version":3},"verdict":{"created_at":null,"id":null,"model_set":{},"one_line_summary":"","pipeline_version":null,"pith_extraction_headline":"","strongest_claim":"","weakest_assumption":""}},"verdict_id":null}}],"author_attestations":[],"timestamp_anchors":[],"storage_attestations":[],"citation_signatures":[],"replication_records":[],"corrections":[],"mirror_hints":[],"record_created":{"event_id":"sha256:f4bd852f0fc84c0a482aaaf1cd284622a4c34db52f84c28885c5f11822cbbd2a","target":"record","created_at":"2026-07-05T09:59:16Z","signer":{"key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signer_id":"pith.science","signer_type":"pith_registry"},"payload":{"attestation_state":"computed","canonical_record":{"metadata":{"abstract_canon_sha256":"47d71667955c77b30f75cfbfdce825653d66e87629a2771eb9bca5345ad71f03","cross_cats_sorted":["cs.AI"],"license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.CL","submitted_at":"2024-06-06T14:49:06Z","title_canon_sha256":"943a1c434b5b4a487de0e9c3e56d98db7b929f9aa4058d84e6e153a7237e5d4c"},"schema_version":"1.0","source":{"id":"2406.04127","kind":"arxiv","version":3}},"canonical_sha256":"30e5f61f07fd6d00137a0097a8c98164c31549318ce8be9191383e268f7a5d91","receipt":{"algorithm":"ed25519","builder_version":"pith-number-builder-2026-05-17-v1","canonical_sha256":"30e5f61f07fd6d00137a0097a8c98164c31549318ce8be9191383e268f7a5d91","first_computed_at":"2026-07-05T09:59:16.123997Z","key_id":"pith-v1-2026-05","kind":"pith_receipt","last_reissued_at":"2026-07-05T09:59:16.123997Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","receipt_version":"0.3","signature_b64":"ZqHFwGFdXTeIao48xidFGTSKhl4V3oZQuisRqS8dSBVfqvQIqcdNa6xfx/nbybtNxVkA8ki5rAHFoiDvjf+dDQ==","signature_status":"signed_v1","signed_at":"2026-07-05T09:59:16.124435Z","signed_message":"canonical_sha256_bytes"},"source_id":"2406.04127","source_kind":"arxiv","source_version":3}}},"equivocations":[],"invalid_events":[],"applied_event_ids":["sha256:f4bd852f0fc84c0a482aaaf1cd284622a4c34db52f84c28885c5f11822cbbd2a","sha256:ab5d83feb80bd68a948f652b5e99b02b1dc4425048990497867e6b3a32bb9eea"],"state_sha256":"07454c630dea94ce6fa87ff41ffcfe2ec03c4782999543322411c2a075791394"},"bundle_signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"rvICk13ajdxNaha+R6mFr/znZcvJvaZcTYk0T/WMU/2K8P18yvNw47egTBLyznmoDWPPGa2MSJ1t8s82FSOpAg==","signed_message":"bundle_sha256_bytes","signed_at":"2026-07-09T03:32:46.709870Z","bundle_sha256":"55a06c8cf694653ec50f18abea69d62493f1794ac4677ca39893fa630d022648"}}