{"bundle_type":"pith_open_graph_bundle","bundle_version":"1.0","pith_number":"pith:2024:MPUKCXH5EBVKFNZJJCPMS556MZ","short_pith_number":"pith:MPUKCXH5","canonical_record":{"source":{"id":"2412.15194","kind":"arxiv","version":1},"metadata":{"license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","primary_cat":"cs.CL","submitted_at":"2024-12-19T18:58:04Z","cross_cats_sorted":["cs.AI","cs.LG","cs.PF"],"title_canon_sha256":"253427a712c66424579a51b7c0de2137b6eb8d06ce0259d84a5b3d5f74e8c2ba","abstract_canon_sha256":"c41233d72282e5e7bf8289eebb291c380b2639644c12fc66bee29537518e4293"},"schema_version":"1.0"},"canonical_sha256":"63e8a15cfd206aa2b729489ec977be667a1222e07a30ad7ccc7b4577dd602e6d","source":{"kind":"arxiv","id":"2412.15194","version":1},"source_aliases":[{"alias_kind":"arxiv","alias_value":"2412.15194","created_at":"2026-07-05T11:27:52Z"},{"alias_kind":"arxiv_version","alias_value":"2412.15194v1","created_at":"2026-07-05T11:27:52Z"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.2412.15194","created_at":"2026-07-05T11:27:52Z"},{"alias_kind":"pith_short_12","alias_value":"MPUKCXH5EBVK","created_at":"2026-07-05T11:27:52Z"},{"alias_kind":"pith_short_16","alias_value":"MPUKCXH5EBVKFNZJ","created_at":"2026-07-05T11:27:52Z"},{"alias_kind":"pith_short_8","alias_value":"MPUKCXH5","created_at":"2026-07-05T11:27:52Z"}],"events":[{"event_type":"record_created","subject_pith_number":"pith:2024:MPUKCXH5EBVKFNZJJCPMS556MZ","target":"record","payload":{"canonical_record":{"source":{"id":"2412.15194","kind":"arxiv","version":1},"metadata":{"license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","primary_cat":"cs.CL","submitted_at":"2024-12-19T18:58:04Z","cross_cats_sorted":["cs.AI","cs.LG","cs.PF"],"title_canon_sha256":"253427a712c66424579a51b7c0de2137b6eb8d06ce0259d84a5b3d5f74e8c2ba","abstract_canon_sha256":"c41233d72282e5e7bf8289eebb291c380b2639644c12fc66bee29537518e4293"},"schema_version":"1.0"},"canonical_sha256":"63e8a15cfd206aa2b729489ec977be667a1222e07a30ad7ccc7b4577dd602e6d","receipt":{"kind":"pith_receipt","key_id":"pith-v1-2026-05","algorithm":"ed25519","signed_at":"2026-07-05T11:27:52.223460Z","signature_b64":"Rp3gocnE2K/YuDeDh6loVndfUEDNLY9olzxEZWsPt498qC4gC4yvmKQKEqKDn8jYLOHPmcEtP3yBd3x352lSBw==","signed_message":"canonical_sha256_bytes","builder_version":"pith-number-builder-2026-05-17-v1","receipt_version":"0.3","canonical_sha256":"63e8a15cfd206aa2b729489ec977be667a1222e07a30ad7ccc7b4577dd602e6d","last_reissued_at":"2026-07-05T11:27:52.222946Z","signature_status":"signed_v1","first_computed_at":"2026-07-05T11:27:52.222946Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"source_kind":"arxiv","source_id":"2412.15194","source_version":1,"attestation_state":"computed"},"signer":{"signer_id":"pith.science","signer_type":"pith_registry","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"created_at":"2026-07-05T11:27:52Z","supersedes":[],"prev_event":null,"signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"lu/nt7ETG3/7AEv4R5N9lU+8qgWiqEtWajKhnFlpAAw9fLFi2ozn8NFLBcEUjw4oCVB0D+SvO9c27nlMLuYiDQ==","signed_message":"open_graph_event_sha256_bytes","signed_at":"2026-08-08T15:46:39.895193Z"},"content_sha256":"632d6dc7451dfbac2c47af79b78fa31779c86fd566f730581ee5d8949ef6c319","schema_version":"1.0","event_id":"sha256:632d6dc7451dfbac2c47af79b78fa31779c86fd566f730581ee5d8949ef6c319"},{"event_type":"graph_snapshot","subject_pith_number":"pith:2024:MPUKCXH5EBVKFNZJJCPMS556MZ","target":"graph","payload":{"graph_snapshot":{"paper":{"title":"MMLU-CF: A Contamination-free Multi-task Language Understanding Benchmark","license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","headline":"","cross_cats":["cs.AI","cs.LG","cs.PF"],"primary_cat":"cs.CL","authors_text":"Furu Wei, Lei Cui, Qihao Zhao, Qinzheng Sun, Qiufeng Yin, Scarlett Li, Shaoguang Mao, Tengchao Lv, Xin Zhang, Yangyu Huang, Ying Xin","submitted_at":"2024-12-19T18:58:04Z","abstract_excerpt":"Multiple-choice question (MCQ) datasets like Massive Multitask Language Understanding (MMLU) are widely used to evaluate the commonsense, understanding, and problem-solving abilities of large language models (LLMs). However, the open-source nature of these benchmarks and the broad sources of training data for LLMs have inevitably led to benchmark contamination, resulting in unreliable evaluation results. To alleviate this issue, we propose a contamination-free and more challenging MCQ benchmark called MMLU-CF. This benchmark reassesses LLMs' understanding of world knowledge by averting both un"},"claims":{"count":0,"items":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"2412.15194","kind":"arxiv","version":1},"verdict":{"id":null,"model_set":{},"created_at":null,"strongest_claim":"","one_line_summary":"","pipeline_version":null,"weakest_assumption":"","pith_extraction_headline":""},"integrity":{"clean":true,"summary":{"advisory":0,"critical":0,"by_detector":{},"informational":0},"endpoint":"/pith/2412.15194/integrity.json","findings":[],"available":true,"detectors_run":[],"snapshot_sha256":"c28c3603d3b5d939e8dc4c7e95fa8dfce3d595e45f758748cecf8e644a296938"},"references":{"count":0,"sample":[],"resolved_work":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","internal_anchors":0},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"author_claims":{"count":0,"strong_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"builder_version":"pith-number-builder-2026-05-17-v1"},"verdict_id":null},"signer":{"signer_id":"pith.science","signer_type":"pith_registry","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"created_at":"2026-07-05T11:27:52Z","supersedes":[],"prev_event":null,"signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"T9W97xId28REDR4uQy+h95WtTPXajs08LegvqaaoXZo9MfCXrAczsFWcQPryzRpmf/uETBMSwouVfNu/yr6HCg==","signed_message":"open_graph_event_sha256_bytes","signed_at":"2026-08-08T15:46:39.896257Z"},"content_sha256":"ab11f2d585e40ff3db6d3ebf769add61634a71ecd5a9c680791d9fd1e2fd353f","schema_version":"1.0","event_id":"sha256:ab11f2d585e40ff3db6d3ebf769add61634a71ecd5a9c680791d9fd1e2fd353f"}],"timestamp_proofs":[],"mirror_hints":[{"mirror_type":"https","name":"Pith Resolver","base_url":"https://pith.science","bundle_url":"https://pith.science/pith/MPUKCXH5EBVKFNZJJCPMS556MZ/bundle.json","state_url":"https://pith.science/pith/MPUKCXH5EBVKFNZJJCPMS556MZ/state.json","well_known_bundle_url":"https://pith.science/.well-known/pith/MPUKCXH5EBVKFNZJJCPMS556MZ/bundle.json","status":"primary"}],"public_keys":[{"key_id":"pith-v1-2026-05","algorithm":"ed25519","format":"raw","public_key_b64":"stVStoiQhXFxp4s2pdzPNoqVNBMojDU/fJ2db5S3CbM=","public_key_hex":"b2d552b68890857171a78b36a5dccf368a953413288c353f7c9d9d6f94b709b3","fingerprint_sha256_b32_first128bits":"RVFV5Z2OI2J3ZUO7ERDEBCYNKS","fingerprint_sha256_hex":"8d4b5ee74e4693bcd1df2446408b0d54","rotates_at":null,"url":"https://pith.science/pith-signing-key.json","notes":"Pith uses this Ed25519 key to sign canonical record SHA-256 digests. Verify with: ed25519_verify(public_key, message=canonical_sha256_bytes, signature=base64decode(signature_b64))."}],"merge_version":"pith-open-graph-merge-v1","built_at":"2026-08-08T15:46:39Z","links":{"resolver":"https://pith.science/pith/MPUKCXH5EBVKFNZJJCPMS556MZ","bundle":"https://pith.science/pith/MPUKCXH5EBVKFNZJJCPMS556MZ/bundle.json","state":"https://pith.science/pith/MPUKCXH5EBVKFNZJJCPMS556MZ/state.json","well_known_bundle":"https://pith.science/.well-known/pith/MPUKCXH5EBVKFNZJJCPMS556MZ/bundle.json"},"state":{"state_type":"pith_open_graph_state","state_version":"1.0","pith_number":"pith:2024:MPUKCXH5EBVKFNZJJCPMS556MZ","merge_version":"pith-open-graph-merge-v1","event_count":2,"valid_event_count":2,"invalid_event_count":0,"equivocation_count":0,"current":{"canonical_record":{"metadata":{"abstract_canon_sha256":"c41233d72282e5e7bf8289eebb291c380b2639644c12fc66bee29537518e4293","cross_cats_sorted":["cs.AI","cs.LG","cs.PF"],"license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","primary_cat":"cs.CL","submitted_at":"2024-12-19T18:58:04Z","title_canon_sha256":"253427a712c66424579a51b7c0de2137b6eb8d06ce0259d84a5b3d5f74e8c2ba"},"schema_version":"1.0","source":{"id":"2412.15194","kind":"arxiv","version":1}},"source_aliases":[{"alias_kind":"arxiv","alias_value":"2412.15194","created_at":"2026-07-05T11:27:52Z"},{"alias_kind":"arxiv_version","alias_value":"2412.15194v1","created_at":"2026-07-05T11:27:52Z"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.2412.15194","created_at":"2026-07-05T11:27:52Z"},{"alias_kind":"pith_short_12","alias_value":"MPUKCXH5EBVK","created_at":"2026-07-05T11:27:52Z"},{"alias_kind":"pith_short_16","alias_value":"MPUKCXH5EBVKFNZJ","created_at":"2026-07-05T11:27:52Z"},{"alias_kind":"pith_short_8","alias_value":"MPUKCXH5","created_at":"2026-07-05T11:27:52Z"}],"graph_snapshots":[{"event_id":"sha256:ab11f2d585e40ff3db6d3ebf769add61634a71ecd5a9c680791d9fd1e2fd353f","target":"graph","created_at":"2026-07-05T11:27:52Z","signer":{"key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signer_id":"pith.science","signer_type":"pith_registry"},"payload":{"graph_snapshot":{"author_claims":{"count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","strong_count":0},"builder_version":"pith-number-builder-2026-05-17-v1","claims":{"count":0,"items":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"integrity":{"available":true,"clean":true,"detectors_run":[],"endpoint":"/pith/2412.15194/integrity.json","findings":[],"snapshot_sha256":"c28c3603d3b5d939e8dc4c7e95fa8dfce3d595e45f758748cecf8e644a296938","summary":{"advisory":0,"by_detector":{},"critical":0,"informational":0}},"paper":{"abstract_excerpt":"Multiple-choice question (MCQ) datasets like Massive Multitask Language Understanding (MMLU) are widely used to evaluate the commonsense, understanding, and problem-solving abilities of large language models (LLMs). However, the open-source nature of these benchmarks and the broad sources of training data for LLMs have inevitably led to benchmark contamination, resulting in unreliable evaluation results. To alleviate this issue, we propose a contamination-free and more challenging MCQ benchmark called MMLU-CF. This benchmark reassesses LLMs' understanding of world knowledge by averting both un","authors_text":"Furu Wei, Lei Cui, Qihao Zhao, Qinzheng Sun, Qiufeng Yin, Scarlett Li, Shaoguang Mao, Tengchao Lv, Xin Zhang, Yangyu Huang, Ying Xin","cross_cats":["cs.AI","cs.LG","cs.PF"],"headline":"","license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","primary_cat":"cs.CL","submitted_at":"2024-12-19T18:58:04Z","title":"MMLU-CF: A Contamination-free Multi-task Language Understanding Benchmark"},"references":{"count":0,"internal_anchors":0,"resolved_work":0,"sample":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"2412.15194","kind":"arxiv","version":1},"verdict":{"created_at":null,"id":null,"model_set":{},"one_line_summary":"","pipeline_version":null,"pith_extraction_headline":"","strongest_claim":"","weakest_assumption":""}},"verdict_id":null}}],"author_attestations":[],"timestamp_anchors":[],"storage_attestations":[],"citation_signatures":[],"replication_records":[],"corrections":[],"mirror_hints":[],"record_created":{"event_id":"sha256:632d6dc7451dfbac2c47af79b78fa31779c86fd566f730581ee5d8949ef6c319","target":"record","created_at":"2026-07-05T11:27:52Z","signer":{"key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signer_id":"pith.science","signer_type":"pith_registry"},"payload":{"attestation_state":"computed","canonical_record":{"metadata":{"abstract_canon_sha256":"c41233d72282e5e7bf8289eebb291c380b2639644c12fc66bee29537518e4293","cross_cats_sorted":["cs.AI","cs.LG","cs.PF"],"license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","primary_cat":"cs.CL","submitted_at":"2024-12-19T18:58:04Z","title_canon_sha256":"253427a712c66424579a51b7c0de2137b6eb8d06ce0259d84a5b3d5f74e8c2ba"},"schema_version":"1.0","source":{"id":"2412.15194","kind":"arxiv","version":1}},"canonical_sha256":"63e8a15cfd206aa2b729489ec977be667a1222e07a30ad7ccc7b4577dd602e6d","receipt":{"algorithm":"ed25519","builder_version":"pith-number-builder-2026-05-17-v1","canonical_sha256":"63e8a15cfd206aa2b729489ec977be667a1222e07a30ad7ccc7b4577dd602e6d","first_computed_at":"2026-07-05T11:27:52.222946Z","key_id":"pith-v1-2026-05","kind":"pith_receipt","last_reissued_at":"2026-07-05T11:27:52.222946Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","receipt_version":"0.3","signature_b64":"Rp3gocnE2K/YuDeDh6loVndfUEDNLY9olzxEZWsPt498qC4gC4yvmKQKEqKDn8jYLOHPmcEtP3yBd3x352lSBw==","signature_status":"signed_v1","signed_at":"2026-07-05T11:27:52.223460Z","signed_message":"canonical_sha256_bytes"},"source_id":"2412.15194","source_kind":"arxiv","source_version":1}}},"equivocations":[],"invalid_events":[],"applied_event_ids":["sha256:632d6dc7451dfbac2c47af79b78fa31779c86fd566f730581ee5d8949ef6c319","sha256:ab11f2d585e40ff3db6d3ebf769add61634a71ecd5a9c680791d9fd1e2fd353f"],"state_sha256":"83350c3d811d5a148163dec9e46171d67caf76004dfcfe56e8a6113c0d531b81"},"bundle_signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"d1X0JkwDwoX4YhVd4bbcKgS0ZsprHUKvTc2iJf9x/aYjfiESahla7wWmND3CkxkJ4uGvE9GfPUmkzxwJTnUwBw==","signed_message":"bundle_sha256_bytes","signed_at":"2026-08-08T15:46:39.902992Z","bundle_sha256":"85c457191a82a090634d4cffe2453ed89fdaf37af255655c083dca6058607e3f"}}