{"bundle_type":"pith_open_graph_bundle","bundle_version":"1.0","pith_number":"pith:2024:KGGC6CAI7QYYUBZA6XWZIIFR2J","short_pith_number":"pith:KGGC6CAI","canonical_record":{"source":{"id":"2406.19470","kind":"arxiv","version":2},"metadata":{"license":"http://creativecommons.org/licenses/by-sa/4.0/","primary_cat":"cs.CL","submitted_at":"2024-06-27T18:21:32Z","cross_cats_sorted":[],"title_canon_sha256":"5e5fc3bacdded614758f3267dedd099b6cab704301ee440fb42c2f87f5b8be0a","abstract_canon_sha256":"082585537ee56c6910c94c4463fbf79d72ed991ac3b0f92d142da82456c972b2"},"schema_version":"1.0"},"canonical_sha256":"518c2f0808fc318a0720f5ed9420b1d243c8198315f4547175daa1b35a4ac79c","source":{"kind":"arxiv","id":"2406.19470","version":2},"source_aliases":[{"alias_kind":"arxiv","alias_value":"2406.19470","created_at":"2026-07-05T09:33:34Z"},{"alias_kind":"arxiv_version","alias_value":"2406.19470v2","created_at":"2026-07-05T09:33:34Z"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.2406.19470","created_at":"2026-07-05T09:33:34Z"},{"alias_kind":"pith_short_12","alias_value":"KGGC6CAI7QYY","created_at":"2026-07-05T09:33:34Z"},{"alias_kind":"pith_short_16","alias_value":"KGGC6CAI7QYYUBZA","created_at":"2026-07-05T09:33:34Z"},{"alias_kind":"pith_short_8","alias_value":"KGGC6CAI","created_at":"2026-07-05T09:33:34Z"}],"events":[{"event_type":"record_created","subject_pith_number":"pith:2024:KGGC6CAI7QYYUBZA6XWZIIFR2J","target":"record","payload":{"canonical_record":{"source":{"id":"2406.19470","kind":"arxiv","version":2},"metadata":{"license":"http://creativecommons.org/licenses/by-sa/4.0/","primary_cat":"cs.CL","submitted_at":"2024-06-27T18:21:32Z","cross_cats_sorted":[],"title_canon_sha256":"5e5fc3bacdded614758f3267dedd099b6cab704301ee440fb42c2f87f5b8be0a","abstract_canon_sha256":"082585537ee56c6910c94c4463fbf79d72ed991ac3b0f92d142da82456c972b2"},"schema_version":"1.0"},"canonical_sha256":"518c2f0808fc318a0720f5ed9420b1d243c8198315f4547175daa1b35a4ac79c","receipt":{"kind":"pith_receipt","key_id":"pith-v1-2026-05","algorithm":"ed25519","signed_at":"2026-07-05T09:33:34.161702Z","signature_b64":"MRVQKyar4eextE7sqlnN2U6zjo2AADgNT6hEhrNbr8ck55/Ef2AoczrZ8jMLDMMxjitiAanRzyeZ438nUpw8Bg==","signed_message":"canonical_sha256_bytes","builder_version":"pith-number-builder-2026-05-17-v1","receipt_version":"0.3","canonical_sha256":"518c2f0808fc318a0720f5ed9420b1d243c8198315f4547175daa1b35a4ac79c","last_reissued_at":"2026-07-05T09:33:34.161203Z","signature_status":"signed_v1","first_computed_at":"2026-07-05T09:33:34.161203Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"source_kind":"arxiv","source_id":"2406.19470","source_version":2,"attestation_state":"computed"},"signer":{"signer_id":"pith.science","signer_type":"pith_registry","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"created_at":"2026-07-05T09:33:34Z","supersedes":[],"prev_event":null,"signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"eJoBwwLdaGCgzeky7KCa9HfIkn91aKXpbzmeFY4BJKafc+sY0vlht1BWKq5iBanW/aCVmu82MoZ4CyWZA+4YDA==","signed_message":"open_graph_event_sha256_bytes","signed_at":"2026-08-09T11:18:16.418771Z"},"content_sha256":"64e3250f9c7072180a6a12d5bf9ff727572184fbd61e889b7b103ad5c5d9e5bf","schema_version":"1.0","event_id":"sha256:64e3250f9c7072180a6a12d5bf9ff727572184fbd61e889b7b103ad5c5d9e5bf"},{"event_type":"graph_snapshot","subject_pith_number":"pith:2024:KGGC6CAI7QYYUBZA6XWZIIFR2J","target":"graph","payload":{"graph_snapshot":{"paper":{"title":"Changing Answer Order Can Decrease MMLU Accuracy","license":"http://creativecommons.org/licenses/by-sa/4.0/","headline":"","cross_cats":[],"primary_cat":"cs.CL","authors_text":"Adina Williams, Candace Ross, David Pantoja, Megan Ung, Vipul Gupta","submitted_at":"2024-06-27T18:21:32Z","abstract_excerpt":"As large language models (LLMs) have grown in prevalence, particular benchmarks have become essential for the evaluation of these models and for understanding model capabilities. Most commonly, we use test accuracy averaged across multiple subtasks in order to rank models on leaderboards, to determine which model is best for our purposes. In this paper, we investigate the robustness of the accuracy measurement on a widely used multiple choice question answering dataset, MMLU. When shuffling the answer label contents, we find that all explored models decrease in accuracy on MMLU, but not every "},"claims":{"count":0,"items":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"2406.19470","kind":"arxiv","version":2},"verdict":{"id":null,"model_set":{},"created_at":null,"strongest_claim":"","one_line_summary":"","pipeline_version":null,"weakest_assumption":"","pith_extraction_headline":""},"integrity":{"clean":true,"summary":{"advisory":0,"critical":0,"by_detector":{},"informational":0},"endpoint":"/pith/2406.19470/integrity.json","findings":[],"available":true,"detectors_run":[],"snapshot_sha256":"c28c3603d3b5d939e8dc4c7e95fa8dfce3d595e45f758748cecf8e644a296938"},"references":{"count":0,"sample":[],"resolved_work":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","internal_anchors":0},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"author_claims":{"count":0,"strong_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"builder_version":"pith-number-builder-2026-05-17-v1"},"verdict_id":null},"signer":{"signer_id":"pith.science","signer_type":"pith_registry","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"created_at":"2026-07-05T09:33:34Z","supersedes":[],"prev_event":null,"signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"WE/PAxaOQVpWalhaBR71hK4fVDjSb4y5K3fosY4q/K2i/0Qhj+xdeEnfJ1G49oqpmUJYv/8w+oU0OsqpuKJmBA==","signed_message":"open_graph_event_sha256_bytes","signed_at":"2026-08-09T11:18:16.419327Z"},"content_sha256":"352f0e17eeb62707b2cf3271f71e47a00781c1d4d1167a1d3561edb8e9691a62","schema_version":"1.0","event_id":"sha256:352f0e17eeb62707b2cf3271f71e47a00781c1d4d1167a1d3561edb8e9691a62"}],"timestamp_proofs":[],"mirror_hints":[{"mirror_type":"https","name":"Pith Resolver","base_url":"https://pith.science","bundle_url":"https://pith.science/pith/KGGC6CAI7QYYUBZA6XWZIIFR2J/bundle.json","state_url":"https://pith.science/pith/KGGC6CAI7QYYUBZA6XWZIIFR2J/state.json","well_known_bundle_url":"https://pith.science/.well-known/pith/KGGC6CAI7QYYUBZA6XWZIIFR2J/bundle.json","status":"primary"}],"public_keys":[{"key_id":"pith-v1-2026-05","algorithm":"ed25519","format":"raw","public_key_b64":"stVStoiQhXFxp4s2pdzPNoqVNBMojDU/fJ2db5S3CbM=","public_key_hex":"b2d552b68890857171a78b36a5dccf368a953413288c353f7c9d9d6f94b709b3","fingerprint_sha256_b32_first128bits":"RVFV5Z2OI2J3ZUO7ERDEBCYNKS","fingerprint_sha256_hex":"8d4b5ee74e4693bcd1df2446408b0d54","rotates_at":null,"url":"https://pith.science/pith-signing-key.json","notes":"Pith uses this Ed25519 key to sign canonical record SHA-256 digests. Verify with: ed25519_verify(public_key, message=canonical_sha256_bytes, signature=base64decode(signature_b64))."}],"merge_version":"pith-open-graph-merge-v1","built_at":"2026-08-09T11:18:16Z","links":{"resolver":"https://pith.science/pith/KGGC6CAI7QYYUBZA6XWZIIFR2J","bundle":"https://pith.science/pith/KGGC6CAI7QYYUBZA6XWZIIFR2J/bundle.json","state":"https://pith.science/pith/KGGC6CAI7QYYUBZA6XWZIIFR2J/state.json","well_known_bundle":"https://pith.science/.well-known/pith/KGGC6CAI7QYYUBZA6XWZIIFR2J/bundle.json"},"state":{"state_type":"pith_open_graph_state","state_version":"1.0","pith_number":"pith:2024:KGGC6CAI7QYYUBZA6XWZIIFR2J","merge_version":"pith-open-graph-merge-v1","event_count":2,"valid_event_count":2,"invalid_event_count":0,"equivocation_count":0,"current":{"canonical_record":{"metadata":{"abstract_canon_sha256":"082585537ee56c6910c94c4463fbf79d72ed991ac3b0f92d142da82456c972b2","cross_cats_sorted":[],"license":"http://creativecommons.org/licenses/by-sa/4.0/","primary_cat":"cs.CL","submitted_at":"2024-06-27T18:21:32Z","title_canon_sha256":"5e5fc3bacdded614758f3267dedd099b6cab704301ee440fb42c2f87f5b8be0a"},"schema_version":"1.0","source":{"id":"2406.19470","kind":"arxiv","version":2}},"source_aliases":[{"alias_kind":"arxiv","alias_value":"2406.19470","created_at":"2026-07-05T09:33:34Z"},{"alias_kind":"arxiv_version","alias_value":"2406.19470v2","created_at":"2026-07-05T09:33:34Z"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.2406.19470","created_at":"2026-07-05T09:33:34Z"},{"alias_kind":"pith_short_12","alias_value":"KGGC6CAI7QYY","created_at":"2026-07-05T09:33:34Z"},{"alias_kind":"pith_short_16","alias_value":"KGGC6CAI7QYYUBZA","created_at":"2026-07-05T09:33:34Z"},{"alias_kind":"pith_short_8","alias_value":"KGGC6CAI","created_at":"2026-07-05T09:33:34Z"}],"graph_snapshots":[{"event_id":"sha256:352f0e17eeb62707b2cf3271f71e47a00781c1d4d1167a1d3561edb8e9691a62","target":"graph","created_at":"2026-07-05T09:33:34Z","signer":{"key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signer_id":"pith.science","signer_type":"pith_registry"},"payload":{"graph_snapshot":{"author_claims":{"count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","strong_count":0},"builder_version":"pith-number-builder-2026-05-17-v1","claims":{"count":0,"items":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"integrity":{"available":true,"clean":true,"detectors_run":[],"endpoint":"/pith/2406.19470/integrity.json","findings":[],"snapshot_sha256":"c28c3603d3b5d939e8dc4c7e95fa8dfce3d595e45f758748cecf8e644a296938","summary":{"advisory":0,"by_detector":{},"critical":0,"informational":0}},"paper":{"abstract_excerpt":"As large language models (LLMs) have grown in prevalence, particular benchmarks have become essential for the evaluation of these models and for understanding model capabilities. Most commonly, we use test accuracy averaged across multiple subtasks in order to rank models on leaderboards, to determine which model is best for our purposes. In this paper, we investigate the robustness of the accuracy measurement on a widely used multiple choice question answering dataset, MMLU. When shuffling the answer label contents, we find that all explored models decrease in accuracy on MMLU, but not every ","authors_text":"Adina Williams, Candace Ross, David Pantoja, Megan Ung, Vipul Gupta","cross_cats":[],"headline":"","license":"http://creativecommons.org/licenses/by-sa/4.0/","primary_cat":"cs.CL","submitted_at":"2024-06-27T18:21:32Z","title":"Changing Answer Order Can Decrease MMLU Accuracy"},"references":{"count":0,"internal_anchors":0,"resolved_work":0,"sample":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"2406.19470","kind":"arxiv","version":2},"verdict":{"created_at":null,"id":null,"model_set":{},"one_line_summary":"","pipeline_version":null,"pith_extraction_headline":"","strongest_claim":"","weakest_assumption":""}},"verdict_id":null}}],"author_attestations":[],"timestamp_anchors":[],"storage_attestations":[],"citation_signatures":[],"replication_records":[],"corrections":[],"mirror_hints":[],"record_created":{"event_id":"sha256:64e3250f9c7072180a6a12d5bf9ff727572184fbd61e889b7b103ad5c5d9e5bf","target":"record","created_at":"2026-07-05T09:33:34Z","signer":{"key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signer_id":"pith.science","signer_type":"pith_registry"},"payload":{"attestation_state":"computed","canonical_record":{"metadata":{"abstract_canon_sha256":"082585537ee56c6910c94c4463fbf79d72ed991ac3b0f92d142da82456c972b2","cross_cats_sorted":[],"license":"http://creativecommons.org/licenses/by-sa/4.0/","primary_cat":"cs.CL","submitted_at":"2024-06-27T18:21:32Z","title_canon_sha256":"5e5fc3bacdded614758f3267dedd099b6cab704301ee440fb42c2f87f5b8be0a"},"schema_version":"1.0","source":{"id":"2406.19470","kind":"arxiv","version":2}},"canonical_sha256":"518c2f0808fc318a0720f5ed9420b1d243c8198315f4547175daa1b35a4ac79c","receipt":{"algorithm":"ed25519","builder_version":"pith-number-builder-2026-05-17-v1","canonical_sha256":"518c2f0808fc318a0720f5ed9420b1d243c8198315f4547175daa1b35a4ac79c","first_computed_at":"2026-07-05T09:33:34.161203Z","key_id":"pith-v1-2026-05","kind":"pith_receipt","last_reissued_at":"2026-07-05T09:33:34.161203Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","receipt_version":"0.3","signature_b64":"MRVQKyar4eextE7sqlnN2U6zjo2AADgNT6hEhrNbr8ck55/Ef2AoczrZ8jMLDMMxjitiAanRzyeZ438nUpw8Bg==","signature_status":"signed_v1","signed_at":"2026-07-05T09:33:34.161702Z","signed_message":"canonical_sha256_bytes"},"source_id":"2406.19470","source_kind":"arxiv","source_version":2}}},"equivocations":[],"invalid_events":[],"applied_event_ids":["sha256:64e3250f9c7072180a6a12d5bf9ff727572184fbd61e889b7b103ad5c5d9e5bf","sha256:352f0e17eeb62707b2cf3271f71e47a00781c1d4d1167a1d3561edb8e9691a62"],"state_sha256":"d9bbe47148aae4fb7fa4054d3fbbce1c8c8a774b2419ca87ec9d7d47106b7e4e"},"bundle_signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"Pri5RLz/7Fe0SUzzEC9b13FeS/lve8zcqU9IDM0sAVZVQZpHJ4v+zFirCAuaPfMAxSvNrOyd4jeijHo1iob/CQ==","signed_message":"bundle_sha256_bytes","signed_at":"2026-08-09T11:18:16.424321Z","bundle_sha256":"4b0600e0a9b8e79b89a53146e60ff123bce0bdf0acb247375fa43bb45a1e38bc"}}