{"bundle_type":"pith_open_graph_bundle","bundle_version":"1.0","pith_number":"pith:2024:GYDNA7ZQN73T2SVKEJH4IMQMFI","short_pith_number":"pith:GYDNA7ZQ","canonical_record":{"source":{"id":"2402.11548","kind":"arxiv","version":2},"metadata":{"license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.CL","submitted_at":"2024-02-18T11:41:07Z","cross_cats_sorted":[],"title_canon_sha256":"ac304ff69de1cc85cfb0e5d5e175745d3543591d17b9d084b831f39e5b048238","abstract_canon_sha256":"db2e4b62a7a59525c43645227774c99d21cbc8abb364d2944dc17963f801d657"},"schema_version":"1.0"},"canonical_sha256":"3606d07f306ff73d4aaa224fc4320c2a044d357406a1d734650753f53f3bc5ad","source":{"kind":"arxiv","id":"2402.11548","version":2},"source_aliases":[{"alias_kind":"arxiv","alias_value":"2402.11548","created_at":"2026-07-05T08:28:06Z"},{"alias_kind":"arxiv_version","alias_value":"2402.11548v2","created_at":"2026-07-05T08:28:06Z"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.2402.11548","created_at":"2026-07-05T08:28:06Z"},{"alias_kind":"pith_short_12","alias_value":"GYDNA7ZQN73T","created_at":"2026-07-05T08:28:06Z"},{"alias_kind":"pith_short_16","alias_value":"GYDNA7ZQN73T2SVK","created_at":"2026-07-05T08:28:06Z"},{"alias_kind":"pith_short_8","alias_value":"GYDNA7ZQ","created_at":"2026-07-05T08:28:06Z"}],"events":[{"event_type":"record_created","subject_pith_number":"pith:2024:GYDNA7ZQN73T2SVKEJH4IMQMFI","target":"record","payload":{"canonical_record":{"source":{"id":"2402.11548","kind":"arxiv","version":2},"metadata":{"license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.CL","submitted_at":"2024-02-18T11:41:07Z","cross_cats_sorted":[],"title_canon_sha256":"ac304ff69de1cc85cfb0e5d5e175745d3543591d17b9d084b831f39e5b048238","abstract_canon_sha256":"db2e4b62a7a59525c43645227774c99d21cbc8abb364d2944dc17963f801d657"},"schema_version":"1.0"},"canonical_sha256":"3606d07f306ff73d4aaa224fc4320c2a044d357406a1d734650753f53f3bc5ad","receipt":{"kind":"pith_receipt","key_id":"pith-v1-2026-05","algorithm":"ed25519","signed_at":"2026-07-05T08:28:06.262837Z","signature_b64":"IcPGfHV7kbN8sT8VWWBqar4gdhjfczfp6+PLNAHXnj3v8lEGzxSGUiWxAZhEpBFwl/r5prwrIoJXBscf73r7CA==","signed_message":"canonical_sha256_bytes","builder_version":"pith-number-builder-2026-05-17-v1","receipt_version":"0.3","canonical_sha256":"3606d07f306ff73d4aaa224fc4320c2a044d357406a1d734650753f53f3bc5ad","last_reissued_at":"2026-07-05T08:28:06.262367Z","signature_status":"signed_v1","first_computed_at":"2026-07-05T08:28:06.262367Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"source_kind":"arxiv","source_id":"2402.11548","source_version":2,"attestation_state":"computed"},"signer":{"signer_id":"pith.science","signer_type":"pith_registry","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"created_at":"2026-07-05T08:28:06Z","supersedes":[],"prev_event":null,"signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"/SpRshbabK9GzyIQZviPXdAupC+k/ux/hRA9SHJSEm19kEvCoE1pFayBGMUoq3N2JvXBsQlAnBcXHtikvstrDw==","signed_message":"open_graph_event_sha256_bytes","signed_at":"2026-08-01T00:46:13.079765Z"},"content_sha256":"d432e5c2e971a22f216dab0fda83b6f200d7d9525e1b6aaf1426039c034cd2de","schema_version":"1.0","event_id":"sha256:d432e5c2e971a22f216dab0fda83b6f200d7d9525e1b6aaf1426039c034cd2de"},{"event_type":"graph_snapshot","subject_pith_number":"pith:2024:GYDNA7ZQN73T2SVKEJH4IMQMFI","target":"graph","payload":{"graph_snapshot":{"paper":{"title":"KMMLU: Measuring Massive Multitask Language Understanding in Korean","license":"http://creativecommons.org/licenses/by/4.0/","headline":"","cross_cats":[],"primary_cat":"cs.CL","authors_text":"Cheonbok Park, Guijin Son, Hanwool Lee, Kang Min Yoo, Niklas Muennighoff, Seungone Kim, Stella Biderman, Sungdong Kim, Taekyoon Choi","submitted_at":"2024-02-18T11:41:07Z","abstract_excerpt":"We propose KMMLU, a new Korean benchmark with 35,030 expert-level multiple-choice questions across 45 subjects ranging from humanities to STEM. While prior Korean benchmarks are translated from existing English benchmarks, KMMLU is collected from original Korean exams, capturing linguistic and cultural aspects of the Korean language. We test 27 public and proprietary LLMs and observe the best public model to score 50.5%, leaving significant room for improvement. This model was primarily trained for English and Chinese, not Korean. Current LLMs tailored to Korean, such as Polyglot-Ko, perform f"},"claims":{"count":0,"items":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"2402.11548","kind":"arxiv","version":2},"verdict":{"id":null,"model_set":{},"created_at":null,"strongest_claim":"","one_line_summary":"","pipeline_version":null,"weakest_assumption":"","pith_extraction_headline":""},"integrity":{"clean":true,"summary":{"advisory":0,"critical":0,"by_detector":{},"informational":0},"endpoint":"/pith/2402.11548/integrity.json","findings":[],"available":true,"detectors_run":[],"snapshot_sha256":"c28c3603d3b5d939e8dc4c7e95fa8dfce3d595e45f758748cecf8e644a296938"},"references":{"count":0,"sample":[],"resolved_work":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","internal_anchors":0},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"author_claims":{"count":0,"strong_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"builder_version":"pith-number-builder-2026-05-17-v1"},"verdict_id":null},"signer":{"signer_id":"pith.science","signer_type":"pith_registry","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"created_at":"2026-07-05T08:28:06Z","supersedes":[],"prev_event":null,"signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"JH0EKnX083gKMr5gFvA46VBZcQNfBFyYhqr83VoteIeyNOYvhZpQr42BRAaaKO8tukEha0bwtCvYO26iQuILBQ==","signed_message":"open_graph_event_sha256_bytes","signed_at":"2026-08-01T00:46:13.080068Z"},"content_sha256":"2e471ccafa8246dd642c4d89ab0a382ef92c6d7c7bc223fc7b61dde726b36334","schema_version":"1.0","event_id":"sha256:2e471ccafa8246dd642c4d89ab0a382ef92c6d7c7bc223fc7b61dde726b36334"}],"timestamp_proofs":[],"mirror_hints":[{"mirror_type":"https","name":"Pith Resolver","base_url":"https://pith.science","bundle_url":"https://pith.science/pith/GYDNA7ZQN73T2SVKEJH4IMQMFI/bundle.json","state_url":"https://pith.science/pith/GYDNA7ZQN73T2SVKEJH4IMQMFI/state.json","well_known_bundle_url":"https://pith.science/.well-known/pith/GYDNA7ZQN73T2SVKEJH4IMQMFI/bundle.json","status":"primary"}],"public_keys":[{"key_id":"pith-v1-2026-05","algorithm":"ed25519","format":"raw","public_key_b64":"stVStoiQhXFxp4s2pdzPNoqVNBMojDU/fJ2db5S3CbM=","public_key_hex":"b2d552b68890857171a78b36a5dccf368a953413288c353f7c9d9d6f94b709b3","fingerprint_sha256_b32_first128bits":"RVFV5Z2OI2J3ZUO7ERDEBCYNKS","fingerprint_sha256_hex":"8d4b5ee74e4693bcd1df2446408b0d54","rotates_at":null,"url":"https://pith.science/pith-signing-key.json","notes":"Pith uses this Ed25519 key to sign canonical record SHA-256 digests. Verify with: ed25519_verify(public_key, message=canonical_sha256_bytes, signature=base64decode(signature_b64))."}],"merge_version":"pith-open-graph-merge-v1","built_at":"2026-08-01T00:46:13Z","links":{"resolver":"https://pith.science/pith/GYDNA7ZQN73T2SVKEJH4IMQMFI","bundle":"https://pith.science/pith/GYDNA7ZQN73T2SVKEJH4IMQMFI/bundle.json","state":"https://pith.science/pith/GYDNA7ZQN73T2SVKEJH4IMQMFI/state.json","well_known_bundle":"https://pith.science/.well-known/pith/GYDNA7ZQN73T2SVKEJH4IMQMFI/bundle.json"},"state":{"state_type":"pith_open_graph_state","state_version":"1.0","pith_number":"pith:2024:GYDNA7ZQN73T2SVKEJH4IMQMFI","merge_version":"pith-open-graph-merge-v1","event_count":2,"valid_event_count":2,"invalid_event_count":0,"equivocation_count":0,"current":{"canonical_record":{"metadata":{"abstract_canon_sha256":"db2e4b62a7a59525c43645227774c99d21cbc8abb364d2944dc17963f801d657","cross_cats_sorted":[],"license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.CL","submitted_at":"2024-02-18T11:41:07Z","title_canon_sha256":"ac304ff69de1cc85cfb0e5d5e175745d3543591d17b9d084b831f39e5b048238"},"schema_version":"1.0","source":{"id":"2402.11548","kind":"arxiv","version":2}},"source_aliases":[{"alias_kind":"arxiv","alias_value":"2402.11548","created_at":"2026-07-05T08:28:06Z"},{"alias_kind":"arxiv_version","alias_value":"2402.11548v2","created_at":"2026-07-05T08:28:06Z"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.2402.11548","created_at":"2026-07-05T08:28:06Z"},{"alias_kind":"pith_short_12","alias_value":"GYDNA7ZQN73T","created_at":"2026-07-05T08:28:06Z"},{"alias_kind":"pith_short_16","alias_value":"GYDNA7ZQN73T2SVK","created_at":"2026-07-05T08:28:06Z"},{"alias_kind":"pith_short_8","alias_value":"GYDNA7ZQ","created_at":"2026-07-05T08:28:06Z"}],"graph_snapshots":[{"event_id":"sha256:2e471ccafa8246dd642c4d89ab0a382ef92c6d7c7bc223fc7b61dde726b36334","target":"graph","created_at":"2026-07-05T08:28:06Z","signer":{"key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signer_id":"pith.science","signer_type":"pith_registry"},"payload":{"graph_snapshot":{"author_claims":{"count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","strong_count":0},"builder_version":"pith-number-builder-2026-05-17-v1","claims":{"count":0,"items":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"integrity":{"available":true,"clean":true,"detectors_run":[],"endpoint":"/pith/2402.11548/integrity.json","findings":[],"snapshot_sha256":"c28c3603d3b5d939e8dc4c7e95fa8dfce3d595e45f758748cecf8e644a296938","summary":{"advisory":0,"by_detector":{},"critical":0,"informational":0}},"paper":{"abstract_excerpt":"We propose KMMLU, a new Korean benchmark with 35,030 expert-level multiple-choice questions across 45 subjects ranging from humanities to STEM. While prior Korean benchmarks are translated from existing English benchmarks, KMMLU is collected from original Korean exams, capturing linguistic and cultural aspects of the Korean language. We test 27 public and proprietary LLMs and observe the best public model to score 50.5%, leaving significant room for improvement. This model was primarily trained for English and Chinese, not Korean. Current LLMs tailored to Korean, such as Polyglot-Ko, perform f","authors_text":"Cheonbok Park, Guijin Son, Hanwool Lee, Kang Min Yoo, Niklas Muennighoff, Seungone Kim, Stella Biderman, Sungdong Kim, Taekyoon Choi","cross_cats":[],"headline":"","license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.CL","submitted_at":"2024-02-18T11:41:07Z","title":"KMMLU: Measuring Massive Multitask Language Understanding in Korean"},"references":{"count":0,"internal_anchors":0,"resolved_work":0,"sample":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"2402.11548","kind":"arxiv","version":2},"verdict":{"created_at":null,"id":null,"model_set":{},"one_line_summary":"","pipeline_version":null,"pith_extraction_headline":"","strongest_claim":"","weakest_assumption":""}},"verdict_id":null}}],"author_attestations":[],"timestamp_anchors":[],"storage_attestations":[],"citation_signatures":[],"replication_records":[],"corrections":[],"mirror_hints":[],"record_created":{"event_id":"sha256:d432e5c2e971a22f216dab0fda83b6f200d7d9525e1b6aaf1426039c034cd2de","target":"record","created_at":"2026-07-05T08:28:06Z","signer":{"key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signer_id":"pith.science","signer_type":"pith_registry"},"payload":{"attestation_state":"computed","canonical_record":{"metadata":{"abstract_canon_sha256":"db2e4b62a7a59525c43645227774c99d21cbc8abb364d2944dc17963f801d657","cross_cats_sorted":[],"license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.CL","submitted_at":"2024-02-18T11:41:07Z","title_canon_sha256":"ac304ff69de1cc85cfb0e5d5e175745d3543591d17b9d084b831f39e5b048238"},"schema_version":"1.0","source":{"id":"2402.11548","kind":"arxiv","version":2}},"canonical_sha256":"3606d07f306ff73d4aaa224fc4320c2a044d357406a1d734650753f53f3bc5ad","receipt":{"algorithm":"ed25519","builder_version":"pith-number-builder-2026-05-17-v1","canonical_sha256":"3606d07f306ff73d4aaa224fc4320c2a044d357406a1d734650753f53f3bc5ad","first_computed_at":"2026-07-05T08:28:06.262367Z","key_id":"pith-v1-2026-05","kind":"pith_receipt","last_reissued_at":"2026-07-05T08:28:06.262367Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","receipt_version":"0.3","signature_b64":"IcPGfHV7kbN8sT8VWWBqar4gdhjfczfp6+PLNAHXnj3v8lEGzxSGUiWxAZhEpBFwl/r5prwrIoJXBscf73r7CA==","signature_status":"signed_v1","signed_at":"2026-07-05T08:28:06.262837Z","signed_message":"canonical_sha256_bytes"},"source_id":"2402.11548","source_kind":"arxiv","source_version":2}}},"equivocations":[],"invalid_events":[],"applied_event_ids":["sha256:d432e5c2e971a22f216dab0fda83b6f200d7d9525e1b6aaf1426039c034cd2de","sha256:2e471ccafa8246dd642c4d89ab0a382ef92c6d7c7bc223fc7b61dde726b36334"],"state_sha256":"f56895759d2254e5ed737116ec014c695d338be84bbd51d6821563d4857e25d0"},"bundle_signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"R85MyAqfaFFbpFyeB9UqFtzTlMjI4HO0yWNk7C64dhoAMxqnQOlG1NQS1bb/vLxncIFOQZs72OvZ6HThfeXuCw==","signed_message":"bundle_sha256_bytes","signed_at":"2026-08-01T00:46:13.084025Z","bundle_sha256":"02bb13f172319215d62c8a8fd926d193cfb7f555433ef6c76bd0d2f278f506bc"}}