{"bundle_type":"pith_open_graph_bundle","bundle_version":"1.0","pith_number":"pith:2024:D76B7DJLIHDC3RGNKXTCBWD2Q4","short_pith_number":"pith:D76B7DJL","canonical_record":{"source":{"id":"2406.17789","kind":"arxiv","version":1},"metadata":{"license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.CL","submitted_at":"2024-05-28T11:13:40Z","cross_cats_sorted":["cs.AI"],"title_canon_sha256":"366d64fa5c9cb6f733d96f5c996b48290b0527aba57935cfee5a7f44c7159aba","abstract_canon_sha256":"756e423170f5e1f5070f5aa71bf7eb40b69ea4ee756192ac50ad320aa71d8f32"},"schema_version":"1.0"},"canonical_sha256":"1ffc1f8d2b41c62dc4cd55e620d87a8736ffa554641e107d31e2f2f808b586b3","source":{"kind":"arxiv","id":"2406.17789","version":1},"source_aliases":[{"alias_kind":"arxiv","alias_value":"2406.17789","created_at":"2026-07-05T08:36:49Z"},{"alias_kind":"arxiv_version","alias_value":"2406.17789v1","created_at":"2026-07-05T08:36:49Z"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.2406.17789","created_at":"2026-07-05T08:36:49Z"},{"alias_kind":"pith_short_12","alias_value":"D76B7DJLIHDC","created_at":"2026-07-05T08:36:49Z"},{"alias_kind":"pith_short_16","alias_value":"D76B7DJLIHDC3RGN","created_at":"2026-07-05T08:36:49Z"},{"alias_kind":"pith_short_8","alias_value":"D76B7DJL","created_at":"2026-07-05T08:36:49Z"}],"events":[{"event_type":"record_created","subject_pith_number":"pith:2024:D76B7DJLIHDC3RGNKXTCBWD2Q4","target":"record","payload":{"canonical_record":{"source":{"id":"2406.17789","kind":"arxiv","version":1},"metadata":{"license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.CL","submitted_at":"2024-05-28T11:13:40Z","cross_cats_sorted":["cs.AI"],"title_canon_sha256":"366d64fa5c9cb6f733d96f5c996b48290b0527aba57935cfee5a7f44c7159aba","abstract_canon_sha256":"756e423170f5e1f5070f5aa71bf7eb40b69ea4ee756192ac50ad320aa71d8f32"},"schema_version":"1.0"},"canonical_sha256":"1ffc1f8d2b41c62dc4cd55e620d87a8736ffa554641e107d31e2f2f808b586b3","receipt":{"kind":"pith_receipt","key_id":"pith-v1-2026-05","algorithm":"ed25519","signed_at":"2026-07-05T08:36:49.846810Z","signature_b64":"P3muh9CrhbvNoyJoLOllNp3ZD9JiVEKnNygtwKuyKHvljBr9pM1hsCGv4OW/fBhP/T1czVEo/7yftxfTK1oQCA==","signed_message":"canonical_sha256_bytes","builder_version":"pith-number-builder-2026-05-17-v1","receipt_version":"0.3","canonical_sha256":"1ffc1f8d2b41c62dc4cd55e620d87a8736ffa554641e107d31e2f2f808b586b3","last_reissued_at":"2026-07-05T08:36:49.846241Z","signature_status":"signed_v1","first_computed_at":"2026-07-05T08:36:49.846241Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"source_kind":"arxiv","source_id":"2406.17789","source_version":1,"attestation_state":"computed"},"signer":{"signer_id":"pith.science","signer_type":"pith_registry","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"created_at":"2026-07-05T08:36:49Z","supersedes":[],"prev_event":null,"signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"KeZwkhZ6Rhs12VAiyrlfTCPLK5EC3UuOJh+KhUoW1rkZmmA51lI3jR9I3qxB8Sgd2bbkjcGMiROoy8hQDH4uAw==","signed_message":"open_graph_event_sha256_bytes","signed_at":"2026-07-07T10:21:20.108152Z"},"content_sha256":"08fb5b96c870b5d65d3c51d1ddd102b5fab1484b38e2d819e7e848bb275c430e","schema_version":"1.0","event_id":"sha256:08fb5b96c870b5d65d3c51d1ddd102b5fab1484b38e2d819e7e848bb275c430e"},{"event_type":"graph_snapshot","subject_pith_number":"pith:2024:D76B7DJLIHDC3RGNKXTCBWD2Q4","target":"graph","payload":{"graph_snapshot":{"paper":{"title":"Spanish and LLM Benchmarks: is MMLU Lost in Translation?","license":"http://creativecommons.org/licenses/by/4.0/","headline":"","cross_cats":["cs.AI"],"primary_cat":"cs.CL","authors_text":"Cristina del Pozo, Irene Plaza, Javier Conde, Mar\\'ia Grandury, Marina Mayor-Rocher, Nina Melero, Pedro Reviriego","submitted_at":"2024-05-28T11:13:40Z","abstract_excerpt":"The evaluation of Large Language Models (LLMs) is a key element in their continuous improvement process and many benchmarks have been developed to assess the performance of LLMs in different tasks and topics. As LLMs become adopted worldwide, evaluating them in languages other than English is increasingly important. However, most LLM benchmarks are simply translated using an automated tool and then run in the target language. This means that the results depend not only on the LLM performance in that language but also on the quality of the translation. In this paper, we consider the case of the"},"claims":{"count":0,"items":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"2406.17789","kind":"arxiv","version":1},"verdict":{"id":null,"model_set":{},"created_at":null,"strongest_claim":"","one_line_summary":"","pipeline_version":null,"weakest_assumption":"","pith_extraction_headline":""},"integrity":{"clean":true,"summary":{"advisory":0,"critical":0,"by_detector":{},"informational":0},"endpoint":"/pith/2406.17789/integrity.json","findings":[],"available":true,"detectors_run":[],"snapshot_sha256":"c28c3603d3b5d939e8dc4c7e95fa8dfce3d595e45f758748cecf8e644a296938"},"references":{"count":0,"sample":[],"resolved_work":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","internal_anchors":0},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"author_claims":{"count":0,"strong_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"builder_version":"pith-number-builder-2026-05-17-v1"},"verdict_id":null},"signer":{"signer_id":"pith.science","signer_type":"pith_registry","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"created_at":"2026-07-05T08:36:49Z","supersedes":[],"prev_event":null,"signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"5bQvvSix8k2z036nG9evxoWr5mDQ8E8hzl24t8+ma4fizxkvq1/fn24AAe99uxFsmozhZ6+xCgsbK+3Aa7OSBA==","signed_message":"open_graph_event_sha256_bytes","signed_at":"2026-07-07T10:21:20.108535Z"},"content_sha256":"01d341394bf9d58d860dd4d1dec8658fb517ac4c60ad3c35ae659c5bd0491599","schema_version":"1.0","event_id":"sha256:01d341394bf9d58d860dd4d1dec8658fb517ac4c60ad3c35ae659c5bd0491599"}],"timestamp_proofs":[],"mirror_hints":[{"mirror_type":"https","name":"Pith Resolver","base_url":"https://pith.science","bundle_url":"https://pith.science/pith/D76B7DJLIHDC3RGNKXTCBWD2Q4/bundle.json","state_url":"https://pith.science/pith/D76B7DJLIHDC3RGNKXTCBWD2Q4/state.json","well_known_bundle_url":"https://pith.science/.well-known/pith/D76B7DJLIHDC3RGNKXTCBWD2Q4/bundle.json","status":"primary"}],"public_keys":[{"key_id":"pith-v1-2026-05","algorithm":"ed25519","format":"raw","public_key_b64":"stVStoiQhXFxp4s2pdzPNoqVNBMojDU/fJ2db5S3CbM=","public_key_hex":"b2d552b68890857171a78b36a5dccf368a953413288c353f7c9d9d6f94b709b3","fingerprint_sha256_b32_first128bits":"RVFV5Z2OI2J3ZUO7ERDEBCYNKS","fingerprint_sha256_hex":"8d4b5ee74e4693bcd1df2446408b0d54","rotates_at":null,"url":"https://pith.science/pith-signing-key.json","notes":"Pith uses this Ed25519 key to sign canonical record SHA-256 digests. Verify with: ed25519_verify(public_key, message=canonical_sha256_bytes, signature=base64decode(signature_b64))."}],"merge_version":"pith-open-graph-merge-v1","built_at":"2026-07-07T10:21:20Z","links":{"resolver":"https://pith.science/pith/D76B7DJLIHDC3RGNKXTCBWD2Q4","bundle":"https://pith.science/pith/D76B7DJLIHDC3RGNKXTCBWD2Q4/bundle.json","state":"https://pith.science/pith/D76B7DJLIHDC3RGNKXTCBWD2Q4/state.json","well_known_bundle":"https://pith.science/.well-known/pith/D76B7DJLIHDC3RGNKXTCBWD2Q4/bundle.json"},"state":{"state_type":"pith_open_graph_state","state_version":"1.0","pith_number":"pith:2024:D76B7DJLIHDC3RGNKXTCBWD2Q4","merge_version":"pith-open-graph-merge-v1","event_count":2,"valid_event_count":2,"invalid_event_count":0,"equivocation_count":0,"current":{"canonical_record":{"metadata":{"abstract_canon_sha256":"756e423170f5e1f5070f5aa71bf7eb40b69ea4ee756192ac50ad320aa71d8f32","cross_cats_sorted":["cs.AI"],"license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.CL","submitted_at":"2024-05-28T11:13:40Z","title_canon_sha256":"366d64fa5c9cb6f733d96f5c996b48290b0527aba57935cfee5a7f44c7159aba"},"schema_version":"1.0","source":{"id":"2406.17789","kind":"arxiv","version":1}},"source_aliases":[{"alias_kind":"arxiv","alias_value":"2406.17789","created_at":"2026-07-05T08:36:49Z"},{"alias_kind":"arxiv_version","alias_value":"2406.17789v1","created_at":"2026-07-05T08:36:49Z"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.2406.17789","created_at":"2026-07-05T08:36:49Z"},{"alias_kind":"pith_short_12","alias_value":"D76B7DJLIHDC","created_at":"2026-07-05T08:36:49Z"},{"alias_kind":"pith_short_16","alias_value":"D76B7DJLIHDC3RGN","created_at":"2026-07-05T08:36:49Z"},{"alias_kind":"pith_short_8","alias_value":"D76B7DJL","created_at":"2026-07-05T08:36:49Z"}],"graph_snapshots":[{"event_id":"sha256:01d341394bf9d58d860dd4d1dec8658fb517ac4c60ad3c35ae659c5bd0491599","target":"graph","created_at":"2026-07-05T08:36:49Z","signer":{"key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signer_id":"pith.science","signer_type":"pith_registry"},"payload":{"graph_snapshot":{"author_claims":{"count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","strong_count":0},"builder_version":"pith-number-builder-2026-05-17-v1","claims":{"count":0,"items":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"integrity":{"available":true,"clean":true,"detectors_run":[],"endpoint":"/pith/2406.17789/integrity.json","findings":[],"snapshot_sha256":"c28c3603d3b5d939e8dc4c7e95fa8dfce3d595e45f758748cecf8e644a296938","summary":{"advisory":0,"by_detector":{},"critical":0,"informational":0}},"paper":{"abstract_excerpt":"The evaluation of Large Language Models (LLMs) is a key element in their continuous improvement process and many benchmarks have been developed to assess the performance of LLMs in different tasks and topics. As LLMs become adopted worldwide, evaluating them in languages other than English is increasingly important. However, most LLM benchmarks are simply translated using an automated tool and then run in the target language. This means that the results depend not only on the LLM performance in that language but also on the quality of the translation. In this paper, we consider the case of the","authors_text":"Cristina del Pozo, Irene Plaza, Javier Conde, Mar\\'ia Grandury, Marina Mayor-Rocher, Nina Melero, Pedro Reviriego","cross_cats":["cs.AI"],"headline":"","license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.CL","submitted_at":"2024-05-28T11:13:40Z","title":"Spanish and LLM Benchmarks: is MMLU Lost in Translation?"},"references":{"count":0,"internal_anchors":0,"resolved_work":0,"sample":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"2406.17789","kind":"arxiv","version":1},"verdict":{"created_at":null,"id":null,"model_set":{},"one_line_summary":"","pipeline_version":null,"pith_extraction_headline":"","strongest_claim":"","weakest_assumption":""}},"verdict_id":null}}],"author_attestations":[],"timestamp_anchors":[],"storage_attestations":[],"citation_signatures":[],"replication_records":[],"corrections":[],"mirror_hints":[],"record_created":{"event_id":"sha256:08fb5b96c870b5d65d3c51d1ddd102b5fab1484b38e2d819e7e848bb275c430e","target":"record","created_at":"2026-07-05T08:36:49Z","signer":{"key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signer_id":"pith.science","signer_type":"pith_registry"},"payload":{"attestation_state":"computed","canonical_record":{"metadata":{"abstract_canon_sha256":"756e423170f5e1f5070f5aa71bf7eb40b69ea4ee756192ac50ad320aa71d8f32","cross_cats_sorted":["cs.AI"],"license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.CL","submitted_at":"2024-05-28T11:13:40Z","title_canon_sha256":"366d64fa5c9cb6f733d96f5c996b48290b0527aba57935cfee5a7f44c7159aba"},"schema_version":"1.0","source":{"id":"2406.17789","kind":"arxiv","version":1}},"canonical_sha256":"1ffc1f8d2b41c62dc4cd55e620d87a8736ffa554641e107d31e2f2f808b586b3","receipt":{"algorithm":"ed25519","builder_version":"pith-number-builder-2026-05-17-v1","canonical_sha256":"1ffc1f8d2b41c62dc4cd55e620d87a8736ffa554641e107d31e2f2f808b586b3","first_computed_at":"2026-07-05T08:36:49.846241Z","key_id":"pith-v1-2026-05","kind":"pith_receipt","last_reissued_at":"2026-07-05T08:36:49.846241Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","receipt_version":"0.3","signature_b64":"P3muh9CrhbvNoyJoLOllNp3ZD9JiVEKnNygtwKuyKHvljBr9pM1hsCGv4OW/fBhP/T1czVEo/7yftxfTK1oQCA==","signature_status":"signed_v1","signed_at":"2026-07-05T08:36:49.846810Z","signed_message":"canonical_sha256_bytes"},"source_id":"2406.17789","source_kind":"arxiv","source_version":1}}},"equivocations":[],"invalid_events":[],"applied_event_ids":["sha256:08fb5b96c870b5d65d3c51d1ddd102b5fab1484b38e2d819e7e848bb275c430e","sha256:01d341394bf9d58d860dd4d1dec8658fb517ac4c60ad3c35ae659c5bd0491599"],"state_sha256":"33321498b2670e1195a4b0683580e809cfd366e58351fecb4c1390f094857aa1"},"bundle_signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"tfsx1h8R0gitV3rHmrZgp61oPKlUsoiv0AnQp2rEIhaiDmcUlHasq1o+h3PNbjlXlE7Rpkoo/+d1BBadvs6iBg==","signed_message":"bundle_sha256_bytes","signed_at":"2026-07-07T10:21:20.110515Z","bundle_sha256":"891736bad30f381986e6ad8970b3eafcafaf94add974e19866bb68fccb696e3f"}}