{"state_type":"pith_open_graph_state","state_version":"1.0","pith_number":"pith:2024:D76B7DJLIHDC3RGNKXTCBWD2Q4","merge_version":"pith-open-graph-merge-v1","event_count":2,"valid_event_count":2,"invalid_event_count":0,"equivocation_count":0,"current":{"canonical_record":{"metadata":{"abstract_canon_sha256":"756e423170f5e1f5070f5aa71bf7eb40b69ea4ee756192ac50ad320aa71d8f32","cross_cats_sorted":["cs.AI"],"license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.CL","submitted_at":"2024-05-28T11:13:40Z","title_canon_sha256":"366d64fa5c9cb6f733d96f5c996b48290b0527aba57935cfee5a7f44c7159aba"},"schema_version":"1.0","source":{"id":"2406.17789","kind":"arxiv","version":1}},"source_aliases":[{"alias_kind":"arxiv","alias_value":"2406.17789","created_at":"2026-07-05T08:36:49Z"},{"alias_kind":"arxiv_version","alias_value":"2406.17789v1","created_at":"2026-07-05T08:36:49Z"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.2406.17789","created_at":"2026-07-05T08:36:49Z"},{"alias_kind":"pith_short_12","alias_value":"D76B7DJLIHDC","created_at":"2026-07-05T08:36:49Z"},{"alias_kind":"pith_short_16","alias_value":"D76B7DJLIHDC3RGN","created_at":"2026-07-05T08:36:49Z"},{"alias_kind":"pith_short_8","alias_value":"D76B7DJL","created_at":"2026-07-05T08:36:49Z"}],"graph_snapshots":[{"event_id":"sha256:01d341394bf9d58d860dd4d1dec8658fb517ac4c60ad3c35ae659c5bd0491599","target":"graph","created_at":"2026-07-05T08:36:49Z","signer":{"key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signer_id":"pith.science","signer_type":"pith_registry"},"payload":{"graph_snapshot":{"author_claims":{"count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","strong_count":0},"builder_version":"pith-number-builder-2026-05-17-v1","claims":{"count":0,"items":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"integrity":{"available":true,"clean":true,"detectors_run":[],"endpoint":"/pith/2406.17789/integrity.json","findings":[],"snapshot_sha256":"c28c3603d3b5d939e8dc4c7e95fa8dfce3d595e45f758748cecf8e644a296938","summary":{"advisory":0,"by_detector":{},"critical":0,"informational":0}},"paper":{"abstract_excerpt":"The evaluation of Large Language Models (LLMs) is a key element in their continuous improvement process and many benchmarks have been developed to assess the performance of LLMs in different tasks and topics. As LLMs become adopted worldwide, evaluating them in languages other than English is increasingly important. However, most LLM benchmarks are simply translated using an automated tool and then run in the target language. This means that the results depend not only on the LLM performance in that language but also on the quality of the translation. In this paper, we consider the case of the","authors_text":"Cristina del Pozo, Irene Plaza, Javier Conde, Mar\\'ia Grandury, Marina Mayor-Rocher, Nina Melero, Pedro Reviriego","cross_cats":["cs.AI"],"headline":"","license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.CL","submitted_at":"2024-05-28T11:13:40Z","title":"Spanish and LLM Benchmarks: is MMLU Lost in Translation?"},"references":{"count":0,"internal_anchors":0,"resolved_work":0,"sample":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"2406.17789","kind":"arxiv","version":1},"verdict":{"created_at":null,"id":null,"model_set":{},"one_line_summary":"","pipeline_version":null,"pith_extraction_headline":"","strongest_claim":"","weakest_assumption":""}},"verdict_id":null}}],"author_attestations":[],"timestamp_anchors":[],"storage_attestations":[],"citation_signatures":[],"replication_records":[],"corrections":[],"mirror_hints":[],"record_created":{"event_id":"sha256:08fb5b96c870b5d65d3c51d1ddd102b5fab1484b38e2d819e7e848bb275c430e","target":"record","created_at":"2026-07-05T08:36:49Z","signer":{"key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signer_id":"pith.science","signer_type":"pith_registry"},"payload":{"attestation_state":"computed","canonical_record":{"metadata":{"abstract_canon_sha256":"756e423170f5e1f5070f5aa71bf7eb40b69ea4ee756192ac50ad320aa71d8f32","cross_cats_sorted":["cs.AI"],"license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.CL","submitted_at":"2024-05-28T11:13:40Z","title_canon_sha256":"366d64fa5c9cb6f733d96f5c996b48290b0527aba57935cfee5a7f44c7159aba"},"schema_version":"1.0","source":{"id":"2406.17789","kind":"arxiv","version":1}},"canonical_sha256":"1ffc1f8d2b41c62dc4cd55e620d87a8736ffa554641e107d31e2f2f808b586b3","receipt":{"algorithm":"ed25519","builder_version":"pith-number-builder-2026-05-17-v1","canonical_sha256":"1ffc1f8d2b41c62dc4cd55e620d87a8736ffa554641e107d31e2f2f808b586b3","first_computed_at":"2026-07-05T08:36:49.846241Z","key_id":"pith-v1-2026-05","kind":"pith_receipt","last_reissued_at":"2026-07-05T08:36:49.846241Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","receipt_version":"0.3","signature_b64":"P3muh9CrhbvNoyJoLOllNp3ZD9JiVEKnNygtwKuyKHvljBr9pM1hsCGv4OW/fBhP/T1czVEo/7yftxfTK1oQCA==","signature_status":"signed_v1","signed_at":"2026-07-05T08:36:49.846810Z","signed_message":"canonical_sha256_bytes"},"source_id":"2406.17789","source_kind":"arxiv","source_version":1}}},"equivocations":[],"invalid_events":[],"applied_event_ids":["sha256:08fb5b96c870b5d65d3c51d1ddd102b5fab1484b38e2d819e7e848bb275c430e","sha256:01d341394bf9d58d860dd4d1dec8658fb517ac4c60ad3c35ae659c5bd0491599"],"state_sha256":"33321498b2670e1195a4b0683580e809cfd366e58351fecb4c1390f094857aa1"}