{"bundle_type":"pith_open_graph_bundle","bundle_version":"1.0","pith_number":"pith:2025:S4UB4TFUYLYC5PYCXMTVDQ2FE5","short_pith_number":"pith:S4UB4TFU","canonical_record":{"source":{"id":"2504.15521","kind":"arxiv","version":1},"metadata":{"license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","primary_cat":"cs.CL","submitted_at":"2025-04-22T01:47:37Z","cross_cats_sorted":[],"title_canon_sha256":"2ed0358a589295804aa699d9cadedaeb020261cfeb14a54c39c1c964c15c66a6","abstract_canon_sha256":"f6506cca950706619fb5c5459a6171ee92d36aec7710e10116a435d0de31929e"},"schema_version":"1.0"},"canonical_sha256":"97281e4cb4c2f02ebf02bb2751c345277b16573f08deba03a7a3e3d904c85b6d","source":{"kind":"arxiv","id":"2504.15521","version":1},"source_aliases":[{"alias_kind":"arxiv","alias_value":"2504.15521","created_at":"2026-07-05T10:52:25Z"},{"alias_kind":"arxiv_version","alias_value":"2504.15521v1","created_at":"2026-07-05T10:52:25Z"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.2504.15521","created_at":"2026-07-05T10:52:25Z"},{"alias_kind":"pith_short_12","alias_value":"S4UB4TFUYLYC","created_at":"2026-07-05T10:52:25Z"},{"alias_kind":"pith_short_16","alias_value":"S4UB4TFUYLYC5PYC","created_at":"2026-07-05T10:52:25Z"},{"alias_kind":"pith_short_8","alias_value":"S4UB4TFU","created_at":"2026-07-05T10:52:25Z"}],"events":[{"event_type":"record_created","subject_pith_number":"pith:2025:S4UB4TFUYLYC5PYCXMTVDQ2FE5","target":"record","payload":{"canonical_record":{"source":{"id":"2504.15521","kind":"arxiv","version":1},"metadata":{"license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","primary_cat":"cs.CL","submitted_at":"2025-04-22T01:47:37Z","cross_cats_sorted":[],"title_canon_sha256":"2ed0358a589295804aa699d9cadedaeb020261cfeb14a54c39c1c964c15c66a6","abstract_canon_sha256":"f6506cca950706619fb5c5459a6171ee92d36aec7710e10116a435d0de31929e"},"schema_version":"1.0"},"canonical_sha256":"97281e4cb4c2f02ebf02bb2751c345277b16573f08deba03a7a3e3d904c85b6d","receipt":{"kind":"pith_receipt","key_id":"pith-v1-2026-05","algorithm":"ed25519","signed_at":"2026-07-05T10:52:25.938835Z","signature_b64":"Ryteu5LkdF6ZodVRg9Dcx7NDlhYM9ixNe3e2v95I4Cf4KrCFqMmfeZWtM/VhEqeI98oA8P5QY0+m8TFAp/UIAA==","signed_message":"canonical_sha256_bytes","builder_version":"pith-number-builder-2026-05-17-v1","receipt_version":"0.3","canonical_sha256":"97281e4cb4c2f02ebf02bb2751c345277b16573f08deba03a7a3e3d904c85b6d","last_reissued_at":"2026-07-05T10:52:25.938339Z","signature_status":"signed_v1","first_computed_at":"2026-07-05T10:52:25.938339Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"source_kind":"arxiv","source_id":"2504.15521","source_version":1,"attestation_state":"computed"},"signer":{"signer_id":"pith.science","signer_type":"pith_registry","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"created_at":"2026-07-05T10:52:25Z","supersedes":[],"prev_event":null,"signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"uMAMa2QABNZOrFH5vO1FHbpB01chWdP0gaNo3oNDpo+Zmov+OcMd7cNdB0Rx9Qp6bRYqlKAVDkDXOttu6OQlDw==","signed_message":"open_graph_event_sha256_bytes","signed_at":"2026-07-07T12:19:31.411760Z"},"content_sha256":"623d5af3a78e3da27f57640a6218ecf5e5bddcef29fb90d3f7a3215abd21c403","schema_version":"1.0","event_id":"sha256:623d5af3a78e3da27f57640a6218ecf5e5bddcef29fb90d3f7a3215abd21c403"},{"event_type":"graph_snapshot","subject_pith_number":"pith:2025:S4UB4TFUYLYC5PYCXMTVDQ2FE5","target":"graph","payload":{"graph_snapshot":{"paper":{"title":"The Bitter Lesson Learned from 2,000+ Multilingual Benchmarks","license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","headline":"","cross_cats":[],"primary_cat":"cs.CL","authors_text":"Chenyang Lyu, Huifeng Yin, Kaifu Zhang, Longyue Wang, Minghao Wu, Sinuo Liu, Weihua Luo, Weixuan Wang, Xintong Wang, Yu Zhao","submitted_at":"2025-04-22T01:47:37Z","abstract_excerpt":"As large language models (LLMs) continue to advance in linguistic capabilities, robust multilingual evaluation has become essential for promoting equitable technological progress. This position paper examines over 2,000 multilingual (non-English) benchmarks from 148 countries, published between 2021 and 2024, to evaluate past, present, and future practices in multilingual benchmarking. Our findings reveal that, despite significant investments amounting to tens of millions of dollars, English remains significantly overrepresented in these benchmarks. Additionally, most benchmarks rely on origin"},"claims":{"count":0,"items":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"2504.15521","kind":"arxiv","version":1},"verdict":{"id":null,"model_set":{},"created_at":null,"strongest_claim":"","one_line_summary":"","pipeline_version":null,"weakest_assumption":"","pith_extraction_headline":""},"integrity":{"clean":true,"summary":{"advisory":0,"critical":0,"by_detector":{},"informational":0},"endpoint":"/pith/2504.15521/integrity.json","findings":[],"available":true,"detectors_run":[],"snapshot_sha256":"c28c3603d3b5d939e8dc4c7e95fa8dfce3d595e45f758748cecf8e644a296938"},"references":{"count":0,"sample":[],"resolved_work":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","internal_anchors":0},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"author_claims":{"count":0,"strong_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"builder_version":"pith-number-builder-2026-05-17-v1"},"verdict_id":null},"signer":{"signer_id":"pith.science","signer_type":"pith_registry","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"created_at":"2026-07-05T10:52:25Z","supersedes":[],"prev_event":null,"signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"sQWN/68ZDmqPn1ZymhgXoxzehM57vZd03vBHoih70yLJUPdg5mZcJLczLvCZa4STW0M5oUVFYAGnqUfe2zTcDA==","signed_message":"open_graph_event_sha256_bytes","signed_at":"2026-07-07T12:19:31.412134Z"},"content_sha256":"5a22eb34f127ca50d24b5165412aadfa9c476deadb122bf50d91beb053598106","schema_version":"1.0","event_id":"sha256:5a22eb34f127ca50d24b5165412aadfa9c476deadb122bf50d91beb053598106"}],"timestamp_proofs":[],"mirror_hints":[{"mirror_type":"https","name":"Pith Resolver","base_url":"https://pith.science","bundle_url":"https://pith.science/pith/S4UB4TFUYLYC5PYCXMTVDQ2FE5/bundle.json","state_url":"https://pith.science/pith/S4UB4TFUYLYC5PYCXMTVDQ2FE5/state.json","well_known_bundle_url":"https://pith.science/.well-known/pith/S4UB4TFUYLYC5PYCXMTVDQ2FE5/bundle.json","status":"primary"}],"public_keys":[{"key_id":"pith-v1-2026-05","algorithm":"ed25519","format":"raw","public_key_b64":"stVStoiQhXFxp4s2pdzPNoqVNBMojDU/fJ2db5S3CbM=","public_key_hex":"b2d552b68890857171a78b36a5dccf368a953413288c353f7c9d9d6f94b709b3","fingerprint_sha256_b32_first128bits":"RVFV5Z2OI2J3ZUO7ERDEBCYNKS","fingerprint_sha256_hex":"8d4b5ee74e4693bcd1df2446408b0d54","rotates_at":null,"url":"https://pith.science/pith-signing-key.json","notes":"Pith uses this Ed25519 key to sign canonical record SHA-256 digests. Verify with: ed25519_verify(public_key, message=canonical_sha256_bytes, signature=base64decode(signature_b64))."}],"merge_version":"pith-open-graph-merge-v1","built_at":"2026-07-07T12:19:31Z","links":{"resolver":"https://pith.science/pith/S4UB4TFUYLYC5PYCXMTVDQ2FE5","bundle":"https://pith.science/pith/S4UB4TFUYLYC5PYCXMTVDQ2FE5/bundle.json","state":"https://pith.science/pith/S4UB4TFUYLYC5PYCXMTVDQ2FE5/state.json","well_known_bundle":"https://pith.science/.well-known/pith/S4UB4TFUYLYC5PYCXMTVDQ2FE5/bundle.json"},"state":{"state_type":"pith_open_graph_state","state_version":"1.0","pith_number":"pith:2025:S4UB4TFUYLYC5PYCXMTVDQ2FE5","merge_version":"pith-open-graph-merge-v1","event_count":2,"valid_event_count":2,"invalid_event_count":0,"equivocation_count":0,"current":{"canonical_record":{"metadata":{"abstract_canon_sha256":"f6506cca950706619fb5c5459a6171ee92d36aec7710e10116a435d0de31929e","cross_cats_sorted":[],"license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","primary_cat":"cs.CL","submitted_at":"2025-04-22T01:47:37Z","title_canon_sha256":"2ed0358a589295804aa699d9cadedaeb020261cfeb14a54c39c1c964c15c66a6"},"schema_version":"1.0","source":{"id":"2504.15521","kind":"arxiv","version":1}},"source_aliases":[{"alias_kind":"arxiv","alias_value":"2504.15521","created_at":"2026-07-05T10:52:25Z"},{"alias_kind":"arxiv_version","alias_value":"2504.15521v1","created_at":"2026-07-05T10:52:25Z"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.2504.15521","created_at":"2026-07-05T10:52:25Z"},{"alias_kind":"pith_short_12","alias_value":"S4UB4TFUYLYC","created_at":"2026-07-05T10:52:25Z"},{"alias_kind":"pith_short_16","alias_value":"S4UB4TFUYLYC5PYC","created_at":"2026-07-05T10:52:25Z"},{"alias_kind":"pith_short_8","alias_value":"S4UB4TFU","created_at":"2026-07-05T10:52:25Z"}],"graph_snapshots":[{"event_id":"sha256:5a22eb34f127ca50d24b5165412aadfa9c476deadb122bf50d91beb053598106","target":"graph","created_at":"2026-07-05T10:52:25Z","signer":{"key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signer_id":"pith.science","signer_type":"pith_registry"},"payload":{"graph_snapshot":{"author_claims":{"count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","strong_count":0},"builder_version":"pith-number-builder-2026-05-17-v1","claims":{"count":0,"items":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"integrity":{"available":true,"clean":true,"detectors_run":[],"endpoint":"/pith/2504.15521/integrity.json","findings":[],"snapshot_sha256":"c28c3603d3b5d939e8dc4c7e95fa8dfce3d595e45f758748cecf8e644a296938","summary":{"advisory":0,"by_detector":{},"critical":0,"informational":0}},"paper":{"abstract_excerpt":"As large language models (LLMs) continue to advance in linguistic capabilities, robust multilingual evaluation has become essential for promoting equitable technological progress. This position paper examines over 2,000 multilingual (non-English) benchmarks from 148 countries, published between 2021 and 2024, to evaluate past, present, and future practices in multilingual benchmarking. Our findings reveal that, despite significant investments amounting to tens of millions of dollars, English remains significantly overrepresented in these benchmarks. Additionally, most benchmarks rely on origin","authors_text":"Chenyang Lyu, Huifeng Yin, Kaifu Zhang, Longyue Wang, Minghao Wu, Sinuo Liu, Weihua Luo, Weixuan Wang, Xintong Wang, Yu Zhao","cross_cats":[],"headline":"","license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","primary_cat":"cs.CL","submitted_at":"2025-04-22T01:47:37Z","title":"The Bitter Lesson Learned from 2,000+ Multilingual Benchmarks"},"references":{"count":0,"internal_anchors":0,"resolved_work":0,"sample":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"2504.15521","kind":"arxiv","version":1},"verdict":{"created_at":null,"id":null,"model_set":{},"one_line_summary":"","pipeline_version":null,"pith_extraction_headline":"","strongest_claim":"","weakest_assumption":""}},"verdict_id":null}}],"author_attestations":[],"timestamp_anchors":[],"storage_attestations":[],"citation_signatures":[],"replication_records":[],"corrections":[],"mirror_hints":[],"record_created":{"event_id":"sha256:623d5af3a78e3da27f57640a6218ecf5e5bddcef29fb90d3f7a3215abd21c403","target":"record","created_at":"2026-07-05T10:52:25Z","signer":{"key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signer_id":"pith.science","signer_type":"pith_registry"},"payload":{"attestation_state":"computed","canonical_record":{"metadata":{"abstract_canon_sha256":"f6506cca950706619fb5c5459a6171ee92d36aec7710e10116a435d0de31929e","cross_cats_sorted":[],"license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","primary_cat":"cs.CL","submitted_at":"2025-04-22T01:47:37Z","title_canon_sha256":"2ed0358a589295804aa699d9cadedaeb020261cfeb14a54c39c1c964c15c66a6"},"schema_version":"1.0","source":{"id":"2504.15521","kind":"arxiv","version":1}},"canonical_sha256":"97281e4cb4c2f02ebf02bb2751c345277b16573f08deba03a7a3e3d904c85b6d","receipt":{"algorithm":"ed25519","builder_version":"pith-number-builder-2026-05-17-v1","canonical_sha256":"97281e4cb4c2f02ebf02bb2751c345277b16573f08deba03a7a3e3d904c85b6d","first_computed_at":"2026-07-05T10:52:25.938339Z","key_id":"pith-v1-2026-05","kind":"pith_receipt","last_reissued_at":"2026-07-05T10:52:25.938339Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","receipt_version":"0.3","signature_b64":"Ryteu5LkdF6ZodVRg9Dcx7NDlhYM9ixNe3e2v95I4Cf4KrCFqMmfeZWtM/VhEqeI98oA8P5QY0+m8TFAp/UIAA==","signature_status":"signed_v1","signed_at":"2026-07-05T10:52:25.938835Z","signed_message":"canonical_sha256_bytes"},"source_id":"2504.15521","source_kind":"arxiv","source_version":1}}},"equivocations":[],"invalid_events":[],"applied_event_ids":["sha256:623d5af3a78e3da27f57640a6218ecf5e5bddcef29fb90d3f7a3215abd21c403","sha256:5a22eb34f127ca50d24b5165412aadfa9c476deadb122bf50d91beb053598106"],"state_sha256":"643f192a0b5c51b011877ae0f192cc188e86a2d56af76c0ca7d911391c38ad00"},"bundle_signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"LC2Ziy4/HD31DxOKVdIRExFbWsTAWJyeWdTkvWWo4SI3G1NB+xH5+mun3U3ykATuDRgjXdc6Mc2jxg81IPwdBA==","signed_message":"bundle_sha256_bytes","signed_at":"2026-07-07T12:19:31.414171Z","bundle_sha256":"fc1aadd000df97a55378a752f074db8ea1399a6655fdffc144115ef500753878"}}