{"bundle_type":"pith_open_graph_bundle","bundle_version":"1.0","pith_number":"pith:2022:BYIGKWLZZAEDMSOWHCHBPOVJ26","short_pith_number":"pith:BYIGKWLZ","canonical_record":{"source":{"id":"2204.14268","kind":"arxiv","version":2},"metadata":{"license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","primary_cat":"cs.CL","submitted_at":"2022-04-29T17:50:36Z","cross_cats_sorted":["cs.AI"],"title_canon_sha256":"0b5c0087b45899654f007eb240613b9ed25127ec754b1b9005533da78022c025","abstract_canon_sha256":"28057cb8d9be8d472b09e91cdc04fdfdddf3513c644bdd6cbd4fc925b6cfe292"},"schema_version":"1.0"},"canonical_sha256":"0e10655979c8083649d6388e17baa9d7b1b8bd600f0070554b57e2b8d9cef5f5","source":{"kind":"arxiv","id":"2204.14268","version":2},"source_aliases":[{"alias_kind":"arxiv","alias_value":"2204.14268","created_at":"2026-07-05T04:56:16Z"},{"alias_kind":"arxiv_version","alias_value":"2204.14268v2","created_at":"2026-07-05T04:56:16Z"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.2204.14268","created_at":"2026-07-05T04:56:16Z"},{"alias_kind":"pith_short_12","alias_value":"BYIGKWLZZAED","created_at":"2026-07-05T04:56:16Z"},{"alias_kind":"pith_short_16","alias_value":"BYIGKWLZZAEDMSOW","created_at":"2026-07-05T04:56:16Z"},{"alias_kind":"pith_short_8","alias_value":"BYIGKWLZ","created_at":"2026-07-05T04:56:16Z"}],"events":[{"event_type":"record_created","subject_pith_number":"pith:2022:BYIGKWLZZAEDMSOWHCHBPOVJ26","target":"record","payload":{"canonical_record":{"source":{"id":"2204.14268","kind":"arxiv","version":2},"metadata":{"license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","primary_cat":"cs.CL","submitted_at":"2022-04-29T17:50:36Z","cross_cats_sorted":["cs.AI"],"title_canon_sha256":"0b5c0087b45899654f007eb240613b9ed25127ec754b1b9005533da78022c025","abstract_canon_sha256":"28057cb8d9be8d472b09e91cdc04fdfdddf3513c644bdd6cbd4fc925b6cfe292"},"schema_version":"1.0"},"canonical_sha256":"0e10655979c8083649d6388e17baa9d7b1b8bd600f0070554b57e2b8d9cef5f5","receipt":{"kind":"pith_receipt","key_id":"pith-v1-2026-05","algorithm":"ed25519","signed_at":"2026-07-05T04:56:16.951525Z","signature_b64":"Vp/af0ijgzjwMbFLlJpr6EODn298rdpNzR263bXHCSoBCemnV6FezQqMq3V7MBkkerjae6ui1j7pRAQowUcwDg==","signed_message":"canonical_sha256_bytes","builder_version":"pith-number-builder-2026-05-17-v1","receipt_version":"0.3","canonical_sha256":"0e10655979c8083649d6388e17baa9d7b1b8bd600f0070554b57e2b8d9cef5f5","last_reissued_at":"2026-07-05T04:56:16.950992Z","signature_status":"signed_v1","first_computed_at":"2026-07-05T04:56:16.950992Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"source_kind":"arxiv","source_id":"2204.14268","source_version":2,"attestation_state":"computed"},"signer":{"signer_id":"pith.science","signer_type":"pith_registry","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"created_at":"2026-07-05T04:56:16Z","supersedes":[],"prev_event":null,"signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"arw7UToZrQViPNcilSkZAdkuxnQFdmoXv9xN+z5d40VriLEoF+BOG+sk3JfYEB6iKMXG4UWnCV/YyDKXqeEADw==","signed_message":"open_graph_event_sha256_bytes","signed_at":"2026-08-03T20:11:02.118993Z"},"content_sha256":"bbcbab370d303a81f4a076cb9bd9d4f7c8c8587e05eb739878f666c9c03c6615","schema_version":"1.0","event_id":"sha256:bbcbab370d303a81f4a076cb9bd9d4f7c8c8587e05eb739878f666c9c03c6615"},{"event_type":"graph_snapshot","subject_pith_number":"pith:2022:BYIGKWLZZAEDMSOWHCHBPOVJ26","target":"graph","payload":{"graph_snapshot":{"paper":{"title":"How Robust is Neural Machine Translation to Language Imbalance in Multilingual Tokenizer Training?","license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","headline":"","cross_cats":["cs.AI"],"primary_cat":"cs.CL","authors_text":"Francisco Guzman, Guillaume Wenzek, James Cross, Mohit Bansal, Naman Goyal, Shiyue Zhang, Vishrav Chaudhary","submitted_at":"2022-04-29T17:50:36Z","abstract_excerpt":"A multilingual tokenizer is a fundamental component of multilingual neural machine translation. It is trained from a multilingual corpus. Since a skewed data distribution is considered to be harmful, a sampling strategy is usually used to balance languages in the corpus. However, few works have systematically answered how language imbalance in tokenizer training affects downstream performance. In this work, we analyze how translation performance changes as the data ratios among languages vary in the tokenizer training corpus. We find that while relatively better performance is often observed w"},"claims":{"count":0,"items":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"2204.14268","kind":"arxiv","version":2},"verdict":{"id":null,"model_set":{},"created_at":null,"strongest_claim":"","one_line_summary":"","pipeline_version":null,"weakest_assumption":"","pith_extraction_headline":""},"integrity":{"clean":true,"summary":{"advisory":0,"critical":0,"by_detector":{},"informational":0},"endpoint":"/pith/2204.14268/integrity.json","findings":[],"available":true,"detectors_run":[],"snapshot_sha256":"c28c3603d3b5d939e8dc4c7e95fa8dfce3d595e45f758748cecf8e644a296938"},"references":{"count":0,"sample":[],"resolved_work":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","internal_anchors":0},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"author_claims":{"count":0,"strong_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"builder_version":"pith-number-builder-2026-05-17-v1"},"verdict_id":null},"signer":{"signer_id":"pith.science","signer_type":"pith_registry","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"created_at":"2026-07-05T04:56:16Z","supersedes":[],"prev_event":null,"signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"nsOuJpby31NQsNDkw1jmM8Th31cbCXnEGnW4aSRPDV/NfuYwjyo+oVwyMci2gHQVmBGzNudvKsx5AUvzJwJjAQ==","signed_message":"open_graph_event_sha256_bytes","signed_at":"2026-08-03T20:11:02.119535Z"},"content_sha256":"33d3d21763761790c0225a0691f129a5adac1e0ff06d734b6336ccc782ca89e7","schema_version":"1.0","event_id":"sha256:33d3d21763761790c0225a0691f129a5adac1e0ff06d734b6336ccc782ca89e7"}],"timestamp_proofs":[],"mirror_hints":[{"mirror_type":"https","name":"Pith Resolver","base_url":"https://pith.science","bundle_url":"https://pith.science/pith/BYIGKWLZZAEDMSOWHCHBPOVJ26/bundle.json","state_url":"https://pith.science/pith/BYIGKWLZZAEDMSOWHCHBPOVJ26/state.json","well_known_bundle_url":"https://pith.science/.well-known/pith/BYIGKWLZZAEDMSOWHCHBPOVJ26/bundle.json","status":"primary"}],"public_keys":[{"key_id":"pith-v1-2026-05","algorithm":"ed25519","format":"raw","public_key_b64":"stVStoiQhXFxp4s2pdzPNoqVNBMojDU/fJ2db5S3CbM=","public_key_hex":"b2d552b68890857171a78b36a5dccf368a953413288c353f7c9d9d6f94b709b3","fingerprint_sha256_b32_first128bits":"RVFV5Z2OI2J3ZUO7ERDEBCYNKS","fingerprint_sha256_hex":"8d4b5ee74e4693bcd1df2446408b0d54","rotates_at":null,"url":"https://pith.science/pith-signing-key.json","notes":"Pith uses this Ed25519 key to sign canonical record SHA-256 digests. Verify with: ed25519_verify(public_key, message=canonical_sha256_bytes, signature=base64decode(signature_b64))."}],"merge_version":"pith-open-graph-merge-v1","built_at":"2026-08-03T20:11:02Z","links":{"resolver":"https://pith.science/pith/BYIGKWLZZAEDMSOWHCHBPOVJ26","bundle":"https://pith.science/pith/BYIGKWLZZAEDMSOWHCHBPOVJ26/bundle.json","state":"https://pith.science/pith/BYIGKWLZZAEDMSOWHCHBPOVJ26/state.json","well_known_bundle":"https://pith.science/.well-known/pith/BYIGKWLZZAEDMSOWHCHBPOVJ26/bundle.json"},"state":{"state_type":"pith_open_graph_state","state_version":"1.0","pith_number":"pith:2022:BYIGKWLZZAEDMSOWHCHBPOVJ26","merge_version":"pith-open-graph-merge-v1","event_count":2,"valid_event_count":2,"invalid_event_count":0,"equivocation_count":0,"current":{"canonical_record":{"metadata":{"abstract_canon_sha256":"28057cb8d9be8d472b09e91cdc04fdfdddf3513c644bdd6cbd4fc925b6cfe292","cross_cats_sorted":["cs.AI"],"license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","primary_cat":"cs.CL","submitted_at":"2022-04-29T17:50:36Z","title_canon_sha256":"0b5c0087b45899654f007eb240613b9ed25127ec754b1b9005533da78022c025"},"schema_version":"1.0","source":{"id":"2204.14268","kind":"arxiv","version":2}},"source_aliases":[{"alias_kind":"arxiv","alias_value":"2204.14268","created_at":"2026-07-05T04:56:16Z"},{"alias_kind":"arxiv_version","alias_value":"2204.14268v2","created_at":"2026-07-05T04:56:16Z"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.2204.14268","created_at":"2026-07-05T04:56:16Z"},{"alias_kind":"pith_short_12","alias_value":"BYIGKWLZZAED","created_at":"2026-07-05T04:56:16Z"},{"alias_kind":"pith_short_16","alias_value":"BYIGKWLZZAEDMSOW","created_at":"2026-07-05T04:56:16Z"},{"alias_kind":"pith_short_8","alias_value":"BYIGKWLZ","created_at":"2026-07-05T04:56:16Z"}],"graph_snapshots":[{"event_id":"sha256:33d3d21763761790c0225a0691f129a5adac1e0ff06d734b6336ccc782ca89e7","target":"graph","created_at":"2026-07-05T04:56:16Z","signer":{"key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signer_id":"pith.science","signer_type":"pith_registry"},"payload":{"graph_snapshot":{"author_claims":{"count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","strong_count":0},"builder_version":"pith-number-builder-2026-05-17-v1","claims":{"count":0,"items":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"integrity":{"available":true,"clean":true,"detectors_run":[],"endpoint":"/pith/2204.14268/integrity.json","findings":[],"snapshot_sha256":"c28c3603d3b5d939e8dc4c7e95fa8dfce3d595e45f758748cecf8e644a296938","summary":{"advisory":0,"by_detector":{},"critical":0,"informational":0}},"paper":{"abstract_excerpt":"A multilingual tokenizer is a fundamental component of multilingual neural machine translation. It is trained from a multilingual corpus. Since a skewed data distribution is considered to be harmful, a sampling strategy is usually used to balance languages in the corpus. However, few works have systematically answered how language imbalance in tokenizer training affects downstream performance. In this work, we analyze how translation performance changes as the data ratios among languages vary in the tokenizer training corpus. We find that while relatively better performance is often observed w","authors_text":"Francisco Guzman, Guillaume Wenzek, James Cross, Mohit Bansal, Naman Goyal, Shiyue Zhang, Vishrav Chaudhary","cross_cats":["cs.AI"],"headline":"","license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","primary_cat":"cs.CL","submitted_at":"2022-04-29T17:50:36Z","title":"How Robust is Neural Machine Translation to Language Imbalance in Multilingual Tokenizer Training?"},"references":{"count":0,"internal_anchors":0,"resolved_work":0,"sample":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"2204.14268","kind":"arxiv","version":2},"verdict":{"created_at":null,"id":null,"model_set":{},"one_line_summary":"","pipeline_version":null,"pith_extraction_headline":"","strongest_claim":"","weakest_assumption":""}},"verdict_id":null}}],"author_attestations":[],"timestamp_anchors":[],"storage_attestations":[],"citation_signatures":[],"replication_records":[],"corrections":[],"mirror_hints":[],"record_created":{"event_id":"sha256:bbcbab370d303a81f4a076cb9bd9d4f7c8c8587e05eb739878f666c9c03c6615","target":"record","created_at":"2026-07-05T04:56:16Z","signer":{"key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signer_id":"pith.science","signer_type":"pith_registry"},"payload":{"attestation_state":"computed","canonical_record":{"metadata":{"abstract_canon_sha256":"28057cb8d9be8d472b09e91cdc04fdfdddf3513c644bdd6cbd4fc925b6cfe292","cross_cats_sorted":["cs.AI"],"license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","primary_cat":"cs.CL","submitted_at":"2022-04-29T17:50:36Z","title_canon_sha256":"0b5c0087b45899654f007eb240613b9ed25127ec754b1b9005533da78022c025"},"schema_version":"1.0","source":{"id":"2204.14268","kind":"arxiv","version":2}},"canonical_sha256":"0e10655979c8083649d6388e17baa9d7b1b8bd600f0070554b57e2b8d9cef5f5","receipt":{"algorithm":"ed25519","builder_version":"pith-number-builder-2026-05-17-v1","canonical_sha256":"0e10655979c8083649d6388e17baa9d7b1b8bd600f0070554b57e2b8d9cef5f5","first_computed_at":"2026-07-05T04:56:16.950992Z","key_id":"pith-v1-2026-05","kind":"pith_receipt","last_reissued_at":"2026-07-05T04:56:16.950992Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","receipt_version":"0.3","signature_b64":"Vp/af0ijgzjwMbFLlJpr6EODn298rdpNzR263bXHCSoBCemnV6FezQqMq3V7MBkkerjae6ui1j7pRAQowUcwDg==","signature_status":"signed_v1","signed_at":"2026-07-05T04:56:16.951525Z","signed_message":"canonical_sha256_bytes"},"source_id":"2204.14268","source_kind":"arxiv","source_version":2}}},"equivocations":[],"invalid_events":[],"applied_event_ids":["sha256:bbcbab370d303a81f4a076cb9bd9d4f7c8c8587e05eb739878f666c9c03c6615","sha256:33d3d21763761790c0225a0691f129a5adac1e0ff06d734b6336ccc782ca89e7"],"state_sha256":"1de36df9bf6a2b7d53d6483f9bc9a987255f4fd0078c025075ee27562ab63330"},"bundle_signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"4yuC8CTbplTJ+2ZtNuRepz0lyJXx+Y15ZcuOaC9S/6C6bJFcSg94ObHVZdzdl1C9jiw0L4cW7dcAhbfOCL+NCA==","signed_message":"bundle_sha256_bytes","signed_at":"2026-08-03T20:11:02.123174Z","bundle_sha256":"827d1d7aeb03260bc016b2cdc7a5ed78ada4c833c7dfac1dd015039d2d58d712"}}