{"bundle_type":"pith_open_graph_bundle","bundle_version":"1.0","pith_number":"pith:2023:36CDGDZM3SAZSJ4RWZIJQ3HUKZ","short_pith_number":"pith:36CDGDZM","canonical_record":{"source":{"id":"2305.15425","kind":"arxiv","version":2},"metadata":{"license":"http://creativecommons.org/licenses/by-nc-sa/4.0/","primary_cat":"cs.CL","submitted_at":"2023-05-17T14:17:57Z","cross_cats_sorted":["cs.LG"],"title_canon_sha256":"cd458dff3fc0b0f9ef61dd2d6667b9b1547e9fc7919ea6c03c8a8c85dbf534f2","abstract_canon_sha256":"8f34be88f08495addf816d16fd57810182f3faa64b32fe155e8fb7a61a5b0dcb"},"schema_version":"1.0"},"canonical_sha256":"df84330f2cdc81992791b650986cf4565d636f1831a543c2d1360d45f496e32b","source":{"kind":"arxiv","id":"2305.15425","version":2},"source_aliases":[{"alias_kind":"arxiv","alias_value":"2305.15425","created_at":"2026-07-05T07:02:50Z"},{"alias_kind":"arxiv_version","alias_value":"2305.15425v2","created_at":"2026-07-05T07:02:50Z"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.2305.15425","created_at":"2026-07-05T07:02:50Z"},{"alias_kind":"pith_short_12","alias_value":"36CDGDZM3SAZ","created_at":"2026-07-05T07:02:50Z"},{"alias_kind":"pith_short_16","alias_value":"36CDGDZM3SAZSJ4R","created_at":"2026-07-05T07:02:50Z"},{"alias_kind":"pith_short_8","alias_value":"36CDGDZM","created_at":"2026-07-05T07:02:50Z"}],"events":[{"event_type":"record_created","subject_pith_number":"pith:2023:36CDGDZM3SAZSJ4RWZIJQ3HUKZ","target":"record","payload":{"canonical_record":{"source":{"id":"2305.15425","kind":"arxiv","version":2},"metadata":{"license":"http://creativecommons.org/licenses/by-nc-sa/4.0/","primary_cat":"cs.CL","submitted_at":"2023-05-17T14:17:57Z","cross_cats_sorted":["cs.LG"],"title_canon_sha256":"cd458dff3fc0b0f9ef61dd2d6667b9b1547e9fc7919ea6c03c8a8c85dbf534f2","abstract_canon_sha256":"8f34be88f08495addf816d16fd57810182f3faa64b32fe155e8fb7a61a5b0dcb"},"schema_version":"1.0"},"canonical_sha256":"df84330f2cdc81992791b650986cf4565d636f1831a543c2d1360d45f496e32b","receipt":{"kind":"pith_receipt","key_id":"pith-v1-2026-05","algorithm":"ed25519","signed_at":"2026-07-05T07:02:50.834951Z","signature_b64":"TFRVJTu+ucamPRV9y9V/o+Ve7+jCKaKr601oCN64FbGbd6CFnwAHL2l1LRkiknIbfwczPepmCtAAhi2UK/gCBw==","signed_message":"canonical_sha256_bytes","builder_version":"pith-number-builder-2026-05-17-v1","receipt_version":"0.3","canonical_sha256":"df84330f2cdc81992791b650986cf4565d636f1831a543c2d1360d45f496e32b","last_reissued_at":"2026-07-05T07:02:50.834446Z","signature_status":"signed_v1","first_computed_at":"2026-07-05T07:02:50.834446Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"source_kind":"arxiv","source_id":"2305.15425","source_version":2,"attestation_state":"computed"},"signer":{"signer_id":"pith.science","signer_type":"pith_registry","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"created_at":"2026-07-05T07:02:50Z","supersedes":[],"prev_event":null,"signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"rf0Do0RR6fPnok7QksWHEcBb1JeW78oh8K1HsTjI+JHQo1/sdwAD6apu65EESfLXU/TY4EZyZXAMWEt3uel/CA==","signed_message":"open_graph_event_sha256_bytes","signed_at":"2026-08-08T10:40:18.695342Z"},"content_sha256":"ba3d44f27c825b2ce9f1cf7d536d7d846b591d4ffb12de50301443995105ab6f","schema_version":"1.0","event_id":"sha256:ba3d44f27c825b2ce9f1cf7d536d7d846b591d4ffb12de50301443995105ab6f"},{"event_type":"graph_snapshot","subject_pith_number":"pith:2023:36CDGDZM3SAZSJ4RWZIJQ3HUKZ","target":"graph","payload":{"graph_snapshot":{"paper":{"title":"Language Model Tokenizers Introduce Unfairness Between Languages","license":"http://creativecommons.org/licenses/by-nc-sa/4.0/","headline":"","cross_cats":["cs.LG"],"primary_cat":"cs.CL","authors_text":"Adel Bibi, Aleksandar Petrov, Emanuele La Malfa, Philip H.S. Torr","submitted_at":"2023-05-17T14:17:57Z","abstract_excerpt":"Recent language models have shown impressive multilingual performance, even when not explicitly trained for it. Despite this, there are concerns about the quality of their outputs across different languages. In this paper, we show how disparity in the treatment of different languages arises at the tokenization stage, well before a model is even invoked. The same text translated into different languages can have drastically different tokenization lengths, with differences up to 15 times in some cases. These disparities persist even for tokenizers that are intentionally trained for multilingual "},"claims":{"count":0,"items":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"2305.15425","kind":"arxiv","version":2},"verdict":{"id":null,"model_set":{},"created_at":null,"strongest_claim":"","one_line_summary":"","pipeline_version":null,"weakest_assumption":"","pith_extraction_headline":""},"integrity":{"clean":true,"summary":{"advisory":0,"critical":0,"by_detector":{},"informational":0},"endpoint":"/pith/2305.15425/integrity.json","findings":[],"available":true,"detectors_run":[],"snapshot_sha256":"c28c3603d3b5d939e8dc4c7e95fa8dfce3d595e45f758748cecf8e644a296938"},"references":{"count":0,"sample":[],"resolved_work":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","internal_anchors":0},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"author_claims":{"count":0,"strong_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"builder_version":"pith-number-builder-2026-05-17-v1"},"verdict_id":null},"signer":{"signer_id":"pith.science","signer_type":"pith_registry","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"created_at":"2026-07-05T07:02:50Z","supersedes":[],"prev_event":null,"signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"kE1zOKTNNsanqZonbNge0fvSkHXHpK7F9yhPomEGiiGDvtnRFvRQPpMiSFltAoByIindyrODfFeY0OWf7R8DAA==","signed_message":"open_graph_event_sha256_bytes","signed_at":"2026-08-08T10:40:18.695946Z"},"content_sha256":"81626f4a93797f107ddf5412741f7ff1af1f078727ecd94ee725080698acb6f8","schema_version":"1.0","event_id":"sha256:81626f4a93797f107ddf5412741f7ff1af1f078727ecd94ee725080698acb6f8"}],"timestamp_proofs":[],"mirror_hints":[{"mirror_type":"https","name":"Pith Resolver","base_url":"https://pith.science","bundle_url":"https://pith.science/pith/36CDGDZM3SAZSJ4RWZIJQ3HUKZ/bundle.json","state_url":"https://pith.science/pith/36CDGDZM3SAZSJ4RWZIJQ3HUKZ/state.json","well_known_bundle_url":"https://pith.science/.well-known/pith/36CDGDZM3SAZSJ4RWZIJQ3HUKZ/bundle.json","status":"primary"}],"public_keys":[{"key_id":"pith-v1-2026-05","algorithm":"ed25519","format":"raw","public_key_b64":"stVStoiQhXFxp4s2pdzPNoqVNBMojDU/fJ2db5S3CbM=","public_key_hex":"b2d552b68890857171a78b36a5dccf368a953413288c353f7c9d9d6f94b709b3","fingerprint_sha256_b32_first128bits":"RVFV5Z2OI2J3ZUO7ERDEBCYNKS","fingerprint_sha256_hex":"8d4b5ee74e4693bcd1df2446408b0d54","rotates_at":null,"url":"https://pith.science/pith-signing-key.json","notes":"Pith uses this Ed25519 key to sign canonical record SHA-256 digests. Verify with: ed25519_verify(public_key, message=canonical_sha256_bytes, signature=base64decode(signature_b64))."}],"merge_version":"pith-open-graph-merge-v1","built_at":"2026-08-08T10:40:18Z","links":{"resolver":"https://pith.science/pith/36CDGDZM3SAZSJ4RWZIJQ3HUKZ","bundle":"https://pith.science/pith/36CDGDZM3SAZSJ4RWZIJQ3HUKZ/bundle.json","state":"https://pith.science/pith/36CDGDZM3SAZSJ4RWZIJQ3HUKZ/state.json","well_known_bundle":"https://pith.science/.well-known/pith/36CDGDZM3SAZSJ4RWZIJQ3HUKZ/bundle.json"},"state":{"state_type":"pith_open_graph_state","state_version":"1.0","pith_number":"pith:2023:36CDGDZM3SAZSJ4RWZIJQ3HUKZ","merge_version":"pith-open-graph-merge-v1","event_count":2,"valid_event_count":2,"invalid_event_count":0,"equivocation_count":0,"current":{"canonical_record":{"metadata":{"abstract_canon_sha256":"8f34be88f08495addf816d16fd57810182f3faa64b32fe155e8fb7a61a5b0dcb","cross_cats_sorted":["cs.LG"],"license":"http://creativecommons.org/licenses/by-nc-sa/4.0/","primary_cat":"cs.CL","submitted_at":"2023-05-17T14:17:57Z","title_canon_sha256":"cd458dff3fc0b0f9ef61dd2d6667b9b1547e9fc7919ea6c03c8a8c85dbf534f2"},"schema_version":"1.0","source":{"id":"2305.15425","kind":"arxiv","version":2}},"source_aliases":[{"alias_kind":"arxiv","alias_value":"2305.15425","created_at":"2026-07-05T07:02:50Z"},{"alias_kind":"arxiv_version","alias_value":"2305.15425v2","created_at":"2026-07-05T07:02:50Z"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.2305.15425","created_at":"2026-07-05T07:02:50Z"},{"alias_kind":"pith_short_12","alias_value":"36CDGDZM3SAZ","created_at":"2026-07-05T07:02:50Z"},{"alias_kind":"pith_short_16","alias_value":"36CDGDZM3SAZSJ4R","created_at":"2026-07-05T07:02:50Z"},{"alias_kind":"pith_short_8","alias_value":"36CDGDZM","created_at":"2026-07-05T07:02:50Z"}],"graph_snapshots":[{"event_id":"sha256:81626f4a93797f107ddf5412741f7ff1af1f078727ecd94ee725080698acb6f8","target":"graph","created_at":"2026-07-05T07:02:50Z","signer":{"key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signer_id":"pith.science","signer_type":"pith_registry"},"payload":{"graph_snapshot":{"author_claims":{"count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","strong_count":0},"builder_version":"pith-number-builder-2026-05-17-v1","claims":{"count":0,"items":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"integrity":{"available":true,"clean":true,"detectors_run":[],"endpoint":"/pith/2305.15425/integrity.json","findings":[],"snapshot_sha256":"c28c3603d3b5d939e8dc4c7e95fa8dfce3d595e45f758748cecf8e644a296938","summary":{"advisory":0,"by_detector":{},"critical":0,"informational":0}},"paper":{"abstract_excerpt":"Recent language models have shown impressive multilingual performance, even when not explicitly trained for it. Despite this, there are concerns about the quality of their outputs across different languages. In this paper, we show how disparity in the treatment of different languages arises at the tokenization stage, well before a model is even invoked. The same text translated into different languages can have drastically different tokenization lengths, with differences up to 15 times in some cases. These disparities persist even for tokenizers that are intentionally trained for multilingual ","authors_text":"Adel Bibi, Aleksandar Petrov, Emanuele La Malfa, Philip H.S. Torr","cross_cats":["cs.LG"],"headline":"","license":"http://creativecommons.org/licenses/by-nc-sa/4.0/","primary_cat":"cs.CL","submitted_at":"2023-05-17T14:17:57Z","title":"Language Model Tokenizers Introduce Unfairness Between Languages"},"references":{"count":0,"internal_anchors":0,"resolved_work":0,"sample":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"2305.15425","kind":"arxiv","version":2},"verdict":{"created_at":null,"id":null,"model_set":{},"one_line_summary":"","pipeline_version":null,"pith_extraction_headline":"","strongest_claim":"","weakest_assumption":""}},"verdict_id":null}}],"author_attestations":[],"timestamp_anchors":[],"storage_attestations":[],"citation_signatures":[],"replication_records":[],"corrections":[],"mirror_hints":[],"record_created":{"event_id":"sha256:ba3d44f27c825b2ce9f1cf7d536d7d846b591d4ffb12de50301443995105ab6f","target":"record","created_at":"2026-07-05T07:02:50Z","signer":{"key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signer_id":"pith.science","signer_type":"pith_registry"},"payload":{"attestation_state":"computed","canonical_record":{"metadata":{"abstract_canon_sha256":"8f34be88f08495addf816d16fd57810182f3faa64b32fe155e8fb7a61a5b0dcb","cross_cats_sorted":["cs.LG"],"license":"http://creativecommons.org/licenses/by-nc-sa/4.0/","primary_cat":"cs.CL","submitted_at":"2023-05-17T14:17:57Z","title_canon_sha256":"cd458dff3fc0b0f9ef61dd2d6667b9b1547e9fc7919ea6c03c8a8c85dbf534f2"},"schema_version":"1.0","source":{"id":"2305.15425","kind":"arxiv","version":2}},"canonical_sha256":"df84330f2cdc81992791b650986cf4565d636f1831a543c2d1360d45f496e32b","receipt":{"algorithm":"ed25519","builder_version":"pith-number-builder-2026-05-17-v1","canonical_sha256":"df84330f2cdc81992791b650986cf4565d636f1831a543c2d1360d45f496e32b","first_computed_at":"2026-07-05T07:02:50.834446Z","key_id":"pith-v1-2026-05","kind":"pith_receipt","last_reissued_at":"2026-07-05T07:02:50.834446Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","receipt_version":"0.3","signature_b64":"TFRVJTu+ucamPRV9y9V/o+Ve7+jCKaKr601oCN64FbGbd6CFnwAHL2l1LRkiknIbfwczPepmCtAAhi2UK/gCBw==","signature_status":"signed_v1","signed_at":"2026-07-05T07:02:50.834951Z","signed_message":"canonical_sha256_bytes"},"source_id":"2305.15425","source_kind":"arxiv","source_version":2}}},"equivocations":[],"invalid_events":[],"applied_event_ids":["sha256:ba3d44f27c825b2ce9f1cf7d536d7d846b591d4ffb12de50301443995105ab6f","sha256:81626f4a93797f107ddf5412741f7ff1af1f078727ecd94ee725080698acb6f8"],"state_sha256":"78d3386295b9a6d9fabf8c7607ccccc14196ae04e1c8164c0dc2ecb4d651ea9b"},"bundle_signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"x7XJgCbtTgl1vwEVwcz6Z7+16jGAalRHeOicBmQrC8D9vn87uGUd0PF2CK25FV6eNK1aOajzwb5gvMTUUc38BA==","signed_message":"bundle_sha256_bytes","signed_at":"2026-08-08T10:40:18.701480Z","bundle_sha256":"90669a3d2779726896cb85da4dd686bc2f8defdcde737dcff9f209f10221d571"}}