{"bundle_type":"pith_open_graph_bundle","bundle_version":"1.0","pith_number":"pith:2024:PJ46MQBAPZUOTHJML4LJ46H4AP","short_pith_number":"pith:PJ46MQBA","canonical_record":{"source":{"id":"2406.11214","kind":"arxiv","version":3},"metadata":{"license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.CL","submitted_at":"2024-06-17T05:13:25Z","cross_cats_sorted":[],"title_canon_sha256":"06536af1412fb6a9d685b5de529e51ab1a33a85b79348235f623c6dddb08ad74","abstract_canon_sha256":"6488332b52fbae82b66f86f9eb7978b5168c59dbb6b369fdf9f7015334afdf35"},"schema_version":"1.0"},"canonical_sha256":"7a79e640207e68e99d2c5f169e78fc03d0ad20b513ac368d6af444b33309019b","source":{"kind":"arxiv","id":"2406.11214","version":3},"source_aliases":[{"alias_kind":"arxiv","alias_value":"2406.11214","created_at":"2026-07-05T09:35:10Z"},{"alias_kind":"arxiv_version","alias_value":"2406.11214v3","created_at":"2026-07-05T09:35:10Z"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.2406.11214","created_at":"2026-07-05T09:35:10Z"},{"alias_kind":"pith_short_12","alias_value":"PJ46MQBAPZUO","created_at":"2026-07-05T09:35:10Z"},{"alias_kind":"pith_short_16","alias_value":"PJ46MQBAPZUOTHJM","created_at":"2026-07-05T09:35:10Z"},{"alias_kind":"pith_short_8","alias_value":"PJ46MQBA","created_at":"2026-07-05T09:35:10Z"}],"events":[{"event_type":"record_created","subject_pith_number":"pith:2024:PJ46MQBAPZUOTHJML4LJ46H4AP","target":"record","payload":{"canonical_record":{"source":{"id":"2406.11214","kind":"arxiv","version":3},"metadata":{"license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.CL","submitted_at":"2024-06-17T05:13:25Z","cross_cats_sorted":[],"title_canon_sha256":"06536af1412fb6a9d685b5de529e51ab1a33a85b79348235f623c6dddb08ad74","abstract_canon_sha256":"6488332b52fbae82b66f86f9eb7978b5168c59dbb6b369fdf9f7015334afdf35"},"schema_version":"1.0"},"canonical_sha256":"7a79e640207e68e99d2c5f169e78fc03d0ad20b513ac368d6af444b33309019b","receipt":{"kind":"pith_receipt","key_id":"pith-v1-2026-05","algorithm":"ed25519","signed_at":"2026-07-05T09:35:10.613469Z","signature_b64":"nQ6VxoHkABsJnPgFouIax4RbEqQvCyjhzWYKLkJmh9zNkouyQjc6DqDazQ4liDmHZhRDEFUDrYT4pyiaS/+2Bg==","signed_message":"canonical_sha256_bytes","builder_version":"pith-number-builder-2026-05-17-v1","receipt_version":"0.3","canonical_sha256":"7a79e640207e68e99d2c5f169e78fc03d0ad20b513ac368d6af444b33309019b","last_reissued_at":"2026-07-05T09:35:10.613008Z","signature_status":"signed_v1","first_computed_at":"2026-07-05T09:35:10.613008Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"source_kind":"arxiv","source_id":"2406.11214","source_version":3,"attestation_state":"computed"},"signer":{"signer_id":"pith.science","signer_type":"pith_registry","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"created_at":"2026-07-05T09:35:10Z","supersedes":[],"prev_event":null,"signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"QCg1jrYjgGuPbk1l5A3JSfprfu0QU95xXPu7CJfiXIZPItXvxOOyj6pppA04Sx9mGgH936e0CNhFiLK2DnrsDg==","signed_message":"open_graph_event_sha256_bytes","signed_at":"2026-08-06T22:40:16.384131Z"},"content_sha256":"c772efb6f056f4cc7a1586fd02a68fdcd82698c1950623645132c93f1ad4a858","schema_version":"1.0","event_id":"sha256:c772efb6f056f4cc7a1586fd02a68fdcd82698c1950623645132c93f1ad4a858"},{"event_type":"graph_snapshot","subject_pith_number":"pith:2024:PJ46MQBAPZUOTHJML4LJ46H4AP","target":"graph","payload":{"graph_snapshot":{"paper":{"title":"Problematic Tokens: Tokenizer Bias in Large Language Models","license":"http://creativecommons.org/licenses/by/4.0/","headline":"","cross_cats":[],"primary_cat":"cs.CL","authors_text":"Jin Yang, Yanbin Lin, Zhiqiang Wang, Zunduo Zhao","submitted_at":"2024-06-17T05:13:25Z","abstract_excerpt":"Recent advancements in large language models(LLMs), such as GPT-4 and GPT-4o, have shown exceptional performance, especially in languages with abundant resources like English, thanks to extensive datasets that ensure robust training. Conversely, these models exhibit limitations when processing under-resourced languages such as Chinese and Korean, where issues including hallucinatory responses remain prevalent. This paper traces the roots of these disparities to the tokenization process inherent to these models. Specifically, it explores how the tokenizers vocabulary, often used to speed up the"},"claims":{"count":0,"items":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"2406.11214","kind":"arxiv","version":3},"verdict":{"id":null,"model_set":{},"created_at":null,"strongest_claim":"","one_line_summary":"","pipeline_version":null,"weakest_assumption":"","pith_extraction_headline":""},"integrity":{"clean":true,"summary":{"advisory":0,"critical":0,"by_detector":{},"informational":0},"endpoint":"/pith/2406.11214/integrity.json","findings":[],"available":true,"detectors_run":[],"snapshot_sha256":"c28c3603d3b5d939e8dc4c7e95fa8dfce3d595e45f758748cecf8e644a296938"},"references":{"count":0,"sample":[],"resolved_work":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","internal_anchors":0},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"author_claims":{"count":0,"strong_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"builder_version":"pith-number-builder-2026-05-17-v1"},"verdict_id":null},"signer":{"signer_id":"pith.science","signer_type":"pith_registry","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"created_at":"2026-07-05T09:35:10Z","supersedes":[],"prev_event":null,"signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"8X0N6KODaRbM+LbBUw2DqEbmBIkpZVLs78ZHXN3JUfNwww63pQzNAujlNNd59JOErSpWOJMu1UXx28xIkiu6DQ==","signed_message":"open_graph_event_sha256_bytes","signed_at":"2026-08-06T22:40:16.384643Z"},"content_sha256":"fe27a6085fa8afbf663411632bf67d52f85d8d8b25dd7d1b637a3d81bc8f852e","schema_version":"1.0","event_id":"sha256:fe27a6085fa8afbf663411632bf67d52f85d8d8b25dd7d1b637a3d81bc8f852e"}],"timestamp_proofs":[],"mirror_hints":[{"mirror_type":"https","name":"Pith Resolver","base_url":"https://pith.science","bundle_url":"https://pith.science/pith/PJ46MQBAPZUOTHJML4LJ46H4AP/bundle.json","state_url":"https://pith.science/pith/PJ46MQBAPZUOTHJML4LJ46H4AP/state.json","well_known_bundle_url":"https://pith.science/.well-known/pith/PJ46MQBAPZUOTHJML4LJ46H4AP/bundle.json","status":"primary"}],"public_keys":[{"key_id":"pith-v1-2026-05","algorithm":"ed25519","format":"raw","public_key_b64":"stVStoiQhXFxp4s2pdzPNoqVNBMojDU/fJ2db5S3CbM=","public_key_hex":"b2d552b68890857171a78b36a5dccf368a953413288c353f7c9d9d6f94b709b3","fingerprint_sha256_b32_first128bits":"RVFV5Z2OI2J3ZUO7ERDEBCYNKS","fingerprint_sha256_hex":"8d4b5ee74e4693bcd1df2446408b0d54","rotates_at":null,"url":"https://pith.science/pith-signing-key.json","notes":"Pith uses this Ed25519 key to sign canonical record SHA-256 digests. Verify with: ed25519_verify(public_key, message=canonical_sha256_bytes, signature=base64decode(signature_b64))."}],"merge_version":"pith-open-graph-merge-v1","built_at":"2026-08-06T22:40:16Z","links":{"resolver":"https://pith.science/pith/PJ46MQBAPZUOTHJML4LJ46H4AP","bundle":"https://pith.science/pith/PJ46MQBAPZUOTHJML4LJ46H4AP/bundle.json","state":"https://pith.science/pith/PJ46MQBAPZUOTHJML4LJ46H4AP/state.json","well_known_bundle":"https://pith.science/.well-known/pith/PJ46MQBAPZUOTHJML4LJ46H4AP/bundle.json"},"state":{"state_type":"pith_open_graph_state","state_version":"1.0","pith_number":"pith:2024:PJ46MQBAPZUOTHJML4LJ46H4AP","merge_version":"pith-open-graph-merge-v1","event_count":2,"valid_event_count":2,"invalid_event_count":0,"equivocation_count":0,"current":{"canonical_record":{"metadata":{"abstract_canon_sha256":"6488332b52fbae82b66f86f9eb7978b5168c59dbb6b369fdf9f7015334afdf35","cross_cats_sorted":[],"license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.CL","submitted_at":"2024-06-17T05:13:25Z","title_canon_sha256":"06536af1412fb6a9d685b5de529e51ab1a33a85b79348235f623c6dddb08ad74"},"schema_version":"1.0","source":{"id":"2406.11214","kind":"arxiv","version":3}},"source_aliases":[{"alias_kind":"arxiv","alias_value":"2406.11214","created_at":"2026-07-05T09:35:10Z"},{"alias_kind":"arxiv_version","alias_value":"2406.11214v3","created_at":"2026-07-05T09:35:10Z"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.2406.11214","created_at":"2026-07-05T09:35:10Z"},{"alias_kind":"pith_short_12","alias_value":"PJ46MQBAPZUO","created_at":"2026-07-05T09:35:10Z"},{"alias_kind":"pith_short_16","alias_value":"PJ46MQBAPZUOTHJM","created_at":"2026-07-05T09:35:10Z"},{"alias_kind":"pith_short_8","alias_value":"PJ46MQBA","created_at":"2026-07-05T09:35:10Z"}],"graph_snapshots":[{"event_id":"sha256:fe27a6085fa8afbf663411632bf67d52f85d8d8b25dd7d1b637a3d81bc8f852e","target":"graph","created_at":"2026-07-05T09:35:10Z","signer":{"key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signer_id":"pith.science","signer_type":"pith_registry"},"payload":{"graph_snapshot":{"author_claims":{"count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","strong_count":0},"builder_version":"pith-number-builder-2026-05-17-v1","claims":{"count":0,"items":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"integrity":{"available":true,"clean":true,"detectors_run":[],"endpoint":"/pith/2406.11214/integrity.json","findings":[],"snapshot_sha256":"c28c3603d3b5d939e8dc4c7e95fa8dfce3d595e45f758748cecf8e644a296938","summary":{"advisory":0,"by_detector":{},"critical":0,"informational":0}},"paper":{"abstract_excerpt":"Recent advancements in large language models(LLMs), such as GPT-4 and GPT-4o, have shown exceptional performance, especially in languages with abundant resources like English, thanks to extensive datasets that ensure robust training. Conversely, these models exhibit limitations when processing under-resourced languages such as Chinese and Korean, where issues including hallucinatory responses remain prevalent. This paper traces the roots of these disparities to the tokenization process inherent to these models. Specifically, it explores how the tokenizers vocabulary, often used to speed up the","authors_text":"Jin Yang, Yanbin Lin, Zhiqiang Wang, Zunduo Zhao","cross_cats":[],"headline":"","license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.CL","submitted_at":"2024-06-17T05:13:25Z","title":"Problematic Tokens: Tokenizer Bias in Large Language Models"},"references":{"count":0,"internal_anchors":0,"resolved_work":0,"sample":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"2406.11214","kind":"arxiv","version":3},"verdict":{"created_at":null,"id":null,"model_set":{},"one_line_summary":"","pipeline_version":null,"pith_extraction_headline":"","strongest_claim":"","weakest_assumption":""}},"verdict_id":null}}],"author_attestations":[],"timestamp_anchors":[],"storage_attestations":[],"citation_signatures":[],"replication_records":[],"corrections":[],"mirror_hints":[],"record_created":{"event_id":"sha256:c772efb6f056f4cc7a1586fd02a68fdcd82698c1950623645132c93f1ad4a858","target":"record","created_at":"2026-07-05T09:35:10Z","signer":{"key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signer_id":"pith.science","signer_type":"pith_registry"},"payload":{"attestation_state":"computed","canonical_record":{"metadata":{"abstract_canon_sha256":"6488332b52fbae82b66f86f9eb7978b5168c59dbb6b369fdf9f7015334afdf35","cross_cats_sorted":[],"license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.CL","submitted_at":"2024-06-17T05:13:25Z","title_canon_sha256":"06536af1412fb6a9d685b5de529e51ab1a33a85b79348235f623c6dddb08ad74"},"schema_version":"1.0","source":{"id":"2406.11214","kind":"arxiv","version":3}},"canonical_sha256":"7a79e640207e68e99d2c5f169e78fc03d0ad20b513ac368d6af444b33309019b","receipt":{"algorithm":"ed25519","builder_version":"pith-number-builder-2026-05-17-v1","canonical_sha256":"7a79e640207e68e99d2c5f169e78fc03d0ad20b513ac368d6af444b33309019b","first_computed_at":"2026-07-05T09:35:10.613008Z","key_id":"pith-v1-2026-05","kind":"pith_receipt","last_reissued_at":"2026-07-05T09:35:10.613008Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","receipt_version":"0.3","signature_b64":"nQ6VxoHkABsJnPgFouIax4RbEqQvCyjhzWYKLkJmh9zNkouyQjc6DqDazQ4liDmHZhRDEFUDrYT4pyiaS/+2Bg==","signature_status":"signed_v1","signed_at":"2026-07-05T09:35:10.613469Z","signed_message":"canonical_sha256_bytes"},"source_id":"2406.11214","source_kind":"arxiv","source_version":3}}},"equivocations":[],"invalid_events":[],"applied_event_ids":["sha256:c772efb6f056f4cc7a1586fd02a68fdcd82698c1950623645132c93f1ad4a858","sha256:fe27a6085fa8afbf663411632bf67d52f85d8d8b25dd7d1b637a3d81bc8f852e"],"state_sha256":"f5dc13e041d728f15399faed0e1860be4342121080d1f3c7f042414e15e497a5"},"bundle_signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"P6fns+gItmyxCic+zl4k2ii3bISYaRpjUs7p0TwjnwZrX/oppisqf7plSMhGdtO5eA7wqkKogVZvyUzciJTGDQ==","signed_message":"bundle_sha256_bytes","signed_at":"2026-08-06T22:40:16.390195Z","bundle_sha256":"3f7b23967327e6be5743bec733ac1a1590d406c78ce9025b1829d284380b8750"}}