{"bundle_type":"pith_open_graph_bundle","bundle_version":"1.0","pith_number":"pith:2024:FG5GIIAZSUKTARA6H76SN774A6","short_pith_number":"pith:FG5GIIAZ","canonical_record":{"source":{"id":"2406.11687","kind":"arxiv","version":3},"metadata":{"license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","primary_cat":"cs.CL","submitted_at":"2024-06-17T16:05:32Z","cross_cats_sorted":[],"title_canon_sha256":"b5f9eec2260d99340d3a65d1442a8b75a433a71980d206ac57b33429d138b3a6","abstract_canon_sha256":"50547f13d50628f7782762ca8e44cfc3d5bce591085798f02437335deabb00df"},"schema_version":"1.0"},"canonical_sha256":"29ba642019951530441e3ffd26fffc07bb904c149c343ee7afafe0cd7f10fba3","source":{"kind":"arxiv","id":"2406.11687","version":3},"source_aliases":[{"alias_kind":"arxiv","alias_value":"2406.11687","created_at":"2026-07-05T09:15:58Z"},{"alias_kind":"arxiv_version","alias_value":"2406.11687v3","created_at":"2026-07-05T09:15:58Z"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.2406.11687","created_at":"2026-07-05T09:15:58Z"},{"alias_kind":"pith_short_12","alias_value":"FG5GIIAZSUKT","created_at":"2026-07-05T09:15:58Z"},{"alias_kind":"pith_short_16","alias_value":"FG5GIIAZSUKTARA6","created_at":"2026-07-05T09:15:58Z"},{"alias_kind":"pith_short_8","alias_value":"FG5GIIAZ","created_at":"2026-07-05T09:15:58Z"}],"events":[{"event_type":"record_created","subject_pith_number":"pith:2024:FG5GIIAZSUKTARA6H76SN774A6","target":"record","payload":{"canonical_record":{"source":{"id":"2406.11687","kind":"arxiv","version":3},"metadata":{"license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","primary_cat":"cs.CL","submitted_at":"2024-06-17T16:05:32Z","cross_cats_sorted":[],"title_canon_sha256":"b5f9eec2260d99340d3a65d1442a8b75a433a71980d206ac57b33429d138b3a6","abstract_canon_sha256":"50547f13d50628f7782762ca8e44cfc3d5bce591085798f02437335deabb00df"},"schema_version":"1.0"},"canonical_sha256":"29ba642019951530441e3ffd26fffc07bb904c149c343ee7afafe0cd7f10fba3","receipt":{"kind":"pith_receipt","key_id":"pith-v1-2026-05","algorithm":"ed25519","signed_at":"2026-07-05T09:15:58.372361Z","signature_b64":"Q4RFaI6SHlxabwtYT+Xw59sG+jCC8CeMQEulhhly5TaZY2fQqwp47Z0fwOv0zPit7F/u/m9tg5nq6LWg6x8eBw==","signed_message":"canonical_sha256_bytes","builder_version":"pith-number-builder-2026-05-17-v1","receipt_version":"0.3","canonical_sha256":"29ba642019951530441e3ffd26fffc07bb904c149c343ee7afafe0cd7f10fba3","last_reissued_at":"2026-07-05T09:15:58.371913Z","signature_status":"signed_v1","first_computed_at":"2026-07-05T09:15:58.371913Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"source_kind":"arxiv","source_id":"2406.11687","source_version":3,"attestation_state":"computed"},"signer":{"signer_id":"pith.science","signer_type":"pith_registry","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"created_at":"2026-07-05T09:15:58Z","supersedes":[],"prev_event":null,"signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"ocHEJUsNBUC2wK+cVjt7AEzY+iU7iIbWzrUiisRshOx2iXwOWdLU/y0MPMhvI+IRRxPUR0+TaQzBFL7CZc0dAg==","signed_message":"open_graph_event_sha256_bytes","signed_at":"2026-07-07T15:12:43.104763Z"},"content_sha256":"e63fec8fe22d0d9865ef7440c1cffdc409cf151e032232b41b52b79e1c97d869","schema_version":"1.0","event_id":"sha256:e63fec8fe22d0d9865ef7440c1cffdc409cf151e032232b41b52b79e1c97d869"},{"event_type":"graph_snapshot","subject_pith_number":"pith:2024:FG5GIIAZSUKTARA6H76SN774A6","target":"graph","payload":{"graph_snapshot":{"paper":{"title":"Tokenization Falling Short: On Subword Robustness in Large Language Models","license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","headline":"","cross_cats":[],"primary_cat":"cs.CL","authors_text":"Qiwei Peng, Xuhong Li, Yekun Chai, Yewei Fang","submitted_at":"2024-06-17T16:05:32Z","abstract_excerpt":"Language models typically tokenize raw text into sequences of subword identifiers from a predefined vocabulary, a process inherently sensitive to typographical errors, length variations, and largely oblivious to the internal structure of tokens--issues we term the curse of tokenization. In this study, we delve into these drawbacks and demonstrate that large language models (LLMs) remain susceptible to these problems. This study systematically investigates these challenges and their impact on LLMs through three critical research questions: (1) complex problem solving, (2) token structure probin"},"claims":{"count":0,"items":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"2406.11687","kind":"arxiv","version":3},"verdict":{"id":null,"model_set":{},"created_at":null,"strongest_claim":"","one_line_summary":"","pipeline_version":null,"weakest_assumption":"","pith_extraction_headline":""},"integrity":{"clean":true,"summary":{"advisory":0,"critical":0,"by_detector":{},"informational":0},"endpoint":"/pith/2406.11687/integrity.json","findings":[],"available":true,"detectors_run":[],"snapshot_sha256":"c28c3603d3b5d939e8dc4c7e95fa8dfce3d595e45f758748cecf8e644a296938"},"references":{"count":0,"sample":[],"resolved_work":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","internal_anchors":0},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"author_claims":{"count":0,"strong_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"builder_version":"pith-number-builder-2026-05-17-v1"},"verdict_id":null},"signer":{"signer_id":"pith.science","signer_type":"pith_registry","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"created_at":"2026-07-05T09:15:58Z","supersedes":[],"prev_event":null,"signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"ft5hLlsjmQvRX/87U6kWxIJbxgOVS3Fb7oLgIE9TmIP7qXUiQ52XELLV49DGFcsjJTyBNn9bf8J9BUh4jA4EDg==","signed_message":"open_graph_event_sha256_bytes","signed_at":"2026-07-07T15:12:43.105209Z"},"content_sha256":"29ba7b56ba90c3b20e4424939587dffe3513dd687e0ea6112c510360eb4fc210","schema_version":"1.0","event_id":"sha256:29ba7b56ba90c3b20e4424939587dffe3513dd687e0ea6112c510360eb4fc210"}],"timestamp_proofs":[],"mirror_hints":[{"mirror_type":"https","name":"Pith Resolver","base_url":"https://pith.science","bundle_url":"https://pith.science/pith/FG5GIIAZSUKTARA6H76SN774A6/bundle.json","state_url":"https://pith.science/pith/FG5GIIAZSUKTARA6H76SN774A6/state.json","well_known_bundle_url":"https://pith.science/.well-known/pith/FG5GIIAZSUKTARA6H76SN774A6/bundle.json","status":"primary"}],"public_keys":[{"key_id":"pith-v1-2026-05","algorithm":"ed25519","format":"raw","public_key_b64":"stVStoiQhXFxp4s2pdzPNoqVNBMojDU/fJ2db5S3CbM=","public_key_hex":"b2d552b68890857171a78b36a5dccf368a953413288c353f7c9d9d6f94b709b3","fingerprint_sha256_b32_first128bits":"RVFV5Z2OI2J3ZUO7ERDEBCYNKS","fingerprint_sha256_hex":"8d4b5ee74e4693bcd1df2446408b0d54","rotates_at":null,"url":"https://pith.science/pith-signing-key.json","notes":"Pith uses this Ed25519 key to sign canonical record SHA-256 digests. Verify with: ed25519_verify(public_key, message=canonical_sha256_bytes, signature=base64decode(signature_b64))."}],"merge_version":"pith-open-graph-merge-v1","built_at":"2026-07-07T15:12:43Z","links":{"resolver":"https://pith.science/pith/FG5GIIAZSUKTARA6H76SN774A6","bundle":"https://pith.science/pith/FG5GIIAZSUKTARA6H76SN774A6/bundle.json","state":"https://pith.science/pith/FG5GIIAZSUKTARA6H76SN774A6/state.json","well_known_bundle":"https://pith.science/.well-known/pith/FG5GIIAZSUKTARA6H76SN774A6/bundle.json"},"state":{"state_type":"pith_open_graph_state","state_version":"1.0","pith_number":"pith:2024:FG5GIIAZSUKTARA6H76SN774A6","merge_version":"pith-open-graph-merge-v1","event_count":2,"valid_event_count":2,"invalid_event_count":0,"equivocation_count":0,"current":{"canonical_record":{"metadata":{"abstract_canon_sha256":"50547f13d50628f7782762ca8e44cfc3d5bce591085798f02437335deabb00df","cross_cats_sorted":[],"license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","primary_cat":"cs.CL","submitted_at":"2024-06-17T16:05:32Z","title_canon_sha256":"b5f9eec2260d99340d3a65d1442a8b75a433a71980d206ac57b33429d138b3a6"},"schema_version":"1.0","source":{"id":"2406.11687","kind":"arxiv","version":3}},"source_aliases":[{"alias_kind":"arxiv","alias_value":"2406.11687","created_at":"2026-07-05T09:15:58Z"},{"alias_kind":"arxiv_version","alias_value":"2406.11687v3","created_at":"2026-07-05T09:15:58Z"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.2406.11687","created_at":"2026-07-05T09:15:58Z"},{"alias_kind":"pith_short_12","alias_value":"FG5GIIAZSUKT","created_at":"2026-07-05T09:15:58Z"},{"alias_kind":"pith_short_16","alias_value":"FG5GIIAZSUKTARA6","created_at":"2026-07-05T09:15:58Z"},{"alias_kind":"pith_short_8","alias_value":"FG5GIIAZ","created_at":"2026-07-05T09:15:58Z"}],"graph_snapshots":[{"event_id":"sha256:29ba7b56ba90c3b20e4424939587dffe3513dd687e0ea6112c510360eb4fc210","target":"graph","created_at":"2026-07-05T09:15:58Z","signer":{"key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signer_id":"pith.science","signer_type":"pith_registry"},"payload":{"graph_snapshot":{"author_claims":{"count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","strong_count":0},"builder_version":"pith-number-builder-2026-05-17-v1","claims":{"count":0,"items":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"integrity":{"available":true,"clean":true,"detectors_run":[],"endpoint":"/pith/2406.11687/integrity.json","findings":[],"snapshot_sha256":"c28c3603d3b5d939e8dc4c7e95fa8dfce3d595e45f758748cecf8e644a296938","summary":{"advisory":0,"by_detector":{},"critical":0,"informational":0}},"paper":{"abstract_excerpt":"Language models typically tokenize raw text into sequences of subword identifiers from a predefined vocabulary, a process inherently sensitive to typographical errors, length variations, and largely oblivious to the internal structure of tokens--issues we term the curse of tokenization. In this study, we delve into these drawbacks and demonstrate that large language models (LLMs) remain susceptible to these problems. This study systematically investigates these challenges and their impact on LLMs through three critical research questions: (1) complex problem solving, (2) token structure probin","authors_text":"Qiwei Peng, Xuhong Li, Yekun Chai, Yewei Fang","cross_cats":[],"headline":"","license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","primary_cat":"cs.CL","submitted_at":"2024-06-17T16:05:32Z","title":"Tokenization Falling Short: On Subword Robustness in Large Language Models"},"references":{"count":0,"internal_anchors":0,"resolved_work":0,"sample":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"2406.11687","kind":"arxiv","version":3},"verdict":{"created_at":null,"id":null,"model_set":{},"one_line_summary":"","pipeline_version":null,"pith_extraction_headline":"","strongest_claim":"","weakest_assumption":""}},"verdict_id":null}}],"author_attestations":[],"timestamp_anchors":[],"storage_attestations":[],"citation_signatures":[],"replication_records":[],"corrections":[],"mirror_hints":[],"record_created":{"event_id":"sha256:e63fec8fe22d0d9865ef7440c1cffdc409cf151e032232b41b52b79e1c97d869","target":"record","created_at":"2026-07-05T09:15:58Z","signer":{"key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signer_id":"pith.science","signer_type":"pith_registry"},"payload":{"attestation_state":"computed","canonical_record":{"metadata":{"abstract_canon_sha256":"50547f13d50628f7782762ca8e44cfc3d5bce591085798f02437335deabb00df","cross_cats_sorted":[],"license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","primary_cat":"cs.CL","submitted_at":"2024-06-17T16:05:32Z","title_canon_sha256":"b5f9eec2260d99340d3a65d1442a8b75a433a71980d206ac57b33429d138b3a6"},"schema_version":"1.0","source":{"id":"2406.11687","kind":"arxiv","version":3}},"canonical_sha256":"29ba642019951530441e3ffd26fffc07bb904c149c343ee7afafe0cd7f10fba3","receipt":{"algorithm":"ed25519","builder_version":"pith-number-builder-2026-05-17-v1","canonical_sha256":"29ba642019951530441e3ffd26fffc07bb904c149c343ee7afafe0cd7f10fba3","first_computed_at":"2026-07-05T09:15:58.371913Z","key_id":"pith-v1-2026-05","kind":"pith_receipt","last_reissued_at":"2026-07-05T09:15:58.371913Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","receipt_version":"0.3","signature_b64":"Q4RFaI6SHlxabwtYT+Xw59sG+jCC8CeMQEulhhly5TaZY2fQqwp47Z0fwOv0zPit7F/u/m9tg5nq6LWg6x8eBw==","signature_status":"signed_v1","signed_at":"2026-07-05T09:15:58.372361Z","signed_message":"canonical_sha256_bytes"},"source_id":"2406.11687","source_kind":"arxiv","source_version":3}}},"equivocations":[],"invalid_events":[],"applied_event_ids":["sha256:e63fec8fe22d0d9865ef7440c1cffdc409cf151e032232b41b52b79e1c97d869","sha256:29ba7b56ba90c3b20e4424939587dffe3513dd687e0ea6112c510360eb4fc210"],"state_sha256":"e84110622f51b09f852c5b02d52bb77f4da208fd05f70cad6d553ed1afc9d56a"},"bundle_signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"LXVUOW6svGLnwFANX6hPLxLHS9xs8W7F3QrymGSFyZOB4zv1sdd4LmvhtB8JvGNKavMgYikjIXfsheoppddaDg==","signed_message":"bundle_sha256_bytes","signed_at":"2026-07-07T15:12:43.107089Z","bundle_sha256":"9cc86284934bb0b98e37d35c7b3169415a780f6ef4cefa235e58fd8b60f13239"}}