{"bundle_type":"pith_open_graph_bundle","bundle_version":"1.0","pith_number":"pith:2024:7UJPNLXOUUOCUKCXF36D3KCEDH","short_pith_number":"pith:7UJPNLXO","canonical_record":{"source":{"id":"2405.17067","kind":"arxiv","version":2},"metadata":{"license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","primary_cat":"cs.CL","submitted_at":"2024-05-27T11:39:59Z","cross_cats_sorted":["cs.AI"],"title_canon_sha256":"e0e685a19accbb37a09fe62f81e1bbd448294e5a0a8ba380fc386f1229a8b230","abstract_canon_sha256":"058fdb77442649a316bc2ab762d4ea8cdcdee36f2eac120897e73b5a95c2796f"},"schema_version":"1.0"},"canonical_sha256":"fd12f6aeeea51c2a28572efc3da84419c33ebc3a988d5647a8c5e81924b4b717","source":{"kind":"arxiv","id":"2405.17067","version":2},"source_aliases":[{"alias_kind":"arxiv","alias_value":"2405.17067","created_at":"2026-07-05T11:03:14Z"},{"alias_kind":"arxiv_version","alias_value":"2405.17067v2","created_at":"2026-07-05T11:03:14Z"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.2405.17067","created_at":"2026-07-05T11:03:14Z"},{"alias_kind":"pith_short_12","alias_value":"7UJPNLXOUUOC","created_at":"2026-07-05T11:03:14Z"},{"alias_kind":"pith_short_16","alias_value":"7UJPNLXOUUOCUKCX","created_at":"2026-07-05T11:03:14Z"},{"alias_kind":"pith_short_8","alias_value":"7UJPNLXO","created_at":"2026-07-05T11:03:14Z"}],"events":[{"event_type":"record_created","subject_pith_number":"pith:2024:7UJPNLXOUUOCUKCXF36D3KCEDH","target":"record","payload":{"canonical_record":{"source":{"id":"2405.17067","kind":"arxiv","version":2},"metadata":{"license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","primary_cat":"cs.CL","submitted_at":"2024-05-27T11:39:59Z","cross_cats_sorted":["cs.AI"],"title_canon_sha256":"e0e685a19accbb37a09fe62f81e1bbd448294e5a0a8ba380fc386f1229a8b230","abstract_canon_sha256":"058fdb77442649a316bc2ab762d4ea8cdcdee36f2eac120897e73b5a95c2796f"},"schema_version":"1.0"},"canonical_sha256":"fd12f6aeeea51c2a28572efc3da84419c33ebc3a988d5647a8c5e81924b4b717","receipt":{"kind":"pith_receipt","key_id":"pith-v1-2026-05","algorithm":"ed25519","signed_at":"2026-07-05T11:03:14.857840Z","signature_b64":"JrxFHBXYVxkwQFSMpq2gz5q6B8n89HvXWOAgXm5PGwFOcHOuomN7HRGDM9bNoyDDzHV0BO5w6NVZEgUR4xMFAw==","signed_message":"canonical_sha256_bytes","builder_version":"pith-number-builder-2026-05-17-v1","receipt_version":"0.3","canonical_sha256":"fd12f6aeeea51c2a28572efc3da84419c33ebc3a988d5647a8c5e81924b4b717","last_reissued_at":"2026-07-05T11:03:14.857396Z","signature_status":"signed_v1","first_computed_at":"2026-07-05T11:03:14.857396Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"source_kind":"arxiv","source_id":"2405.17067","source_version":2,"attestation_state":"computed"},"signer":{"signer_id":"pith.science","signer_type":"pith_registry","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"created_at":"2026-07-05T11:03:14Z","supersedes":[],"prev_event":null,"signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"5+sBl9YN/GNpBU/QkDvgp+hpAdjMEtD3XHOPQln+gGjiY6UtW3qUSD2w5WMPpjsAQHZSt5DjdOm5j9taTEtQCg==","signed_message":"open_graph_event_sha256_bytes","signed_at":"2026-08-08T12:53:37.073996Z"},"content_sha256":"270a7ad71ba3475794cdb56b556638a0398a1afd545243b4ca00aad4cd288d0b","schema_version":"1.0","event_id":"sha256:270a7ad71ba3475794cdb56b556638a0398a1afd545243b4ca00aad4cd288d0b"},{"event_type":"graph_snapshot","subject_pith_number":"pith:2024:7UJPNLXOUUOCUKCXF36D3KCEDH","target":"graph","payload":{"graph_snapshot":{"paper":{"title":"Tokenization Matters! Degrading Large Language Models through Challenging Their Tokenization","license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","headline":"","cross_cats":["cs.AI"],"primary_cat":"cs.CL","authors_text":"Deqing Yang, Dixuan Wang, Guochao Jiang, Jiaqing Liang, Junyuan Jiang, Yanda Li, Zepeng Ding, Ziqin Luo","submitted_at":"2024-05-27T11:39:59Z","abstract_excerpt":"Large Language Models (LLMs) have shown remarkable capabilities in language understanding and generation. Nonetheless, it was also witnessed that LLMs tend to produce inaccurate responses to specific queries. This deficiency can be traced to the tokenization step LLMs must undergo, which is an inevitable limitation inherent to all LLMs. In fact, incorrect tokenization is the critical point that hinders LLMs in understanding the input precisely, thus leading to unsatisfactory output. This defect is more obvious in Chinese scenarios. To demonstrate this flaw of LLMs, we construct an adversarial "},"claims":{"count":0,"items":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"2405.17067","kind":"arxiv","version":2},"verdict":{"id":null,"model_set":{},"created_at":null,"strongest_claim":"","one_line_summary":"","pipeline_version":null,"weakest_assumption":"","pith_extraction_headline":""},"integrity":{"clean":true,"summary":{"advisory":0,"critical":0,"by_detector":{},"informational":0},"endpoint":"/pith/2405.17067/integrity.json","findings":[],"available":true,"detectors_run":[],"snapshot_sha256":"c28c3603d3b5d939e8dc4c7e95fa8dfce3d595e45f758748cecf8e644a296938"},"references":{"count":0,"sample":[],"resolved_work":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","internal_anchors":0},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"author_claims":{"count":0,"strong_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"builder_version":"pith-number-builder-2026-05-17-v1"},"verdict_id":null},"signer":{"signer_id":"pith.science","signer_type":"pith_registry","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"created_at":"2026-07-05T11:03:14Z","supersedes":[],"prev_event":null,"signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"eFTvXxyYqVMi+B3JmAIO/iNCRmyagHnPfjEYuaI1BqQ/zzyA/qYl06BtzsE8sblUhSL+xo/2v7tkfjjL64QlBg==","signed_message":"open_graph_event_sha256_bytes","signed_at":"2026-08-08T12:53:37.074494Z"},"content_sha256":"d3bf12f961cab711a346372a46d10ab06bb073ba9ecb7a7e6bc6f052c61a3838","schema_version":"1.0","event_id":"sha256:d3bf12f961cab711a346372a46d10ab06bb073ba9ecb7a7e6bc6f052c61a3838"}],"timestamp_proofs":[],"mirror_hints":[{"mirror_type":"https","name":"Pith Resolver","base_url":"https://pith.science","bundle_url":"https://pith.science/pith/7UJPNLXOUUOCUKCXF36D3KCEDH/bundle.json","state_url":"https://pith.science/pith/7UJPNLXOUUOCUKCXF36D3KCEDH/state.json","well_known_bundle_url":"https://pith.science/.well-known/pith/7UJPNLXOUUOCUKCXF36D3KCEDH/bundle.json","status":"primary"}],"public_keys":[{"key_id":"pith-v1-2026-05","algorithm":"ed25519","format":"raw","public_key_b64":"stVStoiQhXFxp4s2pdzPNoqVNBMojDU/fJ2db5S3CbM=","public_key_hex":"b2d552b68890857171a78b36a5dccf368a953413288c353f7c9d9d6f94b709b3","fingerprint_sha256_b32_first128bits":"RVFV5Z2OI2J3ZUO7ERDEBCYNKS","fingerprint_sha256_hex":"8d4b5ee74e4693bcd1df2446408b0d54","rotates_at":null,"url":"https://pith.science/pith-signing-key.json","notes":"Pith uses this Ed25519 key to sign canonical record SHA-256 digests. Verify with: ed25519_verify(public_key, message=canonical_sha256_bytes, signature=base64decode(signature_b64))."}],"merge_version":"pith-open-graph-merge-v1","built_at":"2026-08-08T12:53:37Z","links":{"resolver":"https://pith.science/pith/7UJPNLXOUUOCUKCXF36D3KCEDH","bundle":"https://pith.science/pith/7UJPNLXOUUOCUKCXF36D3KCEDH/bundle.json","state":"https://pith.science/pith/7UJPNLXOUUOCUKCXF36D3KCEDH/state.json","well_known_bundle":"https://pith.science/.well-known/pith/7UJPNLXOUUOCUKCXF36D3KCEDH/bundle.json"},"state":{"state_type":"pith_open_graph_state","state_version":"1.0","pith_number":"pith:2024:7UJPNLXOUUOCUKCXF36D3KCEDH","merge_version":"pith-open-graph-merge-v1","event_count":2,"valid_event_count":2,"invalid_event_count":0,"equivocation_count":0,"current":{"canonical_record":{"metadata":{"abstract_canon_sha256":"058fdb77442649a316bc2ab762d4ea8cdcdee36f2eac120897e73b5a95c2796f","cross_cats_sorted":["cs.AI"],"license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","primary_cat":"cs.CL","submitted_at":"2024-05-27T11:39:59Z","title_canon_sha256":"e0e685a19accbb37a09fe62f81e1bbd448294e5a0a8ba380fc386f1229a8b230"},"schema_version":"1.0","source":{"id":"2405.17067","kind":"arxiv","version":2}},"source_aliases":[{"alias_kind":"arxiv","alias_value":"2405.17067","created_at":"2026-07-05T11:03:14Z"},{"alias_kind":"arxiv_version","alias_value":"2405.17067v2","created_at":"2026-07-05T11:03:14Z"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.2405.17067","created_at":"2026-07-05T11:03:14Z"},{"alias_kind":"pith_short_12","alias_value":"7UJPNLXOUUOC","created_at":"2026-07-05T11:03:14Z"},{"alias_kind":"pith_short_16","alias_value":"7UJPNLXOUUOCUKCX","created_at":"2026-07-05T11:03:14Z"},{"alias_kind":"pith_short_8","alias_value":"7UJPNLXO","created_at":"2026-07-05T11:03:14Z"}],"graph_snapshots":[{"event_id":"sha256:d3bf12f961cab711a346372a46d10ab06bb073ba9ecb7a7e6bc6f052c61a3838","target":"graph","created_at":"2026-07-05T11:03:14Z","signer":{"key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signer_id":"pith.science","signer_type":"pith_registry"},"payload":{"graph_snapshot":{"author_claims":{"count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","strong_count":0},"builder_version":"pith-number-builder-2026-05-17-v1","claims":{"count":0,"items":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"integrity":{"available":true,"clean":true,"detectors_run":[],"endpoint":"/pith/2405.17067/integrity.json","findings":[],"snapshot_sha256":"c28c3603d3b5d939e8dc4c7e95fa8dfce3d595e45f758748cecf8e644a296938","summary":{"advisory":0,"by_detector":{},"critical":0,"informational":0}},"paper":{"abstract_excerpt":"Large Language Models (LLMs) have shown remarkable capabilities in language understanding and generation. Nonetheless, it was also witnessed that LLMs tend to produce inaccurate responses to specific queries. This deficiency can be traced to the tokenization step LLMs must undergo, which is an inevitable limitation inherent to all LLMs. In fact, incorrect tokenization is the critical point that hinders LLMs in understanding the input precisely, thus leading to unsatisfactory output. This defect is more obvious in Chinese scenarios. To demonstrate this flaw of LLMs, we construct an adversarial ","authors_text":"Deqing Yang, Dixuan Wang, Guochao Jiang, Jiaqing Liang, Junyuan Jiang, Yanda Li, Zepeng Ding, Ziqin Luo","cross_cats":["cs.AI"],"headline":"","license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","primary_cat":"cs.CL","submitted_at":"2024-05-27T11:39:59Z","title":"Tokenization Matters! Degrading Large Language Models through Challenging Their Tokenization"},"references":{"count":0,"internal_anchors":0,"resolved_work":0,"sample":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"2405.17067","kind":"arxiv","version":2},"verdict":{"created_at":null,"id":null,"model_set":{},"one_line_summary":"","pipeline_version":null,"pith_extraction_headline":"","strongest_claim":"","weakest_assumption":""}},"verdict_id":null}}],"author_attestations":[],"timestamp_anchors":[],"storage_attestations":[],"citation_signatures":[],"replication_records":[],"corrections":[],"mirror_hints":[],"record_created":{"event_id":"sha256:270a7ad71ba3475794cdb56b556638a0398a1afd545243b4ca00aad4cd288d0b","target":"record","created_at":"2026-07-05T11:03:14Z","signer":{"key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signer_id":"pith.science","signer_type":"pith_registry"},"payload":{"attestation_state":"computed","canonical_record":{"metadata":{"abstract_canon_sha256":"058fdb77442649a316bc2ab762d4ea8cdcdee36f2eac120897e73b5a95c2796f","cross_cats_sorted":["cs.AI"],"license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","primary_cat":"cs.CL","submitted_at":"2024-05-27T11:39:59Z","title_canon_sha256":"e0e685a19accbb37a09fe62f81e1bbd448294e5a0a8ba380fc386f1229a8b230"},"schema_version":"1.0","source":{"id":"2405.17067","kind":"arxiv","version":2}},"canonical_sha256":"fd12f6aeeea51c2a28572efc3da84419c33ebc3a988d5647a8c5e81924b4b717","receipt":{"algorithm":"ed25519","builder_version":"pith-number-builder-2026-05-17-v1","canonical_sha256":"fd12f6aeeea51c2a28572efc3da84419c33ebc3a988d5647a8c5e81924b4b717","first_computed_at":"2026-07-05T11:03:14.857396Z","key_id":"pith-v1-2026-05","kind":"pith_receipt","last_reissued_at":"2026-07-05T11:03:14.857396Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","receipt_version":"0.3","signature_b64":"JrxFHBXYVxkwQFSMpq2gz5q6B8n89HvXWOAgXm5PGwFOcHOuomN7HRGDM9bNoyDDzHV0BO5w6NVZEgUR4xMFAw==","signature_status":"signed_v1","signed_at":"2026-07-05T11:03:14.857840Z","signed_message":"canonical_sha256_bytes"},"source_id":"2405.17067","source_kind":"arxiv","source_version":2}}},"equivocations":[],"invalid_events":[],"applied_event_ids":["sha256:270a7ad71ba3475794cdb56b556638a0398a1afd545243b4ca00aad4cd288d0b","sha256:d3bf12f961cab711a346372a46d10ab06bb073ba9ecb7a7e6bc6f052c61a3838"],"state_sha256":"d3a0b519e88ecea13bf0dfca8ecf43a673f831e000905949c470dc1fff3b6531"},"bundle_signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"wzRFFdKEUx9Co9TS4CsqPv8jWcdoyb/eq4qfpUr1TLGsCDhp1f/3VgfCyBn789/v7WJikqrXpi/F5+8ddoqeBw==","signed_message":"bundle_sha256_bytes","signed_at":"2026-08-08T12:53:37.078667Z","bundle_sha256":"0b57524dc5338230480842a16e7e9e9c3a91a5d524e17d3e20162c2600267ef0"}}