{"record_type":"pith_number_record","schema_url":"https://pith.science/schemas/pith-number/v1.json","pith_number":"pith:2026:TMUVX7WKU4ERJAA6TCK6PRFJU5","short_pith_number":"pith:TMUVX7WK","schema_version":"1.0","canonical_sha256":"9b295bfecaa70914801e9895e7c4a9a75a409567bc4d7c77dfc7b5e38a3abee3","source":{"kind":"arxiv","id":"2607.26831","version":1},"attestation_state":"computed","paper":{"title":"Language Models are not Equally Robust to Non-Canonical Tokenization across Languages","license":"http://creativecommons.org/licenses/by-sa/4.0/","headline":"","cross_cats":[],"primary_cat":"cs.CL","authors_text":"Poulami Ghosh, Preethi Jyothi","submitted_at":"2026-07-29T12:23:15Z","abstract_excerpt":"Despite the existence of exponentially many valid tokenizations for a given string, language models operate on a single canonical sequence deterministically produced by the tokenizer, leaving the broader tokenization space largely uncharacterized. In this paper, we investigate this overlooked space by studying the behavior of language models under non-canonical tokenizations across diverse languages. For English, prior work shows that models are largely invariant to alternative tokenizations that represent the same underlying string. We ask whether this invariance generalizes to other language"},"verification_status":{"content_addressed":true,"pith_receipt":true,"author_attested":false,"weak_author_claims":0,"strong_author_claims":0,"externally_anchored":false,"storage_verified":false,"citation_signatures":0,"replication_records":0,"graph_snapshot":true,"references_resolved":false,"formal_links_present":false},"canonical_record":{"source":{"id":"2607.26831","kind":"arxiv","version":1},"metadata":{"license":"http://creativecommons.org/licenses/by-sa/4.0/","primary_cat":"cs.CL","submitted_at":"2026-07-29T12:23:15Z","cross_cats_sorted":[],"title_canon_sha256":"92bb70ba7321b3338807a78dc381d298576453305184b0734ea0b64265dddbc0","abstract_canon_sha256":"6bd4ff528ae8989d911db26a036a9ef33145f98725f5855e5db5d6166a808655"},"schema_version":"1.0"},"receipt":{"kind":"pith_receipt","builder_version":"pith-number-builder-2026-05-17-v1","receipt_version":"0.3","canonical_sha256":"9b295bfecaa70914801e9895e7c4a9a75a409567bc4d7c77dfc7b5e38a3abee3","last_reissued_at":"2026-07-30T01:22:28.122056Z","signature_status":"unsigned_v0","first_computed_at":"2026-07-30T01:22:28.122056Z"},"graph_snapshot":{"paper":{"title":"Language Models are not Equally Robust to Non-Canonical Tokenization across Languages","license":"http://creativecommons.org/licenses/by-sa/4.0/","headline":"","cross_cats":[],"primary_cat":"cs.CL","authors_text":"Poulami Ghosh, Preethi Jyothi","submitted_at":"2026-07-29T12:23:15Z","abstract_excerpt":"Despite the existence of exponentially many valid tokenizations for a given string, language models operate on a single canonical sequence deterministically produced by the tokenizer, leaving the broader tokenization space largely uncharacterized. In this paper, we investigate this overlooked space by studying the behavior of language models under non-canonical tokenizations across diverse languages. For English, prior work shows that models are largely invariant to alternative tokenizations that represent the same underlying string. We ask whether this invariance generalizes to other language"},"claims":{"count":0,"items":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"2607.26831","kind":"arxiv","version":1},"verdict":{"id":null,"model_set":{},"created_at":null,"strongest_claim":"","one_line_summary":"","pipeline_version":null,"weakest_assumption":"","pith_extraction_headline":""},"integrity":{"clean":true,"summary":{"advisory":0,"critical":0,"by_detector":{},"informational":0},"endpoint":"/pith/2607.26831/integrity.json","findings":[],"available":true,"detectors_run":[],"snapshot_sha256":"c28c3603d3b5d939e8dc4c7e95fa8dfce3d595e45f758748cecf8e644a296938"},"references":{"count":0,"sample":[],"resolved_work":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","internal_anchors":0},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"author_claims":{"count":0,"strong_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"builder_version":"pith-number-builder-2026-05-17-v1"},"aliases":[{"alias_kind":"arxiv","alias_value":"2607.26831","created_at":"2026-07-30T01:22:28.127066+00:00"},{"alias_kind":"arxiv_version","alias_value":"2607.26831v1","created_at":"2026-07-30T01:22:28.127066+00:00"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.2607.26831","created_at":"2026-07-30T01:22:28.127066+00:00"},{"alias_kind":"pith_short_12","alias_value":"TMUVX7WKU4ER","created_at":"2026-07-30T01:22:28.127066+00:00"},{"alias_kind":"pith_short_16","alias_value":"TMUVX7WKU4ERJAA6","created_at":"2026-07-30T01:22:28.127066+00:00"},{"alias_kind":"pith_short_8","alias_value":"TMUVX7WK","created_at":"2026-07-30T01:22:28.127066+00:00"}],"events":[],"event_summary":{},"paper_claims":[],"inbound_citations":{"count":0,"internal_anchor_count":0,"sample":[]},"formal_canon":{"evidence_count":0,"sample":[],"anchors":[]},"links":{"html":"https://pith.science/pith/TMUVX7WKU4ERJAA6TCK6PRFJU5","json":"https://pith.science/pith/TMUVX7WKU4ERJAA6TCK6PRFJU5.json","graph_json":"https://pith.science/api/pith-number/TMUVX7WKU4ERJAA6TCK6PRFJU5/graph.json","events_json":"https://pith.science/api/pith-number/TMUVX7WKU4ERJAA6TCK6PRFJU5/events.json","paper":"https://pith.science/paper/TMUVX7WK"},"agent_actions":{"view_html":"https://pith.science/pith/TMUVX7WKU4ERJAA6TCK6PRFJU5","download_json":"https://pith.science/pith/TMUVX7WKU4ERJAA6TCK6PRFJU5.json","view_paper":"https://pith.science/paper/TMUVX7WK","resolve_alias":"https://pith.science/api/pith-number/resolve?arxiv=2607.26831&json=true","fetch_graph":"https://pith.science/api/pith-number/TMUVX7WKU4ERJAA6TCK6PRFJU5/graph.json","fetch_events":"https://pith.science/api/pith-number/TMUVX7WKU4ERJAA6TCK6PRFJU5/events.json","actions":{"anchor_timestamp":"https://pith.science/pith/TMUVX7WKU4ERJAA6TCK6PRFJU5/action/timestamp_anchor","attest_storage":"https://pith.science/pith/TMUVX7WKU4ERJAA6TCK6PRFJU5/action/storage_attestation","attest_author":"https://pith.science/pith/TMUVX7WKU4ERJAA6TCK6PRFJU5/action/author_attestation","sign_citation":"https://pith.science/pith/TMUVX7WKU4ERJAA6TCK6PRFJU5/action/citation_signature","submit_replication":"https://pith.science/pith/TMUVX7WKU4ERJAA6TCK6PRFJU5/action/replication_record"}},"created_at":"2026-07-30T01:22:28.127066+00:00","updated_at":"2026-07-30T01:22:28.127066+00:00"}