{"record_type":"pith_number_record","schema_url":"https://pith.science/schemas/pith-number/v1.json","pith_number":"pith:2025:ZJIZ5JWDQRXMEBCFJ6AGFJXI7B","short_pith_number":"pith:ZJIZ5JWD","schema_version":"1.0","canonical_sha256":"ca519ea6c3846ec204454f8062a6e8f87d55071a05f69a7a6a600a9c2a2409bf","source":{"kind":"arxiv","id":"2502.15343","version":2},"attestation_state":"computed","paper":{"title":"Tokenization is Sensitive to Language Variation","license":"http://creativecommons.org/licenses/by/4.0/","headline":"","cross_cats":[],"primary_cat":"cs.CL","authors_text":"Anna Wegmann, David Jurgens, Dong Nguyen","submitted_at":"2025-02-21T09:58:54Z","abstract_excerpt":"Variation in language is ubiquitous and often systematically linked to regional, social, and contextual factors. Tokenizers split texts into smaller units and might behave differently for less common linguistic forms. This might affect downstream LLM performance differently on two types of tasks: Tasks where the model should be robust to language variation (e.g., for semantic tasks like NLI, labels do not depend on whether a text uses British or American spelling) and tasks where the model should be sensitive to language variation (e.g., for form-based tasks like authorship verification, label"},"verification_status":{"content_addressed":true,"pith_receipt":true,"author_attested":false,"weak_author_claims":0,"strong_author_claims":0,"externally_anchored":false,"storage_verified":false,"citation_signatures":0,"replication_records":0,"graph_snapshot":true,"references_resolved":false,"formal_links_present":false},"canonical_record":{"source":{"id":"2502.15343","kind":"arxiv","version":2},"metadata":{"license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.CL","submitted_at":"2025-02-21T09:58:54Z","cross_cats_sorted":[],"title_canon_sha256":"9e9f18daf9a18abc645f7ca10c1bfcc79d645168dd5fe3aab8efe2e461ca6499","abstract_canon_sha256":"3f04a2973b5b4e9991ba4eadf7142fdde26ae989062c3983723b8d8c3a902def"},"schema_version":"1.0"},"receipt":{"kind":"pith_receipt","key_id":"pith-v1-2026-05","algorithm":"ed25519","signed_at":"2026-07-05T11:31:43.567787Z","signature_b64":"Jln1M85SAiY1/Bdqd4KsgH2wjp65tEZsRd6l7dmX/2LET/Hdf4+LgxWMMTFRKtbFmgEUjDkCj8wrdnau0s49Ag==","signed_message":"canonical_sha256_bytes","builder_version":"pith-number-builder-2026-05-17-v1","receipt_version":"0.3","canonical_sha256":"ca519ea6c3846ec204454f8062a6e8f87d55071a05f69a7a6a600a9c2a2409bf","last_reissued_at":"2026-07-05T11:31:43.567300Z","signature_status":"signed_v1","first_computed_at":"2026-07-05T11:31:43.567300Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"graph_snapshot":{"paper":{"title":"Tokenization is Sensitive to Language Variation","license":"http://creativecommons.org/licenses/by/4.0/","headline":"","cross_cats":[],"primary_cat":"cs.CL","authors_text":"Anna Wegmann, David Jurgens, Dong Nguyen","submitted_at":"2025-02-21T09:58:54Z","abstract_excerpt":"Variation in language is ubiquitous and often systematically linked to regional, social, and contextual factors. Tokenizers split texts into smaller units and might behave differently for less common linguistic forms. This might affect downstream LLM performance differently on two types of tasks: Tasks where the model should be robust to language variation (e.g., for semantic tasks like NLI, labels do not depend on whether a text uses British or American spelling) and tasks where the model should be sensitive to language variation (e.g., for form-based tasks like authorship verification, label"},"claims":{"count":0,"items":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"2502.15343","kind":"arxiv","version":2},"verdict":{"id":null,"model_set":{},"created_at":null,"strongest_claim":"","one_line_summary":"","pipeline_version":null,"weakest_assumption":"","pith_extraction_headline":""},"integrity":{"clean":true,"summary":{"advisory":0,"critical":0,"by_detector":{},"informational":0},"endpoint":"/pith/2502.15343/integrity.json","findings":[],"available":true,"detectors_run":[],"snapshot_sha256":"c28c3603d3b5d939e8dc4c7e95fa8dfce3d595e45f758748cecf8e644a296938"},"references":{"count":0,"sample":[],"resolved_work":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","internal_anchors":0},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"author_claims":{"count":0,"strong_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"builder_version":"pith-number-builder-2026-05-17-v1"},"aliases":[{"alias_kind":"arxiv","alias_value":"2502.15343","created_at":"2026-07-05T11:31:43.567369+00:00"},{"alias_kind":"arxiv_version","alias_value":"2502.15343v2","created_at":"2026-07-05T11:31:43.567369+00:00"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.2502.15343","created_at":"2026-07-05T11:31:43.567369+00:00"},{"alias_kind":"pith_short_12","alias_value":"ZJIZ5JWDQRXM","created_at":"2026-07-05T11:31:43.567369+00:00"},{"alias_kind":"pith_short_16","alias_value":"ZJIZ5JWDQRXMEBCF","created_at":"2026-07-05T11:31:43.567369+00:00"},{"alias_kind":"pith_short_8","alias_value":"ZJIZ5JWD","created_at":"2026-07-05T11:31:43.567369+00:00"}],"events":[],"event_summary":{},"paper_claims":[],"inbound_citations":{"count":1,"internal_anchor_count":1,"sample":[{"citing_arxiv_id":"2505.24689","citing_title":"BPE Stays on SCRIPT: Structured Encoding for Robust Multilingual Pretokenization","ref_index":19,"is_internal_anchor":true}]},"formal_canon":{"evidence_count":0,"sample":[],"anchors":[]},"links":{"html":"https://pith.science/pith/ZJIZ5JWDQRXMEBCFJ6AGFJXI7B","json":"https://pith.science/pith/ZJIZ5JWDQRXMEBCFJ6AGFJXI7B.json","graph_json":"https://pith.science/api/pith-number/ZJIZ5JWDQRXMEBCFJ6AGFJXI7B/graph.json","events_json":"https://pith.science/api/pith-number/ZJIZ5JWDQRXMEBCFJ6AGFJXI7B/events.json","paper":"https://pith.science/paper/ZJIZ5JWD"},"agent_actions":{"view_html":"https://pith.science/pith/ZJIZ5JWDQRXMEBCFJ6AGFJXI7B","download_json":"https://pith.science/pith/ZJIZ5JWDQRXMEBCFJ6AGFJXI7B.json","view_paper":"https://pith.science/paper/ZJIZ5JWD","resolve_alias":"https://pith.science/api/pith-number/resolve?arxiv=2502.15343&json=true","fetch_graph":"https://pith.science/api/pith-number/ZJIZ5JWDQRXMEBCFJ6AGFJXI7B/graph.json","fetch_events":"https://pith.science/api/pith-number/ZJIZ5JWDQRXMEBCFJ6AGFJXI7B/events.json","actions":{"anchor_timestamp":"https://pith.science/pith/ZJIZ5JWDQRXMEBCFJ6AGFJXI7B/action/timestamp_anchor","attest_storage":"https://pith.science/pith/ZJIZ5JWDQRXMEBCFJ6AGFJXI7B/action/storage_attestation","attest_author":"https://pith.science/pith/ZJIZ5JWDQRXMEBCFJ6AGFJXI7B/action/author_attestation","sign_citation":"https://pith.science/pith/ZJIZ5JWDQRXMEBCFJ6AGFJXI7B/action/citation_signature","submit_replication":"https://pith.science/pith/ZJIZ5JWDQRXMEBCFJ6AGFJXI7B/action/replication_record"}},"created_at":"2026-07-05T11:31:43.567369+00:00","updated_at":"2026-07-05T11:31:43.567369+00:00"}