{"state_type":"pith_open_graph_state","state_version":"1.0","pith_number":"pith:2021:36NBI6DQWCHW5TURMK4BDPXFVF","merge_version":"pith-open-graph-merge-v1","event_count":2,"valid_event_count":2,"invalid_event_count":0,"equivocation_count":0,"current":{"canonical_record":{"metadata":{"abstract_canon_sha256":"32753c56c81e4b913065921564af0c443480ba956478af99e17e944a96f46ea0","cross_cats_sorted":[],"license":"http://creativecommons.org/licenses/by-sa/4.0/","primary_cat":"cs.CL","submitted_at":"2021-09-13T08:31:01Z","title_canon_sha256":"41b23abe702b46b0163168e51b1fcab1cf7efb6b114e9f08285d56873062b034"},"schema_version":"1.0","source":{"id":"2109.05772","kind":"arxiv","version":1}},"source_aliases":[{"alias_kind":"arxiv","alias_value":"2109.05772","created_at":"2026-07-05T03:13:42Z"},{"alias_kind":"arxiv_version","alias_value":"2109.05772v1","created_at":"2026-07-05T03:13:42Z"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.2109.05772","created_at":"2026-07-05T03:13:42Z"},{"alias_kind":"pith_short_12","alias_value":"36NBI6DQWCHW","created_at":"2026-07-05T03:13:42Z"},{"alias_kind":"pith_short_16","alias_value":"36NBI6DQWCHW5TUR","created_at":"2026-07-05T03:13:42Z"},{"alias_kind":"pith_short_8","alias_value":"36NBI6DQ","created_at":"2026-07-05T03:13:42Z"}],"graph_snapshots":[{"event_id":"sha256:1f38f7170193960cb8bee95115416041520232abe5df2dc8a4d0c42d5e18f4c7","target":"graph","created_at":"2026-07-05T03:13:42Z","signer":{"key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signer_id":"pith.science","signer_type":"pith_registry"},"payload":{"graph_snapshot":{"author_claims":{"count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","strong_count":0},"builder_version":"pith-number-builder-2026-05-17-v1","claims":{"count":0,"items":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"integrity":{"available":true,"clean":true,"detectors_run":[],"endpoint":"/pith/2109.05772/integrity.json","findings":[],"snapshot_sha256":"c28c3603d3b5d939e8dc4c7e95fa8dfce3d595e45f758748cecf8e644a296938","summary":{"advisory":0,"by_detector":{},"critical":0,"informational":0}},"paper":{"abstract_excerpt":"The size of the vocabulary is a central design choice in large pretrained language models, with respect to both performance and memory requirements. Typically, subword tokenization algorithms such as byte pair encoding and WordPiece are used. In this work, we investigate the compatibility of tokenizations for multilingual static and contextualized embedding spaces and propose a measure that reflects the compatibility of tokenizations across languages. Our goal is to prevent incompatible tokenizations, e.g., \"wine\" (word-level) in English vs.\\ \"v i n\" (character-level) in French, which make it ","authors_text":"Antonis Maronikolakis, Hinrich Sch\\\"utze, Philipp Dufter","cross_cats":[],"headline":"","license":"http://creativecommons.org/licenses/by-sa/4.0/","primary_cat":"cs.CL","submitted_at":"2021-09-13T08:31:01Z","title":"Wine is Not v i n. -- On the Compatibility of Tokenizations Across Languages"},"references":{"count":0,"internal_anchors":0,"resolved_work":0,"sample":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"2109.05772","kind":"arxiv","version":1},"verdict":{"created_at":null,"id":null,"model_set":{},"one_line_summary":"","pipeline_version":null,"pith_extraction_headline":"","strongest_claim":"","weakest_assumption":""}},"verdict_id":null}}],"author_attestations":[],"timestamp_anchors":[],"storage_attestations":[],"citation_signatures":[],"replication_records":[],"corrections":[],"mirror_hints":[],"record_created":{"event_id":"sha256:a7a1550ceef71222f6aac7be36aab298dd5a2071fc4e0ecb60376b528705e0d2","target":"record","created_at":"2026-07-05T03:13:42Z","signer":{"key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signer_id":"pith.science","signer_type":"pith_registry"},"payload":{"attestation_state":"computed","canonical_record":{"metadata":{"abstract_canon_sha256":"32753c56c81e4b913065921564af0c443480ba956478af99e17e944a96f46ea0","cross_cats_sorted":[],"license":"http://creativecommons.org/licenses/by-sa/4.0/","primary_cat":"cs.CL","submitted_at":"2021-09-13T08:31:01Z","title_canon_sha256":"41b23abe702b46b0163168e51b1fcab1cf7efb6b114e9f08285d56873062b034"},"schema_version":"1.0","source":{"id":"2109.05772","kind":"arxiv","version":1}},"canonical_sha256":"df9a147870b08f6ece9162b811bee5a950a89bff6f5e32f8f415463c7b9bae2d","receipt":{"algorithm":"ed25519","builder_version":"pith-number-builder-2026-05-17-v1","canonical_sha256":"df9a147870b08f6ece9162b811bee5a950a89bff6f5e32f8f415463c7b9bae2d","first_computed_at":"2026-07-05T03:13:42.276815Z","key_id":"pith-v1-2026-05","kind":"pith_receipt","last_reissued_at":"2026-07-05T03:13:42.276815Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","receipt_version":"0.3","signature_b64":"4Wta/zHC30Hcvksrl4jQSsQnfHGBtUELqQJwAftzrUkvGLcx+xhjmlCe49XZkjtjaWb+MCcN8fkfVs0TpJeGCA==","signature_status":"signed_v1","signed_at":"2026-07-05T03:13:42.277165Z","signed_message":"canonical_sha256_bytes"},"source_id":"2109.05772","source_kind":"arxiv","source_version":1}}},"equivocations":[],"invalid_events":[],"applied_event_ids":["sha256:a7a1550ceef71222f6aac7be36aab298dd5a2071fc4e0ecb60376b528705e0d2","sha256:1f38f7170193960cb8bee95115416041520232abe5df2dc8a4d0c42d5e18f4c7"],"state_sha256":"6d595cbafff0bcc45fdc9a7a0e7aabd3edc87545fce8e1e74d38245911ee333e"}