{"record_type":"pith_number_record","schema_url":"https://pith.science/schemas/pith-number/v1.json","pith_number":"pith:2024:MA3OHTYIAEWKT7GIK3TX2V76SY","short_pith_number":"pith:MA3OHTYI","schema_version":"1.0","canonical_sha256":"6036e3cf08012ca9fcc856e77d57fe962d1b4ec287d575cfb435247505cbe652","source":{"kind":"arxiv","id":"2404.13292","version":1},"attestation_state":"computed","paper":{"title":"Evaluating Subword Tokenization: Alien Subword Composition and OOV Generalization Challenge","license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","headline":"","cross_cats":["cs.AI"],"primary_cat":"cs.CL","authors_text":"Ekaterina Vylomova, G\\'abor Bella, Khuyagbaatar Batsuren, Omri Uzan, Tsetsuukhei Delgerbaatar, Verna Dankers, Yuval Pinter","submitted_at":"2024-04-20T06:49:15Z","abstract_excerpt":"The popular subword tokenizers of current language models, such as Byte-Pair Encoding (BPE), are known not to respect morpheme boundaries, which affects the downstream performance of the models. While many improved tokenization algorithms have been proposed, their evaluation and cross-comparison is still an open problem. As a solution, we propose a combined intrinsic-extrinsic evaluation framework for subword tokenization. Intrinsic evaluation is based on our new UniMorph Labeller tool that classifies subword tokenization as either morphological or alien. Extrinsic evaluation, in turn, is perf"},"verification_status":{"content_addressed":true,"pith_receipt":true,"author_attested":false,"weak_author_claims":0,"strong_author_claims":0,"externally_anchored":false,"storage_verified":false,"citation_signatures":0,"replication_records":0,"graph_snapshot":true,"references_resolved":false,"formal_links_present":false},"canonical_record":{"source":{"id":"2404.13292","kind":"arxiv","version":1},"metadata":{"license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","primary_cat":"cs.CL","submitted_at":"2024-04-20T06:49:15Z","cross_cats_sorted":["cs.AI"],"title_canon_sha256":"1f19f978fe636e4b7ee8ef4377b8028a9bd5608392fb441fbf8ee401f1f3a12a","abstract_canon_sha256":"e074dd8fb9685921049af5829134a8829fd0d50177fecce01b43b07984271a78"},"schema_version":"1.0"},"receipt":{"kind":"pith_receipt","key_id":"pith-v1-2026-05","algorithm":"ed25519","signed_at":"2026-07-05T08:10:30.085885Z","signature_b64":"wvjaoidyJ+xxskonQpRSOdsYz9DGhaR6ZnGyzBgaB//yOfFcbwSQrSG8tdoZP38JGIYFrXyR9A8qruMpTiLAAw==","signed_message":"canonical_sha256_bytes","builder_version":"pith-number-builder-2026-05-17-v1","receipt_version":"0.3","canonical_sha256":"6036e3cf08012ca9fcc856e77d57fe962d1b4ec287d575cfb435247505cbe652","last_reissued_at":"2026-07-05T08:10:30.085410Z","signature_status":"signed_v1","first_computed_at":"2026-07-05T08:10:30.085410Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"graph_snapshot":{"paper":{"title":"Evaluating Subword Tokenization: Alien Subword Composition and OOV Generalization Challenge","license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","headline":"","cross_cats":["cs.AI"],"primary_cat":"cs.CL","authors_text":"Ekaterina Vylomova, G\\'abor Bella, Khuyagbaatar Batsuren, Omri Uzan, Tsetsuukhei Delgerbaatar, Verna Dankers, Yuval Pinter","submitted_at":"2024-04-20T06:49:15Z","abstract_excerpt":"The popular subword tokenizers of current language models, such as Byte-Pair Encoding (BPE), are known not to respect morpheme boundaries, which affects the downstream performance of the models. While many improved tokenization algorithms have been proposed, their evaluation and cross-comparison is still an open problem. As a solution, we propose a combined intrinsic-extrinsic evaluation framework for subword tokenization. Intrinsic evaluation is based on our new UniMorph Labeller tool that classifies subword tokenization as either morphological or alien. Extrinsic evaluation, in turn, is perf"},"claims":{"count":0,"items":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"2404.13292","kind":"arxiv","version":1},"verdict":{"id":null,"model_set":{},"created_at":null,"strongest_claim":"","one_line_summary":"","pipeline_version":null,"weakest_assumption":"","pith_extraction_headline":""},"integrity":{"clean":true,"summary":{"advisory":0,"critical":0,"by_detector":{},"informational":0},"endpoint":"/pith/2404.13292/integrity.json","findings":[],"available":true,"detectors_run":[],"snapshot_sha256":"c28c3603d3b5d939e8dc4c7e95fa8dfce3d595e45f758748cecf8e644a296938"},"references":{"count":0,"sample":[],"resolved_work":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","internal_anchors":0},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"author_claims":{"count":0,"strong_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"builder_version":"pith-number-builder-2026-05-17-v1"},"aliases":[{"alias_kind":"arxiv","alias_value":"2404.13292","created_at":"2026-07-05T08:10:30.085474+00:00"},{"alias_kind":"arxiv_version","alias_value":"2404.13292v1","created_at":"2026-07-05T08:10:30.085474+00:00"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.2404.13292","created_at":"2026-07-05T08:10:30.085474+00:00"},{"alias_kind":"pith_short_12","alias_value":"MA3OHTYIAEWK","created_at":"2026-07-05T08:10:30.085474+00:00"},{"alias_kind":"pith_short_16","alias_value":"MA3OHTYIAEWKT7GI","created_at":"2026-07-05T08:10:30.085474+00:00"},{"alias_kind":"pith_short_8","alias_value":"MA3OHTYI","created_at":"2026-07-05T08:10:30.085474+00:00"}],"events":[],"event_summary":{},"paper_claims":[],"inbound_citations":{"count":0,"internal_anchor_count":0,"sample":[]},"formal_canon":{"evidence_count":0,"sample":[],"anchors":[]},"links":{"html":"https://pith.science/pith/MA3OHTYIAEWKT7GIK3TX2V76SY","json":"https://pith.science/pith/MA3OHTYIAEWKT7GIK3TX2V76SY.json","graph_json":"https://pith.science/api/pith-number/MA3OHTYIAEWKT7GIK3TX2V76SY/graph.json","events_json":"https://pith.science/api/pith-number/MA3OHTYIAEWKT7GIK3TX2V76SY/events.json","paper":"https://pith.science/paper/MA3OHTYI"},"agent_actions":{"view_html":"https://pith.science/pith/MA3OHTYIAEWKT7GIK3TX2V76SY","download_json":"https://pith.science/pith/MA3OHTYIAEWKT7GIK3TX2V76SY.json","view_paper":"https://pith.science/paper/MA3OHTYI","resolve_alias":"https://pith.science/api/pith-number/resolve?arxiv=2404.13292&json=true","fetch_graph":"https://pith.science/api/pith-number/MA3OHTYIAEWKT7GIK3TX2V76SY/graph.json","fetch_events":"https://pith.science/api/pith-number/MA3OHTYIAEWKT7GIK3TX2V76SY/events.json","actions":{"anchor_timestamp":"https://pith.science/pith/MA3OHTYIAEWKT7GIK3TX2V76SY/action/timestamp_anchor","attest_storage":"https://pith.science/pith/MA3OHTYIAEWKT7GIK3TX2V76SY/action/storage_attestation","attest_author":"https://pith.science/pith/MA3OHTYIAEWKT7GIK3TX2V76SY/action/author_attestation","sign_citation":"https://pith.science/pith/MA3OHTYIAEWKT7GIK3TX2V76SY/action/citation_signature","submit_replication":"https://pith.science/pith/MA3OHTYIAEWKT7GIK3TX2V76SY/action/replication_record"}},"created_at":"2026-07-05T08:10:30.085474+00:00","updated_at":"2026-07-05T08:10:30.085474+00:00"}