{"bundle_type":"pith_open_graph_bundle","bundle_version":"1.0","pith_number":"pith:2024:MA3OHTYIAEWKT7GIK3TX2V76SY","short_pith_number":"pith:MA3OHTYI","canonical_record":{"source":{"id":"2404.13292","kind":"arxiv","version":1},"metadata":{"license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","primary_cat":"cs.CL","submitted_at":"2024-04-20T06:49:15Z","cross_cats_sorted":["cs.AI"],"title_canon_sha256":"1f19f978fe636e4b7ee8ef4377b8028a9bd5608392fb441fbf8ee401f1f3a12a","abstract_canon_sha256":"e074dd8fb9685921049af5829134a8829fd0d50177fecce01b43b07984271a78"},"schema_version":"1.0"},"canonical_sha256":"6036e3cf08012ca9fcc856e77d57fe962d1b4ec287d575cfb435247505cbe652","source":{"kind":"arxiv","id":"2404.13292","version":1},"source_aliases":[{"alias_kind":"arxiv","alias_value":"2404.13292","created_at":"2026-07-05T08:10:30Z"},{"alias_kind":"arxiv_version","alias_value":"2404.13292v1","created_at":"2026-07-05T08:10:30Z"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.2404.13292","created_at":"2026-07-05T08:10:30Z"},{"alias_kind":"pith_short_12","alias_value":"MA3OHTYIAEWK","created_at":"2026-07-05T08:10:30Z"},{"alias_kind":"pith_short_16","alias_value":"MA3OHTYIAEWKT7GI","created_at":"2026-07-05T08:10:30Z"},{"alias_kind":"pith_short_8","alias_value":"MA3OHTYI","created_at":"2026-07-05T08:10:30Z"}],"events":[{"event_type":"record_created","subject_pith_number":"pith:2024:MA3OHTYIAEWKT7GIK3TX2V76SY","target":"record","payload":{"canonical_record":{"source":{"id":"2404.13292","kind":"arxiv","version":1},"metadata":{"license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","primary_cat":"cs.CL","submitted_at":"2024-04-20T06:49:15Z","cross_cats_sorted":["cs.AI"],"title_canon_sha256":"1f19f978fe636e4b7ee8ef4377b8028a9bd5608392fb441fbf8ee401f1f3a12a","abstract_canon_sha256":"e074dd8fb9685921049af5829134a8829fd0d50177fecce01b43b07984271a78"},"schema_version":"1.0"},"canonical_sha256":"6036e3cf08012ca9fcc856e77d57fe962d1b4ec287d575cfb435247505cbe652","receipt":{"kind":"pith_receipt","key_id":"pith-v1-2026-05","algorithm":"ed25519","signed_at":"2026-07-05T08:10:30.085885Z","signature_b64":"wvjaoidyJ+xxskonQpRSOdsYz9DGhaR6ZnGyzBgaB//yOfFcbwSQrSG8tdoZP38JGIYFrXyR9A8qruMpTiLAAw==","signed_message":"canonical_sha256_bytes","builder_version":"pith-number-builder-2026-05-17-v1","receipt_version":"0.3","canonical_sha256":"6036e3cf08012ca9fcc856e77d57fe962d1b4ec287d575cfb435247505cbe652","last_reissued_at":"2026-07-05T08:10:30.085410Z","signature_status":"signed_v1","first_computed_at":"2026-07-05T08:10:30.085410Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"source_kind":"arxiv","source_id":"2404.13292","source_version":1,"attestation_state":"computed"},"signer":{"signer_id":"pith.science","signer_type":"pith_registry","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"created_at":"2026-07-05T08:10:30Z","supersedes":[],"prev_event":null,"signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"aHvpImedlAM05VfyQJsFyCdoZ8MC2H3MTJyT3cdM72qvgpy+LFRpZIEROGaEEL/y9J42zWyOOzhdfsVuSNDpCQ==","signed_message":"open_graph_event_sha256_bytes","signed_at":"2026-07-09T06:54:36.658387Z"},"content_sha256":"94f755e87e9a279bea7f3574af6bbf8130cf30f1d897efc5a7e889f43a681935","schema_version":"1.0","event_id":"sha256:94f755e87e9a279bea7f3574af6bbf8130cf30f1d897efc5a7e889f43a681935"},{"event_type":"graph_snapshot","subject_pith_number":"pith:2024:MA3OHTYIAEWKT7GIK3TX2V76SY","target":"graph","payload":{"graph_snapshot":{"paper":{"title":"Evaluating Subword Tokenization: Alien Subword Composition and OOV Generalization Challenge","license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","headline":"","cross_cats":["cs.AI"],"primary_cat":"cs.CL","authors_text":"Ekaterina Vylomova, G\\'abor Bella, Khuyagbaatar Batsuren, Omri Uzan, Tsetsuukhei Delgerbaatar, Verna Dankers, Yuval Pinter","submitted_at":"2024-04-20T06:49:15Z","abstract_excerpt":"The popular subword tokenizers of current language models, such as Byte-Pair Encoding (BPE), are known not to respect morpheme boundaries, which affects the downstream performance of the models. While many improved tokenization algorithms have been proposed, their evaluation and cross-comparison is still an open problem. As a solution, we propose a combined intrinsic-extrinsic evaluation framework for subword tokenization. Intrinsic evaluation is based on our new UniMorph Labeller tool that classifies subword tokenization as either morphological or alien. Extrinsic evaluation, in turn, is perf"},"claims":{"count":0,"items":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"2404.13292","kind":"arxiv","version":1},"verdict":{"id":null,"model_set":{},"created_at":null,"strongest_claim":"","one_line_summary":"","pipeline_version":null,"weakest_assumption":"","pith_extraction_headline":""},"integrity":{"clean":true,"summary":{"advisory":0,"critical":0,"by_detector":{},"informational":0},"endpoint":"/pith/2404.13292/integrity.json","findings":[],"available":true,"detectors_run":[],"snapshot_sha256":"c28c3603d3b5d939e8dc4c7e95fa8dfce3d595e45f758748cecf8e644a296938"},"references":{"count":0,"sample":[],"resolved_work":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","internal_anchors":0},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"author_claims":{"count":0,"strong_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"builder_version":"pith-number-builder-2026-05-17-v1"},"verdict_id":null},"signer":{"signer_id":"pith.science","signer_type":"pith_registry","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"created_at":"2026-07-05T08:10:30Z","supersedes":[],"prev_event":null,"signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"zR6t4lJpwaxy9Ft0CMO4uk1HnG781x7eJeT/LGr/UJebyO/m4bRj+9afWhHBL6DOTJxrBwqu5M6i9YSENT2PDA==","signed_message":"open_graph_event_sha256_bytes","signed_at":"2026-07-09T06:54:36.658774Z"},"content_sha256":"7c0e6c183b02cd82fbbf392f4b8dabe8f65914ef0b2353adf7b9eb84861745d6","schema_version":"1.0","event_id":"sha256:7c0e6c183b02cd82fbbf392f4b8dabe8f65914ef0b2353adf7b9eb84861745d6"}],"timestamp_proofs":[],"mirror_hints":[{"mirror_type":"https","name":"Pith Resolver","base_url":"https://pith.science","bundle_url":"https://pith.science/pith/MA3OHTYIAEWKT7GIK3TX2V76SY/bundle.json","state_url":"https://pith.science/pith/MA3OHTYIAEWKT7GIK3TX2V76SY/state.json","well_known_bundle_url":"https://pith.science/.well-known/pith/MA3OHTYIAEWKT7GIK3TX2V76SY/bundle.json","status":"primary"}],"public_keys":[{"key_id":"pith-v1-2026-05","algorithm":"ed25519","format":"raw","public_key_b64":"stVStoiQhXFxp4s2pdzPNoqVNBMojDU/fJ2db5S3CbM=","public_key_hex":"b2d552b68890857171a78b36a5dccf368a953413288c353f7c9d9d6f94b709b3","fingerprint_sha256_b32_first128bits":"RVFV5Z2OI2J3ZUO7ERDEBCYNKS","fingerprint_sha256_hex":"8d4b5ee74e4693bcd1df2446408b0d54","rotates_at":null,"url":"https://pith.science/pith-signing-key.json","notes":"Pith uses this Ed25519 key to sign canonical record SHA-256 digests. Verify with: ed25519_verify(public_key, message=canonical_sha256_bytes, signature=base64decode(signature_b64))."}],"merge_version":"pith-open-graph-merge-v1","built_at":"2026-07-09T06:54:36Z","links":{"resolver":"https://pith.science/pith/MA3OHTYIAEWKT7GIK3TX2V76SY","bundle":"https://pith.science/pith/MA3OHTYIAEWKT7GIK3TX2V76SY/bundle.json","state":"https://pith.science/pith/MA3OHTYIAEWKT7GIK3TX2V76SY/state.json","well_known_bundle":"https://pith.science/.well-known/pith/MA3OHTYIAEWKT7GIK3TX2V76SY/bundle.json"},"state":{"state_type":"pith_open_graph_state","state_version":"1.0","pith_number":"pith:2024:MA3OHTYIAEWKT7GIK3TX2V76SY","merge_version":"pith-open-graph-merge-v1","event_count":2,"valid_event_count":2,"invalid_event_count":0,"equivocation_count":0,"current":{"canonical_record":{"metadata":{"abstract_canon_sha256":"e074dd8fb9685921049af5829134a8829fd0d50177fecce01b43b07984271a78","cross_cats_sorted":["cs.AI"],"license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","primary_cat":"cs.CL","submitted_at":"2024-04-20T06:49:15Z","title_canon_sha256":"1f19f978fe636e4b7ee8ef4377b8028a9bd5608392fb441fbf8ee401f1f3a12a"},"schema_version":"1.0","source":{"id":"2404.13292","kind":"arxiv","version":1}},"source_aliases":[{"alias_kind":"arxiv","alias_value":"2404.13292","created_at":"2026-07-05T08:10:30Z"},{"alias_kind":"arxiv_version","alias_value":"2404.13292v1","created_at":"2026-07-05T08:10:30Z"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.2404.13292","created_at":"2026-07-05T08:10:30Z"},{"alias_kind":"pith_short_12","alias_value":"MA3OHTYIAEWK","created_at":"2026-07-05T08:10:30Z"},{"alias_kind":"pith_short_16","alias_value":"MA3OHTYIAEWKT7GI","created_at":"2026-07-05T08:10:30Z"},{"alias_kind":"pith_short_8","alias_value":"MA3OHTYI","created_at":"2026-07-05T08:10:30Z"}],"graph_snapshots":[{"event_id":"sha256:7c0e6c183b02cd82fbbf392f4b8dabe8f65914ef0b2353adf7b9eb84861745d6","target":"graph","created_at":"2026-07-05T08:10:30Z","signer":{"key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signer_id":"pith.science","signer_type":"pith_registry"},"payload":{"graph_snapshot":{"author_claims":{"count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","strong_count":0},"builder_version":"pith-number-builder-2026-05-17-v1","claims":{"count":0,"items":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"integrity":{"available":true,"clean":true,"detectors_run":[],"endpoint":"/pith/2404.13292/integrity.json","findings":[],"snapshot_sha256":"c28c3603d3b5d939e8dc4c7e95fa8dfce3d595e45f758748cecf8e644a296938","summary":{"advisory":0,"by_detector":{},"critical":0,"informational":0}},"paper":{"abstract_excerpt":"The popular subword tokenizers of current language models, such as Byte-Pair Encoding (BPE), are known not to respect morpheme boundaries, which affects the downstream performance of the models. While many improved tokenization algorithms have been proposed, their evaluation and cross-comparison is still an open problem. As a solution, we propose a combined intrinsic-extrinsic evaluation framework for subword tokenization. Intrinsic evaluation is based on our new UniMorph Labeller tool that classifies subword tokenization as either morphological or alien. Extrinsic evaluation, in turn, is perf","authors_text":"Ekaterina Vylomova, G\\'abor Bella, Khuyagbaatar Batsuren, Omri Uzan, Tsetsuukhei Delgerbaatar, Verna Dankers, Yuval Pinter","cross_cats":["cs.AI"],"headline":"","license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","primary_cat":"cs.CL","submitted_at":"2024-04-20T06:49:15Z","title":"Evaluating Subword Tokenization: Alien Subword Composition and OOV Generalization Challenge"},"references":{"count":0,"internal_anchors":0,"resolved_work":0,"sample":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"2404.13292","kind":"arxiv","version":1},"verdict":{"created_at":null,"id":null,"model_set":{},"one_line_summary":"","pipeline_version":null,"pith_extraction_headline":"","strongest_claim":"","weakest_assumption":""}},"verdict_id":null}}],"author_attestations":[],"timestamp_anchors":[],"storage_attestations":[],"citation_signatures":[],"replication_records":[],"corrections":[],"mirror_hints":[],"record_created":{"event_id":"sha256:94f755e87e9a279bea7f3574af6bbf8130cf30f1d897efc5a7e889f43a681935","target":"record","created_at":"2026-07-05T08:10:30Z","signer":{"key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signer_id":"pith.science","signer_type":"pith_registry"},"payload":{"attestation_state":"computed","canonical_record":{"metadata":{"abstract_canon_sha256":"e074dd8fb9685921049af5829134a8829fd0d50177fecce01b43b07984271a78","cross_cats_sorted":["cs.AI"],"license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","primary_cat":"cs.CL","submitted_at":"2024-04-20T06:49:15Z","title_canon_sha256":"1f19f978fe636e4b7ee8ef4377b8028a9bd5608392fb441fbf8ee401f1f3a12a"},"schema_version":"1.0","source":{"id":"2404.13292","kind":"arxiv","version":1}},"canonical_sha256":"6036e3cf08012ca9fcc856e77d57fe962d1b4ec287d575cfb435247505cbe652","receipt":{"algorithm":"ed25519","builder_version":"pith-number-builder-2026-05-17-v1","canonical_sha256":"6036e3cf08012ca9fcc856e77d57fe962d1b4ec287d575cfb435247505cbe652","first_computed_at":"2026-07-05T08:10:30.085410Z","key_id":"pith-v1-2026-05","kind":"pith_receipt","last_reissued_at":"2026-07-05T08:10:30.085410Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","receipt_version":"0.3","signature_b64":"wvjaoidyJ+xxskonQpRSOdsYz9DGhaR6ZnGyzBgaB//yOfFcbwSQrSG8tdoZP38JGIYFrXyR9A8qruMpTiLAAw==","signature_status":"signed_v1","signed_at":"2026-07-05T08:10:30.085885Z","signed_message":"canonical_sha256_bytes"},"source_id":"2404.13292","source_kind":"arxiv","source_version":1}}},"equivocations":[],"invalid_events":[],"applied_event_ids":["sha256:94f755e87e9a279bea7f3574af6bbf8130cf30f1d897efc5a7e889f43a681935","sha256:7c0e6c183b02cd82fbbf392f4b8dabe8f65914ef0b2353adf7b9eb84861745d6"],"state_sha256":"314df6fe26bf8b42aa2721c0b57f1682e8992142fea862c2a06d679171f345e5"},"bundle_signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"wKyabNTn6+5ntGN7LU7P8CULrOL/DwV6vNLG6PH6tgSqAysLf/s/hDRLbK/1R5CxfTV4vNrjPZOW7tGerSzNDw==","signed_message":"bundle_sha256_bytes","signed_at":"2026-07-09T06:54:36.661068Z","bundle_sha256":"13200bf5c9b0b298c3bbb6331f083f1a29d917ba9eda77f3a4184759a045d36e"}}