{"bundle_type":"pith_open_graph_bundle","bundle_version":"1.0","pith_number":"pith:2022:ODTCHLR2MLIRYQAIDTZON74J6V","short_pith_number":"pith:ODTCHLR2","canonical_record":{"source":{"id":"2204.08832","kind":"arxiv","version":1},"metadata":{"license":"http://creativecommons.org/licenses/by-nc-sa/4.0/","primary_cat":"cs.CL","submitted_at":"2022-04-19T12:01:46Z","cross_cats_sorted":[],"title_canon_sha256":"9d61ea2f8abd52b13a9149c0ce8f4b295f718df3be2ffd3268811607d962e9ea","abstract_canon_sha256":"9e39d2d3acfa82fca1b0cf34619060e8a80c35d81a1918ce9c4ba9f1c7075475"},"schema_version":"1.0"},"canonical_sha256":"70e623ae3a62d11c40081cf2e6ff89f56438b4d1a429448896964e3997726be9","source":{"kind":"arxiv","id":"2204.08832","version":1},"source_aliases":[{"alias_kind":"arxiv","alias_value":"2204.08832","created_at":"2026-07-05T05:54:28Z"},{"alias_kind":"arxiv_version","alias_value":"2204.08832v1","created_at":"2026-07-05T05:54:28Z"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.2204.08832","created_at":"2026-07-05T05:54:28Z"},{"alias_kind":"pith_short_12","alias_value":"ODTCHLR2MLIR","created_at":"2026-07-05T05:54:28Z"},{"alias_kind":"pith_short_16","alias_value":"ODTCHLR2MLIRYQAI","created_at":"2026-07-05T05:54:28Z"},{"alias_kind":"pith_short_8","alias_value":"ODTCHLR2","created_at":"2026-07-05T05:54:28Z"}],"events":[{"event_type":"record_created","subject_pith_number":"pith:2022:ODTCHLR2MLIRYQAIDTZON74J6V","target":"record","payload":{"canonical_record":{"source":{"id":"2204.08832","kind":"arxiv","version":1},"metadata":{"license":"http://creativecommons.org/licenses/by-nc-sa/4.0/","primary_cat":"cs.CL","submitted_at":"2022-04-19T12:01:46Z","cross_cats_sorted":[],"title_canon_sha256":"9d61ea2f8abd52b13a9149c0ce8f4b295f718df3be2ffd3268811607d962e9ea","abstract_canon_sha256":"9e39d2d3acfa82fca1b0cf34619060e8a80c35d81a1918ce9c4ba9f1c7075475"},"schema_version":"1.0"},"canonical_sha256":"70e623ae3a62d11c40081cf2e6ff89f56438b4d1a429448896964e3997726be9","receipt":{"kind":"pith_receipt","key_id":"pith-v1-2026-05","algorithm":"ed25519","signed_at":"2026-07-05T05:54:28.788600Z","signature_b64":"PEOYopx8NBsFBB6hlnQxJB+fW0VzHPirHFvZX5xAkMRDnfmLModybg2q//r1nkrxL6+r9SIdsOQ39xc+gQ5qDA==","signed_message":"canonical_sha256_bytes","builder_version":"pith-number-builder-2026-05-17-v1","receipt_version":"0.3","canonical_sha256":"70e623ae3a62d11c40081cf2e6ff89f56438b4d1a429448896964e3997726be9","last_reissued_at":"2026-07-05T05:54:28.788063Z","signature_status":"signed_v1","first_computed_at":"2026-07-05T05:54:28.788063Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"source_kind":"arxiv","source_id":"2204.08832","source_version":1,"attestation_state":"computed"},"signer":{"signer_id":"pith.science","signer_type":"pith_registry","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"created_at":"2026-07-05T05:54:28Z","supersedes":[],"prev_event":null,"signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"tsFjJn/op7vkLmLiVRksuthGKkOBqj9kVdPe73q43OeiIHLsPjl9VpIFJ/7O6xRXKXlRX9mhwtAZNvc4tNAoDw==","signed_message":"open_graph_event_sha256_bytes","signed_at":"2026-07-16T10:30:48.093683Z"},"content_sha256":"21dc8dd1480df33b9efd94728913666056b51964b8d5a22efd3d264081f97a91","schema_version":"1.0","event_id":"sha256:21dc8dd1480df33b9efd94728913666056b51964b8d5a22efd3d264081f97a91"},{"event_type":"graph_snapshot","subject_pith_number":"pith:2022:ODTCHLR2MLIRYQAIDTZON74J6V","target":"graph","payload":{"graph_snapshot":{"paper":{"title":"Impact of Tokenization on Language Models: An Analysis for Turkish","license":"http://creativecommons.org/licenses/by-nc-sa/4.0/","headline":"","cross_cats":[],"primary_cat":"cs.CL","authors_text":"Cagri Toraman, Eyup Halit Yilmaz, Furkan \\c{S}ahinu\\c{c}, Oguzhan Ozcelik","submitted_at":"2022-04-19T12:01:46Z","abstract_excerpt":"Tokenization is an important text preprocessing step to prepare input tokens for deep language models. WordPiece and BPE are de facto methods employed by important models, such as BERT and GPT. However, the impact of tokenization can be different for morphologically rich languages, such as Turkic languages, where many words can be generated by adding prefixes and suffixes. We compare five tokenizers at different granularity levels, i.e. their outputs vary from smallest pieces of characters to the surface form of words, including a Morphological-level tokenizer. We train these tokenizers and pr"},"claims":{"count":0,"items":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"2204.08832","kind":"arxiv","version":1},"verdict":{"id":null,"model_set":{},"created_at":null,"strongest_claim":"","one_line_summary":"","pipeline_version":null,"weakest_assumption":"","pith_extraction_headline":""},"integrity":{"clean":true,"summary":{"advisory":0,"critical":0,"by_detector":{},"informational":0},"endpoint":"/pith/2204.08832/integrity.json","findings":[],"available":true,"detectors_run":[],"snapshot_sha256":"c28c3603d3b5d939e8dc4c7e95fa8dfce3d595e45f758748cecf8e644a296938"},"references":{"count":0,"sample":[],"resolved_work":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","internal_anchors":0},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"author_claims":{"count":0,"strong_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"builder_version":"pith-number-builder-2026-05-17-v1"},"verdict_id":null},"signer":{"signer_id":"pith.science","signer_type":"pith_registry","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"created_at":"2026-07-05T05:54:28Z","supersedes":[],"prev_event":null,"signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"fIg8fsUdNUWCqSENJPT9qMTC5KtpTzfGC3xT+ElDRHSojeX27irIwFwSlFm6ereyaJcFSxK5wVnF8pbA5r/8BQ==","signed_message":"open_graph_event_sha256_bytes","signed_at":"2026-07-16T10:30:48.094061Z"},"content_sha256":"a3bf8cc09a5c59a68745db59d71425f53895e18074e73d7209938ad4ef866fb1","schema_version":"1.0","event_id":"sha256:a3bf8cc09a5c59a68745db59d71425f53895e18074e73d7209938ad4ef866fb1"}],"timestamp_proofs":[],"mirror_hints":[{"mirror_type":"https","name":"Pith Resolver","base_url":"https://pith.science","bundle_url":"https://pith.science/pith/ODTCHLR2MLIRYQAIDTZON74J6V/bundle.json","state_url":"https://pith.science/pith/ODTCHLR2MLIRYQAIDTZON74J6V/state.json","well_known_bundle_url":"https://pith.science/.well-known/pith/ODTCHLR2MLIRYQAIDTZON74J6V/bundle.json","status":"primary"}],"public_keys":[{"key_id":"pith-v1-2026-05","algorithm":"ed25519","format":"raw","public_key_b64":"stVStoiQhXFxp4s2pdzPNoqVNBMojDU/fJ2db5S3CbM=","public_key_hex":"b2d552b68890857171a78b36a5dccf368a953413288c353f7c9d9d6f94b709b3","fingerprint_sha256_b32_first128bits":"RVFV5Z2OI2J3ZUO7ERDEBCYNKS","fingerprint_sha256_hex":"8d4b5ee74e4693bcd1df2446408b0d54","rotates_at":null,"url":"https://pith.science/pith-signing-key.json","notes":"Pith uses this Ed25519 key to sign canonical record SHA-256 digests. Verify with: ed25519_verify(public_key, message=canonical_sha256_bytes, signature=base64decode(signature_b64))."}],"merge_version":"pith-open-graph-merge-v1","built_at":"2026-07-16T10:30:48Z","links":{"resolver":"https://pith.science/pith/ODTCHLR2MLIRYQAIDTZON74J6V","bundle":"https://pith.science/pith/ODTCHLR2MLIRYQAIDTZON74J6V/bundle.json","state":"https://pith.science/pith/ODTCHLR2MLIRYQAIDTZON74J6V/state.json","well_known_bundle":"https://pith.science/.well-known/pith/ODTCHLR2MLIRYQAIDTZON74J6V/bundle.json"},"state":{"state_type":"pith_open_graph_state","state_version":"1.0","pith_number":"pith:2022:ODTCHLR2MLIRYQAIDTZON74J6V","merge_version":"pith-open-graph-merge-v1","event_count":2,"valid_event_count":2,"invalid_event_count":0,"equivocation_count":0,"current":{"canonical_record":{"metadata":{"abstract_canon_sha256":"9e39d2d3acfa82fca1b0cf34619060e8a80c35d81a1918ce9c4ba9f1c7075475","cross_cats_sorted":[],"license":"http://creativecommons.org/licenses/by-nc-sa/4.0/","primary_cat":"cs.CL","submitted_at":"2022-04-19T12:01:46Z","title_canon_sha256":"9d61ea2f8abd52b13a9149c0ce8f4b295f718df3be2ffd3268811607d962e9ea"},"schema_version":"1.0","source":{"id":"2204.08832","kind":"arxiv","version":1}},"source_aliases":[{"alias_kind":"arxiv","alias_value":"2204.08832","created_at":"2026-07-05T05:54:28Z"},{"alias_kind":"arxiv_version","alias_value":"2204.08832v1","created_at":"2026-07-05T05:54:28Z"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.2204.08832","created_at":"2026-07-05T05:54:28Z"},{"alias_kind":"pith_short_12","alias_value":"ODTCHLR2MLIR","created_at":"2026-07-05T05:54:28Z"},{"alias_kind":"pith_short_16","alias_value":"ODTCHLR2MLIRYQAI","created_at":"2026-07-05T05:54:28Z"},{"alias_kind":"pith_short_8","alias_value":"ODTCHLR2","created_at":"2026-07-05T05:54:28Z"}],"graph_snapshots":[{"event_id":"sha256:a3bf8cc09a5c59a68745db59d71425f53895e18074e73d7209938ad4ef866fb1","target":"graph","created_at":"2026-07-05T05:54:28Z","signer":{"key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signer_id":"pith.science","signer_type":"pith_registry"},"payload":{"graph_snapshot":{"author_claims":{"count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","strong_count":0},"builder_version":"pith-number-builder-2026-05-17-v1","claims":{"count":0,"items":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"integrity":{"available":true,"clean":true,"detectors_run":[],"endpoint":"/pith/2204.08832/integrity.json","findings":[],"snapshot_sha256":"c28c3603d3b5d939e8dc4c7e95fa8dfce3d595e45f758748cecf8e644a296938","summary":{"advisory":0,"by_detector":{},"critical":0,"informational":0}},"paper":{"abstract_excerpt":"Tokenization is an important text preprocessing step to prepare input tokens for deep language models. WordPiece and BPE are de facto methods employed by important models, such as BERT and GPT. However, the impact of tokenization can be different for morphologically rich languages, such as Turkic languages, where many words can be generated by adding prefixes and suffixes. We compare five tokenizers at different granularity levels, i.e. their outputs vary from smallest pieces of characters to the surface form of words, including a Morphological-level tokenizer. We train these tokenizers and pr","authors_text":"Cagri Toraman, Eyup Halit Yilmaz, Furkan \\c{S}ahinu\\c{c}, Oguzhan Ozcelik","cross_cats":[],"headline":"","license":"http://creativecommons.org/licenses/by-nc-sa/4.0/","primary_cat":"cs.CL","submitted_at":"2022-04-19T12:01:46Z","title":"Impact of Tokenization on Language Models: An Analysis for Turkish"},"references":{"count":0,"internal_anchors":0,"resolved_work":0,"sample":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"2204.08832","kind":"arxiv","version":1},"verdict":{"created_at":null,"id":null,"model_set":{},"one_line_summary":"","pipeline_version":null,"pith_extraction_headline":"","strongest_claim":"","weakest_assumption":""}},"verdict_id":null}}],"author_attestations":[],"timestamp_anchors":[],"storage_attestations":[],"citation_signatures":[],"replication_records":[],"corrections":[],"mirror_hints":[],"record_created":{"event_id":"sha256:21dc8dd1480df33b9efd94728913666056b51964b8d5a22efd3d264081f97a91","target":"record","created_at":"2026-07-05T05:54:28Z","signer":{"key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signer_id":"pith.science","signer_type":"pith_registry"},"payload":{"attestation_state":"computed","canonical_record":{"metadata":{"abstract_canon_sha256":"9e39d2d3acfa82fca1b0cf34619060e8a80c35d81a1918ce9c4ba9f1c7075475","cross_cats_sorted":[],"license":"http://creativecommons.org/licenses/by-nc-sa/4.0/","primary_cat":"cs.CL","submitted_at":"2022-04-19T12:01:46Z","title_canon_sha256":"9d61ea2f8abd52b13a9149c0ce8f4b295f718df3be2ffd3268811607d962e9ea"},"schema_version":"1.0","source":{"id":"2204.08832","kind":"arxiv","version":1}},"canonical_sha256":"70e623ae3a62d11c40081cf2e6ff89f56438b4d1a429448896964e3997726be9","receipt":{"algorithm":"ed25519","builder_version":"pith-number-builder-2026-05-17-v1","canonical_sha256":"70e623ae3a62d11c40081cf2e6ff89f56438b4d1a429448896964e3997726be9","first_computed_at":"2026-07-05T05:54:28.788063Z","key_id":"pith-v1-2026-05","kind":"pith_receipt","last_reissued_at":"2026-07-05T05:54:28.788063Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","receipt_version":"0.3","signature_b64":"PEOYopx8NBsFBB6hlnQxJB+fW0VzHPirHFvZX5xAkMRDnfmLModybg2q//r1nkrxL6+r9SIdsOQ39xc+gQ5qDA==","signature_status":"signed_v1","signed_at":"2026-07-05T05:54:28.788600Z","signed_message":"canonical_sha256_bytes"},"source_id":"2204.08832","source_kind":"arxiv","source_version":1}}},"equivocations":[],"invalid_events":[],"applied_event_ids":["sha256:21dc8dd1480df33b9efd94728913666056b51964b8d5a22efd3d264081f97a91","sha256:a3bf8cc09a5c59a68745db59d71425f53895e18074e73d7209938ad4ef866fb1"],"state_sha256":"043a79b44452b4cda6395362b52a76988e9f209f2a61ce734fc590b9d83f08c1"},"bundle_signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"7Xb3QvEdwWxB0J3fIhNfNx81QDVjRYiM6Fv3kAFUFzjih6+Neen2MTbH5hXaD/uEymcB4lf5QCQ58ypOGEiiBA==","signed_message":"bundle_sha256_bytes","signed_at":"2026-07-16T10:30:48.096685Z","bundle_sha256":"5f371e6fb53ee125140c76ee5ffeb4aa19eed674e49b9c0dc076980827f812d3"}}