{"bundle_type":"pith_open_graph_bundle","bundle_version":"1.0","pith_number":"pith:2020:GDLRMKVXVZKGJMMZNUDOJEXHUN","short_pith_number":"pith:GDLRMKVX","canonical_record":{"source":{"id":"2010.02534","kind":"arxiv","version":1},"metadata":{"license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","primary_cat":"cs.CL","submitted_at":"2020-10-06T07:20:41Z","cross_cats_sorted":[],"title_canon_sha256":"961fcadd49d2f45b8fe575e16c81ee60f1fd346c304c58243a611fa42f79a60a","abstract_canon_sha256":"61f3e18797c9c3b0ba986d3f5ca50bdb34e7e9fcf1cb840f15d0d7be25883835"},"schema_version":"1.0"},"canonical_sha256":"30d7162ab7ae5464b1996d06e492e7a3521dfd345aa7cdc326a293be6201f88e","source":{"kind":"arxiv","id":"2010.02534","version":1},"source_aliases":[{"alias_kind":"arxiv","alias_value":"2010.02534","created_at":"2026-07-05T01:40:51Z"},{"alias_kind":"arxiv_version","alias_value":"2010.02534v1","created_at":"2026-07-05T01:40:51Z"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.2010.02534","created_at":"2026-07-05T01:40:51Z"},{"alias_kind":"pith_short_12","alias_value":"GDLRMKVXVZKG","created_at":"2026-07-05T01:40:51Z"},{"alias_kind":"pith_short_16","alias_value":"GDLRMKVXVZKGJMMZ","created_at":"2026-07-05T01:40:51Z"},{"alias_kind":"pith_short_8","alias_value":"GDLRMKVX","created_at":"2026-07-05T01:40:51Z"}],"events":[{"event_type":"record_created","subject_pith_number":"pith:2020:GDLRMKVXVZKGJMMZNUDOJEXHUN","target":"record","payload":{"canonical_record":{"source":{"id":"2010.02534","kind":"arxiv","version":1},"metadata":{"license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","primary_cat":"cs.CL","submitted_at":"2020-10-06T07:20:41Z","cross_cats_sorted":[],"title_canon_sha256":"961fcadd49d2f45b8fe575e16c81ee60f1fd346c304c58243a611fa42f79a60a","abstract_canon_sha256":"61f3e18797c9c3b0ba986d3f5ca50bdb34e7e9fcf1cb840f15d0d7be25883835"},"schema_version":"1.0"},"canonical_sha256":"30d7162ab7ae5464b1996d06e492e7a3521dfd345aa7cdc326a293be6201f88e","receipt":{"kind":"pith_receipt","key_id":"pith-v1-2026-05","algorithm":"ed25519","signed_at":"2026-07-05T01:40:51.124179Z","signature_b64":"AShQOoC+1UrwYPG4WXpSf9+4db4ArVZsfg5SlyG+ltoYvwllF7++pM6kVBgghCz5uK2aorHhu0g6JKxoRF6sAw==","signed_message":"canonical_sha256_bytes","builder_version":"pith-number-builder-2026-05-17-v1","receipt_version":"0.3","canonical_sha256":"30d7162ab7ae5464b1996d06e492e7a3521dfd345aa7cdc326a293be6201f88e","last_reissued_at":"2026-07-05T01:40:51.123795Z","signature_status":"signed_v1","first_computed_at":"2026-07-05T01:40:51.123795Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"source_kind":"arxiv","source_id":"2010.02534","source_version":1,"attestation_state":"computed"},"signer":{"signer_id":"pith.science","signer_type":"pith_registry","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"created_at":"2026-07-05T01:40:51Z","supersedes":[],"prev_event":null,"signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"0nwJKMhqiBrtvMSsb4KBtmxImUXoQWRB7q572KWTrLsYwV9niu8IAHS5w4SaUMxJNsVY1JTV4zob0PTxadpCBg==","signed_message":"open_graph_event_sha256_bytes","signed_at":"2026-08-09T01:56:47.232255Z"},"content_sha256":"96ebb0544da1c28f597c349da5ebae38beecbd3b2815207f3faa6e7dfc38762f","schema_version":"1.0","event_id":"sha256:96ebb0544da1c28f597c349da5ebae38beecbd3b2815207f3faa6e7dfc38762f"},{"event_type":"graph_snapshot","subject_pith_number":"pith:2020:GDLRMKVXVZKGJMMZNUDOJEXHUN","target":"graph","payload":{"graph_snapshot":{"paper":{"title":"An Empirical Study of Tokenization Strategies for Various Korean NLP Tasks","license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","headline":"","cross_cats":[],"primary_cat":"cs.CL","authors_text":"Dawoon Jung, Joohong Lee, Kyubyong Park, Seongbo Jang","submitted_at":"2020-10-06T07:20:41Z","abstract_excerpt":"Typically, tokenization is the very first step in most text processing works. As a token serves as an atomic unit that embeds the contextual information of text, how to define a token plays a decisive role in the performance of a model.Even though Byte Pair Encoding (BPE) has been considered the de facto standard tokenization method due to its simplicity and universality, it still remains unclear whether BPE works best across all languages and tasks. In this paper, we test several tokenization strategies in order to answer our primary research question, that is, \"What is the best tokenization "},"claims":{"count":0,"items":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"2010.02534","kind":"arxiv","version":1},"verdict":{"id":null,"model_set":{},"created_at":null,"strongest_claim":"","one_line_summary":"","pipeline_version":null,"weakest_assumption":"","pith_extraction_headline":""},"integrity":{"clean":true,"summary":{"advisory":0,"critical":0,"by_detector":{},"informational":0},"endpoint":"/pith/2010.02534/integrity.json","findings":[],"available":true,"detectors_run":[],"snapshot_sha256":"c28c3603d3b5d939e8dc4c7e95fa8dfce3d595e45f758748cecf8e644a296938"},"references":{"count":0,"sample":[],"resolved_work":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","internal_anchors":0},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"author_claims":{"count":0,"strong_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"builder_version":"pith-number-builder-2026-05-17-v1"},"verdict_id":null},"signer":{"signer_id":"pith.science","signer_type":"pith_registry","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"created_at":"2026-07-05T01:40:51Z","supersedes":[],"prev_event":null,"signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"nZlOqBtX2yOQh46QnrtcAT2+j8LACs4a7H3rnwnuFSen7Wf5AaEwK+q8xMrkxDOXFVQ7PC269KEB/Y9wTNJBCg==","signed_message":"open_graph_event_sha256_bytes","signed_at":"2026-08-09T01:56:47.232779Z"},"content_sha256":"56f8c8779b6e514b5992e24e9ae67391a8a2d27365cb4a5dababc1a0378d9d90","schema_version":"1.0","event_id":"sha256:56f8c8779b6e514b5992e24e9ae67391a8a2d27365cb4a5dababc1a0378d9d90"}],"timestamp_proofs":[],"mirror_hints":[{"mirror_type":"https","name":"Pith Resolver","base_url":"https://pith.science","bundle_url":"https://pith.science/pith/GDLRMKVXVZKGJMMZNUDOJEXHUN/bundle.json","state_url":"https://pith.science/pith/GDLRMKVXVZKGJMMZNUDOJEXHUN/state.json","well_known_bundle_url":"https://pith.science/.well-known/pith/GDLRMKVXVZKGJMMZNUDOJEXHUN/bundle.json","status":"primary"}],"public_keys":[{"key_id":"pith-v1-2026-05","algorithm":"ed25519","format":"raw","public_key_b64":"stVStoiQhXFxp4s2pdzPNoqVNBMojDU/fJ2db5S3CbM=","public_key_hex":"b2d552b68890857171a78b36a5dccf368a953413288c353f7c9d9d6f94b709b3","fingerprint_sha256_b32_first128bits":"RVFV5Z2OI2J3ZUO7ERDEBCYNKS","fingerprint_sha256_hex":"8d4b5ee74e4693bcd1df2446408b0d54","rotates_at":null,"url":"https://pith.science/pith-signing-key.json","notes":"Pith uses this Ed25519 key to sign canonical record SHA-256 digests. Verify with: ed25519_verify(public_key, message=canonical_sha256_bytes, signature=base64decode(signature_b64))."}],"merge_version":"pith-open-graph-merge-v1","built_at":"2026-08-09T01:56:47Z","links":{"resolver":"https://pith.science/pith/GDLRMKVXVZKGJMMZNUDOJEXHUN","bundle":"https://pith.science/pith/GDLRMKVXVZKGJMMZNUDOJEXHUN/bundle.json","state":"https://pith.science/pith/GDLRMKVXVZKGJMMZNUDOJEXHUN/state.json","well_known_bundle":"https://pith.science/.well-known/pith/GDLRMKVXVZKGJMMZNUDOJEXHUN/bundle.json"},"state":{"state_type":"pith_open_graph_state","state_version":"1.0","pith_number":"pith:2020:GDLRMKVXVZKGJMMZNUDOJEXHUN","merge_version":"pith-open-graph-merge-v1","event_count":2,"valid_event_count":2,"invalid_event_count":0,"equivocation_count":0,"current":{"canonical_record":{"metadata":{"abstract_canon_sha256":"61f3e18797c9c3b0ba986d3f5ca50bdb34e7e9fcf1cb840f15d0d7be25883835","cross_cats_sorted":[],"license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","primary_cat":"cs.CL","submitted_at":"2020-10-06T07:20:41Z","title_canon_sha256":"961fcadd49d2f45b8fe575e16c81ee60f1fd346c304c58243a611fa42f79a60a"},"schema_version":"1.0","source":{"id":"2010.02534","kind":"arxiv","version":1}},"source_aliases":[{"alias_kind":"arxiv","alias_value":"2010.02534","created_at":"2026-07-05T01:40:51Z"},{"alias_kind":"arxiv_version","alias_value":"2010.02534v1","created_at":"2026-07-05T01:40:51Z"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.2010.02534","created_at":"2026-07-05T01:40:51Z"},{"alias_kind":"pith_short_12","alias_value":"GDLRMKVXVZKG","created_at":"2026-07-05T01:40:51Z"},{"alias_kind":"pith_short_16","alias_value":"GDLRMKVXVZKGJMMZ","created_at":"2026-07-05T01:40:51Z"},{"alias_kind":"pith_short_8","alias_value":"GDLRMKVX","created_at":"2026-07-05T01:40:51Z"}],"graph_snapshots":[{"event_id":"sha256:56f8c8779b6e514b5992e24e9ae67391a8a2d27365cb4a5dababc1a0378d9d90","target":"graph","created_at":"2026-07-05T01:40:51Z","signer":{"key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signer_id":"pith.science","signer_type":"pith_registry"},"payload":{"graph_snapshot":{"author_claims":{"count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","strong_count":0},"builder_version":"pith-number-builder-2026-05-17-v1","claims":{"count":0,"items":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"integrity":{"available":true,"clean":true,"detectors_run":[],"endpoint":"/pith/2010.02534/integrity.json","findings":[],"snapshot_sha256":"c28c3603d3b5d939e8dc4c7e95fa8dfce3d595e45f758748cecf8e644a296938","summary":{"advisory":0,"by_detector":{},"critical":0,"informational":0}},"paper":{"abstract_excerpt":"Typically, tokenization is the very first step in most text processing works. As a token serves as an atomic unit that embeds the contextual information of text, how to define a token plays a decisive role in the performance of a model.Even though Byte Pair Encoding (BPE) has been considered the de facto standard tokenization method due to its simplicity and universality, it still remains unclear whether BPE works best across all languages and tasks. In this paper, we test several tokenization strategies in order to answer our primary research question, that is, \"What is the best tokenization ","authors_text":"Dawoon Jung, Joohong Lee, Kyubyong Park, Seongbo Jang","cross_cats":[],"headline":"","license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","primary_cat":"cs.CL","submitted_at":"2020-10-06T07:20:41Z","title":"An Empirical Study of Tokenization Strategies for Various Korean NLP Tasks"},"references":{"count":0,"internal_anchors":0,"resolved_work":0,"sample":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"2010.02534","kind":"arxiv","version":1},"verdict":{"created_at":null,"id":null,"model_set":{},"one_line_summary":"","pipeline_version":null,"pith_extraction_headline":"","strongest_claim":"","weakest_assumption":""}},"verdict_id":null}}],"author_attestations":[],"timestamp_anchors":[],"storage_attestations":[],"citation_signatures":[],"replication_records":[],"corrections":[],"mirror_hints":[],"record_created":{"event_id":"sha256:96ebb0544da1c28f597c349da5ebae38beecbd3b2815207f3faa6e7dfc38762f","target":"record","created_at":"2026-07-05T01:40:51Z","signer":{"key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signer_id":"pith.science","signer_type":"pith_registry"},"payload":{"attestation_state":"computed","canonical_record":{"metadata":{"abstract_canon_sha256":"61f3e18797c9c3b0ba986d3f5ca50bdb34e7e9fcf1cb840f15d0d7be25883835","cross_cats_sorted":[],"license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","primary_cat":"cs.CL","submitted_at":"2020-10-06T07:20:41Z","title_canon_sha256":"961fcadd49d2f45b8fe575e16c81ee60f1fd346c304c58243a611fa42f79a60a"},"schema_version":"1.0","source":{"id":"2010.02534","kind":"arxiv","version":1}},"canonical_sha256":"30d7162ab7ae5464b1996d06e492e7a3521dfd345aa7cdc326a293be6201f88e","receipt":{"algorithm":"ed25519","builder_version":"pith-number-builder-2026-05-17-v1","canonical_sha256":"30d7162ab7ae5464b1996d06e492e7a3521dfd345aa7cdc326a293be6201f88e","first_computed_at":"2026-07-05T01:40:51.123795Z","key_id":"pith-v1-2026-05","kind":"pith_receipt","last_reissued_at":"2026-07-05T01:40:51.123795Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","receipt_version":"0.3","signature_b64":"AShQOoC+1UrwYPG4WXpSf9+4db4ArVZsfg5SlyG+ltoYvwllF7++pM6kVBgghCz5uK2aorHhu0g6JKxoRF6sAw==","signature_status":"signed_v1","signed_at":"2026-07-05T01:40:51.124179Z","signed_message":"canonical_sha256_bytes"},"source_id":"2010.02534","source_kind":"arxiv","source_version":1}}},"equivocations":[],"invalid_events":[],"applied_event_ids":["sha256:96ebb0544da1c28f597c349da5ebae38beecbd3b2815207f3faa6e7dfc38762f","sha256:56f8c8779b6e514b5992e24e9ae67391a8a2d27365cb4a5dababc1a0378d9d90"],"state_sha256":"68101c502ed994d0534712608e37a30ba86e57db64a73dc03d1dd04af8f3292a"},"bundle_signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"YqBtsoJ2tCF0Ac/3ZSY4wUKzQaRSwqJwzbUsGwOTT+iJaKOiqc90M8LZjleEdprYnF5CbI3LfACoAnj5XGftBg==","signed_message":"bundle_sha256_bytes","signed_at":"2026-08-09T01:56:47.238252Z","bundle_sha256":"dad63472f22e851b592a47c9687af78f47d241133b6da61d3e73c5e8377116d1"}}