{"bundle_type":"pith_open_graph_bundle","bundle_version":"1.0","pith_number":"pith:2025:6I47WPAYAMENSLHFT7KA6VZPRK","short_pith_number":"pith:6I47WPAY","canonical_record":{"source":{"id":"2506.15889","kind":"arxiv","version":1},"metadata":{"license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.CL","submitted_at":"2025-06-18T21:25:55Z","cross_cats_sorted":[],"title_canon_sha256":"8584be284bd1ac52a7bcc0c2361b33742361f1cbd32c3f61f6b066724b5dd940","abstract_canon_sha256":"d6961bada83953edb4d7f95b3e51ec62dbcd1040d3d6522ae775f374cc813f1d"},"schema_version":"1.0"},"canonical_sha256":"f239fb3c180308d92ce59fd40f572f8a94eef29a1dc0b1407aa7c381cf7b26c6","source":{"kind":"arxiv","id":"2506.15889","version":1},"source_aliases":[{"alias_kind":"arxiv","alias_value":"2506.15889","created_at":"2026-07-05T11:24:25Z"},{"alias_kind":"arxiv_version","alias_value":"2506.15889v1","created_at":"2026-07-05T11:24:25Z"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.2506.15889","created_at":"2026-07-05T11:24:25Z"},{"alias_kind":"pith_short_12","alias_value":"6I47WPAYAMEN","created_at":"2026-07-05T11:24:25Z"},{"alias_kind":"pith_short_16","alias_value":"6I47WPAYAMENSLHF","created_at":"2026-07-05T11:24:25Z"},{"alias_kind":"pith_short_8","alias_value":"6I47WPAY","created_at":"2026-07-05T11:24:25Z"}],"events":[{"event_type":"record_created","subject_pith_number":"pith:2025:6I47WPAYAMENSLHFT7KA6VZPRK","target":"record","payload":{"canonical_record":{"source":{"id":"2506.15889","kind":"arxiv","version":1},"metadata":{"license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.CL","submitted_at":"2025-06-18T21:25:55Z","cross_cats_sorted":[],"title_canon_sha256":"8584be284bd1ac52a7bcc0c2361b33742361f1cbd32c3f61f6b066724b5dd940","abstract_canon_sha256":"d6961bada83953edb4d7f95b3e51ec62dbcd1040d3d6522ae775f374cc813f1d"},"schema_version":"1.0"},"canonical_sha256":"f239fb3c180308d92ce59fd40f572f8a94eef29a1dc0b1407aa7c381cf7b26c6","receipt":{"kind":"pith_receipt","key_id":"pith-v1-2026-05","algorithm":"ed25519","signed_at":"2026-07-05T11:24:25.843711Z","signature_b64":"jdKlAIg3am/PVjZkYS48I1VGhdO5d3G7UcvhTglPNQUgmjX+tQiebKX0DMOfTUdsau7j6OPdgkSYJC8Zli4PDA==","signed_message":"canonical_sha256_bytes","builder_version":"pith-number-builder-2026-05-17-v1","receipt_version":"0.3","canonical_sha256":"f239fb3c180308d92ce59fd40f572f8a94eef29a1dc0b1407aa7c381cf7b26c6","last_reissued_at":"2026-07-05T11:24:25.843141Z","signature_status":"signed_v1","first_computed_at":"2026-07-05T11:24:25.843141Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"source_kind":"arxiv","source_id":"2506.15889","source_version":1,"attestation_state":"computed"},"signer":{"signer_id":"pith.science","signer_type":"pith_registry","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"created_at":"2026-07-05T11:24:25Z","supersedes":[],"prev_event":null,"signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"MPbAFOpuIB53ChHDLLNjTGS7qx720brUo3xR+42Vteisyifdrr9upmn6mxB89+OhTtxZ00UsNDa4g1uVvPhFDA==","signed_message":"open_graph_event_sha256_bytes","signed_at":"2026-07-06T16:59:35.745186Z"},"content_sha256":"b294febe04589788308209be0b187c3b42fa154fc5d24b4c16fad40727fecf2f","schema_version":"1.0","event_id":"sha256:b294febe04589788308209be0b187c3b42fa154fc5d24b4c16fad40727fecf2f"},{"event_type":"graph_snapshot","subject_pith_number":"pith:2025:6I47WPAYAMENSLHFT7KA6VZPRK","target":"graph","payload":{"graph_snapshot":{"paper":{"title":"Entropy-Driven Pre-Tokenization for Byte-Pair Encoding","license":"http://creativecommons.org/licenses/by/4.0/","headline":"","cross_cats":[],"primary_cat":"cs.CL","authors_text":"Chris Tanner, Craig W. Schmidt, Dachuan Zhao, Frank Liang, Jonathan Geuter, Varshini Reddy, Yifan Hu","submitted_at":"2025-06-18T21:25:55Z","abstract_excerpt":"Byte-Pair Encoding (BPE) has become a widely adopted subword tokenization method in modern language models due to its simplicity and strong empirical performance across downstream tasks. However, applying BPE to unsegmented languages such as Chinese presents significant challenges, as its frequency-driven merge operation is agnostic to linguistic boundaries. To address this, we propose two entropy-informed pre-tokenization strategies that guide BPE segmentation using unsupervised information-theoretic cues. The first approach uses pointwise mutual information and left/right entropy to identify"},"claims":{"count":0,"items":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"2506.15889","kind":"arxiv","version":1},"verdict":{"id":null,"model_set":{},"created_at":null,"strongest_claim":"","one_line_summary":"","pipeline_version":null,"weakest_assumption":"","pith_extraction_headline":""},"integrity":{"clean":true,"summary":{"advisory":0,"critical":0,"by_detector":{},"informational":0},"endpoint":"/pith/2506.15889/integrity.json","findings":[],"available":true,"detectors_run":[],"snapshot_sha256":"c28c3603d3b5d939e8dc4c7e95fa8dfce3d595e45f758748cecf8e644a296938"},"references":{"count":0,"sample":[],"resolved_work":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","internal_anchors":0},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"author_claims":{"count":0,"strong_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"builder_version":"pith-number-builder-2026-05-17-v1"},"verdict_id":null},"signer":{"signer_id":"pith.science","signer_type":"pith_registry","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"created_at":"2026-07-05T11:24:25Z","supersedes":[],"prev_event":null,"signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"d8nQYOjBLRs6cDm5R1RMzfB4Tr0W9KJGj/LCAJRywSvXr359qmper9N7tpv1xyORcAoWGhtr+SwB0EOSdsLqCA==","signed_message":"open_graph_event_sha256_bytes","signed_at":"2026-07-06T16:59:35.745600Z"},"content_sha256":"b551a3dc3d8cbe840c1c29d5ec47ab7bad91a0574aa8999a19190b45c69e45db","schema_version":"1.0","event_id":"sha256:b551a3dc3d8cbe840c1c29d5ec47ab7bad91a0574aa8999a19190b45c69e45db"}],"timestamp_proofs":[],"mirror_hints":[{"mirror_type":"https","name":"Pith Resolver","base_url":"https://pith.science","bundle_url":"https://pith.science/pith/6I47WPAYAMENSLHFT7KA6VZPRK/bundle.json","state_url":"https://pith.science/pith/6I47WPAYAMENSLHFT7KA6VZPRK/state.json","well_known_bundle_url":"https://pith.science/.well-known/pith/6I47WPAYAMENSLHFT7KA6VZPRK/bundle.json","status":"primary"}],"public_keys":[{"key_id":"pith-v1-2026-05","algorithm":"ed25519","format":"raw","public_key_b64":"stVStoiQhXFxp4s2pdzPNoqVNBMojDU/fJ2db5S3CbM=","public_key_hex":"b2d552b68890857171a78b36a5dccf368a953413288c353f7c9d9d6f94b709b3","fingerprint_sha256_b32_first128bits":"RVFV5Z2OI2J3ZUO7ERDEBCYNKS","fingerprint_sha256_hex":"8d4b5ee74e4693bcd1df2446408b0d54","rotates_at":null,"url":"https://pith.science/pith-signing-key.json","notes":"Pith uses this Ed25519 key to sign canonical record SHA-256 digests. Verify with: ed25519_verify(public_key, message=canonical_sha256_bytes, signature=base64decode(signature_b64))."}],"merge_version":"pith-open-graph-merge-v1","built_at":"2026-07-06T16:59:35Z","links":{"resolver":"https://pith.science/pith/6I47WPAYAMENSLHFT7KA6VZPRK","bundle":"https://pith.science/pith/6I47WPAYAMENSLHFT7KA6VZPRK/bundle.json","state":"https://pith.science/pith/6I47WPAYAMENSLHFT7KA6VZPRK/state.json","well_known_bundle":"https://pith.science/.well-known/pith/6I47WPAYAMENSLHFT7KA6VZPRK/bundle.json"},"state":{"state_type":"pith_open_graph_state","state_version":"1.0","pith_number":"pith:2025:6I47WPAYAMENSLHFT7KA6VZPRK","merge_version":"pith-open-graph-merge-v1","event_count":2,"valid_event_count":2,"invalid_event_count":0,"equivocation_count":0,"current":{"canonical_record":{"metadata":{"abstract_canon_sha256":"d6961bada83953edb4d7f95b3e51ec62dbcd1040d3d6522ae775f374cc813f1d","cross_cats_sorted":[],"license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.CL","submitted_at":"2025-06-18T21:25:55Z","title_canon_sha256":"8584be284bd1ac52a7bcc0c2361b33742361f1cbd32c3f61f6b066724b5dd940"},"schema_version":"1.0","source":{"id":"2506.15889","kind":"arxiv","version":1}},"source_aliases":[{"alias_kind":"arxiv","alias_value":"2506.15889","created_at":"2026-07-05T11:24:25Z"},{"alias_kind":"arxiv_version","alias_value":"2506.15889v1","created_at":"2026-07-05T11:24:25Z"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.2506.15889","created_at":"2026-07-05T11:24:25Z"},{"alias_kind":"pith_short_12","alias_value":"6I47WPAYAMEN","created_at":"2026-07-05T11:24:25Z"},{"alias_kind":"pith_short_16","alias_value":"6I47WPAYAMENSLHF","created_at":"2026-07-05T11:24:25Z"},{"alias_kind":"pith_short_8","alias_value":"6I47WPAY","created_at":"2026-07-05T11:24:25Z"}],"graph_snapshots":[{"event_id":"sha256:b551a3dc3d8cbe840c1c29d5ec47ab7bad91a0574aa8999a19190b45c69e45db","target":"graph","created_at":"2026-07-05T11:24:25Z","signer":{"key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signer_id":"pith.science","signer_type":"pith_registry"},"payload":{"graph_snapshot":{"author_claims":{"count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","strong_count":0},"builder_version":"pith-number-builder-2026-05-17-v1","claims":{"count":0,"items":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"integrity":{"available":true,"clean":true,"detectors_run":[],"endpoint":"/pith/2506.15889/integrity.json","findings":[],"snapshot_sha256":"c28c3603d3b5d939e8dc4c7e95fa8dfce3d595e45f758748cecf8e644a296938","summary":{"advisory":0,"by_detector":{},"critical":0,"informational":0}},"paper":{"abstract_excerpt":"Byte-Pair Encoding (BPE) has become a widely adopted subword tokenization method in modern language models due to its simplicity and strong empirical performance across downstream tasks. However, applying BPE to unsegmented languages such as Chinese presents significant challenges, as its frequency-driven merge operation is agnostic to linguistic boundaries. To address this, we propose two entropy-informed pre-tokenization strategies that guide BPE segmentation using unsupervised information-theoretic cues. The first approach uses pointwise mutual information and left/right entropy to identify","authors_text":"Chris Tanner, Craig W. Schmidt, Dachuan Zhao, Frank Liang, Jonathan Geuter, Varshini Reddy, Yifan Hu","cross_cats":[],"headline":"","license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.CL","submitted_at":"2025-06-18T21:25:55Z","title":"Entropy-Driven Pre-Tokenization for Byte-Pair Encoding"},"references":{"count":0,"internal_anchors":0,"resolved_work":0,"sample":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"2506.15889","kind":"arxiv","version":1},"verdict":{"created_at":null,"id":null,"model_set":{},"one_line_summary":"","pipeline_version":null,"pith_extraction_headline":"","strongest_claim":"","weakest_assumption":""}},"verdict_id":null}}],"author_attestations":[],"timestamp_anchors":[],"storage_attestations":[],"citation_signatures":[],"replication_records":[],"corrections":[],"mirror_hints":[],"record_created":{"event_id":"sha256:b294febe04589788308209be0b187c3b42fa154fc5d24b4c16fad40727fecf2f","target":"record","created_at":"2026-07-05T11:24:25Z","signer":{"key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signer_id":"pith.science","signer_type":"pith_registry"},"payload":{"attestation_state":"computed","canonical_record":{"metadata":{"abstract_canon_sha256":"d6961bada83953edb4d7f95b3e51ec62dbcd1040d3d6522ae775f374cc813f1d","cross_cats_sorted":[],"license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.CL","submitted_at":"2025-06-18T21:25:55Z","title_canon_sha256":"8584be284bd1ac52a7bcc0c2361b33742361f1cbd32c3f61f6b066724b5dd940"},"schema_version":"1.0","source":{"id":"2506.15889","kind":"arxiv","version":1}},"canonical_sha256":"f239fb3c180308d92ce59fd40f572f8a94eef29a1dc0b1407aa7c381cf7b26c6","receipt":{"algorithm":"ed25519","builder_version":"pith-number-builder-2026-05-17-v1","canonical_sha256":"f239fb3c180308d92ce59fd40f572f8a94eef29a1dc0b1407aa7c381cf7b26c6","first_computed_at":"2026-07-05T11:24:25.843141Z","key_id":"pith-v1-2026-05","kind":"pith_receipt","last_reissued_at":"2026-07-05T11:24:25.843141Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","receipt_version":"0.3","signature_b64":"jdKlAIg3am/PVjZkYS48I1VGhdO5d3G7UcvhTglPNQUgmjX+tQiebKX0DMOfTUdsau7j6OPdgkSYJC8Zli4PDA==","signature_status":"signed_v1","signed_at":"2026-07-05T11:24:25.843711Z","signed_message":"canonical_sha256_bytes"},"source_id":"2506.15889","source_kind":"arxiv","source_version":1}}},"equivocations":[],"invalid_events":[],"applied_event_ids":["sha256:b294febe04589788308209be0b187c3b42fa154fc5d24b4c16fad40727fecf2f","sha256:b551a3dc3d8cbe840c1c29d5ec47ab7bad91a0574aa8999a19190b45c69e45db"],"state_sha256":"c6a1d1bdb1047529d4c44a3e65e0bd0d78760b3b0b6069a17b56bc79d01d492c"},"bundle_signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"PcWof6CpfTXKMULWhXuco3gdcQtjHul3xAn6LJdx0y/ez722k3KITjdcQlrAJ3vuVj5PCENM6dYk81QMAIfEBg==","signed_message":"bundle_sha256_bytes","signed_at":"2026-07-06T16:59:35.747836Z","bundle_sha256":"8b35bb1dc572583977f96ad1b81172c5f60bc41bc3180acdf92ff3fff136d709"}}