{"bundle_type":"pith_open_graph_bundle","bundle_version":"1.0","pith_number":"pith:2026:S5BD5RJBVVE7FRO3UENCMMJHFL","short_pith_number":"pith:S5BD5RJB","canonical_record":{"source":{"id":"2608.00837","kind":"arxiv","version":1},"metadata":{"license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.CL","submitted_at":"2026-08-01T19:34:44Z","cross_cats_sorted":["cs.LG"],"title_canon_sha256":"74fda4ef9340e97fdfefa90aafb231fb96750330bf21d86dd37ba0880f0b4c61","abstract_canon_sha256":"63d23e69d8da809fd1aced6d42ec65d575e4dc137857fb9142898ad19de21151"},"schema_version":"1.0"},"canonical_sha256":"97423ec521ad49f2c5dba11a2631272af645f71bd356a1f864070f72e1a6aedc","source":{"kind":"arxiv","id":"2608.00837","version":1},"source_aliases":[{"alias_kind":"arxiv","alias_value":"2608.00837","created_at":"2026-08-04T01:55:14Z"},{"alias_kind":"arxiv_version","alias_value":"2608.00837v1","created_at":"2026-08-04T01:55:14Z"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.2608.00837","created_at":"2026-08-04T01:55:14Z"},{"alias_kind":"pith_short_12","alias_value":"S5BD5RJBVVE7","created_at":"2026-08-04T01:55:14Z"},{"alias_kind":"pith_short_16","alias_value":"S5BD5RJBVVE7FRO3","created_at":"2026-08-04T01:55:14Z"},{"alias_kind":"pith_short_8","alias_value":"S5BD5RJB","created_at":"2026-08-04T01:55:14Z"}],"events":[{"event_type":"record_created","subject_pith_number":"pith:2026:S5BD5RJBVVE7FRO3UENCMMJHFL","target":"record","payload":{"canonical_record":{"source":{"id":"2608.00837","kind":"arxiv","version":1},"metadata":{"license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.CL","submitted_at":"2026-08-01T19:34:44Z","cross_cats_sorted":["cs.LG"],"title_canon_sha256":"74fda4ef9340e97fdfefa90aafb231fb96750330bf21d86dd37ba0880f0b4c61","abstract_canon_sha256":"63d23e69d8da809fd1aced6d42ec65d575e4dc137857fb9142898ad19de21151"},"schema_version":"1.0"},"canonical_sha256":"97423ec521ad49f2c5dba11a2631272af645f71bd356a1f864070f72e1a6aedc","receipt":{"kind":"pith_receipt","key_id":"pith-v1-2026-05","algorithm":"ed25519","signed_at":"2026-08-04T01:55:14.220180Z","signature_b64":"uFdhpRSFu8Q/E9+aQd+rIrpTAzpkERh2TGPiya09LuMm/tldVTxYp5ikY84bKTRNby02VwC3NeXRzsZZm+H7Dg==","signed_message":"canonical_sha256_bytes","builder_version":"pith-number-builder-2026-05-17-v1","receipt_version":"0.3","canonical_sha256":"97423ec521ad49f2c5dba11a2631272af645f71bd356a1f864070f72e1a6aedc","last_reissued_at":"2026-08-04T01:55:14.218497Z","signature_status":"signed_v1","first_computed_at":"2026-08-04T01:55:14.218497Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"source_kind":"arxiv","source_id":"2608.00837","source_version":1,"attestation_state":"computed"},"signer":{"signer_id":"pith.science","signer_type":"pith_registry","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"created_at":"2026-08-04T01:55:14Z","supersedes":[],"prev_event":null,"signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"C2wuOJluvuzluyRgnoEpVe03tGWJWnj6u/C5mHEJrUDh9W4GVKDVIynZ6HH/O5uYe3s7HI7uS60UzRTuHlnXBw==","signed_message":"open_graph_event_sha256_bytes","signed_at":"2026-08-05T09:14:09.269917Z"},"content_sha256":"420feb5548e52b9754ce45c6e52ef09e7e62da48cd2fe0269f768d19763e1a5b","schema_version":"1.0","event_id":"sha256:420feb5548e52b9754ce45c6e52ef09e7e62da48cd2fe0269f768d19763e1a5b"},{"event_type":"graph_snapshot","subject_pith_number":"pith:2026:S5BD5RJBVVE7FRO3UENCMMJHFL","target":"graph","payload":{"graph_snapshot":{"paper":{"title":"Pruned BPE: Post-training Visibility Pruning and Token Reallocation for Byte Pair Encoding","license":"http://creativecommons.org/licenses/by/4.0/","headline":"","cross_cats":["cs.LG"],"primary_cat":"cs.CL","authors_text":"Kenny Shao","submitted_at":"2026-08-01T19:34:44Z","abstract_excerpt":"Byte Pair Encoding (BPE) is widely used for subword tokenization, but standard BPE exposes every learned merge token to the downstream model, including tokens that mainly serve as intermediate construction units and rarely appear in the final encoded corpus. This paper proposes Pruned BPE, a post-training visibility-pruning and token-reallocation method that separates merge construction from model-visible vocabulary selection. After standard BPE training, tokens are evaluated by final exposure. Low-exposure tokens are retained as internal-only merge nodes, while their visible vocabulary slots "},"claims":{"count":0,"items":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"2608.00837","kind":"arxiv","version":1},"verdict":{"id":null,"model_set":{},"created_at":null,"strongest_claim":"","one_line_summary":"","pipeline_version":null,"weakest_assumption":"","pith_extraction_headline":""},"integrity":{"clean":true,"summary":{"advisory":0,"critical":0,"by_detector":{},"informational":0},"endpoint":"/pith/2608.00837/integrity.json","findings":[],"available":true,"detectors_run":[],"snapshot_sha256":"c28c3603d3b5d939e8dc4c7e95fa8dfce3d595e45f758748cecf8e644a296938"},"references":{"count":0,"sample":[],"resolved_work":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","internal_anchors":0},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"author_claims":{"count":0,"strong_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"builder_version":"pith-number-builder-2026-05-17-v1"},"verdict_id":null},"signer":{"signer_id":"pith.science","signer_type":"pith_registry","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"created_at":"2026-08-04T01:55:14Z","supersedes":[],"prev_event":null,"signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"t+h4G8PZoUxpmtrrc/FPH2vXGJCkODME97spgnLFHETmOSR+ewrkQvI3QElB8Ctcg2QdUwnjAtwcOmV7VtBtAQ==","signed_message":"open_graph_event_sha256_bytes","signed_at":"2026-08-05T09:14:09.270444Z"},"content_sha256":"3872540dcab85a71ecfc7b58d1251a65897875c9dc19e09e95576c7bfb765f6c","schema_version":"1.0","event_id":"sha256:3872540dcab85a71ecfc7b58d1251a65897875c9dc19e09e95576c7bfb765f6c"}],"timestamp_proofs":[],"mirror_hints":[{"mirror_type":"https","name":"Pith Resolver","base_url":"https://pith.science","bundle_url":"https://pith.science/pith/S5BD5RJBVVE7FRO3UENCMMJHFL/bundle.json","state_url":"https://pith.science/pith/S5BD5RJBVVE7FRO3UENCMMJHFL/state.json","well_known_bundle_url":"https://pith.science/.well-known/pith/S5BD5RJBVVE7FRO3UENCMMJHFL/bundle.json","status":"primary"}],"public_keys":[{"key_id":"pith-v1-2026-05","algorithm":"ed25519","format":"raw","public_key_b64":"stVStoiQhXFxp4s2pdzPNoqVNBMojDU/fJ2db5S3CbM=","public_key_hex":"b2d552b68890857171a78b36a5dccf368a953413288c353f7c9d9d6f94b709b3","fingerprint_sha256_b32_first128bits":"RVFV5Z2OI2J3ZUO7ERDEBCYNKS","fingerprint_sha256_hex":"8d4b5ee74e4693bcd1df2446408b0d54","rotates_at":null,"url":"https://pith.science/pith-signing-key.json","notes":"Pith uses this Ed25519 key to sign canonical record SHA-256 digests. Verify with: ed25519_verify(public_key, message=canonical_sha256_bytes, signature=base64decode(signature_b64))."}],"merge_version":"pith-open-graph-merge-v1","built_at":"2026-08-05T09:14:09Z","links":{"resolver":"https://pith.science/pith/S5BD5RJBVVE7FRO3UENCMMJHFL","bundle":"https://pith.science/pith/S5BD5RJBVVE7FRO3UENCMMJHFL/bundle.json","state":"https://pith.science/pith/S5BD5RJBVVE7FRO3UENCMMJHFL/state.json","well_known_bundle":"https://pith.science/.well-known/pith/S5BD5RJBVVE7FRO3UENCMMJHFL/bundle.json"},"state":{"state_type":"pith_open_graph_state","state_version":"1.0","pith_number":"pith:2026:S5BD5RJBVVE7FRO3UENCMMJHFL","merge_version":"pith-open-graph-merge-v1","event_count":2,"valid_event_count":2,"invalid_event_count":0,"equivocation_count":0,"current":{"canonical_record":{"metadata":{"abstract_canon_sha256":"63d23e69d8da809fd1aced6d42ec65d575e4dc137857fb9142898ad19de21151","cross_cats_sorted":["cs.LG"],"license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.CL","submitted_at":"2026-08-01T19:34:44Z","title_canon_sha256":"74fda4ef9340e97fdfefa90aafb231fb96750330bf21d86dd37ba0880f0b4c61"},"schema_version":"1.0","source":{"id":"2608.00837","kind":"arxiv","version":1}},"source_aliases":[{"alias_kind":"arxiv","alias_value":"2608.00837","created_at":"2026-08-04T01:55:14Z"},{"alias_kind":"arxiv_version","alias_value":"2608.00837v1","created_at":"2026-08-04T01:55:14Z"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.2608.00837","created_at":"2026-08-04T01:55:14Z"},{"alias_kind":"pith_short_12","alias_value":"S5BD5RJBVVE7","created_at":"2026-08-04T01:55:14Z"},{"alias_kind":"pith_short_16","alias_value":"S5BD5RJBVVE7FRO3","created_at":"2026-08-04T01:55:14Z"},{"alias_kind":"pith_short_8","alias_value":"S5BD5RJB","created_at":"2026-08-04T01:55:14Z"}],"graph_snapshots":[{"event_id":"sha256:3872540dcab85a71ecfc7b58d1251a65897875c9dc19e09e95576c7bfb765f6c","target":"graph","created_at":"2026-08-04T01:55:14Z","signer":{"key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signer_id":"pith.science","signer_type":"pith_registry"},"payload":{"graph_snapshot":{"author_claims":{"count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","strong_count":0},"builder_version":"pith-number-builder-2026-05-17-v1","claims":{"count":0,"items":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"integrity":{"available":true,"clean":true,"detectors_run":[],"endpoint":"/pith/2608.00837/integrity.json","findings":[],"snapshot_sha256":"c28c3603d3b5d939e8dc4c7e95fa8dfce3d595e45f758748cecf8e644a296938","summary":{"advisory":0,"by_detector":{},"critical":0,"informational":0}},"paper":{"abstract_excerpt":"Byte Pair Encoding (BPE) is widely used for subword tokenization, but standard BPE exposes every learned merge token to the downstream model, including tokens that mainly serve as intermediate construction units and rarely appear in the final encoded corpus. This paper proposes Pruned BPE, a post-training visibility-pruning and token-reallocation method that separates merge construction from model-visible vocabulary selection. After standard BPE training, tokens are evaluated by final exposure. Low-exposure tokens are retained as internal-only merge nodes, while their visible vocabulary slots ","authors_text":"Kenny Shao","cross_cats":["cs.LG"],"headline":"","license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.CL","submitted_at":"2026-08-01T19:34:44Z","title":"Pruned BPE: Post-training Visibility Pruning and Token Reallocation for Byte Pair Encoding"},"references":{"count":0,"internal_anchors":0,"resolved_work":0,"sample":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"2608.00837","kind":"arxiv","version":1},"verdict":{"created_at":null,"id":null,"model_set":{},"one_line_summary":"","pipeline_version":null,"pith_extraction_headline":"","strongest_claim":"","weakest_assumption":""}},"verdict_id":null}}],"author_attestations":[],"timestamp_anchors":[],"storage_attestations":[],"citation_signatures":[],"replication_records":[],"corrections":[],"mirror_hints":[],"record_created":{"event_id":"sha256:420feb5548e52b9754ce45c6e52ef09e7e62da48cd2fe0269f768d19763e1a5b","target":"record","created_at":"2026-08-04T01:55:14Z","signer":{"key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signer_id":"pith.science","signer_type":"pith_registry"},"payload":{"attestation_state":"computed","canonical_record":{"metadata":{"abstract_canon_sha256":"63d23e69d8da809fd1aced6d42ec65d575e4dc137857fb9142898ad19de21151","cross_cats_sorted":["cs.LG"],"license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.CL","submitted_at":"2026-08-01T19:34:44Z","title_canon_sha256":"74fda4ef9340e97fdfefa90aafb231fb96750330bf21d86dd37ba0880f0b4c61"},"schema_version":"1.0","source":{"id":"2608.00837","kind":"arxiv","version":1}},"canonical_sha256":"97423ec521ad49f2c5dba11a2631272af645f71bd356a1f864070f72e1a6aedc","receipt":{"algorithm":"ed25519","builder_version":"pith-number-builder-2026-05-17-v1","canonical_sha256":"97423ec521ad49f2c5dba11a2631272af645f71bd356a1f864070f72e1a6aedc","first_computed_at":"2026-08-04T01:55:14.218497Z","key_id":"pith-v1-2026-05","kind":"pith_receipt","last_reissued_at":"2026-08-04T01:55:14.218497Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","receipt_version":"0.3","signature_b64":"uFdhpRSFu8Q/E9+aQd+rIrpTAzpkERh2TGPiya09LuMm/tldVTxYp5ikY84bKTRNby02VwC3NeXRzsZZm+H7Dg==","signature_status":"signed_v1","signed_at":"2026-08-04T01:55:14.220180Z","signed_message":"canonical_sha256_bytes"},"source_id":"2608.00837","source_kind":"arxiv","source_version":1}}},"equivocations":[],"invalid_events":[],"applied_event_ids":["sha256:420feb5548e52b9754ce45c6e52ef09e7e62da48cd2fe0269f768d19763e1a5b","sha256:3872540dcab85a71ecfc7b58d1251a65897875c9dc19e09e95576c7bfb765f6c"],"state_sha256":"3b4cada5645c939b894eb0873683000a2b1c1ccd2314469974c1d392d9867709"},"bundle_signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"NE9BqvFhMJEit+KsuIZT8fY9dbE5OHcrr6AADAcqQ8fR6J7g+mUnv+rMGhKMo+zB0gYAlt7ewPWM+qKCvc8IDQ==","signed_message":"bundle_sha256_bytes","signed_at":"2026-08-05T09:14:09.274257Z","bundle_sha256":"8aaed98245f579f083b2c18e8964c88e881230b7e3a343de4cbe639f7f1e5cfc"}}