{"record_type":"pith_number_record","schema_url":"https://pith.science/schemas/pith-number/v1.json","pith_number":"pith:2025:B6QMVEAVD3QNSSG5PRAYGSVEE6","short_pith_number":"pith:B6QMVEAV","schema_version":"1.0","canonical_sha256":"0fa0ca90151ee0d948dd7c41834aa427bf0b33741ae76cb7eea3b1ea7c6e04f3","source":{"kind":"arxiv","id":"2506.18639","version":1},"attestation_state":"computed","paper":{"title":"ByteSpan: Information-Driven Subword Tokenisation","license":"http://creativecommons.org/licenses/by/4.0/","headline":"","cross_cats":[],"primary_cat":"cs.CL","authors_text":"Julius Cheng, Paula Buttery, Pietro Lesci, Suchir Salhan, Z\\'ebulon Goriely","submitted_at":"2025-06-23T13:42:00Z","abstract_excerpt":"Recent dynamic tokenisation methods operate directly on bytes and pool their latent representations into patches. This bears similarities to computational models of word segmentation that determine lexical boundaries using spikes in an autoregressive model's prediction error. Inspired by this connection, we explore whether grouping predictable bytes - rather than pooling their representations - can yield a useful fixed subword vocabulary. We propose a new information-driven subword tokeniser, ByteSpan, that uses an external byte-level LM during training to identify contiguous predictable byte "},"verification_status":{"content_addressed":true,"pith_receipt":true,"author_attested":false,"weak_author_claims":0,"strong_author_claims":0,"externally_anchored":false,"storage_verified":false,"citation_signatures":0,"replication_records":0,"graph_snapshot":true,"references_resolved":false,"formal_links_present":false},"canonical_record":{"source":{"id":"2506.18639","kind":"arxiv","version":1},"metadata":{"license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.CL","submitted_at":"2025-06-23T13:42:00Z","cross_cats_sorted":[],"title_canon_sha256":"b306f12ad0178f84999048d766d51db31a8a02b89f23a5009447f5d5febb2d5a","abstract_canon_sha256":"22dc67d3b869b86e126ad172723f692f3216d71a2bf9cb6f56716c31539f775a"},"schema_version":"1.0"},"receipt":{"kind":"pith_receipt","key_id":"pith-v1-2026-05","algorithm":"ed25519","signed_at":"2026-07-05T11:25:53.524378Z","signature_b64":"fvF0hocEue8LflAAwBqbbgd1Cgm5DDI+DRyg41Y/YAEg5M8ZUdoL3QvaTAmjKzOQX+jdryIOTBKJ3RYqkoTBAA==","signed_message":"canonical_sha256_bytes","builder_version":"pith-number-builder-2026-05-17-v1","receipt_version":"0.3","canonical_sha256":"0fa0ca90151ee0d948dd7c41834aa427bf0b33741ae76cb7eea3b1ea7c6e04f3","last_reissued_at":"2026-07-05T11:25:53.523887Z","signature_status":"signed_v1","first_computed_at":"2026-07-05T11:25:53.523887Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"graph_snapshot":{"paper":{"title":"ByteSpan: Information-Driven Subword Tokenisation","license":"http://creativecommons.org/licenses/by/4.0/","headline":"","cross_cats":[],"primary_cat":"cs.CL","authors_text":"Julius Cheng, Paula Buttery, Pietro Lesci, Suchir Salhan, Z\\'ebulon Goriely","submitted_at":"2025-06-23T13:42:00Z","abstract_excerpt":"Recent dynamic tokenisation methods operate directly on bytes and pool their latent representations into patches. This bears similarities to computational models of word segmentation that determine lexical boundaries using spikes in an autoregressive model's prediction error. Inspired by this connection, we explore whether grouping predictable bytes - rather than pooling their representations - can yield a useful fixed subword vocabulary. We propose a new information-driven subword tokeniser, ByteSpan, that uses an external byte-level LM during training to identify contiguous predictable byte "},"claims":{"count":0,"items":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"2506.18639","kind":"arxiv","version":1},"verdict":{"id":null,"model_set":{},"created_at":null,"strongest_claim":"","one_line_summary":"","pipeline_version":null,"weakest_assumption":"","pith_extraction_headline":""},"integrity":{"clean":true,"summary":{"advisory":0,"critical":0,"by_detector":{},"informational":0},"endpoint":"/pith/2506.18639/integrity.json","findings":[],"available":true,"detectors_run":[],"snapshot_sha256":"c28c3603d3b5d939e8dc4c7e95fa8dfce3d595e45f758748cecf8e644a296938"},"references":{"count":0,"sample":[],"resolved_work":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","internal_anchors":0},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"author_claims":{"count":0,"strong_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"builder_version":"pith-number-builder-2026-05-17-v1"},"aliases":[{"alias_kind":"arxiv","alias_value":"2506.18639","created_at":"2026-07-05T11:25:53.523939+00:00"},{"alias_kind":"arxiv_version","alias_value":"2506.18639v1","created_at":"2026-07-05T11:25:53.523939+00:00"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.2506.18639","created_at":"2026-07-05T11:25:53.523939+00:00"},{"alias_kind":"pith_short_12","alias_value":"B6QMVEAVD3QN","created_at":"2026-07-05T11:25:53.523939+00:00"},{"alias_kind":"pith_short_16","alias_value":"B6QMVEAVD3QNSSG5","created_at":"2026-07-05T11:25:53.523939+00:00"},{"alias_kind":"pith_short_8","alias_value":"B6QMVEAV","created_at":"2026-07-05T11:25:53.523939+00:00"}],"events":[],"event_summary":{},"paper_claims":[],"inbound_citations":{"count":1,"internal_anchor_count":0,"sample":[{"citing_arxiv_id":"2604.05192","citing_title":"Faster Superword Tokenization","ref_index":4,"is_internal_anchor":false}]},"formal_canon":{"evidence_count":0,"sample":[],"anchors":[]},"links":{"html":"https://pith.science/pith/B6QMVEAVD3QNSSG5PRAYGSVEE6","json":"https://pith.science/pith/B6QMVEAVD3QNSSG5PRAYGSVEE6.json","graph_json":"https://pith.science/api/pith-number/B6QMVEAVD3QNSSG5PRAYGSVEE6/graph.json","events_json":"https://pith.science/api/pith-number/B6QMVEAVD3QNSSG5PRAYGSVEE6/events.json","paper":"https://pith.science/paper/B6QMVEAV"},"agent_actions":{"view_html":"https://pith.science/pith/B6QMVEAVD3QNSSG5PRAYGSVEE6","download_json":"https://pith.science/pith/B6QMVEAVD3QNSSG5PRAYGSVEE6.json","view_paper":"https://pith.science/paper/B6QMVEAV","resolve_alias":"https://pith.science/api/pith-number/resolve?arxiv=2506.18639&json=true","fetch_graph":"https://pith.science/api/pith-number/B6QMVEAVD3QNSSG5PRAYGSVEE6/graph.json","fetch_events":"https://pith.science/api/pith-number/B6QMVEAVD3QNSSG5PRAYGSVEE6/events.json","actions":{"anchor_timestamp":"https://pith.science/pith/B6QMVEAVD3QNSSG5PRAYGSVEE6/action/timestamp_anchor","attest_storage":"https://pith.science/pith/B6QMVEAVD3QNSSG5PRAYGSVEE6/action/storage_attestation","attest_author":"https://pith.science/pith/B6QMVEAVD3QNSSG5PRAYGSVEE6/action/author_attestation","sign_citation":"https://pith.science/pith/B6QMVEAVD3QNSSG5PRAYGSVEE6/action/citation_signature","submit_replication":"https://pith.science/pith/B6QMVEAVD3QNSSG5PRAYGSVEE6/action/replication_record"}},"created_at":"2026-07-05T11:25:53.523939+00:00","updated_at":"2026-07-05T11:25:53.523939+00:00"}