{"bundle_type":"pith_open_graph_bundle","bundle_version":"1.0","pith_number":"pith:2024:6VYDY36TOA3SRSNA5L4COKHVIS","short_pith_number":"pith:6VYDY36T","canonical_record":{"source":{"id":"2412.03160","kind":"arxiv","version":1},"metadata":{"license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.CL","submitted_at":"2024-12-04T09:38:11Z","cross_cats_sorted":[],"title_canon_sha256":"51707907b1add9f668ad26495fcf90b1bf311c290c7e15059cd4b2d58b58e1f7","abstract_canon_sha256":"0323dbb5b02fcc0c2c4c48e0f609ebaeb9db7677d4ad9ae64ea0fdb450b7c44e"},"schema_version":"1.0"},"canonical_sha256":"f5703c6fd3703728c9a0eaf82728f5448a3fb4c8994748658ed9295425e72cf5","source":{"kind":"arxiv","id":"2412.03160","version":1},"source_aliases":[{"alias_kind":"arxiv","alias_value":"2412.03160","created_at":"2026-07-05T09:44:25Z"},{"alias_kind":"arxiv_version","alias_value":"2412.03160v1","created_at":"2026-07-05T09:44:25Z"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.2412.03160","created_at":"2026-07-05T09:44:25Z"},{"alias_kind":"pith_short_12","alias_value":"6VYDY36TOA3S","created_at":"2026-07-05T09:44:25Z"},{"alias_kind":"pith_short_16","alias_value":"6VYDY36TOA3SRSNA","created_at":"2026-07-05T09:44:25Z"},{"alias_kind":"pith_short_8","alias_value":"6VYDY36T","created_at":"2026-07-05T09:44:25Z"}],"events":[{"event_type":"record_created","subject_pith_number":"pith:2024:6VYDY36TOA3SRSNA5L4COKHVIS","target":"record","payload":{"canonical_record":{"source":{"id":"2412.03160","kind":"arxiv","version":1},"metadata":{"license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.CL","submitted_at":"2024-12-04T09:38:11Z","cross_cats_sorted":[],"title_canon_sha256":"51707907b1add9f668ad26495fcf90b1bf311c290c7e15059cd4b2d58b58e1f7","abstract_canon_sha256":"0323dbb5b02fcc0c2c4c48e0f609ebaeb9db7677d4ad9ae64ea0fdb450b7c44e"},"schema_version":"1.0"},"canonical_sha256":"f5703c6fd3703728c9a0eaf82728f5448a3fb4c8994748658ed9295425e72cf5","receipt":{"kind":"pith_receipt","key_id":"pith-v1-2026-05","algorithm":"ed25519","signed_at":"2026-07-05T09:44:25.758539Z","signature_b64":"Vekc+Lols2hzaW1V5E0puob+bPpjhOy3z6SILeW7C730ViSLGg4zDK4m7GjbnSZVVmYys9Hjrb6oh2JGHfUbDw==","signed_message":"canonical_sha256_bytes","builder_version":"pith-number-builder-2026-05-17-v1","receipt_version":"0.3","canonical_sha256":"f5703c6fd3703728c9a0eaf82728f5448a3fb4c8994748658ed9295425e72cf5","last_reissued_at":"2026-07-05T09:44:25.758084Z","signature_status":"signed_v1","first_computed_at":"2026-07-05T09:44:25.758084Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"source_kind":"arxiv","source_id":"2412.03160","source_version":1,"attestation_state":"computed"},"signer":{"signer_id":"pith.science","signer_type":"pith_registry","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"created_at":"2026-07-05T09:44:25Z","supersedes":[],"prev_event":null,"signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"RF5lM1QIPOAsv2LrE2t58J3XrBiqWyr1hfH+JUdl5wi6Ujrd4YWQTbXZ5eo4QniUuiaYM/QzC3G6hpwYXr0oDw==","signed_message":"open_graph_event_sha256_bytes","signed_at":"2026-08-21T08:04:14.282885Z"},"content_sha256":"0379da477a92fa190e008f3f63eb372dbd51ad861170eb7428ae284cace57dfb","schema_version":"1.0","event_id":"sha256:0379da477a92fa190e008f3f63eb372dbd51ad861170eb7428ae284cace57dfb"},{"event_type":"graph_snapshot","subject_pith_number":"pith:2024:6VYDY36TOA3SRSNA5L4COKHVIS","target":"graph","payload":{"graph_snapshot":{"paper":{"title":"Byte BPE Tokenization as an Inverse string Homomorphism","license":"http://creativecommons.org/licenses/by/4.0/","headline":"","cross_cats":[],"primary_cat":"cs.CL","authors_text":"Chris Wendler, Robert West, Saibo Geng, Sankalp Gambhir","submitted_at":"2024-12-04T09:38:11Z","abstract_excerpt":"Tokenization is an important preprocessing step in the training and inference of large language models (LLMs). While there has been extensive research on the expressive power of the neural achitectures used in LLMs, the impact of tokenization has not been well understood. In this work, we demonstrate that tokenization, irrespective of the algorithm used, acts as an inverse homomorphism between strings and tokens. This suggests that the character space of the source language and the token space of the tokenized language are homomorphic, preserving the structural properties of the source languag"},"claims":{"count":0,"items":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"2412.03160","kind":"arxiv","version":1},"verdict":{"id":null,"model_set":{},"created_at":null,"strongest_claim":"","one_line_summary":"","pipeline_version":null,"weakest_assumption":"","pith_extraction_headline":""},"integrity":{"clean":true,"summary":{"advisory":0,"critical":0,"by_detector":{},"informational":0},"endpoint":"/pith/2412.03160/integrity.json","findings":[],"available":true,"detectors_run":[],"snapshot_sha256":"c28c3603d3b5d939e8dc4c7e95fa8dfce3d595e45f758748cecf8e644a296938"},"references":{"count":0,"sample":[],"resolved_work":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","internal_anchors":0},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"author_claims":{"count":0,"strong_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"builder_version":"pith-number-builder-2026-05-17-v1"},"verdict_id":null},"signer":{"signer_id":"pith.science","signer_type":"pith_registry","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"created_at":"2026-07-05T09:44:25Z","supersedes":[],"prev_event":null,"signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"+RwNk4Jp7KGVh8fiWwIUbq/yXcv1OP8r4AwssYc937uBO4qzXPjUaLxebiAjkPCkvPba4Q8g4jI7r0HtSCUGBg==","signed_message":"open_graph_event_sha256_bytes","signed_at":"2026-08-21T08:04:14.283423Z"},"content_sha256":"c0d628cfc487f5c8960cad4f1dccd5a69d4fe8aed2ca24c1dd088b4abdb8f1c2","schema_version":"1.0","event_id":"sha256:c0d628cfc487f5c8960cad4f1dccd5a69d4fe8aed2ca24c1dd088b4abdb8f1c2"}],"timestamp_proofs":[],"mirror_hints":[{"mirror_type":"https","name":"Pith Resolver","base_url":"https://pith.science","bundle_url":"https://pith.science/pith/6VYDY36TOA3SRSNA5L4COKHVIS/bundle.json","state_url":"https://pith.science/pith/6VYDY36TOA3SRSNA5L4COKHVIS/state.json","well_known_bundle_url":"https://pith.science/.well-known/pith/6VYDY36TOA3SRSNA5L4COKHVIS/bundle.json","status":"primary"}],"public_keys":[{"key_id":"pith-v1-2026-05","algorithm":"ed25519","format":"raw","public_key_b64":"stVStoiQhXFxp4s2pdzPNoqVNBMojDU/fJ2db5S3CbM=","public_key_hex":"b2d552b68890857171a78b36a5dccf368a953413288c353f7c9d9d6f94b709b3","fingerprint_sha256_b32_first128bits":"RVFV5Z2OI2J3ZUO7ERDEBCYNKS","fingerprint_sha256_hex":"8d4b5ee74e4693bcd1df2446408b0d54","rotates_at":null,"url":"https://pith.science/pith-signing-key.json","notes":"Pith uses this Ed25519 key to sign canonical record SHA-256 digests. Verify with: ed25519_verify(public_key, message=canonical_sha256_bytes, signature=base64decode(signature_b64))."}],"merge_version":"pith-open-graph-merge-v1","built_at":"2026-08-21T08:04:14Z","links":{"resolver":"https://pith.science/pith/6VYDY36TOA3SRSNA5L4COKHVIS","bundle":"https://pith.science/pith/6VYDY36TOA3SRSNA5L4COKHVIS/bundle.json","state":"https://pith.science/pith/6VYDY36TOA3SRSNA5L4COKHVIS/state.json","well_known_bundle":"https://pith.science/.well-known/pith/6VYDY36TOA3SRSNA5L4COKHVIS/bundle.json"},"state":{"state_type":"pith_open_graph_state","state_version":"1.0","pith_number":"pith:2024:6VYDY36TOA3SRSNA5L4COKHVIS","merge_version":"pith-open-graph-merge-v1","event_count":2,"valid_event_count":2,"invalid_event_count":0,"equivocation_count":0,"current":{"canonical_record":{"metadata":{"abstract_canon_sha256":"0323dbb5b02fcc0c2c4c48e0f609ebaeb9db7677d4ad9ae64ea0fdb450b7c44e","cross_cats_sorted":[],"license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.CL","submitted_at":"2024-12-04T09:38:11Z","title_canon_sha256":"51707907b1add9f668ad26495fcf90b1bf311c290c7e15059cd4b2d58b58e1f7"},"schema_version":"1.0","source":{"id":"2412.03160","kind":"arxiv","version":1}},"source_aliases":[{"alias_kind":"arxiv","alias_value":"2412.03160","created_at":"2026-07-05T09:44:25Z"},{"alias_kind":"arxiv_version","alias_value":"2412.03160v1","created_at":"2026-07-05T09:44:25Z"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.2412.03160","created_at":"2026-07-05T09:44:25Z"},{"alias_kind":"pith_short_12","alias_value":"6VYDY36TOA3S","created_at":"2026-07-05T09:44:25Z"},{"alias_kind":"pith_short_16","alias_value":"6VYDY36TOA3SRSNA","created_at":"2026-07-05T09:44:25Z"},{"alias_kind":"pith_short_8","alias_value":"6VYDY36T","created_at":"2026-07-05T09:44:25Z"}],"graph_snapshots":[{"event_id":"sha256:c0d628cfc487f5c8960cad4f1dccd5a69d4fe8aed2ca24c1dd088b4abdb8f1c2","target":"graph","created_at":"2026-07-05T09:44:25Z","signer":{"key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signer_id":"pith.science","signer_type":"pith_registry"},"payload":{"graph_snapshot":{"author_claims":{"count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","strong_count":0},"builder_version":"pith-number-builder-2026-05-17-v1","claims":{"count":0,"items":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"integrity":{"available":true,"clean":true,"detectors_run":[],"endpoint":"/pith/2412.03160/integrity.json","findings":[],"snapshot_sha256":"c28c3603d3b5d939e8dc4c7e95fa8dfce3d595e45f758748cecf8e644a296938","summary":{"advisory":0,"by_detector":{},"critical":0,"informational":0}},"paper":{"abstract_excerpt":"Tokenization is an important preprocessing step in the training and inference of large language models (LLMs). While there has been extensive research on the expressive power of the neural achitectures used in LLMs, the impact of tokenization has not been well understood. In this work, we demonstrate that tokenization, irrespective of the algorithm used, acts as an inverse homomorphism between strings and tokens. This suggests that the character space of the source language and the token space of the tokenized language are homomorphic, preserving the structural properties of the source languag","authors_text":"Chris Wendler, Robert West, Saibo Geng, Sankalp Gambhir","cross_cats":[],"headline":"","license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.CL","submitted_at":"2024-12-04T09:38:11Z","title":"Byte BPE Tokenization as an Inverse string Homomorphism"},"references":{"count":0,"internal_anchors":0,"resolved_work":0,"sample":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"2412.03160","kind":"arxiv","version":1},"verdict":{"created_at":null,"id":null,"model_set":{},"one_line_summary":"","pipeline_version":null,"pith_extraction_headline":"","strongest_claim":"","weakest_assumption":""}},"verdict_id":null}}],"author_attestations":[],"timestamp_anchors":[],"storage_attestations":[],"citation_signatures":[],"replication_records":[],"corrections":[],"mirror_hints":[],"record_created":{"event_id":"sha256:0379da477a92fa190e008f3f63eb372dbd51ad861170eb7428ae284cace57dfb","target":"record","created_at":"2026-07-05T09:44:25Z","signer":{"key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signer_id":"pith.science","signer_type":"pith_registry"},"payload":{"attestation_state":"computed","canonical_record":{"metadata":{"abstract_canon_sha256":"0323dbb5b02fcc0c2c4c48e0f609ebaeb9db7677d4ad9ae64ea0fdb450b7c44e","cross_cats_sorted":[],"license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.CL","submitted_at":"2024-12-04T09:38:11Z","title_canon_sha256":"51707907b1add9f668ad26495fcf90b1bf311c290c7e15059cd4b2d58b58e1f7"},"schema_version":"1.0","source":{"id":"2412.03160","kind":"arxiv","version":1}},"canonical_sha256":"f5703c6fd3703728c9a0eaf82728f5448a3fb4c8994748658ed9295425e72cf5","receipt":{"algorithm":"ed25519","builder_version":"pith-number-builder-2026-05-17-v1","canonical_sha256":"f5703c6fd3703728c9a0eaf82728f5448a3fb4c8994748658ed9295425e72cf5","first_computed_at":"2026-07-05T09:44:25.758084Z","key_id":"pith-v1-2026-05","kind":"pith_receipt","last_reissued_at":"2026-07-05T09:44:25.758084Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","receipt_version":"0.3","signature_b64":"Vekc+Lols2hzaW1V5E0puob+bPpjhOy3z6SILeW7C730ViSLGg4zDK4m7GjbnSZVVmYys9Hjrb6oh2JGHfUbDw==","signature_status":"signed_v1","signed_at":"2026-07-05T09:44:25.758539Z","signed_message":"canonical_sha256_bytes"},"source_id":"2412.03160","source_kind":"arxiv","source_version":1}}},"equivocations":[],"invalid_events":[],"applied_event_ids":["sha256:0379da477a92fa190e008f3f63eb372dbd51ad861170eb7428ae284cace57dfb","sha256:c0d628cfc487f5c8960cad4f1dccd5a69d4fe8aed2ca24c1dd088b4abdb8f1c2"],"state_sha256":"62c65ebae430ae839deb86d830311cbba6b9069c6f0cc2fe03e2b98f81a47100"},"bundle_signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"YHLkxCzjzJjep3ZouwaE8bHMjHjBtkpu46zoNBAitm1w/aMparUD1kd+MN7eqj5BpcOcTXWQdvW4gEHfSWZ1AQ==","signed_message":"bundle_sha256_bytes","signed_at":"2026-08-21T08:04:14.289361Z","bundle_sha256":"c11032defde643bae294c6d97ffb1c677da334c4de1da90a88dadb48a9ddd04a"}}