{"bundle_type":"pith_open_graph_bundle","bundle_version":"1.0","pith_number":"pith:2024:H37YC3ZBYAJYCKQWNHOTBRU4VP","short_pith_number":"pith:H37YC3ZB","canonical_record":{"source":{"id":"2407.11606","kind":"arxiv","version":4},"metadata":{"license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.CL","submitted_at":"2024-07-16T11:12:28Z","cross_cats_sorted":["cs.AI","cs.LG"],"title_canon_sha256":"a3e3d95d6f30d1e433ef816f1b1ce20c4b37af68a3952c58bd76830b71322022","abstract_canon_sha256":"fe99fa9569185a34c73732ee71173d7512103cf48bfb781724b1f9611dbb8f27"},"schema_version":"1.0"},"canonical_sha256":"3eff816f21c013812a1669dd30c69cabc66b2626e2f59f2bafee05916677b1c5","source":{"kind":"arxiv","id":"2407.11606","version":4},"source_aliases":[{"alias_kind":"arxiv","alias_value":"2407.11606","created_at":"2026-07-05T10:43:36Z"},{"alias_kind":"arxiv_version","alias_value":"2407.11606v4","created_at":"2026-07-05T10:43:36Z"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.2407.11606","created_at":"2026-07-05T10:43:36Z"},{"alias_kind":"pith_short_12","alias_value":"H37YC3ZBYAJY","created_at":"2026-07-05T10:43:36Z"},{"alias_kind":"pith_short_16","alias_value":"H37YC3ZBYAJYCKQW","created_at":"2026-07-05T10:43:36Z"},{"alias_kind":"pith_short_8","alias_value":"H37YC3ZB","created_at":"2026-07-05T10:43:36Z"}],"events":[{"event_type":"record_created","subject_pith_number":"pith:2024:H37YC3ZBYAJYCKQWNHOTBRU4VP","target":"record","payload":{"canonical_record":{"source":{"id":"2407.11606","kind":"arxiv","version":4},"metadata":{"license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.CL","submitted_at":"2024-07-16T11:12:28Z","cross_cats_sorted":["cs.AI","cs.LG"],"title_canon_sha256":"a3e3d95d6f30d1e433ef816f1b1ce20c4b37af68a3952c58bd76830b71322022","abstract_canon_sha256":"fe99fa9569185a34c73732ee71173d7512103cf48bfb781724b1f9611dbb8f27"},"schema_version":"1.0"},"canonical_sha256":"3eff816f21c013812a1669dd30c69cabc66b2626e2f59f2bafee05916677b1c5","receipt":{"kind":"pith_receipt","key_id":"pith-v1-2026-05","algorithm":"ed25519","signed_at":"2026-07-05T10:43:36.895578Z","signature_b64":"jYsYpCdNAhSHH3Pfaza/2I+76W909KCWt3V/HD6swXMM040dN6yrMHl8rt5UtW4Xfng1asf12jQjSQawUto4Ag==","signed_message":"canonical_sha256_bytes","builder_version":"pith-number-builder-2026-05-17-v1","receipt_version":"0.3","canonical_sha256":"3eff816f21c013812a1669dd30c69cabc66b2626e2f59f2bafee05916677b1c5","last_reissued_at":"2026-07-05T10:43:36.895052Z","signature_status":"signed_v1","first_computed_at":"2026-07-05T10:43:36.895052Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"source_kind":"arxiv","source_id":"2407.11606","source_version":4,"attestation_state":"computed"},"signer":{"signer_id":"pith.science","signer_type":"pith_registry","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"created_at":"2026-07-05T10:43:36Z","supersedes":[],"prev_event":null,"signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"MLKK0HZpmQ3Ov44O4auXZVX9ifKm9L11tq+yA7fcvOilMJPn0mL/fiw+BLyUT5KgYUUZ4eOCp8sj1bI6UCt6BQ==","signed_message":"open_graph_event_sha256_bytes","signed_at":"2026-08-01T12:35:48.537387Z"},"content_sha256":"b0e35be07026c8d3babd5e936a11c8f51446c89e737d9d95b9a8e54ea03f0ab8","schema_version":"1.0","event_id":"sha256:b0e35be07026c8d3babd5e936a11c8f51446c89e737d9d95b9a8e54ea03f0ab8"},{"event_type":"graph_snapshot","subject_pith_number":"pith:2024:H37YC3ZBYAJYCKQWNHOTBRU4VP","target":"graph","payload":{"graph_snapshot":{"paper":{"title":"The Foundations of Tokenization: Statistical and Computational Concerns","license":"http://creativecommons.org/licenses/by/4.0/","headline":"","cross_cats":["cs.AI","cs.LG"],"primary_cat":"cs.CL","authors_text":"Brian DuSell, John Terilla, Juan Luis Gastaldi, Luca Malagutti, Ryan Cotterell, Tim Vieira","submitted_at":"2024-07-16T11:12:28Z","abstract_excerpt":"Tokenization - the practice of converting strings of characters from an alphabet into sequences of tokens over a vocabulary - is a critical step in the NLP pipeline. The use of token representations is widely credited with increased model performance but is also the source of many undesirable behaviors, such as spurious ambiguity or inconsistency. Despite its recognized importance as a standard representation method in NLP, the theoretical underpinnings of tokenization are not yet fully understood. In particular, the impact of tokenization on language model estimation has been investigated pri"},"claims":{"count":0,"items":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"2407.11606","kind":"arxiv","version":4},"verdict":{"id":null,"model_set":{},"created_at":null,"strongest_claim":"","one_line_summary":"","pipeline_version":null,"weakest_assumption":"","pith_extraction_headline":""},"integrity":{"clean":true,"summary":{"advisory":0,"critical":0,"by_detector":{},"informational":0},"endpoint":"/pith/2407.11606/integrity.json","findings":[],"available":true,"detectors_run":[],"snapshot_sha256":"c28c3603d3b5d939e8dc4c7e95fa8dfce3d595e45f758748cecf8e644a296938"},"references":{"count":0,"sample":[],"resolved_work":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","internal_anchors":0},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"author_claims":{"count":0,"strong_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"builder_version":"pith-number-builder-2026-05-17-v1"},"verdict_id":null},"signer":{"signer_id":"pith.science","signer_type":"pith_registry","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"created_at":"2026-07-05T10:43:36Z","supersedes":[],"prev_event":null,"signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"huS7Avt2h2/RN2TcDYpfICUwqmyMrx5uAmsx33LIff9dTW+qxlxGXeKAwMKclVsK+KvWQozqFn1tocscAFgFAg==","signed_message":"open_graph_event_sha256_bytes","signed_at":"2026-08-01T12:35:48.537883Z"},"content_sha256":"fab6ad7f6f0d4c193a6b78e8e3bbe001e0df3b78b15dfb005a38029aedf1fa1a","schema_version":"1.0","event_id":"sha256:fab6ad7f6f0d4c193a6b78e8e3bbe001e0df3b78b15dfb005a38029aedf1fa1a"}],"timestamp_proofs":[],"mirror_hints":[{"mirror_type":"https","name":"Pith Resolver","base_url":"https://pith.science","bundle_url":"https://pith.science/pith/H37YC3ZBYAJYCKQWNHOTBRU4VP/bundle.json","state_url":"https://pith.science/pith/H37YC3ZBYAJYCKQWNHOTBRU4VP/state.json","well_known_bundle_url":"https://pith.science/.well-known/pith/H37YC3ZBYAJYCKQWNHOTBRU4VP/bundle.json","status":"primary"}],"public_keys":[{"key_id":"pith-v1-2026-05","algorithm":"ed25519","format":"raw","public_key_b64":"stVStoiQhXFxp4s2pdzPNoqVNBMojDU/fJ2db5S3CbM=","public_key_hex":"b2d552b68890857171a78b36a5dccf368a953413288c353f7c9d9d6f94b709b3","fingerprint_sha256_b32_first128bits":"RVFV5Z2OI2J3ZUO7ERDEBCYNKS","fingerprint_sha256_hex":"8d4b5ee74e4693bcd1df2446408b0d54","rotates_at":null,"url":"https://pith.science/pith-signing-key.json","notes":"Pith uses this Ed25519 key to sign canonical record SHA-256 digests. Verify with: ed25519_verify(public_key, message=canonical_sha256_bytes, signature=base64decode(signature_b64))."}],"merge_version":"pith-open-graph-merge-v1","built_at":"2026-08-01T12:35:48Z","links":{"resolver":"https://pith.science/pith/H37YC3ZBYAJYCKQWNHOTBRU4VP","bundle":"https://pith.science/pith/H37YC3ZBYAJYCKQWNHOTBRU4VP/bundle.json","state":"https://pith.science/pith/H37YC3ZBYAJYCKQWNHOTBRU4VP/state.json","well_known_bundle":"https://pith.science/.well-known/pith/H37YC3ZBYAJYCKQWNHOTBRU4VP/bundle.json"},"state":{"state_type":"pith_open_graph_state","state_version":"1.0","pith_number":"pith:2024:H37YC3ZBYAJYCKQWNHOTBRU4VP","merge_version":"pith-open-graph-merge-v1","event_count":2,"valid_event_count":2,"invalid_event_count":0,"equivocation_count":0,"current":{"canonical_record":{"metadata":{"abstract_canon_sha256":"fe99fa9569185a34c73732ee71173d7512103cf48bfb781724b1f9611dbb8f27","cross_cats_sorted":["cs.AI","cs.LG"],"license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.CL","submitted_at":"2024-07-16T11:12:28Z","title_canon_sha256":"a3e3d95d6f30d1e433ef816f1b1ce20c4b37af68a3952c58bd76830b71322022"},"schema_version":"1.0","source":{"id":"2407.11606","kind":"arxiv","version":4}},"source_aliases":[{"alias_kind":"arxiv","alias_value":"2407.11606","created_at":"2026-07-05T10:43:36Z"},{"alias_kind":"arxiv_version","alias_value":"2407.11606v4","created_at":"2026-07-05T10:43:36Z"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.2407.11606","created_at":"2026-07-05T10:43:36Z"},{"alias_kind":"pith_short_12","alias_value":"H37YC3ZBYAJY","created_at":"2026-07-05T10:43:36Z"},{"alias_kind":"pith_short_16","alias_value":"H37YC3ZBYAJYCKQW","created_at":"2026-07-05T10:43:36Z"},{"alias_kind":"pith_short_8","alias_value":"H37YC3ZB","created_at":"2026-07-05T10:43:36Z"}],"graph_snapshots":[{"event_id":"sha256:fab6ad7f6f0d4c193a6b78e8e3bbe001e0df3b78b15dfb005a38029aedf1fa1a","target":"graph","created_at":"2026-07-05T10:43:36Z","signer":{"key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signer_id":"pith.science","signer_type":"pith_registry"},"payload":{"graph_snapshot":{"author_claims":{"count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","strong_count":0},"builder_version":"pith-number-builder-2026-05-17-v1","claims":{"count":0,"items":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"integrity":{"available":true,"clean":true,"detectors_run":[],"endpoint":"/pith/2407.11606/integrity.json","findings":[],"snapshot_sha256":"c28c3603d3b5d939e8dc4c7e95fa8dfce3d595e45f758748cecf8e644a296938","summary":{"advisory":0,"by_detector":{},"critical":0,"informational":0}},"paper":{"abstract_excerpt":"Tokenization - the practice of converting strings of characters from an alphabet into sequences of tokens over a vocabulary - is a critical step in the NLP pipeline. The use of token representations is widely credited with increased model performance but is also the source of many undesirable behaviors, such as spurious ambiguity or inconsistency. Despite its recognized importance as a standard representation method in NLP, the theoretical underpinnings of tokenization are not yet fully understood. In particular, the impact of tokenization on language model estimation has been investigated pri","authors_text":"Brian DuSell, John Terilla, Juan Luis Gastaldi, Luca Malagutti, Ryan Cotterell, Tim Vieira","cross_cats":["cs.AI","cs.LG"],"headline":"","license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.CL","submitted_at":"2024-07-16T11:12:28Z","title":"The Foundations of Tokenization: Statistical and Computational Concerns"},"references":{"count":0,"internal_anchors":0,"resolved_work":0,"sample":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"2407.11606","kind":"arxiv","version":4},"verdict":{"created_at":null,"id":null,"model_set":{},"one_line_summary":"","pipeline_version":null,"pith_extraction_headline":"","strongest_claim":"","weakest_assumption":""}},"verdict_id":null}}],"author_attestations":[],"timestamp_anchors":[],"storage_attestations":[],"citation_signatures":[],"replication_records":[],"corrections":[],"mirror_hints":[],"record_created":{"event_id":"sha256:b0e35be07026c8d3babd5e936a11c8f51446c89e737d9d95b9a8e54ea03f0ab8","target":"record","created_at":"2026-07-05T10:43:36Z","signer":{"key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signer_id":"pith.science","signer_type":"pith_registry"},"payload":{"attestation_state":"computed","canonical_record":{"metadata":{"abstract_canon_sha256":"fe99fa9569185a34c73732ee71173d7512103cf48bfb781724b1f9611dbb8f27","cross_cats_sorted":["cs.AI","cs.LG"],"license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.CL","submitted_at":"2024-07-16T11:12:28Z","title_canon_sha256":"a3e3d95d6f30d1e433ef816f1b1ce20c4b37af68a3952c58bd76830b71322022"},"schema_version":"1.0","source":{"id":"2407.11606","kind":"arxiv","version":4}},"canonical_sha256":"3eff816f21c013812a1669dd30c69cabc66b2626e2f59f2bafee05916677b1c5","receipt":{"algorithm":"ed25519","builder_version":"pith-number-builder-2026-05-17-v1","canonical_sha256":"3eff816f21c013812a1669dd30c69cabc66b2626e2f59f2bafee05916677b1c5","first_computed_at":"2026-07-05T10:43:36.895052Z","key_id":"pith-v1-2026-05","kind":"pith_receipt","last_reissued_at":"2026-07-05T10:43:36.895052Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","receipt_version":"0.3","signature_b64":"jYsYpCdNAhSHH3Pfaza/2I+76W909KCWt3V/HD6swXMM040dN6yrMHl8rt5UtW4Xfng1asf12jQjSQawUto4Ag==","signature_status":"signed_v1","signed_at":"2026-07-05T10:43:36.895578Z","signed_message":"canonical_sha256_bytes"},"source_id":"2407.11606","source_kind":"arxiv","source_version":4}}},"equivocations":[],"invalid_events":[],"applied_event_ids":["sha256:b0e35be07026c8d3babd5e936a11c8f51446c89e737d9d95b9a8e54ea03f0ab8","sha256:fab6ad7f6f0d4c193a6b78e8e3bbe001e0df3b78b15dfb005a38029aedf1fa1a"],"state_sha256":"aa32f847aed131c3ec9b5b847e717e3c6beb0fa93a9d523b6d650f68be2d5334"},"bundle_signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"svpexWme4oafVtvApL4e3Vpx+lBwHc1X6uBy12eSps2mAgVjTYM1Qo0AcviOmtiPRTPlUswm9Rin++gle4BRCg==","signed_message":"bundle_sha256_bytes","signed_at":"2026-08-01T12:35:48.551512Z","bundle_sha256":"1e8c61de143c79ef67d10729509bde7fac5423731f5c99785afafcb1a79e11ee"}}