{"record_type":"pith_number_record","schema_url":"https://pith.science/schemas/pith-number/v1.json","pith_number":"pith:2025:FQJBVWDMIRQZC5A23ANOS3G34P","short_pith_number":"pith:FQJBVWDM","schema_version":"1.0","canonical_sha256":"2c121ad86c446191741ad81ae96cdbe3fc42b536c19de5362225e67617d445a5","source":{"kind":"arxiv","id":"2502.20273","version":4},"attestation_state":"computed","paper":{"title":"How Much is Enough? The Diminishing Returns of Tokenization Training Data","license":"http://creativecommons.org/licenses/by/4.0/","headline":"","cross_cats":["cs.CE"],"primary_cat":"cs.CL","authors_text":"Chris Tanner, Craig W. Schmidt, Varshini Reddy, Yuval Pinter","submitted_at":"2025-02-27T17:01:23Z","abstract_excerpt":"Tokenization, a crucial initial step in natural language processing, is governed by several key parameters, such as the tokenization algorithm, vocabulary size, pre-tokenization strategy, inference strategy, and training data corpus. This paper investigates the impact of an often-overlooked hyperparameter, tokenizer training data size. We train BPE, UnigramLM, and WordPiece tokenizers across various vocabulary sizes using English training data ranging from 1GB to 900GB. Our findings reveal diminishing returns as training data size increases beyond roughly 150GB, suggesting a practical limit to"},"verification_status":{"content_addressed":true,"pith_receipt":true,"author_attested":false,"weak_author_claims":0,"strong_author_claims":0,"externally_anchored":false,"storage_verified":false,"citation_signatures":0,"replication_records":0,"graph_snapshot":true,"references_resolved":false,"formal_links_present":false},"canonical_record":{"source":{"id":"2502.20273","kind":"arxiv","version":4},"metadata":{"license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.CL","submitted_at":"2025-02-27T17:01:23Z","cross_cats_sorted":["cs.CE"],"title_canon_sha256":"aa0783ec1d352614dabc77cdac5f0f49dc276410c8700b6bce7eb8425c0b081c","abstract_canon_sha256":"75498295f0a839e96ca624f75a23f6b8fb38a59d02eb8fb7201a069f42464872"},"schema_version":"1.0"},"receipt":{"kind":"pith_receipt","key_id":"pith-v1-2026-05","algorithm":"ed25519","signed_at":"2026-07-05T11:22:17.899156Z","signature_b64":"+6WU1G9xknNmiA7BO66YaDh6xH3dU9ZwwYUbyAyc0rN1jXa/Ut6rVG2X/eZf4DiOZjHbqAc4ap05RTJ4Xw5eBA==","signed_message":"canonical_sha256_bytes","builder_version":"pith-number-builder-2026-05-17-v1","receipt_version":"0.3","canonical_sha256":"2c121ad86c446191741ad81ae96cdbe3fc42b536c19de5362225e67617d445a5","last_reissued_at":"2026-07-05T11:22:17.898653Z","signature_status":"signed_v1","first_computed_at":"2026-07-05T11:22:17.898653Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"graph_snapshot":{"paper":{"title":"How Much is Enough? The Diminishing Returns of Tokenization Training Data","license":"http://creativecommons.org/licenses/by/4.0/","headline":"","cross_cats":["cs.CE"],"primary_cat":"cs.CL","authors_text":"Chris Tanner, Craig W. Schmidt, Varshini Reddy, Yuval Pinter","submitted_at":"2025-02-27T17:01:23Z","abstract_excerpt":"Tokenization, a crucial initial step in natural language processing, is governed by several key parameters, such as the tokenization algorithm, vocabulary size, pre-tokenization strategy, inference strategy, and training data corpus. This paper investigates the impact of an often-overlooked hyperparameter, tokenizer training data size. We train BPE, UnigramLM, and WordPiece tokenizers across various vocabulary sizes using English training data ranging from 1GB to 900GB. Our findings reveal diminishing returns as training data size increases beyond roughly 150GB, suggesting a practical limit to"},"claims":{"count":0,"items":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"2502.20273","kind":"arxiv","version":4},"verdict":{"id":null,"model_set":{},"created_at":null,"strongest_claim":"","one_line_summary":"","pipeline_version":null,"weakest_assumption":"","pith_extraction_headline":""},"integrity":{"clean":true,"summary":{"advisory":0,"critical":0,"by_detector":{},"informational":0},"endpoint":"/pith/2502.20273/integrity.json","findings":[],"available":true,"detectors_run":[],"snapshot_sha256":"c28c3603d3b5d939e8dc4c7e95fa8dfce3d595e45f758748cecf8e644a296938"},"references":{"count":0,"sample":[],"resolved_work":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","internal_anchors":0},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"author_claims":{"count":0,"strong_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"builder_version":"pith-number-builder-2026-05-17-v1"},"aliases":[{"alias_kind":"arxiv","alias_value":"2502.20273","created_at":"2026-07-05T11:22:17.898713+00:00"},{"alias_kind":"arxiv_version","alias_value":"2502.20273v4","created_at":"2026-07-05T11:22:17.898713+00:00"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.2502.20273","created_at":"2026-07-05T11:22:17.898713+00:00"},{"alias_kind":"pith_short_12","alias_value":"FQJBVWDMIRQZ","created_at":"2026-07-05T11:22:17.898713+00:00"},{"alias_kind":"pith_short_16","alias_value":"FQJBVWDMIRQZC5A2","created_at":"2026-07-05T11:22:17.898713+00:00"},{"alias_kind":"pith_short_8","alias_value":"FQJBVWDM","created_at":"2026-07-05T11:22:17.898713+00:00"}],"events":[],"event_summary":{},"paper_claims":[],"inbound_citations":{"count":1,"internal_anchor_count":0,"sample":[{"citing_arxiv_id":"2604.05192","citing_title":"Faster Superword Tokenization","ref_index":12,"is_internal_anchor":false}]},"formal_canon":{"evidence_count":0,"sample":[],"anchors":[]},"links":{"html":"https://pith.science/pith/FQJBVWDMIRQZC5A23ANOS3G34P","json":"https://pith.science/pith/FQJBVWDMIRQZC5A23ANOS3G34P.json","graph_json":"https://pith.science/api/pith-number/FQJBVWDMIRQZC5A23ANOS3G34P/graph.json","events_json":"https://pith.science/api/pith-number/FQJBVWDMIRQZC5A23ANOS3G34P/events.json","paper":"https://pith.science/paper/FQJBVWDM"},"agent_actions":{"view_html":"https://pith.science/pith/FQJBVWDMIRQZC5A23ANOS3G34P","download_json":"https://pith.science/pith/FQJBVWDMIRQZC5A23ANOS3G34P.json","view_paper":"https://pith.science/paper/FQJBVWDM","resolve_alias":"https://pith.science/api/pith-number/resolve?arxiv=2502.20273&json=true","fetch_graph":"https://pith.science/api/pith-number/FQJBVWDMIRQZC5A23ANOS3G34P/graph.json","fetch_events":"https://pith.science/api/pith-number/FQJBVWDMIRQZC5A23ANOS3G34P/events.json","actions":{"anchor_timestamp":"https://pith.science/pith/FQJBVWDMIRQZC5A23ANOS3G34P/action/timestamp_anchor","attest_storage":"https://pith.science/pith/FQJBVWDMIRQZC5A23ANOS3G34P/action/storage_attestation","attest_author":"https://pith.science/pith/FQJBVWDMIRQZC5A23ANOS3G34P/action/author_attestation","sign_citation":"https://pith.science/pith/FQJBVWDMIRQZC5A23ANOS3G34P/action/citation_signature","submit_replication":"https://pith.science/pith/FQJBVWDMIRQZC5A23ANOS3G34P/action/replication_record"}},"created_at":"2026-07-05T11:22:17.898713+00:00","updated_at":"2026-07-05T11:22:17.898713+00:00"}