{"bundle_type":"pith_open_graph_bundle","bundle_version":"1.0","pith_number":"pith:2025:F537CTPTMZWEKD5YPLKZE6AQDI","short_pith_number":"pith:F537CTPT","canonical_record":{"source":{"id":"2507.22543","kind":"arxiv","version":1},"metadata":{"license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.LG","submitted_at":"2025-07-30T10:16:23Z","cross_cats_sorted":["cs.CL"],"title_canon_sha256":"51e44ff066de2f3b1ca1db1d5415e854909c29e1fef7fdc48d0cabe63f603c57","abstract_canon_sha256":"5727c554eeb324d94d79abc993c42ecf41bc448186baf6c2d37980ecdc9a1355"},"schema_version":"1.0"},"canonical_sha256":"2f77f14df3666c450fb87ad59278101a1316296bfd57a2bce53327aa842d30e4","source":{"kind":"arxiv","id":"2507.22543","version":1},"source_aliases":[{"alias_kind":"arxiv","alias_value":"2507.22543","created_at":"2026-07-05T11:45:44Z"},{"alias_kind":"arxiv_version","alias_value":"2507.22543v1","created_at":"2026-07-05T11:45:44Z"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.2507.22543","created_at":"2026-07-05T11:45:44Z"},{"alias_kind":"pith_short_12","alias_value":"F537CTPTMZWE","created_at":"2026-07-05T11:45:44Z"},{"alias_kind":"pith_short_16","alias_value":"F537CTPTMZWEKD5Y","created_at":"2026-07-05T11:45:44Z"},{"alias_kind":"pith_short_8","alias_value":"F537CTPT","created_at":"2026-07-05T11:45:44Z"}],"events":[{"event_type":"record_created","subject_pith_number":"pith:2025:F537CTPTMZWEKD5YPLKZE6AQDI","target":"record","payload":{"canonical_record":{"source":{"id":"2507.22543","kind":"arxiv","version":1},"metadata":{"license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.LG","submitted_at":"2025-07-30T10:16:23Z","cross_cats_sorted":["cs.CL"],"title_canon_sha256":"51e44ff066de2f3b1ca1db1d5415e854909c29e1fef7fdc48d0cabe63f603c57","abstract_canon_sha256":"5727c554eeb324d94d79abc993c42ecf41bc448186baf6c2d37980ecdc9a1355"},"schema_version":"1.0"},"canonical_sha256":"2f77f14df3666c450fb87ad59278101a1316296bfd57a2bce53327aa842d30e4","receipt":{"kind":"pith_receipt","key_id":"pith-v1-2026-05","algorithm":"ed25519","signed_at":"2026-07-05T11:45:44.338144Z","signature_b64":"0mb+2M74FBBnwFufadbMkoVCCTdg6NS+B81oHT/oleZzq/3juBgIFAk8LTIq06nB8mljbzjRwCu5vsMQTj/rBA==","signed_message":"canonical_sha256_bytes","builder_version":"pith-number-builder-2026-05-17-v1","receipt_version":"0.3","canonical_sha256":"2f77f14df3666c450fb87ad59278101a1316296bfd57a2bce53327aa842d30e4","last_reissued_at":"2026-07-05T11:45:44.337653Z","signature_status":"signed_v1","first_computed_at":"2026-07-05T11:45:44.337653Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"source_kind":"arxiv","source_id":"2507.22543","source_version":1,"attestation_state":"computed"},"signer":{"signer_id":"pith.science","signer_type":"pith_registry","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"created_at":"2026-07-05T11:45:44Z","supersedes":[],"prev_event":null,"signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"LGgesI+KVZEEYXh6RAji6lwIb57taNIvE80z0uxgbTt+wRUKS+CM+Magbpl9jTdOocc2qGAU2SVFVqivoH2GBw==","signed_message":"open_graph_event_sha256_bytes","signed_at":"2026-08-05T20:52:52.885181Z"},"content_sha256":"a90ed3783d5d77a73a0b84c62bf89db9fb528b9403a9ab92a33096c164eba78e","schema_version":"1.0","event_id":"sha256:a90ed3783d5d77a73a0b84c62bf89db9fb528b9403a9ab92a33096c164eba78e"},{"event_type":"graph_snapshot","subject_pith_number":"pith:2025:F537CTPTMZWEKD5YPLKZE6AQDI","target":"graph","payload":{"graph_snapshot":{"paper":{"title":"Pre-trained Models Perform the Best When Token Distributions Follow Zipf's Law","license":"http://creativecommons.org/licenses/by/4.0/","headline":"","cross_cats":["cs.CL"],"primary_cat":"cs.LG","authors_text":"Meng Jiang, Qingkai Zeng, Yanjin He","submitted_at":"2025-07-30T10:16:23Z","abstract_excerpt":"Tokenization is a fundamental step in natural language processing (NLP) and other sequence modeling domains, where the choice of vocabulary size significantly impacts model performance. Despite its importance, selecting an optimal vocabulary size remains underexplored, typically relying on heuristics or dataset-specific choices. In this work, we propose a principled method for determining the vocabulary size by analyzing token frequency distributions through Zipf's law. We show that downstream task performance correlates with how closely token distributions follow power-law behavior, and that "},"claims":{"count":0,"items":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"2507.22543","kind":"arxiv","version":1},"verdict":{"id":null,"model_set":{},"created_at":null,"strongest_claim":"","one_line_summary":"","pipeline_version":null,"weakest_assumption":"","pith_extraction_headline":""},"integrity":{"clean":true,"summary":{"advisory":0,"critical":0,"by_detector":{},"informational":0},"endpoint":"/pith/2507.22543/integrity.json","findings":[],"available":true,"detectors_run":[],"snapshot_sha256":"c28c3603d3b5d939e8dc4c7e95fa8dfce3d595e45f758748cecf8e644a296938"},"references":{"count":0,"sample":[],"resolved_work":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","internal_anchors":0},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"author_claims":{"count":0,"strong_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"builder_version":"pith-number-builder-2026-05-17-v1"},"verdict_id":null},"signer":{"signer_id":"pith.science","signer_type":"pith_registry","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"created_at":"2026-07-05T11:45:44Z","supersedes":[],"prev_event":null,"signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"UuLfc5f5u9tHeUHxbv8irEpWZxbyVf/qV9sTYEj4RBpmr4esiHNe/FQyUVbtTUYfx4ZeozdPWapQGsvjGHV0DA==","signed_message":"open_graph_event_sha256_bytes","signed_at":"2026-08-05T20:52:52.885595Z"},"content_sha256":"2bde6362f717f97f684674d228f009f4f1d343a2fedc6615f9004f531c3abed7","schema_version":"1.0","event_id":"sha256:2bde6362f717f97f684674d228f009f4f1d343a2fedc6615f9004f531c3abed7"}],"timestamp_proofs":[],"mirror_hints":[{"mirror_type":"https","name":"Pith Resolver","base_url":"https://pith.science","bundle_url":"https://pith.science/pith/F537CTPTMZWEKD5YPLKZE6AQDI/bundle.json","state_url":"https://pith.science/pith/F537CTPTMZWEKD5YPLKZE6AQDI/state.json","well_known_bundle_url":"https://pith.science/.well-known/pith/F537CTPTMZWEKD5YPLKZE6AQDI/bundle.json","status":"primary"}],"public_keys":[{"key_id":"pith-v1-2026-05","algorithm":"ed25519","format":"raw","public_key_b64":"stVStoiQhXFxp4s2pdzPNoqVNBMojDU/fJ2db5S3CbM=","public_key_hex":"b2d552b68890857171a78b36a5dccf368a953413288c353f7c9d9d6f94b709b3","fingerprint_sha256_b32_first128bits":"RVFV5Z2OI2J3ZUO7ERDEBCYNKS","fingerprint_sha256_hex":"8d4b5ee74e4693bcd1df2446408b0d54","rotates_at":null,"url":"https://pith.science/pith-signing-key.json","notes":"Pith uses this Ed25519 key to sign canonical record SHA-256 digests. Verify with: ed25519_verify(public_key, message=canonical_sha256_bytes, signature=base64decode(signature_b64))."}],"merge_version":"pith-open-graph-merge-v1","built_at":"2026-08-05T20:52:52Z","links":{"resolver":"https://pith.science/pith/F537CTPTMZWEKD5YPLKZE6AQDI","bundle":"https://pith.science/pith/F537CTPTMZWEKD5YPLKZE6AQDI/bundle.json","state":"https://pith.science/pith/F537CTPTMZWEKD5YPLKZE6AQDI/state.json","well_known_bundle":"https://pith.science/.well-known/pith/F537CTPTMZWEKD5YPLKZE6AQDI/bundle.json"},"state":{"state_type":"pith_open_graph_state","state_version":"1.0","pith_number":"pith:2025:F537CTPTMZWEKD5YPLKZE6AQDI","merge_version":"pith-open-graph-merge-v1","event_count":2,"valid_event_count":2,"invalid_event_count":0,"equivocation_count":0,"current":{"canonical_record":{"metadata":{"abstract_canon_sha256":"5727c554eeb324d94d79abc993c42ecf41bc448186baf6c2d37980ecdc9a1355","cross_cats_sorted":["cs.CL"],"license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.LG","submitted_at":"2025-07-30T10:16:23Z","title_canon_sha256":"51e44ff066de2f3b1ca1db1d5415e854909c29e1fef7fdc48d0cabe63f603c57"},"schema_version":"1.0","source":{"id":"2507.22543","kind":"arxiv","version":1}},"source_aliases":[{"alias_kind":"arxiv","alias_value":"2507.22543","created_at":"2026-07-05T11:45:44Z"},{"alias_kind":"arxiv_version","alias_value":"2507.22543v1","created_at":"2026-07-05T11:45:44Z"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.2507.22543","created_at":"2026-07-05T11:45:44Z"},{"alias_kind":"pith_short_12","alias_value":"F537CTPTMZWE","created_at":"2026-07-05T11:45:44Z"},{"alias_kind":"pith_short_16","alias_value":"F537CTPTMZWEKD5Y","created_at":"2026-07-05T11:45:44Z"},{"alias_kind":"pith_short_8","alias_value":"F537CTPT","created_at":"2026-07-05T11:45:44Z"}],"graph_snapshots":[{"event_id":"sha256:2bde6362f717f97f684674d228f009f4f1d343a2fedc6615f9004f531c3abed7","target":"graph","created_at":"2026-07-05T11:45:44Z","signer":{"key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signer_id":"pith.science","signer_type":"pith_registry"},"payload":{"graph_snapshot":{"author_claims":{"count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","strong_count":0},"builder_version":"pith-number-builder-2026-05-17-v1","claims":{"count":0,"items":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"integrity":{"available":true,"clean":true,"detectors_run":[],"endpoint":"/pith/2507.22543/integrity.json","findings":[],"snapshot_sha256":"c28c3603d3b5d939e8dc4c7e95fa8dfce3d595e45f758748cecf8e644a296938","summary":{"advisory":0,"by_detector":{},"critical":0,"informational":0}},"paper":{"abstract_excerpt":"Tokenization is a fundamental step in natural language processing (NLP) and other sequence modeling domains, where the choice of vocabulary size significantly impacts model performance. Despite its importance, selecting an optimal vocabulary size remains underexplored, typically relying on heuristics or dataset-specific choices. In this work, we propose a principled method for determining the vocabulary size by analyzing token frequency distributions through Zipf's law. We show that downstream task performance correlates with how closely token distributions follow power-law behavior, and that ","authors_text":"Meng Jiang, Qingkai Zeng, Yanjin He","cross_cats":["cs.CL"],"headline":"","license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.LG","submitted_at":"2025-07-30T10:16:23Z","title":"Pre-trained Models Perform the Best When Token Distributions Follow Zipf's Law"},"references":{"count":0,"internal_anchors":0,"resolved_work":0,"sample":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"2507.22543","kind":"arxiv","version":1},"verdict":{"created_at":null,"id":null,"model_set":{},"one_line_summary":"","pipeline_version":null,"pith_extraction_headline":"","strongest_claim":"","weakest_assumption":""}},"verdict_id":null}}],"author_attestations":[],"timestamp_anchors":[],"storage_attestations":[],"citation_signatures":[],"replication_records":[],"corrections":[],"mirror_hints":[],"record_created":{"event_id":"sha256:a90ed3783d5d77a73a0b84c62bf89db9fb528b9403a9ab92a33096c164eba78e","target":"record","created_at":"2026-07-05T11:45:44Z","signer":{"key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signer_id":"pith.science","signer_type":"pith_registry"},"payload":{"attestation_state":"computed","canonical_record":{"metadata":{"abstract_canon_sha256":"5727c554eeb324d94d79abc993c42ecf41bc448186baf6c2d37980ecdc9a1355","cross_cats_sorted":["cs.CL"],"license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.LG","submitted_at":"2025-07-30T10:16:23Z","title_canon_sha256":"51e44ff066de2f3b1ca1db1d5415e854909c29e1fef7fdc48d0cabe63f603c57"},"schema_version":"1.0","source":{"id":"2507.22543","kind":"arxiv","version":1}},"canonical_sha256":"2f77f14df3666c450fb87ad59278101a1316296bfd57a2bce53327aa842d30e4","receipt":{"algorithm":"ed25519","builder_version":"pith-number-builder-2026-05-17-v1","canonical_sha256":"2f77f14df3666c450fb87ad59278101a1316296bfd57a2bce53327aa842d30e4","first_computed_at":"2026-07-05T11:45:44.337653Z","key_id":"pith-v1-2026-05","kind":"pith_receipt","last_reissued_at":"2026-07-05T11:45:44.337653Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","receipt_version":"0.3","signature_b64":"0mb+2M74FBBnwFufadbMkoVCCTdg6NS+B81oHT/oleZzq/3juBgIFAk8LTIq06nB8mljbzjRwCu5vsMQTj/rBA==","signature_status":"signed_v1","signed_at":"2026-07-05T11:45:44.338144Z","signed_message":"canonical_sha256_bytes"},"source_id":"2507.22543","source_kind":"arxiv","source_version":1}}},"equivocations":[],"invalid_events":[],"applied_event_ids":["sha256:a90ed3783d5d77a73a0b84c62bf89db9fb528b9403a9ab92a33096c164eba78e","sha256:2bde6362f717f97f684674d228f009f4f1d343a2fedc6615f9004f531c3abed7"],"state_sha256":"99eae38958d25cddd0745a8e3f0fae9f5cbfc4a6e46a730aa09530392f4b3c20"},"bundle_signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"nXF7m69L8pM31Usy6UMLhh5zKxvWfj8ftsL6PywxYz+CSxUEOW7mKHeBTYlGUyBVgQdqVkboG3ZAGHyazy32BA==","signed_message":"bundle_sha256_bytes","signed_at":"2026-08-05T20:52:52.888136Z","bundle_sha256":"bda2c7de1852dc0ef8858eadd14c96cb04cba3479527a02631d837f30c03a1af"}}