{"bundle_type":"pith_open_graph_bundle","bundle_version":"1.0","pith_number":"pith:2025:EB5FDIVGKJQI7J7NQQBZK3B3TG","short_pith_number":"pith:EB5FDIVG","canonical_record":{"source":{"id":"2506.03101","kind":"arxiv","version":1},"metadata":{"license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.CL","submitted_at":"2025-06-03T17:35:56Z","cross_cats_sorted":[],"title_canon_sha256":"1104674e1740cca8febc0581499605eaf2ef1f8ebc81f617911fc5e67544058e","abstract_canon_sha256":"3ee61e12ee34464d180fc33505a0782ae195f479a6fc530fb18eedb43150d954"},"schema_version":"1.0"},"canonical_sha256":"207a51a2a652608fa7ed8403956c3b9993444ebb1e877c6cd98d86db0587606b","source":{"kind":"arxiv","id":"2506.03101","version":1},"source_aliases":[{"alias_kind":"arxiv","alias_value":"2506.03101","created_at":"2026-07-05T11:15:14Z"},{"alias_kind":"arxiv_version","alias_value":"2506.03101v1","created_at":"2026-07-05T11:15:14Z"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.2506.03101","created_at":"2026-07-05T11:15:14Z"},{"alias_kind":"pith_short_12","alias_value":"EB5FDIVGKJQI","created_at":"2026-07-05T11:15:14Z"},{"alias_kind":"pith_short_16","alias_value":"EB5FDIVGKJQI7J7N","created_at":"2026-07-05T11:15:14Z"},{"alias_kind":"pith_short_8","alias_value":"EB5FDIVG","created_at":"2026-07-05T11:15:14Z"}],"events":[{"event_type":"record_created","subject_pith_number":"pith:2025:EB5FDIVGKJQI7J7NQQBZK3B3TG","target":"record","payload":{"canonical_record":{"source":{"id":"2506.03101","kind":"arxiv","version":1},"metadata":{"license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.CL","submitted_at":"2025-06-03T17:35:56Z","cross_cats_sorted":[],"title_canon_sha256":"1104674e1740cca8febc0581499605eaf2ef1f8ebc81f617911fc5e67544058e","abstract_canon_sha256":"3ee61e12ee34464d180fc33505a0782ae195f479a6fc530fb18eedb43150d954"},"schema_version":"1.0"},"canonical_sha256":"207a51a2a652608fa7ed8403956c3b9993444ebb1e877c6cd98d86db0587606b","receipt":{"kind":"pith_receipt","key_id":"pith-v1-2026-05","algorithm":"ed25519","signed_at":"2026-07-05T11:15:14.888163Z","signature_b64":"M7eOzz6X9QpnYBo1N+hDBw4Lj8i1BWuC9qsqyqMOE3tKKm5IujWmUO8JqpLxDtXEA0iIXeSirJPAQaZAbKreDw==","signed_message":"canonical_sha256_bytes","builder_version":"pith-number-builder-2026-05-17-v1","receipt_version":"0.3","canonical_sha256":"207a51a2a652608fa7ed8403956c3b9993444ebb1e877c6cd98d86db0587606b","last_reissued_at":"2026-07-05T11:15:14.887654Z","signature_status":"signed_v1","first_computed_at":"2026-07-05T11:15:14.887654Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"source_kind":"arxiv","source_id":"2506.03101","source_version":1,"attestation_state":"computed"},"signer":{"signer_id":"pith.science","signer_type":"pith_registry","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"created_at":"2026-07-05T11:15:14Z","supersedes":[],"prev_event":null,"signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"3kKcoTYYPMerZTpSBxe6UiVb4UAMbd3RyLrIV0rJuPg967tX+Bn7Tf2co0WD4W6LVK12rXJ+qpwq4SR8ZebyAQ==","signed_message":"open_graph_event_sha256_bytes","signed_at":"2026-08-08T11:51:38.690325Z"},"content_sha256":"407564be54581fead6d4c17ae5af3591f66eed64f9abf3d0a896a771c6370f5c","schema_version":"1.0","event_id":"sha256:407564be54581fead6d4c17ae5af3591f66eed64f9abf3d0a896a771c6370f5c"},{"event_type":"graph_snapshot","subject_pith_number":"pith:2025:EB5FDIVGKJQI7J7NQQBZK3B3TG","target":"graph","payload":{"graph_snapshot":{"paper":{"title":"Beyond Text Compression: Evaluating Tokenizers Across Scales","license":"http://creativecommons.org/licenses/by/4.0/","headline":"","cross_cats":[],"primary_cat":"cs.CL","authors_text":"Ant\\'onio V. Lopes, Hendra Setiawan, Jonas F. Lotz, Leonardo Emili, Stephan Peitz","submitted_at":"2025-06-03T17:35:56Z","abstract_excerpt":"The choice of tokenizer can profoundly impact language model performance, yet accessible and reliable evaluations of tokenizer quality remain an open challenge. Inspired by scaling consistency, we show that smaller models can accurately predict significant differences in tokenizer impact on larger models at a fraction of the compute cost. By systematically evaluating both English-centric and multilingual tokenizers, we find that tokenizer choice has negligible effects on tasks in English but results in consistent performance differences in multilingual settings. We propose new intrinsic tokeni"},"claims":{"count":0,"items":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"2506.03101","kind":"arxiv","version":1},"verdict":{"id":null,"model_set":{},"created_at":null,"strongest_claim":"","one_line_summary":"","pipeline_version":null,"weakest_assumption":"","pith_extraction_headline":""},"integrity":{"clean":true,"summary":{"advisory":0,"critical":0,"by_detector":{},"informational":0},"endpoint":"/pith/2506.03101/integrity.json","findings":[],"available":true,"detectors_run":[],"snapshot_sha256":"c28c3603d3b5d939e8dc4c7e95fa8dfce3d595e45f758748cecf8e644a296938"},"references":{"count":0,"sample":[],"resolved_work":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","internal_anchors":0},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"author_claims":{"count":0,"strong_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"builder_version":"pith-number-builder-2026-05-17-v1"},"verdict_id":null},"signer":{"signer_id":"pith.science","signer_type":"pith_registry","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"created_at":"2026-07-05T11:15:14Z","supersedes":[],"prev_event":null,"signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"V7bMCQGbuqx+3+iyzzyxlgMCwOiJH6lSzE2kjugjRVPkfavf8okk7x55bvPLihC9nY1pKMEH25GpvqXcJnqbDA==","signed_message":"open_graph_event_sha256_bytes","signed_at":"2026-08-08T11:51:38.690847Z"},"content_sha256":"d57db1124552fb38361b6fa3baf1063f491756bc3cdbfd33829c42b9d1d5c627","schema_version":"1.0","event_id":"sha256:d57db1124552fb38361b6fa3baf1063f491756bc3cdbfd33829c42b9d1d5c627"}],"timestamp_proofs":[],"mirror_hints":[{"mirror_type":"https","name":"Pith Resolver","base_url":"https://pith.science","bundle_url":"https://pith.science/pith/EB5FDIVGKJQI7J7NQQBZK3B3TG/bundle.json","state_url":"https://pith.science/pith/EB5FDIVGKJQI7J7NQQBZK3B3TG/state.json","well_known_bundle_url":"https://pith.science/.well-known/pith/EB5FDIVGKJQI7J7NQQBZK3B3TG/bundle.json","status":"primary"}],"public_keys":[{"key_id":"pith-v1-2026-05","algorithm":"ed25519","format":"raw","public_key_b64":"stVStoiQhXFxp4s2pdzPNoqVNBMojDU/fJ2db5S3CbM=","public_key_hex":"b2d552b68890857171a78b36a5dccf368a953413288c353f7c9d9d6f94b709b3","fingerprint_sha256_b32_first128bits":"RVFV5Z2OI2J3ZUO7ERDEBCYNKS","fingerprint_sha256_hex":"8d4b5ee74e4693bcd1df2446408b0d54","rotates_at":null,"url":"https://pith.science/pith-signing-key.json","notes":"Pith uses this Ed25519 key to sign canonical record SHA-256 digests. Verify with: ed25519_verify(public_key, message=canonical_sha256_bytes, signature=base64decode(signature_b64))."}],"merge_version":"pith-open-graph-merge-v1","built_at":"2026-08-08T11:51:38Z","links":{"resolver":"https://pith.science/pith/EB5FDIVGKJQI7J7NQQBZK3B3TG","bundle":"https://pith.science/pith/EB5FDIVGKJQI7J7NQQBZK3B3TG/bundle.json","state":"https://pith.science/pith/EB5FDIVGKJQI7J7NQQBZK3B3TG/state.json","well_known_bundle":"https://pith.science/.well-known/pith/EB5FDIVGKJQI7J7NQQBZK3B3TG/bundle.json"},"state":{"state_type":"pith_open_graph_state","state_version":"1.0","pith_number":"pith:2025:EB5FDIVGKJQI7J7NQQBZK3B3TG","merge_version":"pith-open-graph-merge-v1","event_count":2,"valid_event_count":2,"invalid_event_count":0,"equivocation_count":0,"current":{"canonical_record":{"metadata":{"abstract_canon_sha256":"3ee61e12ee34464d180fc33505a0782ae195f479a6fc530fb18eedb43150d954","cross_cats_sorted":[],"license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.CL","submitted_at":"2025-06-03T17:35:56Z","title_canon_sha256":"1104674e1740cca8febc0581499605eaf2ef1f8ebc81f617911fc5e67544058e"},"schema_version":"1.0","source":{"id":"2506.03101","kind":"arxiv","version":1}},"source_aliases":[{"alias_kind":"arxiv","alias_value":"2506.03101","created_at":"2026-07-05T11:15:14Z"},{"alias_kind":"arxiv_version","alias_value":"2506.03101v1","created_at":"2026-07-05T11:15:14Z"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.2506.03101","created_at":"2026-07-05T11:15:14Z"},{"alias_kind":"pith_short_12","alias_value":"EB5FDIVGKJQI","created_at":"2026-07-05T11:15:14Z"},{"alias_kind":"pith_short_16","alias_value":"EB5FDIVGKJQI7J7N","created_at":"2026-07-05T11:15:14Z"},{"alias_kind":"pith_short_8","alias_value":"EB5FDIVG","created_at":"2026-07-05T11:15:14Z"}],"graph_snapshots":[{"event_id":"sha256:d57db1124552fb38361b6fa3baf1063f491756bc3cdbfd33829c42b9d1d5c627","target":"graph","created_at":"2026-07-05T11:15:14Z","signer":{"key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signer_id":"pith.science","signer_type":"pith_registry"},"payload":{"graph_snapshot":{"author_claims":{"count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","strong_count":0},"builder_version":"pith-number-builder-2026-05-17-v1","claims":{"count":0,"items":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"integrity":{"available":true,"clean":true,"detectors_run":[],"endpoint":"/pith/2506.03101/integrity.json","findings":[],"snapshot_sha256":"c28c3603d3b5d939e8dc4c7e95fa8dfce3d595e45f758748cecf8e644a296938","summary":{"advisory":0,"by_detector":{},"critical":0,"informational":0}},"paper":{"abstract_excerpt":"The choice of tokenizer can profoundly impact language model performance, yet accessible and reliable evaluations of tokenizer quality remain an open challenge. Inspired by scaling consistency, we show that smaller models can accurately predict significant differences in tokenizer impact on larger models at a fraction of the compute cost. By systematically evaluating both English-centric and multilingual tokenizers, we find that tokenizer choice has negligible effects on tasks in English but results in consistent performance differences in multilingual settings. We propose new intrinsic tokeni","authors_text":"Ant\\'onio V. Lopes, Hendra Setiawan, Jonas F. Lotz, Leonardo Emili, Stephan Peitz","cross_cats":[],"headline":"","license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.CL","submitted_at":"2025-06-03T17:35:56Z","title":"Beyond Text Compression: Evaluating Tokenizers Across Scales"},"references":{"count":0,"internal_anchors":0,"resolved_work":0,"sample":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"2506.03101","kind":"arxiv","version":1},"verdict":{"created_at":null,"id":null,"model_set":{},"one_line_summary":"","pipeline_version":null,"pith_extraction_headline":"","strongest_claim":"","weakest_assumption":""}},"verdict_id":null}}],"author_attestations":[],"timestamp_anchors":[],"storage_attestations":[],"citation_signatures":[],"replication_records":[],"corrections":[],"mirror_hints":[],"record_created":{"event_id":"sha256:407564be54581fead6d4c17ae5af3591f66eed64f9abf3d0a896a771c6370f5c","target":"record","created_at":"2026-07-05T11:15:14Z","signer":{"key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signer_id":"pith.science","signer_type":"pith_registry"},"payload":{"attestation_state":"computed","canonical_record":{"metadata":{"abstract_canon_sha256":"3ee61e12ee34464d180fc33505a0782ae195f479a6fc530fb18eedb43150d954","cross_cats_sorted":[],"license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.CL","submitted_at":"2025-06-03T17:35:56Z","title_canon_sha256":"1104674e1740cca8febc0581499605eaf2ef1f8ebc81f617911fc5e67544058e"},"schema_version":"1.0","source":{"id":"2506.03101","kind":"arxiv","version":1}},"canonical_sha256":"207a51a2a652608fa7ed8403956c3b9993444ebb1e877c6cd98d86db0587606b","receipt":{"algorithm":"ed25519","builder_version":"pith-number-builder-2026-05-17-v1","canonical_sha256":"207a51a2a652608fa7ed8403956c3b9993444ebb1e877c6cd98d86db0587606b","first_computed_at":"2026-07-05T11:15:14.887654Z","key_id":"pith-v1-2026-05","kind":"pith_receipt","last_reissued_at":"2026-07-05T11:15:14.887654Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","receipt_version":"0.3","signature_b64":"M7eOzz6X9QpnYBo1N+hDBw4Lj8i1BWuC9qsqyqMOE3tKKm5IujWmUO8JqpLxDtXEA0iIXeSirJPAQaZAbKreDw==","signature_status":"signed_v1","signed_at":"2026-07-05T11:15:14.888163Z","signed_message":"canonical_sha256_bytes"},"source_id":"2506.03101","source_kind":"arxiv","source_version":1}}},"equivocations":[],"invalid_events":[],"applied_event_ids":["sha256:407564be54581fead6d4c17ae5af3591f66eed64f9abf3d0a896a771c6370f5c","sha256:d57db1124552fb38361b6fa3baf1063f491756bc3cdbfd33829c42b9d1d5c627"],"state_sha256":"047155b253bc8ee0fa02d65539ac9c0b53cad29c34873572a6614338ae89e753"},"bundle_signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"jxD/owdTrcBB2mrzx5BVwRUiLk3QVN8ENBnU1cQs36MK3/2i3tXIp79+4TMCSMZs+Win7mj89LP1FBlVCCr0Bw==","signed_message":"bundle_sha256_bytes","signed_at":"2026-08-08T11:51:38.694597Z","bundle_sha256":"aaa0157a4830dc82f07864293e8adeef5cd19f998eff1cbf0c190115494de0bc"}}