{"bundle_type":"pith_open_graph_bundle","bundle_version":"1.0","pith_number":"pith:2024:3OAAKWSNXFGQXWSBZZ4QKJK5YY","short_pith_number":"pith:3OAAKWSN","canonical_record":{"source":{"id":"2403.06265","kind":"arxiv","version":2},"metadata":{"license":"http://creativecommons.org/licenses/by-sa/4.0/","primary_cat":"cs.CL","submitted_at":"2024-03-10T17:02:53Z","cross_cats_sorted":["cs.AI","cs.LG"],"title_canon_sha256":"532bb59baf42edd4a8b1e38c7d2046451d859b2309f50df501a4c779c05eadfd","abstract_canon_sha256":"2fbeb2442b8e6d9d725db4bbb2c72c276aa8a5a5b48b4e71f54c9ca014def395"},"schema_version":"1.0"},"canonical_sha256":"db80055a4db94d0bda41ce7905255dc610766feb29af0530b0a9357f1a0d9c0a","source":{"kind":"arxiv","id":"2403.06265","version":2},"source_aliases":[{"alias_kind":"arxiv","alias_value":"2403.06265","created_at":"2026-07-05T08:35:43Z"},{"alias_kind":"arxiv_version","alias_value":"2403.06265v2","created_at":"2026-07-05T08:35:43Z"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.2403.06265","created_at":"2026-07-05T08:35:43Z"},{"alias_kind":"pith_short_12","alias_value":"3OAAKWSNXFGQ","created_at":"2026-07-05T08:35:43Z"},{"alias_kind":"pith_short_16","alias_value":"3OAAKWSNXFGQXWSB","created_at":"2026-07-05T08:35:43Z"},{"alias_kind":"pith_short_8","alias_value":"3OAAKWSN","created_at":"2026-07-05T08:35:43Z"}],"events":[{"event_type":"record_created","subject_pith_number":"pith:2024:3OAAKWSNXFGQXWSBZZ4QKJK5YY","target":"record","payload":{"canonical_record":{"source":{"id":"2403.06265","kind":"arxiv","version":2},"metadata":{"license":"http://creativecommons.org/licenses/by-sa/4.0/","primary_cat":"cs.CL","submitted_at":"2024-03-10T17:02:53Z","cross_cats_sorted":["cs.AI","cs.LG"],"title_canon_sha256":"532bb59baf42edd4a8b1e38c7d2046451d859b2309f50df501a4c779c05eadfd","abstract_canon_sha256":"2fbeb2442b8e6d9d725db4bbb2c72c276aa8a5a5b48b4e71f54c9ca014def395"},"schema_version":"1.0"},"canonical_sha256":"db80055a4db94d0bda41ce7905255dc610766feb29af0530b0a9357f1a0d9c0a","receipt":{"kind":"pith_receipt","key_id":"pith-v1-2026-05","algorithm":"ed25519","signed_at":"2026-07-05T08:35:43.942135Z","signature_b64":"aRNnLAqNlm4+I/YuR058rMbAJc6MHZS5e0JBxbNFz/zHo9n/muyorYu+TcrBtvirHviQPK843mECZPTslTXlAA==","signed_message":"canonical_sha256_bytes","builder_version":"pith-number-builder-2026-05-17-v1","receipt_version":"0.3","canonical_sha256":"db80055a4db94d0bda41ce7905255dc610766feb29af0530b0a9357f1a0d9c0a","last_reissued_at":"2026-07-05T08:35:43.941639Z","signature_status":"signed_v1","first_computed_at":"2026-07-05T08:35:43.941639Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"source_kind":"arxiv","source_id":"2403.06265","source_version":2,"attestation_state":"computed"},"signer":{"signer_id":"pith.science","signer_type":"pith_registry","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"created_at":"2026-07-05T08:35:43Z","supersedes":[],"prev_event":null,"signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"Q59xoK7OWbbWiFEojBKpHO3zB2RqtAP1reX9bQGnDeImeT2XCBPVF7Av190v8E4LqRWw8oYoAVaJeFjnlzpLDA==","signed_message":"open_graph_event_sha256_bytes","signed_at":"2026-08-14T13:56:52.141326Z"},"content_sha256":"77cc2043803755fe5b0e0f1e0bc86fea0eb5ddc746fbcb774b8344f484ae9ec4","schema_version":"1.0","event_id":"sha256:77cc2043803755fe5b0e0f1e0bc86fea0eb5ddc746fbcb774b8344f484ae9ec4"},{"event_type":"graph_snapshot","subject_pith_number":"pith:2024:3OAAKWSNXFGQXWSBZZ4QKJK5YY","target":"graph","payload":{"graph_snapshot":{"paper":{"title":"Unpacking Tokenization: Evaluating Text Compression and its Correlation with Model Performance","license":"http://creativecommons.org/licenses/by-sa/4.0/","headline":"","cross_cats":["cs.AI","cs.LG"],"primary_cat":"cs.CL","authors_text":"Avi Caciularu, Idan Szpektor, Kris Cao, Matan Eyal, Omer Goldman, Reut Tsarfaty","submitted_at":"2024-03-10T17:02:53Z","abstract_excerpt":"Despite it being the cornerstone of BPE, the most common tokenization algorithm, the importance of compression in the tokenization process is still unclear. In this paper, we argue for the theoretical importance of compression, that can be viewed as 0-gram language modeling where equal probability is assigned to all tokens. We also demonstrate the empirical importance of compression for downstream success of pre-trained language models. We control the compression ability of several BPE tokenizers by varying the amount of documents available during their training: from 1 million documents to a "},"claims":{"count":0,"items":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"2403.06265","kind":"arxiv","version":2},"verdict":{"id":null,"model_set":{},"created_at":null,"strongest_claim":"","one_line_summary":"","pipeline_version":null,"weakest_assumption":"","pith_extraction_headline":""},"integrity":{"clean":true,"summary":{"advisory":0,"critical":0,"by_detector":{},"informational":0},"endpoint":"/pith/2403.06265/integrity.json","findings":[],"available":true,"detectors_run":[],"snapshot_sha256":"c28c3603d3b5d939e8dc4c7e95fa8dfce3d595e45f758748cecf8e644a296938"},"references":{"count":0,"sample":[],"resolved_work":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","internal_anchors":0},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"author_claims":{"count":0,"strong_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"builder_version":"pith-number-builder-2026-05-17-v1"},"verdict_id":null},"signer":{"signer_id":"pith.science","signer_type":"pith_registry","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"created_at":"2026-07-05T08:35:43Z","supersedes":[],"prev_event":null,"signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"0PP0rApva2BMgCQOp6lFSAfH7VxQvlRdNRsBFod3wa8ItPcw0AteIGJyi8jJIUm1+QDEXJj4AkWFLfGPDh/HCA==","signed_message":"open_graph_event_sha256_bytes","signed_at":"2026-08-14T13:56:52.142588Z"},"content_sha256":"9ce9622afe8e1df33a97d900ee5617e73a52de2cadf6375f0cd7ede365573eee","schema_version":"1.0","event_id":"sha256:9ce9622afe8e1df33a97d900ee5617e73a52de2cadf6375f0cd7ede365573eee"}],"timestamp_proofs":[],"mirror_hints":[{"mirror_type":"https","name":"Pith Resolver","base_url":"https://pith.science","bundle_url":"https://pith.science/pith/3OAAKWSNXFGQXWSBZZ4QKJK5YY/bundle.json","state_url":"https://pith.science/pith/3OAAKWSNXFGQXWSBZZ4QKJK5YY/state.json","well_known_bundle_url":"https://pith.science/.well-known/pith/3OAAKWSNXFGQXWSBZZ4QKJK5YY/bundle.json","status":"primary"}],"public_keys":[{"key_id":"pith-v1-2026-05","algorithm":"ed25519","format":"raw","public_key_b64":"stVStoiQhXFxp4s2pdzPNoqVNBMojDU/fJ2db5S3CbM=","public_key_hex":"b2d552b68890857171a78b36a5dccf368a953413288c353f7c9d9d6f94b709b3","fingerprint_sha256_b32_first128bits":"RVFV5Z2OI2J3ZUO7ERDEBCYNKS","fingerprint_sha256_hex":"8d4b5ee74e4693bcd1df2446408b0d54","rotates_at":null,"url":"https://pith.science/pith-signing-key.json","notes":"Pith uses this Ed25519 key to sign canonical record SHA-256 digests. Verify with: ed25519_verify(public_key, message=canonical_sha256_bytes, signature=base64decode(signature_b64))."}],"merge_version":"pith-open-graph-merge-v1","built_at":"2026-08-14T13:56:52Z","links":{"resolver":"https://pith.science/pith/3OAAKWSNXFGQXWSBZZ4QKJK5YY","bundle":"https://pith.science/pith/3OAAKWSNXFGQXWSBZZ4QKJK5YY/bundle.json","state":"https://pith.science/pith/3OAAKWSNXFGQXWSBZZ4QKJK5YY/state.json","well_known_bundle":"https://pith.science/.well-known/pith/3OAAKWSNXFGQXWSBZZ4QKJK5YY/bundle.json"},"state":{"state_type":"pith_open_graph_state","state_version":"1.0","pith_number":"pith:2024:3OAAKWSNXFGQXWSBZZ4QKJK5YY","merge_version":"pith-open-graph-merge-v1","event_count":2,"valid_event_count":2,"invalid_event_count":0,"equivocation_count":0,"current":{"canonical_record":{"metadata":{"abstract_canon_sha256":"2fbeb2442b8e6d9d725db4bbb2c72c276aa8a5a5b48b4e71f54c9ca014def395","cross_cats_sorted":["cs.AI","cs.LG"],"license":"http://creativecommons.org/licenses/by-sa/4.0/","primary_cat":"cs.CL","submitted_at":"2024-03-10T17:02:53Z","title_canon_sha256":"532bb59baf42edd4a8b1e38c7d2046451d859b2309f50df501a4c779c05eadfd"},"schema_version":"1.0","source":{"id":"2403.06265","kind":"arxiv","version":2}},"source_aliases":[{"alias_kind":"arxiv","alias_value":"2403.06265","created_at":"2026-07-05T08:35:43Z"},{"alias_kind":"arxiv_version","alias_value":"2403.06265v2","created_at":"2026-07-05T08:35:43Z"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.2403.06265","created_at":"2026-07-05T08:35:43Z"},{"alias_kind":"pith_short_12","alias_value":"3OAAKWSNXFGQ","created_at":"2026-07-05T08:35:43Z"},{"alias_kind":"pith_short_16","alias_value":"3OAAKWSNXFGQXWSB","created_at":"2026-07-05T08:35:43Z"},{"alias_kind":"pith_short_8","alias_value":"3OAAKWSN","created_at":"2026-07-05T08:35:43Z"}],"graph_snapshots":[{"event_id":"sha256:9ce9622afe8e1df33a97d900ee5617e73a52de2cadf6375f0cd7ede365573eee","target":"graph","created_at":"2026-07-05T08:35:43Z","signer":{"key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signer_id":"pith.science","signer_type":"pith_registry"},"payload":{"graph_snapshot":{"author_claims":{"count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","strong_count":0},"builder_version":"pith-number-builder-2026-05-17-v1","claims":{"count":0,"items":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"integrity":{"available":true,"clean":true,"detectors_run":[],"endpoint":"/pith/2403.06265/integrity.json","findings":[],"snapshot_sha256":"c28c3603d3b5d939e8dc4c7e95fa8dfce3d595e45f758748cecf8e644a296938","summary":{"advisory":0,"by_detector":{},"critical":0,"informational":0}},"paper":{"abstract_excerpt":"Despite it being the cornerstone of BPE, the most common tokenization algorithm, the importance of compression in the tokenization process is still unclear. In this paper, we argue for the theoretical importance of compression, that can be viewed as 0-gram language modeling where equal probability is assigned to all tokens. We also demonstrate the empirical importance of compression for downstream success of pre-trained language models. We control the compression ability of several BPE tokenizers by varying the amount of documents available during their training: from 1 million documents to a ","authors_text":"Avi Caciularu, Idan Szpektor, Kris Cao, Matan Eyal, Omer Goldman, Reut Tsarfaty","cross_cats":["cs.AI","cs.LG"],"headline":"","license":"http://creativecommons.org/licenses/by-sa/4.0/","primary_cat":"cs.CL","submitted_at":"2024-03-10T17:02:53Z","title":"Unpacking Tokenization: Evaluating Text Compression and its Correlation with Model Performance"},"references":{"count":0,"internal_anchors":0,"resolved_work":0,"sample":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"2403.06265","kind":"arxiv","version":2},"verdict":{"created_at":null,"id":null,"model_set":{},"one_line_summary":"","pipeline_version":null,"pith_extraction_headline":"","strongest_claim":"","weakest_assumption":""}},"verdict_id":null}}],"author_attestations":[],"timestamp_anchors":[],"storage_attestations":[],"citation_signatures":[],"replication_records":[],"corrections":[],"mirror_hints":[],"record_created":{"event_id":"sha256:77cc2043803755fe5b0e0f1e0bc86fea0eb5ddc746fbcb774b8344f484ae9ec4","target":"record","created_at":"2026-07-05T08:35:43Z","signer":{"key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signer_id":"pith.science","signer_type":"pith_registry"},"payload":{"attestation_state":"computed","canonical_record":{"metadata":{"abstract_canon_sha256":"2fbeb2442b8e6d9d725db4bbb2c72c276aa8a5a5b48b4e71f54c9ca014def395","cross_cats_sorted":["cs.AI","cs.LG"],"license":"http://creativecommons.org/licenses/by-sa/4.0/","primary_cat":"cs.CL","submitted_at":"2024-03-10T17:02:53Z","title_canon_sha256":"532bb59baf42edd4a8b1e38c7d2046451d859b2309f50df501a4c779c05eadfd"},"schema_version":"1.0","source":{"id":"2403.06265","kind":"arxiv","version":2}},"canonical_sha256":"db80055a4db94d0bda41ce7905255dc610766feb29af0530b0a9357f1a0d9c0a","receipt":{"algorithm":"ed25519","builder_version":"pith-number-builder-2026-05-17-v1","canonical_sha256":"db80055a4db94d0bda41ce7905255dc610766feb29af0530b0a9357f1a0d9c0a","first_computed_at":"2026-07-05T08:35:43.941639Z","key_id":"pith-v1-2026-05","kind":"pith_receipt","last_reissued_at":"2026-07-05T08:35:43.941639Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","receipt_version":"0.3","signature_b64":"aRNnLAqNlm4+I/YuR058rMbAJc6MHZS5e0JBxbNFz/zHo9n/muyorYu+TcrBtvirHviQPK843mECZPTslTXlAA==","signature_status":"signed_v1","signed_at":"2026-07-05T08:35:43.942135Z","signed_message":"canonical_sha256_bytes"},"source_id":"2403.06265","source_kind":"arxiv","source_version":2}}},"equivocations":[],"invalid_events":[],"applied_event_ids":["sha256:77cc2043803755fe5b0e0f1e0bc86fea0eb5ddc746fbcb774b8344f484ae9ec4","sha256:9ce9622afe8e1df33a97d900ee5617e73a52de2cadf6375f0cd7ede365573eee"],"state_sha256":"e9b5d07bdaac465ae105c221d6159a21874b91955cff4803475f743b9e886fef"},"bundle_signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"oINc8g3uPHGNzOwKY2+8peJrOMT5iWWbKD0JtAv8yGBL5HGTzQM/eiS9MzE6o296nUsZ2CffAATsLvmcKTEyBQ==","signed_message":"bundle_sha256_bytes","signed_at":"2026-08-14T13:56:52.148231Z","bundle_sha256":"f9ba244d2b6420ab3500b4b2e796c3944b7212a8a45a353f71c7a4559c462be3"}}