{"bundle_type":"pith_open_graph_bundle","bundle_version":"1.0","pith_number":"pith:2025:A4S4ANG2VKIXQLKHARGAC4FCDG","short_pith_number":"pith:A4S4ANG2","canonical_record":{"source":{"id":"2505.09738","kind":"arxiv","version":1},"metadata":{"license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","primary_cat":"cs.CL","submitted_at":"2025-05-14T19:00:27Z","cross_cats_sorted":["cs.AI"],"title_canon_sha256":"7697e623705c0a9b9c4d3dab159c2865cf8a197718d23ded0e7ffc4aa8c1c3a3","abstract_canon_sha256":"b71c932e768b740c7a5f4230f12eeb8ca29ef946fb2d3ad6f4f74d24a17088cf"},"schema_version":"1.0"},"canonical_sha256":"0725c034daaa91782d47044c0170a2198f3ee437a2dc95d07c7c36f6e4d792a4","source":{"kind":"arxiv","id":"2505.09738","version":1},"source_aliases":[{"alias_kind":"arxiv","alias_value":"2505.09738","created_at":"2026-07-05T11:03:27Z"},{"alias_kind":"arxiv_version","alias_value":"2505.09738v1","created_at":"2026-07-05T11:03:27Z"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.2505.09738","created_at":"2026-07-05T11:03:27Z"},{"alias_kind":"pith_short_12","alias_value":"A4S4ANG2VKIX","created_at":"2026-07-05T11:03:27Z"},{"alias_kind":"pith_short_16","alias_value":"A4S4ANG2VKIXQLKH","created_at":"2026-07-05T11:03:27Z"},{"alias_kind":"pith_short_8","alias_value":"A4S4ANG2","created_at":"2026-07-05T11:03:27Z"}],"events":[{"event_type":"record_created","subject_pith_number":"pith:2025:A4S4ANG2VKIXQLKHARGAC4FCDG","target":"record","payload":{"canonical_record":{"source":{"id":"2505.09738","kind":"arxiv","version":1},"metadata":{"license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","primary_cat":"cs.CL","submitted_at":"2025-05-14T19:00:27Z","cross_cats_sorted":["cs.AI"],"title_canon_sha256":"7697e623705c0a9b9c4d3dab159c2865cf8a197718d23ded0e7ffc4aa8c1c3a3","abstract_canon_sha256":"b71c932e768b740c7a5f4230f12eeb8ca29ef946fb2d3ad6f4f74d24a17088cf"},"schema_version":"1.0"},"canonical_sha256":"0725c034daaa91782d47044c0170a2198f3ee437a2dc95d07c7c36f6e4d792a4","receipt":{"kind":"pith_receipt","key_id":"pith-v1-2026-05","algorithm":"ed25519","signed_at":"2026-07-05T11:03:27.640309Z","signature_b64":"AAx3jOYe4zcmHIqxU4hA5NcIIPB/3hhW917iNXbASQiXQx1KQdRKISkgC97bZuoQeWPv4dhlwg+WO82teZVoCw==","signed_message":"canonical_sha256_bytes","builder_version":"pith-number-builder-2026-05-17-v1","receipt_version":"0.3","canonical_sha256":"0725c034daaa91782d47044c0170a2198f3ee437a2dc95d07c7c36f6e4d792a4","last_reissued_at":"2026-07-05T11:03:27.639870Z","signature_status":"signed_v1","first_computed_at":"2026-07-05T11:03:27.639870Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"source_kind":"arxiv","source_id":"2505.09738","source_version":1,"attestation_state":"computed"},"signer":{"signer_id":"pith.science","signer_type":"pith_registry","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"created_at":"2026-07-05T11:03:27Z","supersedes":[],"prev_event":null,"signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"H2/PeMn/0T7ropbTDZkOyVEy9OAM4Pat7CFDuBGwZp0xYvLjwjFNu1SXhjaEG8hUTYYNtf/bHsNXhwA8nkHcCw==","signed_message":"open_graph_event_sha256_bytes","signed_at":"2026-08-03T22:03:26.332421Z"},"content_sha256":"e61bd9749943d9bc496ae0192460af3f2fcd7946bad2382d17ee23befc860f1c","schema_version":"1.0","event_id":"sha256:e61bd9749943d9bc496ae0192460af3f2fcd7946bad2382d17ee23befc860f1c"},{"event_type":"graph_snapshot","subject_pith_number":"pith:2025:A4S4ANG2VKIXQLKHARGAC4FCDG","target":"graph","payload":{"graph_snapshot":{"paper":{"title":"Achieving Tokenizer Flexibility in Language Models through Heuristic Adaptation and Supertoken Learning","license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","headline":"","cross_cats":["cs.AI"],"primary_cat":"cs.CL","authors_text":"Adarsh Shirawalmath, Adithya Kamath, Shaurya Sharthak, Vinayak Pahalwan","submitted_at":"2025-05-14T19:00:27Z","abstract_excerpt":"Pretrained language models (LLMs) are often constrained by their fixed tokenization schemes, leading to inefficiencies and performance limitations, particularly for multilingual or specialized applications. This tokenizer lock-in presents significant challenges. standard methods to overcome this often require prohibitive computational resources. Although tokenizer replacement with heuristic initialization aims to reduce this burden, existing methods often require exhaustive residual fine-tuning and still may not fully preserve semantic nuances or adequately address the underlying compression i"},"claims":{"count":0,"items":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"2505.09738","kind":"arxiv","version":1},"verdict":{"id":null,"model_set":{},"created_at":null,"strongest_claim":"","one_line_summary":"","pipeline_version":null,"weakest_assumption":"","pith_extraction_headline":""},"integrity":{"clean":true,"summary":{"advisory":0,"critical":0,"by_detector":{},"informational":0},"endpoint":"/pith/2505.09738/integrity.json","findings":[],"available":true,"detectors_run":[],"snapshot_sha256":"c28c3603d3b5d939e8dc4c7e95fa8dfce3d595e45f758748cecf8e644a296938"},"references":{"count":0,"sample":[],"resolved_work":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","internal_anchors":0},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"author_claims":{"count":0,"strong_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"builder_version":"pith-number-builder-2026-05-17-v1"},"verdict_id":null},"signer":{"signer_id":"pith.science","signer_type":"pith_registry","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"created_at":"2026-07-05T11:03:27Z","supersedes":[],"prev_event":null,"signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"zgxg03FPKm+1LTQWm7+k0IdoZY2BC6Ik/A2Zwj0uNKM1xbOg4O+OSwuYMJaDfQ2g3K3oaqe3rBiKE7SSyRe/AA==","signed_message":"open_graph_event_sha256_bytes","signed_at":"2026-08-03T22:03:26.332806Z"},"content_sha256":"05323673140b287d2e2bfa5646bab29bb2e105af0077e201ead3b42f537e46b9","schema_version":"1.0","event_id":"sha256:05323673140b287d2e2bfa5646bab29bb2e105af0077e201ead3b42f537e46b9"}],"timestamp_proofs":[],"mirror_hints":[{"mirror_type":"https","name":"Pith Resolver","base_url":"https://pith.science","bundle_url":"https://pith.science/pith/A4S4ANG2VKIXQLKHARGAC4FCDG/bundle.json","state_url":"https://pith.science/pith/A4S4ANG2VKIXQLKHARGAC4FCDG/state.json","well_known_bundle_url":"https://pith.science/.well-known/pith/A4S4ANG2VKIXQLKHARGAC4FCDG/bundle.json","status":"primary"}],"public_keys":[{"key_id":"pith-v1-2026-05","algorithm":"ed25519","format":"raw","public_key_b64":"stVStoiQhXFxp4s2pdzPNoqVNBMojDU/fJ2db5S3CbM=","public_key_hex":"b2d552b68890857171a78b36a5dccf368a953413288c353f7c9d9d6f94b709b3","fingerprint_sha256_b32_first128bits":"RVFV5Z2OI2J3ZUO7ERDEBCYNKS","fingerprint_sha256_hex":"8d4b5ee74e4693bcd1df2446408b0d54","rotates_at":null,"url":"https://pith.science/pith-signing-key.json","notes":"Pith uses this Ed25519 key to sign canonical record SHA-256 digests. Verify with: ed25519_verify(public_key, message=canonical_sha256_bytes, signature=base64decode(signature_b64))."}],"merge_version":"pith-open-graph-merge-v1","built_at":"2026-08-03T22:03:26Z","links":{"resolver":"https://pith.science/pith/A4S4ANG2VKIXQLKHARGAC4FCDG","bundle":"https://pith.science/pith/A4S4ANG2VKIXQLKHARGAC4FCDG/bundle.json","state":"https://pith.science/pith/A4S4ANG2VKIXQLKHARGAC4FCDG/state.json","well_known_bundle":"https://pith.science/.well-known/pith/A4S4ANG2VKIXQLKHARGAC4FCDG/bundle.json"},"state":{"state_type":"pith_open_graph_state","state_version":"1.0","pith_number":"pith:2025:A4S4ANG2VKIXQLKHARGAC4FCDG","merge_version":"pith-open-graph-merge-v1","event_count":2,"valid_event_count":2,"invalid_event_count":0,"equivocation_count":0,"current":{"canonical_record":{"metadata":{"abstract_canon_sha256":"b71c932e768b740c7a5f4230f12eeb8ca29ef946fb2d3ad6f4f74d24a17088cf","cross_cats_sorted":["cs.AI"],"license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","primary_cat":"cs.CL","submitted_at":"2025-05-14T19:00:27Z","title_canon_sha256":"7697e623705c0a9b9c4d3dab159c2865cf8a197718d23ded0e7ffc4aa8c1c3a3"},"schema_version":"1.0","source":{"id":"2505.09738","kind":"arxiv","version":1}},"source_aliases":[{"alias_kind":"arxiv","alias_value":"2505.09738","created_at":"2026-07-05T11:03:27Z"},{"alias_kind":"arxiv_version","alias_value":"2505.09738v1","created_at":"2026-07-05T11:03:27Z"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.2505.09738","created_at":"2026-07-05T11:03:27Z"},{"alias_kind":"pith_short_12","alias_value":"A4S4ANG2VKIX","created_at":"2026-07-05T11:03:27Z"},{"alias_kind":"pith_short_16","alias_value":"A4S4ANG2VKIXQLKH","created_at":"2026-07-05T11:03:27Z"},{"alias_kind":"pith_short_8","alias_value":"A4S4ANG2","created_at":"2026-07-05T11:03:27Z"}],"graph_snapshots":[{"event_id":"sha256:05323673140b287d2e2bfa5646bab29bb2e105af0077e201ead3b42f537e46b9","target":"graph","created_at":"2026-07-05T11:03:27Z","signer":{"key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signer_id":"pith.science","signer_type":"pith_registry"},"payload":{"graph_snapshot":{"author_claims":{"count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","strong_count":0},"builder_version":"pith-number-builder-2026-05-17-v1","claims":{"count":0,"items":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"integrity":{"available":true,"clean":true,"detectors_run":[],"endpoint":"/pith/2505.09738/integrity.json","findings":[],"snapshot_sha256":"c28c3603d3b5d939e8dc4c7e95fa8dfce3d595e45f758748cecf8e644a296938","summary":{"advisory":0,"by_detector":{},"critical":0,"informational":0}},"paper":{"abstract_excerpt":"Pretrained language models (LLMs) are often constrained by their fixed tokenization schemes, leading to inefficiencies and performance limitations, particularly for multilingual or specialized applications. This tokenizer lock-in presents significant challenges. standard methods to overcome this often require prohibitive computational resources. Although tokenizer replacement with heuristic initialization aims to reduce this burden, existing methods often require exhaustive residual fine-tuning and still may not fully preserve semantic nuances or adequately address the underlying compression i","authors_text":"Adarsh Shirawalmath, Adithya Kamath, Shaurya Sharthak, Vinayak Pahalwan","cross_cats":["cs.AI"],"headline":"","license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","primary_cat":"cs.CL","submitted_at":"2025-05-14T19:00:27Z","title":"Achieving Tokenizer Flexibility in Language Models through Heuristic Adaptation and Supertoken Learning"},"references":{"count":0,"internal_anchors":0,"resolved_work":0,"sample":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"2505.09738","kind":"arxiv","version":1},"verdict":{"created_at":null,"id":null,"model_set":{},"one_line_summary":"","pipeline_version":null,"pith_extraction_headline":"","strongest_claim":"","weakest_assumption":""}},"verdict_id":null}}],"author_attestations":[],"timestamp_anchors":[],"storage_attestations":[],"citation_signatures":[],"replication_records":[],"corrections":[],"mirror_hints":[],"record_created":{"event_id":"sha256:e61bd9749943d9bc496ae0192460af3f2fcd7946bad2382d17ee23befc860f1c","target":"record","created_at":"2026-07-05T11:03:27Z","signer":{"key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signer_id":"pith.science","signer_type":"pith_registry"},"payload":{"attestation_state":"computed","canonical_record":{"metadata":{"abstract_canon_sha256":"b71c932e768b740c7a5f4230f12eeb8ca29ef946fb2d3ad6f4f74d24a17088cf","cross_cats_sorted":["cs.AI"],"license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","primary_cat":"cs.CL","submitted_at":"2025-05-14T19:00:27Z","title_canon_sha256":"7697e623705c0a9b9c4d3dab159c2865cf8a197718d23ded0e7ffc4aa8c1c3a3"},"schema_version":"1.0","source":{"id":"2505.09738","kind":"arxiv","version":1}},"canonical_sha256":"0725c034daaa91782d47044c0170a2198f3ee437a2dc95d07c7c36f6e4d792a4","receipt":{"algorithm":"ed25519","builder_version":"pith-number-builder-2026-05-17-v1","canonical_sha256":"0725c034daaa91782d47044c0170a2198f3ee437a2dc95d07c7c36f6e4d792a4","first_computed_at":"2026-07-05T11:03:27.639870Z","key_id":"pith-v1-2026-05","kind":"pith_receipt","last_reissued_at":"2026-07-05T11:03:27.639870Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","receipt_version":"0.3","signature_b64":"AAx3jOYe4zcmHIqxU4hA5NcIIPB/3hhW917iNXbASQiXQx1KQdRKISkgC97bZuoQeWPv4dhlwg+WO82teZVoCw==","signature_status":"signed_v1","signed_at":"2026-07-05T11:03:27.640309Z","signed_message":"canonical_sha256_bytes"},"source_id":"2505.09738","source_kind":"arxiv","source_version":1}}},"equivocations":[],"invalid_events":[],"applied_event_ids":["sha256:e61bd9749943d9bc496ae0192460af3f2fcd7946bad2382d17ee23befc860f1c","sha256:05323673140b287d2e2bfa5646bab29bb2e105af0077e201ead3b42f537e46b9"],"state_sha256":"fab64b16032df376c717404f8b5dbf080fd6324bd4ff43546a3ad9f6a35d2aa3"},"bundle_signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"wKz+kiXOfn5UMzm2tQiR6Y914Kj1UlPTUjbj35ye+9p9CrmQaQS1YKLQ1pp0YEaeJTE58MGM129shvYTW7xWAA==","signed_message":"bundle_sha256_bytes","signed_at":"2026-08-03T22:03:26.335478Z","bundle_sha256":"3e1923e4d49d38863e45b824dc4d122286c046c4487e508e757826b47e876f1a"}}