{"bundle_type":"pith_open_graph_bundle","bundle_version":"1.0","pith_number":"pith:2024:KXKNNAO4UP64EDUTY3VELUC4YL","short_pith_number":"pith:KXKNNAO4","canonical_record":{"source":{"id":"2401.07793","kind":"arxiv","version":1},"metadata":{"license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.CL","submitted_at":"2024-01-15T16:00:50Z","cross_cats_sorted":[],"title_canon_sha256":"8aa40df1eb93221f490541fd17f438d33102d7df04e31f120381225914931337","abstract_canon_sha256":"d093d0bbdc5473d90f6098c363d6b68ca93cb5e35b41ac14a497c058c7b334ee"},"schema_version":"1.0"},"canonical_sha256":"55d4d681dca3fdc20e93c6ea45d05cc2f309f4a05ac12ed66ad2739362b1800c","source":{"kind":"arxiv","id":"2401.07793","version":1},"source_aliases":[{"alias_kind":"arxiv","alias_value":"2401.07793","created_at":"2026-07-05T07:34:03Z"},{"alias_kind":"arxiv_version","alias_value":"2401.07793v1","created_at":"2026-07-05T07:34:03Z"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.2401.07793","created_at":"2026-07-05T07:34:03Z"},{"alias_kind":"pith_short_12","alias_value":"KXKNNAO4UP64","created_at":"2026-07-05T07:34:03Z"},{"alias_kind":"pith_short_16","alias_value":"KXKNNAO4UP64EDUT","created_at":"2026-07-05T07:34:03Z"},{"alias_kind":"pith_short_8","alias_value":"KXKNNAO4","created_at":"2026-07-05T07:34:03Z"}],"events":[{"event_type":"record_created","subject_pith_number":"pith:2024:KXKNNAO4UP64EDUTY3VELUC4YL","target":"record","payload":{"canonical_record":{"source":{"id":"2401.07793","kind":"arxiv","version":1},"metadata":{"license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.CL","submitted_at":"2024-01-15T16:00:50Z","cross_cats_sorted":[],"title_canon_sha256":"8aa40df1eb93221f490541fd17f438d33102d7df04e31f120381225914931337","abstract_canon_sha256":"d093d0bbdc5473d90f6098c363d6b68ca93cb5e35b41ac14a497c058c7b334ee"},"schema_version":"1.0"},"canonical_sha256":"55d4d681dca3fdc20e93c6ea45d05cc2f309f4a05ac12ed66ad2739362b1800c","receipt":{"kind":"pith_receipt","key_id":"pith-v1-2026-05","algorithm":"ed25519","signed_at":"2026-07-05T07:34:03.022034Z","signature_b64":"xWpTDCZrz+MIb9cad7NF69q8Tg52s1IreqRtYDoGWWQZe6id9HJALMTsjpq4Kq1lVayeAcG/3mH6OPePxHXdCA==","signed_message":"canonical_sha256_bytes","builder_version":"pith-number-builder-2026-05-17-v1","receipt_version":"0.3","canonical_sha256":"55d4d681dca3fdc20e93c6ea45d05cc2f309f4a05ac12ed66ad2739362b1800c","last_reissued_at":"2026-07-05T07:34:03.021609Z","signature_status":"signed_v1","first_computed_at":"2026-07-05T07:34:03.021609Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"source_kind":"arxiv","source_id":"2401.07793","source_version":1,"attestation_state":"computed"},"signer":{"signer_id":"pith.science","signer_type":"pith_registry","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"created_at":"2026-07-05T07:34:03Z","supersedes":[],"prev_event":null,"signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"eNBlAQ9koEMy5cIv5PNpVMqPZcBC+6uTT41lsinwudLahCc1bQ+sTzqKm/gXnAL9lqexkVo4AHsdsOPAjgCTBQ==","signed_message":"open_graph_event_sha256_bytes","signed_at":"2026-08-21T15:44:58.884067Z"},"content_sha256":"0ba56c886d07a6106dd4fa0e55094158ac81202b29329218473805d96bc69ef0","schema_version":"1.0","event_id":"sha256:0ba56c886d07a6106dd4fa0e55094158ac81202b29329218473805d96bc69ef0"},{"event_type":"graph_snapshot","subject_pith_number":"pith:2024:KXKNNAO4UP64EDUTY3VELUC4YL","target":"graph","payload":{"graph_snapshot":{"paper":{"title":"Flexibly Scaling Large Language Models Contexts Through Extensible Tokenization","license":"http://creativecommons.org/licenses/by/4.0/","headline":"","cross_cats":[],"primary_cat":"cs.CL","authors_text":"Ninglu Shao, Peitian Zhang, Shitao Xiao, Zheng Liu","submitted_at":"2024-01-15T16:00:50Z","abstract_excerpt":"Large language models (LLMs) are in need of sufficient contexts to handle many critical applications, such as retrieval augmented generation and few-shot learning. However, due to the constrained window size, the LLMs can only access to the information within a limited context. Although the size of context window can be extended by fine-tuning, it will result in a substantial cost in both training and inference stage. In this paper, we present Extensible Tokenization as an alternative method which realizes the flexible scaling of LLMs' context. Extensible Tokenization stands as a midware in be"},"claims":{"count":0,"items":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"2401.07793","kind":"arxiv","version":1},"verdict":{"id":null,"model_set":{},"created_at":null,"strongest_claim":"","one_line_summary":"","pipeline_version":null,"weakest_assumption":"","pith_extraction_headline":""},"integrity":{"clean":true,"summary":{"advisory":0,"critical":0,"by_detector":{},"informational":0},"endpoint":"/pith/2401.07793/integrity.json","findings":[],"available":true,"detectors_run":[],"snapshot_sha256":"c28c3603d3b5d939e8dc4c7e95fa8dfce3d595e45f758748cecf8e644a296938"},"references":{"count":0,"sample":[],"resolved_work":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","internal_anchors":0},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"author_claims":{"count":0,"strong_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"builder_version":"pith-number-builder-2026-05-17-v1"},"verdict_id":null},"signer":{"signer_id":"pith.science","signer_type":"pith_registry","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"created_at":"2026-07-05T07:34:03Z","supersedes":[],"prev_event":null,"signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"FdVd+nF/sJLF5lhgT7W4LecC/6n25Zt2ttIz/W63GxN34cqLt4vPY+pzZuooEljYkTUshWVtISetH8t9ZAIFAg==","signed_message":"open_graph_event_sha256_bytes","signed_at":"2026-08-21T15:44:58.884572Z"},"content_sha256":"7bc83da1d580d8ef706cf258a9db5d674117841bc6c1245f6f7577112040427e","schema_version":"1.0","event_id":"sha256:7bc83da1d580d8ef706cf258a9db5d674117841bc6c1245f6f7577112040427e"}],"timestamp_proofs":[],"mirror_hints":[{"mirror_type":"https","name":"Pith Resolver","base_url":"https://pith.science","bundle_url":"https://pith.science/pith/KXKNNAO4UP64EDUTY3VELUC4YL/bundle.json","state_url":"https://pith.science/pith/KXKNNAO4UP64EDUTY3VELUC4YL/state.json","well_known_bundle_url":"https://pith.science/.well-known/pith/KXKNNAO4UP64EDUTY3VELUC4YL/bundle.json","status":"primary"}],"public_keys":[{"key_id":"pith-v1-2026-05","algorithm":"ed25519","format":"raw","public_key_b64":"stVStoiQhXFxp4s2pdzPNoqVNBMojDU/fJ2db5S3CbM=","public_key_hex":"b2d552b68890857171a78b36a5dccf368a953413288c353f7c9d9d6f94b709b3","fingerprint_sha256_b32_first128bits":"RVFV5Z2OI2J3ZUO7ERDEBCYNKS","fingerprint_sha256_hex":"8d4b5ee74e4693bcd1df2446408b0d54","rotates_at":null,"url":"https://pith.science/pith-signing-key.json","notes":"Pith uses this Ed25519 key to sign canonical record SHA-256 digests. Verify with: ed25519_verify(public_key, message=canonical_sha256_bytes, signature=base64decode(signature_b64))."}],"merge_version":"pith-open-graph-merge-v1","built_at":"2026-08-21T15:44:58Z","links":{"resolver":"https://pith.science/pith/KXKNNAO4UP64EDUTY3VELUC4YL","bundle":"https://pith.science/pith/KXKNNAO4UP64EDUTY3VELUC4YL/bundle.json","state":"https://pith.science/pith/KXKNNAO4UP64EDUTY3VELUC4YL/state.json","well_known_bundle":"https://pith.science/.well-known/pith/KXKNNAO4UP64EDUTY3VELUC4YL/bundle.json"},"state":{"state_type":"pith_open_graph_state","state_version":"1.0","pith_number":"pith:2024:KXKNNAO4UP64EDUTY3VELUC4YL","merge_version":"pith-open-graph-merge-v1","event_count":2,"valid_event_count":2,"invalid_event_count":0,"equivocation_count":0,"current":{"canonical_record":{"metadata":{"abstract_canon_sha256":"d093d0bbdc5473d90f6098c363d6b68ca93cb5e35b41ac14a497c058c7b334ee","cross_cats_sorted":[],"license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.CL","submitted_at":"2024-01-15T16:00:50Z","title_canon_sha256":"8aa40df1eb93221f490541fd17f438d33102d7df04e31f120381225914931337"},"schema_version":"1.0","source":{"id":"2401.07793","kind":"arxiv","version":1}},"source_aliases":[{"alias_kind":"arxiv","alias_value":"2401.07793","created_at":"2026-07-05T07:34:03Z"},{"alias_kind":"arxiv_version","alias_value":"2401.07793v1","created_at":"2026-07-05T07:34:03Z"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.2401.07793","created_at":"2026-07-05T07:34:03Z"},{"alias_kind":"pith_short_12","alias_value":"KXKNNAO4UP64","created_at":"2026-07-05T07:34:03Z"},{"alias_kind":"pith_short_16","alias_value":"KXKNNAO4UP64EDUT","created_at":"2026-07-05T07:34:03Z"},{"alias_kind":"pith_short_8","alias_value":"KXKNNAO4","created_at":"2026-07-05T07:34:03Z"}],"graph_snapshots":[{"event_id":"sha256:7bc83da1d580d8ef706cf258a9db5d674117841bc6c1245f6f7577112040427e","target":"graph","created_at":"2026-07-05T07:34:03Z","signer":{"key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signer_id":"pith.science","signer_type":"pith_registry"},"payload":{"graph_snapshot":{"author_claims":{"count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","strong_count":0},"builder_version":"pith-number-builder-2026-05-17-v1","claims":{"count":0,"items":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"integrity":{"available":true,"clean":true,"detectors_run":[],"endpoint":"/pith/2401.07793/integrity.json","findings":[],"snapshot_sha256":"c28c3603d3b5d939e8dc4c7e95fa8dfce3d595e45f758748cecf8e644a296938","summary":{"advisory":0,"by_detector":{},"critical":0,"informational":0}},"paper":{"abstract_excerpt":"Large language models (LLMs) are in need of sufficient contexts to handle many critical applications, such as retrieval augmented generation and few-shot learning. However, due to the constrained window size, the LLMs can only access to the information within a limited context. Although the size of context window can be extended by fine-tuning, it will result in a substantial cost in both training and inference stage. In this paper, we present Extensible Tokenization as an alternative method which realizes the flexible scaling of LLMs' context. Extensible Tokenization stands as a midware in be","authors_text":"Ninglu Shao, Peitian Zhang, Shitao Xiao, Zheng Liu","cross_cats":[],"headline":"","license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.CL","submitted_at":"2024-01-15T16:00:50Z","title":"Flexibly Scaling Large Language Models Contexts Through Extensible Tokenization"},"references":{"count":0,"internal_anchors":0,"resolved_work":0,"sample":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"2401.07793","kind":"arxiv","version":1},"verdict":{"created_at":null,"id":null,"model_set":{},"one_line_summary":"","pipeline_version":null,"pith_extraction_headline":"","strongest_claim":"","weakest_assumption":""}},"verdict_id":null}}],"author_attestations":[],"timestamp_anchors":[],"storage_attestations":[],"citation_signatures":[],"replication_records":[],"corrections":[],"mirror_hints":[],"record_created":{"event_id":"sha256:0ba56c886d07a6106dd4fa0e55094158ac81202b29329218473805d96bc69ef0","target":"record","created_at":"2026-07-05T07:34:03Z","signer":{"key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signer_id":"pith.science","signer_type":"pith_registry"},"payload":{"attestation_state":"computed","canonical_record":{"metadata":{"abstract_canon_sha256":"d093d0bbdc5473d90f6098c363d6b68ca93cb5e35b41ac14a497c058c7b334ee","cross_cats_sorted":[],"license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.CL","submitted_at":"2024-01-15T16:00:50Z","title_canon_sha256":"8aa40df1eb93221f490541fd17f438d33102d7df04e31f120381225914931337"},"schema_version":"1.0","source":{"id":"2401.07793","kind":"arxiv","version":1}},"canonical_sha256":"55d4d681dca3fdc20e93c6ea45d05cc2f309f4a05ac12ed66ad2739362b1800c","receipt":{"algorithm":"ed25519","builder_version":"pith-number-builder-2026-05-17-v1","canonical_sha256":"55d4d681dca3fdc20e93c6ea45d05cc2f309f4a05ac12ed66ad2739362b1800c","first_computed_at":"2026-07-05T07:34:03.021609Z","key_id":"pith-v1-2026-05","kind":"pith_receipt","last_reissued_at":"2026-07-05T07:34:03.021609Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","receipt_version":"0.3","signature_b64":"xWpTDCZrz+MIb9cad7NF69q8Tg52s1IreqRtYDoGWWQZe6id9HJALMTsjpq4Kq1lVayeAcG/3mH6OPePxHXdCA==","signature_status":"signed_v1","signed_at":"2026-07-05T07:34:03.022034Z","signed_message":"canonical_sha256_bytes"},"source_id":"2401.07793","source_kind":"arxiv","source_version":1}}},"equivocations":[],"invalid_events":[],"applied_event_ids":["sha256:0ba56c886d07a6106dd4fa0e55094158ac81202b29329218473805d96bc69ef0","sha256:7bc83da1d580d8ef706cf258a9db5d674117841bc6c1245f6f7577112040427e"],"state_sha256":"f96da2c0a249626f47907df17851bf8b02ddebaa76dfb1c0ad32a5525c0b9daa"},"bundle_signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"d3zT7vwwB9KhcENVkEUdn/ZDpu/yu43ABhHvhrS907xZIxxr/KnCid8QW3F3EJLAe2NT5cAIfUffDNmyUMlsCQ==","signed_message":"bundle_sha256_bytes","signed_at":"2026-08-21T15:44:58.889042Z","bundle_sha256":"a046c5b95a45a6cb1b8a46b793fe37dc0db622a49e639cedf885374ed6957f99"}}