{"state_type":"pith_open_graph_state","state_version":"1.0","pith_number":"pith:2025:AOGI57O5HGFJEUKEPEUI54MUP6","merge_version":"pith-open-graph-merge-v1","event_count":2,"valid_event_count":2,"invalid_event_count":0,"equivocation_count":0,"current":{"canonical_record":{"metadata":{"abstract_canon_sha256":"89bb27ee89552de86329b53140cba4d2fcd84e77053b2d50c462a809e3014d5a","cross_cats_sorted":[],"license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","primary_cat":"cs.IR","submitted_at":"2025-02-18T02:29:51Z","title_canon_sha256":"4b6d9fa79dcf4ab2471b1b0ce03f2bae0ec669344d20dcc84726d5659d8f8b88"},"schema_version":"1.0","source":{"id":"2502.12448","kind":"arxiv","version":1}},"source_aliases":[{"alias_kind":"arxiv","alias_value":"2502.12448","created_at":"2026-07-05T10:15:55Z"},{"alias_kind":"arxiv_version","alias_value":"2502.12448v1","created_at":"2026-07-05T10:15:55Z"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.2502.12448","created_at":"2026-07-05T10:15:55Z"},{"alias_kind":"pith_short_12","alias_value":"AOGI57O5HGFJ","created_at":"2026-07-05T10:15:55Z"},{"alias_kind":"pith_short_16","alias_value":"AOGI57O5HGFJEUKE","created_at":"2026-07-05T10:15:55Z"},{"alias_kind":"pith_short_8","alias_value":"AOGI57O5","created_at":"2026-07-05T10:15:55Z"}],"graph_snapshots":[{"event_id":"sha256:f38120d2c3af0824dc072b9d01d6601e0a84eadb6ef06c952210f3914d9e5a39","target":"graph","created_at":"2026-07-05T10:15:55Z","signer":{"key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signer_id":"pith.science","signer_type":"pith_registry"},"payload":{"graph_snapshot":{"author_claims":{"count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","strong_count":0},"builder_version":"pith-number-builder-2026-05-17-v1","claims":{"count":0,"items":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"integrity":{"available":true,"clean":true,"detectors_run":[],"endpoint":"/pith/2502.12448/integrity.json","findings":[],"snapshot_sha256":"c28c3603d3b5d939e8dc4c7e95fa8dfce3d595e45f758748cecf8e644a296938","summary":{"advisory":0,"by_detector":{},"critical":0,"informational":0}},"paper":{"abstract_excerpt":"Discrete tokenizers have emerged as indispensable components in modern machine learning systems, particularly within the context of autoregressive modeling and large language models (LLMs). These tokenizers serve as the critical interface that transforms raw, unstructured data from diverse modalities into discrete tokens, enabling LLMs to operate effectively across a wide range of tasks. Despite their central role in generation, comprehension, and recommendation systems, a comprehensive survey dedicated to discrete tokenizers remains conspicuously absent in the literature. This paper addresses","authors_text":"Ben Xue, Jian Jia, Jingtong Gao, Junhao Wang, Kun Gai, Peng Jiang, Qingpeng Cai, Quan Chen, Xiangyu Zhao","cross_cats":[],"headline":"","license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","primary_cat":"cs.IR","submitted_at":"2025-02-18T02:29:51Z","title":"From Principles to Applications: A Comprehensive Survey of Discrete Tokenizers in Generation, Comprehension, Recommendation, and Information Retrieval"},"references":{"count":0,"internal_anchors":0,"resolved_work":0,"sample":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"2502.12448","kind":"arxiv","version":1},"verdict":{"created_at":null,"id":null,"model_set":{},"one_line_summary":"","pipeline_version":null,"pith_extraction_headline":"","strongest_claim":"","weakest_assumption":""}},"verdict_id":null}}],"author_attestations":[],"timestamp_anchors":[],"storage_attestations":[],"citation_signatures":[],"replication_records":[],"corrections":[],"mirror_hints":[],"record_created":{"event_id":"sha256:b4e10a6fa4b0aaf468891b515ccabd76675b2fdb7e71873c4b0d4768afdb8b7f","target":"record","created_at":"2026-07-05T10:15:55Z","signer":{"key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signer_id":"pith.science","signer_type":"pith_registry"},"payload":{"attestation_state":"computed","canonical_record":{"metadata":{"abstract_canon_sha256":"89bb27ee89552de86329b53140cba4d2fcd84e77053b2d50c462a809e3014d5a","cross_cats_sorted":[],"license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","primary_cat":"cs.IR","submitted_at":"2025-02-18T02:29:51Z","title_canon_sha256":"4b6d9fa79dcf4ab2471b1b0ce03f2bae0ec669344d20dcc84726d5659d8f8b88"},"schema_version":"1.0","source":{"id":"2502.12448","kind":"arxiv","version":1}},"canonical_sha256":"038c8efddd398a92514479288ef1947f90f46c9b52e5ca1ccc9ac3876345c316","receipt":{"algorithm":"ed25519","builder_version":"pith-number-builder-2026-05-17-v1","canonical_sha256":"038c8efddd398a92514479288ef1947f90f46c9b52e5ca1ccc9ac3876345c316","first_computed_at":"2026-07-05T10:15:55.886350Z","key_id":"pith-v1-2026-05","kind":"pith_receipt","last_reissued_at":"2026-07-05T10:15:55.886350Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","receipt_version":"0.3","signature_b64":"KyO69BJcBZbiRyFhEjP6GtJ48vYyUoTu51U4NvDHnSVMww2EG3yGIzCAPuRlWnXztsLNJPlTSEgXJ5qwKziyDQ==","signature_status":"signed_v1","signed_at":"2026-07-05T10:15:55.886797Z","signed_message":"canonical_sha256_bytes"},"source_id":"2502.12448","source_kind":"arxiv","source_version":1}}},"equivocations":[],"invalid_events":[],"applied_event_ids":["sha256:b4e10a6fa4b0aaf468891b515ccabd76675b2fdb7e71873c4b0d4768afdb8b7f","sha256:f38120d2c3af0824dc072b9d01d6601e0a84eadb6ef06c952210f3914d9e5a39"],"state_sha256":"5902487355f293958257f81442a41aa0168a0f0d0474df5f00015dd160f1a86c"}