{"state_type":"pith_open_graph_state","state_version":"1.0","pith_number":"pith:2024:TYEJVBMA3P6GDKMK5QWAN63ASG","merge_version":"pith-open-graph-merge-v1","event_count":2,"valid_event_count":2,"invalid_event_count":0,"equivocation_count":0,"current":{"canonical_record":{"metadata":{"abstract_canon_sha256":"6c672f4ddfd034c4b55e7414ea2450c7dad8251acd02b968a15c159ab02938ed","cross_cats_sorted":["cs.AI"],"license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","primary_cat":"cs.CL","submitted_at":"2024-10-08T09:53:35Z","title_canon_sha256":"c2e65c97e9a860e5c4ddfac027f2206bd3519daa2e1ee75170bd7c0d53d83c81"},"schema_version":"1.0","source":{"id":"2410.05864","kind":"arxiv","version":4}},"source_aliases":[{"alias_kind":"arxiv","alias_value":"2410.05864","created_at":"2026-07-05T10:22:51Z"},{"alias_kind":"arxiv_version","alias_value":"2410.05864v4","created_at":"2026-07-05T10:22:51Z"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.2410.05864","created_at":"2026-07-05T10:22:51Z"},{"alias_kind":"pith_short_12","alias_value":"TYEJVBMA3P6G","created_at":"2026-07-05T10:22:51Z"},{"alias_kind":"pith_short_16","alias_value":"TYEJVBMA3P6GDKMK","created_at":"2026-07-05T10:22:51Z"},{"alias_kind":"pith_short_8","alias_value":"TYEJVBMA","created_at":"2026-07-05T10:22:51Z"}],"graph_snapshots":[{"event_id":"sha256:ae6349b88572124a570011b8cf7e5f4812128e3ae854904074ec07f84ac7bd2f","target":"graph","created_at":"2026-07-05T10:22:51Z","signer":{"key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signer_id":"pith.science","signer_type":"pith_registry"},"payload":{"graph_snapshot":{"author_claims":{"count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","strong_count":0},"builder_version":"pith-number-builder-2026-05-17-v1","claims":{"count":0,"items":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"integrity":{"available":true,"clean":true,"detectors_run":[],"endpoint":"/pith/2410.05864/integrity.json","findings":[],"snapshot_sha256":"c28c3603d3b5d939e8dc4c7e95fa8dfce3d595e45f758748cecf8e644a296938","summary":{"advisory":0,"by_detector":{},"critical":0,"informational":0}},"paper":{"abstract_excerpt":"Natural language is composed of words, but modern large language models (LLMs) process sub-words as input. A natural question raised by this discrepancy is whether LLMs encode words internally, and if so how. We present evidence that LLMs engage in an intrinsic detokenization process, where sub-word sequences are combined into coherent whole-word representations at their last token. Our experiments show that this process primarily takes place within the early and middle layers of the model. We further demonstrate its robustness to arbitrary splits (e.g., \"cats\" to \"ca\" and \"ts\"), typos, and im","authors_text":"Guy Kaplan, Matanel Oren, Roy Schwartz, Yuval Reif","cross_cats":["cs.AI"],"headline":"","license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","primary_cat":"cs.CL","submitted_at":"2024-10-08T09:53:35Z","title":"From Tokens to Words: On the Inner Lexicon of LLMs"},"references":{"count":0,"internal_anchors":0,"resolved_work":0,"sample":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"2410.05864","kind":"arxiv","version":4},"verdict":{"created_at":null,"id":null,"model_set":{},"one_line_summary":"","pipeline_version":null,"pith_extraction_headline":"","strongest_claim":"","weakest_assumption":""}},"verdict_id":null}}],"author_attestations":[],"timestamp_anchors":[],"storage_attestations":[],"citation_signatures":[],"replication_records":[],"corrections":[],"mirror_hints":[],"record_created":{"event_id":"sha256:86ea0d7cdf1fe31fa4f01d543e0bd0ff5c3aebfcc94e0714ca887f9c06d6affd","target":"record","created_at":"2026-07-05T10:22:51Z","signer":{"key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signer_id":"pith.science","signer_type":"pith_registry"},"payload":{"attestation_state":"computed","canonical_record":{"metadata":{"abstract_canon_sha256":"6c672f4ddfd034c4b55e7414ea2450c7dad8251acd02b968a15c159ab02938ed","cross_cats_sorted":["cs.AI"],"license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","primary_cat":"cs.CL","submitted_at":"2024-10-08T09:53:35Z","title_canon_sha256":"c2e65c97e9a860e5c4ddfac027f2206bd3519daa2e1ee75170bd7c0d53d83c81"},"schema_version":"1.0","source":{"id":"2410.05864","kind":"arxiv","version":4}},"canonical_sha256":"9e089a8580dbfc61a98aec2c06fb60919339e30d40614a013d6694e54086d497","receipt":{"algorithm":"ed25519","builder_version":"pith-number-builder-2026-05-17-v1","canonical_sha256":"9e089a8580dbfc61a98aec2c06fb60919339e30d40614a013d6694e54086d497","first_computed_at":"2026-07-05T10:22:51.818360Z","key_id":"pith-v1-2026-05","kind":"pith_receipt","last_reissued_at":"2026-07-05T10:22:51.818360Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","receipt_version":"0.3","signature_b64":"houxa15lTjiu49XvxQo1EUfHfo9iZ2ePCEXw8SmxTMUMX6yTMdHTLMPFbvICDpoANr67mxsrmfmXlw3+JnkiBg==","signature_status":"signed_v1","signed_at":"2026-07-05T10:22:51.819007Z","signed_message":"canonical_sha256_bytes"},"source_id":"2410.05864","source_kind":"arxiv","source_version":4}}},"equivocations":[],"invalid_events":[],"applied_event_ids":["sha256:86ea0d7cdf1fe31fa4f01d543e0bd0ff5c3aebfcc94e0714ca887f9c06d6affd","sha256:ae6349b88572124a570011b8cf7e5f4812128e3ae854904074ec07f84ac7bd2f"],"state_sha256":"60a7d532e398793c2fc81d8c5f826891f35bbcac1aa0f33ca90753836719ca4e"}