{"record_type":"pith_number_record","schema_url":"https://pith.science/schemas/pith-number/v1.json","pith_number":"pith:2021:6AA4GCM2ZVMOKIIHF445YND5B6","short_pith_number":"pith:6AA4GCM2","schema_version":"1.0","canonical_sha256":"f001c3099acd58e521072f39dc347d0f8ac95ac04eb313f7749a6756bfbb557c","source":{"kind":"arxiv","id":"2108.11193","version":2},"attestation_state":"computed","paper":{"title":"Models In a Spelling Bee: Language Models Implicitly Learn the Character Composition of Tokens","license":"http://creativecommons.org/licenses/by/4.0/","headline":"","cross_cats":["cs.AI","cs.LG"],"primary_cat":"cs.CL","authors_text":"Itay Itzhak, Omer Levy","submitted_at":"2021-08-25T11:48:05Z","abstract_excerpt":"Standard pretrained language models operate on sequences of subword tokens without direct access to the characters that compose each token's string representation. We probe the embedding layer of pretrained language models and show that models learn the internal character composition of whole word and subword tokens to a surprising extent, without ever seeing the characters coupled with the tokens. Our results show that the embedding layer of RoBERTa holds enough information to accurately spell up to a third of the vocabulary and reach high average character ngram overlap on all token types. W"},"verification_status":{"content_addressed":true,"pith_receipt":true,"author_attested":false,"weak_author_claims":0,"strong_author_claims":0,"externally_anchored":false,"storage_verified":false,"citation_signatures":0,"replication_records":0,"graph_snapshot":true,"references_resolved":false,"formal_links_present":false},"canonical_record":{"source":{"id":"2108.11193","kind":"arxiv","version":2},"metadata":{"license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.CL","submitted_at":"2021-08-25T11:48:05Z","cross_cats_sorted":["cs.AI","cs.LG"],"title_canon_sha256":"094740be2239232e7bab0710a3425b98d67d469a23ef2bede34e2e3d12bc5b82","abstract_canon_sha256":"8820a46a6b6a0e1593f284fc4dd5e65193f65d3fc37d3778aa436b1ad481f886"},"schema_version":"1.0"},"receipt":{"kind":"pith_receipt","key_id":"pith-v1-2026-05","algorithm":"ed25519","signed_at":"2026-07-05T04:30:04.040690Z","signature_b64":"/C6rltyra0D2QOuDabhgS0eBpTqJsip9qJ5V7snubp+hNXPeil7vacGL2XHwQMOAi3jx93S24IUWYlVJnkR8Ag==","signed_message":"canonical_sha256_bytes","builder_version":"pith-number-builder-2026-05-17-v1","receipt_version":"0.3","canonical_sha256":"f001c3099acd58e521072f39dc347d0f8ac95ac04eb313f7749a6756bfbb557c","last_reissued_at":"2026-07-05T04:30:04.040152Z","signature_status":"signed_v1","first_computed_at":"2026-07-05T04:30:04.040152Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"graph_snapshot":{"paper":{"title":"Models In a Spelling Bee: Language Models Implicitly Learn the Character Composition of Tokens","license":"http://creativecommons.org/licenses/by/4.0/","headline":"","cross_cats":["cs.AI","cs.LG"],"primary_cat":"cs.CL","authors_text":"Itay Itzhak, Omer Levy","submitted_at":"2021-08-25T11:48:05Z","abstract_excerpt":"Standard pretrained language models operate on sequences of subword tokens without direct access to the characters that compose each token's string representation. We probe the embedding layer of pretrained language models and show that models learn the internal character composition of whole word and subword tokens to a surprising extent, without ever seeing the characters coupled with the tokens. Our results show that the embedding layer of RoBERTa holds enough information to accurately spell up to a third of the vocabulary and reach high average character ngram overlap on all token types. W"},"claims":{"count":0,"items":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"2108.11193","kind":"arxiv","version":2},"verdict":{"id":null,"model_set":{},"created_at":null,"strongest_claim":"","one_line_summary":"","pipeline_version":null,"weakest_assumption":"","pith_extraction_headline":""},"integrity":{"clean":true,"summary":{"advisory":0,"critical":0,"by_detector":{},"informational":0},"endpoint":"/pith/2108.11193/integrity.json","findings":[],"available":true,"detectors_run":[],"snapshot_sha256":"c28c3603d3b5d939e8dc4c7e95fa8dfce3d595e45f758748cecf8e644a296938"},"references":{"count":0,"sample":[],"resolved_work":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","internal_anchors":0},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"author_claims":{"count":0,"strong_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"builder_version":"pith-number-builder-2026-05-17-v1"},"aliases":[{"alias_kind":"arxiv","alias_value":"2108.11193","created_at":"2026-07-05T04:30:04.040208+00:00"},{"alias_kind":"arxiv_version","alias_value":"2108.11193v2","created_at":"2026-07-05T04:30:04.040208+00:00"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.2108.11193","created_at":"2026-07-05T04:30:04.040208+00:00"},{"alias_kind":"pith_short_12","alias_value":"6AA4GCM2ZVMO","created_at":"2026-07-05T04:30:04.040208+00:00"},{"alias_kind":"pith_short_16","alias_value":"6AA4GCM2ZVMOKIIH","created_at":"2026-07-05T04:30:04.040208+00:00"},{"alias_kind":"pith_short_8","alias_value":"6AA4GCM2","created_at":"2026-07-05T04:30:04.040208+00:00"}],"events":[],"event_summary":{},"paper_claims":[],"inbound_citations":{"count":1,"internal_anchor_count":1,"sample":[{"citing_arxiv_id":"2507.16323","citing_title":"SpeLLM: Character-Level Multi-Head Decoding","ref_index":10,"is_internal_anchor":true}]},"formal_canon":{"evidence_count":0,"sample":[],"anchors":[]},"links":{"html":"https://pith.science/pith/6AA4GCM2ZVMOKIIHF445YND5B6","json":"https://pith.science/pith/6AA4GCM2ZVMOKIIHF445YND5B6.json","graph_json":"https://pith.science/api/pith-number/6AA4GCM2ZVMOKIIHF445YND5B6/graph.json","events_json":"https://pith.science/api/pith-number/6AA4GCM2ZVMOKIIHF445YND5B6/events.json","paper":"https://pith.science/paper/6AA4GCM2"},"agent_actions":{"view_html":"https://pith.science/pith/6AA4GCM2ZVMOKIIHF445YND5B6","download_json":"https://pith.science/pith/6AA4GCM2ZVMOKIIHF445YND5B6.json","view_paper":"https://pith.science/paper/6AA4GCM2","resolve_alias":"https://pith.science/api/pith-number/resolve?arxiv=2108.11193&json=true","fetch_graph":"https://pith.science/api/pith-number/6AA4GCM2ZVMOKIIHF445YND5B6/graph.json","fetch_events":"https://pith.science/api/pith-number/6AA4GCM2ZVMOKIIHF445YND5B6/events.json","actions":{"anchor_timestamp":"https://pith.science/pith/6AA4GCM2ZVMOKIIHF445YND5B6/action/timestamp_anchor","attest_storage":"https://pith.science/pith/6AA4GCM2ZVMOKIIHF445YND5B6/action/storage_attestation","attest_author":"https://pith.science/pith/6AA4GCM2ZVMOKIIHF445YND5B6/action/author_attestation","sign_citation":"https://pith.science/pith/6AA4GCM2ZVMOKIIHF445YND5B6/action/citation_signature","submit_replication":"https://pith.science/pith/6AA4GCM2ZVMOKIIHF445YND5B6/action/replication_record"}},"created_at":"2026-07-05T04:30:04.040208+00:00","updated_at":"2026-07-05T04:30:04.040208+00:00"}