{"record_type":"pith_number_record","schema_url":"https://pith.science/schemas/pith-number/v1.json","pith_number":"pith:2024:RHLZSOENGQ22NHEAGAUCK6ELLG","short_pith_number":"pith:RHLZSOEN","schema_version":"1.0","canonical_sha256":"89d799388d3435a69c80302825788b59a5355f1815004663f3a3036312a035b5","source":{"kind":"arxiv","id":"2406.10851","version":2},"attestation_state":"computed","paper":{"title":"Leading Whitespaces of Language Models' Subword Vocabulary Pose a Confound for Calculating Word Probabilities","license":"http://creativecommons.org/licenses/by/4.0/","headline":"","cross_cats":[],"primary_cat":"cs.CL","authors_text":"Byung-Doh Oh, William Schuler","submitted_at":"2024-06-16T08:44:56Z","abstract_excerpt":"Predictions of word-by-word conditional probabilities from Transformer-based language models are often evaluated to model the incremental processing difficulty of human readers. In this paper, we argue that there is a confound posed by the most common method of aggregating subword probabilities of such language models into word probabilities. This is due to the fact that tokens in the subword vocabulary of most language models have leading whitespaces and therefore do not naturally define stop probabilities of words. We first prove that this can result in distributions over word probabilities "},"verification_status":{"content_addressed":true,"pith_receipt":true,"author_attested":false,"weak_author_claims":0,"strong_author_claims":0,"externally_anchored":false,"storage_verified":false,"citation_signatures":0,"replication_records":0,"graph_snapshot":true,"references_resolved":false,"formal_links_present":false},"canonical_record":{"source":{"id":"2406.10851","kind":"arxiv","version":2},"metadata":{"license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.CL","submitted_at":"2024-06-16T08:44:56Z","cross_cats_sorted":[],"title_canon_sha256":"24e599b7e69e5b9993c9695a76c6fcabc6536aed796392a8ce4b6878fce936eb","abstract_canon_sha256":"2c2a54a6ba98435d5ea422c5cbdca553461921220ee936fa1b1725055b50c256"},"schema_version":"1.0"},"receipt":{"kind":"pith_receipt","key_id":"pith-v1-2026-05","algorithm":"ed25519","signed_at":"2026-07-05T09:13:03.734363Z","signature_b64":"T++pybE+/CkEWpvs7ld7ou/H8dpUW4BMeWxKWtd+O1CSEDzQNYeonZp9H4+ns6SfcyNyVOVZ9ljBkVKUS/7uBw==","signed_message":"canonical_sha256_bytes","builder_version":"pith-number-builder-2026-05-17-v1","receipt_version":"0.3","canonical_sha256":"89d799388d3435a69c80302825788b59a5355f1815004663f3a3036312a035b5","last_reissued_at":"2026-07-05T09:13:03.733919Z","signature_status":"signed_v1","first_computed_at":"2026-07-05T09:13:03.733919Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"graph_snapshot":{"paper":{"title":"Leading Whitespaces of Language Models' Subword Vocabulary Pose a Confound for Calculating Word Probabilities","license":"http://creativecommons.org/licenses/by/4.0/","headline":"","cross_cats":[],"primary_cat":"cs.CL","authors_text":"Byung-Doh Oh, William Schuler","submitted_at":"2024-06-16T08:44:56Z","abstract_excerpt":"Predictions of word-by-word conditional probabilities from Transformer-based language models are often evaluated to model the incremental processing difficulty of human readers. In this paper, we argue that there is a confound posed by the most common method of aggregating subword probabilities of such language models into word probabilities. This is due to the fact that tokens in the subword vocabulary of most language models have leading whitespaces and therefore do not naturally define stop probabilities of words. We first prove that this can result in distributions over word probabilities "},"claims":{"count":0,"items":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"2406.10851","kind":"arxiv","version":2},"verdict":{"id":null,"model_set":{},"created_at":null,"strongest_claim":"","one_line_summary":"","pipeline_version":null,"weakest_assumption":"","pith_extraction_headline":""},"integrity":{"clean":true,"summary":{"advisory":0,"critical":0,"by_detector":{},"informational":0},"endpoint":"/pith/2406.10851/integrity.json","findings":[],"available":true,"detectors_run":[],"snapshot_sha256":"c28c3603d3b5d939e8dc4c7e95fa8dfce3d595e45f758748cecf8e644a296938"},"references":{"count":0,"sample":[],"resolved_work":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","internal_anchors":0},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"author_claims":{"count":0,"strong_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"builder_version":"pith-number-builder-2026-05-17-v1"},"aliases":[{"alias_kind":"arxiv","alias_value":"2406.10851","created_at":"2026-07-05T09:13:03.733980+00:00"},{"alias_kind":"arxiv_version","alias_value":"2406.10851v2","created_at":"2026-07-05T09:13:03.733980+00:00"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.2406.10851","created_at":"2026-07-05T09:13:03.733980+00:00"},{"alias_kind":"pith_short_12","alias_value":"RHLZSOENGQ22","created_at":"2026-07-05T09:13:03.733980+00:00"},{"alias_kind":"pith_short_16","alias_value":"RHLZSOENGQ22NHEA","created_at":"2026-07-05T09:13:03.733980+00:00"},{"alias_kind":"pith_short_8","alias_value":"RHLZSOEN","created_at":"2026-07-05T09:13:03.733980+00:00"}],"events":[],"event_summary":{},"paper_claims":[],"inbound_citations":{"count":1,"internal_anchor_count":1,"sample":[{"citing_arxiv_id":"2412.10369","citing_title":"A Grounded Typology of Word Classes","ref_index":43,"is_internal_anchor":true}]},"formal_canon":{"evidence_count":0,"sample":[],"anchors":[]},"links":{"html":"https://pith.science/pith/RHLZSOENGQ22NHEAGAUCK6ELLG","json":"https://pith.science/pith/RHLZSOENGQ22NHEAGAUCK6ELLG.json","graph_json":"https://pith.science/api/pith-number/RHLZSOENGQ22NHEAGAUCK6ELLG/graph.json","events_json":"https://pith.science/api/pith-number/RHLZSOENGQ22NHEAGAUCK6ELLG/events.json","paper":"https://pith.science/paper/RHLZSOEN"},"agent_actions":{"view_html":"https://pith.science/pith/RHLZSOENGQ22NHEAGAUCK6ELLG","download_json":"https://pith.science/pith/RHLZSOENGQ22NHEAGAUCK6ELLG.json","view_paper":"https://pith.science/paper/RHLZSOEN","resolve_alias":"https://pith.science/api/pith-number/resolve?arxiv=2406.10851&json=true","fetch_graph":"https://pith.science/api/pith-number/RHLZSOENGQ22NHEAGAUCK6ELLG/graph.json","fetch_events":"https://pith.science/api/pith-number/RHLZSOENGQ22NHEAGAUCK6ELLG/events.json","actions":{"anchor_timestamp":"https://pith.science/pith/RHLZSOENGQ22NHEAGAUCK6ELLG/action/timestamp_anchor","attest_storage":"https://pith.science/pith/RHLZSOENGQ22NHEAGAUCK6ELLG/action/storage_attestation","attest_author":"https://pith.science/pith/RHLZSOENGQ22NHEAGAUCK6ELLG/action/author_attestation","sign_citation":"https://pith.science/pith/RHLZSOENGQ22NHEAGAUCK6ELLG/action/citation_signature","submit_replication":"https://pith.science/pith/RHLZSOENGQ22NHEAGAUCK6ELLG/action/replication_record"}},"created_at":"2026-07-05T09:13:03.733980+00:00","updated_at":"2026-07-05T09:13:03.733980+00:00"}