{"record_type":"pith_number_record","schema_url":"https://pith.science/schemas/pith-number/v1.json","pith_number":"pith:2018:6SWPWALIPXPRECOPMZDDPUKLJJ","short_pith_number":"pith:6SWPWALI","schema_version":"1.0","canonical_sha256":"f4acfb01687ddf1209cf664637d14b4a7c65a6a8487c51d5310c6d41bd864b33","source":{"kind":"arxiv","id":"1804.10959","version":1},"attestation_state":"computed","paper":{"title":"Subword Regularization: Improving Neural Network Translation Models with Multiple Subword Candidates","license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","headline":"","cross_cats":[],"primary_cat":"cs.CL","authors_text":"Taku Kudo","submitted_at":"2018-04-29T16:13:44Z","abstract_excerpt":"Subword units are an effective way to alleviate the open vocabulary problems in neural machine translation (NMT). While sentences are usually converted into unique subword sequences, subword segmentation is potentially ambiguous and multiple segmentations are possible even with the same vocabulary. The question addressed in this paper is whether it is possible to harness the segmentation ambiguity as a noise to improve the robustness of NMT. We present a simple regularization method, subword regularization, which trains the model with multiple subword segmentations probabilistically sampled du"},"verification_status":{"content_addressed":true,"pith_receipt":true,"author_attested":false,"weak_author_claims":0,"strong_author_claims":0,"externally_anchored":false,"storage_verified":false,"citation_signatures":0,"replication_records":0,"graph_snapshot":true,"references_resolved":false,"formal_links_present":false},"canonical_record":{"source":{"id":"1804.10959","kind":"arxiv","version":1},"metadata":{"license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","primary_cat":"cs.CL","submitted_at":"2018-04-29T16:13:44Z","cross_cats_sorted":[],"title_canon_sha256":"a4f2ec8267050cb7845f5983d057e5a1a2ecf9d2016eee62c9de635dd83f8516","abstract_canon_sha256":"7262b0bbdd41b5992d995ff1c54142761a439a8f770c6b734f57351e35a5937b"},"schema_version":"1.0"},"receipt":{"kind":"pith_receipt","key_id":"pith-v1-2026-05","algorithm":"ed25519","signed_at":"2026-05-18T00:17:15.071268Z","signature_b64":"QY+sVqFcFCkLxn9rgqVNPqAuAx8YmUMnv0LTOxDvbY9mAN006zMs5mp0lByjh5esXcr+RSr5FI2t+KS8JZxzAA==","signed_message":"canonical_sha256_bytes","builder_version":"pith-number-builder-2026-05-17-v1","receipt_version":"0.3","canonical_sha256":"f4acfb01687ddf1209cf664637d14b4a7c65a6a8487c51d5310c6d41bd864b33","last_reissued_at":"2026-05-18T00:17:15.070645Z","signature_status":"signed_v1","first_computed_at":"2026-05-18T00:17:15.070645Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"graph_snapshot":{"paper":{"title":"Subword Regularization: Improving Neural Network Translation Models with Multiple Subword Candidates","license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","headline":"","cross_cats":[],"primary_cat":"cs.CL","authors_text":"Taku Kudo","submitted_at":"2018-04-29T16:13:44Z","abstract_excerpt":"Subword units are an effective way to alleviate the open vocabulary problems in neural machine translation (NMT). While sentences are usually converted into unique subword sequences, subword segmentation is potentially ambiguous and multiple segmentations are possible even with the same vocabulary. The question addressed in this paper is whether it is possible to harness the segmentation ambiguity as a noise to improve the robustness of NMT. We present a simple regularization method, subword regularization, which trains the model with multiple subword segmentations probabilistically sampled du"},"claims":{"count":0,"items":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"1804.10959","kind":"arxiv","version":1},"verdict":{"id":null,"model_set":{},"created_at":null,"strongest_claim":"","one_line_summary":"","pipeline_version":null,"weakest_assumption":"","pith_extraction_headline":""},"references":{"count":0,"sample":[],"resolved_work":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","internal_anchors":0},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"author_claims":{"count":0,"strong_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"builder_version":"pith-number-builder-2026-05-17-v1"},"aliases":[{"alias_kind":"arxiv","alias_value":"1804.10959","created_at":"2026-05-18T00:17:15.070728+00:00"},{"alias_kind":"arxiv_version","alias_value":"1804.10959v1","created_at":"2026-05-18T00:17:15.070728+00:00"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.1804.10959","created_at":"2026-05-18T00:17:15.070728+00:00"},{"alias_kind":"pith_short_12","alias_value":"6SWPWALIPXPR","created_at":"2026-05-18T12:32:11.075285+00:00"},{"alias_kind":"pith_short_16","alias_value":"6SWPWALIPXPRECOP","created_at":"2026-05-18T12:32:11.075285+00:00"},{"alias_kind":"pith_short_8","alias_value":"6SWPWALI","created_at":"2026-05-18T12:32:11.075285+00:00"}],"events":[],"event_summary":{},"paper_claims":[],"inbound_citations":{"count":6,"internal_anchor_count":2,"sample":[{"citing_arxiv_id":"2605.17762","citing_title":"Surface-Form Neural Sparse Retrieval: Robust Fuzzy Matching for Industrial Music Search","ref_index":18,"is_internal_anchor":true},{"citing_arxiv_id":"2205.01917","citing_title":"CoCa: Contrastive Captioners are Image-Text Foundation Models","ref_index":44,"is_internal_anchor":true},{"citing_arxiv_id":"2605.09156","citing_title":"Lost in Translation? Exploring the Shift in Grammatical Gender from Latin to Occitan","ref_index":24,"is_internal_anchor":false},{"citing_arxiv_id":"1910.10683","citing_title":"Exploring the Limits of Transfer Learning with a Unified Text-to-Text Transformer","ref_index":40,"is_internal_anchor":false},{"citing_arxiv_id":"2604.10665","citing_title":"HeceTokenizer: A Syllable-Based Tokenization Approach for Turkish Retrieval","ref_index":8,"is_internal_anchor":false},{"citing_arxiv_id":"2604.17105","citing_title":"How Tokenization Limits Phonological Knowledge Representation in Language Models and How to Improve Them","ref_index":31,"is_internal_anchor":false}]},"formal_canon":{"evidence_count":0,"sample":[],"anchors":[]},"links":{"html":"https://pith.science/pith/6SWPWALIPXPRECOPMZDDPUKLJJ","json":"https://pith.science/pith/6SWPWALIPXPRECOPMZDDPUKLJJ.json","graph_json":"https://pith.science/api/pith-number/6SWPWALIPXPRECOPMZDDPUKLJJ/graph.json","events_json":"https://pith.science/api/pith-number/6SWPWALIPXPRECOPMZDDPUKLJJ/events.json","paper":"https://pith.science/paper/6SWPWALI"},"agent_actions":{"view_html":"https://pith.science/pith/6SWPWALIPXPRECOPMZDDPUKLJJ","download_json":"https://pith.science/pith/6SWPWALIPXPRECOPMZDDPUKLJJ.json","view_paper":"https://pith.science/paper/6SWPWALI","resolve_alias":"https://pith.science/api/pith-number/resolve?arxiv=1804.10959&json=true","fetch_graph":"https://pith.science/api/pith-number/6SWPWALIPXPRECOPMZDDPUKLJJ/graph.json","fetch_events":"https://pith.science/api/pith-number/6SWPWALIPXPRECOPMZDDPUKLJJ/events.json","actions":{"anchor_timestamp":"https://pith.science/pith/6SWPWALIPXPRECOPMZDDPUKLJJ/action/timestamp_anchor","attest_storage":"https://pith.science/pith/6SWPWALIPXPRECOPMZDDPUKLJJ/action/storage_attestation","attest_author":"https://pith.science/pith/6SWPWALIPXPRECOPMZDDPUKLJJ/action/author_attestation","sign_citation":"https://pith.science/pith/6SWPWALIPXPRECOPMZDDPUKLJJ/action/citation_signature","submit_replication":"https://pith.science/pith/6SWPWALIPXPRECOPMZDDPUKLJJ/action/replication_record"}},"created_at":"2026-05-18T00:17:15.070728+00:00","updated_at":"2026-05-18T00:17:15.070728+00:00"}