{"work":{"id":"1b427a55-e10e-463d-b44e-7fe0fd535403","openalex_id":null,"doi":null,"arxiv_id":"1901.07291","raw_key":null,"title":"Cross-lingual Language Model Pretraining","authors":null,"authors_text":"Guillaume Lample and Alexis Conneau","year":2019,"venue":"cs.CL","abstract":"Recent studies have demonstrated the efficiency of generative pretraining for English natural language understanding. In this work, we extend this approach to multiple languages and show the effectiveness of cross-lingual pretraining. We propose two methods to learn cross-lingual language models (XLMs): one unsupervised that only relies on monolingual data, and one supervised that leverages parallel data with a new cross-lingual language model objective. We obtain state-of-the-art results on cross-lingual classification, unsupervised and supervised machine translation. On XNLI, our approach pushes the state of the art by an absolute gain of 4.9% accuracy. On unsupervised machine translation, we obtain 34.3 BLEU on WMT'16 German-English, improving the previous state of the art by more than 9 BLEU. On supervised machine translation, we obtain a new state of the art of 38.5 BLEU on WMT'16 Romanian-English, outperforming the previous best approach by more than 4 BLEU. Our code and pretrained models will be made publicly available.","external_url":"https://arxiv.org/abs/1901.07291","cited_by_count":null,"metadata_source":"pith","metadata_fetched_at":"2026-05-18T06:38:36.994432+00:00","pith_arxiv_id":"1901.07291","created_at":"2026-05-09T04:47:44.386612+00:00","updated_at":"2026-05-18T06:38:36.994432+00:00","title_quality_ok":true,"display_title":"Cross- lingual language model pretraining","render_title":"Cross- lingual language model pretraining"},"hub":{"state":{"work_id":"1b427a55-e10e-463d-b44e-7fe0fd535403","tier":"hub","tier_reason":"10+ Pith inbound or 1,000+ external citations","pith_inbound_count":12,"external_cited_by_count":null,"distinct_field_count":4,"first_pith_cited_at":"2019-07-26T17:48:29+00:00","last_pith_cited_at":"2026-05-07T09:14:46+00:00","author_build_status":"not_needed","summary_status":"needed","contexts_status":"needed","graph_status":"needed","ask_index_status":"not_needed","reader_status":"not_needed","recognition_status":"not_needed","updated_at":"2026-05-20T13:01:53.857289+00:00","tier_text":"hub"},"tier":"hub","role_counts":[{"context_role":"background","n":1}],"polarity_counts":[{"context_polarity":"background","n":1}],"runs":{},"summary":{},"graph":{},"authors":[]}}