{"record_type":"pith_number_record","schema_url":"https://pith.science/schemas/pith-number/v1.json","pith_number":"pith:2023:CWGHMEMW4JCPDHENVPUFRVB66U","short_pith_number":"pith:CWGHMEMW","schema_version":"1.0","canonical_sha256":"158c761196e244f19c8dabe858d43ef52a0ef11ef088f26169d956e7b0c0fa27","source":{"kind":"arxiv","id":"2305.10429","version":4},"attestation_state":"computed","paper":{"title":"DoReMi: Optimizing Data Mixtures Speeds Up Language Model Pretraining","license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","headline":"","cross_cats":["cs.LG"],"primary_cat":"cs.CL","authors_text":"Adams Wei Yu, Hanxiao Liu, Hieu Pham, Nan Du, Percy Liang, Quoc V. Le, Sang Michael Xie, Tengyu Ma, Xuanyi Dong, Yifeng Lu","submitted_at":"2023-05-17T17:58:13Z","abstract_excerpt":"The mixture proportions of pretraining data domains (e.g., Wikipedia, books, web text) greatly affect language model (LM) performance. In this paper, we propose Domain Reweighting with Minimax Optimization (DoReMi), which first trains a small proxy model using group distributionally robust optimization (Group DRO) over domains to produce domain weights (mixture proportions) without knowledge of downstream tasks. We then resample a dataset with these domain weights and train a larger, full-sized model. In our experiments, we use DoReMi on a 280M-parameter proxy model to set the domain weights f"},"verification_status":{"content_addressed":true,"pith_receipt":true,"author_attested":false,"weak_author_claims":0,"strong_author_claims":0,"externally_anchored":false,"storage_verified":false,"citation_signatures":0,"replication_records":0,"graph_snapshot":true,"references_resolved":false,"formal_links_present":false},"canonical_record":{"source":{"id":"2305.10429","kind":"arxiv","version":4},"metadata":{"license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","primary_cat":"cs.CL","submitted_at":"2023-05-17T17:58:13Z","cross_cats_sorted":["cs.LG"],"title_canon_sha256":"09e9671ae95ded18ffafdf0c1d0bda7690860ca9c99d28de5eabd68b6b5aa6f0","abstract_canon_sha256":"080c8e43d4f39066a368bf7a5ddecb760bc25eb88dd1bc9de7118ce58eb05037"},"schema_version":"1.0"},"receipt":{"kind":"pith_receipt","key_id":"pith-v1-2026-05","algorithm":"ed25519","signed_at":"2026-07-05T07:14:53.565311Z","signature_b64":"VYMXwy2Dq30AmZbJ9R8sguVHPds6FDnwmTTZAZq/NA5jt0fu73LKs7Rnd1AlEwx41lAqxDLL5Po+V98WKCKpDQ==","signed_message":"canonical_sha256_bytes","builder_version":"pith-number-builder-2026-05-17-v1","receipt_version":"0.3","canonical_sha256":"158c761196e244f19c8dabe858d43ef52a0ef11ef088f26169d956e7b0c0fa27","last_reissued_at":"2026-07-05T07:14:53.564770Z","signature_status":"signed_v1","first_computed_at":"2026-07-05T07:14:53.564770Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"graph_snapshot":{"paper":{"title":"DoReMi: Optimizing Data Mixtures Speeds Up Language Model Pretraining","license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","headline":"","cross_cats":["cs.LG"],"primary_cat":"cs.CL","authors_text":"Adams Wei Yu, Hanxiao Liu, Hieu Pham, Nan Du, Percy Liang, Quoc V. Le, Sang Michael Xie, Tengyu Ma, Xuanyi Dong, Yifeng Lu","submitted_at":"2023-05-17T17:58:13Z","abstract_excerpt":"The mixture proportions of pretraining data domains (e.g., Wikipedia, books, web text) greatly affect language model (LM) performance. In this paper, we propose Domain Reweighting with Minimax Optimization (DoReMi), which first trains a small proxy model using group distributionally robust optimization (Group DRO) over domains to produce domain weights (mixture proportions) without knowledge of downstream tasks. We then resample a dataset with these domain weights and train a larger, full-sized model. In our experiments, we use DoReMi on a 280M-parameter proxy model to set the domain weights f"},"claims":{"count":0,"items":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"2305.10429","kind":"arxiv","version":4},"verdict":{"id":null,"model_set":{},"created_at":null,"strongest_claim":"","one_line_summary":"","pipeline_version":null,"weakest_assumption":"","pith_extraction_headline":""},"integrity":{"clean":true,"summary":{"advisory":0,"critical":0,"by_detector":{},"informational":0},"endpoint":"/pith/2305.10429/integrity.json","findings":[],"available":true,"detectors_run":[],"snapshot_sha256":"c28c3603d3b5d939e8dc4c7e95fa8dfce3d595e45f758748cecf8e644a296938"},"references":{"count":0,"sample":[],"resolved_work":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","internal_anchors":0},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"author_claims":{"count":0,"strong_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"builder_version":"pith-number-builder-2026-05-17-v1"},"aliases":[{"alias_kind":"arxiv","alias_value":"2305.10429","created_at":"2026-07-05T07:14:53.564833+00:00"},{"alias_kind":"arxiv_version","alias_value":"2305.10429v4","created_at":"2026-07-05T07:14:53.564833+00:00"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.2305.10429","created_at":"2026-07-05T07:14:53.564833+00:00"},{"alias_kind":"pith_short_12","alias_value":"CWGHMEMW4JCP","created_at":"2026-07-05T07:14:53.564833+00:00"},{"alias_kind":"pith_short_16","alias_value":"CWGHMEMW4JCPDHEN","created_at":"2026-07-05T07:14:53.564833+00:00"},{"alias_kind":"pith_short_8","alias_value":"CWGHMEMW","created_at":"2026-07-05T07:14:53.564833+00:00"}],"events":[],"event_summary":{},"paper_claims":[],"inbound_citations":{"count":7,"internal_anchor_count":0,"sample":[{"citing_arxiv_id":"2606.26050","citing_title":"Natural Ungrokking: Asymmetric Control of Which Rules Survive Pretraining","ref_index":46,"is_internal_anchor":false},{"citing_arxiv_id":"2607.02266","citing_title":"HERMES: A Multi-Granularity Labeling Substrate for Pre-training Data Mixtures","ref_index":1,"is_internal_anchor":false},{"citing_arxiv_id":"2606.12365","citing_title":"Ambient Diffusion Policy: Imitation Learning from Suboptimal Data in Robotics","ref_index":30,"is_internal_anchor":false},{"citing_arxiv_id":"2310.10631","citing_title":"Llemma: An Open Language Model For Mathematics","ref_index":197,"is_internal_anchor":false},{"citing_arxiv_id":"2508.15202","citing_title":"Fin-PRM: A Domain-Specialized Process Reward Model for Financial Reasoning in Large Language Models","ref_index":24,"is_internal_anchor":false},{"citing_arxiv_id":"2605.13225","citing_title":"Mix, Don't Tune: Bilingual Pre-Training Outperforms Hyperparameter Search in Data-Constrained Settings","ref_index":26,"is_internal_anchor":false},{"citing_arxiv_id":"2303.18223","citing_title":"A Survey of Large Language Models","ref_index":61,"is_internal_anchor":false}]},"formal_canon":{"evidence_count":0,"sample":[],"anchors":[]},"links":{"html":"https://pith.science/pith/CWGHMEMW4JCPDHENVPUFRVB66U","json":"https://pith.science/pith/CWGHMEMW4JCPDHENVPUFRVB66U.json","graph_json":"https://pith.science/api/pith-number/CWGHMEMW4JCPDHENVPUFRVB66U/graph.json","events_json":"https://pith.science/api/pith-number/CWGHMEMW4JCPDHENVPUFRVB66U/events.json","paper":"https://pith.science/paper/CWGHMEMW"},"agent_actions":{"view_html":"https://pith.science/pith/CWGHMEMW4JCPDHENVPUFRVB66U","download_json":"https://pith.science/pith/CWGHMEMW4JCPDHENVPUFRVB66U.json","view_paper":"https://pith.science/paper/CWGHMEMW","resolve_alias":"https://pith.science/api/pith-number/resolve?arxiv=2305.10429&json=true","fetch_graph":"https://pith.science/api/pith-number/CWGHMEMW4JCPDHENVPUFRVB66U/graph.json","fetch_events":"https://pith.science/api/pith-number/CWGHMEMW4JCPDHENVPUFRVB66U/events.json","actions":{"anchor_timestamp":"https://pith.science/pith/CWGHMEMW4JCPDHENVPUFRVB66U/action/timestamp_anchor","attest_storage":"https://pith.science/pith/CWGHMEMW4JCPDHENVPUFRVB66U/action/storage_attestation","attest_author":"https://pith.science/pith/CWGHMEMW4JCPDHENVPUFRVB66U/action/author_attestation","sign_citation":"https://pith.science/pith/CWGHMEMW4JCPDHENVPUFRVB66U/action/citation_signature","submit_replication":"https://pith.science/pith/CWGHMEMW4JCPDHENVPUFRVB66U/action/replication_record"}},"created_at":"2026-07-05T07:14:53.564833+00:00","updated_at":"2026-07-05T07:14:53.564833+00:00"}