{"record_type":"pith_number_record","schema_url":"https://pith.science/schemas/pith-number/v1.json","pith_number":"pith:2025:DXBWIRUNPB4JV4CMY7O5MJJ467","short_pith_number":"pith:DXBWIRUN","schema_version":"1.0","canonical_sha256":"1dc364468d78789af04cc7ddd6253cf7f22cca97299493ee5eaa8b6a027a3245","source":{"kind":"arxiv","id":"2502.03009","version":2},"attestation_state":"computed","paper":{"title":"Scaling Laws for Upcycling Mixture-of-Experts Language Models","license":"http://creativecommons.org/licenses/by/4.0/","headline":"","cross_cats":["cs.CL"],"primary_cat":"cs.LG","authors_text":"Seng Pei Liew, Sho Takase, Takuya Kato","submitted_at":"2025-02-05T09:11:13Z","abstract_excerpt":"Pretraining large language models (LLMs) is resource-intensive, often requiring months of training time even with high-end GPU clusters. There are two approaches of mitigating such computational demands: reusing smaller models to train larger ones (upcycling), and training computationally efficient models like mixture-of-experts (MoE). In this paper, we study the upcycling of LLMs to MoE models, of which the scaling behavior remains underexplored. Through extensive experiments, we identify empirical scaling laws that describe how performance depends on dataset size and model configuration. Par"},"verification_status":{"content_addressed":true,"pith_receipt":true,"author_attested":false,"weak_author_claims":0,"strong_author_claims":0,"externally_anchored":false,"storage_verified":false,"citation_signatures":0,"replication_records":0,"graph_snapshot":true,"references_resolved":false,"formal_links_present":false},"canonical_record":{"source":{"id":"2502.03009","kind":"arxiv","version":2},"metadata":{"license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.LG","submitted_at":"2025-02-05T09:11:13Z","cross_cats_sorted":["cs.CL"],"title_canon_sha256":"a35a8c71f0376c3a1f734c027fe193b502fb312ac30c3139d8af28baea5c8550","abstract_canon_sha256":"6af4c0477b75537ca94d0f83701bb69e8d039e3905edef8131ae4b1c878d2c1f"},"schema_version":"1.0"},"receipt":{"kind":"pith_receipt","key_id":"pith-v1-2026-05","algorithm":"ed25519","signed_at":"2026-07-05T11:21:41.662255Z","signature_b64":"Xp8401r6BghBslIOABbyGhSthsQVWfzZzTvaf0HWfbuixl7T2lyxv94jLQMAHlfjK7EeToEMnRXHgjXtO9e+Aw==","signed_message":"canonical_sha256_bytes","builder_version":"pith-number-builder-2026-05-17-v1","receipt_version":"0.3","canonical_sha256":"1dc364468d78789af04cc7ddd6253cf7f22cca97299493ee5eaa8b6a027a3245","last_reissued_at":"2026-07-05T11:21:41.661740Z","signature_status":"signed_v1","first_computed_at":"2026-07-05T11:21:41.661740Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"graph_snapshot":{"paper":{"title":"Scaling Laws for Upcycling Mixture-of-Experts Language Models","license":"http://creativecommons.org/licenses/by/4.0/","headline":"","cross_cats":["cs.CL"],"primary_cat":"cs.LG","authors_text":"Seng Pei Liew, Sho Takase, Takuya Kato","submitted_at":"2025-02-05T09:11:13Z","abstract_excerpt":"Pretraining large language models (LLMs) is resource-intensive, often requiring months of training time even with high-end GPU clusters. There are two approaches of mitigating such computational demands: reusing smaller models to train larger ones (upcycling), and training computationally efficient models like mixture-of-experts (MoE). In this paper, we study the upcycling of LLMs to MoE models, of which the scaling behavior remains underexplored. Through extensive experiments, we identify empirical scaling laws that describe how performance depends on dataset size and model configuration. Par"},"claims":{"count":0,"items":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"2502.03009","kind":"arxiv","version":2},"verdict":{"id":null,"model_set":{},"created_at":null,"strongest_claim":"","one_line_summary":"","pipeline_version":null,"weakest_assumption":"","pith_extraction_headline":""},"integrity":{"clean":true,"summary":{"advisory":0,"critical":0,"by_detector":{},"informational":0},"endpoint":"/pith/2502.03009/integrity.json","findings":[],"available":true,"detectors_run":[],"snapshot_sha256":"c28c3603d3b5d939e8dc4c7e95fa8dfce3d595e45f758748cecf8e644a296938"},"references":{"count":0,"sample":[],"resolved_work":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","internal_anchors":0},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"author_claims":{"count":0,"strong_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"builder_version":"pith-number-builder-2026-05-17-v1"},"aliases":[{"alias_kind":"arxiv","alias_value":"2502.03009","created_at":"2026-07-05T11:21:41.661806+00:00"},{"alias_kind":"arxiv_version","alias_value":"2502.03009v2","created_at":"2026-07-05T11:21:41.661806+00:00"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.2502.03009","created_at":"2026-07-05T11:21:41.661806+00:00"},{"alias_kind":"pith_short_12","alias_value":"DXBWIRUNPB4J","created_at":"2026-07-05T11:21:41.661806+00:00"},{"alias_kind":"pith_short_16","alias_value":"DXBWIRUNPB4JV4CM","created_at":"2026-07-05T11:21:41.661806+00:00"},{"alias_kind":"pith_short_8","alias_value":"DXBWIRUN","created_at":"2026-07-05T11:21:41.661806+00:00"}],"events":[],"event_summary":{},"paper_claims":[],"inbound_citations":{"count":2,"internal_anchor_count":0,"sample":[{"citing_arxiv_id":"2604.19835","citing_title":"Expert Upcycling: Shifting the Compute-Efficient Frontier of Mixture-of-Experts","ref_index":32,"is_internal_anchor":false},{"citing_arxiv_id":"2604.19835","citing_title":"Expert Upcycling: Shifting the Compute-Efficient Frontier of Mixture-of-Experts","ref_index":32,"is_internal_anchor":false}]},"formal_canon":{"evidence_count":0,"sample":[],"anchors":[]},"links":{"html":"https://pith.science/pith/DXBWIRUNPB4JV4CMY7O5MJJ467","json":"https://pith.science/pith/DXBWIRUNPB4JV4CMY7O5MJJ467.json","graph_json":"https://pith.science/api/pith-number/DXBWIRUNPB4JV4CMY7O5MJJ467/graph.json","events_json":"https://pith.science/api/pith-number/DXBWIRUNPB4JV4CMY7O5MJJ467/events.json","paper":"https://pith.science/paper/DXBWIRUN"},"agent_actions":{"view_html":"https://pith.science/pith/DXBWIRUNPB4JV4CMY7O5MJJ467","download_json":"https://pith.science/pith/DXBWIRUNPB4JV4CMY7O5MJJ467.json","view_paper":"https://pith.science/paper/DXBWIRUN","resolve_alias":"https://pith.science/api/pith-number/resolve?arxiv=2502.03009&json=true","fetch_graph":"https://pith.science/api/pith-number/DXBWIRUNPB4JV4CMY7O5MJJ467/graph.json","fetch_events":"https://pith.science/api/pith-number/DXBWIRUNPB4JV4CMY7O5MJJ467/events.json","actions":{"anchor_timestamp":"https://pith.science/pith/DXBWIRUNPB4JV4CMY7O5MJJ467/action/timestamp_anchor","attest_storage":"https://pith.science/pith/DXBWIRUNPB4JV4CMY7O5MJJ467/action/storage_attestation","attest_author":"https://pith.science/pith/DXBWIRUNPB4JV4CMY7O5MJJ467/action/author_attestation","sign_citation":"https://pith.science/pith/DXBWIRUNPB4JV4CMY7O5MJJ467/action/citation_signature","submit_replication":"https://pith.science/pith/DXBWIRUNPB4JV4CMY7O5MJJ467/action/replication_record"}},"created_at":"2026-07-05T11:21:41.661806+00:00","updated_at":"2026-07-05T11:21:41.661806+00:00"}