{"bundle_type":"pith_open_graph_bundle","bundle_version":"1.0","pith_number":"pith:2024:RW5XONPRALUDJDUGBESKZJT2PU","short_pith_number":"pith:RW5XONPR","canonical_record":{"source":{"id":"2406.06563","kind":"arxiv","version":1},"metadata":{"license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.CL","submitted_at":"2024-06-03T03:58:41Z","cross_cats_sorted":["cs.AI"],"title_canon_sha256":"19be60bf4420a9da3c11bd67d528d09e86c9af4e75b575b31331cdc395641525","abstract_canon_sha256":"70e6a03e57a3b2cb107beef0010d4d9d631c63401d32aa1285188fe444594fa1"},"schema_version":"1.0"},"canonical_sha256":"8dbb7735f102e8348e860924aca67a7d08187c5ca7fa346374c42620a351bd02","source":{"kind":"arxiv","id":"2406.06563","version":1},"source_aliases":[{"alias_kind":"arxiv","alias_value":"2406.06563","created_at":"2026-07-05T08:29:47Z"},{"alias_kind":"arxiv_version","alias_value":"2406.06563v1","created_at":"2026-07-05T08:29:47Z"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.2406.06563","created_at":"2026-07-05T08:29:47Z"},{"alias_kind":"pith_short_12","alias_value":"RW5XONPRALUD","created_at":"2026-07-05T08:29:47Z"},{"alias_kind":"pith_short_16","alias_value":"RW5XONPRALUDJDUG","created_at":"2026-07-05T08:29:47Z"},{"alias_kind":"pith_short_8","alias_value":"RW5XONPR","created_at":"2026-07-05T08:29:47Z"}],"events":[{"event_type":"record_created","subject_pith_number":"pith:2024:RW5XONPRALUDJDUGBESKZJT2PU","target":"record","payload":{"canonical_record":{"source":{"id":"2406.06563","kind":"arxiv","version":1},"metadata":{"license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.CL","submitted_at":"2024-06-03T03:58:41Z","cross_cats_sorted":["cs.AI"],"title_canon_sha256":"19be60bf4420a9da3c11bd67d528d09e86c9af4e75b575b31331cdc395641525","abstract_canon_sha256":"70e6a03e57a3b2cb107beef0010d4d9d631c63401d32aa1285188fe444594fa1"},"schema_version":"1.0"},"canonical_sha256":"8dbb7735f102e8348e860924aca67a7d08187c5ca7fa346374c42620a351bd02","receipt":{"kind":"pith_receipt","key_id":"pith-v1-2026-05","algorithm":"ed25519","signed_at":"2026-07-05T08:29:47.524159Z","signature_b64":"dFGZVznwsekFi5V0aWqubTB1yPPl/STRXXQ0G7Uj6xFY8d2YTfkaUayK5vYvtzlFMwMR3dlJ8l/gtPRrxBbGCQ==","signed_message":"canonical_sha256_bytes","builder_version":"pith-number-builder-2026-05-17-v1","receipt_version":"0.3","canonical_sha256":"8dbb7735f102e8348e860924aca67a7d08187c5ca7fa346374c42620a351bd02","last_reissued_at":"2026-07-05T08:29:47.523700Z","signature_status":"signed_v1","first_computed_at":"2026-07-05T08:29:47.523700Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"source_kind":"arxiv","source_id":"2406.06563","source_version":1,"attestation_state":"computed"},"signer":{"signer_id":"pith.science","signer_type":"pith_registry","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"created_at":"2026-07-05T08:29:47Z","supersedes":[],"prev_event":null,"signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"MxcWl9pw6yzCplU36I6FN23NTsDfHrSrMELLJj5yz9o7ikJrwRVDEvgVbyVtUTsS9oB3x3OtO3Plf/AurKPKCg==","signed_message":"open_graph_event_sha256_bytes","signed_at":"2026-07-06T10:03:37.581978Z"},"content_sha256":"b69ee5c7c319338e10038a44263597502078cf7988a931f570fb24d242e604f6","schema_version":"1.0","event_id":"sha256:b69ee5c7c319338e10038a44263597502078cf7988a931f570fb24d242e604f6"},{"event_type":"graph_snapshot","subject_pith_number":"pith:2024:RW5XONPRALUDJDUGBESKZJT2PU","target":"graph","payload":{"graph_snapshot":{"paper":{"title":"Skywork-MoE: A Deep Dive into Training Techniques for Mixture-of-Experts Language Models","license":"http://creativecommons.org/licenses/by/4.0/","headline":"","cross_cats":["cs.AI"],"primary_cat":"cs.CL","authors_text":"Biye Li, Bo Zhu, Cheng cheng, Han Fang, Jianhao Zhang, Liang Zeng, Liang Zhao, Peng Cheng, Rui Hu, Shuicheng Yan, Tianwen Wei, Weiwei L\\\"u, Xiaokun Wang, Xiaoyu Zhang, Yahui Zhou, Yutuan Ma","submitted_at":"2024-06-03T03:58:41Z","abstract_excerpt":"In this technical report, we introduce the training methodologies implemented in the development of Skywork-MoE, a high-performance mixture-of-experts (MoE) large language model (LLM) with 146 billion parameters and 16 experts. It is initialized from the pre-existing dense checkpoints of our Skywork-13B model. We explore the comparative effectiveness of upcycling versus training from scratch initializations. Our findings suggest that the choice between these two approaches should consider both the performance of the existing dense checkpoints and the MoE training budget. We highlight two innov"},"claims":{"count":0,"items":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"2406.06563","kind":"arxiv","version":1},"verdict":{"id":null,"model_set":{},"created_at":null,"strongest_claim":"","one_line_summary":"","pipeline_version":null,"weakest_assumption":"","pith_extraction_headline":""},"integrity":{"clean":true,"summary":{"advisory":0,"critical":0,"by_detector":{},"informational":0},"endpoint":"/pith/2406.06563/integrity.json","findings":[],"available":true,"detectors_run":[],"snapshot_sha256":"c28c3603d3b5d939e8dc4c7e95fa8dfce3d595e45f758748cecf8e644a296938"},"references":{"count":0,"sample":[],"resolved_work":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","internal_anchors":0},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"author_claims":{"count":0,"strong_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"builder_version":"pith-number-builder-2026-05-17-v1"},"verdict_id":null},"signer":{"signer_id":"pith.science","signer_type":"pith_registry","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"created_at":"2026-07-05T08:29:47Z","supersedes":[],"prev_event":null,"signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"3vKmZFQqlYtjkbKmRerdhpZbDXTrPzqbPDAZFPWtFkeqQLr+eEk4heP32AMa666Dntgn0FsxIvI/CmzGn3RWBA==","signed_message":"open_graph_event_sha256_bytes","signed_at":"2026-07-06T10:03:37.582369Z"},"content_sha256":"5bc0800cd95aaf14e84e0627afe71584faacbe6d3f1a356682286b8bb2abfbd7","schema_version":"1.0","event_id":"sha256:5bc0800cd95aaf14e84e0627afe71584faacbe6d3f1a356682286b8bb2abfbd7"}],"timestamp_proofs":[],"mirror_hints":[{"mirror_type":"https","name":"Pith Resolver","base_url":"https://pith.science","bundle_url":"https://pith.science/pith/RW5XONPRALUDJDUGBESKZJT2PU/bundle.json","state_url":"https://pith.science/pith/RW5XONPRALUDJDUGBESKZJT2PU/state.json","well_known_bundle_url":"https://pith.science/.well-known/pith/RW5XONPRALUDJDUGBESKZJT2PU/bundle.json","status":"primary"}],"public_keys":[{"key_id":"pith-v1-2026-05","algorithm":"ed25519","format":"raw","public_key_b64":"stVStoiQhXFxp4s2pdzPNoqVNBMojDU/fJ2db5S3CbM=","public_key_hex":"b2d552b68890857171a78b36a5dccf368a953413288c353f7c9d9d6f94b709b3","fingerprint_sha256_b32_first128bits":"RVFV5Z2OI2J3ZUO7ERDEBCYNKS","fingerprint_sha256_hex":"8d4b5ee74e4693bcd1df2446408b0d54","rotates_at":null,"url":"https://pith.science/pith-signing-key.json","notes":"Pith uses this Ed25519 key to sign canonical record SHA-256 digests. Verify with: ed25519_verify(public_key, message=canonical_sha256_bytes, signature=base64decode(signature_b64))."}],"merge_version":"pith-open-graph-merge-v1","built_at":"2026-07-06T10:03:37Z","links":{"resolver":"https://pith.science/pith/RW5XONPRALUDJDUGBESKZJT2PU","bundle":"https://pith.science/pith/RW5XONPRALUDJDUGBESKZJT2PU/bundle.json","state":"https://pith.science/pith/RW5XONPRALUDJDUGBESKZJT2PU/state.json","well_known_bundle":"https://pith.science/.well-known/pith/RW5XONPRALUDJDUGBESKZJT2PU/bundle.json"},"state":{"state_type":"pith_open_graph_state","state_version":"1.0","pith_number":"pith:2024:RW5XONPRALUDJDUGBESKZJT2PU","merge_version":"pith-open-graph-merge-v1","event_count":2,"valid_event_count":2,"invalid_event_count":0,"equivocation_count":0,"current":{"canonical_record":{"metadata":{"abstract_canon_sha256":"70e6a03e57a3b2cb107beef0010d4d9d631c63401d32aa1285188fe444594fa1","cross_cats_sorted":["cs.AI"],"license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.CL","submitted_at":"2024-06-03T03:58:41Z","title_canon_sha256":"19be60bf4420a9da3c11bd67d528d09e86c9af4e75b575b31331cdc395641525"},"schema_version":"1.0","source":{"id":"2406.06563","kind":"arxiv","version":1}},"source_aliases":[{"alias_kind":"arxiv","alias_value":"2406.06563","created_at":"2026-07-05T08:29:47Z"},{"alias_kind":"arxiv_version","alias_value":"2406.06563v1","created_at":"2026-07-05T08:29:47Z"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.2406.06563","created_at":"2026-07-05T08:29:47Z"},{"alias_kind":"pith_short_12","alias_value":"RW5XONPRALUD","created_at":"2026-07-05T08:29:47Z"},{"alias_kind":"pith_short_16","alias_value":"RW5XONPRALUDJDUG","created_at":"2026-07-05T08:29:47Z"},{"alias_kind":"pith_short_8","alias_value":"RW5XONPR","created_at":"2026-07-05T08:29:47Z"}],"graph_snapshots":[{"event_id":"sha256:5bc0800cd95aaf14e84e0627afe71584faacbe6d3f1a356682286b8bb2abfbd7","target":"graph","created_at":"2026-07-05T08:29:47Z","signer":{"key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signer_id":"pith.science","signer_type":"pith_registry"},"payload":{"graph_snapshot":{"author_claims":{"count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","strong_count":0},"builder_version":"pith-number-builder-2026-05-17-v1","claims":{"count":0,"items":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"integrity":{"available":true,"clean":true,"detectors_run":[],"endpoint":"/pith/2406.06563/integrity.json","findings":[],"snapshot_sha256":"c28c3603d3b5d939e8dc4c7e95fa8dfce3d595e45f758748cecf8e644a296938","summary":{"advisory":0,"by_detector":{},"critical":0,"informational":0}},"paper":{"abstract_excerpt":"In this technical report, we introduce the training methodologies implemented in the development of Skywork-MoE, a high-performance mixture-of-experts (MoE) large language model (LLM) with 146 billion parameters and 16 experts. It is initialized from the pre-existing dense checkpoints of our Skywork-13B model. We explore the comparative effectiveness of upcycling versus training from scratch initializations. Our findings suggest that the choice between these two approaches should consider both the performance of the existing dense checkpoints and the MoE training budget. We highlight two innov","authors_text":"Biye Li, Bo Zhu, Cheng cheng, Han Fang, Jianhao Zhang, Liang Zeng, Liang Zhao, Peng Cheng, Rui Hu, Shuicheng Yan, Tianwen Wei, Weiwei L\\\"u, Xiaokun Wang, Xiaoyu Zhang, Yahui Zhou, Yutuan Ma","cross_cats":["cs.AI"],"headline":"","license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.CL","submitted_at":"2024-06-03T03:58:41Z","title":"Skywork-MoE: A Deep Dive into Training Techniques for Mixture-of-Experts Language Models"},"references":{"count":0,"internal_anchors":0,"resolved_work":0,"sample":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"2406.06563","kind":"arxiv","version":1},"verdict":{"created_at":null,"id":null,"model_set":{},"one_line_summary":"","pipeline_version":null,"pith_extraction_headline":"","strongest_claim":"","weakest_assumption":""}},"verdict_id":null}}],"author_attestations":[],"timestamp_anchors":[],"storage_attestations":[],"citation_signatures":[],"replication_records":[],"corrections":[],"mirror_hints":[],"record_created":{"event_id":"sha256:b69ee5c7c319338e10038a44263597502078cf7988a931f570fb24d242e604f6","target":"record","created_at":"2026-07-05T08:29:47Z","signer":{"key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signer_id":"pith.science","signer_type":"pith_registry"},"payload":{"attestation_state":"computed","canonical_record":{"metadata":{"abstract_canon_sha256":"70e6a03e57a3b2cb107beef0010d4d9d631c63401d32aa1285188fe444594fa1","cross_cats_sorted":["cs.AI"],"license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.CL","submitted_at":"2024-06-03T03:58:41Z","title_canon_sha256":"19be60bf4420a9da3c11bd67d528d09e86c9af4e75b575b31331cdc395641525"},"schema_version":"1.0","source":{"id":"2406.06563","kind":"arxiv","version":1}},"canonical_sha256":"8dbb7735f102e8348e860924aca67a7d08187c5ca7fa346374c42620a351bd02","receipt":{"algorithm":"ed25519","builder_version":"pith-number-builder-2026-05-17-v1","canonical_sha256":"8dbb7735f102e8348e860924aca67a7d08187c5ca7fa346374c42620a351bd02","first_computed_at":"2026-07-05T08:29:47.523700Z","key_id":"pith-v1-2026-05","kind":"pith_receipt","last_reissued_at":"2026-07-05T08:29:47.523700Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","receipt_version":"0.3","signature_b64":"dFGZVznwsekFi5V0aWqubTB1yPPl/STRXXQ0G7Uj6xFY8d2YTfkaUayK5vYvtzlFMwMR3dlJ8l/gtPRrxBbGCQ==","signature_status":"signed_v1","signed_at":"2026-07-05T08:29:47.524159Z","signed_message":"canonical_sha256_bytes"},"source_id":"2406.06563","source_kind":"arxiv","source_version":1}}},"equivocations":[],"invalid_events":[],"applied_event_ids":["sha256:b69ee5c7c319338e10038a44263597502078cf7988a931f570fb24d242e604f6","sha256:5bc0800cd95aaf14e84e0627afe71584faacbe6d3f1a356682286b8bb2abfbd7"],"state_sha256":"0f13565c5c4c70f4afa5c62a254d57e45f0c2dc912ff75a3b788ce8b883054d3"},"bundle_signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"EytartvYMpzaCfzg0wYthMSFwNHP/dGFKJmEXbZHOaeZ6XhcLXFqXQTu7nSu3RnhvmrwuS2wxu4DtQFwwfOaDw==","signed_message":"bundle_sha256_bytes","signed_at":"2026-07-06T10:03:37.584336Z","bundle_sha256":"9c8c927405e8fa46d079b00821307e59d4c9e34829acb67babae621af21bdaa6"}}