{"bundle_type":"pith_open_graph_bundle","bundle_version":"1.0","pith_number":"pith:2024:LUU4NGDJYCFK4A4QTHVPEZKG3M","short_pith_number":"pith:LUU4NGDJ","canonical_record":{"source":{"id":"2410.07524","kind":"arxiv","version":2},"metadata":{"license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","primary_cat":"cs.CL","submitted_at":"2024-10-10T01:36:03Z","cross_cats_sorted":["cs.AI","cs.LG"],"title_canon_sha256":"71acd7b0c7e3c19ade3d4443e21543799523a60e8cbb7858ba625f601bb7d9bc","abstract_canon_sha256":"7e8c808f788e551fb5920ec128e1bc6a1baaa9e41548ebe07b3ba35a72ceb608"},"schema_version":"1.0"},"canonical_sha256":"5d29c69869c08aae039099eaf26546db093bf6121b61e136c8f87da12500f2ed","source":{"kind":"arxiv","id":"2410.07524","version":2},"source_aliases":[{"alias_kind":"arxiv","alias_value":"2410.07524","created_at":"2026-07-05T11:21:36Z"},{"alias_kind":"arxiv_version","alias_value":"2410.07524v2","created_at":"2026-07-05T11:21:36Z"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.2410.07524","created_at":"2026-07-05T11:21:36Z"},{"alias_kind":"pith_short_12","alias_value":"LUU4NGDJYCFK","created_at":"2026-07-05T11:21:36Z"},{"alias_kind":"pith_short_16","alias_value":"LUU4NGDJYCFK4A4Q","created_at":"2026-07-05T11:21:36Z"},{"alias_kind":"pith_short_8","alias_value":"LUU4NGDJ","created_at":"2026-07-05T11:21:36Z"}],"events":[{"event_type":"record_created","subject_pith_number":"pith:2024:LUU4NGDJYCFK4A4QTHVPEZKG3M","target":"record","payload":{"canonical_record":{"source":{"id":"2410.07524","kind":"arxiv","version":2},"metadata":{"license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","primary_cat":"cs.CL","submitted_at":"2024-10-10T01:36:03Z","cross_cats_sorted":["cs.AI","cs.LG"],"title_canon_sha256":"71acd7b0c7e3c19ade3d4443e21543799523a60e8cbb7858ba625f601bb7d9bc","abstract_canon_sha256":"7e8c808f788e551fb5920ec128e1bc6a1baaa9e41548ebe07b3ba35a72ceb608"},"schema_version":"1.0"},"canonical_sha256":"5d29c69869c08aae039099eaf26546db093bf6121b61e136c8f87da12500f2ed","receipt":{"kind":"pith_receipt","key_id":"pith-v1-2026-05","algorithm":"ed25519","signed_at":"2026-07-05T11:21:36.784668Z","signature_b64":"VNCY9X1nY0mA9QBTuN4599q+tkEH/fVCqzbS7OJ+WsGIhczqqnXeGQjcTTaukAMhsw/6XQkn/7jAI9HQmEc5BQ==","signed_message":"canonical_sha256_bytes","builder_version":"pith-number-builder-2026-05-17-v1","receipt_version":"0.3","canonical_sha256":"5d29c69869c08aae039099eaf26546db093bf6121b61e136c8f87da12500f2ed","last_reissued_at":"2026-07-05T11:21:36.784186Z","signature_status":"signed_v1","first_computed_at":"2026-07-05T11:21:36.784186Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"source_kind":"arxiv","source_id":"2410.07524","source_version":2,"attestation_state":"computed"},"signer":{"signer_id":"pith.science","signer_type":"pith_registry","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"created_at":"2026-07-05T11:21:36Z","supersedes":[],"prev_event":null,"signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"YxWM+fLc7naFkNnqLPozz7GvUaQ4Gg5SrglH3GiAV+VgnYFttfjifInzWWeG6cxT8fQWNm7HBBS8pjlWZXd9Ag==","signed_message":"open_graph_event_sha256_bytes","signed_at":"2026-08-04T11:59:41.972731Z"},"content_sha256":"9e8a897054273f459c5f3474a5530e6cedeb0e282da71a42b633af22a96fabe7","schema_version":"1.0","event_id":"sha256:9e8a897054273f459c5f3474a5530e6cedeb0e282da71a42b633af22a96fabe7"},{"event_type":"graph_snapshot","subject_pith_number":"pith:2024:LUU4NGDJYCFK4A4QTHVPEZKG3M","target":"graph","payload":{"graph_snapshot":{"paper":{"title":"Upcycling Large Language Models into Mixture of Experts","license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","headline":"","cross_cats":["cs.AI","cs.LG"],"primary_cat":"cs.CL","authors_text":"Abhinav Khattar, Ashwath Aithal, Bryan Catanzaro, Ethan He, Mohammad Shoeybi, Ryan Prenger, Shiqing Fan, Tong Liu, Vijay Korthikanti, Zijie Yan","submitted_at":"2024-10-10T01:36:03Z","abstract_excerpt":"Upcycling pre-trained dense language models into sparse mixture-of-experts (MoE) models is an efficient approach to increase the model capacity of already trained models. However, optimal techniques for upcycling at scale remain unclear. In this work, we conduct an extensive study of upcycling methods and hyperparameters for billion-parameter scale language models. We propose a novel \"virtual group\" initialization scheme and weight scaling approach to enable upcycling into fine-grained MoE architectures. Through ablations, we find that upcycling outperforms continued dense model training. In a"},"claims":{"count":0,"items":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"2410.07524","kind":"arxiv","version":2},"verdict":{"id":null,"model_set":{},"created_at":null,"strongest_claim":"","one_line_summary":"","pipeline_version":null,"weakest_assumption":"","pith_extraction_headline":""},"integrity":{"clean":true,"summary":{"advisory":0,"critical":0,"by_detector":{},"informational":0},"endpoint":"/pith/2410.07524/integrity.json","findings":[],"available":true,"detectors_run":[],"snapshot_sha256":"c28c3603d3b5d939e8dc4c7e95fa8dfce3d595e45f758748cecf8e644a296938"},"references":{"count":0,"sample":[],"resolved_work":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","internal_anchors":0},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"author_claims":{"count":0,"strong_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"builder_version":"pith-number-builder-2026-05-17-v1"},"verdict_id":null},"signer":{"signer_id":"pith.science","signer_type":"pith_registry","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"created_at":"2026-07-05T11:21:36Z","supersedes":[],"prev_event":null,"signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"5dI0zfXL/dpQTh7iN3dushtCPUz+MnkNjM9X2CA7SS9WXMTO2e8sMQf+0kaAmrDv3YKQjtpwQP5O6aw/B9AyCg==","signed_message":"open_graph_event_sha256_bytes","signed_at":"2026-08-04T11:59:41.973235Z"},"content_sha256":"ff7e7a9468ae922cc49315d9dc459881b4e5f09d839fe5de6cd39fbb2383dc1b","schema_version":"1.0","event_id":"sha256:ff7e7a9468ae922cc49315d9dc459881b4e5f09d839fe5de6cd39fbb2383dc1b"}],"timestamp_proofs":[],"mirror_hints":[{"mirror_type":"https","name":"Pith Resolver","base_url":"https://pith.science","bundle_url":"https://pith.science/pith/LUU4NGDJYCFK4A4QTHVPEZKG3M/bundle.json","state_url":"https://pith.science/pith/LUU4NGDJYCFK4A4QTHVPEZKG3M/state.json","well_known_bundle_url":"https://pith.science/.well-known/pith/LUU4NGDJYCFK4A4QTHVPEZKG3M/bundle.json","status":"primary"}],"public_keys":[{"key_id":"pith-v1-2026-05","algorithm":"ed25519","format":"raw","public_key_b64":"stVStoiQhXFxp4s2pdzPNoqVNBMojDU/fJ2db5S3CbM=","public_key_hex":"b2d552b68890857171a78b36a5dccf368a953413288c353f7c9d9d6f94b709b3","fingerprint_sha256_b32_first128bits":"RVFV5Z2OI2J3ZUO7ERDEBCYNKS","fingerprint_sha256_hex":"8d4b5ee74e4693bcd1df2446408b0d54","rotates_at":null,"url":"https://pith.science/pith-signing-key.json","notes":"Pith uses this Ed25519 key to sign canonical record SHA-256 digests. Verify with: ed25519_verify(public_key, message=canonical_sha256_bytes, signature=base64decode(signature_b64))."}],"merge_version":"pith-open-graph-merge-v1","built_at":"2026-08-04T11:59:41Z","links":{"resolver":"https://pith.science/pith/LUU4NGDJYCFK4A4QTHVPEZKG3M","bundle":"https://pith.science/pith/LUU4NGDJYCFK4A4QTHVPEZKG3M/bundle.json","state":"https://pith.science/pith/LUU4NGDJYCFK4A4QTHVPEZKG3M/state.json","well_known_bundle":"https://pith.science/.well-known/pith/LUU4NGDJYCFK4A4QTHVPEZKG3M/bundle.json"},"state":{"state_type":"pith_open_graph_state","state_version":"1.0","pith_number":"pith:2024:LUU4NGDJYCFK4A4QTHVPEZKG3M","merge_version":"pith-open-graph-merge-v1","event_count":2,"valid_event_count":2,"invalid_event_count":0,"equivocation_count":0,"current":{"canonical_record":{"metadata":{"abstract_canon_sha256":"7e8c808f788e551fb5920ec128e1bc6a1baaa9e41548ebe07b3ba35a72ceb608","cross_cats_sorted":["cs.AI","cs.LG"],"license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","primary_cat":"cs.CL","submitted_at":"2024-10-10T01:36:03Z","title_canon_sha256":"71acd7b0c7e3c19ade3d4443e21543799523a60e8cbb7858ba625f601bb7d9bc"},"schema_version":"1.0","source":{"id":"2410.07524","kind":"arxiv","version":2}},"source_aliases":[{"alias_kind":"arxiv","alias_value":"2410.07524","created_at":"2026-07-05T11:21:36Z"},{"alias_kind":"arxiv_version","alias_value":"2410.07524v2","created_at":"2026-07-05T11:21:36Z"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.2410.07524","created_at":"2026-07-05T11:21:36Z"},{"alias_kind":"pith_short_12","alias_value":"LUU4NGDJYCFK","created_at":"2026-07-05T11:21:36Z"},{"alias_kind":"pith_short_16","alias_value":"LUU4NGDJYCFK4A4Q","created_at":"2026-07-05T11:21:36Z"},{"alias_kind":"pith_short_8","alias_value":"LUU4NGDJ","created_at":"2026-07-05T11:21:36Z"}],"graph_snapshots":[{"event_id":"sha256:ff7e7a9468ae922cc49315d9dc459881b4e5f09d839fe5de6cd39fbb2383dc1b","target":"graph","created_at":"2026-07-05T11:21:36Z","signer":{"key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signer_id":"pith.science","signer_type":"pith_registry"},"payload":{"graph_snapshot":{"author_claims":{"count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","strong_count":0},"builder_version":"pith-number-builder-2026-05-17-v1","claims":{"count":0,"items":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"integrity":{"available":true,"clean":true,"detectors_run":[],"endpoint":"/pith/2410.07524/integrity.json","findings":[],"snapshot_sha256":"c28c3603d3b5d939e8dc4c7e95fa8dfce3d595e45f758748cecf8e644a296938","summary":{"advisory":0,"by_detector":{},"critical":0,"informational":0}},"paper":{"abstract_excerpt":"Upcycling pre-trained dense language models into sparse mixture-of-experts (MoE) models is an efficient approach to increase the model capacity of already trained models. However, optimal techniques for upcycling at scale remain unclear. In this work, we conduct an extensive study of upcycling methods and hyperparameters for billion-parameter scale language models. We propose a novel \"virtual group\" initialization scheme and weight scaling approach to enable upcycling into fine-grained MoE architectures. Through ablations, we find that upcycling outperforms continued dense model training. In a","authors_text":"Abhinav Khattar, Ashwath Aithal, Bryan Catanzaro, Ethan He, Mohammad Shoeybi, Ryan Prenger, Shiqing Fan, Tong Liu, Vijay Korthikanti, Zijie Yan","cross_cats":["cs.AI","cs.LG"],"headline":"","license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","primary_cat":"cs.CL","submitted_at":"2024-10-10T01:36:03Z","title":"Upcycling Large Language Models into Mixture of Experts"},"references":{"count":0,"internal_anchors":0,"resolved_work":0,"sample":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"2410.07524","kind":"arxiv","version":2},"verdict":{"created_at":null,"id":null,"model_set":{},"one_line_summary":"","pipeline_version":null,"pith_extraction_headline":"","strongest_claim":"","weakest_assumption":""}},"verdict_id":null}}],"author_attestations":[],"timestamp_anchors":[],"storage_attestations":[],"citation_signatures":[],"replication_records":[],"corrections":[],"mirror_hints":[],"record_created":{"event_id":"sha256:9e8a897054273f459c5f3474a5530e6cedeb0e282da71a42b633af22a96fabe7","target":"record","created_at":"2026-07-05T11:21:36Z","signer":{"key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signer_id":"pith.science","signer_type":"pith_registry"},"payload":{"attestation_state":"computed","canonical_record":{"metadata":{"abstract_canon_sha256":"7e8c808f788e551fb5920ec128e1bc6a1baaa9e41548ebe07b3ba35a72ceb608","cross_cats_sorted":["cs.AI","cs.LG"],"license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","primary_cat":"cs.CL","submitted_at":"2024-10-10T01:36:03Z","title_canon_sha256":"71acd7b0c7e3c19ade3d4443e21543799523a60e8cbb7858ba625f601bb7d9bc"},"schema_version":"1.0","source":{"id":"2410.07524","kind":"arxiv","version":2}},"canonical_sha256":"5d29c69869c08aae039099eaf26546db093bf6121b61e136c8f87da12500f2ed","receipt":{"algorithm":"ed25519","builder_version":"pith-number-builder-2026-05-17-v1","canonical_sha256":"5d29c69869c08aae039099eaf26546db093bf6121b61e136c8f87da12500f2ed","first_computed_at":"2026-07-05T11:21:36.784186Z","key_id":"pith-v1-2026-05","kind":"pith_receipt","last_reissued_at":"2026-07-05T11:21:36.784186Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","receipt_version":"0.3","signature_b64":"VNCY9X1nY0mA9QBTuN4599q+tkEH/fVCqzbS7OJ+WsGIhczqqnXeGQjcTTaukAMhsw/6XQkn/7jAI9HQmEc5BQ==","signature_status":"signed_v1","signed_at":"2026-07-05T11:21:36.784668Z","signed_message":"canonical_sha256_bytes"},"source_id":"2410.07524","source_kind":"arxiv","source_version":2}}},"equivocations":[],"invalid_events":[],"applied_event_ids":["sha256:9e8a897054273f459c5f3474a5530e6cedeb0e282da71a42b633af22a96fabe7","sha256:ff7e7a9468ae922cc49315d9dc459881b4e5f09d839fe5de6cd39fbb2383dc1b"],"state_sha256":"eeb2d3ae49f78cf23886fdb5082e7b6c1fdbe923883b3f52ebf00b7573967453"},"bundle_signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"icONQprMfdCAL9qRrC+RoO6XxcuzFfgd7aG0r8zTGj+schawI3ISm5LaIcr/LqqzXQ6dB/vBPoD0to6KoSkDAg==","signed_message":"bundle_sha256_bytes","signed_at":"2026-08-04T11:59:41.977132Z","bundle_sha256":"dbc0e434affc0524ca2b90dae31878d09217469f41585882a1fa1c9a5aa302f0"}}