{"bundle_type":"pith_open_graph_bundle","bundle_version":"1.0","pith_number":"pith:2021:PLYSZZHUAB6KZHDWA6ZVTWCRH3","short_pith_number":"pith:PLYSZZHU","canonical_record":{"source":{"id":"2112.10684","kind":"arxiv","version":2},"metadata":{"license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","primary_cat":"cs.CL","submitted_at":"2021-12-20T17:05:11Z","cross_cats_sorted":["cs.AI","cs.LG"],"title_canon_sha256":"dae3d138b54c45c136dd76eda2af00f608e89d5d1b10701e371cb92a60385378","abstract_canon_sha256":"867c077d61e703cdc2346fb740f101cb4a90bf1171acf2015afb1b5f8940527c"},"schema_version":"1.0"},"canonical_sha256":"7af12ce4f4007cac9c7607b359d8513ef322b3faab86d43478ba922aeb80594e","source":{"kind":"arxiv","id":"2112.10684","version":2},"source_aliases":[{"alias_kind":"arxiv","alias_value":"2112.10684","created_at":"2026-07-05T05:10:35Z"},{"alias_kind":"arxiv_version","alias_value":"2112.10684v2","created_at":"2026-07-05T05:10:35Z"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.2112.10684","created_at":"2026-07-05T05:10:35Z"},{"alias_kind":"pith_short_12","alias_value":"PLYSZZHUAB6K","created_at":"2026-07-05T05:10:35Z"},{"alias_kind":"pith_short_16","alias_value":"PLYSZZHUAB6KZHDW","created_at":"2026-07-05T05:10:35Z"},{"alias_kind":"pith_short_8","alias_value":"PLYSZZHU","created_at":"2026-07-05T05:10:35Z"}],"events":[{"event_type":"record_created","subject_pith_number":"pith:2021:PLYSZZHUAB6KZHDWA6ZVTWCRH3","target":"record","payload":{"canonical_record":{"source":{"id":"2112.10684","kind":"arxiv","version":2},"metadata":{"license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","primary_cat":"cs.CL","submitted_at":"2021-12-20T17:05:11Z","cross_cats_sorted":["cs.AI","cs.LG"],"title_canon_sha256":"dae3d138b54c45c136dd76eda2af00f608e89d5d1b10701e371cb92a60385378","abstract_canon_sha256":"867c077d61e703cdc2346fb740f101cb4a90bf1171acf2015afb1b5f8940527c"},"schema_version":"1.0"},"canonical_sha256":"7af12ce4f4007cac9c7607b359d8513ef322b3faab86d43478ba922aeb80594e","receipt":{"kind":"pith_receipt","key_id":"pith-v1-2026-05","algorithm":"ed25519","signed_at":"2026-07-05T05:10:35.746925Z","signature_b64":"OlZrkVVbFu7OAu/xPRwrtcOfFiiMfsddvY7jG1jEguCeLadTSzi65Yy8kh3FhtHJerwVO4njapjk2D1ijvE9DA==","signed_message":"canonical_sha256_bytes","builder_version":"pith-number-builder-2026-05-17-v1","receipt_version":"0.3","canonical_sha256":"7af12ce4f4007cac9c7607b359d8513ef322b3faab86d43478ba922aeb80594e","last_reissued_at":"2026-07-05T05:10:35.746444Z","signature_status":"signed_v1","first_computed_at":"2026-07-05T05:10:35.746444Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"source_kind":"arxiv","source_id":"2112.10684","source_version":2,"attestation_state":"computed"},"signer":{"signer_id":"pith.science","signer_type":"pith_registry","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"created_at":"2026-07-05T05:10:35Z","supersedes":[],"prev_event":null,"signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"egs/IqzyWzMXU3s6fsdem22W3yu5yraqK5cVT+gbTl7WGFH99tJnHZBB8ycTw8EiR6r4T2PlgEAjRmesuPofAw==","signed_message":"open_graph_event_sha256_bytes","signed_at":"2026-08-18T23:51:23.396158Z"},"content_sha256":"d35fa323d6296d575577e105873262f78d9a97870b2c9febfb6957fa0ffa0ec2","schema_version":"1.0","event_id":"sha256:d35fa323d6296d575577e105873262f78d9a97870b2c9febfb6957fa0ffa0ec2"},{"event_type":"graph_snapshot","subject_pith_number":"pith:2021:PLYSZZHUAB6KZHDWA6ZVTWCRH3","target":"graph","payload":{"graph_snapshot":{"paper":{"title":"Efficient Large Scale Language Modeling with Mixtures of Experts","license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","headline":"","cross_cats":["cs.AI","cs.LG"],"primary_cat":"cs.CL","authors_text":"Brian O'Horo, Giri Anantharaman, Halil Akin, Jeff Wang, Jingfei Du, Louis Martin, Luke Zettlemoyer, Mandeep Baines, Mikel Artetxe, Mona Diab, Myle Ott, Naman Goyal, Punit Singh Koura, Ramakanth Pasunuru, Sam Shleifer, Shruti Bhosale, Shuohui Chen, Srinivasan Iyer, Todor Mihaylov, Ves Stoyanov, Xian Li, Xing Zhou, Xi Victoria Lin, Zornitsa Kozareva","submitted_at":"2021-12-20T17:05:11Z","abstract_excerpt":"Mixture of Experts layers (MoEs) enable efficient scaling of language models through conditional computation. This paper presents a detailed empirical study of how autoregressive MoE language models scale in comparison with dense models in a wide range of settings: in- and out-of-domain language modeling, zero- and few-shot priming, and full-shot fine-tuning. With the exception of fine-tuning, we find MoEs to be substantially more compute efficient. At more modest training budgets, MoEs can match the performance of dense models using $\\sim$4 times less compute. This gap narrows at scale, but o"},"claims":{"count":0,"items":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"2112.10684","kind":"arxiv","version":2},"verdict":{"id":null,"model_set":{},"created_at":null,"strongest_claim":"","one_line_summary":"","pipeline_version":null,"weakest_assumption":"","pith_extraction_headline":""},"integrity":{"clean":true,"summary":{"advisory":0,"critical":0,"by_detector":{},"informational":0},"endpoint":"/pith/2112.10684/integrity.json","findings":[],"available":true,"detectors_run":[],"snapshot_sha256":"c28c3603d3b5d939e8dc4c7e95fa8dfce3d595e45f758748cecf8e644a296938"},"references":{"count":0,"sample":[],"resolved_work":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","internal_anchors":0},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"author_claims":{"count":0,"strong_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"builder_version":"pith-number-builder-2026-05-17-v1"},"verdict_id":null},"signer":{"signer_id":"pith.science","signer_type":"pith_registry","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"created_at":"2026-07-05T05:10:35Z","supersedes":[],"prev_event":null,"signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"ztwmxcrojaTHqpMebtVRzcbdsn4KShKx/h1kVjpZIsRrHJdtBYzW1PEyHbTNkob9IcqbXDVcxH4pdOFX4SVIBQ==","signed_message":"open_graph_event_sha256_bytes","signed_at":"2026-08-18T23:51:23.396897Z"},"content_sha256":"a9e77152226d3f7a7d0a05a7b8ffb0b2883a05c4e5f119664317bad66e060360","schema_version":"1.0","event_id":"sha256:a9e77152226d3f7a7d0a05a7b8ffb0b2883a05c4e5f119664317bad66e060360"}],"timestamp_proofs":[],"mirror_hints":[{"mirror_type":"https","name":"Pith Resolver","base_url":"https://pith.science","bundle_url":"https://pith.science/pith/PLYSZZHUAB6KZHDWA6ZVTWCRH3/bundle.json","state_url":"https://pith.science/pith/PLYSZZHUAB6KZHDWA6ZVTWCRH3/state.json","well_known_bundle_url":"https://pith.science/.well-known/pith/PLYSZZHUAB6KZHDWA6ZVTWCRH3/bundle.json","status":"primary"}],"public_keys":[{"key_id":"pith-v1-2026-05","algorithm":"ed25519","format":"raw","public_key_b64":"stVStoiQhXFxp4s2pdzPNoqVNBMojDU/fJ2db5S3CbM=","public_key_hex":"b2d552b68890857171a78b36a5dccf368a953413288c353f7c9d9d6f94b709b3","fingerprint_sha256_b32_first128bits":"RVFV5Z2OI2J3ZUO7ERDEBCYNKS","fingerprint_sha256_hex":"8d4b5ee74e4693bcd1df2446408b0d54","rotates_at":null,"url":"https://pith.science/pith-signing-key.json","notes":"Pith uses this Ed25519 key to sign canonical record SHA-256 digests. Verify with: ed25519_verify(public_key, message=canonical_sha256_bytes, signature=base64decode(signature_b64))."}],"merge_version":"pith-open-graph-merge-v1","built_at":"2026-08-18T23:51:23Z","links":{"resolver":"https://pith.science/pith/PLYSZZHUAB6KZHDWA6ZVTWCRH3","bundle":"https://pith.science/pith/PLYSZZHUAB6KZHDWA6ZVTWCRH3/bundle.json","state":"https://pith.science/pith/PLYSZZHUAB6KZHDWA6ZVTWCRH3/state.json","well_known_bundle":"https://pith.science/.well-known/pith/PLYSZZHUAB6KZHDWA6ZVTWCRH3/bundle.json"},"state":{"state_type":"pith_open_graph_state","state_version":"1.0","pith_number":"pith:2021:PLYSZZHUAB6KZHDWA6ZVTWCRH3","merge_version":"pith-open-graph-merge-v1","event_count":2,"valid_event_count":2,"invalid_event_count":0,"equivocation_count":0,"current":{"canonical_record":{"metadata":{"abstract_canon_sha256":"867c077d61e703cdc2346fb740f101cb4a90bf1171acf2015afb1b5f8940527c","cross_cats_sorted":["cs.AI","cs.LG"],"license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","primary_cat":"cs.CL","submitted_at":"2021-12-20T17:05:11Z","title_canon_sha256":"dae3d138b54c45c136dd76eda2af00f608e89d5d1b10701e371cb92a60385378"},"schema_version":"1.0","source":{"id":"2112.10684","kind":"arxiv","version":2}},"source_aliases":[{"alias_kind":"arxiv","alias_value":"2112.10684","created_at":"2026-07-05T05:10:35Z"},{"alias_kind":"arxiv_version","alias_value":"2112.10684v2","created_at":"2026-07-05T05:10:35Z"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.2112.10684","created_at":"2026-07-05T05:10:35Z"},{"alias_kind":"pith_short_12","alias_value":"PLYSZZHUAB6K","created_at":"2026-07-05T05:10:35Z"},{"alias_kind":"pith_short_16","alias_value":"PLYSZZHUAB6KZHDW","created_at":"2026-07-05T05:10:35Z"},{"alias_kind":"pith_short_8","alias_value":"PLYSZZHU","created_at":"2026-07-05T05:10:35Z"}],"graph_snapshots":[{"event_id":"sha256:a9e77152226d3f7a7d0a05a7b8ffb0b2883a05c4e5f119664317bad66e060360","target":"graph","created_at":"2026-07-05T05:10:35Z","signer":{"key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signer_id":"pith.science","signer_type":"pith_registry"},"payload":{"graph_snapshot":{"author_claims":{"count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","strong_count":0},"builder_version":"pith-number-builder-2026-05-17-v1","claims":{"count":0,"items":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"integrity":{"available":true,"clean":true,"detectors_run":[],"endpoint":"/pith/2112.10684/integrity.json","findings":[],"snapshot_sha256":"c28c3603d3b5d939e8dc4c7e95fa8dfce3d595e45f758748cecf8e644a296938","summary":{"advisory":0,"by_detector":{},"critical":0,"informational":0}},"paper":{"abstract_excerpt":"Mixture of Experts layers (MoEs) enable efficient scaling of language models through conditional computation. This paper presents a detailed empirical study of how autoregressive MoE language models scale in comparison with dense models in a wide range of settings: in- and out-of-domain language modeling, zero- and few-shot priming, and full-shot fine-tuning. With the exception of fine-tuning, we find MoEs to be substantially more compute efficient. At more modest training budgets, MoEs can match the performance of dense models using $\\sim$4 times less compute. This gap narrows at scale, but o","authors_text":"Brian O'Horo, Giri Anantharaman, Halil Akin, Jeff Wang, Jingfei Du, Louis Martin, Luke Zettlemoyer, Mandeep Baines, Mikel Artetxe, Mona Diab, Myle Ott, Naman Goyal, Punit Singh Koura, Ramakanth Pasunuru, Sam Shleifer, Shruti Bhosale, Shuohui Chen, Srinivasan Iyer, Todor Mihaylov, Ves Stoyanov, Xian Li, Xing Zhou, Xi Victoria Lin, Zornitsa Kozareva","cross_cats":["cs.AI","cs.LG"],"headline":"","license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","primary_cat":"cs.CL","submitted_at":"2021-12-20T17:05:11Z","title":"Efficient Large Scale Language Modeling with Mixtures of Experts"},"references":{"count":0,"internal_anchors":0,"resolved_work":0,"sample":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"2112.10684","kind":"arxiv","version":2},"verdict":{"created_at":null,"id":null,"model_set":{},"one_line_summary":"","pipeline_version":null,"pith_extraction_headline":"","strongest_claim":"","weakest_assumption":""}},"verdict_id":null}}],"author_attestations":[],"timestamp_anchors":[],"storage_attestations":[],"citation_signatures":[],"replication_records":[],"corrections":[],"mirror_hints":[],"record_created":{"event_id":"sha256:d35fa323d6296d575577e105873262f78d9a97870b2c9febfb6957fa0ffa0ec2","target":"record","created_at":"2026-07-05T05:10:35Z","signer":{"key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signer_id":"pith.science","signer_type":"pith_registry"},"payload":{"attestation_state":"computed","canonical_record":{"metadata":{"abstract_canon_sha256":"867c077d61e703cdc2346fb740f101cb4a90bf1171acf2015afb1b5f8940527c","cross_cats_sorted":["cs.AI","cs.LG"],"license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","primary_cat":"cs.CL","submitted_at":"2021-12-20T17:05:11Z","title_canon_sha256":"dae3d138b54c45c136dd76eda2af00f608e89d5d1b10701e371cb92a60385378"},"schema_version":"1.0","source":{"id":"2112.10684","kind":"arxiv","version":2}},"canonical_sha256":"7af12ce4f4007cac9c7607b359d8513ef322b3faab86d43478ba922aeb80594e","receipt":{"algorithm":"ed25519","builder_version":"pith-number-builder-2026-05-17-v1","canonical_sha256":"7af12ce4f4007cac9c7607b359d8513ef322b3faab86d43478ba922aeb80594e","first_computed_at":"2026-07-05T05:10:35.746444Z","key_id":"pith-v1-2026-05","kind":"pith_receipt","last_reissued_at":"2026-07-05T05:10:35.746444Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","receipt_version":"0.3","signature_b64":"OlZrkVVbFu7OAu/xPRwrtcOfFiiMfsddvY7jG1jEguCeLadTSzi65Yy8kh3FhtHJerwVO4njapjk2D1ijvE9DA==","signature_status":"signed_v1","signed_at":"2026-07-05T05:10:35.746925Z","signed_message":"canonical_sha256_bytes"},"source_id":"2112.10684","source_kind":"arxiv","source_version":2}}},"equivocations":[],"invalid_events":[],"applied_event_ids":["sha256:d35fa323d6296d575577e105873262f78d9a97870b2c9febfb6957fa0ffa0ec2","sha256:a9e77152226d3f7a7d0a05a7b8ffb0b2883a05c4e5f119664317bad66e060360"],"state_sha256":"326f3a1d445dad2d35280b71cad0758a8a7795ebeca436d259494b60797f568e"},"bundle_signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"ab2N0FyqePMWlDb4rT7hR+oi2Aici+w2JFMIq+bvcdC4sLWnD9mGfUYOmyY3sah8el3ab+hrk40WJrN2Q20LDg==","signed_message":"bundle_sha256_bytes","signed_at":"2026-08-18T23:51:23.403181Z","bundle_sha256":"6b22a101219c27f528b01b3bec2c75a765143f9c1a2778564ca819041ae5daa4"}}