{"record_type":"pith_number_record","schema_url":"https://pith.science/schemas/pith-number/v1.json","pith_number":"pith:2024:G7L2PM7TEHTWL2BHKMXQVD6NIM","short_pith_number":"pith:G7L2PM7T","schema_version":"1.0","canonical_sha256":"37d7a7b3f321e765e827532f0a8fcd43235945334e55e883b4c0ce03cc1000a3","source":{"kind":"arxiv","id":"2408.15237","version":4},"attestation_state":"computed","paper":{"title":"The Mamba in the Llama: Distilling and Accelerating Hybrid Models","license":"http://creativecommons.org/licenses/by/4.0/","headline":"","cross_cats":["cs.AI"],"primary_cat":"cs.LG","authors_text":"Alexander M. Rush, Avner May, Daniele Paliotta, Junxiong Wang, Tri Dao","submitted_at":"2024-08-27T17:56:11Z","abstract_excerpt":"Linear RNN architectures, like Mamba, can be competitive with Transformer models in language modeling while having advantageous deployment characteristics. Given the focus on training large-scale Transformer models, we consider the challenge of converting these pretrained models for deployment. We demonstrate that it is feasible to distill large Transformers into linear RNNs by reusing the linear projection weights from attention layers with academic GPU resources. The resulting hybrid model, which incorporates a quarter of the attention layers, achieves performance comparable to the original "},"verification_status":{"content_addressed":true,"pith_receipt":true,"author_attested":false,"weak_author_claims":0,"strong_author_claims":0,"externally_anchored":false,"storage_verified":false,"citation_signatures":0,"replication_records":0,"graph_snapshot":true,"references_resolved":false,"formal_links_present":false},"canonical_record":{"source":{"id":"2408.15237","kind":"arxiv","version":4},"metadata":{"license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.LG","submitted_at":"2024-08-27T17:56:11Z","cross_cats_sorted":["cs.AI"],"title_canon_sha256":"28ca1f1dd53655a90600a368bdb24ba0989f752357a41ce19f2507a29135ca99","abstract_canon_sha256":"1dab2ab50dad1e7fbe7b63f3bfeaaed867c5d3cd839ba40a3f6553ca04dd579a"},"schema_version":"1.0"},"receipt":{"kind":"pith_receipt","key_id":"pith-v1-2026-05","algorithm":"ed25519","signed_at":"2026-07-05T11:28:11.101128Z","signature_b64":"ufaL+UFOMkFW3lC0k/mhxtmUN9iyH62w+fpJZRRHNlpZBAgS8LnbKeYE3NEE1fTiSWmbEZE5kvTroNfSVh9PDQ==","signed_message":"canonical_sha256_bytes","builder_version":"pith-number-builder-2026-05-17-v1","receipt_version":"0.3","canonical_sha256":"37d7a7b3f321e765e827532f0a8fcd43235945334e55e883b4c0ce03cc1000a3","last_reissued_at":"2026-07-05T11:28:11.100606Z","signature_status":"signed_v1","first_computed_at":"2026-07-05T11:28:11.100606Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"graph_snapshot":{"paper":{"title":"The Mamba in the Llama: Distilling and Accelerating Hybrid Models","license":"http://creativecommons.org/licenses/by/4.0/","headline":"","cross_cats":["cs.AI"],"primary_cat":"cs.LG","authors_text":"Alexander M. Rush, Avner May, Daniele Paliotta, Junxiong Wang, Tri Dao","submitted_at":"2024-08-27T17:56:11Z","abstract_excerpt":"Linear RNN architectures, like Mamba, can be competitive with Transformer models in language modeling while having advantageous deployment characteristics. Given the focus on training large-scale Transformer models, we consider the challenge of converting these pretrained models for deployment. We demonstrate that it is feasible to distill large Transformers into linear RNNs by reusing the linear projection weights from attention layers with academic GPU resources. The resulting hybrid model, which incorporates a quarter of the attention layers, achieves performance comparable to the original "},"claims":{"count":0,"items":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"2408.15237","kind":"arxiv","version":4},"verdict":{"id":null,"model_set":{},"created_at":null,"strongest_claim":"","one_line_summary":"","pipeline_version":null,"weakest_assumption":"","pith_extraction_headline":""},"integrity":{"clean":true,"summary":{"advisory":0,"critical":0,"by_detector":{},"informational":0},"endpoint":"/pith/2408.15237/integrity.json","findings":[],"available":true,"detectors_run":[],"snapshot_sha256":"c28c3603d3b5d939e8dc4c7e95fa8dfce3d595e45f758748cecf8e644a296938"},"references":{"count":0,"sample":[],"resolved_work":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","internal_anchors":0},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"author_claims":{"count":0,"strong_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"builder_version":"pith-number-builder-2026-05-17-v1"},"aliases":[{"alias_kind":"arxiv","alias_value":"2408.15237","created_at":"2026-07-05T11:28:11.100669+00:00"},{"alias_kind":"arxiv_version","alias_value":"2408.15237v4","created_at":"2026-07-05T11:28:11.100669+00:00"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.2408.15237","created_at":"2026-07-05T11:28:11.100669+00:00"},{"alias_kind":"pith_short_12","alias_value":"G7L2PM7TEHTW","created_at":"2026-07-05T11:28:11.100669+00:00"},{"alias_kind":"pith_short_16","alias_value":"G7L2PM7TEHTWL2BH","created_at":"2026-07-05T11:28:11.100669+00:00"},{"alias_kind":"pith_short_8","alias_value":"G7L2PM7T","created_at":"2026-07-05T11:28:11.100669+00:00"}],"events":[],"event_summary":{},"paper_claims":[],"inbound_citations":{"count":6,"internal_anchor_count":0,"sample":[{"citing_arxiv_id":"2606.06479","citing_title":"Pretraining Recurrent Networks without Recurrence","ref_index":131,"is_internal_anchor":false},{"citing_arxiv_id":"2410.13846","citing_title":"LightTransfer: Your Long-Context LLM is Secretly a Hybrid Model with Effortless Adaptation","ref_index":33,"is_internal_anchor":false},{"citing_arxiv_id":"2410.17891","citing_title":"Scaling Diffusion Language Models via Adaptation from Autoregressive Models","ref_index":188,"is_internal_anchor":false},{"citing_arxiv_id":"2502.13189","citing_title":"MoBA: Mixture of Block Attention for Long-Context LLMs","ref_index":54,"is_internal_anchor":false},{"citing_arxiv_id":"2604.14191","citing_title":"Attention to Mamba: A Recipe for Cross-Architecture Distillation","ref_index":31,"is_internal_anchor":false},{"citing_arxiv_id":"2605.01048","citing_title":"Compared to What? Baselines and Metrics for Counterfactual Prompting","ref_index":85,"is_internal_anchor":false}]},"formal_canon":{"evidence_count":0,"sample":[],"anchors":[]},"links":{"html":"https://pith.science/pith/G7L2PM7TEHTWL2BHKMXQVD6NIM","json":"https://pith.science/pith/G7L2PM7TEHTWL2BHKMXQVD6NIM.json","graph_json":"https://pith.science/api/pith-number/G7L2PM7TEHTWL2BHKMXQVD6NIM/graph.json","events_json":"https://pith.science/api/pith-number/G7L2PM7TEHTWL2BHKMXQVD6NIM/events.json","paper":"https://pith.science/paper/G7L2PM7T"},"agent_actions":{"view_html":"https://pith.science/pith/G7L2PM7TEHTWL2BHKMXQVD6NIM","download_json":"https://pith.science/pith/G7L2PM7TEHTWL2BHKMXQVD6NIM.json","view_paper":"https://pith.science/paper/G7L2PM7T","resolve_alias":"https://pith.science/api/pith-number/resolve?arxiv=2408.15237&json=true","fetch_graph":"https://pith.science/api/pith-number/G7L2PM7TEHTWL2BHKMXQVD6NIM/graph.json","fetch_events":"https://pith.science/api/pith-number/G7L2PM7TEHTWL2BHKMXQVD6NIM/events.json","actions":{"anchor_timestamp":"https://pith.science/pith/G7L2PM7TEHTWL2BHKMXQVD6NIM/action/timestamp_anchor","attest_storage":"https://pith.science/pith/G7L2PM7TEHTWL2BHKMXQVD6NIM/action/storage_attestation","attest_author":"https://pith.science/pith/G7L2PM7TEHTWL2BHKMXQVD6NIM/action/author_attestation","sign_citation":"https://pith.science/pith/G7L2PM7TEHTWL2BHKMXQVD6NIM/action/citation_signature","submit_replication":"https://pith.science/pith/G7L2PM7TEHTWL2BHKMXQVD6NIM/action/replication_record"}},"created_at":"2026-07-05T11:28:11.100669+00:00","updated_at":"2026-07-05T11:28:11.100669+00:00"}