{"record_type":"pith_number_record","schema_url":"https://pith.science/schemas/pith-number/v1.json","pith_number":"pith:2024:NUGXK6Y4YSCE545ILLRI6N2ZGR","short_pith_number":"pith:NUGXK6Y4","schema_version":"1.0","canonical_sha256":"6d0d757b1cc4844ef3a85ae28f37593443d6dfa2b6d9217075eb06fc2e7b7c31","source":{"kind":"arxiv","id":"2409.12517","version":2},"attestation_state":"computed","paper":{"title":"Scaling FP8 training to trillion-token LLMs","license":"http://creativecommons.org/licenses/by-nc-nd/4.0/","headline":"","cross_cats":["cs.AI"],"primary_cat":"cs.LG","authors_text":"Brian Chmiel, Daniel Soudry, Maxim Fishman, Ron Banner","submitted_at":"2024-09-19T07:15:58Z","abstract_excerpt":"We train, for the first time, large language models using FP8 precision on datasets up to 2 trillion tokens -- a 20-fold increase over previous limits. Through these extended training runs, we uncover critical instabilities in FP8 training that were not observable in earlier works with shorter durations. We trace these instabilities to outlier amplification by the SwiGLU activation function. Interestingly, we show, both analytically and empirically, that this amplification happens only over prolonged training periods, and link it to a SwiGLU weight alignment process. To address this newly iden"},"verification_status":{"content_addressed":true,"pith_receipt":true,"author_attested":false,"weak_author_claims":0,"strong_author_claims":0,"externally_anchored":false,"storage_verified":false,"citation_signatures":0,"replication_records":0,"graph_snapshot":true,"references_resolved":false,"formal_links_present":false},"canonical_record":{"source":{"id":"2409.12517","kind":"arxiv","version":2},"metadata":{"license":"http://creativecommons.org/licenses/by-nc-nd/4.0/","primary_cat":"cs.LG","submitted_at":"2024-09-19T07:15:58Z","cross_cats_sorted":["cs.AI"],"title_canon_sha256":"9a1a7ec3f7c6beb04465ea9b3a27caf24513566228e1ed75f298132dd59f7611","abstract_canon_sha256":"9d934df82c6b51ac1b5c36f22290ff649864a57d70fa5ced999bb575b0be72aa"},"schema_version":"1.0"},"receipt":{"kind":"pith_receipt","key_id":"pith-v1-2026-05","algorithm":"ed25519","signed_at":"2026-07-05T10:11:24.427284Z","signature_b64":"jrbWFqPR0aOEZc/wF1H/uG4hFNJxb1ia4+IptCU4jT5TTqkidd04qZKAxpE0SfmazvzEByF+1cc+uWRdIPx6Ag==","signed_message":"canonical_sha256_bytes","builder_version":"pith-number-builder-2026-05-17-v1","receipt_version":"0.3","canonical_sha256":"6d0d757b1cc4844ef3a85ae28f37593443d6dfa2b6d9217075eb06fc2e7b7c31","last_reissued_at":"2026-07-05T10:11:24.426732Z","signature_status":"signed_v1","first_computed_at":"2026-07-05T10:11:24.426732Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"graph_snapshot":{"paper":{"title":"Scaling FP8 training to trillion-token LLMs","license":"http://creativecommons.org/licenses/by-nc-nd/4.0/","headline":"","cross_cats":["cs.AI"],"primary_cat":"cs.LG","authors_text":"Brian Chmiel, Daniel Soudry, Maxim Fishman, Ron Banner","submitted_at":"2024-09-19T07:15:58Z","abstract_excerpt":"We train, for the first time, large language models using FP8 precision on datasets up to 2 trillion tokens -- a 20-fold increase over previous limits. Through these extended training runs, we uncover critical instabilities in FP8 training that were not observable in earlier works with shorter durations. We trace these instabilities to outlier amplification by the SwiGLU activation function. Interestingly, we show, both analytically and empirically, that this amplification happens only over prolonged training periods, and link it to a SwiGLU weight alignment process. To address this newly iden"},"claims":{"count":0,"items":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"2409.12517","kind":"arxiv","version":2},"verdict":{"id":null,"model_set":{},"created_at":null,"strongest_claim":"","one_line_summary":"","pipeline_version":null,"weakest_assumption":"","pith_extraction_headline":""},"integrity":{"clean":true,"summary":{"advisory":0,"critical":0,"by_detector":{},"informational":0},"endpoint":"/pith/2409.12517/integrity.json","findings":[],"available":true,"detectors_run":[],"snapshot_sha256":"c28c3603d3b5d939e8dc4c7e95fa8dfce3d595e45f758748cecf8e644a296938"},"references":{"count":0,"sample":[],"resolved_work":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","internal_anchors":0},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"author_claims":{"count":0,"strong_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"builder_version":"pith-number-builder-2026-05-17-v1"},"aliases":[{"alias_kind":"arxiv","alias_value":"2409.12517","created_at":"2026-07-05T10:11:24.426790+00:00"},{"alias_kind":"arxiv_version","alias_value":"2409.12517v2","created_at":"2026-07-05T10:11:24.426790+00:00"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.2409.12517","created_at":"2026-07-05T10:11:24.426790+00:00"},{"alias_kind":"pith_short_12","alias_value":"NUGXK6Y4YSCE","created_at":"2026-07-05T10:11:24.426790+00:00"},{"alias_kind":"pith_short_16","alias_value":"NUGXK6Y4YSCE545I","created_at":"2026-07-05T10:11:24.426790+00:00"},{"alias_kind":"pith_short_8","alias_value":"NUGXK6Y4","created_at":"2026-07-05T10:11:24.426790+00:00"}],"events":[],"event_summary":{},"paper_claims":[],"inbound_citations":{"count":8,"internal_anchor_count":0,"sample":[{"citing_arxiv_id":"2606.10493","citing_title":"Achieving Cloud-Grade SLOs for Local Mixture-of-Experts Inference through CPU-GPU Hybrid Design","ref_index":14,"is_internal_anchor":false},{"citing_arxiv_id":"2605.09370","citing_title":"From Detection to Recovery: Operational Analysis on LLM Pre-training with 504 GPUs","ref_index":49,"is_internal_anchor":false},{"citing_arxiv_id":"2605.23191","citing_title":"Expand More, Shrink Less: Shaping Effective-Rank Dynamics for Dense Scaling in Recommendation","ref_index":7,"is_internal_anchor":false},{"citing_arxiv_id":"2412.04468","citing_title":"NVILA: Efficient Frontier Visual Language Models","ref_index":86,"is_internal_anchor":false},{"citing_arxiv_id":"2510.04212","citing_title":"Why Low-Precision Transformer Training Fails: An Analysis on Flash Attention","ref_index":4,"is_internal_anchor":false},{"citing_arxiv_id":"2605.10886","citing_title":"LoKA: Low-precision Kernel Applications for Recommendation Models At Scale","ref_index":30,"is_internal_anchor":false},{"citing_arxiv_id":"2605.10886","citing_title":"LoKA: Low-precision Kernel Applications for Recommendation Models At Scale","ref_index":30,"is_internal_anchor":false},{"citing_arxiv_id":"2605.09370","citing_title":"From Detection to Recovery: Operational Analysis on LLM Pre-training with 504 GPUs","ref_index":49,"is_internal_anchor":false}]},"formal_canon":{"evidence_count":0,"sample":[],"anchors":[]},"links":{"html":"https://pith.science/pith/NUGXK6Y4YSCE545ILLRI6N2ZGR","json":"https://pith.science/pith/NUGXK6Y4YSCE545ILLRI6N2ZGR.json","graph_json":"https://pith.science/api/pith-number/NUGXK6Y4YSCE545ILLRI6N2ZGR/graph.json","events_json":"https://pith.science/api/pith-number/NUGXK6Y4YSCE545ILLRI6N2ZGR/events.json","paper":"https://pith.science/paper/NUGXK6Y4"},"agent_actions":{"view_html":"https://pith.science/pith/NUGXK6Y4YSCE545ILLRI6N2ZGR","download_json":"https://pith.science/pith/NUGXK6Y4YSCE545ILLRI6N2ZGR.json","view_paper":"https://pith.science/paper/NUGXK6Y4","resolve_alias":"https://pith.science/api/pith-number/resolve?arxiv=2409.12517&json=true","fetch_graph":"https://pith.science/api/pith-number/NUGXK6Y4YSCE545ILLRI6N2ZGR/graph.json","fetch_events":"https://pith.science/api/pith-number/NUGXK6Y4YSCE545ILLRI6N2ZGR/events.json","actions":{"anchor_timestamp":"https://pith.science/pith/NUGXK6Y4YSCE545ILLRI6N2ZGR/action/timestamp_anchor","attest_storage":"https://pith.science/pith/NUGXK6Y4YSCE545ILLRI6N2ZGR/action/storage_attestation","attest_author":"https://pith.science/pith/NUGXK6Y4YSCE545ILLRI6N2ZGR/action/author_attestation","sign_citation":"https://pith.science/pith/NUGXK6Y4YSCE545ILLRI6N2ZGR/action/citation_signature","submit_replication":"https://pith.science/pith/NUGXK6Y4YSCE545ILLRI6N2ZGR/action/replication_record"}},"created_at":"2026-07-05T10:11:24.426790+00:00","updated_at":"2026-07-05T10:11:24.426790+00:00"}