{"record_type":"pith_number_record","schema_url":"https://pith.science/schemas/pith-number/v1.json","pith_number":"pith:2025:OIH2LAFA3AYNMNPD4GMQCU4Q6I","short_pith_number":"pith:OIH2LAFA","schema_version":"1.0","canonical_sha256":"720fa580a0d830d635e3e199015390f214144c560869f50144b4a54521203450","source":{"kind":"arxiv","id":"2509.02046","version":2},"attestation_state":"computed","paper":{"title":"Fantastic Pretraining Optimizers and Where to Find Them","license":"http://creativecommons.org/licenses/by/4.0/","headline":"","cross_cats":["cs.AI","stat.ML"],"primary_cat":"cs.LG","authors_text":"David Hall, Kaiyue Wen, Percy Liang, Tengyu Ma","submitted_at":"2025-09-02T07:43:22Z","abstract_excerpt":"AdamW has long been the dominant optimizer in language model pretraining, despite numerous claims that alternative optimizers offer 1.4 to 2x speedup. We posit that two methodological shortcomings have obscured fair comparisons and hindered practical adoption: (i) unequal hyperparameter tuning and (ii) limited or misleading evaluation setups. To address these two issues, we conduct a systematic study of ten deep learning optimizers across four model scales (0.1B-1.2B parameters) and data-to-model ratios (1-8x the Chinchilla optimum). We find that fair and informative comparisons require rigoro"},"verification_status":{"content_addressed":true,"pith_receipt":true,"author_attested":false,"weak_author_claims":0,"strong_author_claims":0,"externally_anchored":false,"storage_verified":false,"citation_signatures":0,"replication_records":0,"graph_snapshot":true,"references_resolved":false,"formal_links_present":false},"canonical_record":{"source":{"id":"2509.02046","kind":"arxiv","version":2},"metadata":{"license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.LG","submitted_at":"2025-09-02T07:43:22Z","cross_cats_sorted":["cs.AI","stat.ML"],"title_canon_sha256":"35321aba0ebc26ab3f0249a95845499e857e175ca39e44d5389b5536bac99e7a","abstract_canon_sha256":"1dea35b4bf504722a3df51d3dfc6c4bba8dced032ae4ee44278afb745f870b7b"},"schema_version":"1.0"},"receipt":{"kind":"pith_receipt","key_id":"pith-v1-2026-05","algorithm":"ed25519","signed_at":"2026-07-05T12:05:21.205428Z","signature_b64":"X+4cbf3XLncQH/Gm1fC0c/XETRVZYkLsOhOyvS8l1TDR7AG2xn4hoKyge+SEY2p69V/196cWKnNnX7Rj25VZBw==","signed_message":"canonical_sha256_bytes","builder_version":"pith-number-builder-2026-05-17-v1","receipt_version":"0.3","canonical_sha256":"720fa580a0d830d635e3e199015390f214144c560869f50144b4a54521203450","last_reissued_at":"2026-07-05T12:05:21.204934Z","signature_status":"signed_v1","first_computed_at":"2026-07-05T12:05:21.204934Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"graph_snapshot":{"paper":{"title":"Fantastic Pretraining Optimizers and Where to Find Them","license":"http://creativecommons.org/licenses/by/4.0/","headline":"","cross_cats":["cs.AI","stat.ML"],"primary_cat":"cs.LG","authors_text":"David Hall, Kaiyue Wen, Percy Liang, Tengyu Ma","submitted_at":"2025-09-02T07:43:22Z","abstract_excerpt":"AdamW has long been the dominant optimizer in language model pretraining, despite numerous claims that alternative optimizers offer 1.4 to 2x speedup. We posit that two methodological shortcomings have obscured fair comparisons and hindered practical adoption: (i) unequal hyperparameter tuning and (ii) limited or misleading evaluation setups. To address these two issues, we conduct a systematic study of ten deep learning optimizers across four model scales (0.1B-1.2B parameters) and data-to-model ratios (1-8x the Chinchilla optimum). We find that fair and informative comparisons require rigoro"},"claims":{"count":0,"items":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"2509.02046","kind":"arxiv","version":2},"verdict":{"id":null,"model_set":{},"created_at":null,"strongest_claim":"","one_line_summary":"","pipeline_version":null,"weakest_assumption":"","pith_extraction_headline":""},"integrity":{"clean":true,"summary":{"advisory":0,"critical":0,"by_detector":{},"informational":0},"endpoint":"/pith/2509.02046/integrity.json","findings":[],"available":true,"detectors_run":[],"snapshot_sha256":"c28c3603d3b5d939e8dc4c7e95fa8dfce3d595e45f758748cecf8e644a296938"},"references":{"count":0,"sample":[],"resolved_work":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","internal_anchors":0},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"author_claims":{"count":0,"strong_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"builder_version":"pith-number-builder-2026-05-17-v1"},"aliases":[{"alias_kind":"arxiv","alias_value":"2509.02046","created_at":"2026-07-05T12:05:21.204990+00:00"},{"alias_kind":"arxiv_version","alias_value":"2509.02046v2","created_at":"2026-07-05T12:05:21.204990+00:00"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.2509.02046","created_at":"2026-07-05T12:05:21.204990+00:00"},{"alias_kind":"pith_short_12","alias_value":"OIH2LAFA3AYN","created_at":"2026-07-05T12:05:21.204990+00:00"},{"alias_kind":"pith_short_16","alias_value":"OIH2LAFA3AYNMNPD","created_at":"2026-07-05T12:05:21.204990+00:00"},{"alias_kind":"pith_short_8","alias_value":"OIH2LAFA","created_at":"2026-07-05T12:05:21.204990+00:00"}],"events":[],"event_summary":{},"paper_claims":[],"inbound_citations":{"count":23,"internal_anchor_count":0,"sample":[{"citing_arxiv_id":"2606.23676","citing_title":"Open Problem: Is AdamW Effective Under Heavy-Tailed Noise?","ref_index":10,"is_internal_anchor":false},{"citing_arxiv_id":"2606.11387","citing_title":"Small Experiments, Cheaper Decisions: A Case Study in Staged Promotion for Micro-Pretraining","ref_index":7,"is_internal_anchor":false},{"citing_arxiv_id":"2606.06470","citing_title":"PC Layer: Polynomial Weight Preconditioning for Improving LLM Pre-Training","ref_index":99,"is_internal_anchor":false},{"citing_arxiv_id":"2606.04662","citing_title":"Why Muon Outperforms Adam: A Curvature Perspective","ref_index":193,"is_internal_anchor":false},{"citing_arxiv_id":"2606.04058","citing_title":"Spectral Scaling Laws of Muon","ref_index":13,"is_internal_anchor":false},{"citing_arxiv_id":"2606.03899","citing_title":"Denoise First, Orthogonalize Later: Understanding Momentum in Muon via Spectral Filtering","ref_index":62,"is_internal_anchor":false},{"citing_arxiv_id":"2605.08678","citing_title":"MLS-Bench: A Holistic and Rigorous Assessment of AI Systems on Building Better AI","ref_index":111,"is_internal_anchor":false},{"citing_arxiv_id":"2605.28585","citing_title":"Outer-Momentum Restarting in High-Dimensional Two-Phase Optimization","ref_index":27,"is_internal_anchor":false},{"citing_arxiv_id":"2605.31371","citing_title":"Softsign: Smooth Sign in Your Optimizer For Better Parameter Heterogeneity Handling","ref_index":51,"is_internal_anchor":false},{"citing_arxiv_id":"2603.10067","citing_title":"HTMuon: Improving Muon via Heavy-Tailed Spectral Correction","ref_index":30,"is_internal_anchor":false},{"citing_arxiv_id":"2605.22644","citing_title":"Why SGD is not Brownian Motion: A New Perspective on Stochastic Dynamics","ref_index":193,"is_internal_anchor":false},{"citing_arxiv_id":"2510.10777","citing_title":"Preconditioned Norms: A Unified Framework for Steepest Descent, Quasi-Newton and Adaptive Methods","ref_index":22,"is_internal_anchor":false},{"citing_arxiv_id":"2602.22437","citing_title":"veScale-FSDP: Flexible and High-Performance FSDP at Scale","ref_index":31,"is_internal_anchor":false},{"citing_arxiv_id":"2603.20527","citing_title":"RMNP: Row-Momentum Normalized Preconditioning for Scalable Matrix-Based Optimization","ref_index":25,"is_internal_anchor":false},{"citing_arxiv_id":"2603.26554","citing_title":"Sharp Capacity Scaling of Spectral Optimizers in Learning Associative Memory","ref_index":59,"is_internal_anchor":false},{"citing_arxiv_id":"2605.12316","citing_title":"Autoregressive Learning in Joint KL: Sharp Oracle Bounds and Lower Bounds","ref_index":51,"is_internal_anchor":false},{"citing_arxiv_id":"2604.27077","citing_title":"Learning Rate Transfer in Normalized Transformers","ref_index":16,"is_internal_anchor":false},{"citing_arxiv_id":"2605.09238","citing_title":"Intrinsic Muon: Spectral Optimization on Riemannian Matrix Manifolds","ref_index":64,"is_internal_anchor":false},{"citing_arxiv_id":"2605.08678","citing_title":"MLS-Bench: A Holistic and Rigorous Assessment of AI Systems on Building Better AI","ref_index":109,"is_internal_anchor":false},{"citing_arxiv_id":"2605.03769","citing_title":"Nora: Normalized Orthogonal Row Alignment for Scalable Matrix Optimizer","ref_index":25,"is_internal_anchor":false},{"citing_arxiv_id":"2605.06654","citing_title":"Optimizer-Model Consistency: Full Finetuning with the Same Optimizer as Pretraining Forgets Less","ref_index":33,"is_internal_anchor":false},{"citing_arxiv_id":"2605.06615","citing_title":"When and Why SignSGD Outperforms SGD: A Theoretical Study Based on $\\ell_1$-norm Lower Bounds","ref_index":41,"is_internal_anchor":false},{"citing_arxiv_id":"2604.09258","citing_title":"Nexus: Same Pretraining Loss, Better Downstream Generalization via Common Minima","ref_index":45,"is_internal_anchor":false}]},"formal_canon":{"evidence_count":0,"sample":[],"anchors":[]},"links":{"html":"https://pith.science/pith/OIH2LAFA3AYNMNPD4GMQCU4Q6I","json":"https://pith.science/pith/OIH2LAFA3AYNMNPD4GMQCU4Q6I.json","graph_json":"https://pith.science/api/pith-number/OIH2LAFA3AYNMNPD4GMQCU4Q6I/graph.json","events_json":"https://pith.science/api/pith-number/OIH2LAFA3AYNMNPD4GMQCU4Q6I/events.json","paper":"https://pith.science/paper/OIH2LAFA"},"agent_actions":{"view_html":"https://pith.science/pith/OIH2LAFA3AYNMNPD4GMQCU4Q6I","download_json":"https://pith.science/pith/OIH2LAFA3AYNMNPD4GMQCU4Q6I.json","view_paper":"https://pith.science/paper/OIH2LAFA","resolve_alias":"https://pith.science/api/pith-number/resolve?arxiv=2509.02046&json=true","fetch_graph":"https://pith.science/api/pith-number/OIH2LAFA3AYNMNPD4GMQCU4Q6I/graph.json","fetch_events":"https://pith.science/api/pith-number/OIH2LAFA3AYNMNPD4GMQCU4Q6I/events.json","actions":{"anchor_timestamp":"https://pith.science/pith/OIH2LAFA3AYNMNPD4GMQCU4Q6I/action/timestamp_anchor","attest_storage":"https://pith.science/pith/OIH2LAFA3AYNMNPD4GMQCU4Q6I/action/storage_attestation","attest_author":"https://pith.science/pith/OIH2LAFA3AYNMNPD4GMQCU4Q6I/action/author_attestation","sign_citation":"https://pith.science/pith/OIH2LAFA3AYNMNPD4GMQCU4Q6I/action/citation_signature","submit_replication":"https://pith.science/pith/OIH2LAFA3AYNMNPD4GMQCU4Q6I/action/replication_record"}},"created_at":"2026-07-05T12:05:21.204990+00:00","updated_at":"2026-07-05T12:05:21.204990+00:00"}