{"record_type":"pith_number_record","schema_url":"https://pith.science/schemas/pith-number/v1.json","pith_number":"pith:2025:PYZQ7EYQUEV2NDKFISQX64Z7U5","short_pith_number":"pith:PYZQ7EYQ","schema_version":"1.0","canonical_sha256":"7e330f9310a12ba68d4544a17f733fa7622c51012eda9a418c2d829e9dbd8d9a","source":{"kind":"arxiv","id":"2506.12543","version":1},"attestation_state":"computed","paper":{"title":"Is your batch size the problem? Revisiting the Adam-SGD gap in language modeling","license":"http://creativecommons.org/licenses/by/4.0/","headline":"","cross_cats":["math.OC"],"primary_cat":"cs.LG","authors_text":"Antonio Orvieto, Jonas Geiping, Teodora Sre\\'ckovi\\'c","submitted_at":"2025-06-14T15:37:31Z","abstract_excerpt":"Adam is known to perform significantly better than Stochastic Gradient Descent (SGD) in language models, a phenomenon for which a number of explanations have been proposed. In this work, we revisit this \"optimizer gap\" through a series of comprehensively tuned baseline training runs for language modeling with Transformers. We exhaustively study how momentum, gradient clipping, and batch size affect the gap between SGD and Adam. Our empirical findings show that SGD with momentum can actually perform similarly to Adam in small-batch settings, if tuned correctly. We revisit existing explanations "},"verification_status":{"content_addressed":true,"pith_receipt":true,"author_attested":false,"weak_author_claims":0,"strong_author_claims":0,"externally_anchored":false,"storage_verified":false,"citation_signatures":0,"replication_records":0,"graph_snapshot":true,"references_resolved":false,"formal_links_present":false},"canonical_record":{"source":{"id":"2506.12543","kind":"arxiv","version":1},"metadata":{"license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.LG","submitted_at":"2025-06-14T15:37:31Z","cross_cats_sorted":["math.OC"],"title_canon_sha256":"2dc1b20ce21fb812f7bb97abd825da1d419be904c5c40dc1ce7bbd0e2e071824","abstract_canon_sha256":"1db946658505b424f255425b7d82754c4be19c548dbd4d05b07e5066388581b1"},"schema_version":"1.0"},"receipt":{"kind":"pith_receipt","key_id":"pith-v1-2026-05","algorithm":"ed25519","signed_at":"2026-07-05T11:21:55.034926Z","signature_b64":"9KebPrw+HXO0tmzHgJDn8wbmXYn5wpoUht8LAqXW9SLwXYwTNcYz+nuvAFlrI6NhE+mQwQBL+aauTJzH7yF9Cg==","signed_message":"canonical_sha256_bytes","builder_version":"pith-number-builder-2026-05-17-v1","receipt_version":"0.3","canonical_sha256":"7e330f9310a12ba68d4544a17f733fa7622c51012eda9a418c2d829e9dbd8d9a","last_reissued_at":"2026-07-05T11:21:55.034474Z","signature_status":"signed_v1","first_computed_at":"2026-07-05T11:21:55.034474Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"graph_snapshot":{"paper":{"title":"Is your batch size the problem? Revisiting the Adam-SGD gap in language modeling","license":"http://creativecommons.org/licenses/by/4.0/","headline":"","cross_cats":["math.OC"],"primary_cat":"cs.LG","authors_text":"Antonio Orvieto, Jonas Geiping, Teodora Sre\\'ckovi\\'c","submitted_at":"2025-06-14T15:37:31Z","abstract_excerpt":"Adam is known to perform significantly better than Stochastic Gradient Descent (SGD) in language models, a phenomenon for which a number of explanations have been proposed. In this work, we revisit this \"optimizer gap\" through a series of comprehensively tuned baseline training runs for language modeling with Transformers. We exhaustively study how momentum, gradient clipping, and batch size affect the gap between SGD and Adam. Our empirical findings show that SGD with momentum can actually perform similarly to Adam in small-batch settings, if tuned correctly. We revisit existing explanations "},"claims":{"count":0,"items":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"2506.12543","kind":"arxiv","version":1},"verdict":{"id":null,"model_set":{},"created_at":null,"strongest_claim":"","one_line_summary":"","pipeline_version":null,"weakest_assumption":"","pith_extraction_headline":""},"integrity":{"clean":true,"summary":{"advisory":0,"critical":0,"by_detector":{},"informational":0},"endpoint":"/pith/2506.12543/integrity.json","findings":[],"available":true,"detectors_run":[],"snapshot_sha256":"c28c3603d3b5d939e8dc4c7e95fa8dfce3d595e45f758748cecf8e644a296938"},"references":{"count":0,"sample":[],"resolved_work":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","internal_anchors":0},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"author_claims":{"count":0,"strong_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"builder_version":"pith-number-builder-2026-05-17-v1"},"aliases":[{"alias_kind":"arxiv","alias_value":"2506.12543","created_at":"2026-07-05T11:21:55.034532+00:00"},{"alias_kind":"arxiv_version","alias_value":"2506.12543v1","created_at":"2026-07-05T11:21:55.034532+00:00"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.2506.12543","created_at":"2026-07-05T11:21:55.034532+00:00"},{"alias_kind":"pith_short_12","alias_value":"PYZQ7EYQUEV2","created_at":"2026-07-05T11:21:55.034532+00:00"},{"alias_kind":"pith_short_16","alias_value":"PYZQ7EYQUEV2NDKF","created_at":"2026-07-05T11:21:55.034532+00:00"},{"alias_kind":"pith_short_8","alias_value":"PYZQ7EYQ","created_at":"2026-07-05T11:21:55.034532+00:00"}],"events":[],"event_summary":{},"paper_claims":[],"inbound_citations":{"count":5,"internal_anchor_count":0,"sample":[{"citing_arxiv_id":"2605.29273","citing_title":"A Theoretical and Experimental Study of a Novel Adaptive Learning Algorithm","ref_index":11,"is_internal_anchor":false},{"citing_arxiv_id":"2606.23591","citing_title":"Quantifying the Agreement Between Data-Influence and Data-Similarity to Understand LLM Behavior","ref_index":48,"is_internal_anchor":false},{"citing_arxiv_id":"2605.22644","citing_title":"Why SGD is not Brownian Motion: A New Perspective on Stochastic Dynamics","ref_index":235,"is_internal_anchor":false},{"citing_arxiv_id":"2605.17787","citing_title":"Revisiting the Adam-SGD Gap in LLM Pre-Training: The Role of Large Effective Learning Rates","ref_index":9,"is_internal_anchor":false},{"citing_arxiv_id":"2605.14200","citing_title":"How to Scale Mixture-of-Experts: From muP to the Maximally Scale-Stable Parameterization","ref_index":41,"is_internal_anchor":false}]},"formal_canon":{"evidence_count":0,"sample":[],"anchors":[]},"links":{"html":"https://pith.science/pith/PYZQ7EYQUEV2NDKFISQX64Z7U5","json":"https://pith.science/pith/PYZQ7EYQUEV2NDKFISQX64Z7U5.json","graph_json":"https://pith.science/api/pith-number/PYZQ7EYQUEV2NDKFISQX64Z7U5/graph.json","events_json":"https://pith.science/api/pith-number/PYZQ7EYQUEV2NDKFISQX64Z7U5/events.json","paper":"https://pith.science/paper/PYZQ7EYQ"},"agent_actions":{"view_html":"https://pith.science/pith/PYZQ7EYQUEV2NDKFISQX64Z7U5","download_json":"https://pith.science/pith/PYZQ7EYQUEV2NDKFISQX64Z7U5.json","view_paper":"https://pith.science/paper/PYZQ7EYQ","resolve_alias":"https://pith.science/api/pith-number/resolve?arxiv=2506.12543&json=true","fetch_graph":"https://pith.science/api/pith-number/PYZQ7EYQUEV2NDKFISQX64Z7U5/graph.json","fetch_events":"https://pith.science/api/pith-number/PYZQ7EYQUEV2NDKFISQX64Z7U5/events.json","actions":{"anchor_timestamp":"https://pith.science/pith/PYZQ7EYQUEV2NDKFISQX64Z7U5/action/timestamp_anchor","attest_storage":"https://pith.science/pith/PYZQ7EYQUEV2NDKFISQX64Z7U5/action/storage_attestation","attest_author":"https://pith.science/pith/PYZQ7EYQUEV2NDKFISQX64Z7U5/action/author_attestation","sign_citation":"https://pith.science/pith/PYZQ7EYQUEV2NDKFISQX64Z7U5/action/citation_signature","submit_replication":"https://pith.science/pith/PYZQ7EYQUEV2NDKFISQX64Z7U5/action/replication_record"}},"created_at":"2026-07-05T11:21:55.034532+00:00","updated_at":"2026-07-05T11:21:55.034532+00:00"}