{"record_type":"pith_number_record","schema_url":"https://pith.science/schemas/pith-number/v1.json","pith_number":"pith:2019:NAC62DBTP6WY4EWWSCR4OUM4IX","short_pith_number":"pith:NAC62DBT","schema_version":"1.0","canonical_sha256":"6805ed0c337fad8e12d690a3c7519c45ce8a2669ba4baa4d6986435188dd3d65","source":{"kind":"arxiv","id":"1904.09237","version":1},"attestation_state":"computed","paper":{"title":"On the Convergence of Adam and Beyond","license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","headline":"","cross_cats":["math.OC","stat.ML"],"primary_cat":"cs.LG","authors_text":"Sanjiv Kumar, Sashank J. Reddi, Satyen Kale","submitted_at":"2019-04-19T16:21:38Z","abstract_excerpt":"Several recently proposed stochastic optimization methods that have been successfully used in training deep networks such as RMSProp, Adam, Adadelta, Nadam are based on using gradient updates scaled by square roots of exponential moving averages of squared past gradients. In many applications, e.g. learning with large output spaces, it has been empirically observed that these algorithms fail to converge to an optimal solution (or a critical point in nonconvex settings). We show that one cause for such failures is the exponential moving average used in the algorithms. We provide an explicit exa"},"verification_status":{"content_addressed":true,"pith_receipt":true,"author_attested":false,"weak_author_claims":0,"strong_author_claims":0,"externally_anchored":false,"storage_verified":false,"citation_signatures":0,"replication_records":0,"graph_snapshot":true,"references_resolved":false,"formal_links_present":false},"canonical_record":{"source":{"id":"1904.09237","kind":"arxiv","version":1},"metadata":{"license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","primary_cat":"cs.LG","submitted_at":"2019-04-19T16:21:38Z","cross_cats_sorted":["math.OC","stat.ML"],"title_canon_sha256":"001a4bb98f91b0563412da3a3e0ea9a3c9d07dfc27c216809c1a47144e79d99e","abstract_canon_sha256":"60105a2d8716f6d93b8eae7ae7a0f267b98a9741432f708151ec04b7892e1730"},"schema_version":"1.0"},"receipt":{"kind":"pith_receipt","key_id":"pith-v1-2026-05","algorithm":"ed25519","signed_at":"2026-05-17T23:48:07.910892Z","signature_b64":"gHkFA12ruy49vkvg+jlTBvZU96jV6NOp1+n/EwUhr/MSUUl27NUHAKmI3vU4xJXSzy2M4xtBYbleVwkHsDZ4DQ==","signed_message":"canonical_sha256_bytes","builder_version":"pith-number-builder-2026-05-17-v1","receipt_version":"0.3","canonical_sha256":"6805ed0c337fad8e12d690a3c7519c45ce8a2669ba4baa4d6986435188dd3d65","last_reissued_at":"2026-05-17T23:48:07.910468Z","signature_status":"signed_v1","first_computed_at":"2026-05-17T23:48:07.910468Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"graph_snapshot":{"paper":{"title":"On the Convergence of Adam and Beyond","license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","headline":"","cross_cats":["math.OC","stat.ML"],"primary_cat":"cs.LG","authors_text":"Sanjiv Kumar, Sashank J. Reddi, Satyen Kale","submitted_at":"2019-04-19T16:21:38Z","abstract_excerpt":"Several recently proposed stochastic optimization methods that have been successfully used in training deep networks such as RMSProp, Adam, Adadelta, Nadam are based on using gradient updates scaled by square roots of exponential moving averages of squared past gradients. In many applications, e.g. learning with large output spaces, it has been empirically observed that these algorithms fail to converge to an optimal solution (or a critical point in nonconvex settings). We show that one cause for such failures is the exponential moving average used in the algorithms. We provide an explicit exa"},"claims":{"count":0,"items":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"1904.09237","kind":"arxiv","version":1},"verdict":{"id":null,"model_set":{},"created_at":null,"strongest_claim":"","one_line_summary":"","pipeline_version":null,"weakest_assumption":"","pith_extraction_headline":""},"references":{"count":0,"sample":[],"resolved_work":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","internal_anchors":0},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"author_claims":{"count":0,"strong_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"builder_version":"pith-number-builder-2026-05-17-v1"},"aliases":[{"alias_kind":"arxiv","alias_value":"1904.09237","created_at":"2026-05-17T23:48:07.910536+00:00"},{"alias_kind":"arxiv_version","alias_value":"1904.09237v1","created_at":"2026-05-17T23:48:07.910536+00:00"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.1904.09237","created_at":"2026-05-17T23:48:07.910536+00:00"},{"alias_kind":"pith_short_12","alias_value":"NAC62DBTP6WY","created_at":"2026-05-18T12:33:24.271573+00:00"},{"alias_kind":"pith_short_16","alias_value":"NAC62DBTP6WY4EWW","created_at":"2026-05-18T12:33:24.271573+00:00"},{"alias_kind":"pith_short_8","alias_value":"NAC62DBT","created_at":"2026-05-18T12:33:24.271573+00:00"}],"events":[],"event_summary":{},"paper_claims":[],"inbound_citations":{"count":31,"internal_anchor_count":17,"sample":[{"citing_arxiv_id":"2212.08989","citing_title":"Deep learning applied to computational mechanics: A comprehensive review, state of the art, and the classics","ref_index":184,"is_internal_anchor":true},{"citing_arxiv_id":"2307.07030","citing_title":"Accelerated Gradient Methods for Nonconvex Optimization: Escape Trajectories From Strict Saddle Points and Convergence to Local Minima","ref_index":72,"is_internal_anchor":true},{"citing_arxiv_id":"2502.06238","citing_title":"XNet-Enhanced Deep BSDE Method and Numerical Analysis","ref_index":28,"is_internal_anchor":true},{"citing_arxiv_id":"2505.13196","citing_title":"A Physics-Inspired Optimizer: Velocity Regularized Adam","ref_index":19,"is_internal_anchor":true},{"citing_arxiv_id":"2605.22644","citing_title":"Why SGD is not Brownian Motion: A New Perspective on Stochastic Dynamics","ref_index":140,"is_internal_anchor":true},{"citing_arxiv_id":"2505.16487","citing_title":"Generative Prior-Guided Neural Interface Reconstruction for 3D Electrical Impedance Tomography","ref_index":37,"is_internal_anchor":true},{"citing_arxiv_id":"2507.21901","citing_title":"Communication-Efficient Decentralized Stochastic Minimax Optimization","ref_index":76,"is_internal_anchor":true},{"citing_arxiv_id":"2003.00295","citing_title":"Adaptive Federated Optimization","ref_index":234,"is_internal_anchor":true},{"citing_arxiv_id":"2605.19073","citing_title":"Riemannian Networks over Full-Rank Correlation Matrices","ref_index":86,"is_internal_anchor":true},{"citing_arxiv_id":"2605.19392","citing_title":"Understanding Dynamics of Adam in Zero-Sum Games: An ODE Approach","ref_index":249,"is_internal_anchor":true},{"citing_arxiv_id":"2605.16875","citing_title":"Stochastic Optimization and Data Science","ref_index":51,"is_internal_anchor":true},{"citing_arxiv_id":"2605.15435","citing_title":"On the Stability of Growth in Structural Plasticity","ref_index":35,"is_internal_anchor":true},{"citing_arxiv_id":"2506.01897","citing_title":"MLorc: Momentum Low-rank Compression for Memory Efficient Large Language Model Adaptation","ref_index":10,"is_internal_anchor":true},{"citing_arxiv_id":"2509.15816","citing_title":"On the Convergence of Muon and Beyond","ref_index":39,"is_internal_anchor":true},{"citing_arxiv_id":"2401.00096","citing_title":"A foundation model for atomistic materials chemistry","ref_index":142,"is_internal_anchor":true},{"citing_arxiv_id":"2602.19945","citing_title":"DP-FedAdamW: An Efficient Optimizer for Differentially Private Federated Large Models","ref_index":58,"is_internal_anchor":true},{"citing_arxiv_id":"2605.13445","citing_title":"VLTI/PIONIER imaging of post-AGB binaries. An INSPIRING hunt for inner rim substructures in circumbinary discs","ref_index":109,"is_internal_anchor":true},{"citing_arxiv_id":"2605.12492","citing_title":"Pion: A Spectrum-Preserving Optimizer via Orthogonal Equivalence Transformation","ref_index":64,"is_internal_anchor":false},{"citing_arxiv_id":"2605.03425","citing_title":"FIBER: A Differentially Private Optimizer with Filter-Aware Innovation Bias Correction","ref_index":45,"is_internal_anchor":false},{"citing_arxiv_id":"2604.28175","citing_title":"Strait: Perceiving Priority and Interference in ML Inference Serving","ref_index":75,"is_internal_anchor":false},{"citing_arxiv_id":"2605.06654","citing_title":"Optimizer-Model Consistency: Full Finetuning with the Same Optimizer as Pretraining Forgets Less","ref_index":21,"is_internal_anchor":false},{"citing_arxiv_id":"2605.02317","citing_title":"Anon: Extrapolating Adaptivity Beyond SGD and Adam","ref_index":13,"is_internal_anchor":false},{"citing_arxiv_id":"2605.00711","citing_title":"A Line-search-free Method for Adaptive Decentralized Optimization","ref_index":20,"is_internal_anchor":false},{"citing_arxiv_id":"2604.12221","citing_title":"BarbieGait: An Identity-Consistent Synthetic Human Dataset with Versatile Cloth-Changing for Gait Recognition","ref_index":52,"is_internal_anchor":false},{"citing_arxiv_id":"2604.10689","citing_title":"Communication-Efficient Gluon in Federated Learning","ref_index":31,"is_internal_anchor":false}]},"formal_canon":{"evidence_count":0,"sample":[],"anchors":[]},"links":{"html":"https://pith.science/pith/NAC62DBTP6WY4EWWSCR4OUM4IX","json":"https://pith.science/pith/NAC62DBTP6WY4EWWSCR4OUM4IX.json","graph_json":"https://pith.science/api/pith-number/NAC62DBTP6WY4EWWSCR4OUM4IX/graph.json","events_json":"https://pith.science/api/pith-number/NAC62DBTP6WY4EWWSCR4OUM4IX/events.json","paper":"https://pith.science/paper/NAC62DBT"},"agent_actions":{"view_html":"https://pith.science/pith/NAC62DBTP6WY4EWWSCR4OUM4IX","download_json":"https://pith.science/pith/NAC62DBTP6WY4EWWSCR4OUM4IX.json","view_paper":"https://pith.science/paper/NAC62DBT","resolve_alias":"https://pith.science/api/pith-number/resolve?arxiv=1904.09237&json=true","fetch_graph":"https://pith.science/api/pith-number/NAC62DBTP6WY4EWWSCR4OUM4IX/graph.json","fetch_events":"https://pith.science/api/pith-number/NAC62DBTP6WY4EWWSCR4OUM4IX/events.json","actions":{"anchor_timestamp":"https://pith.science/pith/NAC62DBTP6WY4EWWSCR4OUM4IX/action/timestamp_anchor","attest_storage":"https://pith.science/pith/NAC62DBTP6WY4EWWSCR4OUM4IX/action/storage_attestation","attest_author":"https://pith.science/pith/NAC62DBTP6WY4EWWSCR4OUM4IX/action/author_attestation","sign_citation":"https://pith.science/pith/NAC62DBTP6WY4EWWSCR4OUM4IX/action/citation_signature","submit_replication":"https://pith.science/pith/NAC62DBTP6WY4EWWSCR4OUM4IX/action/replication_record"}},"created_at":"2026-05-17T23:48:07.910536+00:00","updated_at":"2026-05-17T23:48:07.910536+00:00"}