{"record_type":"pith_number_record","schema_url":"https://pith.science/schemas/pith-number/v1.json","pith_number":"pith:2023:7L7PLV7VTM6W4HMICIRE5O5HJ6","short_pith_number":"pith:7L7PLV7V","schema_version":"1.0","canonical_sha256":"fafef5d7f59b3d6e1d8812224ebba74f8873976393b61b6ca1792506939c81e0","source":{"kind":"arxiv","id":"2310.04415","version":2},"attestation_state":"computed","paper":{"title":"Why Do We Need Weight Decay in Modern Deep Learning?","license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","headline":"","cross_cats":[],"primary_cat":"cs.LG","authors_text":"Aditya Varre, Francesco D'Angelo, Maksym Andriushchenko, Nicolas Flammarion","submitted_at":"2023-10-06T17:58:21Z","abstract_excerpt":"Weight decay is a broadly used technique for training state-of-the-art deep networks from image classification to large language models. Despite its widespread usage and being extensively studied in the classical literature, its role remains poorly understood for deep learning. In this work, we highlight that the role of weight decay in modern deep learning is different from its regularization effect studied in classical learning theory. For deep networks on vision tasks trained with multipass SGD, we show how weight decay modifies the optimization dynamics enhancing the ever-present implicit "},"verification_status":{"content_addressed":true,"pith_receipt":true,"author_attested":false,"weak_author_claims":0,"strong_author_claims":0,"externally_anchored":false,"storage_verified":false,"citation_signatures":0,"replication_records":0,"graph_snapshot":true,"references_resolved":false,"formal_links_present":false},"canonical_record":{"source":{"id":"2310.04415","kind":"arxiv","version":2},"metadata":{"license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","primary_cat":"cs.LG","submitted_at":"2023-10-06T17:58:21Z","cross_cats_sorted":[],"title_canon_sha256":"e3301d36be51cd97e2119422334ba8ce31cc2abbabf1b9e9063c42b2bf9e7ed8","abstract_canon_sha256":"37f8e48784afcedb2cab90a2f01593ef74e67b8237777510ace9b8bc06e3b36c"},"schema_version":"1.0"},"receipt":{"kind":"pith_receipt","key_id":"pith-v1-2026-05","algorithm":"ed25519","signed_at":"2026-07-05T09:30:57.025535Z","signature_b64":"M7ejfPs4TVn53G53SRDWUyq5JOuX7R438sg2ryxuNbl3S2uHY10E3Ykou7nB8MjMegu+O4v0g5QzcPuyJKbqAQ==","signed_message":"canonical_sha256_bytes","builder_version":"pith-number-builder-2026-05-17-v1","receipt_version":"0.3","canonical_sha256":"fafef5d7f59b3d6e1d8812224ebba74f8873976393b61b6ca1792506939c81e0","last_reissued_at":"2026-07-05T09:30:57.025048Z","signature_status":"signed_v1","first_computed_at":"2026-07-05T09:30:57.025048Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"graph_snapshot":{"paper":{"title":"Why Do We Need Weight Decay in Modern Deep Learning?","license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","headline":"","cross_cats":[],"primary_cat":"cs.LG","authors_text":"Aditya Varre, Francesco D'Angelo, Maksym Andriushchenko, Nicolas Flammarion","submitted_at":"2023-10-06T17:58:21Z","abstract_excerpt":"Weight decay is a broadly used technique for training state-of-the-art deep networks from image classification to large language models. Despite its widespread usage and being extensively studied in the classical literature, its role remains poorly understood for deep learning. In this work, we highlight that the role of weight decay in modern deep learning is different from its regularization effect studied in classical learning theory. For deep networks on vision tasks trained with multipass SGD, we show how weight decay modifies the optimization dynamics enhancing the ever-present implicit "},"claims":{"count":0,"items":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"2310.04415","kind":"arxiv","version":2},"verdict":{"id":null,"model_set":{},"created_at":null,"strongest_claim":"","one_line_summary":"","pipeline_version":null,"weakest_assumption":"","pith_extraction_headline":""},"integrity":{"clean":true,"summary":{"advisory":0,"critical":0,"by_detector":{},"informational":0},"endpoint":"/pith/2310.04415/integrity.json","findings":[],"available":true,"detectors_run":[],"snapshot_sha256":"c28c3603d3b5d939e8dc4c7e95fa8dfce3d595e45f758748cecf8e644a296938"},"references":{"count":0,"sample":[],"resolved_work":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","internal_anchors":0},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"author_claims":{"count":0,"strong_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"builder_version":"pith-number-builder-2026-05-17-v1"},"aliases":[{"alias_kind":"arxiv","alias_value":"2310.04415","created_at":"2026-07-05T09:30:57.025107+00:00"},{"alias_kind":"arxiv_version","alias_value":"2310.04415v2","created_at":"2026-07-05T09:30:57.025107+00:00"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.2310.04415","created_at":"2026-07-05T09:30:57.025107+00:00"},{"alias_kind":"pith_short_12","alias_value":"7L7PLV7VTM6W","created_at":"2026-07-05T09:30:57.025107+00:00"},{"alias_kind":"pith_short_16","alias_value":"7L7PLV7VTM6W4HMI","created_at":"2026-07-05T09:30:57.025107+00:00"},{"alias_kind":"pith_short_8","alias_value":"7L7PLV7V","created_at":"2026-07-05T09:30:57.025107+00:00"}],"events":[],"event_summary":{},"paper_claims":[],"inbound_citations":{"count":10,"internal_anchor_count":0,"sample":[{"citing_arxiv_id":"2606.25971","citing_title":"Improving Neural Network Training by Decoupling the Magnitude and Direction of Weight Vectors","ref_index":177,"is_internal_anchor":false},{"citing_arxiv_id":"2605.23061","citing_title":"Anytime Training with Schedule-Free Spectral Optimization","ref_index":58,"is_internal_anchor":false},{"citing_arxiv_id":"2502.07529","citing_title":"Training Deep Learning Models with Norm-Constrained LMOs","ref_index":172,"is_internal_anchor":false},{"citing_arxiv_id":"2605.20441","citing_title":"Weight Decay Regimes in Grokking Transformers: Cheap Online Diagnostics","ref_index":7,"is_internal_anchor":false},{"citing_arxiv_id":"2605.18845","citing_title":"First-Passage Prediction of Grokking Delay: ACalibrated Law under AdamW with Causal Validation","ref_index":1,"is_internal_anchor":false},{"citing_arxiv_id":"2605.15290","citing_title":"GQA-{\\mu}P: The maximal parameterization update for grouped query attention","ref_index":2,"is_internal_anchor":false},{"citing_arxiv_id":"2412.06464","citing_title":"Gated Delta Networks: Improving Mamba2 with Delta Rule","ref_index":293,"is_internal_anchor":false},{"citing_arxiv_id":"2605.10161","citing_title":"OUIDecay: Adaptive Layer-wise Weight Decay for CNNs Using Online Activation Patterns","ref_index":2,"is_internal_anchor":false},{"citing_arxiv_id":"2604.23434","citing_title":"When Does Removing LayerNorm Help? Activation Bounding as a Regime-Dependent Implicit Regularizer","ref_index":8,"is_internal_anchor":false},{"citing_arxiv_id":"2605.04418","citing_title":"Demystifying Manifold Constraints in LLM Pre-training","ref_index":18,"is_internal_anchor":false}]},"formal_canon":{"evidence_count":0,"sample":[],"anchors":[]},"links":{"html":"https://pith.science/pith/7L7PLV7VTM6W4HMICIRE5O5HJ6","json":"https://pith.science/pith/7L7PLV7VTM6W4HMICIRE5O5HJ6.json","graph_json":"https://pith.science/api/pith-number/7L7PLV7VTM6W4HMICIRE5O5HJ6/graph.json","events_json":"https://pith.science/api/pith-number/7L7PLV7VTM6W4HMICIRE5O5HJ6/events.json","paper":"https://pith.science/paper/7L7PLV7V"},"agent_actions":{"view_html":"https://pith.science/pith/7L7PLV7VTM6W4HMICIRE5O5HJ6","download_json":"https://pith.science/pith/7L7PLV7VTM6W4HMICIRE5O5HJ6.json","view_paper":"https://pith.science/paper/7L7PLV7V","resolve_alias":"https://pith.science/api/pith-number/resolve?arxiv=2310.04415&json=true","fetch_graph":"https://pith.science/api/pith-number/7L7PLV7VTM6W4HMICIRE5O5HJ6/graph.json","fetch_events":"https://pith.science/api/pith-number/7L7PLV7VTM6W4HMICIRE5O5HJ6/events.json","actions":{"anchor_timestamp":"https://pith.science/pith/7L7PLV7VTM6W4HMICIRE5O5HJ6/action/timestamp_anchor","attest_storage":"https://pith.science/pith/7L7PLV7VTM6W4HMICIRE5O5HJ6/action/storage_attestation","attest_author":"https://pith.science/pith/7L7PLV7VTM6W4HMICIRE5O5HJ6/action/author_attestation","sign_citation":"https://pith.science/pith/7L7PLV7VTM6W4HMICIRE5O5HJ6/action/citation_signature","submit_replication":"https://pith.science/pith/7L7PLV7VTM6W4HMICIRE5O5HJ6/action/replication_record"}},"created_at":"2026-07-05T09:30:57.025107+00:00","updated_at":"2026-07-05T09:30:57.025107+00:00"}