{"state_type":"pith_open_graph_state","state_version":"1.0","pith_number":"pith:2024:I6DYAHCRPXIDWQD2YN7B6XO2OP","merge_version":"pith-open-graph-merge-v1","event_count":2,"valid_event_count":2,"invalid_event_count":0,"equivocation_count":0,"current":{"canonical_record":{"metadata":{"abstract_canon_sha256":"b198faf1cd92e511d937f9847a533da60d64c338a64f7d2ca6c8d42e4cc9eee9","cross_cats_sorted":["cs.AI"],"license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.LG","submitted_at":"2024-05-22T14:43:02Z","title_canon_sha256":"60f5a6dba065aad38c2755b3ffa9a7b638871bd98da2ae4e67cbbb3fbbda9809"},"schema_version":"1.0","source":{"id":"2405.13698","kind":"arxiv","version":3}},"source_aliases":[{"alias_kind":"arxiv","alias_value":"2405.13698","created_at":"2026-07-05T11:13:09Z"},{"alias_kind":"arxiv_version","alias_value":"2405.13698v3","created_at":"2026-07-05T11:13:09Z"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.2405.13698","created_at":"2026-07-05T11:13:09Z"},{"alias_kind":"pith_short_12","alias_value":"I6DYAHCRPXID","created_at":"2026-07-05T11:13:09Z"},{"alias_kind":"pith_short_16","alias_value":"I6DYAHCRPXIDWQD2","created_at":"2026-07-05T11:13:09Z"},{"alias_kind":"pith_short_8","alias_value":"I6DYAHCR","created_at":"2026-07-05T11:13:09Z"}],"graph_snapshots":[{"event_id":"sha256:4bb086f04cc240e0a216244b5ffc2b14dcd3bf8413bc427211ca7ff2448d9cdb","target":"graph","created_at":"2026-07-05T11:13:09Z","signer":{"key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signer_id":"pith.science","signer_type":"pith_registry"},"payload":{"graph_snapshot":{"author_claims":{"count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","strong_count":0},"builder_version":"pith-number-builder-2026-05-17-v1","claims":{"count":0,"items":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"integrity":{"available":true,"clean":true,"detectors_run":[],"endpoint":"/pith/2405.13698/integrity.json","findings":[],"snapshot_sha256":"c28c3603d3b5d939e8dc4c7e95fa8dfce3d595e45f758748cecf8e644a296938","summary":{"advisory":0,"by_detector":{},"critical":0,"informational":0}},"paper":{"abstract_excerpt":"The scaling of the optimal AdamW weight decay hyperparameter with model and dataset size is critical as we seek to build larger models, but is poorly understood. We show that weights learned by AdamW can be understood as an exponential moving average (EMA) of recent updates. This gives critical insights for how to set the weight decay in AdamW, and how the weight decay should scale with model and dataset size. In particular, the key hyperparameter for an exponential moving average is the EMA timescale. Intuitively, the EMA timescale can be understood as the number of recent iterations the EMA ","authors_text":"Laurence Aitchison, Xi Wang","cross_cats":["cs.AI"],"headline":"","license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.LG","submitted_at":"2024-05-22T14:43:02Z","title":"How to set AdamW's weight decay as you scale model and dataset size"},"references":{"count":0,"internal_anchors":0,"resolved_work":0,"sample":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"2405.13698","kind":"arxiv","version":3},"verdict":{"created_at":null,"id":null,"model_set":{},"one_line_summary":"","pipeline_version":null,"pith_extraction_headline":"","strongest_claim":"","weakest_assumption":""}},"verdict_id":null}}],"author_attestations":[],"timestamp_anchors":[],"storage_attestations":[],"citation_signatures":[],"replication_records":[],"corrections":[],"mirror_hints":[],"record_created":{"event_id":"sha256:551994cc63568edf710eff7f17347d0acb427b88312eb6d92597ebb8673a663d","target":"record","created_at":"2026-07-05T11:13:09Z","signer":{"key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signer_id":"pith.science","signer_type":"pith_registry"},"payload":{"attestation_state":"computed","canonical_record":{"metadata":{"abstract_canon_sha256":"b198faf1cd92e511d937f9847a533da60d64c338a64f7d2ca6c8d42e4cc9eee9","cross_cats_sorted":["cs.AI"],"license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.LG","submitted_at":"2024-05-22T14:43:02Z","title_canon_sha256":"60f5a6dba065aad38c2755b3ffa9a7b638871bd98da2ae4e67cbbb3fbbda9809"},"schema_version":"1.0","source":{"id":"2405.13698","kind":"arxiv","version":3}},"canonical_sha256":"4787801c517dd03b407ac37e1f5dda73ff17dfd8f5338e24dfc28964e629a4e1","receipt":{"algorithm":"ed25519","builder_version":"pith-number-builder-2026-05-17-v1","canonical_sha256":"4787801c517dd03b407ac37e1f5dda73ff17dfd8f5338e24dfc28964e629a4e1","first_computed_at":"2026-07-05T11:13:09.861774Z","key_id":"pith-v1-2026-05","kind":"pith_receipt","last_reissued_at":"2026-07-05T11:13:09.861774Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","receipt_version":"0.3","signature_b64":"4vLZPjGd5aJr4Luiwusc3Gq1R49VkM8/8OIa/ZIU1rwnugVHByXZUKK0u+WopGkh5hxFKqXYL0gBPhwqVeF/DA==","signature_status":"signed_v1","signed_at":"2026-07-05T11:13:09.862334Z","signed_message":"canonical_sha256_bytes"},"source_id":"2405.13698","source_kind":"arxiv","source_version":3}}},"equivocations":[],"invalid_events":[],"applied_event_ids":["sha256:551994cc63568edf710eff7f17347d0acb427b88312eb6d92597ebb8673a663d","sha256:4bb086f04cc240e0a216244b5ffc2b14dcd3bf8413bc427211ca7ff2448d9cdb"],"state_sha256":"88a71eabfd84f40e09cf8833e21c96a1b5f1ff51c6cbdae3b01bfbec5b269ca0"}