{"record_type":"pith_number_record","schema_url":"https://pith.science/schemas/pith-number/v1.json","pith_number":"pith:2023:7DSU5DOOKHNQKEEI7U4LJHPVM7","short_pith_number":"pith:7DSU5DOO","schema_version":"1.0","canonical_sha256":"f8e54e8dce51db051088fd38b49df567c931f0d3af9fa708a736b67f4670a5ed","source":{"kind":"arxiv","id":"2306.03241","version":2},"attestation_state":"computed","paper":{"title":"Early Weight Averaging meets High Learning Rates for LLM Pre-training","license":"http://creativecommons.org/licenses/by/4.0/","headline":"","cross_cats":["cs.AI","cs.CL"],"primary_cat":"cs.LG","authors_text":"Abhishek Kumar, Atula Neerkaje, Jean Kaddour, Sujay Sanghavi, Sunny Sanyal","submitted_at":"2023-06-05T20:51:44Z","abstract_excerpt":"Training Large Language Models (LLMs) incurs significant cost; hence, any strategy that accelerates model convergence is helpful. In this paper, we investigate the ability of a simple idea checkpoint averaging along the trajectory of a training run to improve both convergence and generalization quite early on during training. Here we show that models trained with high learning rates observe higher gains due to checkpoint averaging. Furthermore, these gains are amplified when checkpoints are sampled with considerable spacing in training steps. Our training recipe outperforms conventional traini"},"verification_status":{"content_addressed":true,"pith_receipt":true,"author_attested":false,"weak_author_claims":0,"strong_author_claims":0,"externally_anchored":false,"storage_verified":false,"citation_signatures":0,"replication_records":0,"graph_snapshot":true,"references_resolved":false,"formal_links_present":false},"canonical_record":{"source":{"id":"2306.03241","kind":"arxiv","version":2},"metadata":{"license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.LG","submitted_at":"2023-06-05T20:51:44Z","cross_cats_sorted":["cs.AI","cs.CL"],"title_canon_sha256":"f8e9386f740d6ced8df2dd261d3d4771cb3531ac736bdf479442c3b4163ba42d","abstract_canon_sha256":"f4dc5a07c16be19f593adc4e38bffc434701165999da60f80b894116940b242b"},"schema_version":"1.0"},"receipt":{"kind":"pith_receipt","key_id":"pith-v1-2026-05","algorithm":"ed25519","signed_at":"2026-07-05T07:22:49.879383Z","signature_b64":"ALgOA6al/u3BG13z31NWb4C8kU/hPIfRfVTr24PWUTg1mi+wS4MZt+sc0r5FY9pXRnHTNJo4iFfnFuGj0ptuCw==","signed_message":"canonical_sha256_bytes","builder_version":"pith-number-builder-2026-05-17-v1","receipt_version":"0.3","canonical_sha256":"f8e54e8dce51db051088fd38b49df567c931f0d3af9fa708a736b67f4670a5ed","last_reissued_at":"2026-07-05T07:22:49.878949Z","signature_status":"signed_v1","first_computed_at":"2026-07-05T07:22:49.878949Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"graph_snapshot":{"paper":{"title":"Early Weight Averaging meets High Learning Rates for LLM Pre-training","license":"http://creativecommons.org/licenses/by/4.0/","headline":"","cross_cats":["cs.AI","cs.CL"],"primary_cat":"cs.LG","authors_text":"Abhishek Kumar, Atula Neerkaje, Jean Kaddour, Sujay Sanghavi, Sunny Sanyal","submitted_at":"2023-06-05T20:51:44Z","abstract_excerpt":"Training Large Language Models (LLMs) incurs significant cost; hence, any strategy that accelerates model convergence is helpful. In this paper, we investigate the ability of a simple idea checkpoint averaging along the trajectory of a training run to improve both convergence and generalization quite early on during training. Here we show that models trained with high learning rates observe higher gains due to checkpoint averaging. Furthermore, these gains are amplified when checkpoints are sampled with considerable spacing in training steps. Our training recipe outperforms conventional traini"},"claims":{"count":0,"items":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"2306.03241","kind":"arxiv","version":2},"verdict":{"id":null,"model_set":{},"created_at":null,"strongest_claim":"","one_line_summary":"","pipeline_version":null,"weakest_assumption":"","pith_extraction_headline":""},"integrity":{"clean":true,"summary":{"advisory":0,"critical":0,"by_detector":{},"informational":0},"endpoint":"/pith/2306.03241/integrity.json","findings":[],"available":true,"detectors_run":[],"snapshot_sha256":"c28c3603d3b5d939e8dc4c7e95fa8dfce3d595e45f758748cecf8e644a296938"},"references":{"count":0,"sample":[],"resolved_work":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","internal_anchors":0},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"author_claims":{"count":0,"strong_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"builder_version":"pith-number-builder-2026-05-17-v1"},"aliases":[{"alias_kind":"arxiv","alias_value":"2306.03241","created_at":"2026-07-05T07:22:49.879013+00:00"},{"alias_kind":"arxiv_version","alias_value":"2306.03241v2","created_at":"2026-07-05T07:22:49.879013+00:00"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.2306.03241","created_at":"2026-07-05T07:22:49.879013+00:00"},{"alias_kind":"pith_short_12","alias_value":"7DSU5DOOKHNQ","created_at":"2026-07-05T07:22:49.879013+00:00"},{"alias_kind":"pith_short_16","alias_value":"7DSU5DOOKHNQKEEI","created_at":"2026-07-05T07:22:49.879013+00:00"},{"alias_kind":"pith_short_8","alias_value":"7DSU5DOO","created_at":"2026-07-05T07:22:49.879013+00:00"}],"events":[],"event_summary":{},"paper_claims":[],"inbound_citations":{"count":3,"internal_anchor_count":0,"sample":[{"citing_arxiv_id":"2607.02291","citing_title":"Optimizing Visual Generative Models via Distribution-wise Rewards","ref_index":26,"is_internal_anchor":false},{"citing_arxiv_id":"2605.12419","citing_title":"ORBIT: Preserving Foundational Language Capabilities in GenRetrieval via Origin-Regulated Merging","ref_index":15,"is_internal_anchor":false},{"citing_arxiv_id":"2604.11446","citing_title":"Low-rank Optimization Trajectories Modeling for LLM RLVR Acceleration","ref_index":25,"is_internal_anchor":false}]},"formal_canon":{"evidence_count":0,"sample":[],"anchors":[]},"links":{"html":"https://pith.science/pith/7DSU5DOOKHNQKEEI7U4LJHPVM7","json":"https://pith.science/pith/7DSU5DOOKHNQKEEI7U4LJHPVM7.json","graph_json":"https://pith.science/api/pith-number/7DSU5DOOKHNQKEEI7U4LJHPVM7/graph.json","events_json":"https://pith.science/api/pith-number/7DSU5DOOKHNQKEEI7U4LJHPVM7/events.json","paper":"https://pith.science/paper/7DSU5DOO"},"agent_actions":{"view_html":"https://pith.science/pith/7DSU5DOOKHNQKEEI7U4LJHPVM7","download_json":"https://pith.science/pith/7DSU5DOOKHNQKEEI7U4LJHPVM7.json","view_paper":"https://pith.science/paper/7DSU5DOO","resolve_alias":"https://pith.science/api/pith-number/resolve?arxiv=2306.03241&json=true","fetch_graph":"https://pith.science/api/pith-number/7DSU5DOOKHNQKEEI7U4LJHPVM7/graph.json","fetch_events":"https://pith.science/api/pith-number/7DSU5DOOKHNQKEEI7U4LJHPVM7/events.json","actions":{"anchor_timestamp":"https://pith.science/pith/7DSU5DOOKHNQKEEI7U4LJHPVM7/action/timestamp_anchor","attest_storage":"https://pith.science/pith/7DSU5DOOKHNQKEEI7U4LJHPVM7/action/storage_attestation","attest_author":"https://pith.science/pith/7DSU5DOOKHNQKEEI7U4LJHPVM7/action/author_attestation","sign_citation":"https://pith.science/pith/7DSU5DOOKHNQKEEI7U4LJHPVM7/action/citation_signature","submit_replication":"https://pith.science/pith/7DSU5DOOKHNQKEEI7U4LJHPVM7/action/replication_record"}},"created_at":"2026-07-05T07:22:49.879013+00:00","updated_at":"2026-07-05T07:22:49.879013+00:00"}