{"bundle_type":"pith_open_graph_bundle","bundle_version":"1.0","pith_number":"pith:2024:NMCJO3JWVDE6EQQKNGCIAZYYRR","short_pith_number":"pith:NMCJO3JW","canonical_record":{"source":{"id":"2409.12903","kind":"arxiv","version":2},"metadata":{"license":"http://creativecommons.org/licenses/by-sa/4.0/","primary_cat":"cs.CL","submitted_at":"2024-09-19T16:50:26Z","cross_cats_sorted":["cs.AI","cs.LG"],"title_canon_sha256":"0edd43995332d1282a774453b6561a89eee42546930814555e98982b7baff5b7","abstract_canon_sha256":"b37b10906ca341a62c6bc0c38eda748452002e8f184fef7ab0858d1edc82dc33"},"schema_version":"1.0"},"canonical_sha256":"6b04976d36a8c9e2420a69848067188c6bbf680832a44ceb8d95a7f6b1cb147a","source":{"kind":"arxiv","id":"2409.12903","version":2},"source_aliases":[{"alias_kind":"arxiv","alias_value":"2409.12903","created_at":"2026-07-05T09:09:27Z"},{"alias_kind":"arxiv_version","alias_value":"2409.12903v2","created_at":"2026-07-05T09:09:27Z"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.2409.12903","created_at":"2026-07-05T09:09:27Z"},{"alias_kind":"pith_short_12","alias_value":"NMCJO3JWVDE6","created_at":"2026-07-05T09:09:27Z"},{"alias_kind":"pith_short_16","alias_value":"NMCJO3JWVDE6EQQK","created_at":"2026-07-05T09:09:27Z"},{"alias_kind":"pith_short_8","alias_value":"NMCJO3JW","created_at":"2026-07-05T09:09:27Z"}],"events":[{"event_type":"record_created","subject_pith_number":"pith:2024:NMCJO3JWVDE6EQQKNGCIAZYYRR","target":"record","payload":{"canonical_record":{"source":{"id":"2409.12903","kind":"arxiv","version":2},"metadata":{"license":"http://creativecommons.org/licenses/by-sa/4.0/","primary_cat":"cs.CL","submitted_at":"2024-09-19T16:50:26Z","cross_cats_sorted":["cs.AI","cs.LG"],"title_canon_sha256":"0edd43995332d1282a774453b6561a89eee42546930814555e98982b7baff5b7","abstract_canon_sha256":"b37b10906ca341a62c6bc0c38eda748452002e8f184fef7ab0858d1edc82dc33"},"schema_version":"1.0"},"canonical_sha256":"6b04976d36a8c9e2420a69848067188c6bbf680832a44ceb8d95a7f6b1cb147a","receipt":{"kind":"pith_receipt","key_id":"pith-v1-2026-05","algorithm":"ed25519","signed_at":"2026-07-05T09:09:27.006708Z","signature_b64":"4rApp14EOXalX/vwOsFxcinzJwO9q5IbZ4DT9RUOp7bzR7hp6jo5WpBMUZ1Zd/ZRdOCriXbJquerx3D1bNeNCg==","signed_message":"canonical_sha256_bytes","builder_version":"pith-number-builder-2026-05-17-v1","receipt_version":"0.3","canonical_sha256":"6b04976d36a8c9e2420a69848067188c6bbf680832a44ceb8d95a7f6b1cb147a","last_reissued_at":"2026-07-05T09:09:27.006205Z","signature_status":"signed_v1","first_computed_at":"2026-07-05T09:09:27.006205Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"source_kind":"arxiv","source_id":"2409.12903","source_version":2,"attestation_state":"computed"},"signer":{"signer_id":"pith.science","signer_type":"pith_registry","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"created_at":"2026-07-05T09:09:27Z","supersedes":[],"prev_event":null,"signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"AiZp4/iVXpQETYvc+956wWjUdaPqBjiXrKDEOYy3DT8J6Y3DUAKA7dndpljszRhzwZS3XDP76PWByEhCE72RBQ==","signed_message":"open_graph_event_sha256_bytes","signed_at":"2026-08-06T18:31:31.101412Z"},"content_sha256":"4eb969a8936f6b3e60ba0d7fa0cf8363dc591378e006b5f9521b4ff94636e7ff","schema_version":"1.0","event_id":"sha256:4eb969a8936f6b3e60ba0d7fa0cf8363dc591378e006b5f9521b4ff94636e7ff"},{"event_type":"graph_snapshot","subject_pith_number":"pith:2024:NMCJO3JWVDE6EQQKNGCIAZYYRR","target":"graph","payload":{"graph_snapshot":{"paper":{"title":"Scaling Smart: Accelerating Large Language Model Pre-training with Small Model Initialization","license":"http://creativecommons.org/licenses/by-sa/4.0/","headline":"","cross_cats":["cs.AI","cs.LG"],"primary_cat":"cs.CL","authors_text":"Devang Naik, Fartash Faghri, Iman Mirzadeh, Keivan Alizadeh Vahid, Mehrdad Farajtabar, Minsik Cho, Mohammad Samragh, Moin Nabi","submitted_at":"2024-09-19T16:50:26Z","abstract_excerpt":"The pre-training phase of language models often begins with randomly initialized parameters. With the current trends in scaling models, training their large number of parameters can be extremely slow and costly. In contrast, small language models are less expensive to train, but they often cannot achieve the accuracy of large models. In this paper, we explore an intriguing idea to connect these two different regimes: Can we develop a method to initialize large language models using smaller pre-trained models? Will such initialization bring any benefits in terms of training time and final accur"},"claims":{"count":0,"items":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"2409.12903","kind":"arxiv","version":2},"verdict":{"id":null,"model_set":{},"created_at":null,"strongest_claim":"","one_line_summary":"","pipeline_version":null,"weakest_assumption":"","pith_extraction_headline":""},"integrity":{"clean":true,"summary":{"advisory":0,"critical":0,"by_detector":{},"informational":0},"endpoint":"/pith/2409.12903/integrity.json","findings":[],"available":true,"detectors_run":[],"snapshot_sha256":"c28c3603d3b5d939e8dc4c7e95fa8dfce3d595e45f758748cecf8e644a296938"},"references":{"count":0,"sample":[],"resolved_work":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","internal_anchors":0},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"author_claims":{"count":0,"strong_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"builder_version":"pith-number-builder-2026-05-17-v1"},"verdict_id":null},"signer":{"signer_id":"pith.science","signer_type":"pith_registry","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"created_at":"2026-07-05T09:09:27Z","supersedes":[],"prev_event":null,"signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"PyPVpjis/Mnc6/aU7Kicmt2RrRAkFn14t5qi0m3PLURAUYAxmT0GL71PKLbgr7iKY5BKEEzxCiProVqdA5ckDA==","signed_message":"open_graph_event_sha256_bytes","signed_at":"2026-08-06T18:31:31.104685Z"},"content_sha256":"9afc283e5c04fefd210dc17d22c28bf110de7db7fbff5851b74820595712dd3a","schema_version":"1.0","event_id":"sha256:9afc283e5c04fefd210dc17d22c28bf110de7db7fbff5851b74820595712dd3a"}],"timestamp_proofs":[],"mirror_hints":[{"mirror_type":"https","name":"Pith Resolver","base_url":"https://pith.science","bundle_url":"https://pith.science/pith/NMCJO3JWVDE6EQQKNGCIAZYYRR/bundle.json","state_url":"https://pith.science/pith/NMCJO3JWVDE6EQQKNGCIAZYYRR/state.json","well_known_bundle_url":"https://pith.science/.well-known/pith/NMCJO3JWVDE6EQQKNGCIAZYYRR/bundle.json","status":"primary"}],"public_keys":[{"key_id":"pith-v1-2026-05","algorithm":"ed25519","format":"raw","public_key_b64":"stVStoiQhXFxp4s2pdzPNoqVNBMojDU/fJ2db5S3CbM=","public_key_hex":"b2d552b68890857171a78b36a5dccf368a953413288c353f7c9d9d6f94b709b3","fingerprint_sha256_b32_first128bits":"RVFV5Z2OI2J3ZUO7ERDEBCYNKS","fingerprint_sha256_hex":"8d4b5ee74e4693bcd1df2446408b0d54","rotates_at":null,"url":"https://pith.science/pith-signing-key.json","notes":"Pith uses this Ed25519 key to sign canonical record SHA-256 digests. Verify with: ed25519_verify(public_key, message=canonical_sha256_bytes, signature=base64decode(signature_b64))."}],"merge_version":"pith-open-graph-merge-v1","built_at":"2026-08-06T18:31:31Z","links":{"resolver":"https://pith.science/pith/NMCJO3JWVDE6EQQKNGCIAZYYRR","bundle":"https://pith.science/pith/NMCJO3JWVDE6EQQKNGCIAZYYRR/bundle.json","state":"https://pith.science/pith/NMCJO3JWVDE6EQQKNGCIAZYYRR/state.json","well_known_bundle":"https://pith.science/.well-known/pith/NMCJO3JWVDE6EQQKNGCIAZYYRR/bundle.json"},"state":{"state_type":"pith_open_graph_state","state_version":"1.0","pith_number":"pith:2024:NMCJO3JWVDE6EQQKNGCIAZYYRR","merge_version":"pith-open-graph-merge-v1","event_count":2,"valid_event_count":2,"invalid_event_count":0,"equivocation_count":0,"current":{"canonical_record":{"metadata":{"abstract_canon_sha256":"b37b10906ca341a62c6bc0c38eda748452002e8f184fef7ab0858d1edc82dc33","cross_cats_sorted":["cs.AI","cs.LG"],"license":"http://creativecommons.org/licenses/by-sa/4.0/","primary_cat":"cs.CL","submitted_at":"2024-09-19T16:50:26Z","title_canon_sha256":"0edd43995332d1282a774453b6561a89eee42546930814555e98982b7baff5b7"},"schema_version":"1.0","source":{"id":"2409.12903","kind":"arxiv","version":2}},"source_aliases":[{"alias_kind":"arxiv","alias_value":"2409.12903","created_at":"2026-07-05T09:09:27Z"},{"alias_kind":"arxiv_version","alias_value":"2409.12903v2","created_at":"2026-07-05T09:09:27Z"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.2409.12903","created_at":"2026-07-05T09:09:27Z"},{"alias_kind":"pith_short_12","alias_value":"NMCJO3JWVDE6","created_at":"2026-07-05T09:09:27Z"},{"alias_kind":"pith_short_16","alias_value":"NMCJO3JWVDE6EQQK","created_at":"2026-07-05T09:09:27Z"},{"alias_kind":"pith_short_8","alias_value":"NMCJO3JW","created_at":"2026-07-05T09:09:27Z"}],"graph_snapshots":[{"event_id":"sha256:9afc283e5c04fefd210dc17d22c28bf110de7db7fbff5851b74820595712dd3a","target":"graph","created_at":"2026-07-05T09:09:27Z","signer":{"key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signer_id":"pith.science","signer_type":"pith_registry"},"payload":{"graph_snapshot":{"author_claims":{"count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","strong_count":0},"builder_version":"pith-number-builder-2026-05-17-v1","claims":{"count":0,"items":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"integrity":{"available":true,"clean":true,"detectors_run":[],"endpoint":"/pith/2409.12903/integrity.json","findings":[],"snapshot_sha256":"c28c3603d3b5d939e8dc4c7e95fa8dfce3d595e45f758748cecf8e644a296938","summary":{"advisory":0,"by_detector":{},"critical":0,"informational":0}},"paper":{"abstract_excerpt":"The pre-training phase of language models often begins with randomly initialized parameters. With the current trends in scaling models, training their large number of parameters can be extremely slow and costly. In contrast, small language models are less expensive to train, but they often cannot achieve the accuracy of large models. In this paper, we explore an intriguing idea to connect these two different regimes: Can we develop a method to initialize large language models using smaller pre-trained models? Will such initialization bring any benefits in terms of training time and final accur","authors_text":"Devang Naik, Fartash Faghri, Iman Mirzadeh, Keivan Alizadeh Vahid, Mehrdad Farajtabar, Minsik Cho, Mohammad Samragh, Moin Nabi","cross_cats":["cs.AI","cs.LG"],"headline":"","license":"http://creativecommons.org/licenses/by-sa/4.0/","primary_cat":"cs.CL","submitted_at":"2024-09-19T16:50:26Z","title":"Scaling Smart: Accelerating Large Language Model Pre-training with Small Model Initialization"},"references":{"count":0,"internal_anchors":0,"resolved_work":0,"sample":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"2409.12903","kind":"arxiv","version":2},"verdict":{"created_at":null,"id":null,"model_set":{},"one_line_summary":"","pipeline_version":null,"pith_extraction_headline":"","strongest_claim":"","weakest_assumption":""}},"verdict_id":null}}],"author_attestations":[],"timestamp_anchors":[],"storage_attestations":[],"citation_signatures":[],"replication_records":[],"corrections":[],"mirror_hints":[],"record_created":{"event_id":"sha256:4eb969a8936f6b3e60ba0d7fa0cf8363dc591378e006b5f9521b4ff94636e7ff","target":"record","created_at":"2026-07-05T09:09:27Z","signer":{"key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signer_id":"pith.science","signer_type":"pith_registry"},"payload":{"attestation_state":"computed","canonical_record":{"metadata":{"abstract_canon_sha256":"b37b10906ca341a62c6bc0c38eda748452002e8f184fef7ab0858d1edc82dc33","cross_cats_sorted":["cs.AI","cs.LG"],"license":"http://creativecommons.org/licenses/by-sa/4.0/","primary_cat":"cs.CL","submitted_at":"2024-09-19T16:50:26Z","title_canon_sha256":"0edd43995332d1282a774453b6561a89eee42546930814555e98982b7baff5b7"},"schema_version":"1.0","source":{"id":"2409.12903","kind":"arxiv","version":2}},"canonical_sha256":"6b04976d36a8c9e2420a69848067188c6bbf680832a44ceb8d95a7f6b1cb147a","receipt":{"algorithm":"ed25519","builder_version":"pith-number-builder-2026-05-17-v1","canonical_sha256":"6b04976d36a8c9e2420a69848067188c6bbf680832a44ceb8d95a7f6b1cb147a","first_computed_at":"2026-07-05T09:09:27.006205Z","key_id":"pith-v1-2026-05","kind":"pith_receipt","last_reissued_at":"2026-07-05T09:09:27.006205Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","receipt_version":"0.3","signature_b64":"4rApp14EOXalX/vwOsFxcinzJwO9q5IbZ4DT9RUOp7bzR7hp6jo5WpBMUZ1Zd/ZRdOCriXbJquerx3D1bNeNCg==","signature_status":"signed_v1","signed_at":"2026-07-05T09:09:27.006708Z","signed_message":"canonical_sha256_bytes"},"source_id":"2409.12903","source_kind":"arxiv","source_version":2}}},"equivocations":[],"invalid_events":[],"applied_event_ids":["sha256:4eb969a8936f6b3e60ba0d7fa0cf8363dc591378e006b5f9521b4ff94636e7ff","sha256:9afc283e5c04fefd210dc17d22c28bf110de7db7fbff5851b74820595712dd3a"],"state_sha256":"23a5b3e7235bcd53b86ab7537b9900a585d5f002051236d15277a3fb34feaaca"},"bundle_signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"zH/bjFhjmVBV2FWu7p84C9ReJ3D8cV0d39z3rZhhJmD/8p9+st8FJrwDU7+LKvYd4D7CPnVp1ctYffpM0FxaAg==","signed_message":"bundle_sha256_bytes","signed_at":"2026-08-06T18:31:31.112102Z","bundle_sha256":"01c138c96f29a841128de30a415eeb054e747a1ccf74ec2821d1fa36a0e49016"}}