{"bundle_type":"pith_open_graph_bundle","bundle_version":"1.0","pith_number":"pith:2024:JMASYTRWUTLX2VZNE4UTIO3ZCZ","short_pith_number":"pith:JMASYTRW","canonical_record":{"source":{"id":"2410.05052","kind":"arxiv","version":1},"metadata":{"license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.CL","submitted_at":"2024-10-07T14:09:58Z","cross_cats_sorted":[],"title_canon_sha256":"b5db77dec77fffb1ebe750b2e0c12c85030f426fab04630af20afc20dda055ce","abstract_canon_sha256":"7dd2d04ad87bbf6ee1ce7e4950400fb93bd812d4d7b9bae3579b9ebfe9ed1c52"},"schema_version":"1.0"},"canonical_sha256":"4b012c4e36a4d77d572d2729343b79164cea5a85d2e3a3d624bdafaa98c3f73d","source":{"kind":"arxiv","id":"2410.05052","version":1},"source_aliases":[{"alias_kind":"arxiv","alias_value":"2410.05052","created_at":"2026-07-05T09:17:00Z"},{"alias_kind":"arxiv_version","alias_value":"2410.05052v1","created_at":"2026-07-05T09:17:00Z"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.2410.05052","created_at":"2026-07-05T09:17:00Z"},{"alias_kind":"pith_short_12","alias_value":"JMASYTRWUTLX","created_at":"2026-07-05T09:17:00Z"},{"alias_kind":"pith_short_16","alias_value":"JMASYTRWUTLX2VZN","created_at":"2026-07-05T09:17:00Z"},{"alias_kind":"pith_short_8","alias_value":"JMASYTRW","created_at":"2026-07-05T09:17:00Z"}],"events":[{"event_type":"record_created","subject_pith_number":"pith:2024:JMASYTRWUTLX2VZNE4UTIO3ZCZ","target":"record","payload":{"canonical_record":{"source":{"id":"2410.05052","kind":"arxiv","version":1},"metadata":{"license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.CL","submitted_at":"2024-10-07T14:09:58Z","cross_cats_sorted":[],"title_canon_sha256":"b5db77dec77fffb1ebe750b2e0c12c85030f426fab04630af20afc20dda055ce","abstract_canon_sha256":"7dd2d04ad87bbf6ee1ce7e4950400fb93bd812d4d7b9bae3579b9ebfe9ed1c52"},"schema_version":"1.0"},"canonical_sha256":"4b012c4e36a4d77d572d2729343b79164cea5a85d2e3a3d624bdafaa98c3f73d","receipt":{"kind":"pith_receipt","key_id":"pith-v1-2026-05","algorithm":"ed25519","signed_at":"2026-07-05T09:17:00.707008Z","signature_b64":"4UspDvVUVFxyQsy7s+Iw6HkCc+IMeLrwgrSuzyrbX1A5ZNWUAix1UCxNlERvFA/3xlSW53wS1oDMKJtXXD8EBQ==","signed_message":"canonical_sha256_bytes","builder_version":"pith-number-builder-2026-05-17-v1","receipt_version":"0.3","canonical_sha256":"4b012c4e36a4d77d572d2729343b79164cea5a85d2e3a3d624bdafaa98c3f73d","last_reissued_at":"2026-07-05T09:17:00.706593Z","signature_status":"signed_v1","first_computed_at":"2026-07-05T09:17:00.706593Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"source_kind":"arxiv","source_id":"2410.05052","source_version":1,"attestation_state":"computed"},"signer":{"signer_id":"pith.science","signer_type":"pith_registry","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"created_at":"2026-07-05T09:17:00Z","supersedes":[],"prev_event":null,"signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"ljD+Uh0AZS62Dr3bo9iKTBaDhCizrdGBAwwGSCxcNdijdEGRrrZWdpxUxgjJ84TCEDhRIZJsPifb4GxU/79RBQ==","signed_message":"open_graph_event_sha256_bytes","signed_at":"2026-07-08T16:19:42.311277Z"},"content_sha256":"72a43c2ed31a824c281cf697837db87ce915fe29987289153e5eeec18c3c9df7","schema_version":"1.0","event_id":"sha256:72a43c2ed31a824c281cf697837db87ce915fe29987289153e5eeec18c3c9df7"},{"event_type":"graph_snapshot","subject_pith_number":"pith:2024:JMASYTRWUTLX2VZNE4UTIO3ZCZ","target":"graph","payload":{"graph_snapshot":{"paper":{"title":"Initialization of Large Language Models via Reparameterization to Mitigate Loss Spikes","license":"http://creativecommons.org/licenses/by/4.0/","headline":"","cross_cats":[],"primary_cat":"cs.CL","authors_text":"Kosuke Nishida, Kuniko Saito, Kyosuke Nishida","submitted_at":"2024-10-07T14:09:58Z","abstract_excerpt":"Loss spikes, a phenomenon in which the loss value diverges suddenly, is a fundamental issue in the pre-training of large language models. This paper supposes that the non-uniformity of the norm of the parameters is one of the causes of loss spikes. Here, in training of neural networks, the scale of the gradients is required to be kept constant throughout the layers to avoid the vanishing and exploding gradients problem. However, to meet these requirements in the Transformer model, the norm of the model parameters must be non-uniform, and thus, parameters whose norm is smaller are more sensitiv"},"claims":{"count":0,"items":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"2410.05052","kind":"arxiv","version":1},"verdict":{"id":null,"model_set":{},"created_at":null,"strongest_claim":"","one_line_summary":"","pipeline_version":null,"weakest_assumption":"","pith_extraction_headline":""},"integrity":{"clean":true,"summary":{"advisory":0,"critical":0,"by_detector":{},"informational":0},"endpoint":"/pith/2410.05052/integrity.json","findings":[],"available":true,"detectors_run":[],"snapshot_sha256":"c28c3603d3b5d939e8dc4c7e95fa8dfce3d595e45f758748cecf8e644a296938"},"references":{"count":0,"sample":[],"resolved_work":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","internal_anchors":0},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"author_claims":{"count":0,"strong_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"builder_version":"pith-number-builder-2026-05-17-v1"},"verdict_id":null},"signer":{"signer_id":"pith.science","signer_type":"pith_registry","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"created_at":"2026-07-05T09:17:00Z","supersedes":[],"prev_event":null,"signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"Fw9V8Vwce39iFXrMx7cq0jKbpOnPYudNjmNekPZuk7GpiV52xLfEZDFHVa2X2CLBYDmXSrQBaSF8bQ+yi1XBDg==","signed_message":"open_graph_event_sha256_bytes","signed_at":"2026-07-08T16:19:42.311696Z"},"content_sha256":"b44d3e8f198ab16a5daf8236d02e02def32ae837e3451c33714f9f5182f9169e","schema_version":"1.0","event_id":"sha256:b44d3e8f198ab16a5daf8236d02e02def32ae837e3451c33714f9f5182f9169e"}],"timestamp_proofs":[],"mirror_hints":[{"mirror_type":"https","name":"Pith Resolver","base_url":"https://pith.science","bundle_url":"https://pith.science/pith/JMASYTRWUTLX2VZNE4UTIO3ZCZ/bundle.json","state_url":"https://pith.science/pith/JMASYTRWUTLX2VZNE4UTIO3ZCZ/state.json","well_known_bundle_url":"https://pith.science/.well-known/pith/JMASYTRWUTLX2VZNE4UTIO3ZCZ/bundle.json","status":"primary"}],"public_keys":[{"key_id":"pith-v1-2026-05","algorithm":"ed25519","format":"raw","public_key_b64":"stVStoiQhXFxp4s2pdzPNoqVNBMojDU/fJ2db5S3CbM=","public_key_hex":"b2d552b68890857171a78b36a5dccf368a953413288c353f7c9d9d6f94b709b3","fingerprint_sha256_b32_first128bits":"RVFV5Z2OI2J3ZUO7ERDEBCYNKS","fingerprint_sha256_hex":"8d4b5ee74e4693bcd1df2446408b0d54","rotates_at":null,"url":"https://pith.science/pith-signing-key.json","notes":"Pith uses this Ed25519 key to sign canonical record SHA-256 digests. Verify with: ed25519_verify(public_key, message=canonical_sha256_bytes, signature=base64decode(signature_b64))."}],"merge_version":"pith-open-graph-merge-v1","built_at":"2026-07-08T16:19:42Z","links":{"resolver":"https://pith.science/pith/JMASYTRWUTLX2VZNE4UTIO3ZCZ","bundle":"https://pith.science/pith/JMASYTRWUTLX2VZNE4UTIO3ZCZ/bundle.json","state":"https://pith.science/pith/JMASYTRWUTLX2VZNE4UTIO3ZCZ/state.json","well_known_bundle":"https://pith.science/.well-known/pith/JMASYTRWUTLX2VZNE4UTIO3ZCZ/bundle.json"},"state":{"state_type":"pith_open_graph_state","state_version":"1.0","pith_number":"pith:2024:JMASYTRWUTLX2VZNE4UTIO3ZCZ","merge_version":"pith-open-graph-merge-v1","event_count":2,"valid_event_count":2,"invalid_event_count":0,"equivocation_count":0,"current":{"canonical_record":{"metadata":{"abstract_canon_sha256":"7dd2d04ad87bbf6ee1ce7e4950400fb93bd812d4d7b9bae3579b9ebfe9ed1c52","cross_cats_sorted":[],"license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.CL","submitted_at":"2024-10-07T14:09:58Z","title_canon_sha256":"b5db77dec77fffb1ebe750b2e0c12c85030f426fab04630af20afc20dda055ce"},"schema_version":"1.0","source":{"id":"2410.05052","kind":"arxiv","version":1}},"source_aliases":[{"alias_kind":"arxiv","alias_value":"2410.05052","created_at":"2026-07-05T09:17:00Z"},{"alias_kind":"arxiv_version","alias_value":"2410.05052v1","created_at":"2026-07-05T09:17:00Z"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.2410.05052","created_at":"2026-07-05T09:17:00Z"},{"alias_kind":"pith_short_12","alias_value":"JMASYTRWUTLX","created_at":"2026-07-05T09:17:00Z"},{"alias_kind":"pith_short_16","alias_value":"JMASYTRWUTLX2VZN","created_at":"2026-07-05T09:17:00Z"},{"alias_kind":"pith_short_8","alias_value":"JMASYTRW","created_at":"2026-07-05T09:17:00Z"}],"graph_snapshots":[{"event_id":"sha256:b44d3e8f198ab16a5daf8236d02e02def32ae837e3451c33714f9f5182f9169e","target":"graph","created_at":"2026-07-05T09:17:00Z","signer":{"key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signer_id":"pith.science","signer_type":"pith_registry"},"payload":{"graph_snapshot":{"author_claims":{"count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","strong_count":0},"builder_version":"pith-number-builder-2026-05-17-v1","claims":{"count":0,"items":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"integrity":{"available":true,"clean":true,"detectors_run":[],"endpoint":"/pith/2410.05052/integrity.json","findings":[],"snapshot_sha256":"c28c3603d3b5d939e8dc4c7e95fa8dfce3d595e45f758748cecf8e644a296938","summary":{"advisory":0,"by_detector":{},"critical":0,"informational":0}},"paper":{"abstract_excerpt":"Loss spikes, a phenomenon in which the loss value diverges suddenly, is a fundamental issue in the pre-training of large language models. This paper supposes that the non-uniformity of the norm of the parameters is one of the causes of loss spikes. Here, in training of neural networks, the scale of the gradients is required to be kept constant throughout the layers to avoid the vanishing and exploding gradients problem. However, to meet these requirements in the Transformer model, the norm of the model parameters must be non-uniform, and thus, parameters whose norm is smaller are more sensitiv","authors_text":"Kosuke Nishida, Kuniko Saito, Kyosuke Nishida","cross_cats":[],"headline":"","license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.CL","submitted_at":"2024-10-07T14:09:58Z","title":"Initialization of Large Language Models via Reparameterization to Mitigate Loss Spikes"},"references":{"count":0,"internal_anchors":0,"resolved_work":0,"sample":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"2410.05052","kind":"arxiv","version":1},"verdict":{"created_at":null,"id":null,"model_set":{},"one_line_summary":"","pipeline_version":null,"pith_extraction_headline":"","strongest_claim":"","weakest_assumption":""}},"verdict_id":null}}],"author_attestations":[],"timestamp_anchors":[],"storage_attestations":[],"citation_signatures":[],"replication_records":[],"corrections":[],"mirror_hints":[],"record_created":{"event_id":"sha256:72a43c2ed31a824c281cf697837db87ce915fe29987289153e5eeec18c3c9df7","target":"record","created_at":"2026-07-05T09:17:00Z","signer":{"key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signer_id":"pith.science","signer_type":"pith_registry"},"payload":{"attestation_state":"computed","canonical_record":{"metadata":{"abstract_canon_sha256":"7dd2d04ad87bbf6ee1ce7e4950400fb93bd812d4d7b9bae3579b9ebfe9ed1c52","cross_cats_sorted":[],"license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.CL","submitted_at":"2024-10-07T14:09:58Z","title_canon_sha256":"b5db77dec77fffb1ebe750b2e0c12c85030f426fab04630af20afc20dda055ce"},"schema_version":"1.0","source":{"id":"2410.05052","kind":"arxiv","version":1}},"canonical_sha256":"4b012c4e36a4d77d572d2729343b79164cea5a85d2e3a3d624bdafaa98c3f73d","receipt":{"algorithm":"ed25519","builder_version":"pith-number-builder-2026-05-17-v1","canonical_sha256":"4b012c4e36a4d77d572d2729343b79164cea5a85d2e3a3d624bdafaa98c3f73d","first_computed_at":"2026-07-05T09:17:00.706593Z","key_id":"pith-v1-2026-05","kind":"pith_receipt","last_reissued_at":"2026-07-05T09:17:00.706593Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","receipt_version":"0.3","signature_b64":"4UspDvVUVFxyQsy7s+Iw6HkCc+IMeLrwgrSuzyrbX1A5ZNWUAix1UCxNlERvFA/3xlSW53wS1oDMKJtXXD8EBQ==","signature_status":"signed_v1","signed_at":"2026-07-05T09:17:00.707008Z","signed_message":"canonical_sha256_bytes"},"source_id":"2410.05052","source_kind":"arxiv","source_version":1}}},"equivocations":[],"invalid_events":[],"applied_event_ids":["sha256:72a43c2ed31a824c281cf697837db87ce915fe29987289153e5eeec18c3c9df7","sha256:b44d3e8f198ab16a5daf8236d02e02def32ae837e3451c33714f9f5182f9169e"],"state_sha256":"bbdafba88641999cb2b457e0e2fd47b8ac47c484570ee7fea1344ac297d3f47e"},"bundle_signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"gFaKfaMYWQjATeyFmxrDwA9ZxPODiHU0i+5GIbbvi68FFDphftfai8rtr3w3h064oiFw5zLb3xND7MEtDHj4AQ==","signed_message":"bundle_sha256_bytes","signed_at":"2026-07-08T16:19:42.313740Z","bundle_sha256":"4242fc9745539f60967131d87959d1d521e54e9911e357eee76b9c940932c64c"}}