{"bundle_type":"pith_open_graph_bundle","bundle_version":"1.0","pith_number":"pith:2020:XO623QCLIJ4IAJZGR2EGCVU5CW","short_pith_number":"pith:XO623QCL","canonical_record":{"source":{"id":"2010.09697","kind":"arxiv","version":5},"metadata":{"license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","primary_cat":"cs.LG","submitted_at":"2020-10-19T17:40:38Z","cross_cats_sorted":["cs.CL"],"title_canon_sha256":"42420202fe1aee0fdc109f45599d08ec4d72d46d584bf8a99b338dda8276de53","abstract_canon_sha256":"7952a381eb584be59494641d30eef823d6d0c91bf49484db9e3108a340afd8b5"},"schema_version":"1.0"},"canonical_sha256":"bbbdadc04b42788027268e8861569d15ba978ae712466afef159e693bbe575a2","source":{"kind":"arxiv","id":"2010.09697","version":5},"source_aliases":[{"alias_kind":"arxiv","alias_value":"2010.09697","created_at":"2026-07-05T05:49:11Z"},{"alias_kind":"arxiv_version","alias_value":"2010.09697v5","created_at":"2026-07-05T05:49:11Z"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.2010.09697","created_at":"2026-07-05T05:49:11Z"},{"alias_kind":"pith_short_12","alias_value":"XO623QCLIJ4I","created_at":"2026-07-05T05:49:11Z"},{"alias_kind":"pith_short_16","alias_value":"XO623QCLIJ4IAJZG","created_at":"2026-07-05T05:49:11Z"},{"alias_kind":"pith_short_8","alias_value":"XO623QCL","created_at":"2026-07-05T05:49:11Z"}],"events":[{"event_type":"record_created","subject_pith_number":"pith:2020:XO623QCLIJ4IAJZGR2EGCVU5CW","target":"record","payload":{"canonical_record":{"source":{"id":"2010.09697","kind":"arxiv","version":5},"metadata":{"license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","primary_cat":"cs.LG","submitted_at":"2020-10-19T17:40:38Z","cross_cats_sorted":["cs.CL"],"title_canon_sha256":"42420202fe1aee0fdc109f45599d08ec4d72d46d584bf8a99b338dda8276de53","abstract_canon_sha256":"7952a381eb584be59494641d30eef823d6d0c91bf49484db9e3108a340afd8b5"},"schema_version":"1.0"},"canonical_sha256":"bbbdadc04b42788027268e8861569d15ba978ae712466afef159e693bbe575a2","receipt":{"kind":"pith_receipt","key_id":"pith-v1-2026-05","algorithm":"ed25519","signed_at":"2026-07-05T05:49:11.294422Z","signature_b64":"KNvnyRCm72AJamPc7uS9DYSVG4fvyC8D3nb56ADTU09iAizkbeRPdXXONb0wQkT9Ftr8XctpoNs1YqjnMgw8Cw==","signed_message":"canonical_sha256_bytes","builder_version":"pith-number-builder-2026-05-17-v1","receipt_version":"0.3","canonical_sha256":"bbbdadc04b42788027268e8861569d15ba978ae712466afef159e693bbe575a2","last_reissued_at":"2026-07-05T05:49:11.293772Z","signature_status":"signed_v1","first_computed_at":"2026-07-05T05:49:11.293772Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"source_kind":"arxiv","source_id":"2010.09697","source_version":5,"attestation_state":"computed"},"signer":{"signer_id":"pith.science","signer_type":"pith_registry","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"created_at":"2026-07-05T05:49:11Z","supersedes":[],"prev_event":null,"signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"e15EymRM6chR7Y0qaK63s6A1xJ2zHzirdNlzB40EqFw7RBE3nMfCNX+7Rf7D3P5giGUmjw/zeCh2NxzwGYBVAg==","signed_message":"open_graph_event_sha256_bytes","signed_at":"2026-08-15T05:47:20.914799Z"},"content_sha256":"2b7ed1d8cae4466e51447e33dc7cdee4cb87e8a34aa8c997c2be79913c6bcb26","schema_version":"1.0","event_id":"sha256:2b7ed1d8cae4466e51447e33dc7cdee4cb87e8a34aa8c997c2be79913c6bcb26"},{"event_type":"graph_snapshot","subject_pith_number":"pith:2020:XO623QCLIJ4IAJZGR2EGCVU5CW","target":"graph","payload":{"graph_snapshot":{"paper":{"title":"Effects of Parameter Norm Growth During Transformer Training: Inductive Bias from Gradient Descent","license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","headline":"","cross_cats":["cs.CL"],"primary_cat":"cs.LG","authors_text":"Noah Smith, Roy Schwartz, Vivek Ramanujan, William Merrill, Yoav Goldberg","submitted_at":"2020-10-19T17:40:38Z","abstract_excerpt":"The capacity of neural networks like the widely adopted transformer is known to be very high. Evidence is emerging that they learn successfully due to inductive bias in the training routine, typically a variant of gradient descent (GD). To better understand this bias, we study the tendency for transformer parameters to grow in magnitude ($\\ell_2$ norm) during training, and its implications for the emergent representations within self attention layers. Empirically, we document norm growth in the training of transformer language models, including T5 during its pretraining. As the parameters grow"},"claims":{"count":0,"items":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"2010.09697","kind":"arxiv","version":5},"verdict":{"id":null,"model_set":{},"created_at":null,"strongest_claim":"","one_line_summary":"","pipeline_version":null,"weakest_assumption":"","pith_extraction_headline":""},"integrity":{"clean":true,"summary":{"advisory":0,"critical":0,"by_detector":{},"informational":0},"endpoint":"/pith/2010.09697/integrity.json","findings":[],"available":true,"detectors_run":[],"snapshot_sha256":"c28c3603d3b5d939e8dc4c7e95fa8dfce3d595e45f758748cecf8e644a296938"},"references":{"count":0,"sample":[],"resolved_work":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","internal_anchors":0},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"author_claims":{"count":0,"strong_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"builder_version":"pith-number-builder-2026-05-17-v1"},"verdict_id":null},"signer":{"signer_id":"pith.science","signer_type":"pith_registry","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"created_at":"2026-07-05T05:49:11Z","supersedes":[],"prev_event":null,"signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"ml9QrJ6xBM8VFQTHOns4iwpYDY1lWgDPuhajoBwQ/ie/3pnikpG+NJTLhRK7+wJBmNSB+629QFd/7TZgF4JfBg==","signed_message":"open_graph_event_sha256_bytes","signed_at":"2026-08-15T05:47:20.915313Z"},"content_sha256":"932d84fb5d829ad2d192626f7ac62a462dfaa1abd8d6ba8fd9ff9914a4ed322e","schema_version":"1.0","event_id":"sha256:932d84fb5d829ad2d192626f7ac62a462dfaa1abd8d6ba8fd9ff9914a4ed322e"}],"timestamp_proofs":[],"mirror_hints":[{"mirror_type":"https","name":"Pith Resolver","base_url":"https://pith.science","bundle_url":"https://pith.science/pith/XO623QCLIJ4IAJZGR2EGCVU5CW/bundle.json","state_url":"https://pith.science/pith/XO623QCLIJ4IAJZGR2EGCVU5CW/state.json","well_known_bundle_url":"https://pith.science/.well-known/pith/XO623QCLIJ4IAJZGR2EGCVU5CW/bundle.json","status":"primary"}],"public_keys":[{"key_id":"pith-v1-2026-05","algorithm":"ed25519","format":"raw","public_key_b64":"stVStoiQhXFxp4s2pdzPNoqVNBMojDU/fJ2db5S3CbM=","public_key_hex":"b2d552b68890857171a78b36a5dccf368a953413288c353f7c9d9d6f94b709b3","fingerprint_sha256_b32_first128bits":"RVFV5Z2OI2J3ZUO7ERDEBCYNKS","fingerprint_sha256_hex":"8d4b5ee74e4693bcd1df2446408b0d54","rotates_at":null,"url":"https://pith.science/pith-signing-key.json","notes":"Pith uses this Ed25519 key to sign canonical record SHA-256 digests. Verify with: ed25519_verify(public_key, message=canonical_sha256_bytes, signature=base64decode(signature_b64))."}],"merge_version":"pith-open-graph-merge-v1","built_at":"2026-08-15T05:47:20Z","links":{"resolver":"https://pith.science/pith/XO623QCLIJ4IAJZGR2EGCVU5CW","bundle":"https://pith.science/pith/XO623QCLIJ4IAJZGR2EGCVU5CW/bundle.json","state":"https://pith.science/pith/XO623QCLIJ4IAJZGR2EGCVU5CW/state.json","well_known_bundle":"https://pith.science/.well-known/pith/XO623QCLIJ4IAJZGR2EGCVU5CW/bundle.json"},"state":{"state_type":"pith_open_graph_state","state_version":"1.0","pith_number":"pith:2020:XO623QCLIJ4IAJZGR2EGCVU5CW","merge_version":"pith-open-graph-merge-v1","event_count":2,"valid_event_count":2,"invalid_event_count":0,"equivocation_count":0,"current":{"canonical_record":{"metadata":{"abstract_canon_sha256":"7952a381eb584be59494641d30eef823d6d0c91bf49484db9e3108a340afd8b5","cross_cats_sorted":["cs.CL"],"license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","primary_cat":"cs.LG","submitted_at":"2020-10-19T17:40:38Z","title_canon_sha256":"42420202fe1aee0fdc109f45599d08ec4d72d46d584bf8a99b338dda8276de53"},"schema_version":"1.0","source":{"id":"2010.09697","kind":"arxiv","version":5}},"source_aliases":[{"alias_kind":"arxiv","alias_value":"2010.09697","created_at":"2026-07-05T05:49:11Z"},{"alias_kind":"arxiv_version","alias_value":"2010.09697v5","created_at":"2026-07-05T05:49:11Z"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.2010.09697","created_at":"2026-07-05T05:49:11Z"},{"alias_kind":"pith_short_12","alias_value":"XO623QCLIJ4I","created_at":"2026-07-05T05:49:11Z"},{"alias_kind":"pith_short_16","alias_value":"XO623QCLIJ4IAJZG","created_at":"2026-07-05T05:49:11Z"},{"alias_kind":"pith_short_8","alias_value":"XO623QCL","created_at":"2026-07-05T05:49:11Z"}],"graph_snapshots":[{"event_id":"sha256:932d84fb5d829ad2d192626f7ac62a462dfaa1abd8d6ba8fd9ff9914a4ed322e","target":"graph","created_at":"2026-07-05T05:49:11Z","signer":{"key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signer_id":"pith.science","signer_type":"pith_registry"},"payload":{"graph_snapshot":{"author_claims":{"count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","strong_count":0},"builder_version":"pith-number-builder-2026-05-17-v1","claims":{"count":0,"items":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"integrity":{"available":true,"clean":true,"detectors_run":[],"endpoint":"/pith/2010.09697/integrity.json","findings":[],"snapshot_sha256":"c28c3603d3b5d939e8dc4c7e95fa8dfce3d595e45f758748cecf8e644a296938","summary":{"advisory":0,"by_detector":{},"critical":0,"informational":0}},"paper":{"abstract_excerpt":"The capacity of neural networks like the widely adopted transformer is known to be very high. Evidence is emerging that they learn successfully due to inductive bias in the training routine, typically a variant of gradient descent (GD). To better understand this bias, we study the tendency for transformer parameters to grow in magnitude ($\\ell_2$ norm) during training, and its implications for the emergent representations within self attention layers. Empirically, we document norm growth in the training of transformer language models, including T5 during its pretraining. As the parameters grow","authors_text":"Noah Smith, Roy Schwartz, Vivek Ramanujan, William Merrill, Yoav Goldberg","cross_cats":["cs.CL"],"headline":"","license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","primary_cat":"cs.LG","submitted_at":"2020-10-19T17:40:38Z","title":"Effects of Parameter Norm Growth During Transformer Training: Inductive Bias from Gradient Descent"},"references":{"count":0,"internal_anchors":0,"resolved_work":0,"sample":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"2010.09697","kind":"arxiv","version":5},"verdict":{"created_at":null,"id":null,"model_set":{},"one_line_summary":"","pipeline_version":null,"pith_extraction_headline":"","strongest_claim":"","weakest_assumption":""}},"verdict_id":null}}],"author_attestations":[],"timestamp_anchors":[],"storage_attestations":[],"citation_signatures":[],"replication_records":[],"corrections":[],"mirror_hints":[],"record_created":{"event_id":"sha256:2b7ed1d8cae4466e51447e33dc7cdee4cb87e8a34aa8c997c2be79913c6bcb26","target":"record","created_at":"2026-07-05T05:49:11Z","signer":{"key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signer_id":"pith.science","signer_type":"pith_registry"},"payload":{"attestation_state":"computed","canonical_record":{"metadata":{"abstract_canon_sha256":"7952a381eb584be59494641d30eef823d6d0c91bf49484db9e3108a340afd8b5","cross_cats_sorted":["cs.CL"],"license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","primary_cat":"cs.LG","submitted_at":"2020-10-19T17:40:38Z","title_canon_sha256":"42420202fe1aee0fdc109f45599d08ec4d72d46d584bf8a99b338dda8276de53"},"schema_version":"1.0","source":{"id":"2010.09697","kind":"arxiv","version":5}},"canonical_sha256":"bbbdadc04b42788027268e8861569d15ba978ae712466afef159e693bbe575a2","receipt":{"algorithm":"ed25519","builder_version":"pith-number-builder-2026-05-17-v1","canonical_sha256":"bbbdadc04b42788027268e8861569d15ba978ae712466afef159e693bbe575a2","first_computed_at":"2026-07-05T05:49:11.293772Z","key_id":"pith-v1-2026-05","kind":"pith_receipt","last_reissued_at":"2026-07-05T05:49:11.293772Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","receipt_version":"0.3","signature_b64":"KNvnyRCm72AJamPc7uS9DYSVG4fvyC8D3nb56ADTU09iAizkbeRPdXXONb0wQkT9Ftr8XctpoNs1YqjnMgw8Cw==","signature_status":"signed_v1","signed_at":"2026-07-05T05:49:11.294422Z","signed_message":"canonical_sha256_bytes"},"source_id":"2010.09697","source_kind":"arxiv","source_version":5}}},"equivocations":[],"invalid_events":[],"applied_event_ids":["sha256:2b7ed1d8cae4466e51447e33dc7cdee4cb87e8a34aa8c997c2be79913c6bcb26","sha256:932d84fb5d829ad2d192626f7ac62a462dfaa1abd8d6ba8fd9ff9914a4ed322e"],"state_sha256":"58d2e80bc9422e29ee018379055f88b80e76aca6c619f9ed14c684ce1b75fa08"},"bundle_signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"yGVm1+p0aGjeTfvMG284lMxVq/ymIhXRwcHwiL7jBx/dqFFsHQKaMyO6mlxxq0BIt3LiJ5nPv67sOTxNeGPdBQ==","signed_message":"bundle_sha256_bytes","signed_at":"2026-08-15T05:47:20.918433Z","bundle_sha256":"5b8c86007f793baef2638545cc48d1d30390c28d5cea64fc689f66ded3d8479d"}}