{"bundle_type":"pith_open_graph_bundle","bundle_version":"1.0","pith_number":"pith:2023:647CWRTCVSWMAGFIJFPGPMMYWA","short_pith_number":"pith:647CWRTC","canonical_record":{"source":{"id":"2309.14322","kind":"arxiv","version":2},"metadata":{"license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","primary_cat":"cs.LG","submitted_at":"2023-09-25T17:48:51Z","cross_cats_sorted":[],"title_canon_sha256":"692522a05eafd3228d7544e2a06deaf5da8ba56e40d3abf861602d862b391de5","abstract_canon_sha256":"e3a198a9dc3da292a34f42811b74e4af7ee2aa1c10bbe952465d9c4039e7d793"},"schema_version":"1.0"},"canonical_sha256":"f73e2b4662acacc018a8495e67b198b0288e669ba39fee2cae4b847469edc3de","source":{"kind":"arxiv","id":"2309.14322","version":2},"source_aliases":[{"alias_kind":"arxiv","alias_value":"2309.14322","created_at":"2026-07-05T07:01:25Z"},{"alias_kind":"arxiv_version","alias_value":"2309.14322v2","created_at":"2026-07-05T07:01:25Z"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.2309.14322","created_at":"2026-07-05T07:01:25Z"},{"alias_kind":"pith_short_12","alias_value":"647CWRTCVSWM","created_at":"2026-07-05T07:01:25Z"},{"alias_kind":"pith_short_16","alias_value":"647CWRTCVSWMAGFI","created_at":"2026-07-05T07:01:25Z"},{"alias_kind":"pith_short_8","alias_value":"647CWRTC","created_at":"2026-07-05T07:01:25Z"}],"events":[{"event_type":"record_created","subject_pith_number":"pith:2023:647CWRTCVSWMAGFIJFPGPMMYWA","target":"record","payload":{"canonical_record":{"source":{"id":"2309.14322","kind":"arxiv","version":2},"metadata":{"license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","primary_cat":"cs.LG","submitted_at":"2023-09-25T17:48:51Z","cross_cats_sorted":[],"title_canon_sha256":"692522a05eafd3228d7544e2a06deaf5da8ba56e40d3abf861602d862b391de5","abstract_canon_sha256":"e3a198a9dc3da292a34f42811b74e4af7ee2aa1c10bbe952465d9c4039e7d793"},"schema_version":"1.0"},"canonical_sha256":"f73e2b4662acacc018a8495e67b198b0288e669ba39fee2cae4b847469edc3de","receipt":{"kind":"pith_receipt","key_id":"pith-v1-2026-05","algorithm":"ed25519","signed_at":"2026-07-05T07:01:25.755306Z","signature_b64":"fJMzcPk7n86j3NFHrdaBFHVlvqUJOneYRjXscmbJxFDdjQZgmIV93JSYThErXSdszRZHQk9e51xipvRf2w3FDA==","signed_message":"canonical_sha256_bytes","builder_version":"pith-number-builder-2026-05-17-v1","receipt_version":"0.3","canonical_sha256":"f73e2b4662acacc018a8495e67b198b0288e669ba39fee2cae4b847469edc3de","last_reissued_at":"2026-07-05T07:01:25.754842Z","signature_status":"signed_v1","first_computed_at":"2026-07-05T07:01:25.754842Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"source_kind":"arxiv","source_id":"2309.14322","source_version":2,"attestation_state":"computed"},"signer":{"signer_id":"pith.science","signer_type":"pith_registry","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"created_at":"2026-07-05T07:01:25Z","supersedes":[],"prev_event":null,"signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"inomAlUx/MW4U5HtjejkSXtVUaBEL5FUULx9Xq6veljMOP7oC0zVCanNcw1cHwxRvO1SNpSNDYgBiVW82NJ/CQ==","signed_message":"open_graph_event_sha256_bytes","signed_at":"2026-07-06T19:01:07.206514Z"},"content_sha256":"3b08b13dfdeca8d2c025e8757ea3af8fa1aec0cac6b8cc4add5d504323be0863","schema_version":"1.0","event_id":"sha256:3b08b13dfdeca8d2c025e8757ea3af8fa1aec0cac6b8cc4add5d504323be0863"},{"event_type":"graph_snapshot","subject_pith_number":"pith:2023:647CWRTCVSWMAGFIJFPGPMMYWA","target":"graph","payload":{"graph_snapshot":{"paper":{"title":"Small-scale proxies for large-scale Transformer training instabilities","license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","headline":"","cross_cats":[],"primary_cat":"cs.LG","authors_text":"Abhishek Kumar, Alex Alemi, Ben Adlam, Izzeddin Gur, Jaehoon Lee, Jascha Sohl-Dickstein, Jeffrey Pennington, John D. Co-Reyes, Justin Gilmer, Katie Everett, Kelvin Xu, Lechao Xiao, Mitchell Wortsman, Peter J. Liu, Roman Novak, Simon Kornblith","submitted_at":"2023-09-25T17:48:51Z","abstract_excerpt":"Teams that have trained large Transformer-based models have reported training instabilities at large scale that did not appear when training with the same hyperparameters at smaller scales. Although the causes of such instabilities are of scientific interest, the amount of resources required to reproduce them has made investigation difficult. In this work, we seek ways to reproduce and study training stability and instability at smaller scales. First, we focus on two sources of training instability described in previous work: the growth of logits in attention layers (Dehghani et al., 2023) and"},"claims":{"count":0,"items":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"2309.14322","kind":"arxiv","version":2},"verdict":{"id":null,"model_set":{},"created_at":null,"strongest_claim":"","one_line_summary":"","pipeline_version":null,"weakest_assumption":"","pith_extraction_headline":""},"integrity":{"clean":true,"summary":{"advisory":0,"critical":0,"by_detector":{},"informational":0},"endpoint":"/pith/2309.14322/integrity.json","findings":[],"available":true,"detectors_run":[],"snapshot_sha256":"c28c3603d3b5d939e8dc4c7e95fa8dfce3d595e45f758748cecf8e644a296938"},"references":{"count":0,"sample":[],"resolved_work":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","internal_anchors":0},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"author_claims":{"count":0,"strong_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"builder_version":"pith-number-builder-2026-05-17-v1"},"verdict_id":null},"signer":{"signer_id":"pith.science","signer_type":"pith_registry","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"created_at":"2026-07-05T07:01:25Z","supersedes":[],"prev_event":null,"signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"8BLQTPTGQhPRuIhXOUbV4C+ZesjTzop4awIJil54IiPP0nfVM0o/EtvVhUGjPsTE/K4BEwP+FA4/WK3tNhpFCA==","signed_message":"open_graph_event_sha256_bytes","signed_at":"2026-07-06T19:01:07.206921Z"},"content_sha256":"a1195beea9e0283a93c701a19c8624af562abe04a5ccb251b8c013a0d8fef063","schema_version":"1.0","event_id":"sha256:a1195beea9e0283a93c701a19c8624af562abe04a5ccb251b8c013a0d8fef063"}],"timestamp_proofs":[],"mirror_hints":[{"mirror_type":"https","name":"Pith Resolver","base_url":"https://pith.science","bundle_url":"https://pith.science/pith/647CWRTCVSWMAGFIJFPGPMMYWA/bundle.json","state_url":"https://pith.science/pith/647CWRTCVSWMAGFIJFPGPMMYWA/state.json","well_known_bundle_url":"https://pith.science/.well-known/pith/647CWRTCVSWMAGFIJFPGPMMYWA/bundle.json","status":"primary"}],"public_keys":[{"key_id":"pith-v1-2026-05","algorithm":"ed25519","format":"raw","public_key_b64":"stVStoiQhXFxp4s2pdzPNoqVNBMojDU/fJ2db5S3CbM=","public_key_hex":"b2d552b68890857171a78b36a5dccf368a953413288c353f7c9d9d6f94b709b3","fingerprint_sha256_b32_first128bits":"RVFV5Z2OI2J3ZUO7ERDEBCYNKS","fingerprint_sha256_hex":"8d4b5ee74e4693bcd1df2446408b0d54","rotates_at":null,"url":"https://pith.science/pith-signing-key.json","notes":"Pith uses this Ed25519 key to sign canonical record SHA-256 digests. Verify with: ed25519_verify(public_key, message=canonical_sha256_bytes, signature=base64decode(signature_b64))."}],"merge_version":"pith-open-graph-merge-v1","built_at":"2026-07-06T19:01:07Z","links":{"resolver":"https://pith.science/pith/647CWRTCVSWMAGFIJFPGPMMYWA","bundle":"https://pith.science/pith/647CWRTCVSWMAGFIJFPGPMMYWA/bundle.json","state":"https://pith.science/pith/647CWRTCVSWMAGFIJFPGPMMYWA/state.json","well_known_bundle":"https://pith.science/.well-known/pith/647CWRTCVSWMAGFIJFPGPMMYWA/bundle.json"},"state":{"state_type":"pith_open_graph_state","state_version":"1.0","pith_number":"pith:2023:647CWRTCVSWMAGFIJFPGPMMYWA","merge_version":"pith-open-graph-merge-v1","event_count":2,"valid_event_count":2,"invalid_event_count":0,"equivocation_count":0,"current":{"canonical_record":{"metadata":{"abstract_canon_sha256":"e3a198a9dc3da292a34f42811b74e4af7ee2aa1c10bbe952465d9c4039e7d793","cross_cats_sorted":[],"license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","primary_cat":"cs.LG","submitted_at":"2023-09-25T17:48:51Z","title_canon_sha256":"692522a05eafd3228d7544e2a06deaf5da8ba56e40d3abf861602d862b391de5"},"schema_version":"1.0","source":{"id":"2309.14322","kind":"arxiv","version":2}},"source_aliases":[{"alias_kind":"arxiv","alias_value":"2309.14322","created_at":"2026-07-05T07:01:25Z"},{"alias_kind":"arxiv_version","alias_value":"2309.14322v2","created_at":"2026-07-05T07:01:25Z"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.2309.14322","created_at":"2026-07-05T07:01:25Z"},{"alias_kind":"pith_short_12","alias_value":"647CWRTCVSWM","created_at":"2026-07-05T07:01:25Z"},{"alias_kind":"pith_short_16","alias_value":"647CWRTCVSWMAGFI","created_at":"2026-07-05T07:01:25Z"},{"alias_kind":"pith_short_8","alias_value":"647CWRTC","created_at":"2026-07-05T07:01:25Z"}],"graph_snapshots":[{"event_id":"sha256:a1195beea9e0283a93c701a19c8624af562abe04a5ccb251b8c013a0d8fef063","target":"graph","created_at":"2026-07-05T07:01:25Z","signer":{"key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signer_id":"pith.science","signer_type":"pith_registry"},"payload":{"graph_snapshot":{"author_claims":{"count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","strong_count":0},"builder_version":"pith-number-builder-2026-05-17-v1","claims":{"count":0,"items":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"integrity":{"available":true,"clean":true,"detectors_run":[],"endpoint":"/pith/2309.14322/integrity.json","findings":[],"snapshot_sha256":"c28c3603d3b5d939e8dc4c7e95fa8dfce3d595e45f758748cecf8e644a296938","summary":{"advisory":0,"by_detector":{},"critical":0,"informational":0}},"paper":{"abstract_excerpt":"Teams that have trained large Transformer-based models have reported training instabilities at large scale that did not appear when training with the same hyperparameters at smaller scales. Although the causes of such instabilities are of scientific interest, the amount of resources required to reproduce them has made investigation difficult. In this work, we seek ways to reproduce and study training stability and instability at smaller scales. First, we focus on two sources of training instability described in previous work: the growth of logits in attention layers (Dehghani et al., 2023) and","authors_text":"Abhishek Kumar, Alex Alemi, Ben Adlam, Izzeddin Gur, Jaehoon Lee, Jascha Sohl-Dickstein, Jeffrey Pennington, John D. Co-Reyes, Justin Gilmer, Katie Everett, Kelvin Xu, Lechao Xiao, Mitchell Wortsman, Peter J. Liu, Roman Novak, Simon Kornblith","cross_cats":[],"headline":"","license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","primary_cat":"cs.LG","submitted_at":"2023-09-25T17:48:51Z","title":"Small-scale proxies for large-scale Transformer training instabilities"},"references":{"count":0,"internal_anchors":0,"resolved_work":0,"sample":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"2309.14322","kind":"arxiv","version":2},"verdict":{"created_at":null,"id":null,"model_set":{},"one_line_summary":"","pipeline_version":null,"pith_extraction_headline":"","strongest_claim":"","weakest_assumption":""}},"verdict_id":null}}],"author_attestations":[],"timestamp_anchors":[],"storage_attestations":[],"citation_signatures":[],"replication_records":[],"corrections":[],"mirror_hints":[],"record_created":{"event_id":"sha256:3b08b13dfdeca8d2c025e8757ea3af8fa1aec0cac6b8cc4add5d504323be0863","target":"record","created_at":"2026-07-05T07:01:25Z","signer":{"key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signer_id":"pith.science","signer_type":"pith_registry"},"payload":{"attestation_state":"computed","canonical_record":{"metadata":{"abstract_canon_sha256":"e3a198a9dc3da292a34f42811b74e4af7ee2aa1c10bbe952465d9c4039e7d793","cross_cats_sorted":[],"license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","primary_cat":"cs.LG","submitted_at":"2023-09-25T17:48:51Z","title_canon_sha256":"692522a05eafd3228d7544e2a06deaf5da8ba56e40d3abf861602d862b391de5"},"schema_version":"1.0","source":{"id":"2309.14322","kind":"arxiv","version":2}},"canonical_sha256":"f73e2b4662acacc018a8495e67b198b0288e669ba39fee2cae4b847469edc3de","receipt":{"algorithm":"ed25519","builder_version":"pith-number-builder-2026-05-17-v1","canonical_sha256":"f73e2b4662acacc018a8495e67b198b0288e669ba39fee2cae4b847469edc3de","first_computed_at":"2026-07-05T07:01:25.754842Z","key_id":"pith-v1-2026-05","kind":"pith_receipt","last_reissued_at":"2026-07-05T07:01:25.754842Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","receipt_version":"0.3","signature_b64":"fJMzcPk7n86j3NFHrdaBFHVlvqUJOneYRjXscmbJxFDdjQZgmIV93JSYThErXSdszRZHQk9e51xipvRf2w3FDA==","signature_status":"signed_v1","signed_at":"2026-07-05T07:01:25.755306Z","signed_message":"canonical_sha256_bytes"},"source_id":"2309.14322","source_kind":"arxiv","source_version":2}}},"equivocations":[],"invalid_events":[],"applied_event_ids":["sha256:3b08b13dfdeca8d2c025e8757ea3af8fa1aec0cac6b8cc4add5d504323be0863","sha256:a1195beea9e0283a93c701a19c8624af562abe04a5ccb251b8c013a0d8fef063"],"state_sha256":"949e40ce95398b82b4e7852fb4724c86e10ddaa6da6612a662cdb2f79edd0ab2"},"bundle_signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"vUdBOmZVlx6Rm2VIFko8z9G0rQ1tHLUdRqxtcQIPYO4a4ifoj9i0ZgpPBjLNIP5B8MopkkIRc1JoRAbyPZDUAA==","signed_message":"bundle_sha256_bytes","signed_at":"2026-07-06T19:01:07.208855Z","bundle_sha256":"b4b32a20bed2c6f72a8423dc3084a0395da7994c956c6d356f0b3f9a58910d8f"}}