{"bundle_type":"pith_open_graph_bundle","bundle_version":"1.0","pith_number":"pith:2023:PYLO2YHLBEBEZ6KRJMNHBXMKQP","short_pith_number":"pith:PYLO2YHL","canonical_record":{"source":{"id":"2303.06296","kind":"arxiv","version":2},"metadata":{"license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","primary_cat":"cs.LG","submitted_at":"2023-03-11T03:30:47Z","cross_cats_sorted":["cs.AI","cs.CL","cs.CV","stat.ML"],"title_canon_sha256":"03a2678f137d44fed0baf035f88f427c81af5f44af6e6384c4ec463d431946f0","abstract_canon_sha256":"f3bb4cc74780264b2ac009e8d332ac2af03acf691bd0549c663d170eedad12d9"},"schema_version":"1.0"},"canonical_sha256":"7e16ed60eb09024cf9514b1a70dd8a83dd272136e4c855eacec3555438db9fc3","source":{"kind":"arxiv","id":"2303.06296","version":2},"source_aliases":[{"alias_kind":"arxiv","alias_value":"2303.06296","created_at":"2026-07-05T06:34:28Z"},{"alias_kind":"arxiv_version","alias_value":"2303.06296v2","created_at":"2026-07-05T06:34:28Z"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.2303.06296","created_at":"2026-07-05T06:34:28Z"},{"alias_kind":"pith_short_12","alias_value":"PYLO2YHLBEBE","created_at":"2026-07-05T06:34:28Z"},{"alias_kind":"pith_short_16","alias_value":"PYLO2YHLBEBEZ6KR","created_at":"2026-07-05T06:34:28Z"},{"alias_kind":"pith_short_8","alias_value":"PYLO2YHL","created_at":"2026-07-05T06:34:28Z"}],"events":[{"event_type":"record_created","subject_pith_number":"pith:2023:PYLO2YHLBEBEZ6KRJMNHBXMKQP","target":"record","payload":{"canonical_record":{"source":{"id":"2303.06296","kind":"arxiv","version":2},"metadata":{"license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","primary_cat":"cs.LG","submitted_at":"2023-03-11T03:30:47Z","cross_cats_sorted":["cs.AI","cs.CL","cs.CV","stat.ML"],"title_canon_sha256":"03a2678f137d44fed0baf035f88f427c81af5f44af6e6384c4ec463d431946f0","abstract_canon_sha256":"f3bb4cc74780264b2ac009e8d332ac2af03acf691bd0549c663d170eedad12d9"},"schema_version":"1.0"},"canonical_sha256":"7e16ed60eb09024cf9514b1a70dd8a83dd272136e4c855eacec3555438db9fc3","receipt":{"kind":"pith_receipt","key_id":"pith-v1-2026-05","algorithm":"ed25519","signed_at":"2026-07-05T06:34:28.225967Z","signature_b64":"cLQknWm/V7mzlnQnnIb1A9giJXIFxC4xgIzwtOIUQEgUI+60GaOS3hPXfGs/wu7NB3Fyj06kglN+ulgOYgdbDw==","signed_message":"canonical_sha256_bytes","builder_version":"pith-number-builder-2026-05-17-v1","receipt_version":"0.3","canonical_sha256":"7e16ed60eb09024cf9514b1a70dd8a83dd272136e4c855eacec3555438db9fc3","last_reissued_at":"2026-07-05T06:34:28.225428Z","signature_status":"signed_v1","first_computed_at":"2026-07-05T06:34:28.225428Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"source_kind":"arxiv","source_id":"2303.06296","source_version":2,"attestation_state":"computed"},"signer":{"signer_id":"pith.science","signer_type":"pith_registry","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"created_at":"2026-07-05T06:34:28Z","supersedes":[],"prev_event":null,"signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"TZs9dWjx/xITIGv+U0vqzFqYrcafNu8BID2k2ys3Q3ukkdHx/GYylOiaXRydH9DQ8TCQXdBZ0fanv0Hq5jXqBg==","signed_message":"open_graph_event_sha256_bytes","signed_at":"2026-08-09T23:21:56.838540Z"},"content_sha256":"c8192a9f8dc46c6f52871e3ea6d519ea8bb0dec3279fd88ec9a359c2b76d54d6","schema_version":"1.0","event_id":"sha256:c8192a9f8dc46c6f52871e3ea6d519ea8bb0dec3279fd88ec9a359c2b76d54d6"},{"event_type":"graph_snapshot","subject_pith_number":"pith:2023:PYLO2YHLBEBEZ6KRJMNHBXMKQP","target":"graph","payload":{"graph_snapshot":{"paper":{"title":"Stabilizing Transformer Training by Preventing Attention Entropy Collapse","license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","headline":"","cross_cats":["cs.AI","cs.CL","cs.CV","stat.ML"],"primary_cat":"cs.LG","authors_text":"Dan Busbridge, Etai Littwin, Jason Ramapuram, Jiatao Gu, Josh Susskind, Shuangfei Zhai, Tatiana Likhomanenko, Yizhe Zhang","submitted_at":"2023-03-11T03:30:47Z","abstract_excerpt":"Training stability is of great importance to Transformers. In this work, we investigate the training dynamics of Transformers by examining the evolution of the attention layers. In particular, we track the attention entropy for each attention head during the course of training, which is a proxy for model sharpness. We identify a common pattern across different architectures and tasks, where low attention entropy is accompanied by high training instability, which can take the form of oscillating loss or divergence. We denote the pathologically low attention entropy, corresponding to highly conc"},"claims":{"count":0,"items":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"2303.06296","kind":"arxiv","version":2},"verdict":{"id":null,"model_set":{},"created_at":null,"strongest_claim":"","one_line_summary":"","pipeline_version":null,"weakest_assumption":"","pith_extraction_headline":""},"integrity":{"clean":true,"summary":{"advisory":0,"critical":0,"by_detector":{},"informational":0},"endpoint":"/pith/2303.06296/integrity.json","findings":[],"available":true,"detectors_run":[],"snapshot_sha256":"c28c3603d3b5d939e8dc4c7e95fa8dfce3d595e45f758748cecf8e644a296938"},"references":{"count":0,"sample":[],"resolved_work":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","internal_anchors":0},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"author_claims":{"count":0,"strong_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"builder_version":"pith-number-builder-2026-05-17-v1"},"verdict_id":null},"signer":{"signer_id":"pith.science","signer_type":"pith_registry","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"created_at":"2026-07-05T06:34:28Z","supersedes":[],"prev_event":null,"signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"F2d7bNBuOgmCUXplRqKbfNZAd7MgMabwxXk7UDIa391QY72YM0t2m2/hFLELB9oxYBMhxHXDpEEB77KlZFuiDw==","signed_message":"open_graph_event_sha256_bytes","signed_at":"2026-08-09T23:21:56.839040Z"},"content_sha256":"e84475bd3c8bd1c6426c18ae319e1d945b4e9e96338a4f36d3e69ce6fbc71a59","schema_version":"1.0","event_id":"sha256:e84475bd3c8bd1c6426c18ae319e1d945b4e9e96338a4f36d3e69ce6fbc71a59"}],"timestamp_proofs":[],"mirror_hints":[{"mirror_type":"https","name":"Pith Resolver","base_url":"https://pith.science","bundle_url":"https://pith.science/pith/PYLO2YHLBEBEZ6KRJMNHBXMKQP/bundle.json","state_url":"https://pith.science/pith/PYLO2YHLBEBEZ6KRJMNHBXMKQP/state.json","well_known_bundle_url":"https://pith.science/.well-known/pith/PYLO2YHLBEBEZ6KRJMNHBXMKQP/bundle.json","status":"primary"}],"public_keys":[{"key_id":"pith-v1-2026-05","algorithm":"ed25519","format":"raw","public_key_b64":"stVStoiQhXFxp4s2pdzPNoqVNBMojDU/fJ2db5S3CbM=","public_key_hex":"b2d552b68890857171a78b36a5dccf368a953413288c353f7c9d9d6f94b709b3","fingerprint_sha256_b32_first128bits":"RVFV5Z2OI2J3ZUO7ERDEBCYNKS","fingerprint_sha256_hex":"8d4b5ee74e4693bcd1df2446408b0d54","rotates_at":null,"url":"https://pith.science/pith-signing-key.json","notes":"Pith uses this Ed25519 key to sign canonical record SHA-256 digests. Verify with: ed25519_verify(public_key, message=canonical_sha256_bytes, signature=base64decode(signature_b64))."}],"merge_version":"pith-open-graph-merge-v1","built_at":"2026-08-09T23:21:56Z","links":{"resolver":"https://pith.science/pith/PYLO2YHLBEBEZ6KRJMNHBXMKQP","bundle":"https://pith.science/pith/PYLO2YHLBEBEZ6KRJMNHBXMKQP/bundle.json","state":"https://pith.science/pith/PYLO2YHLBEBEZ6KRJMNHBXMKQP/state.json","well_known_bundle":"https://pith.science/.well-known/pith/PYLO2YHLBEBEZ6KRJMNHBXMKQP/bundle.json"},"state":{"state_type":"pith_open_graph_state","state_version":"1.0","pith_number":"pith:2023:PYLO2YHLBEBEZ6KRJMNHBXMKQP","merge_version":"pith-open-graph-merge-v1","event_count":2,"valid_event_count":2,"invalid_event_count":0,"equivocation_count":0,"current":{"canonical_record":{"metadata":{"abstract_canon_sha256":"f3bb4cc74780264b2ac009e8d332ac2af03acf691bd0549c663d170eedad12d9","cross_cats_sorted":["cs.AI","cs.CL","cs.CV","stat.ML"],"license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","primary_cat":"cs.LG","submitted_at":"2023-03-11T03:30:47Z","title_canon_sha256":"03a2678f137d44fed0baf035f88f427c81af5f44af6e6384c4ec463d431946f0"},"schema_version":"1.0","source":{"id":"2303.06296","kind":"arxiv","version":2}},"source_aliases":[{"alias_kind":"arxiv","alias_value":"2303.06296","created_at":"2026-07-05T06:34:28Z"},{"alias_kind":"arxiv_version","alias_value":"2303.06296v2","created_at":"2026-07-05T06:34:28Z"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.2303.06296","created_at":"2026-07-05T06:34:28Z"},{"alias_kind":"pith_short_12","alias_value":"PYLO2YHLBEBE","created_at":"2026-07-05T06:34:28Z"},{"alias_kind":"pith_short_16","alias_value":"PYLO2YHLBEBEZ6KR","created_at":"2026-07-05T06:34:28Z"},{"alias_kind":"pith_short_8","alias_value":"PYLO2YHL","created_at":"2026-07-05T06:34:28Z"}],"graph_snapshots":[{"event_id":"sha256:e84475bd3c8bd1c6426c18ae319e1d945b4e9e96338a4f36d3e69ce6fbc71a59","target":"graph","created_at":"2026-07-05T06:34:28Z","signer":{"key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signer_id":"pith.science","signer_type":"pith_registry"},"payload":{"graph_snapshot":{"author_claims":{"count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","strong_count":0},"builder_version":"pith-number-builder-2026-05-17-v1","claims":{"count":0,"items":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"integrity":{"available":true,"clean":true,"detectors_run":[],"endpoint":"/pith/2303.06296/integrity.json","findings":[],"snapshot_sha256":"c28c3603d3b5d939e8dc4c7e95fa8dfce3d595e45f758748cecf8e644a296938","summary":{"advisory":0,"by_detector":{},"critical":0,"informational":0}},"paper":{"abstract_excerpt":"Training stability is of great importance to Transformers. In this work, we investigate the training dynamics of Transformers by examining the evolution of the attention layers. In particular, we track the attention entropy for each attention head during the course of training, which is a proxy for model sharpness. We identify a common pattern across different architectures and tasks, where low attention entropy is accompanied by high training instability, which can take the form of oscillating loss or divergence. We denote the pathologically low attention entropy, corresponding to highly conc","authors_text":"Dan Busbridge, Etai Littwin, Jason Ramapuram, Jiatao Gu, Josh Susskind, Shuangfei Zhai, Tatiana Likhomanenko, Yizhe Zhang","cross_cats":["cs.AI","cs.CL","cs.CV","stat.ML"],"headline":"","license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","primary_cat":"cs.LG","submitted_at":"2023-03-11T03:30:47Z","title":"Stabilizing Transformer Training by Preventing Attention Entropy Collapse"},"references":{"count":0,"internal_anchors":0,"resolved_work":0,"sample":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"2303.06296","kind":"arxiv","version":2},"verdict":{"created_at":null,"id":null,"model_set":{},"one_line_summary":"","pipeline_version":null,"pith_extraction_headline":"","strongest_claim":"","weakest_assumption":""}},"verdict_id":null}}],"author_attestations":[],"timestamp_anchors":[],"storage_attestations":[],"citation_signatures":[],"replication_records":[],"corrections":[],"mirror_hints":[],"record_created":{"event_id":"sha256:c8192a9f8dc46c6f52871e3ea6d519ea8bb0dec3279fd88ec9a359c2b76d54d6","target":"record","created_at":"2026-07-05T06:34:28Z","signer":{"key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signer_id":"pith.science","signer_type":"pith_registry"},"payload":{"attestation_state":"computed","canonical_record":{"metadata":{"abstract_canon_sha256":"f3bb4cc74780264b2ac009e8d332ac2af03acf691bd0549c663d170eedad12d9","cross_cats_sorted":["cs.AI","cs.CL","cs.CV","stat.ML"],"license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","primary_cat":"cs.LG","submitted_at":"2023-03-11T03:30:47Z","title_canon_sha256":"03a2678f137d44fed0baf035f88f427c81af5f44af6e6384c4ec463d431946f0"},"schema_version":"1.0","source":{"id":"2303.06296","kind":"arxiv","version":2}},"canonical_sha256":"7e16ed60eb09024cf9514b1a70dd8a83dd272136e4c855eacec3555438db9fc3","receipt":{"algorithm":"ed25519","builder_version":"pith-number-builder-2026-05-17-v1","canonical_sha256":"7e16ed60eb09024cf9514b1a70dd8a83dd272136e4c855eacec3555438db9fc3","first_computed_at":"2026-07-05T06:34:28.225428Z","key_id":"pith-v1-2026-05","kind":"pith_receipt","last_reissued_at":"2026-07-05T06:34:28.225428Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","receipt_version":"0.3","signature_b64":"cLQknWm/V7mzlnQnnIb1A9giJXIFxC4xgIzwtOIUQEgUI+60GaOS3hPXfGs/wu7NB3Fyj06kglN+ulgOYgdbDw==","signature_status":"signed_v1","signed_at":"2026-07-05T06:34:28.225967Z","signed_message":"canonical_sha256_bytes"},"source_id":"2303.06296","source_kind":"arxiv","source_version":2}}},"equivocations":[],"invalid_events":[],"applied_event_ids":["sha256:c8192a9f8dc46c6f52871e3ea6d519ea8bb0dec3279fd88ec9a359c2b76d54d6","sha256:e84475bd3c8bd1c6426c18ae319e1d945b4e9e96338a4f36d3e69ce6fbc71a59"],"state_sha256":"cc2ef70db0d9cbc4d59ae62656aea9d325a3a033c2cb4adc46ebb478ab04875b"},"bundle_signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"HwtWmqsEJwrtcDdjdiHesoNJhBO/oKMAF6UcjZ1chSct+uFiZcxPRbotk+kqyBkb7D/W6dqXnwBNv4BcYubWDw==","signed_message":"bundle_sha256_bytes","signed_at":"2026-08-09T23:21:56.842447Z","bundle_sha256":"abafbb03878fe4d7ccff5f021935f8ad3321a0bb08569877fa00124ea6f209ba"}}