{"bundle_type":"pith_open_graph_bundle","bundle_version":"1.0","pith_number":"pith:2024:P5HJPFSOGA2V6QTYBSHVIBTARR","short_pith_number":"pith:P5HJPFSO","canonical_record":{"source":{"id":"2405.18781","kind":"arxiv","version":2},"metadata":{"license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.LG","submitted_at":"2024-05-29T05:41:28Z","cross_cats_sorted":["stat.ML"],"title_canon_sha256":"6712ad6ed2d3a14282a8b3cf659a716ef2187b303dce150c6ca4752318c726ec","abstract_canon_sha256":"8a9fedbb8f67528c5849c486a45e5c2714d55f7426fd074320786b7ef617e4dd"},"schema_version":"1.0"},"canonical_sha256":"7f4e97964e30355f42780c8f5406608c7d981d0164459de9d21794953d192bf4","source":{"kind":"arxiv","id":"2405.18781","version":2},"source_aliases":[{"alias_kind":"arxiv","alias_value":"2405.18781","created_at":"2026-07-05T09:29:29Z"},{"alias_kind":"arxiv_version","alias_value":"2405.18781v2","created_at":"2026-07-05T09:29:29Z"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.2405.18781","created_at":"2026-07-05T09:29:29Z"},{"alias_kind":"pith_short_12","alias_value":"P5HJPFSOGA2V","created_at":"2026-07-05T09:29:29Z"},{"alias_kind":"pith_short_16","alias_value":"P5HJPFSOGA2V6QTY","created_at":"2026-07-05T09:29:29Z"},{"alias_kind":"pith_short_8","alias_value":"P5HJPFSO","created_at":"2026-07-05T09:29:29Z"}],"events":[{"event_type":"record_created","subject_pith_number":"pith:2024:P5HJPFSOGA2V6QTYBSHVIBTARR","target":"record","payload":{"canonical_record":{"source":{"id":"2405.18781","kind":"arxiv","version":2},"metadata":{"license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.LG","submitted_at":"2024-05-29T05:41:28Z","cross_cats_sorted":["stat.ML"],"title_canon_sha256":"6712ad6ed2d3a14282a8b3cf659a716ef2187b303dce150c6ca4752318c726ec","abstract_canon_sha256":"8a9fedbb8f67528c5849c486a45e5c2714d55f7426fd074320786b7ef617e4dd"},"schema_version":"1.0"},"canonical_sha256":"7f4e97964e30355f42780c8f5406608c7d981d0164459de9d21794953d192bf4","receipt":{"kind":"pith_receipt","key_id":"pith-v1-2026-05","algorithm":"ed25519","signed_at":"2026-07-05T09:29:29.329648Z","signature_b64":"+uxRFn1AFgU+NGNQ5tgyqTO2lnq0R6HE1qYeHx0XgwZFOl1Qg6tCunsEzOdDhj2EK2VXdYP9fx2gupojNSrRAw==","signed_message":"canonical_sha256_bytes","builder_version":"pith-number-builder-2026-05-17-v1","receipt_version":"0.3","canonical_sha256":"7f4e97964e30355f42780c8f5406608c7d981d0164459de9d21794953d192bf4","last_reissued_at":"2026-07-05T09:29:29.329081Z","signature_status":"signed_v1","first_computed_at":"2026-07-05T09:29:29.329081Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"source_kind":"arxiv","source_id":"2405.18781","source_version":2,"attestation_state":"computed"},"signer":{"signer_id":"pith.science","signer_type":"pith_registry","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"created_at":"2026-07-05T09:29:29Z","supersedes":[],"prev_event":null,"signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"vT+0wCJW1c0+RIwx+E0ErStkOfT0Npo8Ad05ll+C0uGsD7XW39SaSzMlia7hnZbJ+Sd+GyAvpkkkH2BFPnq1BQ==","signed_message":"open_graph_event_sha256_bytes","signed_at":"2026-07-06T18:31:40.326787Z"},"content_sha256":"bffe3d62b23d6e47ff18fd83fccd51d6742340310e3fe4ae030b1ebbefdcd0da","schema_version":"1.0","event_id":"sha256:bffe3d62b23d6e47ff18fd83fccd51d6742340310e3fe4ae030b1ebbefdcd0da"},{"event_type":"graph_snapshot","subject_pith_number":"pith:2024:P5HJPFSOGA2V6QTYBSHVIBTARR","target":"graph","payload":{"graph_snapshot":{"paper":{"title":"On the Role of Attention Masks and LayerNorm in Transformers","license":"http://creativecommons.org/licenses/by/4.0/","headline":"","cross_cats":["stat.ML"],"primary_cat":"cs.LG","authors_text":"Ali Jadbabaie, Amir Ajorlou, Stefanie Jegelka, Xinyi Wu, Yifei Wang","submitted_at":"2024-05-29T05:41:28Z","abstract_excerpt":"Self-attention is the key mechanism of transformers, which are the essential building blocks of modern foundation models. Recent studies have shown that pure self-attention suffers from an increasing degree of rank collapse as depth increases, limiting model expressivity and further utilization of model depth. The existing literature on rank collapse, however, has mostly overlooked other critical components in transformers that may alleviate the rank collapse issue. In this paper, we provide a general analysis of rank collapse under self-attention, taking into account the effects of attention "},"claims":{"count":0,"items":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"2405.18781","kind":"arxiv","version":2},"verdict":{"id":null,"model_set":{},"created_at":null,"strongest_claim":"","one_line_summary":"","pipeline_version":null,"weakest_assumption":"","pith_extraction_headline":""},"integrity":{"clean":true,"summary":{"advisory":0,"critical":0,"by_detector":{},"informational":0},"endpoint":"/pith/2405.18781/integrity.json","findings":[],"available":true,"detectors_run":[],"snapshot_sha256":"c28c3603d3b5d939e8dc4c7e95fa8dfce3d595e45f758748cecf8e644a296938"},"references":{"count":0,"sample":[],"resolved_work":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","internal_anchors":0},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"author_claims":{"count":0,"strong_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"builder_version":"pith-number-builder-2026-05-17-v1"},"verdict_id":null},"signer":{"signer_id":"pith.science","signer_type":"pith_registry","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"created_at":"2026-07-05T09:29:29Z","supersedes":[],"prev_event":null,"signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"ftBaRxPjbPdDaTwPsBEvLfuVC1XyhDIFWIxlCaCLnZAZmsyo139EXI55QA7TrCWcGwTI6l2iPQozeEW6/OAdDw==","signed_message":"open_graph_event_sha256_bytes","signed_at":"2026-07-06T18:31:40.327149Z"},"content_sha256":"d83baedc6e34a016485c94345a9a02cabc4336f9e7c2dae99064402304947995","schema_version":"1.0","event_id":"sha256:d83baedc6e34a016485c94345a9a02cabc4336f9e7c2dae99064402304947995"}],"timestamp_proofs":[],"mirror_hints":[{"mirror_type":"https","name":"Pith Resolver","base_url":"https://pith.science","bundle_url":"https://pith.science/pith/P5HJPFSOGA2V6QTYBSHVIBTARR/bundle.json","state_url":"https://pith.science/pith/P5HJPFSOGA2V6QTYBSHVIBTARR/state.json","well_known_bundle_url":"https://pith.science/.well-known/pith/P5HJPFSOGA2V6QTYBSHVIBTARR/bundle.json","status":"primary"}],"public_keys":[{"key_id":"pith-v1-2026-05","algorithm":"ed25519","format":"raw","public_key_b64":"stVStoiQhXFxp4s2pdzPNoqVNBMojDU/fJ2db5S3CbM=","public_key_hex":"b2d552b68890857171a78b36a5dccf368a953413288c353f7c9d9d6f94b709b3","fingerprint_sha256_b32_first128bits":"RVFV5Z2OI2J3ZUO7ERDEBCYNKS","fingerprint_sha256_hex":"8d4b5ee74e4693bcd1df2446408b0d54","rotates_at":null,"url":"https://pith.science/pith-signing-key.json","notes":"Pith uses this Ed25519 key to sign canonical record SHA-256 digests. Verify with: ed25519_verify(public_key, message=canonical_sha256_bytes, signature=base64decode(signature_b64))."}],"merge_version":"pith-open-graph-merge-v1","built_at":"2026-07-06T18:31:40Z","links":{"resolver":"https://pith.science/pith/P5HJPFSOGA2V6QTYBSHVIBTARR","bundle":"https://pith.science/pith/P5HJPFSOGA2V6QTYBSHVIBTARR/bundle.json","state":"https://pith.science/pith/P5HJPFSOGA2V6QTYBSHVIBTARR/state.json","well_known_bundle":"https://pith.science/.well-known/pith/P5HJPFSOGA2V6QTYBSHVIBTARR/bundle.json"},"state":{"state_type":"pith_open_graph_state","state_version":"1.0","pith_number":"pith:2024:P5HJPFSOGA2V6QTYBSHVIBTARR","merge_version":"pith-open-graph-merge-v1","event_count":2,"valid_event_count":2,"invalid_event_count":0,"equivocation_count":0,"current":{"canonical_record":{"metadata":{"abstract_canon_sha256":"8a9fedbb8f67528c5849c486a45e5c2714d55f7426fd074320786b7ef617e4dd","cross_cats_sorted":["stat.ML"],"license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.LG","submitted_at":"2024-05-29T05:41:28Z","title_canon_sha256":"6712ad6ed2d3a14282a8b3cf659a716ef2187b303dce150c6ca4752318c726ec"},"schema_version":"1.0","source":{"id":"2405.18781","kind":"arxiv","version":2}},"source_aliases":[{"alias_kind":"arxiv","alias_value":"2405.18781","created_at":"2026-07-05T09:29:29Z"},{"alias_kind":"arxiv_version","alias_value":"2405.18781v2","created_at":"2026-07-05T09:29:29Z"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.2405.18781","created_at":"2026-07-05T09:29:29Z"},{"alias_kind":"pith_short_12","alias_value":"P5HJPFSOGA2V","created_at":"2026-07-05T09:29:29Z"},{"alias_kind":"pith_short_16","alias_value":"P5HJPFSOGA2V6QTY","created_at":"2026-07-05T09:29:29Z"},{"alias_kind":"pith_short_8","alias_value":"P5HJPFSO","created_at":"2026-07-05T09:29:29Z"}],"graph_snapshots":[{"event_id":"sha256:d83baedc6e34a016485c94345a9a02cabc4336f9e7c2dae99064402304947995","target":"graph","created_at":"2026-07-05T09:29:29Z","signer":{"key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signer_id":"pith.science","signer_type":"pith_registry"},"payload":{"graph_snapshot":{"author_claims":{"count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","strong_count":0},"builder_version":"pith-number-builder-2026-05-17-v1","claims":{"count":0,"items":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"integrity":{"available":true,"clean":true,"detectors_run":[],"endpoint":"/pith/2405.18781/integrity.json","findings":[],"snapshot_sha256":"c28c3603d3b5d939e8dc4c7e95fa8dfce3d595e45f758748cecf8e644a296938","summary":{"advisory":0,"by_detector":{},"critical":0,"informational":0}},"paper":{"abstract_excerpt":"Self-attention is the key mechanism of transformers, which are the essential building blocks of modern foundation models. Recent studies have shown that pure self-attention suffers from an increasing degree of rank collapse as depth increases, limiting model expressivity and further utilization of model depth. The existing literature on rank collapse, however, has mostly overlooked other critical components in transformers that may alleviate the rank collapse issue. In this paper, we provide a general analysis of rank collapse under self-attention, taking into account the effects of attention ","authors_text":"Ali Jadbabaie, Amir Ajorlou, Stefanie Jegelka, Xinyi Wu, Yifei Wang","cross_cats":["stat.ML"],"headline":"","license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.LG","submitted_at":"2024-05-29T05:41:28Z","title":"On the Role of Attention Masks and LayerNorm in Transformers"},"references":{"count":0,"internal_anchors":0,"resolved_work":0,"sample":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"2405.18781","kind":"arxiv","version":2},"verdict":{"created_at":null,"id":null,"model_set":{},"one_line_summary":"","pipeline_version":null,"pith_extraction_headline":"","strongest_claim":"","weakest_assumption":""}},"verdict_id":null}}],"author_attestations":[],"timestamp_anchors":[],"storage_attestations":[],"citation_signatures":[],"replication_records":[],"corrections":[],"mirror_hints":[],"record_created":{"event_id":"sha256:bffe3d62b23d6e47ff18fd83fccd51d6742340310e3fe4ae030b1ebbefdcd0da","target":"record","created_at":"2026-07-05T09:29:29Z","signer":{"key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signer_id":"pith.science","signer_type":"pith_registry"},"payload":{"attestation_state":"computed","canonical_record":{"metadata":{"abstract_canon_sha256":"8a9fedbb8f67528c5849c486a45e5c2714d55f7426fd074320786b7ef617e4dd","cross_cats_sorted":["stat.ML"],"license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.LG","submitted_at":"2024-05-29T05:41:28Z","title_canon_sha256":"6712ad6ed2d3a14282a8b3cf659a716ef2187b303dce150c6ca4752318c726ec"},"schema_version":"1.0","source":{"id":"2405.18781","kind":"arxiv","version":2}},"canonical_sha256":"7f4e97964e30355f42780c8f5406608c7d981d0164459de9d21794953d192bf4","receipt":{"algorithm":"ed25519","builder_version":"pith-number-builder-2026-05-17-v1","canonical_sha256":"7f4e97964e30355f42780c8f5406608c7d981d0164459de9d21794953d192bf4","first_computed_at":"2026-07-05T09:29:29.329081Z","key_id":"pith-v1-2026-05","kind":"pith_receipt","last_reissued_at":"2026-07-05T09:29:29.329081Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","receipt_version":"0.3","signature_b64":"+uxRFn1AFgU+NGNQ5tgyqTO2lnq0R6HE1qYeHx0XgwZFOl1Qg6tCunsEzOdDhj2EK2VXdYP9fx2gupojNSrRAw==","signature_status":"signed_v1","signed_at":"2026-07-05T09:29:29.329648Z","signed_message":"canonical_sha256_bytes"},"source_id":"2405.18781","source_kind":"arxiv","source_version":2}}},"equivocations":[],"invalid_events":[],"applied_event_ids":["sha256:bffe3d62b23d6e47ff18fd83fccd51d6742340310e3fe4ae030b1ebbefdcd0da","sha256:d83baedc6e34a016485c94345a9a02cabc4336f9e7c2dae99064402304947995"],"state_sha256":"db7e2c00ac63dcade70a371822da19a3c823f3a8dc3e64f2e1eb67a6ca170141"},"bundle_signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"D38rYuhOAeHp4ItX4yTLwEd0VvC44Qih0SREuN5n9Y2Rn3CJWCPPw97iKOSrKXJ7jsBjmZ8Ez0e9u+IgILaCBg==","signed_message":"bundle_sha256_bytes","signed_at":"2026-07-06T18:31:40.329093Z","bundle_sha256":"d3ff61da61e0e9b78fe353daa5819fe6977110243ac104e4c472ad94d4ad4c11"}}