{"record_type":"pith_number_record","schema_url":"https://pith.science/schemas/pith-number/v1.json","pith_number":"pith:2021:3526PL7AKLYDEVVHXLXC57VVA7","short_pith_number":"pith:3526PL7A","schema_version":"1.0","canonical_sha256":"df75e7afe052f03256a7baee2efeb507e8295ccf751dcc4bd6ebaded53beeef8","source":{"kind":"arxiv","id":"2106.08823","version":1},"attestation_state":"computed","paper":{"title":"Eigen Analysis of Self-Attention and its Reconstruction from Partial Computation","license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","headline":"","cross_cats":[],"primary_cat":"cs.LG","authors_text":"Andreas Veit, Ayan Chakrabarti, Himanshu Jain, Michal Lukasik, Sanjiv Kumar, Srinadh Bhojanapalli","submitted_at":"2021-06-16T14:38:42Z","abstract_excerpt":"State-of-the-art transformer models use pairwise dot-product based self-attention, which comes at a computational cost quadratic in the input sequence length. In this paper, we investigate the global structure of attention scores computed using this dot product mechanism on a typical distribution of inputs, and study the principal components of their variation. Through eigen analysis of full attention score matrices, as well as of their individual rows, we find that most of the variation among attention scores lie in a low-dimensional eigenspace. Moreover, we find significant overlap between t"},"verification_status":{"content_addressed":true,"pith_receipt":true,"author_attested":false,"weak_author_claims":0,"strong_author_claims":0,"externally_anchored":false,"storage_verified":false,"citation_signatures":0,"replication_records":0,"graph_snapshot":true,"references_resolved":false,"formal_links_present":false},"canonical_record":{"source":{"id":"2106.08823","kind":"arxiv","version":1},"metadata":{"license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","primary_cat":"cs.LG","submitted_at":"2021-06-16T14:38:42Z","cross_cats_sorted":[],"title_canon_sha256":"8f1ef8448a3401a80d1bdca417ed6edf368befde4c8979434fa74e59140726a8","abstract_canon_sha256":"330fcc7e2b5a73bbb5575697068b9c00c668915f7feb8967656959c6413aeade"},"schema_version":"1.0"},"receipt":{"kind":"pith_receipt","key_id":"pith-v1-2026-05","algorithm":"ed25519","signed_at":"2026-07-05T02:50:01.754530Z","signature_b64":"7FoPjdUNe5WxZFkVky/BV7r0gR1O4d7pLoJEM0aIoTK4kGeG5IaA67G0GhbvW16z5st9iMZ1S6nhledsouNIDQ==","signed_message":"canonical_sha256_bytes","builder_version":"pith-number-builder-2026-05-17-v1","receipt_version":"0.3","canonical_sha256":"df75e7afe052f03256a7baee2efeb507e8295ccf751dcc4bd6ebaded53beeef8","last_reissued_at":"2026-07-05T02:50:01.754173Z","signature_status":"signed_v1","first_computed_at":"2026-07-05T02:50:01.754173Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"graph_snapshot":{"paper":{"title":"Eigen Analysis of Self-Attention and its Reconstruction from Partial Computation","license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","headline":"","cross_cats":[],"primary_cat":"cs.LG","authors_text":"Andreas Veit, Ayan Chakrabarti, Himanshu Jain, Michal Lukasik, Sanjiv Kumar, Srinadh Bhojanapalli","submitted_at":"2021-06-16T14:38:42Z","abstract_excerpt":"State-of-the-art transformer models use pairwise dot-product based self-attention, which comes at a computational cost quadratic in the input sequence length. In this paper, we investigate the global structure of attention scores computed using this dot product mechanism on a typical distribution of inputs, and study the principal components of their variation. Through eigen analysis of full attention score matrices, as well as of their individual rows, we find that most of the variation among attention scores lie in a low-dimensional eigenspace. Moreover, we find significant overlap between t"},"claims":{"count":0,"items":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"2106.08823","kind":"arxiv","version":1},"verdict":{"id":null,"model_set":{},"created_at":null,"strongest_claim":"","one_line_summary":"","pipeline_version":null,"weakest_assumption":"","pith_extraction_headline":""},"integrity":{"clean":true,"summary":{"advisory":0,"critical":0,"by_detector":{},"informational":0},"endpoint":"/pith/2106.08823/integrity.json","findings":[],"available":true,"detectors_run":[],"snapshot_sha256":"c28c3603d3b5d939e8dc4c7e95fa8dfce3d595e45f758748cecf8e644a296938"},"references":{"count":0,"sample":[],"resolved_work":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","internal_anchors":0},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"author_claims":{"count":0,"strong_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"builder_version":"pith-number-builder-2026-05-17-v1"},"aliases":[{"alias_kind":"arxiv","alias_value":"2106.08823","created_at":"2026-07-05T02:50:01.754234+00:00"},{"alias_kind":"arxiv_version","alias_value":"2106.08823v1","created_at":"2026-07-05T02:50:01.754234+00:00"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.2106.08823","created_at":"2026-07-05T02:50:01.754234+00:00"},{"alias_kind":"pith_short_12","alias_value":"3526PL7AKLYD","created_at":"2026-07-05T02:50:01.754234+00:00"},{"alias_kind":"pith_short_16","alias_value":"3526PL7AKLYDEVVH","created_at":"2026-07-05T02:50:01.754234+00:00"},{"alias_kind":"pith_short_8","alias_value":"3526PL7A","created_at":"2026-07-05T02:50:01.754234+00:00"}],"events":[],"event_summary":{},"paper_claims":[],"inbound_citations":{"count":0,"internal_anchor_count":0,"sample":[]},"formal_canon":{"evidence_count":0,"sample":[],"anchors":[]},"links":{"html":"https://pith.science/pith/3526PL7AKLYDEVVHXLXC57VVA7","json":"https://pith.science/pith/3526PL7AKLYDEVVHXLXC57VVA7.json","graph_json":"https://pith.science/api/pith-number/3526PL7AKLYDEVVHXLXC57VVA7/graph.json","events_json":"https://pith.science/api/pith-number/3526PL7AKLYDEVVHXLXC57VVA7/events.json","paper":"https://pith.science/paper/3526PL7A"},"agent_actions":{"view_html":"https://pith.science/pith/3526PL7AKLYDEVVHXLXC57VVA7","download_json":"https://pith.science/pith/3526PL7AKLYDEVVHXLXC57VVA7.json","view_paper":"https://pith.science/paper/3526PL7A","resolve_alias":"https://pith.science/api/pith-number/resolve?arxiv=2106.08823&json=true","fetch_graph":"https://pith.science/api/pith-number/3526PL7AKLYDEVVHXLXC57VVA7/graph.json","fetch_events":"https://pith.science/api/pith-number/3526PL7AKLYDEVVHXLXC57VVA7/events.json","actions":{"anchor_timestamp":"https://pith.science/pith/3526PL7AKLYDEVVHXLXC57VVA7/action/timestamp_anchor","attest_storage":"https://pith.science/pith/3526PL7AKLYDEVVHXLXC57VVA7/action/storage_attestation","attest_author":"https://pith.science/pith/3526PL7AKLYDEVVHXLXC57VVA7/action/author_attestation","sign_citation":"https://pith.science/pith/3526PL7AKLYDEVVHXLXC57VVA7/action/citation_signature","submit_replication":"https://pith.science/pith/3526PL7AKLYDEVVHXLXC57VVA7/action/replication_record"}},"created_at":"2026-07-05T02:50:01.754234+00:00","updated_at":"2026-07-05T02:50:01.754234+00:00"}