{"record_type":"pith_number_record","schema_url":"https://pith.science/schemas/pith-number/v1.json","pith_number":"pith:2023:6GPIG3HVUSYNBOC7IJM5AYLOJH","short_pith_number":"pith:6GPIG3HV","schema_version":"1.0","canonical_sha256":"f19e836cf5a4b0d0b85f4259d0616e49ddb8698409ca06cdada300e615c526c2","source":{"kind":"arxiv","id":"2305.05465","version":6},"attestation_state":"computed","paper":{"title":"The emergence of clusters in self-attention dynamics","license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","headline":"","cross_cats":["math.AP","stat.ML"],"primary_cat":"cs.LG","authors_text":"Borjan Geshkovski, Cyril Letrouit, Philippe Rigollet, Yury Polyanskiy","submitted_at":"2023-05-09T14:04:42Z","abstract_excerpt":"Viewing Transformers as interacting particle systems, we describe the geometry of learned representations when the weights are not time dependent. We show that particles, representing tokens, tend to cluster toward particular limiting objects as time tends to infinity. Cluster locations are determined by the initial tokens, confirming context-awareness of representations learned by Transformers. Using techniques from dynamical systems and partial differential equations, we show that the type of limiting object that emerges depends on the spectrum of the value matrix. Additionally, in the one-d"},"verification_status":{"content_addressed":true,"pith_receipt":true,"author_attested":false,"weak_author_claims":0,"strong_author_claims":0,"externally_anchored":false,"storage_verified":false,"citation_signatures":0,"replication_records":0,"graph_snapshot":true,"references_resolved":false,"formal_links_present":false},"canonical_record":{"source":{"id":"2305.05465","kind":"arxiv","version":6},"metadata":{"license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","primary_cat":"cs.LG","submitted_at":"2023-05-09T14:04:42Z","cross_cats_sorted":["math.AP","stat.ML"],"title_canon_sha256":"d4ea358df604a4f5d643d6862daf5e12cc7b833359535b71429063b1d2192dae","abstract_canon_sha256":"dcfe944728d85ab9c7747deb059f573af69721fb54fb8df995856fa61b374eb5"},"schema_version":"1.0"},"receipt":{"kind":"pith_receipt","key_id":"pith-v1-2026-05","algorithm":"ed25519","signed_at":"2026-07-05T07:44:17.966552Z","signature_b64":"PBc/v4rkfIDeaMU8Rel9rskJGAg2K+Xuc4wqgWg/lbv8FsW2UJzjp73ae2PZCHNb0YVTjXMK0K7ak2czyh8XCQ==","signed_message":"canonical_sha256_bytes","builder_version":"pith-number-builder-2026-05-17-v1","receipt_version":"0.3","canonical_sha256":"f19e836cf5a4b0d0b85f4259d0616e49ddb8698409ca06cdada300e615c526c2","last_reissued_at":"2026-07-05T07:44:17.966121Z","signature_status":"signed_v1","first_computed_at":"2026-07-05T07:44:17.966121Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"graph_snapshot":{"paper":{"title":"The emergence of clusters in self-attention dynamics","license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","headline":"","cross_cats":["math.AP","stat.ML"],"primary_cat":"cs.LG","authors_text":"Borjan Geshkovski, Cyril Letrouit, Philippe Rigollet, Yury Polyanskiy","submitted_at":"2023-05-09T14:04:42Z","abstract_excerpt":"Viewing Transformers as interacting particle systems, we describe the geometry of learned representations when the weights are not time dependent. We show that particles, representing tokens, tend to cluster toward particular limiting objects as time tends to infinity. Cluster locations are determined by the initial tokens, confirming context-awareness of representations learned by Transformers. Using techniques from dynamical systems and partial differential equations, we show that the type of limiting object that emerges depends on the spectrum of the value matrix. Additionally, in the one-d"},"claims":{"count":0,"items":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"2305.05465","kind":"arxiv","version":6},"verdict":{"id":null,"model_set":{},"created_at":null,"strongest_claim":"","one_line_summary":"","pipeline_version":null,"weakest_assumption":"","pith_extraction_headline":""},"integrity":{"clean":true,"summary":{"advisory":0,"critical":0,"by_detector":{},"informational":0},"endpoint":"/pith/2305.05465/integrity.json","findings":[],"available":true,"detectors_run":[],"snapshot_sha256":"c28c3603d3b5d939e8dc4c7e95fa8dfce3d595e45f758748cecf8e644a296938"},"references":{"count":0,"sample":[],"resolved_work":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","internal_anchors":0},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"author_claims":{"count":0,"strong_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"builder_version":"pith-number-builder-2026-05-17-v1"},"aliases":[{"alias_kind":"arxiv","alias_value":"2305.05465","created_at":"2026-07-05T07:44:17.966183+00:00"},{"alias_kind":"arxiv_version","alias_value":"2305.05465v6","created_at":"2026-07-05T07:44:17.966183+00:00"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.2305.05465","created_at":"2026-07-05T07:44:17.966183+00:00"},{"alias_kind":"pith_short_12","alias_value":"6GPIG3HVUSYN","created_at":"2026-07-05T07:44:17.966183+00:00"},{"alias_kind":"pith_short_16","alias_value":"6GPIG3HVUSYNBOC7","created_at":"2026-07-05T07:44:17.966183+00:00"},{"alias_kind":"pith_short_8","alias_value":"6GPIG3HV","created_at":"2026-07-05T07:44:17.966183+00:00"}],"events":[],"event_summary":{},"paper_claims":[],"inbound_citations":{"count":6,"internal_anchor_count":6,"sample":[{"citing_arxiv_id":"2606.05140","citing_title":"Phase transitions for the noisy transformer model in arbitrary dimension","ref_index":14,"is_internal_anchor":true},{"citing_arxiv_id":"2407.01602","citing_title":"Clustering in pure-attention hardmax transformers and its role in sentiment analysis","ref_index":13,"is_internal_anchor":true},{"citing_arxiv_id":"2605.12809","citing_title":"Correcting Influence: Unboxing LLM Outputs with Orthogonal Latent Spaces","ref_index":103,"is_internal_anchor":true},{"citing_arxiv_id":"2605.12697","citing_title":"A Unified Framework for Critical Scaling of Inverse Temperature in Self-Attention","ref_index":24,"is_internal_anchor":true},{"citing_arxiv_id":"2605.04279","citing_title":"Gradient Flow Structure and Quantitative Dynamics of Multi-Head Self-Attention","ref_index":9,"is_internal_anchor":true},{"citing_arxiv_id":"2605.04279","citing_title":"Gradient Flow Structure and Quantitative Dynamics of Multi-Head Self-Attention","ref_index":9,"is_internal_anchor":true}]},"formal_canon":{"evidence_count":0,"sample":[],"anchors":[]},"links":{"html":"https://pith.science/pith/6GPIG3HVUSYNBOC7IJM5AYLOJH","json":"https://pith.science/pith/6GPIG3HVUSYNBOC7IJM5AYLOJH.json","graph_json":"https://pith.science/api/pith-number/6GPIG3HVUSYNBOC7IJM5AYLOJH/graph.json","events_json":"https://pith.science/api/pith-number/6GPIG3HVUSYNBOC7IJM5AYLOJH/events.json","paper":"https://pith.science/paper/6GPIG3HV"},"agent_actions":{"view_html":"https://pith.science/pith/6GPIG3HVUSYNBOC7IJM5AYLOJH","download_json":"https://pith.science/pith/6GPIG3HVUSYNBOC7IJM5AYLOJH.json","view_paper":"https://pith.science/paper/6GPIG3HV","resolve_alias":"https://pith.science/api/pith-number/resolve?arxiv=2305.05465&json=true","fetch_graph":"https://pith.science/api/pith-number/6GPIG3HVUSYNBOC7IJM5AYLOJH/graph.json","fetch_events":"https://pith.science/api/pith-number/6GPIG3HVUSYNBOC7IJM5AYLOJH/events.json","actions":{"anchor_timestamp":"https://pith.science/pith/6GPIG3HVUSYNBOC7IJM5AYLOJH/action/timestamp_anchor","attest_storage":"https://pith.science/pith/6GPIG3HVUSYNBOC7IJM5AYLOJH/action/storage_attestation","attest_author":"https://pith.science/pith/6GPIG3HVUSYNBOC7IJM5AYLOJH/action/author_attestation","sign_citation":"https://pith.science/pith/6GPIG3HVUSYNBOC7IJM5AYLOJH/action/citation_signature","submit_replication":"https://pith.science/pith/6GPIG3HVUSYNBOC7IJM5AYLOJH/action/replication_record"}},"created_at":"2026-07-05T07:44:17.966183+00:00","updated_at":"2026-07-05T07:44:17.966183+00:00"}