{"record_type":"pith_number_record","schema_url":"https://pith.science/schemas/pith-number/v1.json","pith_number":"pith:2023:DOSBXT2RLFAJ32CJM6EPRXMFYD","short_pith_number":"pith:DOSBXT2R","schema_version":"1.0","canonical_sha256":"1ba41bcf5159409de8496788f8dd85c0f584a2ea0e0f83ba86593f3c40b7e552","source":{"kind":"arxiv","id":"2311.05928","version":2},"attestation_state":"computed","paper":{"title":"The Shape of Learning: Anisotropy and Intrinsic Dimensions in Transformer-Based Models","license":"http://creativecommons.org/publicdomain/zero/1.0/","headline":"","cross_cats":["cs.AI","cs.IT","cs.LG","math.GN","math.IT"],"primary_cat":"cs.CL","authors_text":"Andrey Kuznetsov, Anton Razzhigaev, Denis Dimitrov, Elizaveta Goncharova, Ivan Oseledets, Matvey Mikhalchuk","submitted_at":"2023-11-10T08:25:02Z","abstract_excerpt":"In this study, we present an investigation into the anisotropy dynamics and intrinsic dimension of embeddings in transformer architectures, focusing on the dichotomy between encoders and decoders. Our findings reveal that the anisotropy profile in transformer decoders exhibits a distinct bell-shaped curve, with the highest anisotropy concentrations in the middle layers. This pattern diverges from the more uniformly distributed anisotropy observed in encoders. In addition, we found that the intrinsic dimension of embeddings increases in the initial phases of training, indicating an expansion in"},"verification_status":{"content_addressed":true,"pith_receipt":true,"author_attested":false,"weak_author_claims":0,"strong_author_claims":0,"externally_anchored":false,"storage_verified":false,"citation_signatures":0,"replication_records":0,"graph_snapshot":true,"references_resolved":false,"formal_links_present":false},"canonical_record":{"source":{"id":"2311.05928","kind":"arxiv","version":2},"metadata":{"license":"http://creativecommons.org/publicdomain/zero/1.0/","primary_cat":"cs.CL","submitted_at":"2023-11-10T08:25:02Z","cross_cats_sorted":["cs.AI","cs.IT","cs.LG","math.GN","math.IT"],"title_canon_sha256":"8aa358eecb545b309281b93217c91ad388131dcc438c6a525f566ae09b507dfc","abstract_canon_sha256":"7db70afd19bfdc6993066bd43293399393de09d8343226437ab608ac8c5eb89a"},"schema_version":"1.0"},"receipt":{"kind":"pith_receipt","key_id":"pith-v1-2026-05","algorithm":"ed25519","signed_at":"2026-07-05T07:49:15.950637Z","signature_b64":"9KksY54s9BZhzUI0JxXIiETkwFD/JOol9LJgDKOLzJenyMwpYZMecN4hiezMCR0rIT/6TNDXeWYdYbebC+BABA==","signed_message":"canonical_sha256_bytes","builder_version":"pith-number-builder-2026-05-17-v1","receipt_version":"0.3","canonical_sha256":"1ba41bcf5159409de8496788f8dd85c0f584a2ea0e0f83ba86593f3c40b7e552","last_reissued_at":"2026-07-05T07:49:15.950196Z","signature_status":"signed_v1","first_computed_at":"2026-07-05T07:49:15.950196Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"graph_snapshot":{"paper":{"title":"The Shape of Learning: Anisotropy and Intrinsic Dimensions in Transformer-Based Models","license":"http://creativecommons.org/publicdomain/zero/1.0/","headline":"","cross_cats":["cs.AI","cs.IT","cs.LG","math.GN","math.IT"],"primary_cat":"cs.CL","authors_text":"Andrey Kuznetsov, Anton Razzhigaev, Denis Dimitrov, Elizaveta Goncharova, Ivan Oseledets, Matvey Mikhalchuk","submitted_at":"2023-11-10T08:25:02Z","abstract_excerpt":"In this study, we present an investigation into the anisotropy dynamics and intrinsic dimension of embeddings in transformer architectures, focusing on the dichotomy between encoders and decoders. Our findings reveal that the anisotropy profile in transformer decoders exhibits a distinct bell-shaped curve, with the highest anisotropy concentrations in the middle layers. This pattern diverges from the more uniformly distributed anisotropy observed in encoders. In addition, we found that the intrinsic dimension of embeddings increases in the initial phases of training, indicating an expansion in"},"claims":{"count":0,"items":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"2311.05928","kind":"arxiv","version":2},"verdict":{"id":null,"model_set":{},"created_at":null,"strongest_claim":"","one_line_summary":"","pipeline_version":null,"weakest_assumption":"","pith_extraction_headline":""},"integrity":{"clean":true,"summary":{"advisory":0,"critical":0,"by_detector":{},"informational":0},"endpoint":"/pith/2311.05928/integrity.json","findings":[],"available":true,"detectors_run":[],"snapshot_sha256":"c28c3603d3b5d939e8dc4c7e95fa8dfce3d595e45f758748cecf8e644a296938"},"references":{"count":0,"sample":[],"resolved_work":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","internal_anchors":0},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"author_claims":{"count":0,"strong_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"builder_version":"pith-number-builder-2026-05-17-v1"},"aliases":[{"alias_kind":"arxiv","alias_value":"2311.05928","created_at":"2026-07-05T07:49:15.950272+00:00"},{"alias_kind":"arxiv_version","alias_value":"2311.05928v2","created_at":"2026-07-05T07:49:15.950272+00:00"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.2311.05928","created_at":"2026-07-05T07:49:15.950272+00:00"},{"alias_kind":"pith_short_12","alias_value":"DOSBXT2RLFAJ","created_at":"2026-07-05T07:49:15.950272+00:00"},{"alias_kind":"pith_short_16","alias_value":"DOSBXT2RLFAJ32CJ","created_at":"2026-07-05T07:49:15.950272+00:00"},{"alias_kind":"pith_short_8","alias_value":"DOSBXT2R","created_at":"2026-07-05T07:49:15.950272+00:00"}],"events":[],"event_summary":{},"paper_claims":[],"inbound_citations":{"count":2,"internal_anchor_count":0,"sample":[{"citing_arxiv_id":"2606.07604","citing_title":"Contribution Weights: A Geometrical Analysis of Self-Attention Transformers","ref_index":73,"is_internal_anchor":false},{"citing_arxiv_id":"2505.03205","citing_title":"Transformers for Learning on Noisy and Task-Level Manifolds: Approximation and Generalization Insights","ref_index":10,"is_internal_anchor":false}]},"formal_canon":{"evidence_count":0,"sample":[],"anchors":[]},"links":{"html":"https://pith.science/pith/DOSBXT2RLFAJ32CJM6EPRXMFYD","json":"https://pith.science/pith/DOSBXT2RLFAJ32CJM6EPRXMFYD.json","graph_json":"https://pith.science/api/pith-number/DOSBXT2RLFAJ32CJM6EPRXMFYD/graph.json","events_json":"https://pith.science/api/pith-number/DOSBXT2RLFAJ32CJM6EPRXMFYD/events.json","paper":"https://pith.science/paper/DOSBXT2R"},"agent_actions":{"view_html":"https://pith.science/pith/DOSBXT2RLFAJ32CJM6EPRXMFYD","download_json":"https://pith.science/pith/DOSBXT2RLFAJ32CJM6EPRXMFYD.json","view_paper":"https://pith.science/paper/DOSBXT2R","resolve_alias":"https://pith.science/api/pith-number/resolve?arxiv=2311.05928&json=true","fetch_graph":"https://pith.science/api/pith-number/DOSBXT2RLFAJ32CJM6EPRXMFYD/graph.json","fetch_events":"https://pith.science/api/pith-number/DOSBXT2RLFAJ32CJM6EPRXMFYD/events.json","actions":{"anchor_timestamp":"https://pith.science/pith/DOSBXT2RLFAJ32CJM6EPRXMFYD/action/timestamp_anchor","attest_storage":"https://pith.science/pith/DOSBXT2RLFAJ32CJM6EPRXMFYD/action/storage_attestation","attest_author":"https://pith.science/pith/DOSBXT2RLFAJ32CJM6EPRXMFYD/action/author_attestation","sign_citation":"https://pith.science/pith/DOSBXT2RLFAJ32CJM6EPRXMFYD/action/citation_signature","submit_replication":"https://pith.science/pith/DOSBXT2RLFAJ32CJM6EPRXMFYD/action/replication_record"}},"created_at":"2026-07-05T07:49:15.950272+00:00","updated_at":"2026-07-05T07:49:15.950272+00:00"}