{"record_type":"pith_number_record","schema_url":"https://pith.science/schemas/pith-number/v1.json","pith_number":"pith:2019:5BCQ7KQRKSAVD25RVM4RRY6TNW","short_pith_number":"pith:5BCQ7KQR","schema_version":"1.0","canonical_sha256":"e8450faa11548151ebb1ab3918e3d36d829f91407ad7877928fc8fe2546f6fbf","source":{"kind":"arxiv","id":"1909.00512","version":1},"attestation_state":"computed","paper":{"title":"How Contextual are Contextualized Word Representations? Comparing the Geometry of BERT, ELMo, and GPT-2 Embeddings","license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","headline":"","cross_cats":[],"primary_cat":"cs.CL","authors_text":"Kawin Ethayarajh","submitted_at":"2019-09-02T01:51:46Z","abstract_excerpt":"Replacing static word embeddings with contextualized word representations has yielded significant improvements on many NLP tasks. However, just how contextual are the contextualized representations produced by models such as ELMo and BERT? Are there infinitely many context-specific representations for each word, or are words essentially assigned one of a finite number of word-sense representations? For one, we find that the contextualized representations of all words are not isotropic in any layer of the contextualizing model. While representations of the same word in different contexts still "},"verification_status":{"content_addressed":true,"pith_receipt":true,"author_attested":false,"weak_author_claims":0,"strong_author_claims":0,"externally_anchored":false,"storage_verified":false,"citation_signatures":0,"replication_records":0,"graph_snapshot":true,"references_resolved":false,"formal_links_present":false},"canonical_record":{"source":{"id":"1909.00512","kind":"arxiv","version":1},"metadata":{"license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","primary_cat":"cs.CL","submitted_at":"2019-09-02T01:51:46Z","cross_cats_sorted":[],"title_canon_sha256":"fd39e15e86653b97d190a81348af690ca9ebf0de5659e543acc0df07522910e3","abstract_canon_sha256":"1defcb13386141c0b68439f40d6b57f747b9e6f4444e61a62e908a0ef0e74788"},"schema_version":"1.0"},"receipt":{"kind":"pith_receipt","key_id":"pith-v1-2026-05","algorithm":"ed25519","signed_at":"2026-07-05T00:01:10.372882Z","signature_b64":"QoYdPCwZFiQgrL14MCk2t8d5V7UzGs4V/b2X4FLOLt6+35g1Dv9gTUCsuf+MSD8KkgkcVuo0V9UoJhE7/SRABA==","signed_message":"canonical_sha256_bytes","builder_version":"pith-number-builder-2026-05-17-v1","receipt_version":"0.3","canonical_sha256":"e8450faa11548151ebb1ab3918e3d36d829f91407ad7877928fc8fe2546f6fbf","last_reissued_at":"2026-07-05T00:01:10.372494Z","signature_status":"signed_v1","first_computed_at":"2026-07-05T00:01:10.372494Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"graph_snapshot":{"paper":{"title":"How Contextual are Contextualized Word Representations? Comparing the Geometry of BERT, ELMo, and GPT-2 Embeddings","license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","headline":"","cross_cats":[],"primary_cat":"cs.CL","authors_text":"Kawin Ethayarajh","submitted_at":"2019-09-02T01:51:46Z","abstract_excerpt":"Replacing static word embeddings with contextualized word representations has yielded significant improvements on many NLP tasks. However, just how contextual are the contextualized representations produced by models such as ELMo and BERT? Are there infinitely many context-specific representations for each word, or are words essentially assigned one of a finite number of word-sense representations? For one, we find that the contextualized representations of all words are not isotropic in any layer of the contextualizing model. While representations of the same word in different contexts still "},"claims":{"count":0,"items":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"1909.00512","kind":"arxiv","version":1},"verdict":{"id":null,"model_set":{},"created_at":null,"strongest_claim":"","one_line_summary":"","pipeline_version":null,"weakest_assumption":"","pith_extraction_headline":""},"integrity":{"clean":true,"summary":{"advisory":0,"critical":0,"by_detector":{},"informational":0},"endpoint":"/pith/1909.00512/integrity.json","findings":[],"available":true,"detectors_run":[],"snapshot_sha256":"c28c3603d3b5d939e8dc4c7e95fa8dfce3d595e45f758748cecf8e644a296938"},"references":{"count":0,"sample":[],"resolved_work":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","internal_anchors":0},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"author_claims":{"count":0,"strong_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"builder_version":"pith-number-builder-2026-05-17-v1"},"aliases":[{"alias_kind":"arxiv","alias_value":"1909.00512","created_at":"2026-07-05T00:01:10.372552+00:00"},{"alias_kind":"arxiv_version","alias_value":"1909.00512v1","created_at":"2026-07-05T00:01:10.372552+00:00"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.1909.00512","created_at":"2026-07-05T00:01:10.372552+00:00"},{"alias_kind":"pith_short_12","alias_value":"5BCQ7KQRKSAV","created_at":"2026-07-05T00:01:10.372552+00:00"},{"alias_kind":"pith_short_16","alias_value":"5BCQ7KQRKSAVD25R","created_at":"2026-07-05T00:01:10.372552+00:00"},{"alias_kind":"pith_short_8","alias_value":"5BCQ7KQR","created_at":"2026-07-05T00:01:10.372552+00:00"}],"events":[],"event_summary":{},"paper_claims":[],"inbound_citations":{"count":19,"internal_anchor_count":1,"sample":[{"citing_arxiv_id":"2607.07047","citing_title":"Riemannian Geometry for Pre-trained Language Model Embeddings","ref_index":19,"is_internal_anchor":true},{"citing_arxiv_id":"2606.24937","citing_title":"The Hitchhiker's Guide to Agentic AI: From Foundations to Systems","ref_index":40,"is_internal_anchor":false},{"citing_arxiv_id":"2606.22019","citing_title":"Channel Location Constrains the Auditability of Subliminal Learning","ref_index":7,"is_internal_anchor":false},{"citing_arxiv_id":"2605.24956","citing_title":"NITP: Next Implicit Token Prediction for LLM Pre-training","ref_index":9,"is_internal_anchor":false},{"citing_arxiv_id":"2606.01844","citing_title":"Decoupled Residual Quantization for Robust Semantic IDs in Recommendation","ref_index":13,"is_internal_anchor":false},{"citing_arxiv_id":"2606.01671","citing_title":"When Meaning Travels: A Granular Lens on Hybrid-MoE's Role in Idiomatic Understanding for Language Models","ref_index":82,"is_internal_anchor":false},{"citing_arxiv_id":"2605.24956","citing_title":"NITP: Next Implicit Token Prediction for LLM Pre-training","ref_index":9,"is_internal_anchor":false},{"citing_arxiv_id":"2606.28343","citing_title":"The Crowded Embedding Space: A Mean-Field Mechanism for Emergent Marginalization in Retrieval-Augmented Agents","ref_index":4,"is_internal_anchor":false},{"citing_arxiv_id":"2606.29148","citing_title":"GPC: Large-Scale Generative Pretraining for Transferable Motor Control","ref_index":292,"is_internal_anchor":false},{"citing_arxiv_id":"2605.30596","citing_title":"Improving Relative Representations with Learned Anchors and Whitened Inner Products","ref_index":3,"is_internal_anchor":false},{"citing_arxiv_id":"2605.17355","citing_title":"HyperPersona: A Multi-Level Hypergraph Framework for Text-Based Automatic Personality Prediction","ref_index":60,"is_internal_anchor":false},{"citing_arxiv_id":"2605.15394","citing_title":"Representation Without Reward: A JEPA Audit for LLM Fine-Tuning","ref_index":26,"is_internal_anchor":false},{"citing_arxiv_id":"2605.12813","citing_title":"REALISTA: Realistic Latent Adversarial Attacks that Elicit LLM Hallucinations","ref_index":21,"is_internal_anchor":false},{"citing_arxiv_id":"2604.27914","citing_title":"Geometry-Calibrated Conformal Abstention for Language Models","ref_index":43,"is_internal_anchor":false},{"citing_arxiv_id":"2605.05741","citing_title":"HyperLens: Quantifying Cognitive Effort in LLMs with Fine-grained Confidence Trajectory","ref_index":28,"is_internal_anchor":false},{"citing_arxiv_id":"2605.00380","citing_title":"ResRL: Boosting LLM Reasoning via Negative Sample Projection Residual Reinforcement Learning","ref_index":15,"is_internal_anchor":false},{"citing_arxiv_id":"2605.00380","citing_title":"ResRL: Boosting LLM Reasoning via Negative Sample Projection Residual Reinforcement Learning","ref_index":15,"is_internal_anchor":false},{"citing_arxiv_id":"2604.17054","citing_title":"mEOL: Training-Free Instruction-Guided Multimodal Embedder for Vector Graphics and Image Retrieval","ref_index":12,"is_internal_anchor":false},{"citing_arxiv_id":"2604.17257","citing_title":"REZE: Representation Regularization for Domain-adaptive Text Embedding Pre-finetuning","ref_index":41,"is_internal_anchor":false}]},"formal_canon":{"evidence_count":0,"sample":[],"anchors":[]},"links":{"html":"https://pith.science/pith/5BCQ7KQRKSAVD25RVM4RRY6TNW","json":"https://pith.science/pith/5BCQ7KQRKSAVD25RVM4RRY6TNW.json","graph_json":"https://pith.science/api/pith-number/5BCQ7KQRKSAVD25RVM4RRY6TNW/graph.json","events_json":"https://pith.science/api/pith-number/5BCQ7KQRKSAVD25RVM4RRY6TNW/events.json","paper":"https://pith.science/paper/5BCQ7KQR"},"agent_actions":{"view_html":"https://pith.science/pith/5BCQ7KQRKSAVD25RVM4RRY6TNW","download_json":"https://pith.science/pith/5BCQ7KQRKSAVD25RVM4RRY6TNW.json","view_paper":"https://pith.science/paper/5BCQ7KQR","resolve_alias":"https://pith.science/api/pith-number/resolve?arxiv=1909.00512&json=true","fetch_graph":"https://pith.science/api/pith-number/5BCQ7KQRKSAVD25RVM4RRY6TNW/graph.json","fetch_events":"https://pith.science/api/pith-number/5BCQ7KQRKSAVD25RVM4RRY6TNW/events.json","actions":{"anchor_timestamp":"https://pith.science/pith/5BCQ7KQRKSAVD25RVM4RRY6TNW/action/timestamp_anchor","attest_storage":"https://pith.science/pith/5BCQ7KQRKSAVD25RVM4RRY6TNW/action/storage_attestation","attest_author":"https://pith.science/pith/5BCQ7KQRKSAVD25RVM4RRY6TNW/action/author_attestation","sign_citation":"https://pith.science/pith/5BCQ7KQRKSAVD25RVM4RRY6TNW/action/citation_signature","submit_replication":"https://pith.science/pith/5BCQ7KQRKSAVD25RVM4RRY6TNW/action/replication_record"}},"created_at":"2026-07-05T00:01:10.372552+00:00","updated_at":"2026-07-05T00:01:10.372552+00:00"}