{"record_type":"pith_number_record","schema_url":"https://pith.science/schemas/pith-number/v1.json","pith_number":"pith:2021:CZXF64POAH27C5T7XVW2QVL32D","short_pith_number":"pith:CZXF64PO","schema_version":"1.0","canonical_sha256":"166e5f71ee01f5f1767fbd6da8557bd0f6294ebb40a8ce09b47f878f10840369","source":{"kind":"arxiv","id":"2109.03127","version":3},"attestation_state":"computed","paper":{"title":"Rare Tokens Degenerate All Tokens: Improving Neural Text Generation via Adaptive Gradient Gating for Rare Token Embeddings","license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","headline":"","cross_cats":[],"primary_cat":"cs.CL","authors_text":"Heeseung Kim, Jongyoon Song, Sangwon Yu, Seong-min Lee, Sungroh Yoon, Woo-Jong Ryu","submitted_at":"2021-09-07T14:48:12Z","abstract_excerpt":"Recent studies have determined that the learned token embeddings of large-scale neural language models are degenerated to be anisotropic with a narrow-cone shape. This phenomenon, called the representation degeneration problem, facilitates an increase in the overall similarity between token embeddings that negatively affect the performance of the models. Although the existing methods that address the degeneration problem based on observations of the phenomenon triggered by the problem improves the performance of the text generation, the training dynamics of token embeddings behind the degenera"},"verification_status":{"content_addressed":true,"pith_receipt":true,"author_attested":false,"weak_author_claims":0,"strong_author_claims":0,"externally_anchored":false,"storage_verified":false,"citation_signatures":0,"replication_records":0,"graph_snapshot":true,"references_resolved":false,"formal_links_present":false},"canonical_record":{"source":{"id":"2109.03127","kind":"arxiv","version":3},"metadata":{"license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","primary_cat":"cs.CL","submitted_at":"2021-09-07T14:48:12Z","cross_cats_sorted":[],"title_canon_sha256":"5aaecddedd808f0a50048735a8ad5bcae523f41b4e2597a2e1e368cdd3350e20","abstract_canon_sha256":"4b6130f92453b89a75de885c51f0a6ade49d29e24fef10f5a148939d2c45c68d"},"schema_version":"1.0"},"receipt":{"kind":"pith_receipt","key_id":"pith-v1-2026-05","algorithm":"ed25519","signed_at":"2026-07-05T04:30:04.125296Z","signature_b64":"lPCo4dUwxvGT9LwmZfWSOpzWsBwVGGgt84mxlkLLSxTP4neeIHuOh4QZAmIXteCSs6vkMWzR1jB+0Rv6VjjFAA==","signed_message":"canonical_sha256_bytes","builder_version":"pith-number-builder-2026-05-17-v1","receipt_version":"0.3","canonical_sha256":"166e5f71ee01f5f1767fbd6da8557bd0f6294ebb40a8ce09b47f878f10840369","last_reissued_at":"2026-07-05T04:30:04.124898Z","signature_status":"signed_v1","first_computed_at":"2026-07-05T04:30:04.124898Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"graph_snapshot":{"paper":{"title":"Rare Tokens Degenerate All Tokens: Improving Neural Text Generation via Adaptive Gradient Gating for Rare Token Embeddings","license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","headline":"","cross_cats":[],"primary_cat":"cs.CL","authors_text":"Heeseung Kim, Jongyoon Song, Sangwon Yu, Seong-min Lee, Sungroh Yoon, Woo-Jong Ryu","submitted_at":"2021-09-07T14:48:12Z","abstract_excerpt":"Recent studies have determined that the learned token embeddings of large-scale neural language models are degenerated to be anisotropic with a narrow-cone shape. This phenomenon, called the representation degeneration problem, facilitates an increase in the overall similarity between token embeddings that negatively affect the performance of the models. Although the existing methods that address the degeneration problem based on observations of the phenomenon triggered by the problem improves the performance of the text generation, the training dynamics of token embeddings behind the degenera"},"claims":{"count":0,"items":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"2109.03127","kind":"arxiv","version":3},"verdict":{"id":null,"model_set":{},"created_at":null,"strongest_claim":"","one_line_summary":"","pipeline_version":null,"weakest_assumption":"","pith_extraction_headline":""},"integrity":{"clean":true,"summary":{"advisory":0,"critical":0,"by_detector":{},"informational":0},"endpoint":"/pith/2109.03127/integrity.json","findings":[],"available":true,"detectors_run":[],"snapshot_sha256":"c28c3603d3b5d939e8dc4c7e95fa8dfce3d595e45f758748cecf8e644a296938"},"references":{"count":0,"sample":[],"resolved_work":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","internal_anchors":0},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"author_claims":{"count":0,"strong_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"builder_version":"pith-number-builder-2026-05-17-v1"},"aliases":[{"alias_kind":"arxiv","alias_value":"2109.03127","created_at":"2026-07-05T04:30:04.124955+00:00"},{"alias_kind":"arxiv_version","alias_value":"2109.03127v3","created_at":"2026-07-05T04:30:04.124955+00:00"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.2109.03127","created_at":"2026-07-05T04:30:04.124955+00:00"},{"alias_kind":"pith_short_12","alias_value":"CZXF64POAH27","created_at":"2026-07-05T04:30:04.124955+00:00"},{"alias_kind":"pith_short_16","alias_value":"CZXF64POAH27C5T7","created_at":"2026-07-05T04:30:04.124955+00:00"},{"alias_kind":"pith_short_8","alias_value":"CZXF64PO","created_at":"2026-07-05T04:30:04.124955+00:00"}],"events":[],"event_summary":{},"paper_claims":[],"inbound_citations":{"count":0,"internal_anchor_count":0,"sample":[]},"formal_canon":{"evidence_count":0,"sample":[],"anchors":[]},"links":{"html":"https://pith.science/pith/CZXF64POAH27C5T7XVW2QVL32D","json":"https://pith.science/pith/CZXF64POAH27C5T7XVW2QVL32D.json","graph_json":"https://pith.science/api/pith-number/CZXF64POAH27C5T7XVW2QVL32D/graph.json","events_json":"https://pith.science/api/pith-number/CZXF64POAH27C5T7XVW2QVL32D/events.json","paper":"https://pith.science/paper/CZXF64PO"},"agent_actions":{"view_html":"https://pith.science/pith/CZXF64POAH27C5T7XVW2QVL32D","download_json":"https://pith.science/pith/CZXF64POAH27C5T7XVW2QVL32D.json","view_paper":"https://pith.science/paper/CZXF64PO","resolve_alias":"https://pith.science/api/pith-number/resolve?arxiv=2109.03127&json=true","fetch_graph":"https://pith.science/api/pith-number/CZXF64POAH27C5T7XVW2QVL32D/graph.json","fetch_events":"https://pith.science/api/pith-number/CZXF64POAH27C5T7XVW2QVL32D/events.json","actions":{"anchor_timestamp":"https://pith.science/pith/CZXF64POAH27C5T7XVW2QVL32D/action/timestamp_anchor","attest_storage":"https://pith.science/pith/CZXF64POAH27C5T7XVW2QVL32D/action/storage_attestation","attest_author":"https://pith.science/pith/CZXF64POAH27C5T7XVW2QVL32D/action/author_attestation","sign_citation":"https://pith.science/pith/CZXF64POAH27C5T7XVW2QVL32D/action/citation_signature","submit_replication":"https://pith.science/pith/CZXF64POAH27C5T7XVW2QVL32D/action/replication_record"}},"created_at":"2026-07-05T04:30:04.124955+00:00","updated_at":"2026-07-05T04:30:04.124955+00:00"}