{"record_type":"pith_number_record","schema_url":"https://pith.science/schemas/pith-number/v1.json","pith_number":"pith:2024:YPCNMCKDNGCOBPZ35H43HXEOQK","short_pith_number":"pith:YPCNMCKD","schema_version":"1.0","canonical_sha256":"c3c4d609436984e0bf3be9f9b3dc8e828155b20ccc1482b599210e487b5d8359","source":{"kind":"arxiv","id":"2404.02305","version":1},"attestation_state":"computed","paper":{"title":"Collapse of Self-trained Language Models","license":"http://creativecommons.org/licenses/by/4.0/","headline":"","cross_cats":["cs.AI"],"primary_cat":"cs.CL","authors_text":"David Herel, Tomas Mikolov","submitted_at":"2024-04-02T21:03:37Z","abstract_excerpt":"In various fields of knowledge creation, including science, new ideas often build on pre-existing information. In this work, we explore this concept within the context of language models. Specifically, we explore the potential of self-training models on their own outputs, akin to how humans learn and build on their previous thoughts and actions. While this approach is intuitively appealing, our research reveals its practical limitations. We find that extended self-training of the GPT-2 model leads to a significant degradation in performance, resulting in repetitive and collapsed token output."},"verification_status":{"content_addressed":true,"pith_receipt":true,"author_attested":false,"weak_author_claims":0,"strong_author_claims":0,"externally_anchored":false,"storage_verified":false,"citation_signatures":0,"replication_records":0,"graph_snapshot":true,"references_resolved":false,"formal_links_present":false},"canonical_record":{"source":{"id":"2404.02305","kind":"arxiv","version":1},"metadata":{"license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.CL","submitted_at":"2024-04-02T21:03:37Z","cross_cats_sorted":["cs.AI"],"title_canon_sha256":"6bde97baf5177420f595fa2fac7d902cb9e2ffe41f692557b5dc7ce34003ae81","abstract_canon_sha256":"17afa03050097c0ab5aeae0b0eb91a71702d9d532dcc364c3873c06af444762d"},"schema_version":"1.0"},"receipt":{"kind":"pith_receipt","key_id":"pith-v1-2026-05","algorithm":"ed25519","signed_at":"2026-07-05T08:03:53.819390Z","signature_b64":"3mgsWyhzkMPa9qs9lZTSI9SZ1cjqlrwekaHyI+Wa2EUk2u8hxrPLvAzc/E2bgFWeUZp9O70y+m+lCmpLpXrFAQ==","signed_message":"canonical_sha256_bytes","builder_version":"pith-number-builder-2026-05-17-v1","receipt_version":"0.3","canonical_sha256":"c3c4d609436984e0bf3be9f9b3dc8e828155b20ccc1482b599210e487b5d8359","last_reissued_at":"2026-07-05T08:03:53.818939Z","signature_status":"signed_v1","first_computed_at":"2026-07-05T08:03:53.818939Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"graph_snapshot":{"paper":{"title":"Collapse of Self-trained Language Models","license":"http://creativecommons.org/licenses/by/4.0/","headline":"","cross_cats":["cs.AI"],"primary_cat":"cs.CL","authors_text":"David Herel, Tomas Mikolov","submitted_at":"2024-04-02T21:03:37Z","abstract_excerpt":"In various fields of knowledge creation, including science, new ideas often build on pre-existing information. In this work, we explore this concept within the context of language models. Specifically, we explore the potential of self-training models on their own outputs, akin to how humans learn and build on their previous thoughts and actions. While this approach is intuitively appealing, our research reveals its practical limitations. We find that extended self-training of the GPT-2 model leads to a significant degradation in performance, resulting in repetitive and collapsed token output."},"claims":{"count":0,"items":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"2404.02305","kind":"arxiv","version":1},"verdict":{"id":null,"model_set":{},"created_at":null,"strongest_claim":"","one_line_summary":"","pipeline_version":null,"weakest_assumption":"","pith_extraction_headline":""},"integrity":{"clean":true,"summary":{"advisory":0,"critical":0,"by_detector":{},"informational":0},"endpoint":"/pith/2404.02305/integrity.json","findings":[],"available":true,"detectors_run":[],"snapshot_sha256":"c28c3603d3b5d939e8dc4c7e95fa8dfce3d595e45f758748cecf8e644a296938"},"references":{"count":0,"sample":[],"resolved_work":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","internal_anchors":0},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"author_claims":{"count":0,"strong_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"builder_version":"pith-number-builder-2026-05-17-v1"},"aliases":[{"alias_kind":"arxiv","alias_value":"2404.02305","created_at":"2026-07-05T08:03:53.818993+00:00"},{"alias_kind":"arxiv_version","alias_value":"2404.02305v1","created_at":"2026-07-05T08:03:53.818993+00:00"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.2404.02305","created_at":"2026-07-05T08:03:53.818993+00:00"},{"alias_kind":"pith_short_12","alias_value":"YPCNMCKDNGCO","created_at":"2026-07-05T08:03:53.818993+00:00"},{"alias_kind":"pith_short_16","alias_value":"YPCNMCKDNGCOBPZ3","created_at":"2026-07-05T08:03:53.818993+00:00"},{"alias_kind":"pith_short_8","alias_value":"YPCNMCKD","created_at":"2026-07-05T08:03:53.818993+00:00"}],"events":[],"event_summary":{},"paper_claims":[],"inbound_citations":{"count":3,"internal_anchor_count":0,"sample":[{"citing_arxiv_id":"2606.26091","citing_title":"On-Policy Self-Distillation with Sampled Demonstrations Reduces Output Diversity","ref_index":205,"is_internal_anchor":false},{"citing_arxiv_id":"2606.23611","citing_title":"Data Selection Through Iterative Self-Filtering for Vision-Language Settings","ref_index":184,"is_internal_anchor":false},{"citing_arxiv_id":"2605.20602","citing_title":"Self-Training Doesn't Flatten Language -- It Restructures It: Surface Markers Amplify While Deep Syntax Dies","ref_index":12,"is_internal_anchor":false}]},"formal_canon":{"evidence_count":0,"sample":[],"anchors":[]},"links":{"html":"https://pith.science/pith/YPCNMCKDNGCOBPZ35H43HXEOQK","json":"https://pith.science/pith/YPCNMCKDNGCOBPZ35H43HXEOQK.json","graph_json":"https://pith.science/api/pith-number/YPCNMCKDNGCOBPZ35H43HXEOQK/graph.json","events_json":"https://pith.science/api/pith-number/YPCNMCKDNGCOBPZ35H43HXEOQK/events.json","paper":"https://pith.science/paper/YPCNMCKD"},"agent_actions":{"view_html":"https://pith.science/pith/YPCNMCKDNGCOBPZ35H43HXEOQK","download_json":"https://pith.science/pith/YPCNMCKDNGCOBPZ35H43HXEOQK.json","view_paper":"https://pith.science/paper/YPCNMCKD","resolve_alias":"https://pith.science/api/pith-number/resolve?arxiv=2404.02305&json=true","fetch_graph":"https://pith.science/api/pith-number/YPCNMCKDNGCOBPZ35H43HXEOQK/graph.json","fetch_events":"https://pith.science/api/pith-number/YPCNMCKDNGCOBPZ35H43HXEOQK/events.json","actions":{"anchor_timestamp":"https://pith.science/pith/YPCNMCKDNGCOBPZ35H43HXEOQK/action/timestamp_anchor","attest_storage":"https://pith.science/pith/YPCNMCKDNGCOBPZ35H43HXEOQK/action/storage_attestation","attest_author":"https://pith.science/pith/YPCNMCKDNGCOBPZ35H43HXEOQK/action/author_attestation","sign_citation":"https://pith.science/pith/YPCNMCKDNGCOBPZ35H43HXEOQK/action/citation_signature","submit_replication":"https://pith.science/pith/YPCNMCKDNGCOBPZ35H43HXEOQK/action/replication_record"}},"created_at":"2026-07-05T08:03:53.818993+00:00","updated_at":"2026-07-05T08:03:53.818993+00:00"}