{"record_type":"pith_number_record","schema_url":"https://pith.science/schemas/pith-number/v1.json","pith_number":"pith:2022:WHDFRWDA4MMY345DDJ7NHD5LHS","short_pith_number":"pith:WHDFRWDA","schema_version":"1.0","canonical_sha256":"b1c658d860e3198df3a31a7ed38fab3cab2799239bc51ca596e2061fcaeb7dac","source":{"kind":"arxiv","id":"2205.05131","version":3},"attestation_state":"computed","paper":{"title":"UL2: Unifying Language Learning Paradigms","license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","headline":"","cross_cats":[],"primary_cat":"cs.CL","authors_text":"Dara Bahri, Denny Zhou, Donald Metzler, Huaixiu Steven Zheng, Hyung Won Chung, Jason Wei, Mostafa Dehghani, Neil Houlsby, Siamak Shakeri, Tal Schuster, Vinh Q. Tran, Xavier Garcia, Xuezhi Wang, Yi Tay","submitted_at":"2022-05-10T19:32:20Z","abstract_excerpt":"Existing pre-trained models are generally geared towards a particular class of problems. To date, there seems to be still no consensus on what the right architecture and pre-training setup should be. This paper presents a unified framework for pre-training models that are universally effective across datasets and setups. We begin by disentangling architectural archetypes with pre-training objectives -- two concepts that are commonly conflated. Next, we present a generalized & unified perspective for self-supervision in NLP and show how different pre-training objectives can be cast as one anoth"},"verification_status":{"content_addressed":true,"pith_receipt":true,"author_attested":false,"weak_author_claims":0,"strong_author_claims":0,"externally_anchored":false,"storage_verified":false,"citation_signatures":0,"replication_records":0,"graph_snapshot":true,"references_resolved":false,"formal_links_present":false},"canonical_record":{"source":{"id":"2205.05131","kind":"arxiv","version":3},"metadata":{"license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","primary_cat":"cs.CL","submitted_at":"2022-05-10T19:32:20Z","cross_cats_sorted":[],"title_canon_sha256":"22d189e1404a3a46ee2e98c5010afc8f98d600327efc31c4f091d81e6cc29357","abstract_canon_sha256":"a11b4c754f6e2e62c21ce251cf3b86ef298d1ebeb8a3bb99db61d9de85a2b2de"},"schema_version":"1.0"},"receipt":{"kind":"pith_receipt","key_id":"pith-v1-2026-05","algorithm":"ed25519","signed_at":"2026-07-05T05:46:15.508694Z","signature_b64":"EzOoJ0bT2PQFoeQjIlyO7qERkApE6m/bMgb+7CXtuvj/Lb7sfrU4oEjCh7s5CbT2nZXTrDKv4bMbQBkDxdgPAg==","signed_message":"canonical_sha256_bytes","builder_version":"pith-number-builder-2026-05-17-v1","receipt_version":"0.3","canonical_sha256":"b1c658d860e3198df3a31a7ed38fab3cab2799239bc51ca596e2061fcaeb7dac","last_reissued_at":"2026-07-05T05:46:15.508110Z","signature_status":"signed_v1","first_computed_at":"2026-07-05T05:46:15.508110Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"graph_snapshot":{"paper":{"title":"UL2: Unifying Language Learning Paradigms","license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","headline":"","cross_cats":[],"primary_cat":"cs.CL","authors_text":"Dara Bahri, Denny Zhou, Donald Metzler, Huaixiu Steven Zheng, Hyung Won Chung, Jason Wei, Mostafa Dehghani, Neil Houlsby, Siamak Shakeri, Tal Schuster, Vinh Q. Tran, Xavier Garcia, Xuezhi Wang, Yi Tay","submitted_at":"2022-05-10T19:32:20Z","abstract_excerpt":"Existing pre-trained models are generally geared towards a particular class of problems. To date, there seems to be still no consensus on what the right architecture and pre-training setup should be. This paper presents a unified framework for pre-training models that are universally effective across datasets and setups. We begin by disentangling architectural archetypes with pre-training objectives -- two concepts that are commonly conflated. Next, we present a generalized & unified perspective for self-supervision in NLP and show how different pre-training objectives can be cast as one anoth"},"claims":{"count":0,"items":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"2205.05131","kind":"arxiv","version":3},"verdict":{"id":null,"model_set":{},"created_at":null,"strongest_claim":"","one_line_summary":"","pipeline_version":null,"weakest_assumption":"","pith_extraction_headline":""},"integrity":{"clean":true,"summary":{"advisory":0,"critical":0,"by_detector":{},"informational":0},"endpoint":"/pith/2205.05131/integrity.json","findings":[],"available":true,"detectors_run":[],"snapshot_sha256":"c28c3603d3b5d939e8dc4c7e95fa8dfce3d595e45f758748cecf8e644a296938"},"references":{"count":0,"sample":[],"resolved_work":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","internal_anchors":0},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"author_claims":{"count":0,"strong_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"builder_version":"pith-number-builder-2026-05-17-v1"},"aliases":[{"alias_kind":"arxiv","alias_value":"2205.05131","created_at":"2026-07-05T05:46:15.508176+00:00"},{"alias_kind":"arxiv_version","alias_value":"2205.05131v3","created_at":"2026-07-05T05:46:15.508176+00:00"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.2205.05131","created_at":"2026-07-05T05:46:15.508176+00:00"},{"alias_kind":"pith_short_12","alias_value":"WHDFRWDA4MMY","created_at":"2026-07-05T05:46:15.508176+00:00"},{"alias_kind":"pith_short_16","alias_value":"WHDFRWDA4MMY345D","created_at":"2026-07-05T05:46:15.508176+00:00"},{"alias_kind":"pith_short_8","alias_value":"WHDFRWDA","created_at":"2026-07-05T05:46:15.508176+00:00"}],"events":[],"event_summary":{},"paper_claims":[],"inbound_citations":{"count":21,"internal_anchor_count":0,"sample":[{"citing_arxiv_id":"2606.16246","citing_title":"Demystifying Training-Time Augmentation for Data-Constrained Language Model Pretraining","ref_index":8,"is_internal_anchor":false},{"citing_arxiv_id":"2605.24981","citing_title":"Large Language Model Selection with Limited Annotations","ref_index":107,"is_internal_anchor":false},{"citing_arxiv_id":"2605.27767","citing_title":"UniMaia: Steering Chess Policies with Language for Human-like Play","ref_index":2,"is_internal_anchor":false},{"citing_arxiv_id":"2301.13688","citing_title":"The Flan Collection: Designing Data and Methods for Effective Instruction Tuning","ref_index":57,"is_internal_anchor":false},{"citing_arxiv_id":"2502.01941","citing_title":"Semantic Integrity Matters: Benchmarking and Preserving High-Density Reasoning in KV Cache Compression","ref_index":4,"is_internal_anchor":false},{"citing_arxiv_id":"2305.07922","citing_title":"CodeT5+: Open Code Large Language Models for Code Understanding and Generation","ref_index":28,"is_internal_anchor":false},{"citing_arxiv_id":"2305.16264","citing_title":"Scaling Data-Constrained Language Models","ref_index":114,"is_internal_anchor":false},{"citing_arxiv_id":"2207.14255","citing_title":"Efficient Training of Language Models to Fill in the Middle","ref_index":140,"is_internal_anchor":false},{"citing_arxiv_id":"2207.14255","citing_title":"Efficient Training of Language Models to Fill in the Middle","ref_index":93,"is_internal_anchor":false},{"citing_arxiv_id":"2404.19737","citing_title":"Better & Faster Large Language Models via Multi-token Prediction","ref_index":14,"is_internal_anchor":false},{"citing_arxiv_id":"2509.20354","citing_title":"EmbeddingGemma: Powerful and Lightweight Text Representations","ref_index":23,"is_internal_anchor":false},{"citing_arxiv_id":"2303.17564","citing_title":"BloombergGPT: A Large Language Model for Finance","ref_index":116,"is_internal_anchor":false},{"citing_arxiv_id":"2605.03244","citing_title":"S^2tory: Story Spine Distillation for Movie Script Summarization","ref_index":22,"is_internal_anchor":false},{"citing_arxiv_id":"2605.08094","citing_title":"MedThink: Enhancing Diagnostic Accuracy in Small Models via Teacher-Guided Reasoning Correction","ref_index":14,"is_internal_anchor":false},{"citing_arxiv_id":"2402.06196","citing_title":"Large Language Models: A Survey","ref_index":92,"is_internal_anchor":false},{"citing_arxiv_id":"2410.13720","citing_title":"Movie Gen: A Cast of Media Foundation Models","ref_index":65,"is_internal_anchor":false},{"citing_arxiv_id":"2412.20404","citing_title":"Open-Sora: Democratizing Efficient Video Production for All","ref_index":30,"is_internal_anchor":false},{"citing_arxiv_id":"2206.07682","citing_title":"Emergent Abilities of Large Language Models","ref_index":82,"is_internal_anchor":false},{"citing_arxiv_id":"2305.06161","citing_title":"StarCoder: may the source be with you!","ref_index":214,"is_internal_anchor":false},{"citing_arxiv_id":"2201.11903","citing_title":"Chain-of-Thought Prompting Elicits Reasoning in Large Language Models","ref_index":67,"is_internal_anchor":false},{"citing_arxiv_id":"2605.04291","citing_title":"Leveraging Pretrained Language Models as Energy Functions for Glauber Dynamics Text Diffusion","ref_index":120,"is_internal_anchor":false}]},"formal_canon":{"evidence_count":0,"sample":[],"anchors":[]},"links":{"html":"https://pith.science/pith/WHDFRWDA4MMY345DDJ7NHD5LHS","json":"https://pith.science/pith/WHDFRWDA4MMY345DDJ7NHD5LHS.json","graph_json":"https://pith.science/api/pith-number/WHDFRWDA4MMY345DDJ7NHD5LHS/graph.json","events_json":"https://pith.science/api/pith-number/WHDFRWDA4MMY345DDJ7NHD5LHS/events.json","paper":"https://pith.science/paper/WHDFRWDA"},"agent_actions":{"view_html":"https://pith.science/pith/WHDFRWDA4MMY345DDJ7NHD5LHS","download_json":"https://pith.science/pith/WHDFRWDA4MMY345DDJ7NHD5LHS.json","view_paper":"https://pith.science/paper/WHDFRWDA","resolve_alias":"https://pith.science/api/pith-number/resolve?arxiv=2205.05131&json=true","fetch_graph":"https://pith.science/api/pith-number/WHDFRWDA4MMY345DDJ7NHD5LHS/graph.json","fetch_events":"https://pith.science/api/pith-number/WHDFRWDA4MMY345DDJ7NHD5LHS/events.json","actions":{"anchor_timestamp":"https://pith.science/pith/WHDFRWDA4MMY345DDJ7NHD5LHS/action/timestamp_anchor","attest_storage":"https://pith.science/pith/WHDFRWDA4MMY345DDJ7NHD5LHS/action/storage_attestation","attest_author":"https://pith.science/pith/WHDFRWDA4MMY345DDJ7NHD5LHS/action/author_attestation","sign_citation":"https://pith.science/pith/WHDFRWDA4MMY345DDJ7NHD5LHS/action/citation_signature","submit_replication":"https://pith.science/pith/WHDFRWDA4MMY345DDJ7NHD5LHS/action/replication_record"}},"created_at":"2026-07-05T05:46:15.508176+00:00","updated_at":"2026-07-05T05:46:15.508176+00:00"}