{"bundle_type":"pith_open_graph_bundle","bundle_version":"1.0","pith_number":"pith:2022:APCCNCY42HHIJXSKVRWKP2A7K3","short_pith_number":"pith:APCCNCY4","canonical_record":{"source":{"id":"2212.12017","kind":"arxiv","version":3},"metadata":{"license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.CL","submitted_at":"2022-12-22T19:56:09Z","cross_cats_sorted":[],"title_canon_sha256":"50f144c76cd8960a3a65c93eec8753ba5b9444987ea030fc5c004d82595e8a06","abstract_canon_sha256":"025164ad055811875b0f90849498f9de497302ffe14bb73802b0234deb1bbe90"},"schema_version":"1.0"},"canonical_sha256":"03c4268b1cd1ce84de4aac6ca7e81f56d23f69fada537f6a0c40966b36b8a18f","source":{"kind":"arxiv","id":"2212.12017","version":3},"source_aliases":[{"alias_kind":"arxiv","alias_value":"2212.12017","created_at":"2026-05-17T23:38:14Z"},{"alias_kind":"arxiv_version","alias_value":"2212.12017v3","created_at":"2026-05-17T23:38:14Z"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.2212.12017","created_at":"2026-05-17T23:38:14Z"},{"alias_kind":"pith_short_12","alias_value":"APCCNCY42HHI","created_at":"2026-05-18T12:33:33Z"},{"alias_kind":"pith_short_16","alias_value":"APCCNCY42HHIJXSK","created_at":"2026-05-18T12:33:33Z"},{"alias_kind":"pith_short_8","alias_value":"APCCNCY4","created_at":"2026-05-18T12:33:33Z"}],"events":[{"event_type":"record_created","subject_pith_number":"pith:2022:APCCNCY42HHIJXSKVRWKP2A7K3","target":"record","payload":{"canonical_record":{"source":{"id":"2212.12017","kind":"arxiv","version":3},"metadata":{"license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.CL","submitted_at":"2022-12-22T19:56:09Z","cross_cats_sorted":[],"title_canon_sha256":"50f144c76cd8960a3a65c93eec8753ba5b9444987ea030fc5c004d82595e8a06","abstract_canon_sha256":"025164ad055811875b0f90849498f9de497302ffe14bb73802b0234deb1bbe90"},"schema_version":"1.0"},"canonical_sha256":"03c4268b1cd1ce84de4aac6ca7e81f56d23f69fada537f6a0c40966b36b8a18f","receipt":{"kind":"pith_receipt","key_id":"pith-v1-2026-05","algorithm":"ed25519","signed_at":"2026-05-17T23:38:14.863845Z","signature_b64":"q88eL8fWs7g/vjoDqCzAFfMcEbFOCPpS+Q7/i6LNPXSAS2YWSI1JTrkbLv3D/w8XgEKc9cyzWYdDzqB2sASUDw==","signed_message":"canonical_sha256_bytes","builder_version":"pith-number-builder-2026-05-17-v1","receipt_version":"0.3","canonical_sha256":"03c4268b1cd1ce84de4aac6ca7e81f56d23f69fada537f6a0c40966b36b8a18f","last_reissued_at":"2026-05-17T23:38:14.863298Z","signature_status":"signed_v1","first_computed_at":"2026-05-17T23:38:14.863298Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"source_kind":"arxiv","source_id":"2212.12017","source_version":3,"attestation_state":"computed"},"signer":{"signer_id":"pith.science","signer_type":"pith_registry","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"created_at":"2026-05-17T23:38:14Z","supersedes":[],"prev_event":null,"signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"diLkCuK/9Q4sgMBBdsIpz3eYwxLGrIKi1MEDuKreMbfYvCJY+fFwVh15X31Da0Tiv7inBDlmo0rse/u1h5f8Dg==","signed_message":"open_graph_event_sha256_bytes","signed_at":"2026-05-24T07:00:00.864130Z"},"content_sha256":"9d8370de3605a8def4ba67e3332f11c70352a16c706d455b0c0963125cf19b71","schema_version":"1.0","event_id":"sha256:9d8370de3605a8def4ba67e3332f11c70352a16c706d455b0c0963125cf19b71"},{"event_type":"graph_snapshot","subject_pith_number":"pith:2022:APCCNCY42HHIJXSKVRWKP2A7K3","target":"graph","payload":{"graph_snapshot":{"paper":{"title":"OPT-IML: Scaling Language Model Instruction Meta Learning through the Lens of Generalization","license":"http://creativecommons.org/licenses/by/4.0/","headline":"Instruction-tuning on a 2000-task benchmark produces models that generalize to held-out categories, tasks, and instances.","cross_cats":[],"primary_cat":"cs.CL","authors_text":"Asli Celikyilmaz, Brian O'Horo, Christopher Dewan, Daniel Simig, Gabriel Pereyra, Jeff Wang, Kurt Shuster, Luke Zettlemoyer, Ping Yu, Punit Singh Koura, Qing Liu, Ramakanth Pasunuru, Srinivasan Iyer, Tianlu Wang, Todor Mihaylov, Ves Stoyanov, Xian Li, Xi Victoria Lin","submitted_at":"2022-12-22T19:56:09Z","abstract_excerpt":"Recent work has shown that fine-tuning large pre-trained language models on a collection of tasks described via instructions, a.k.a. instruction-tuning, improves their zero and few-shot generalization to unseen tasks. However, there is a limited understanding of the performance trade-offs of different decisions made during the instruction-tuning process. These decisions include the scale and diversity of the instruction-tuning benchmark, different task sampling strategies, fine-tuning with and without demonstrations, training using specialized datasets for reasoning and dialogue, and finally, "},"claims":{"count":4,"items":[{"kind":"strongest_claim","text":"OPT-IML demonstrates all three generalization abilities at both scales on four different evaluation benchmarks with diverse tasks and input formats -- PromptSource, FLAN, Super-NaturalInstructions, and UnifiedSKG.","source":"verdict.strongest_claim","status":"machine_extracted","claim_id":"C1","attestation":"unclaimed"},{"kind":"weakest_assumption","text":"That the consolidation of tasks from eight existing benchmarks into 2000 tasks and the defined held-out category/task/instance splits provide a representative and unbiased measure of generalization to truly unseen NLP problems.","source":"verdict.weakest_assumption","status":"machine_extracted","claim_id":"C2","attestation":"unclaimed"},{"kind":"one_line_summary","text":"OPT-IML 30B and 175B models, trained on a new 2000-task instruction benchmark, demonstrate generalization to held-out categories, tasks, and instances while outperforming base OPT and competing with benchmark-specific models.","source":"verdict.one_line_summary","status":"machine_extracted","claim_id":"C3","attestation":"unclaimed"},{"kind":"headline","text":"Instruction-tuning on a 2000-task benchmark produces models that generalize to held-out categories, tasks, and instances.","source":"verdict.pith_extraction.headline","status":"machine_extracted","claim_id":"C4","attestation":"unclaimed"}],"snapshot_sha256":"2695d9dc9fc29a142a4ce0c2d84f1b74dbad1ff924d64ab2ee33d1847edfb483"},"source":{"id":"2212.12017","kind":"arxiv","version":3},"verdict":{"id":"09c7bd3e-91cb-47fd-abf3-052f2bcd2891","model_set":{"reader":"grok-4.3"},"created_at":"2026-05-17T06:02:41.527056Z","strongest_claim":"OPT-IML demonstrates all three generalization abilities at both scales on four different evaluation benchmarks with diverse tasks and input formats -- PromptSource, FLAN, Super-NaturalInstructions, and UnifiedSKG.","one_line_summary":"OPT-IML 30B and 175B models, trained on a new 2000-task instruction benchmark, demonstrate generalization to held-out categories, tasks, and instances while outperforming base OPT and competing with benchmark-specific models.","pipeline_version":"pith-pipeline@v0.9.0","weakest_assumption":"That the consolidation of tasks from eight existing benchmarks into 2000 tasks and the defined held-out category/task/instance splits provide a representative and unbiased measure of generalization to truly unseen NLP problems.","pith_extraction_headline":"Instruction-tuning on a 2000-task benchmark produces models that generalize to held-out categories, tasks, and instances."},"references":{"count":0,"sample":[],"resolved_work":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","internal_anchors":0},"formal_canon":{"evidence_count":2,"snapshot_sha256":"bb8120a1cd8778e55f62cce5ef6748e50bb11ab03f5b17f1a6932fcb64af250c"},"author_claims":{"count":0,"strong_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"builder_version":"pith-number-builder-2026-05-17-v1"},"verdict_id":"09c7bd3e-91cb-47fd-abf3-052f2bcd2891"},"signer":{"signer_id":"pith.science","signer_type":"pith_registry","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"created_at":"2026-05-17T23:38:14Z","supersedes":[],"prev_event":null,"signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"SUbZIF/Ld093cOl9x1gSWSx9hFA9PlnKBOjg5cs1OJreYi6LMexJcyEH0sWmY6U9NoddEs2sVZ7LYDuNXVyMBQ==","signed_message":"open_graph_event_sha256_bytes","signed_at":"2026-05-24T07:00:00.865026Z"},"content_sha256":"af2563b2b1fb760e1794294650ef31a360076573012e7e773fb8e67b6f9bdfa1","schema_version":"1.0","event_id":"sha256:af2563b2b1fb760e1794294650ef31a360076573012e7e773fb8e67b6f9bdfa1"}],"timestamp_proofs":[],"mirror_hints":[{"mirror_type":"https","name":"Pith Resolver","base_url":"https://pith.science","bundle_url":"https://pith.science/pith/APCCNCY42HHIJXSKVRWKP2A7K3/bundle.json","state_url":"https://pith.science/pith/APCCNCY42HHIJXSKVRWKP2A7K3/state.json","well_known_bundle_url":"https://pith.science/.well-known/pith/APCCNCY42HHIJXSKVRWKP2A7K3/bundle.json","status":"primary"}],"public_keys":[{"key_id":"pith-v1-2026-05","algorithm":"ed25519","format":"raw","public_key_b64":"stVStoiQhXFxp4s2pdzPNoqVNBMojDU/fJ2db5S3CbM=","public_key_hex":"b2d552b68890857171a78b36a5dccf368a953413288c353f7c9d9d6f94b709b3","fingerprint_sha256_b32_first128bits":"RVFV5Z2OI2J3ZUO7ERDEBCYNKS","fingerprint_sha256_hex":"8d4b5ee74e4693bcd1df2446408b0d54","rotates_at":null,"url":"https://pith.science/pith-signing-key.json","notes":"Pith uses this Ed25519 key to sign canonical record SHA-256 digests. Verify with: ed25519_verify(public_key, message=canonical_sha256_bytes, signature=base64decode(signature_b64))."}],"merge_version":"pith-open-graph-merge-v1","built_at":"2026-05-24T07:00:00Z","links":{"resolver":"https://pith.science/pith/APCCNCY42HHIJXSKVRWKP2A7K3","bundle":"https://pith.science/pith/APCCNCY42HHIJXSKVRWKP2A7K3/bundle.json","state":"https://pith.science/pith/APCCNCY42HHIJXSKVRWKP2A7K3/state.json","well_known_bundle":"https://pith.science/.well-known/pith/APCCNCY42HHIJXSKVRWKP2A7K3/bundle.json"},"state":{"state_type":"pith_open_graph_state","state_version":"1.0","pith_number":"pith:2022:APCCNCY42HHIJXSKVRWKP2A7K3","merge_version":"pith-open-graph-merge-v1","event_count":2,"valid_event_count":2,"invalid_event_count":0,"equivocation_count":0,"current":{"canonical_record":{"metadata":{"abstract_canon_sha256":"025164ad055811875b0f90849498f9de497302ffe14bb73802b0234deb1bbe90","cross_cats_sorted":[],"license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.CL","submitted_at":"2022-12-22T19:56:09Z","title_canon_sha256":"50f144c76cd8960a3a65c93eec8753ba5b9444987ea030fc5c004d82595e8a06"},"schema_version":"1.0","source":{"id":"2212.12017","kind":"arxiv","version":3}},"source_aliases":[{"alias_kind":"arxiv","alias_value":"2212.12017","created_at":"2026-05-17T23:38:14Z"},{"alias_kind":"arxiv_version","alias_value":"2212.12017v3","created_at":"2026-05-17T23:38:14Z"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.2212.12017","created_at":"2026-05-17T23:38:14Z"},{"alias_kind":"pith_short_12","alias_value":"APCCNCY42HHI","created_at":"2026-05-18T12:33:33Z"},{"alias_kind":"pith_short_16","alias_value":"APCCNCY42HHIJXSK","created_at":"2026-05-18T12:33:33Z"},{"alias_kind":"pith_short_8","alias_value":"APCCNCY4","created_at":"2026-05-18T12:33:33Z"}],"graph_snapshots":[{"event_id":"sha256:af2563b2b1fb760e1794294650ef31a360076573012e7e773fb8e67b6f9bdfa1","target":"graph","created_at":"2026-05-17T23:38:14Z","signer":{"key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signer_id":"pith.science","signer_type":"pith_registry"},"payload":{"graph_snapshot":{"author_claims":{"count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","strong_count":0},"builder_version":"pith-number-builder-2026-05-17-v1","claims":{"count":4,"items":[{"attestation":"unclaimed","claim_id":"C1","kind":"strongest_claim","source":"verdict.strongest_claim","status":"machine_extracted","text":"OPT-IML demonstrates all three generalization abilities at both scales on four different evaluation benchmarks with diverse tasks and input formats -- PromptSource, FLAN, Super-NaturalInstructions, and UnifiedSKG."},{"attestation":"unclaimed","claim_id":"C2","kind":"weakest_assumption","source":"verdict.weakest_assumption","status":"machine_extracted","text":"That the consolidation of tasks from eight existing benchmarks into 2000 tasks and the defined held-out category/task/instance splits provide a representative and unbiased measure of generalization to truly unseen NLP problems."},{"attestation":"unclaimed","claim_id":"C3","kind":"one_line_summary","source":"verdict.one_line_summary","status":"machine_extracted","text":"OPT-IML 30B and 175B models, trained on a new 2000-task instruction benchmark, demonstrate generalization to held-out categories, tasks, and instances while outperforming base OPT and competing with benchmark-specific models."},{"attestation":"unclaimed","claim_id":"C4","kind":"headline","source":"verdict.pith_extraction.headline","status":"machine_extracted","text":"Instruction-tuning on a 2000-task benchmark produces models that generalize to held-out categories, tasks, and instances."}],"snapshot_sha256":"2695d9dc9fc29a142a4ce0c2d84f1b74dbad1ff924d64ab2ee33d1847edfb483"},"formal_canon":{"evidence_count":2,"snapshot_sha256":"bb8120a1cd8778e55f62cce5ef6748e50bb11ab03f5b17f1a6932fcb64af250c"},"paper":{"abstract_excerpt":"Recent work has shown that fine-tuning large pre-trained language models on a collection of tasks described via instructions, a.k.a. instruction-tuning, improves their zero and few-shot generalization to unseen tasks. However, there is a limited understanding of the performance trade-offs of different decisions made during the instruction-tuning process. These decisions include the scale and diversity of the instruction-tuning benchmark, different task sampling strategies, fine-tuning with and without demonstrations, training using specialized datasets for reasoning and dialogue, and finally, ","authors_text":"Asli Celikyilmaz, Brian O'Horo, Christopher Dewan, Daniel Simig, Gabriel Pereyra, Jeff Wang, Kurt Shuster, Luke Zettlemoyer, Ping Yu, Punit Singh Koura, Qing Liu, Ramakanth Pasunuru, Srinivasan Iyer, Tianlu Wang, Todor Mihaylov, Ves Stoyanov, Xian Li, Xi Victoria Lin","cross_cats":[],"headline":"Instruction-tuning on a 2000-task benchmark produces models that generalize to held-out categories, tasks, and instances.","license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.CL","submitted_at":"2022-12-22T19:56:09Z","title":"OPT-IML: Scaling Language Model Instruction Meta Learning through the Lens of Generalization"},"references":{"count":0,"internal_anchors":0,"resolved_work":0,"sample":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"2212.12017","kind":"arxiv","version":3},"verdict":{"created_at":"2026-05-17T06:02:41.527056Z","id":"09c7bd3e-91cb-47fd-abf3-052f2bcd2891","model_set":{"reader":"grok-4.3"},"one_line_summary":"OPT-IML 30B and 175B models, trained on a new 2000-task instruction benchmark, demonstrate generalization to held-out categories, tasks, and instances while outperforming base OPT and competing with benchmark-specific models.","pipeline_version":"pith-pipeline@v0.9.0","pith_extraction_headline":"Instruction-tuning on a 2000-task benchmark produces models that generalize to held-out categories, tasks, and instances.","strongest_claim":"OPT-IML demonstrates all three generalization abilities at both scales on four different evaluation benchmarks with diverse tasks and input formats -- PromptSource, FLAN, Super-NaturalInstructions, and UnifiedSKG.","weakest_assumption":"That the consolidation of tasks from eight existing benchmarks into 2000 tasks and the defined held-out category/task/instance splits provide a representative and unbiased measure of generalization to truly unseen NLP problems."}},"verdict_id":"09c7bd3e-91cb-47fd-abf3-052f2bcd2891"}}],"author_attestations":[],"timestamp_anchors":[],"storage_attestations":[],"citation_signatures":[],"replication_records":[],"corrections":[],"mirror_hints":[],"record_created":{"event_id":"sha256:9d8370de3605a8def4ba67e3332f11c70352a16c706d455b0c0963125cf19b71","target":"record","created_at":"2026-05-17T23:38:14Z","signer":{"key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signer_id":"pith.science","signer_type":"pith_registry"},"payload":{"attestation_state":"computed","canonical_record":{"metadata":{"abstract_canon_sha256":"025164ad055811875b0f90849498f9de497302ffe14bb73802b0234deb1bbe90","cross_cats_sorted":[],"license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.CL","submitted_at":"2022-12-22T19:56:09Z","title_canon_sha256":"50f144c76cd8960a3a65c93eec8753ba5b9444987ea030fc5c004d82595e8a06"},"schema_version":"1.0","source":{"id":"2212.12017","kind":"arxiv","version":3}},"canonical_sha256":"03c4268b1cd1ce84de4aac6ca7e81f56d23f69fada537f6a0c40966b36b8a18f","receipt":{"algorithm":"ed25519","builder_version":"pith-number-builder-2026-05-17-v1","canonical_sha256":"03c4268b1cd1ce84de4aac6ca7e81f56d23f69fada537f6a0c40966b36b8a18f","first_computed_at":"2026-05-17T23:38:14.863298Z","key_id":"pith-v1-2026-05","kind":"pith_receipt","last_reissued_at":"2026-05-17T23:38:14.863298Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","receipt_version":"0.3","signature_b64":"q88eL8fWs7g/vjoDqCzAFfMcEbFOCPpS+Q7/i6LNPXSAS2YWSI1JTrkbLv3D/w8XgEKc9cyzWYdDzqB2sASUDw==","signature_status":"signed_v1","signed_at":"2026-05-17T23:38:14.863845Z","signed_message":"canonical_sha256_bytes"},"source_id":"2212.12017","source_kind":"arxiv","source_version":3}}},"equivocations":[],"invalid_events":[],"applied_event_ids":["sha256:9d8370de3605a8def4ba67e3332f11c70352a16c706d455b0c0963125cf19b71","sha256:af2563b2b1fb760e1794294650ef31a360076573012e7e773fb8e67b6f9bdfa1"],"state_sha256":"d8d2e0a2ad38b466b1d3cb5abf18b6c7e5d67a680fc15986eef357051f8a0537"},"bundle_signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"Emk3p4K8tQkMXBN1UqIPchtiOxaqbhF9DPZMOPxo3HZumnsR7Bo9eyx7iiIX+sNHy9iVpcPXjzZNn3oStvP6BQ==","signed_message":"bundle_sha256_bytes","signed_at":"2026-05-24T07:00:00.868434Z","bundle_sha256":"4d291b9ccf9314dd7f43ceb4273a07bd0ff203f12c29fbeb6f8a0957f5e5d313"}}