{"record_type":"pith_number_record","schema_url":"https://pith.science/schemas/pith-number/v1.json","pith_number":"pith:2022:TRRS5IT2GHTQ4SJS3552DBVHYS","short_pith_number":"pith:TRRS5IT2","schema_version":"1.0","canonical_sha256":"9c632ea27a31e70e4932df7ba186a7c49aec1ad1af2f8b945a131ecc28216e5f","source":{"kind":"arxiv","id":"2210.09261","version":1},"attestation_state":"computed","paper":{"title":"Challenging BIG-Bench Tasks and Whether Chain-of-Thought Can Solve Them","license":"http://creativecommons.org/licenses/by-nc-sa/4.0/","headline":"Chain-of-thought prompting lets current models surpass average humans on 17 of 23 BIG-Bench Hard tasks.","cross_cats":["cs.AI"],"primary_cat":"cs.CL","authors_text":"Aakanksha Chowdhery, Denny Zhou, Ed H. Chi, Hyung Won Chung, Jason Wei, Mirac Suzgun, Nathanael Sch\\\"arli, Nathan Scales, Quoc V. Le, Sebastian Gehrmann, Yi Tay","submitted_at":"2022-10-17T17:08:26Z","abstract_excerpt":"BIG-Bench (Srivastava et al., 2022) is a diverse evaluation suite that focuses on tasks believed to be beyond the capabilities of current language models. Language models have already made good progress on this benchmark, with the best model in the BIG-Bench paper outperforming average reported human-rater results on 65% of the BIG-Bench tasks via few-shot prompting. But on what tasks do language models fall short of average human-rater performance, and are those tasks actually unsolvable by current language models?\n  In this work, we focus on a suite of 23 challenging BIG-Bench tasks which we"},"verification_status":{"content_addressed":true,"pith_receipt":true,"author_attested":false,"weak_author_claims":0,"strong_author_claims":0,"externally_anchored":false,"storage_verified":false,"citation_signatures":0,"replication_records":0,"graph_snapshot":true,"references_resolved":true,"formal_links_present":false},"canonical_record":{"source":{"id":"2210.09261","kind":"arxiv","version":1},"metadata":{"license":"http://creativecommons.org/licenses/by-nc-sa/4.0/","primary_cat":"cs.CL","submitted_at":"2022-10-17T17:08:26Z","cross_cats_sorted":["cs.AI"],"title_canon_sha256":"18ef4a27704c509abd2f75c40a443aeda786c987da847eea56365bf08165d9b6","abstract_canon_sha256":"19b2f797d21adaeb164d1511924c3c019f94f58cbabe0e47ceb3b95b35741065"},"schema_version":"1.0"},"receipt":{"kind":"pith_receipt","key_id":"pith-v1-2026-05","algorithm":"ed25519","signed_at":"2026-07-05T05:07:20.598098Z","signature_b64":"Ondktz8SyShGc3ddQnR4nrvl7WOZTrNUHePgQwEJqxYu/2rSZ2KQP/ss5K93sBNbcj5vLsyCEY7Z5ak0ebUVDQ==","signed_message":"canonical_sha256_bytes","builder_version":"pith-number-builder-2026-05-17-v1","receipt_version":"0.3","canonical_sha256":"9c632ea27a31e70e4932df7ba186a7c49aec1ad1af2f8b945a131ecc28216e5f","last_reissued_at":"2026-07-05T05:07:20.597704Z","signature_status":"signed_v1","first_computed_at":"2026-07-05T05:07:20.597704Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"graph_snapshot":{"paper":{"title":"Challenging BIG-Bench Tasks and Whether Chain-of-Thought Can Solve Them","license":"http://creativecommons.org/licenses/by-nc-sa/4.0/","headline":"Chain-of-thought prompting lets current models surpass average humans on 17 of 23 BIG-Bench Hard tasks.","cross_cats":["cs.AI"],"primary_cat":"cs.CL","authors_text":"Aakanksha Chowdhery, Denny Zhou, Ed H. Chi, Hyung Won Chung, Jason Wei, Mirac Suzgun, Nathanael Sch\\\"arli, Nathan Scales, Quoc V. Le, Sebastian Gehrmann, Yi Tay","submitted_at":"2022-10-17T17:08:26Z","abstract_excerpt":"BIG-Bench (Srivastava et al., 2022) is a diverse evaluation suite that focuses on tasks believed to be beyond the capabilities of current language models. Language models have already made good progress on this benchmark, with the best model in the BIG-Bench paper outperforming average reported human-rater results on 65% of the BIG-Bench tasks via few-shot prompting. But on what tasks do language models fall short of average human-rater performance, and are those tasks actually unsolvable by current language models?\n  In this work, we focus on a suite of 23 challenging BIG-Bench tasks which we"},"claims":{"count":4,"items":[{"kind":"strongest_claim","text":"Applying chain-of-thought (CoT) prompting to BBH tasks enables PaLM to surpass the average human-rater performance on 10 of the 23 tasks, and Codex (code-davinci-002) to surpass the average human-rater performance on 17 of the 23 tasks.","source":"verdict.strongest_claim","status":"machine_extracted","claim_id":"C1","attestation":"unclaimed"},{"kind":"weakest_assumption","text":"That the average human-rater scores reported in the original BIG-Bench paper constitute a stable and fair external benchmark against which model performance can be directly compared without additional controls for prompt sensitivity or rater variability.","source":"verdict.weakest_assumption","status":"machine_extracted","claim_id":"C2","attestation":"unclaimed"},{"kind":"one_line_summary","text":"Chain-of-thought prompting enables large language models to surpass average human performance on 17 of 23 challenging BIG-Bench tasks.","source":"verdict.one_line_summary","status":"machine_extracted","claim_id":"C3","attestation":"unclaimed"},{"kind":"headline","text":"Chain-of-thought prompting lets current models surpass average humans on 17 of 23 BIG-Bench Hard tasks.","source":"verdict.pith_extraction.headline","status":"machine_extracted","claim_id":"C4","attestation":"unclaimed"}],"snapshot_sha256":"684a57a19a3b830704835fb530a540e96a944023443976be45ef871a63a1382f"},"source":{"id":"2210.09261","kind":"arxiv","version":1},"verdict":{"id":"47ea85c3-c3c3-4a28-8442-c48a1e428f60","model_set":{"reader":"grok-4.3"},"created_at":"2026-05-11T07:09:17.108348Z","strongest_claim":"Applying chain-of-thought (CoT) prompting to BBH tasks enables PaLM to surpass the average human-rater performance on 10 of the 23 tasks, and Codex (code-davinci-002) to surpass the average human-rater performance on 17 of the 23 tasks.","one_line_summary":"Chain-of-thought prompting enables large language models to surpass average human performance on 17 of 23 challenging BIG-Bench tasks.","pipeline_version":"pith-pipeline@v0.9.0","weakest_assumption":"That the average human-rater scores reported in the original BIG-Bench paper constitute a stable and fair external benchmark against which model performance can be directly compared without additional controls for prompt sensitivity or rater variability.","pith_extraction_headline":"Chain-of-thought prompting lets current models surpass average humans on 17 of 23 BIG-Bench Hard tasks."},"integrity":{"clean":true,"summary":{"advisory":0,"critical":0,"by_detector":{},"informational":0},"endpoint":"/pith/2210.09261/integrity.json","findings":[],"available":true,"detectors_run":[],"snapshot_sha256":"c28c3603d3b5d939e8dc4c7e95fa8dfce3d595e45f758748cecf8e644a296938"},"references":{"count":42,"sample":[{"doi":"","year":null,"title":"Program Synthesis with Large Language Models","work_id":"fd241a05-03b9-4de2-9588-9d77ce176125","ref_index":1,"cited_arxiv_id":"2108.07732","is_internal_anchor":true},{"doi":"","year":1901,"title":"Language models are few-shot learners","work_id":"f93ff324-f230-4a46-97b9-6b103c35585d","ref_index":2,"cited_arxiv_id":"","is_internal_anchor":false},{"doi":"","year":null,"title":"Evaluating Large Language Models Trained on Code","work_id":"042493e9-b26f-4b4e-bbde-382072ca9b08","ref_index":3,"cited_arxiv_id":"2107.03374","is_internal_anchor":true},{"doi":"","year":null,"title":"Smith, and Tao Yu","work_id":"f8fe6642-904b-44b0-a533-ae63c3a213e1","ref_index":4,"cited_arxiv_id":"","is_internal_anchor":false},{"doi":"","year":null,"title":"PaLM: Scaling Language Modeling with Pathways","work_id":"a94f3ef7-2c49-4445-93fe-6ec16aafd966","ref_index":5,"cited_arxiv_id":"2204.02311","is_internal_anchor":true}],"resolved_work":42,"snapshot_sha256":"0cc76cb2921f126c5edafe21e4352b5f3ab46500adf613191b9e05c2f6b26ea9","internal_anchors":15},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"author_claims":{"count":0,"strong_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"builder_version":"pith-number-builder-2026-05-17-v1"},"aliases":[{"alias_kind":"arxiv","alias_value":"2210.09261","created_at":"2026-07-05T05:07:20.597765+00:00"},{"alias_kind":"arxiv_version","alias_value":"2210.09261v1","created_at":"2026-07-05T05:07:20.597765+00:00"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.2210.09261","created_at":"2026-07-05T05:07:20.597765+00:00"},{"alias_kind":"pith_short_12","alias_value":"TRRS5IT2GHTQ","created_at":"2026-07-05T05:07:20.597765+00:00"},{"alias_kind":"pith_short_16","alias_value":"TRRS5IT2GHTQ4SJS","created_at":"2026-07-05T05:07:20.597765+00:00"},{"alias_kind":"pith_short_8","alias_value":"TRRS5IT2","created_at":"2026-07-05T05:07:20.597765+00:00"}],"events":[],"event_summary":{},"paper_claims":[],"inbound_citations":{"count":120,"internal_anchor_count":120,"sample":[{"citing_arxiv_id":"2607.05196","citing_title":"Unified Audio Intelligence Without Regressing on Text Intelligence","ref_index":283,"is_internal_anchor":true},{"citing_arxiv_id":"2606.25331","citing_title":"Improved Large Language Diffusion Models","ref_index":39,"is_internal_anchor":true},{"citing_arxiv_id":"2606.26587","citing_title":"SharQ: Bridging Activation Sparsity and FP4 Quantization for LLM Inference","ref_index":93,"is_internal_anchor":true},{"citing_arxiv_id":"2606.18431","citing_title":"Beyond Prediction: Tail-Aware Scheduling for LLM Inference","ref_index":46,"is_internal_anchor":true},{"citing_arxiv_id":"2606.18307","citing_title":"DRIFT: Refining Instruction Data via On-Policy Data Attribution","ref_index":20,"is_internal_anchor":true},{"citing_arxiv_id":"2607.02464","citing_title":"Will Scaling Improve Social Simulation with LLMs?","ref_index":36,"is_internal_anchor":true},{"citing_arxiv_id":"2607.01740","citing_title":"Meta-Benchmarks for Financial-Services LLM Evaluation","ref_index":23,"is_internal_anchor":true},{"citing_arxiv_id":"2606.12397","citing_title":"Redesign Mixture-of-Experts Routers with Manifold Power Iteration","ref_index":14,"is_internal_anchor":true},{"citing_arxiv_id":"2606.09186","citing_title":"DuplexOmni: Real-Time Listening, Seeing, Thinking, and Speaking for Full-Duplex Interaction","ref_index":18,"is_internal_anchor":true},{"citing_arxiv_id":"2606.28615","citing_title":"What LLMs explain is not what they believe: Evaluating explanation sufficiency under models' own input beliefs","ref_index":145,"is_internal_anchor":true},{"citing_arxiv_id":"2605.00419","citing_title":"Rethinking LLM Ensembling from the Perspective of Mixture Models","ref_index":13,"is_internal_anchor":true},{"citing_arxiv_id":"2605.07053","citing_title":"GSM-SEM: Benchmark and Framework for Generating Semantically Variant Augmentations","ref_index":30,"is_internal_anchor":true},{"citing_arxiv_id":"2605.08704","citing_title":"AgentPSO: Evolving Agent Reasoning Skill via Multi-agent Particle Swarm Optimization","ref_index":37,"is_internal_anchor":true},{"citing_arxiv_id":"2606.30383","citing_title":"Whose Side Is Your Agent On? Multi-Party Principal Loyalty in LLM Agents","ref_index":36,"is_internal_anchor":true},{"citing_arxiv_id":"2605.24981","citing_title":"Large Language Model Selection with Limited Annotations","ref_index":104,"is_internal_anchor":true},{"citing_arxiv_id":"2605.26184","citing_title":"GAC: Noise-Aware Adaptive Mixing for Hybrid SFT-RL Post-Training","ref_index":22,"is_internal_anchor":true},{"citing_arxiv_id":"2605.25558","citing_title":"Beyond Query Memorization: Large Language Model Routing with Query Decomposition and Historical Matching","ref_index":15,"is_internal_anchor":true},{"citing_arxiv_id":"2605.26496","citing_title":"Dense2MoE: Pushing the Pareto Frontier of On-Device LLMs via Unified Pruning and Upcycling","ref_index":14,"is_internal_anchor":true},{"citing_arxiv_id":"2606.27632","citing_title":"Yuvion LLM: An Adversarially-Aware Large Language Model for Content And AI Safety","ref_index":31,"is_internal_anchor":true},{"citing_arxiv_id":"2605.31268","citing_title":"Mellum2 Technical Report","ref_index":70,"is_internal_anchor":true},{"citing_arxiv_id":"2606.05976","citing_title":"The Self-Correction Illusion: LLMs Correct Others but Not Themselves","ref_index":35,"is_internal_anchor":true},{"citing_arxiv_id":"2211.01910","citing_title":"Large Language Models Are Human-Level Prompt Engineers","ref_index":33,"is_internal_anchor":true},{"citing_arxiv_id":"2301.13688","citing_title":"The Flan Collection: Designing Data and Methods for Effective Instruction Tuning","ref_index":55,"is_internal_anchor":true},{"citing_arxiv_id":"2309.10305","citing_title":"Baichuan 2: Open Large-scale Language Models","ref_index":65,"is_internal_anchor":true},{"citing_arxiv_id":"2310.06825","citing_title":"Mistral 7B","ref_index":23,"is_internal_anchor":true}]},"formal_canon":{"evidence_count":0,"sample":[],"anchors":[]},"links":{"html":"https://pith.science/pith/TRRS5IT2GHTQ4SJS3552DBVHYS","json":"https://pith.science/pith/TRRS5IT2GHTQ4SJS3552DBVHYS.json","graph_json":"https://pith.science/api/pith-number/TRRS5IT2GHTQ4SJS3552DBVHYS/graph.json","events_json":"https://pith.science/api/pith-number/TRRS5IT2GHTQ4SJS3552DBVHYS/events.json","paper":"https://pith.science/paper/TRRS5IT2"},"agent_actions":{"view_html":"https://pith.science/pith/TRRS5IT2GHTQ4SJS3552DBVHYS","download_json":"https://pith.science/pith/TRRS5IT2GHTQ4SJS3552DBVHYS.json","view_paper":"https://pith.science/paper/TRRS5IT2","resolve_alias":"https://pith.science/api/pith-number/resolve?arxiv=2210.09261&json=true","fetch_graph":"https://pith.science/api/pith-number/TRRS5IT2GHTQ4SJS3552DBVHYS/graph.json","fetch_events":"https://pith.science/api/pith-number/TRRS5IT2GHTQ4SJS3552DBVHYS/events.json","actions":{"anchor_timestamp":"https://pith.science/pith/TRRS5IT2GHTQ4SJS3552DBVHYS/action/timestamp_anchor","attest_storage":"https://pith.science/pith/TRRS5IT2GHTQ4SJS3552DBVHYS/action/storage_attestation","attest_author":"https://pith.science/pith/TRRS5IT2GHTQ4SJS3552DBVHYS/action/author_attestation","sign_citation":"https://pith.science/pith/TRRS5IT2GHTQ4SJS3552DBVHYS/action/citation_signature","submit_replication":"https://pith.science/pith/TRRS5IT2GHTQ4SJS3552DBVHYS/action/replication_record"}},"created_at":"2026-07-05T05:07:20.597765+00:00","updated_at":"2026-07-05T05:07:20.597765+00:00"}