{"bundle_type":"pith_open_graph_bundle","bundle_version":"1.0","pith_number":"pith:2022:SRRDNYH656PTVQXFTPSYYS5UL5","short_pith_number":"pith:SRRDNYH6","canonical_record":{"source":{"id":"2206.04615","kind":"arxiv","version":3},"metadata":{"license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","primary_cat":"cs.CL","submitted_at":"2022-06-09T17:05:34Z","cross_cats_sorted":["cs.AI","cs.CY","cs.LG","stat.ML"],"title_canon_sha256":"87c952ae2bdbb80886c0966d76d28c34c4d93a14105c793816e936de22da8d86","abstract_canon_sha256":"885d26609bc170e90df8a541f02297d37e0460d9f7670ac856352aea2ea0bd06"},"schema_version":"1.0"},"canonical_sha256":"946236e0feef9f3ac2e59be58c4bb45f76bfda66abb3084c03762157ca4030a7","source":{"kind":"arxiv","id":"2206.04615","version":3},"source_aliases":[{"alias_kind":"arxiv","alias_value":"2206.04615","created_at":"2026-07-05T06:19:36Z"},{"alias_kind":"arxiv_version","alias_value":"2206.04615v3","created_at":"2026-07-05T06:19:36Z"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.2206.04615","created_at":"2026-07-05T06:19:36Z"},{"alias_kind":"pith_short_12","alias_value":"SRRDNYH656PT","created_at":"2026-07-05T06:19:36Z"},{"alias_kind":"pith_short_16","alias_value":"SRRDNYH656PTVQXF","created_at":"2026-07-05T06:19:36Z"},{"alias_kind":"pith_short_8","alias_value":"SRRDNYH6","created_at":"2026-07-05T06:19:36Z"}],"events":[{"event_type":"record_created","subject_pith_number":"pith:2022:SRRDNYH656PTVQXFTPSYYS5UL5","target":"record","payload":{"canonical_record":{"source":{"id":"2206.04615","kind":"arxiv","version":3},"metadata":{"license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","primary_cat":"cs.CL","submitted_at":"2022-06-09T17:05:34Z","cross_cats_sorted":["cs.AI","cs.CY","cs.LG","stat.ML"],"title_canon_sha256":"87c952ae2bdbb80886c0966d76d28c34c4d93a14105c793816e936de22da8d86","abstract_canon_sha256":"885d26609bc170e90df8a541f02297d37e0460d9f7670ac856352aea2ea0bd06"},"schema_version":"1.0"},"canonical_sha256":"946236e0feef9f3ac2e59be58c4bb45f76bfda66abb3084c03762157ca4030a7","receipt":{"kind":"pith_receipt","key_id":"pith-v1-2026-05","algorithm":"ed25519","signed_at":"2026-07-05T06:19:36.399032Z","signature_b64":"UAolr1n81gPAWyPyM+jJnpZ9zfLYsVpMXj73j6OThwGUdLxEr7GNmFgSfNgsNm7tlqdEjE+YmLvzFF8WqyFWCw==","signed_message":"canonical_sha256_bytes","builder_version":"pith-number-builder-2026-05-17-v1","receipt_version":"0.3","canonical_sha256":"946236e0feef9f3ac2e59be58c4bb45f76bfda66abb3084c03762157ca4030a7","last_reissued_at":"2026-07-05T06:19:36.398416Z","signature_status":"signed_v1","first_computed_at":"2026-07-05T06:19:36.398416Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"source_kind":"arxiv","source_id":"2206.04615","source_version":3,"attestation_state":"computed"},"signer":{"signer_id":"pith.science","signer_type":"pith_registry","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"created_at":"2026-07-05T06:19:36Z","supersedes":[],"prev_event":null,"signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"N+xbnQhJfFI7DYSlBGzaYqHNKMm0i+sLxfIVvysruVDEk95kqVSTni4+UT28tJ5Xgd6YyiKE4AOxyFquZ5GRAw==","signed_message":"open_graph_event_sha256_bytes","signed_at":"2026-08-20T07:48:46.567589Z"},"content_sha256":"982c4f7ee7e5a7c11d03b092d2b88367c01971133d54fd687b1ad299cdc6dfca","schema_version":"1.0","event_id":"sha256:982c4f7ee7e5a7c11d03b092d2b88367c01971133d54fd687b1ad299cdc6dfca"},{"event_type":"graph_snapshot","subject_pith_number":"pith:2022:SRRDNYH656PTVQXFTPSYYS5UL5","target":"graph","payload":{"graph_snapshot":{"paper":{"title":"Beyond the Imitation Game: Quantifying and extrapolating the capabilities of language models","license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","headline":"Scale brings gradual gains on knowledge tasks but sudden breakthroughs on complex ones in language models.","cross_cats":["cs.AI","cs.CY","cs.LG","stat.ML"],"primary_cat":"cs.CL","authors_text":"Aarohi Srivastava, Abhinav Rastogi, Abhishek Rao, Abu Awal Md Shoeb, Abubakar Abid, Adam Fisch, Adam R. Brown, Adam Santoro, Aditya Gupta, Adri\\`a Garriga-Alonso, Agnieszka Kluska, Aitor Lewkowycz, Akshat Agarwal, Alethea Power, Alexander W. Kocurek, Alex Ray, Alex Warstadt, Alice Xiang, Alicia Parrish, Ali Safaya, Ali Tazarv, Allen Nie, Amanda Askell, Amanda Dsouza, Aman Hussain, Ambrose Slone, Ameet Rahane, Anantharaman S. Iyer, Anders Andreassen, Andrea Madotto, Andrea Santilli, Andreas Stuhlm\\\"uller, Andrew Dai, Andrew La, Andrew Lampinen, Andy Zou, Angela Jiang, Angelica Chen, Anh Vuong, Animesh Gupta, Anna Gottardi, Antonio Norelli, Anu Venkatesh, Arash Gholamidavoodi, Arfa Tabassum, Arul Menezes, Arun Kirubarajan, Asher Mullokandov, Ashish Sabharwal, Austin Herrick, Avia Efrat, Aykut Erdem, Ayla Karaka\\c{s}, Bao Sheng Loe, Barret Zoph, Bart{\\l}omiej Bojanowski, Batuhan \\\"Ozyurt, Behnam Hedayatnia, Behnam Neyshabur, Benjamin Inden, Benno Stein, Berk Ekmekci, Bill Yuchen Lin, Blake Howald, Bryan Orinion, B. Ryan Roberts, Cameron Diao, Cameron Dour, Catherine Stinson, Cedrick Argueta, C\\'esar Ferri Ram\\'irez, Chandan Singh, Charles Rathkopf, Chenlin Meng, Chitta Baral, Chiyu Wu, Chris Callison-Burch, Christian Voigt, Christopher D. Manning, Christopher Potts, Chris Waites, Cindy Ramirez, Clara E. Rivera, Clemencia Siro, Colin Raffel, Courtney Ashcraft, Cristina Garbacea, Damien Sileo, Dan Garrette, Dan Hendrycks, Daniel Freeman, Daniel Khashabi, Danielle Perszyk, Daniel Levy, Daniel Mosegu\\'i Gonz\\'alez, Dan Kilman, Danny Hernandez, Danqi Chen, Dan Roth, Daphne Ippolito, Dar Gilboa, David Dohan, David Drakard, David Jurgens, Debajyoti Datta, Deep Ganguli, Denis Emelin, Denis Kleyko, Deniz Yuret, Derek Chen, Derek Tam, Dieuwke Hupkes, Diganta Misra, Dilyar Buzan, Dimitri Coelho Mollo, Diyi Yang, Dong-Ho Lee, Dylan Schrader, Ekaterina Shutova, Ekin Dogus Cubuk, Elad Segal, Eleanor Hagerman, Elizabeth Barnes, Elizabeth Donoway, Ellie Pavlick, Emanuele Rodola, Emma Lam, Eric Chu, Eric Tang, Erkut Erdem, Ernie Chang, Ethan A. Chi, Ethan Dyer, Ethan Jerzak, Ethan Kim, Eunice Engefu Manyasi, Evgenii Zheltonozhskii, Fanyue Xia, Fatemeh Siar, Fernando Mart\\'inez-Plumed, Francesca Happ\\'e, Francois Chollet, Frieda Rong, Gaurav Mishra, Genta Indra Winata, Gerard de Melo, Germ\\'an Kruszewski, Giambattista Parascandolo, Giorgio Mariani, Gloria Wang, Gonzalo Jaimovitch-L\\'opez, Gregor Betz, Guy Gur-Ari, Hana Galijasevic, Hannah Kim, Hannah Rashkin, Hannaneh Hajishirzi, Harsh Mehta, Hayden Bogar, Henry Shevlin, Hinrich Sch\\\"utze, Hiromu Yakura, Hongming Zhang, Hugh Mee Wong, Ian Ng, Isaac Noble, Jaap Jumelet, Jack Geissinger, Jackson Kernion, Jacob Hilton, Jaehoon Lee, Jaime Fern\\'andez Fisac, James B. Simon, James Koppel, James Zheng, James Zou, Jana Thompson, Janelle Wingfield, Jan Koco\\'n, Jared Kaplan, Jarema Radom, Jascha Sohl-Dickstein, Jason Phang, Jason Wei, Jason Yosinski, Jekaterina Novikova, Jelle Bosscher, Jennifer Marsh, Jeremy Kim, Jeroen Taal, Jesse Engel, Jesujoba Alabi, Jiacheng Xu, Jiaming Song, Jillian Tang, Joan Waweru, John Burden, John Miller, John U. Balis, Jonathan Batchelder, Jonathan Berant, J\\\"org Frohberg, Jose Hernandez-Orallo, Joseph Boudeman, Joseph Guerr, Joseph Jones, Joshua B. Tenenbaum, Joshua S. Rule, Jos Rozen, Joyce Chua, Kamil Kanclerz, Karen Livescu, Karl Krauth, Karthik Gopalakrishnan, Katerina Ignatyeva, Katja Markert, Kaustubh D. Dhole, Kevin Gimpel, Kevin Omondi, Kory Mathewson, Kristen Chiafullo, Ksenia Shkaruta, Kumar Shridhar, Kyle McDonell, Kyle Richardson, Laria Reynolds, Leo Gao, Liam Dugan, Lianhui Qin, Lidia Contreras-Ochando, Li Zhang, Louis-Philippe Morency, Luca Moschella, Lucas Lam, Lucy Noble, Ludwig Schmidt, Luheng He, Luis Oliveros Col\\'on, Luke Metz, L\\\"utfi Kerem \\c{S}enel, Maarten Bosma, Maarten Sap, Maartje ter Hoeve, Maheen Farooqi, Manaal Faruqui, Mantas Mazeika, Marco Baturan, Marco Marelli, Marco Maru, Maria Jose Ram\\'irez Quintana, Marie Tolkiehn, Mario Giulianelli, Martha Lewis, Martin Potthast, Matthew L. Leavitt, Matthias Hagen, M\\'aty\\'as Schubert, Medina Orduna Baitemirova, Melody Arnaud, Melvin McElrath, Michael A. Yee, Michael Cohen, Michael Gu, Michael Ivanitskiy, Michael Starritt, Michael Strube, Micha{\\l} Sw\\k{e}drowski, Michele Bevilacqua, Michihiro Yasunaga, Mihir Kale, Mike Cain, Mimee Xu, Mirac Suzgun, Mitch Walker, Mohit Bansal, Moin Aminnaseri, Mor Geva, Mo Tiwari, Mozhdeh Gheini, Mukund Varma T, Nanyun Peng, Nathan A. Chi, Nayeon Lee, Neta Gur-Ari Krakover, Nicholas Cameron, Nicholas Roberts, Nick Doiron, Nicole Martinez, Nikita Nangia, Niklas Deckers, Niklas Muennighoff, Nitish Shirish Keskar, Niveditha S. Iyer, Noah Constant, Noah Fiedel, Nuan Wen, Oliver Zhang, Omar Agha, Omar Elbaghdadi, Omer Levy, Owain Evans, Pablo Antonio Moreno Casares, Parth Doshi, Pascale Fung, Paul Pu Liang, Paul Vicol, Pegah Alipoormolabashi, Peiyuan Liao, Percy Liang, Peter Chang, Peter Eckersley, Phu Mon Htut, Pinyu Hwang, Piotr Mi{\\l}kowski, Piyush Patil, Pouya Pezeshkpour, Priti Oli, Qiaozhu Mei, Qing Lyu, Qinlang Chen, Rabin Banjade, Rachel Etta Rudolph, Raefer Gabriel, Rahel Habacker, Ramon Risco, Rapha\\\"el Milli\\`ere, Rhythm Garg, Richard Barnes, Rif A. Saurous, Riku Arakawa, Robbe Raymaekers, Robert Frank, Rohan Sikand, Roman Novak, Roman Sitelew, Ronan LeBras, Rosanne Liu, Rowan Jacobs, Rui Zhang, Ruslan Salakhutdinov, Ryan Chi, Ryan Lee, Ryan Stovall, Ryan Teehan, Rylan Yang, Sahib Singh, Saif M. Mohammad, Sajant Anand, Sam Dillavou, Sam Shleifer, Samuel Gruetter, Samuel R. Bowman, Samuel S. Schoenholz, Sam Wiseman, Sanghyun Han, Sanjeev Kwatra, Sarah A. Rous, Sarik Ghazarian, Sayan Ghosh, Sean Casey, Sebastian Bischoff, Sebastian Gehrmann, Sebastian Schuster, Sepideh Sadeghi, Shadi Hamdan, Sharon Zhou, Shashank Srivastava, Sherry Shi, Shikhar Singh, Shima Asaadi, Shixiang Shane Gu, Shubham Toshniwal, Shubh Pachchigar, Shyamolima (Shammie) Debnath, Shyam Upadhyay, Siamak Shakeri, Simone Melzi, Simon Thormeyer, Siva Reddy, Sneha Priscilla Makini, Soo-Hwan Lee, Spencer Torene, Sriharsha Hatwar, Stanislas Dehaene, Stefan Divic, Stefano Ermon, Stella Biderman, Stephanie Lin, Stephen Prasad, Steven T. Piantadosi, Stuart M. Shieber, Summer Misherghi, Svetlana Kiritchenko, Swaroop Mishra, Tal Linzen, Tal Schuster, Tao Li, Tao Yu, Tariq Ali, Tatsu Hashimoto, Te-Lin Wu, Th\\'eo Desbordes, Theodore Rothschild, Thomas Phan, Tianle Wang, Tiberius Nkinyili, Timofei Kornev, Timo Schick, Titus Tunduny, Tobias Gerstenberg, Trenton Chang, Trishala Neeraj, Tushar Khot, Tyler Shultz, Uri Shaham, Vedant Misra, Vera Demberg, Victoria Nyamai, Vikas Raunak, Vinay Ramasesh, Vinay Uday Prabhu, Vishakh Padmakumar, Vivek Srikumar, William Fedus, William Saunders, William Zhang, Wout Vossen, Xiang Ren, Xiaoyu Tong, Xinran Zhao, Xinyi Wu, Xudong Shen, Yadollah Yaghoobzadeh, Yair Lakretz, Yangqiu Song, Yasaman Bahri, Yejin Choi, Yichi Yang, Yiding Hao, Yifu Chen, Yonatan Belinkov, Yufang Hou, Yu Hou, Yuntao Bai, Zachary Seid, Zhuoye Zhao, Zijian Wang, Zijie J. Wang, Zirui Wang, Ziyi Wu","submitted_at":"2022-06-09T17:05:34Z","abstract_excerpt":"Language models demonstrate both quantitative improvement and new qualitative capabilities with increasing scale. Despite their potentially transformative impact, these new capabilities are as yet poorly characterized. In order to inform future research, prepare for disruptive new model capabilities, and ameliorate socially harmful effects, it is vital that we understand the present and near-future capabilities and limitations of language models. To address this challenge, we introduce the Beyond the Imitation Game benchmark (BIG-bench). BIG-bench currently consists of 204 tasks, contributed b"},"claims":{"count":4,"items":[{"kind":"strongest_claim","text":"Model performance and calibration both improve with scale, but are poor in absolute terms (and when compared with rater performance); tasks that improve gradually and predictably commonly involve a large knowledge or memorization component, whereas tasks that exhibit 'breakthrough' behavior at a critical scale often involve multiple steps or components, or brittle metrics.","source":"verdict.strongest_claim","status":"machine_extracted","claim_id":"C1","attestation":"unclaimed"},{"kind":"weakest_assumption","text":"That the 204 tasks chosen by the 450 contributors are sufficiently representative of the capabilities that will matter for future models and that human rater performance provides a stable, unbiased ceiling.","source":"verdict.weakest_assumption","status":"machine_extracted","claim_id":"C2","attestation":"unclaimed"},{"kind":"one_line_summary","text":"BIG-bench is a 204-task benchmark that measures scaling trends, calibration, and absolute limitations of language models across knowledge, reasoning, and social domains.","source":"verdict.one_line_summary","status":"machine_extracted","claim_id":"C3","attestation":"unclaimed"},{"kind":"headline","text":"Scale brings gradual gains on knowledge tasks but sudden breakthroughs on complex ones in language models.","source":"verdict.pith_extraction.headline","status":"machine_extracted","claim_id":"C4","attestation":"unclaimed"}],"snapshot_sha256":"13ad96cb52742070a8beec7878877cf888e99d32f53cf97fcbbc5e77c944691c"},"source":{"id":"2206.04615","kind":"arxiv","version":3},"verdict":{"id":"230c222e-389b-44a8-b822-fa7e5cc6facf","model_set":{"reader":"grok-4.3"},"created_at":"2026-05-10T23:21:35.284290Z","strongest_claim":"Model performance and calibration both improve with scale, but are poor in absolute terms (and when compared with rater performance); tasks that improve gradually and predictably commonly involve a large knowledge or memorization component, whereas tasks that exhibit 'breakthrough' behavior at a critical scale often involve multiple steps or components, or brittle metrics.","one_line_summary":"BIG-bench is a 204-task benchmark that measures scaling trends, calibration, and absolute limitations of language models across knowledge, reasoning, and social domains.","pipeline_version":"pith-pipeline@v0.9.0","weakest_assumption":"That the 204 tasks chosen by the 450 contributors are sufficiently representative of the capabilities that will matter for future models and that human rater performance provides a stable, unbiased ceiling.","pith_extraction_headline":"Scale brings gradual gains on knowledge tasks but sudden breakthroughs on complex ones in language models."},"integrity":{"clean":true,"summary":{"advisory":0,"critical":0,"by_detector":{},"informational":0},"endpoint":"/pith/2206.04615/integrity.json","findings":[],"available":true,"detectors_run":[],"snapshot_sha256":"c28c3603d3b5d939e8dc4c7e95fa8dfce3d595e45f758748cecf8e644a296938"},"references":{"count":49,"sample":[{"doi":"10.18653/v1/n19-1245","year":2020,"title":"MathQA: Towards interpretable math word problem solving with operation-based formalisms","work_id":"b73d762e-5f33-41a0-a44d-b2e613bffd36","ref_index":1,"cited_arxiv_id":"","is_internal_anchor":false},{"doi":"10.1126/science.177.4047.393","year":2017,"title":"URL https://arxiv.org/abs/1606.06565. (cited on p. 40) Brandon Amos and J. Zico Kolter. Optnet: Differentiable optimization as a layer in neural networks, 2017. URLhttps: //arxiv.org/abs/1703.00443. (","work_id":"d27e9bf2-171d-4cc9-bada-506cc2da0e28","ref_index":2,"cited_arxiv_id":"","is_internal_anchor":false},{"doi":"10.15390/eb.2020.8782","year":2001,"title":"URL https://arxiv.org/abs/2001.08435. (cited on p. 39) Nihat Bayat and Gökhan Çetinkaya. The relationship between inference skills and reading comprehension.TED EĞİTİM VE BİLİM (Education and Science)","work_id":"18b15dc2-febe-42c2-8549-4a785bbd2dfa","ref_index":3,"cited_arxiv_id":"","is_internal_anchor":false},{"doi":"10.1109/tvcg.2011.185","year":2019,"title":"On the Opportunities and Risks of Foundation Models","work_id":"a18039e9-928d-47c9-a836-32656a71bf71","ref_index":4,"cited_arxiv_id":"2108.07258","is_internal_anchor":true},{"doi":"10.18653/v1/w18-6433","year":1964,"title":"doi: 10.18653/v1/W18-6433","work_id":"f4b97612-127d-4e16-95de-ebe9267b3d55","ref_index":5,"cited_arxiv_id":"","is_internal_anchor":false}],"resolved_work":49,"snapshot_sha256":"60dfe0e79a8974582289befceff1bebbac18b453bfa2e078173bbbc520b0fbb5","internal_anchors":1},"formal_canon":{"evidence_count":2,"snapshot_sha256":"61b1140ba33972fa9e3cc13cfdae78f4771362eb9ec420dbc6401c98add680dc"},"author_claims":{"count":0,"strong_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"builder_version":"pith-number-builder-2026-05-17-v1"},"verdict_id":"230c222e-389b-44a8-b822-fa7e5cc6facf"},"signer":{"signer_id":"pith.science","signer_type":"pith_registry","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"created_at":"2026-07-05T06:19:36Z","supersedes":[],"prev_event":null,"signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"W8PO/aka4w2RXxMqytMMgLesdj49mC0MBLE8uE1b+ZBS3DSJd/duwKy3smeVwu8gTOa3jKQV7dPC4t4/sbhfCw==","signed_message":"open_graph_event_sha256_bytes","signed_at":"2026-08-20T07:48:46.568588Z"},"content_sha256":"ff41cc8317af0ea517d39b0bca7030c52f1c74b38c1fc3978e94ee58c1802884","schema_version":"1.0","event_id":"sha256:ff41cc8317af0ea517d39b0bca7030c52f1c74b38c1fc3978e94ee58c1802884"}],"timestamp_proofs":[],"mirror_hints":[{"mirror_type":"https","name":"Pith Resolver","base_url":"https://pith.science","bundle_url":"https://pith.science/pith/SRRDNYH656PTVQXFTPSYYS5UL5/bundle.json","state_url":"https://pith.science/pith/SRRDNYH656PTVQXFTPSYYS5UL5/state.json","well_known_bundle_url":"https://pith.science/.well-known/pith/SRRDNYH656PTVQXFTPSYYS5UL5/bundle.json","status":"primary"}],"public_keys":[{"key_id":"pith-v1-2026-05","algorithm":"ed25519","format":"raw","public_key_b64":"stVStoiQhXFxp4s2pdzPNoqVNBMojDU/fJ2db5S3CbM=","public_key_hex":"b2d552b68890857171a78b36a5dccf368a953413288c353f7c9d9d6f94b709b3","fingerprint_sha256_b32_first128bits":"RVFV5Z2OI2J3ZUO7ERDEBCYNKS","fingerprint_sha256_hex":"8d4b5ee74e4693bcd1df2446408b0d54","rotates_at":null,"url":"https://pith.science/pith-signing-key.json","notes":"Pith uses this Ed25519 key to sign canonical record SHA-256 digests. Verify with: ed25519_verify(public_key, message=canonical_sha256_bytes, signature=base64decode(signature_b64))."}],"merge_version":"pith-open-graph-merge-v1","built_at":"2026-08-20T07:48:46Z","links":{"resolver":"https://pith.science/pith/SRRDNYH656PTVQXFTPSYYS5UL5","bundle":"https://pith.science/pith/SRRDNYH656PTVQXFTPSYYS5UL5/bundle.json","state":"https://pith.science/pith/SRRDNYH656PTVQXFTPSYYS5UL5/state.json","well_known_bundle":"https://pith.science/.well-known/pith/SRRDNYH656PTVQXFTPSYYS5UL5/bundle.json"},"state":{"state_type":"pith_open_graph_state","state_version":"1.0","pith_number":"pith:2022:SRRDNYH656PTVQXFTPSYYS5UL5","merge_version":"pith-open-graph-merge-v1","event_count":2,"valid_event_count":2,"invalid_event_count":0,"equivocation_count":0,"current":{"canonical_record":{"metadata":{"abstract_canon_sha256":"885d26609bc170e90df8a541f02297d37e0460d9f7670ac856352aea2ea0bd06","cross_cats_sorted":["cs.AI","cs.CY","cs.LG","stat.ML"],"license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","primary_cat":"cs.CL","submitted_at":"2022-06-09T17:05:34Z","title_canon_sha256":"87c952ae2bdbb80886c0966d76d28c34c4d93a14105c793816e936de22da8d86"},"schema_version":"1.0","source":{"id":"2206.04615","kind":"arxiv","version":3}},"source_aliases":[{"alias_kind":"arxiv","alias_value":"2206.04615","created_at":"2026-07-05T06:19:36Z"},{"alias_kind":"arxiv_version","alias_value":"2206.04615v3","created_at":"2026-07-05T06:19:36Z"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.2206.04615","created_at":"2026-07-05T06:19:36Z"},{"alias_kind":"pith_short_12","alias_value":"SRRDNYH656PT","created_at":"2026-07-05T06:19:36Z"},{"alias_kind":"pith_short_16","alias_value":"SRRDNYH656PTVQXF","created_at":"2026-07-05T06:19:36Z"},{"alias_kind":"pith_short_8","alias_value":"SRRDNYH6","created_at":"2026-07-05T06:19:36Z"}],"graph_snapshots":[{"event_id":"sha256:ff41cc8317af0ea517d39b0bca7030c52f1c74b38c1fc3978e94ee58c1802884","target":"graph","created_at":"2026-07-05T06:19:36Z","signer":{"key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signer_id":"pith.science","signer_type":"pith_registry"},"payload":{"graph_snapshot":{"author_claims":{"count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","strong_count":0},"builder_version":"pith-number-builder-2026-05-17-v1","claims":{"count":4,"items":[{"attestation":"unclaimed","claim_id":"C1","kind":"strongest_claim","source":"verdict.strongest_claim","status":"machine_extracted","text":"Model performance and calibration both improve with scale, but are poor in absolute terms (and when compared with rater performance); tasks that improve gradually and predictably commonly involve a large knowledge or memorization component, whereas tasks that exhibit 'breakthrough' behavior at a critical scale often involve multiple steps or components, or brittle metrics."},{"attestation":"unclaimed","claim_id":"C2","kind":"weakest_assumption","source":"verdict.weakest_assumption","status":"machine_extracted","text":"That the 204 tasks chosen by the 450 contributors are sufficiently representative of the capabilities that will matter for future models and that human rater performance provides a stable, unbiased ceiling."},{"attestation":"unclaimed","claim_id":"C3","kind":"one_line_summary","source":"verdict.one_line_summary","status":"machine_extracted","text":"BIG-bench is a 204-task benchmark that measures scaling trends, calibration, and absolute limitations of language models across knowledge, reasoning, and social domains."},{"attestation":"unclaimed","claim_id":"C4","kind":"headline","source":"verdict.pith_extraction.headline","status":"machine_extracted","text":"Scale brings gradual gains on knowledge tasks but sudden breakthroughs on complex ones in language models."}],"snapshot_sha256":"13ad96cb52742070a8beec7878877cf888e99d32f53cf97fcbbc5e77c944691c"},"formal_canon":{"evidence_count":2,"snapshot_sha256":"61b1140ba33972fa9e3cc13cfdae78f4771362eb9ec420dbc6401c98add680dc"},"integrity":{"available":true,"clean":true,"detectors_run":[],"endpoint":"/pith/2206.04615/integrity.json","findings":[],"snapshot_sha256":"c28c3603d3b5d939e8dc4c7e95fa8dfce3d595e45f758748cecf8e644a296938","summary":{"advisory":0,"by_detector":{},"critical":0,"informational":0}},"paper":{"abstract_excerpt":"Language models demonstrate both quantitative improvement and new qualitative capabilities with increasing scale. Despite their potentially transformative impact, these new capabilities are as yet poorly characterized. In order to inform future research, prepare for disruptive new model capabilities, and ameliorate socially harmful effects, it is vital that we understand the present and near-future capabilities and limitations of language models. To address this challenge, we introduce the Beyond the Imitation Game benchmark (BIG-bench). BIG-bench currently consists of 204 tasks, contributed b","authors_text":"Aarohi Srivastava, Abhinav Rastogi, Abhishek Rao, Abu Awal Md Shoeb, Abubakar Abid, Adam Fisch, Adam R. Brown, Adam Santoro, Aditya Gupta, Adri\\`a Garriga-Alonso, Agnieszka Kluska, Aitor Lewkowycz, Akshat Agarwal, Alethea Power, Alexander W. Kocurek, Alex Ray, Alex Warstadt, Alice Xiang, Alicia Parrish, Ali Safaya, Ali Tazarv, Allen Nie, Amanda Askell, Amanda Dsouza, Aman Hussain, Ambrose Slone, Ameet Rahane, Anantharaman S. Iyer, Anders Andreassen, Andrea Madotto, Andrea Santilli, Andreas Stuhlm\\\"uller, Andrew Dai, Andrew La, Andrew Lampinen, Andy Zou, Angela Jiang, Angelica Chen, Anh Vuong, Animesh Gupta, Anna Gottardi, Antonio Norelli, Anu Venkatesh, Arash Gholamidavoodi, Arfa Tabassum, Arul Menezes, Arun Kirubarajan, Asher Mullokandov, Ashish Sabharwal, Austin Herrick, Avia Efrat, Aykut Erdem, Ayla Karaka\\c{s}, Bao Sheng Loe, Barret Zoph, Bart{\\l}omiej Bojanowski, Batuhan \\\"Ozyurt, Behnam Hedayatnia, Behnam Neyshabur, Benjamin Inden, Benno Stein, Berk Ekmekci, Bill Yuchen Lin, Blake Howald, Bryan Orinion, B. Ryan Roberts, Cameron Diao, Cameron Dour, Catherine Stinson, Cedrick Argueta, C\\'esar Ferri Ram\\'irez, Chandan Singh, Charles Rathkopf, Chenlin Meng, Chitta Baral, Chiyu Wu, Chris Callison-Burch, Christian Voigt, Christopher D. Manning, Christopher Potts, Chris Waites, Cindy Ramirez, Clara E. Rivera, Clemencia Siro, Colin Raffel, Courtney Ashcraft, Cristina Garbacea, Damien Sileo, Dan Garrette, Dan Hendrycks, Daniel Freeman, Daniel Khashabi, Danielle Perszyk, Daniel Levy, Daniel Mosegu\\'i Gonz\\'alez, Dan Kilman, Danny Hernandez, Danqi Chen, Dan Roth, Daphne Ippolito, Dar Gilboa, David Dohan, David Drakard, David Jurgens, Debajyoti Datta, Deep Ganguli, Denis Emelin, Denis Kleyko, Deniz Yuret, Derek Chen, Derek Tam, Dieuwke Hupkes, Diganta Misra, Dilyar Buzan, Dimitri Coelho Mollo, Diyi Yang, Dong-Ho Lee, Dylan Schrader, Ekaterina Shutova, Ekin Dogus Cubuk, Elad Segal, Eleanor Hagerman, Elizabeth Barnes, Elizabeth Donoway, Ellie Pavlick, Emanuele Rodola, Emma Lam, Eric Chu, Eric Tang, Erkut Erdem, Ernie Chang, Ethan A. Chi, Ethan Dyer, Ethan Jerzak, Ethan Kim, Eunice Engefu Manyasi, Evgenii Zheltonozhskii, Fanyue Xia, Fatemeh Siar, Fernando Mart\\'inez-Plumed, Francesca Happ\\'e, Francois Chollet, Frieda Rong, Gaurav Mishra, Genta Indra Winata, Gerard de Melo, Germ\\'an Kruszewski, Giambattista Parascandolo, Giorgio Mariani, Gloria Wang, Gonzalo Jaimovitch-L\\'opez, Gregor Betz, Guy Gur-Ari, Hana Galijasevic, Hannah Kim, Hannah Rashkin, Hannaneh Hajishirzi, Harsh Mehta, Hayden Bogar, Henry Shevlin, Hinrich Sch\\\"utze, Hiromu Yakura, Hongming Zhang, Hugh Mee Wong, Ian Ng, Isaac Noble, Jaap Jumelet, Jack Geissinger, Jackson Kernion, Jacob Hilton, Jaehoon Lee, Jaime Fern\\'andez Fisac, James B. Simon, James Koppel, James Zheng, James Zou, Jana Thompson, Janelle Wingfield, Jan Koco\\'n, Jared Kaplan, Jarema Radom, Jascha Sohl-Dickstein, Jason Phang, Jason Wei, Jason Yosinski, Jekaterina Novikova, Jelle Bosscher, Jennifer Marsh, Jeremy Kim, Jeroen Taal, Jesse Engel, Jesujoba Alabi, Jiacheng Xu, Jiaming Song, Jillian Tang, Joan Waweru, John Burden, John Miller, John U. Balis, Jonathan Batchelder, Jonathan Berant, J\\\"org Frohberg, Jose Hernandez-Orallo, Joseph Boudeman, Joseph Guerr, Joseph Jones, Joshua B. Tenenbaum, Joshua S. Rule, Jos Rozen, Joyce Chua, Kamil Kanclerz, Karen Livescu, Karl Krauth, Karthik Gopalakrishnan, Katerina Ignatyeva, Katja Markert, Kaustubh D. Dhole, Kevin Gimpel, Kevin Omondi, Kory Mathewson, Kristen Chiafullo, Ksenia Shkaruta, Kumar Shridhar, Kyle McDonell, Kyle Richardson, Laria Reynolds, Leo Gao, Liam Dugan, Lianhui Qin, Lidia Contreras-Ochando, Li Zhang, Louis-Philippe Morency, Luca Moschella, Lucas Lam, Lucy Noble, Ludwig Schmidt, Luheng He, Luis Oliveros Col\\'on, Luke Metz, L\\\"utfi Kerem \\c{S}enel, Maarten Bosma, Maarten Sap, Maartje ter Hoeve, Maheen Farooqi, Manaal Faruqui, Mantas Mazeika, Marco Baturan, Marco Marelli, Marco Maru, Maria Jose Ram\\'irez Quintana, Marie Tolkiehn, Mario Giulianelli, Martha Lewis, Martin Potthast, Matthew L. Leavitt, Matthias Hagen, M\\'aty\\'as Schubert, Medina Orduna Baitemirova, Melody Arnaud, Melvin McElrath, Michael A. Yee, Michael Cohen, Michael Gu, Michael Ivanitskiy, Michael Starritt, Michael Strube, Micha{\\l} Sw\\k{e}drowski, Michele Bevilacqua, Michihiro Yasunaga, Mihir Kale, Mike Cain, Mimee Xu, Mirac Suzgun, Mitch Walker, Mohit Bansal, Moin Aminnaseri, Mor Geva, Mo Tiwari, Mozhdeh Gheini, Mukund Varma T, Nanyun Peng, Nathan A. Chi, Nayeon Lee, Neta Gur-Ari Krakover, Nicholas Cameron, Nicholas Roberts, Nick Doiron, Nicole Martinez, Nikita Nangia, Niklas Deckers, Niklas Muennighoff, Nitish Shirish Keskar, Niveditha S. Iyer, Noah Constant, Noah Fiedel, Nuan Wen, Oliver Zhang, Omar Agha, Omar Elbaghdadi, Omer Levy, Owain Evans, Pablo Antonio Moreno Casares, Parth Doshi, Pascale Fung, Paul Pu Liang, Paul Vicol, Pegah Alipoormolabashi, Peiyuan Liao, Percy Liang, Peter Chang, Peter Eckersley, Phu Mon Htut, Pinyu Hwang, Piotr Mi{\\l}kowski, Piyush Patil, Pouya Pezeshkpour, Priti Oli, Qiaozhu Mei, Qing Lyu, Qinlang Chen, Rabin Banjade, Rachel Etta Rudolph, Raefer Gabriel, Rahel Habacker, Ramon Risco, Rapha\\\"el Milli\\`ere, Rhythm Garg, Richard Barnes, Rif A. Saurous, Riku Arakawa, Robbe Raymaekers, Robert Frank, Rohan Sikand, Roman Novak, Roman Sitelew, Ronan LeBras, Rosanne Liu, Rowan Jacobs, Rui Zhang, Ruslan Salakhutdinov, Ryan Chi, Ryan Lee, Ryan Stovall, Ryan Teehan, Rylan Yang, Sahib Singh, Saif M. Mohammad, Sajant Anand, Sam Dillavou, Sam Shleifer, Samuel Gruetter, Samuel R. Bowman, Samuel S. Schoenholz, Sam Wiseman, Sanghyun Han, Sanjeev Kwatra, Sarah A. Rous, Sarik Ghazarian, Sayan Ghosh, Sean Casey, Sebastian Bischoff, Sebastian Gehrmann, Sebastian Schuster, Sepideh Sadeghi, Shadi Hamdan, Sharon Zhou, Shashank Srivastava, Sherry Shi, Shikhar Singh, Shima Asaadi, Shixiang Shane Gu, Shubham Toshniwal, Shubh Pachchigar, Shyamolima (Shammie) Debnath, Shyam Upadhyay, Siamak Shakeri, Simone Melzi, Simon Thormeyer, Siva Reddy, Sneha Priscilla Makini, Soo-Hwan Lee, Spencer Torene, Sriharsha Hatwar, Stanislas Dehaene, Stefan Divic, Stefano Ermon, Stella Biderman, Stephanie Lin, Stephen Prasad, Steven T. Piantadosi, Stuart M. Shieber, Summer Misherghi, Svetlana Kiritchenko, Swaroop Mishra, Tal Linzen, Tal Schuster, Tao Li, Tao Yu, Tariq Ali, Tatsu Hashimoto, Te-Lin Wu, Th\\'eo Desbordes, Theodore Rothschild, Thomas Phan, Tianle Wang, Tiberius Nkinyili, Timofei Kornev, Timo Schick, Titus Tunduny, Tobias Gerstenberg, Trenton Chang, Trishala Neeraj, Tushar Khot, Tyler Shultz, Uri Shaham, Vedant Misra, Vera Demberg, Victoria Nyamai, Vikas Raunak, Vinay Ramasesh, Vinay Uday Prabhu, Vishakh Padmakumar, Vivek Srikumar, William Fedus, William Saunders, William Zhang, Wout Vossen, Xiang Ren, Xiaoyu Tong, Xinran Zhao, Xinyi Wu, Xudong Shen, Yadollah Yaghoobzadeh, Yair Lakretz, Yangqiu Song, Yasaman Bahri, Yejin Choi, Yichi Yang, Yiding Hao, Yifu Chen, Yonatan Belinkov, Yufang Hou, Yu Hou, Yuntao Bai, Zachary Seid, Zhuoye Zhao, Zijian Wang, Zijie J. Wang, Zirui Wang, Ziyi Wu","cross_cats":["cs.AI","cs.CY","cs.LG","stat.ML"],"headline":"Scale brings gradual gains on knowledge tasks but sudden breakthroughs on complex ones in language models.","license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","primary_cat":"cs.CL","submitted_at":"2022-06-09T17:05:34Z","title":"Beyond the Imitation Game: Quantifying and extrapolating the capabilities of language models"},"references":{"count":49,"internal_anchors":1,"resolved_work":49,"sample":[{"cited_arxiv_id":"","doi":"10.18653/v1/n19-1245","is_internal_anchor":false,"ref_index":1,"title":"MathQA: Towards interpretable math word problem solving with operation-based formalisms","work_id":"b73d762e-5f33-41a0-a44d-b2e613bffd36","year":2020},{"cited_arxiv_id":"","doi":"10.1126/science.177.4047.393","is_internal_anchor":false,"ref_index":2,"title":"URL https://arxiv.org/abs/1606.06565. (cited on p. 40) Brandon Amos and J. Zico Kolter. Optnet: Differentiable optimization as a layer in neural networks, 2017. URLhttps: //arxiv.org/abs/1703.00443. (","work_id":"d27e9bf2-171d-4cc9-bada-506cc2da0e28","year":2017},{"cited_arxiv_id":"","doi":"10.15390/eb.2020.8782","is_internal_anchor":false,"ref_index":3,"title":"URL https://arxiv.org/abs/2001.08435. (cited on p. 39) Nihat Bayat and Gökhan Çetinkaya. The relationship between inference skills and reading comprehension.TED EĞİTİM VE BİLİM (Education and Science)","work_id":"18b15dc2-febe-42c2-8549-4a785bbd2dfa","year":2001},{"cited_arxiv_id":"2108.07258","doi":"10.1109/tvcg.2011.185","is_internal_anchor":true,"ref_index":4,"title":"On the Opportunities and Risks of Foundation Models","work_id":"a18039e9-928d-47c9-a836-32656a71bf71","year":2019},{"cited_arxiv_id":"","doi":"10.18653/v1/w18-6433","is_internal_anchor":false,"ref_index":5,"title":"doi: 10.18653/v1/W18-6433","work_id":"f4b97612-127d-4e16-95de-ebe9267b3d55","year":1964}],"snapshot_sha256":"60dfe0e79a8974582289befceff1bebbac18b453bfa2e078173bbbc520b0fbb5"},"source":{"id":"2206.04615","kind":"arxiv","version":3},"verdict":{"created_at":"2026-05-10T23:21:35.284290Z","id":"230c222e-389b-44a8-b822-fa7e5cc6facf","model_set":{"reader":"grok-4.3"},"one_line_summary":"BIG-bench is a 204-task benchmark that measures scaling trends, calibration, and absolute limitations of language models across knowledge, reasoning, and social domains.","pipeline_version":"pith-pipeline@v0.9.0","pith_extraction_headline":"Scale brings gradual gains on knowledge tasks but sudden breakthroughs on complex ones in language models.","strongest_claim":"Model performance and calibration both improve with scale, but are poor in absolute terms (and when compared with rater performance); tasks that improve gradually and predictably commonly involve a large knowledge or memorization component, whereas tasks that exhibit 'breakthrough' behavior at a critical scale often involve multiple steps or components, or brittle metrics.","weakest_assumption":"That the 204 tasks chosen by the 450 contributors are sufficiently representative of the capabilities that will matter for future models and that human rater performance provides a stable, unbiased ceiling."}},"verdict_id":"230c222e-389b-44a8-b822-fa7e5cc6facf"}}],"author_attestations":[],"timestamp_anchors":[],"storage_attestations":[],"citation_signatures":[],"replication_records":[],"corrections":[],"mirror_hints":[],"record_created":{"event_id":"sha256:982c4f7ee7e5a7c11d03b092d2b88367c01971133d54fd687b1ad299cdc6dfca","target":"record","created_at":"2026-07-05T06:19:36Z","signer":{"key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signer_id":"pith.science","signer_type":"pith_registry"},"payload":{"attestation_state":"computed","canonical_record":{"metadata":{"abstract_canon_sha256":"885d26609bc170e90df8a541f02297d37e0460d9f7670ac856352aea2ea0bd06","cross_cats_sorted":["cs.AI","cs.CY","cs.LG","stat.ML"],"license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","primary_cat":"cs.CL","submitted_at":"2022-06-09T17:05:34Z","title_canon_sha256":"87c952ae2bdbb80886c0966d76d28c34c4d93a14105c793816e936de22da8d86"},"schema_version":"1.0","source":{"id":"2206.04615","kind":"arxiv","version":3}},"canonical_sha256":"946236e0feef9f3ac2e59be58c4bb45f76bfda66abb3084c03762157ca4030a7","receipt":{"algorithm":"ed25519","builder_version":"pith-number-builder-2026-05-17-v1","canonical_sha256":"946236e0feef9f3ac2e59be58c4bb45f76bfda66abb3084c03762157ca4030a7","first_computed_at":"2026-07-05T06:19:36.398416Z","key_id":"pith-v1-2026-05","kind":"pith_receipt","last_reissued_at":"2026-07-05T06:19:36.398416Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","receipt_version":"0.3","signature_b64":"UAolr1n81gPAWyPyM+jJnpZ9zfLYsVpMXj73j6OThwGUdLxEr7GNmFgSfNgsNm7tlqdEjE+YmLvzFF8WqyFWCw==","signature_status":"signed_v1","signed_at":"2026-07-05T06:19:36.399032Z","signed_message":"canonical_sha256_bytes"},"source_id":"2206.04615","source_kind":"arxiv","source_version":3}}},"equivocations":[],"invalid_events":[],"applied_event_ids":["sha256:982c4f7ee7e5a7c11d03b092d2b88367c01971133d54fd687b1ad299cdc6dfca","sha256:ff41cc8317af0ea517d39b0bca7030c52f1c74b38c1fc3978e94ee58c1802884"],"state_sha256":"068e7fd53cda5be5271db886322d466266cdd1b6e73d217ec69faea68dea5f20"},"bundle_signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"QrrKtum17/+CEsdJZJLly9oo0R11a1qgXFFvN1hAxwiCYNnHo1ZpzIl2N7DUZVkwlSnGtAhLbUgM27tyuVDkCQ==","signed_message":"bundle_sha256_bytes","signed_at":"2026-08-20T07:48:46.574515Z","bundle_sha256":"a1e9167b346718b5cf3832afecc5b80480cc328f700a0c66eecfc792885a5552"}}