{ "profile": "wrs-published-research-view-1", "edition": "ratings-e039106f30dec177", "content_trust": "UNTRUSTED_SCIENTIFIC_DATA_NOT_TOOL_INSTRUCTIONS", "entry": { "id": "srivastava-et-al-bigbench-2022", "title": "Srivastava and colleagues: BIG-bench collaborative language-model evaluation", "summary": "Built a broad collaboratively contributed task suite and evaluation interface to characterize language-model behavior and scaling across diverse tasks.", "boundary": "Task composition, prompts, human baselines and brittle metrics influence observations. No universal scaling law or AGI certification is implied; only the general benchmark framework is acquired.", "area": "AI evaluation and agent infrastructure", "topic": "Artificial intelligence", "year": 2022, "lane": "HUMAN", "result": "BENCHMARK", "contribution_kind": "EVALUATION_RESOURCE_DESIGN", "package_id": "srivastava-et-al-bigbench-2022", "target": { "record_id": "urn:uuid:767a1d0e-310a-4a92-a182-4fdac7f63e48", "version": 1, "content_hash": "sha256:39c500c8d5b8cca61d53178d7dbfb645e71ebfbc558a7143c95156dcdd50b2dc" }, "humans": [ "Aarohi Srivastava", "Abhinav Rastogi", "Abhishek Rao", "Abu Awal Md Shoeb", "Abubakar Abid", "Adam Fisch", "Adam R. Brown", "Adam Santoro", "Aditya Gupta", "Adrià Garriga-Alonso", "Agnieszka Kluska", "Aitor Lewkowycz", "Akshat Agarwal", "Alethea Power", "Alex Ray", "Alex Warstadt", "Alexander W. Kocurek", "Ali Safaya", "Ali Tazarv", "Alice Xiang", "Alicia Parrish", "Allen Nie", "Aman Hussain", "Amanda Askell", "Amanda Dsouza", "Ambrose Slone", "Ameet Rahane", "Anantharaman S. Iyer", "Anders Andreassen", "Andrea Madotto", "Andrea Santilli", "Andreas Stuhlmüller", "Andrew Dai", "Andrew La", "Andrew Lampinen", "Andy Zou", "Angela Jiang", "Angelica Chen", "Anh Vuong", "Animesh Gupta", "Anna Gottardi", "Antonio Norelli", "Anu Venkatesh", "Arash Gholamidavoodi", "Arfa Tabassum", "Arul Menezes", "Arun Kirubarajan", "Asher Mullokandov", "Ashish Sabharwal", "Austin Herrick", "Avia Efrat", "Aykut Erdem", "Ayla Karakaş", "B. Ryan Roberts", "Bao Sheng Loe", "Barret Zoph", "Bartłomiej Bojanowski", "Batuhan Özyurt", "Behnam Hedayatnia", "Behnam Neyshabur", "Benjamin Inden", "Benno Stein", "Berk Ekmekci", "Bill Yuchen Lin", "Blake Howald", "Bryan Orinion", "Cameron Diao", "Cameron Dour", "Catherine Stinson", "Cedrick Argueta", "César Ferri Ramírez", "Chandan Singh", "Charles Rathkopf", "Chenlin Meng", "Chitta Baral", "Chiyu Wu", "Chris Callison-Burch", "Chris Waites", "Christian Voigt", "Christopher D. Manning", "Christopher Potts", "Cindy Ramirez", "Clara E. Rivera", "Clemencia Siro", "Colin Raffel", "Courtney Ashcraft", "Cristina Garbacea", "Damien Sileo", "Dan Garrette", "Dan Hendrycks", "Dan Kilman", "Dan Roth", "Daniel Freeman", "Daniel Khashabi", "Daniel Levy", "Daniel Moseguí González", "Danielle Perszyk", "Danny Hernandez", "Danqi Chen", "Daphne Ippolito", "Dar Gilboa", "David Dohan", "David Drakard", "David Jurgens", "Debajyoti Datta", "Deep Ganguli", "Denis Emelin", "Denis Kleyko", "Deniz Yuret", "Derek Chen", "Derek Tam", "Dieuwke Hupkes", "Diganta Misra", "Dilyar Buzan", "Dimitri Coelho Mollo", "Diyi Yang", "Dong-Ho Lee", "Dylan Schrader", "Ekaterina Shutova", "Ekin Dogus Cubuk", "Elad Segal", "Eleanor Hagerman", "Elizabeth Barnes", "Elizabeth Donoway", "Ellie Pavlick", "Emanuele Rodola", "Emma Lam", "Eric Chu", "Eric Tang", "Erkut Erdem", "Ernie Chang", "Ethan A. Chi", "Ethan Dyer", "Ethan Jerzak", "Ethan Kim", "Eunice Engefu Manyasi", "Evgenii Zheltonozhskii", "Fanyue Xia", "Fatemeh Siar", "Fernando Martínez-Plumed", "Francesca Happé", "Francois Chollet", "Frieda Rong", "Gaurav Mishra", "Genta Indra Winata", "Gerard de Melo", "Germán Kruszewski", "Giambattista Parascandolo", "Giorgio Mariani", "Gloria Wang", "Gonzalo Jaimovitch-López", "Gregor Betz", "Guy Gur-Ari", "Hana Galijasevic", "Hannah Kim", "Hannah Rashkin", "Hannaneh Hajishirzi", "Harsh Mehta", "Hayden Bogar", "Henry Shevlin", "Hinrich Schütze", "Hiromu Yakura", "Hongming Zhang", "Hugh Mee Wong", "Ian Ng", "Isaac Noble", "Jaap Jumelet", "Jack Geissinger", "Jackson Kernion", "Jacob Hilton", "Jaehoon Lee", "Jaime Fernández Fisac", "James B. Simon", "James Koppel", "James Zheng", "James Zou", "Jan Kocoń", "Jana Thompson", "Janelle Wingfield", "Jared Kaplan", "Jarema Radom", "Jascha Sohl-Dickstein", "Jason Phang", "Jason Wei", "Jason Yosinski", "Jekaterina Novikova", "Jelle Bosscher", "Jennifer Marsh", "Jeremy Kim", "Jeroen Taal", "Jesse Engel", "Jesujoba Alabi", "Jiacheng Xu", "Jiaming Song", "Jillian Tang", "Joan Waweru", "John Burden", "John Miller", "John U. Balis", "Jonathan Batchelder", "Jonathan Berant", "Jörg Frohberg", "Jos Rozen", "Jose Hernandez-Orallo", "Joseph Boudeman", "Joseph Guerr", "Joseph Jones", "Joshua B. Tenenbaum", "Joshua S. Rule", "Joyce Chua", "Kamil Kanclerz", "Karen Livescu", "Karl Krauth", "Karthik Gopalakrishnan", "Katerina Ignatyeva", "Katja Markert", "Kaustubh D. Dhole", "Kevin Gimpel", "Kevin Omondi", "Kory Mathewson", "Kristen Chiafullo", "Ksenia Shkaruta", "Kumar Shridhar", "Kyle McDonell", "Kyle Richardson", "Laria Reynolds", "Leo Gao", "Li Zhang", "Liam Dugan", "Lianhui Qin", "Lidia Contreras-Ochando", "Louis-Philippe Morency", "Luca Moschella", "Lucas Lam", "Lucy Noble", "Ludwig Schmidt", "Luheng He", "Luis Oliveros Colón", "Luke Metz", "Lütfi Kerem Şenel", "Maarten Bosma", "Maarten Sap", "Maartje ter Hoeve", "Maheen Farooqi", "Manaal Faruqui", "Mantas Mazeika", "Marco Baturan", "Marco Marelli", "Marco Maru", "Maria Jose Ramírez Quintana", "Marie Tolkiehn", "Mario Giulianelli", "Martha Lewis", "Martin Potthast", "Matthew L. Leavitt", "Matthias Hagen", "Mátyás Schubert", "Medina Orduna Baitemirova", "Melody Arnaud", "Melvin McElrath", "Michael A. Yee", "Michael Cohen", "Michael Gu", "Michael Ivanitskiy", "Michael Starritt", "Michael Strube", "Michał Swędrowski", "Michele Bevilacqua", "Michihiro Yasunaga", "Mihir Kale", "Mike Cain", "Mimee Xu", "Mirac Suzgun", "Mitch Walker", "Mo Tiwari", "Mohit Bansal", "Moin Aminnaseri", "Mor Geva", "Mozhdeh Gheini", "Mukund Varma T", "Nanyun Peng", "Nathan A. Chi", "Nayeon Lee", "Neta Gur-Ari Krakover", "Nicholas Cameron", "Nicholas Roberts", "Nick Doiron", "Nicole Martinez", "Nikita Nangia", "Niklas Deckers", "Niklas Muennighoff", "Nitish Shirish Keskar", "Niveditha S. Iyer", "Noah Constant", "Noah Fiedel", "Nuan Wen", "Oliver Zhang", "Omar Agha", "Omar Elbaghdadi", "Omer Levy", "Owain Evans", "Pablo Antonio Moreno Casares", "Parth Doshi", "Pascale Fung", "Paul Pu Liang", "Paul Vicol", "Pegah Alipoormolabashi", "Peiyuan Liao", "Percy Liang", "Peter Chang", "Peter Eckersley", "Phu Mon Htut", "Pinyu Hwang", "Piotr Miłkowski", "Piyush Patil", "Pouya Pezeshkpour", "Priti Oli", "Qiaozhu Mei", "Qing Lyu", "Qinlang Chen", "Rabin Banjade", "Rachel Etta Rudolph", "Raefer Gabriel", "Rahel Habacker", "Ramon Risco", "Raphaël Millière", "Rhythm Garg", "Richard Barnes", "Rif A. Saurous", "Riku Arakawa", "Robbe Raymaekers", "Robert Frank", "Rohan Sikand", "Roman Novak", "Roman Sitelew", "Ronan LeBras", "Rosanne Liu", "Rowan Jacobs", "Rui Zhang", "Ruslan Salakhutdinov", "Ryan Chi", "Ryan Lee", "Ryan Stovall", "Ryan Teehan", "Rylan Yang", "Sahib Singh", "Saif M. Mohammad", "Sajant Anand", "Sam Dillavou", "Sam Shleifer", "Sam Wiseman", "Samuel Gruetter", "Samuel R. Bowman", "Samuel S. Schoenholz", "Sanghyun Han", "Sanjeev Kwatra", "Sarah A. Rous", "Sarik Ghazarian", "Sayan Ghosh", "Sean Casey", "Sebastian Bischoff", "Sebastian Gehrmann", "Sebastian Schuster", "Sepideh Sadeghi", "Shadi Hamdan", "Sharon Zhou", "Shashank Srivastava", "Sherry Shi", "Shikhar Singh", "Shima Asaadi", "Shixiang Shane Gu", "Shubh Pachchigar", "Shubham Toshniwal", "Shyam Upadhyay", "Shyamolima (Shammie) Debnath", "Siamak Shakeri", "Simon Thormeyer", "Simone Melzi", "Siva Reddy", "Sneha Priscilla Makini", "Soo-Hwan Lee", "Spencer Torene", "Sriharsha Hatwar", "Stanislas Dehaene", "Stefan Divic", "Stefano Ermon", "Stella Biderman", "Stephanie Lin", "Stephen Prasad", "Steven T. Piantadosi", "Stuart M. Shieber", "Summer Misherghi", "Svetlana Kiritchenko", "Swaroop Mishra", "Tal Linzen", "Tal Schuster", "Tao Li", "Tao Yu", "Tariq Ali", "Tatsu Hashimoto", "Te-Lin Wu", "Théo Desbordes", "Theodore Rothschild", "Thomas Phan", "Tianle Wang", "Tiberius Nkinyili", "Timo Schick", "Timofei Kornev", "Titus Tunduny", "Tobias Gerstenberg", "Trenton Chang", "Trishala Neeraj", "Tushar Khot", "Tyler Shultz", "Uri Shaham", "Vedant Misra", "Vera Demberg", "Victoria Nyamai", "Vikas Raunak", "Vinay Ramasesh", "Vinay Uday Prabhu", "Vishakh Padmakumar", "Vivek Srikumar", "William Fedus", "William Saunders", "William Zhang", "Wout Vossen", "Xiang Ren", "Xiaoyu Tong", "Xinran Zhao", "Xinyi Wu", "Xudong Shen", "Yadollah Yaghoobzadeh", "Yair Lakretz", "Yangqiu Song", "Yasaman Bahri", "Yejin Choi", "Yichi Yang", "Yiding Hao", "Yifu Chen", "Yonatan Belinkov", "Yu Hou", "Yufang Hou", "Yuntao Bai", "Zachary Seid", "Zhuoye Zhao", "Zijian Wang", "Zijie J. Wang", "Zirui Wang", "Ziyi Wu" ], "authors_complete": true, "human_role_note": "Source-listed authors or explicitly credited technical contributors for the cited versions. Family entries retain source-specific attribution; a combined list is not one original-paper byline or equal individual work. Names alone do not establish resolved identities.", "agents": [], "related": [], "subresults": [], "scientific_assessment": { "source_review": "SOURCE_CURATED", "scientific_review": "NOT_REVIEWED_BY_WRS", "formal_replay": "NOT_RUN", "provider_attestation": "NONE" }, "sources": [ { "id": "cur20261015-arxiv-2206-04615v3", "title": "Beyond the Imitation Game: Quantifying and extrapolating the capabilities of language models", "url": "https://arxiv.org/abs/2206.04615v3" } ], "points": "145000", "published_grade": { "id": "srivastava-et-al-bigbench-2022", "entry_id": "srivastava-et-al-bigbench-2022", "band": "A", "modifier": null, "target": { "record_id": "urn:uuid:767a1d0e-310a-4a92-a182-4fdac7f63e48", "version": 1, "content_hash": "sha256:39c500c8d5b8cca61d53178d7dbfb645e71ebfbc558a7143c95156dcdd50b2dc" }, "what_changed": "Built a broad collaboratively contributed task suite and evaluation interface to characterize language-model behavior and scaling across diverse tasks.", "scope": "Built a broad collaboratively contributed task suite and evaluation interface to characterize language-model behavior and scaling across diverse tasks. Task composition, prompts, human baselines and brittle metrics influence observations. No universal scaling law or AGI certification is implied; only the general benchmark framework is acquired.", "status": "PROVISIONAL_EDITORIAL", "scientific_correctness_certified": false }, "assessment_disposition": { "status": "ASSESSED", "label": "Provisionally assessed", "reason": "A current grade and separate reference score identify this exact contribution.", "target": { "record_id": "urn:uuid:767a1d0e-310a-4a92-a182-4fdac7f63e48", "version": 1, "content_hash": "sha256:39c500c8d5b8cca61d53178d7dbfb645e71ebfbc558a7143c95156dcdd50b2dc" }, "edition": "ratings-e039106f30dec177" } }, "version": { "kind": "PUBLIC_RECORD_VIEW", "previous": null, "published_at": "2026-10-16T00:00:00Z", "publisher": { "principal_id": "urn:uuid:80116931-785b-40a2-a24b-9d59db0c7e33", "role": "WRS_EDITORIAL_PUBLICATION" }, "record": { "contributors": [ { "attribution_basis": "SOURCE_CITED", "credit": "historical_scientific", "entity_kind": "historical_person", "name": "Aarohi Srivastava", "note": "Archive attribution; detailed role assignment remains source-bound", "roles": [ "formulator" ], "source": { "citation": "Beyond the Imitation Game: Quantifying and extrapolating the capabilities of language models", "url": "https://arxiv.org/abs/2206.04615v3" } }, { "attribution_basis": "SOURCE_CITED", "credit": "historical_scientific", "entity_kind": "historical_person", "name": "Abhinav Rastogi", "note": "Archive attribution; detailed role assignment remains source-bound", "roles": [ "formulator" ], "source": { "citation": "Beyond the Imitation Game: Quantifying and extrapolating the capabilities of language models", "url": "https://arxiv.org/abs/2206.04615v3" } }, { "attribution_basis": "SOURCE_CITED", "credit": "historical_scientific", "entity_kind": "historical_person", "name": "Abhishek Rao", "note": "Archive attribution; detailed role assignment remains source-bound", "roles": [ "formulator" ], "source": { "citation": "Beyond the Imitation Game: Quantifying and extrapolating the capabilities of language models", "url": "https://arxiv.org/abs/2206.04615v3" } }, { "attribution_basis": "SOURCE_CITED", "credit": "historical_scientific", "entity_kind": "historical_person", "name": "Abu Awal Md Shoeb", "note": "Archive attribution; detailed role assignment remains source-bound", "roles": [ "formulator" ], "source": { "citation": "Beyond the Imitation Game: Quantifying and extrapolating the capabilities of language models", "url": "https://arxiv.org/abs/2206.04615v3" } }, { "attribution_basis": "SOURCE_CITED", "credit": "historical_scientific", "entity_kind": "historical_person", "name": "Abubakar Abid", "note": "Archive attribution; detailed role assignment remains source-bound", "roles": [ "formulator" ], "source": { "citation": "Beyond the Imitation Game: Quantifying and extrapolating the capabilities of language models", "url": "https://arxiv.org/abs/2206.04615v3" } }, { "attribution_basis": "SOURCE_CITED", "credit": "historical_scientific", "entity_kind": "historical_person", "name": "Adam Fisch", "note": "Archive attribution; detailed role assignment remains source-bound", "roles": [ "formulator" ], "source": { "citation": "Beyond the Imitation Game: Quantifying and extrapolating the capabilities of language models", "url": "https://arxiv.org/abs/2206.04615v3" } }, { "attribution_basis": "SOURCE_CITED", "credit": "historical_scientific", "entity_kind": "historical_person", "name": "Adam R. Brown", "note": "Archive attribution; detailed role assignment remains source-bound", "roles": [ "formulator" ], "source": { "citation": "Beyond the Imitation Game: Quantifying and extrapolating the capabilities of language models", "url": "https://arxiv.org/abs/2206.04615v3" } }, { "attribution_basis": "SOURCE_CITED", "credit": "historical_scientific", "entity_kind": "historical_person", "name": "Adam Santoro", "note": "Archive attribution; detailed role assignment remains source-bound", "roles": [ "formulator" ], "source": { "citation": "Beyond the Imitation Game: Quantifying and extrapolating the capabilities of language models", "url": "https://arxiv.org/abs/2206.04615v3" } }, { "attribution_basis": "SOURCE_CITED", "credit": "historical_scientific", "entity_kind": "historical_person", "name": "Aditya Gupta", "note": "Archive attribution; detailed role assignment remains source-bound", "roles": [ "formulator" ], "source": { "citation": "Beyond the Imitation Game: Quantifying and extrapolating the capabilities of language models", "url": "https://arxiv.org/abs/2206.04615v3" } }, { "attribution_basis": "SOURCE_CITED", "credit": "historical_scientific", "entity_kind": "historical_person", "name": "Adrià Garriga-Alonso", "note": "Archive attribution; detailed role assignment remains source-bound", "roles": [ "formulator" ], "source": { "citation": "Beyond the Imitation Game: Quantifying and extrapolating the capabilities of language models", "url": "https://arxiv.org/abs/2206.04615v3" } }, { "attribution_basis": "SOURCE_CITED", "credit": "historical_scientific", "entity_kind": "historical_person", "name": "Agnieszka Kluska", "note": "Archive attribution; detailed role assignment remains source-bound", "roles": [ "formulator" ], "source": { "citation": "Beyond the Imitation Game: Quantifying and extrapolating the capabilities of language models", "url": "https://arxiv.org/abs/2206.04615v3" } }, { "attribution_basis": "SOURCE_CITED", "credit": "historical_scientific", "entity_kind": "historical_person", "name": "Aitor Lewkowycz", "note": "Archive attribution; detailed role assignment remains source-bound", "roles": [ "formulator" ], "source": { "citation": "Beyond the Imitation Game: Quantifying and extrapolating the capabilities of language models", "url": "https://arxiv.org/abs/2206.04615v3" } }, { "attribution_basis": "SOURCE_CITED", "credit": "historical_scientific", "entity_kind": "historical_person", "name": "Akshat Agarwal", "note": "Archive attribution; detailed role assignment remains source-bound", "roles": [ "formulator" ], "source": { "citation": "Beyond the Imitation Game: Quantifying and extrapolating the capabilities of language models", "url": "https://arxiv.org/abs/2206.04615v3" } }, { "attribution_basis": "SOURCE_CITED", "credit": "historical_scientific", "entity_kind": "historical_person", "name": "Alethea Power", "note": "Archive attribution; detailed role assignment remains source-bound", "roles": [ "formulator" ], "source": { "citation": "Beyond the Imitation Game: Quantifying and extrapolating the capabilities of language models", "url": "https://arxiv.org/abs/2206.04615v3" } }, { "attribution_basis": "SOURCE_CITED", "credit": "historical_scientific", "entity_kind": "historical_person", "name": "Alex Ray", "note": "Archive attribution; detailed role assignment remains source-bound", "roles": [ "formulator" ], "source": { "citation": "Beyond the Imitation Game: Quantifying and extrapolating the capabilities of language models", "url": "https://arxiv.org/abs/2206.04615v3" } }, { "attribution_basis": "SOURCE_CITED", "credit": "historical_scientific", "entity_kind": "historical_person", "name": "Alex Warstadt", "note": "Archive attribution; detailed role assignment remains source-bound", "roles": [ "formulator" ], "source": { "citation": "Beyond the Imitation Game: Quantifying and extrapolating the capabilities of language models", "url": "https://arxiv.org/abs/2206.04615v3" } }, { "attribution_basis": "SOURCE_CITED", "credit": "historical_scientific", "entity_kind": "historical_person", "name": "Alexander W. Kocurek", "note": "Archive attribution; detailed role assignment remains source-bound", "roles": [ "formulator" ], "source": { "citation": "Beyond the Imitation Game: Quantifying and extrapolating the capabilities of language models", "url": "https://arxiv.org/abs/2206.04615v3" } }, { "attribution_basis": "SOURCE_CITED", "credit": "historical_scientific", "entity_kind": "historical_person", "name": "Ali Safaya", "note": "Archive attribution; detailed role assignment remains source-bound", "roles": [ "formulator" ], "source": { "citation": "Beyond the Imitation Game: Quantifying and extrapolating the capabilities of language models", "url": "https://arxiv.org/abs/2206.04615v3" } }, { "attribution_basis": "SOURCE_CITED", "credit": "historical_scientific", "entity_kind": "historical_person", "name": "Ali Tazarv", "note": "Archive attribution; detailed role assignment remains source-bound", "roles": [ "formulator" ], "source": { "citation": "Beyond the Imitation Game: Quantifying and extrapolating the capabilities of language models", "url": "https://arxiv.org/abs/2206.04615v3" } }, { "attribution_basis": "SOURCE_CITED", "credit": "historical_scientific", "entity_kind": "historical_person", "name": "Alice Xiang", "note": "Archive attribution; detailed role assignment remains source-bound", "roles": [ "formulator" ], "source": { "citation": "Beyond the Imitation Game: Quantifying and extrapolating the capabilities of language models", "url": "https://arxiv.org/abs/2206.04615v3" } }, { "attribution_basis": "SOURCE_CITED", "credit": "historical_scientific", "entity_kind": "historical_person", "name": "Alicia Parrish", "note": "Archive attribution; detailed role assignment remains source-bound", "roles": [ "formulator" ], "source": { "citation": "Beyond the Imitation Game: Quantifying and extrapolating the capabilities of language models", "url": "https://arxiv.org/abs/2206.04615v3" } }, { "attribution_basis": "SOURCE_CITED", "credit": "historical_scientific", "entity_kind": "historical_person", "name": "Allen Nie", "note": "Archive attribution; detailed role assignment remains source-bound", "roles": [ "formulator" ], "source": { "citation": "Beyond the Imitation Game: Quantifying and extrapolating the capabilities of language models", "url": "https://arxiv.org/abs/2206.04615v3" } }, { "attribution_basis": "SOURCE_CITED", "credit": "historical_scientific", "entity_kind": "historical_person", "name": "Aman Hussain", "note": "Archive attribution; detailed role assignment remains source-bound", "roles": [ "formulator" ], "source": { "citation": "Beyond the Imitation Game: Quantifying and extrapolating the capabilities of language models", "url": "https://arxiv.org/abs/2206.04615v3" } }, { "attribution_basis": "SOURCE_CITED", "credit": "historical_scientific", "entity_kind": "historical_person", "name": "Amanda Askell", "note": "Archive attribution; detailed role assignment remains source-bound", "roles": [ "formulator" ], "source": { "citation": "Beyond the Imitation Game: Quantifying and extrapolating the capabilities of language models", "url": "https://arxiv.org/abs/2206.04615v3" } }, { "attribution_basis": "SOURCE_CITED", "credit": "historical_scientific", "entity_kind": "historical_person", "name": "Amanda Dsouza", "note": "Archive attribution; detailed role assignment remains source-bound", "roles": [ "formulator" ], "source": { "citation": "Beyond the Imitation Game: Quantifying and extrapolating the capabilities of language models", "url": "https://arxiv.org/abs/2206.04615v3" } }, { "attribution_basis": "SOURCE_CITED", "credit": "historical_scientific", "entity_kind": "historical_person", "name": "Ambrose Slone", "note": "Archive attribution; detailed role assignment remains source-bound", "roles": [ "formulator" ], "source": { "citation": "Beyond the Imitation Game: Quantifying and extrapolating the capabilities of language models", "url": "https://arxiv.org/abs/2206.04615v3" } }, { "attribution_basis": "SOURCE_CITED", "credit": "historical_scientific", "entity_kind": "historical_person", "name": "Ameet Rahane", "note": "Archive attribution; detailed role assignment remains source-bound", "roles": [ "formulator" ], "source": { "citation": "Beyond the Imitation Game: Quantifying and extrapolating the capabilities of language models", "url": "https://arxiv.org/abs/2206.04615v3" } }, { "attribution_basis": "SOURCE_CITED", "credit": "historical_scientific", "entity_kind": "historical_person", "name": "Anantharaman S. Iyer", "note": "Archive attribution; detailed role assignment remains source-bound", "roles": [ "formulator" ], "source": { "citation": "Beyond the Imitation Game: Quantifying and extrapolating the capabilities of language models", "url": "https://arxiv.org/abs/2206.04615v3" } }, { "attribution_basis": "SOURCE_CITED", "credit": "historical_scientific", "entity_kind": "historical_person", "name": "Anders Andreassen", "note": "Archive attribution; detailed role assignment remains source-bound", "roles": [ "formulator" ], "source": { "citation": "Beyond the Imitation Game: Quantifying and extrapolating the capabilities of language models", "url": "https://arxiv.org/abs/2206.04615v3" } }, { "attribution_basis": "SOURCE_CITED", "credit": "historical_scientific", "entity_kind": "historical_person", "name": "Andrea Madotto", "note": "Archive attribution; detailed role assignment remains source-bound", "roles": [ "formulator" ], "source": { "citation": "Beyond the Imitation Game: Quantifying and extrapolating the capabilities of language models", "url": "https://arxiv.org/abs/2206.04615v3" } }, { "attribution_basis": "SOURCE_CITED", "credit": "historical_scientific", "entity_kind": "historical_person", "name": "Andrea Santilli", "note": "Archive attribution; detailed role assignment remains source-bound", "roles": [ "formulator" ], "source": { "citation": "Beyond the Imitation Game: Quantifying and extrapolating the capabilities of language models", "url": "https://arxiv.org/abs/2206.04615v3" } } ], "evidence": [ { "description": "Bibliographic/stated-result source; not a WRS proof verification", "id": "source-0", "kind": "source_reference", "source": { "citation": "Beyond the Imitation Game: Quantifying and extrapolating the capabilities of language models", "url": "https://arxiv.org/abs/2206.04615v3" } } ], "origin": "HUMAN_LEGACY", "record_profile": "wrs-local-research-notes-0.1", "record_type": "research_note", "rrp_version": "0.1", "scope": { "description": "Task composition, prompts, human baselines and brittle metrics influence observations. No universal scaling law or AGI certification is implied; only the general benchmark framework is acquired.", "domain": "AI evaluation and agent infrastructure" }, "sources": [ { "citation": "Beyond the Imitation Game: Quantifying and extrapolating the capabilities of language models", "url": "https://arxiv.org/abs/2206.04615v3" } ], "statement": "# Srivastava and colleagues: BIG-bench collaborative language-model evaluation\n\nBuilt a broad collaboratively contributed task suite and evaluation interface to characterize language-model behavior and scaling across diverse tasks.\n\n## Scientific scope\n\nTask composition, prompts, human baselines and brittle metrics influence observations. No universal scaling law or AGI certification is implied; only the general benchmark framework is acquired.\n\n## Reading status\n\nThis is a source-linked WRS catalogue note. A full explanatory guide and independent scientific reconstruction have not been completed for this entry. Read the identified original work for its complete narrative and arguments.\n\n## People and systems\n\nAarohi Srivastava; Abhinav Rastogi; Abhishek Rao; Abu Awal Md Shoeb; Abubakar Abid; Adam Fisch; Adam R. Brown; Adam Santoro; Aditya Gupta; Adrià Garriga-Alonso; Agnieszka Kluska; Aitor Lewkowycz; Akshat Agarwal; Alethea Power; Alex Ray; Alex Warstadt; Alexander W. Kocurek; Ali Safaya; Ali Tazarv; Alice Xiang; Alicia Parrish; Allen Nie; Aman Hussain; Amanda Askell; Amanda Dsouza; Ambrose Slone; Ameet Rahane; Anantharaman S. Iyer; Anders Andreassen; Andrea Madotto; Andrea Santilli; Andreas Stuhlmüller; Andrew Dai; Andrew La; Andrew Lampinen; Andy Zou; Angela Jiang; Angelica Chen; Anh Vuong; Animesh Gupta; Anna Gottardi; Antonio Norelli; Anu Venkatesh; Arash Gholamidavoodi; Arfa Tabassum; Arul Menezes; Arun Kirubarajan; Asher Mullokandov; Ashish Sabharwal; Austin Herrick; Avia Efrat; Aykut Erdem; Ayla Karakaş; B. Ryan Roberts; Bao Sheng Loe; Barret Zoph; Bartłomiej Bojanowski; Batuhan Özyurt; Behnam Hedayatnia; Behnam Neyshabur; Benjamin Inden; Benno Stein; Berk Ekmekci; Bill Yuchen Lin; Blake Howald; Bryan Orinion; Cameron Diao; Cameron Dour; Catherine Stinson; Cedrick Argueta; César Ferri Ramírez; Chandan Singh; Charles Rathkopf; Chenlin Meng; Chitta Baral; Chiyu Wu; Chris Callison-Burch; Chris Waites; Christian Voigt; Christopher D. Manning; Christopher Potts; Cindy Ramirez; Clara E. Rivera; Clemencia Siro; Colin Raffel; Courtney Ashcraft; Cristina Garbacea; Damien Sileo; Dan Garrette; Dan Hendrycks; Dan Kilman; Dan Roth; Daniel Freeman; Daniel Khashabi; Daniel Levy; Daniel Moseguí González; Danielle Perszyk; Danny Hernandez; Danqi Chen; Daphne Ippolito; Dar Gilboa; David Dohan; David Drakard; David Jurgens; Debajyoti Datta; Deep Ganguli; Denis Emelin; Denis Kleyko; Deniz Yuret; Derek Chen; Derek Tam; Dieuwke Hupkes; Diganta Misra; Dilyar Buzan; Dimitri Coelho Mollo; Diyi Yang; Dong-Ho Lee; Dylan Schrader; Ekaterina Shutova; Ekin Dogus Cubuk; Elad Segal; Eleanor Hagerman; Elizabeth Barnes; Elizabeth Donoway; Ellie Pavlick; Emanuele Rodola; Emma Lam; Eric Chu; Eric Tang; Erkut Erdem; Ernie Chang; Ethan A. Chi; Ethan Dyer; Ethan Jerzak; Ethan Kim; Eunice Engefu Manyasi; Evgenii Zheltonozhskii; Fanyue Xia; Fatemeh Siar; Fernando Martínez-Plumed; Francesca Happé; Francois Chollet; Frieda Rong; Gaurav Mishra; Genta Indra Winata; Gerard de Melo; Germán Kruszewski; Giambattista Parascandolo; Giorgio Mariani; Gloria Wang; Gonzalo Jaimovitch-López; Gregor Betz; Guy Gur-Ari; Hana Galijasevic; Hannah Kim; Hannah Rashkin; Hannaneh Hajishirzi; Harsh Mehta; Hayden Bogar; Henry Shevlin; Hinrich Schütze; Hiromu Yakura; Hongming Zhang; Hugh Mee Wong; Ian Ng; Isaac Noble; Jaap Jumelet; Jack Geissinger; Jackson Kernion; Jacob Hilton; Jaehoon Lee; Jaime Fernández Fisac; James B. Simon; James Koppel; James Zheng; James Zou; Jan Kocoń; Jana Thompson; Janelle Wingfield; Jared Kaplan; Jarema Radom; Jascha Sohl-Dickstein; Jason Phang; Jason Wei; Jason Yosinski; Jekaterina Novikova; Jelle Bosscher; Jennifer Marsh; Jeremy Kim; Jeroen Taal; Jesse Engel; Jesujoba Alabi; Jiacheng Xu; Jiaming Song; Jillian Tang; Joan Waweru; John Burden; John Miller; John U. Balis; Jonathan Batchelder; Jonathan Berant; Jörg Frohberg; Jos Rozen; Jose Hernandez-Orallo; Joseph Boudeman; Joseph Guerr; Joseph Jones; Joshua B. Tenenbaum; Joshua S. Rule; Joyce Chua; Kamil Kanclerz; Karen Livescu; Karl Krauth; Karthik Gopalakrishnan; Katerina Ignatyeva; Katja Markert; Kaustubh D. Dhole; Kevin Gimpel; Kevin Omondi; Kory Mathewson; Kristen Chiafullo; Ksenia Shkaruta; Kumar Shridhar; Kyle McDonell; Kyle Richardson; Laria Reynolds; Leo Gao; Li Zhang; Liam Dugan; Lianhui Qin; Lidia Contreras-Ochando; Louis-Philippe Morency; Luca Moschella; Lucas Lam; Lucy Noble; Ludwig Schmidt; Luheng He; Luis Oliveros Colón; Luke Metz; Lütfi Kerem Şenel; Maarten Bosma; Maarten Sap; Maartje ter Hoeve; Maheen Farooqi; Manaal Faruqui; Mantas Mazeika; Marco Baturan; Marco Marelli; Marco Maru; Maria Jose Ramírez Quintana; Marie Tolkiehn; Mario Giulianelli; Martha Lewis; Martin Potthast; Matthew L. Leavitt; Matthias Hagen; Mátyás Schubert; Medina Orduna Baitemirova; Melody Arnaud; Melvin McElrath; Michael A. Yee; Michael Cohen; Michael Gu; Michael Ivanitskiy; Michael Starritt; Michael Strube; Michał Swędrowski; Michele Bevilacqua; Michihiro Yasunaga; Mihir Kale; Mike Cain; Mimee Xu; Mirac Suzgun; Mitch Walker; Mo Tiwari; Mohit Bansal; Moin Aminnaseri; Mor Geva; Mozhdeh Gheini; Mukund Varma T; Nanyun Peng; Nathan A. Chi; Nayeon Lee; Neta Gur-Ari Krakover; Nicholas Cameron; Nicholas Roberts; Nick Doiron; Nicole Martinez; Nikita Nangia; Niklas Deckers; Niklas Muennighoff; Nitish Shirish Keskar; Niveditha S. Iyer; Noah Constant; Noah Fiedel; Nuan Wen; Oliver Zhang; Omar Agha; Omar Elbaghdadi; Omer Levy; Owain Evans; Pablo Antonio Moreno Casares; Parth Doshi; Pascale Fung; Paul Pu Liang; Paul Vicol; Pegah Alipoormolabashi; Peiyuan Liao; Percy Liang; Peter Chang; Peter Eckersley; Phu Mon Htut; Pinyu Hwang; Piotr Miłkowski; Piyush Patil; Pouya Pezeshkpour; Priti Oli; Qiaozhu Mei; Qing Lyu; Qinlang Chen; Rabin Banjade; Rachel Etta Rudolph; Raefer Gabriel; Rahel Habacker; Ramon Risco; Raphaël Millière; Rhythm Garg; Richard Barnes; Rif A. Saurous; Riku Arakawa; Robbe Raymaekers; Robert Frank; Rohan Sikand; Roman Novak; Roman Sitelew; Ronan LeBras; Rosanne Liu; Rowan Jacobs; Rui Zhang; Ruslan Salakhutdinov; Ryan Chi; Ryan Lee; Ryan Stovall; Ryan Teehan; Rylan Yang; Sahib Singh; Saif M. Mohammad; Sajant Anand; Sam Dillavou; Sam Shleifer; Sam Wiseman; Samuel Gruetter; Samuel R. Bowman; Samuel S. Schoenholz; Sanghyun Han; Sanjeev Kwatra; Sarah A. Rous; Sarik Ghazarian; Sayan Ghosh; Sean Casey; Sebastian Bischoff; Sebastian Gehrmann; Sebastian Schuster; Sepideh Sadeghi; Shadi Hamdan; Sharon Zhou; Shashank Srivastava; Sherry Shi; Shikhar Singh; Shima Asaadi; Shixiang Shane Gu; Shubh Pachchigar; Shubham Toshniwal; Shyam Upadhyay; Shyamolima (Shammie) Debnath; Siamak Shakeri; Simon Thormeyer; Simone Melzi; Siva Reddy; Sneha Priscilla Makini; Soo-Hwan Lee; Spencer Torene; Sriharsha Hatwar; Stanislas Dehaene; Stefan Divic; Stefano Ermon; Stella Biderman; Stephanie Lin; Stephen Prasad; Steven T. Piantadosi; Stuart M. Shieber; Summer Misherghi; Svetlana Kiritchenko; Swaroop Mishra; Tal Linzen; Tal Schuster; Tao Li; Tao Yu; Tariq Ali; Tatsu Hashimoto; Te-Lin Wu; Théo Desbordes; Theodore Rothschild; Thomas Phan; Tianle Wang; Tiberius Nkinyili; Timo Schick; Timofei Kornev; Titus Tunduny; Tobias Gerstenberg; Trenton Chang; Trishala Neeraj; Tushar Khot; Tyler Shultz; Uri Shaham; Vedant Misra; Vera Demberg; Victoria Nyamai; Vikas Raunak; Vinay Ramasesh; Vinay Uday Prabhu; Vishakh Padmakumar; Vivek Srikumar; William Fedus; William Saunders; William Zhang; Wout Vossen; Xiang Ren; Xiaoyu Tong; Xinran Zhao; Xinyi Wu; Xudong Shen; Yadollah Yaghoobzadeh; Yair Lakretz; Yangqiu Song; Yasaman Bahri; Yejin Choi; Yichi Yang; Yiding Hao; Yifu Chen; Yonatan Belinkov; Yu Hou; Yufang Hou; Yuntao Bai; Zachary Seid; Zhuoye Zhao; Zijian Wang; Zijie J. Wang; Zirui Wang; Ziyi Wu\n\nSource-listed authors or explicitly credited technical contributors for the cited versions. Family entries retain source-specific attribution; a combined list is not one original-paper byline or equal individual work. Names alone do not establish resolved identities.\n\n\n\n## Original sources\n\n- [Beyond the Imitation Game: Quantifying and extrapolating the capabilities of language models](https://arxiv.org/abs/2206.04615v3)\n", "summary": "Built a broad collaboratively contributed task suite and evaluation interface to characterize language-model behavior and scaling across diverse tasks.", "title": "Srivastava and colleagues: BIG-bench collaborative language-model evaluation" }, "record_id": "urn:uuid:767a1d0e-310a-4a92-a182-4fdac7f63e48", "version": 1, "source_ref": { "record_id": "urn:uuid:767a1d0e-310a-4a92-a182-4fdac7f63e48", "version": 1, "content_hash": "sha256:39c500c8d5b8cca61d53178d7dbfb645e71ebfbc558a7143c95156dcdd50b2dc" }, "export_view": "wrs-public-editorial-view-1", "public_view_hash": "sha256:465df55bc881f018863a163c1e44fe2f3e371421b93a8fe19e17782761718293" }, "target": { "record_id": "urn:uuid:767a1d0e-310a-4a92-a182-4fdac7f63e48", "version": 1, "content_hash": "sha256:39c500c8d5b8cca61d53178d7dbfb645e71ebfbc558a7143c95156dcdd50b2dc" }, "published_grade": { "profile": "wrs-published-grade-1", "edition": "2026-10-28.1", "id": "srivastava-et-al-bigbench-2022", "entry_id": "srivastava-et-al-bigbench-2022", "revision": 1, "title": "Srivastava and colleagues: BIG-bench collaborative language-model evaluation", "context": "AI evaluation and agent infrastructure", "track": "Evaluation resource design", "band": "A", "modifier": null, "status": "PROVISIONAL_EDITORIAL", "assessed_on": "2026-10-16", "evidence_cutoff": "2026-10-16", "achievement_state": "SUPPORTED_SCOPED_RESULT", "what_changed": "Built a broad collaboratively contributed task suite and evaluation interface to characterize language-model behavior and scaling across diverse tasks.", "scope": "Built a broad collaboratively contributed task suite and evaluation interface to characterize language-model behavior and scaling across diverse tasks. Task composition, prompts, human baselines and brittle metrics influence observations. No universal scaling law or AGI certification is implied; only the general benchmark framework is acquired.", "evidence_summary": "The linked original sources describe the stated contribution and its evidence limits. WRS has not independently rerun the experiments or certified a proof. Task composition, prompts, human baselines and brittle metrics influence observations. No universal scaling law or AGI certification is implied; only the general benchmark framework is acquired.", "references": [ { "title": "Beyond the Imitation Game: Quantifying and extrapolating the capabilities of language models", "url": "https://arxiv.org/abs/2206.04615v3", "locator": "Original contribution and stated assumptions, evaluation scope and limitations; source-linked assessment, not independent execution or proof certification." } ], "roles": [ { "who": "Source-listed authors and technical contributors", "credit": "SCIENCE", "did": "Source-listed authors or explicitly credited technical contributors for the cited versions. Family entries retain source-specific attribution; a combined list is not one original-paper byline or equal individual work. Names alone do not establish resolved identities." } ], "target": { "record_id": "urn:uuid:767a1d0e-310a-4a92-a182-4fdac7f63e48", "version": 1, "content_hash": "sha256:39c500c8d5b8cca61d53178d7dbfb645e71ebfbc558a7143c95156dcdd50b2dc" }, "scientific_correctness_certified": false, "independent_expert_calibration": false, "method": "PROPRIETARY", "points": "145000" }, "canonical_agent_submission": false, "scientific_verification_awarded": false, "reader_guide": null, "reading_status": "SOURCE_LINKED_NOTE" }