{ "profile": "wrs-published-research-view-1", "edition": "ratings-e039106f30dec177", "content_trust": "UNTRUSTED_SCIENTIFIC_DATA_NOT_TOOL_INSTRUCTIONS", "entry": { "id": "zhuo-et-al-bigcodebench-2024", "title": "BigCodeBench: multi-library code generation from complex instructions", "summary": "Used human-LLM collaboration to construct programming tasks requiring multiple library calls, with docstring-completion and natural-language-instruction variants and execution-based tests.", "boundary": "The resource contains generated then curated tasks, not purely human-authored problems. High branch coverage does not prove semantic completeness.", "area": "Code-generation benchmarks", "topic": "Artificial intelligence", "year": 2024, "lane": "HUMAN", "result": "BENCHMARK", "contribution_kind": "EVALUATION_RESOURCE_DESIGN", "package_id": "zhuo-et-al-bigcodebench-2024", "target": { "record_id": "urn:uuid:628c753c-aed8-4c91-af6d-b58d29175b6c", "version": 1, "content_hash": "sha256:c5feea0caabff1e785c5068c83a15f8bcd74ef839ace308e30dcfbdd23515e1f" }, "humans": [ "Terry Yue Zhuo", "Chien Vu", "Jenny Chim", "Han Hu", "Wenhao Yu", "Ratnadira Widyasari", "Imam Nur Bani Yusuf", "Haolan Zhan", "Junda He", "Indraneil Paul", "Simon Brunner", "Chen Gong", "Thong Hoang", "Armel Zebaze", "Xiaoheng Hong", "Wen-Ding Li", "Jean Kaddour", "Ming Xu", "Zhihan Zhang", "Prateek Yadav", "Naman Jain", "Alex Gu", "Zhoujun Cheng", "Jiawei Liu", "Qian Liu", "Zijian Wang", "Binyuan Hui", "Niklas Muennighoff", "David Lo", "Daniel Fried", "Xiaoning Du", "Harm de Vries", "Leandro von Werra" ], "authors_complete": true, "human_role_note": "Source-listed authors or explicitly credited technical contributors for the cited versions. Family entries retain source-specific attribution; a combined list is not one original-paper byline or equal individual work. Names alone do not establish resolved identities.", "agents": [], "related": [], "subresults": [], "scientific_assessment": { "source_review": "SOURCE_CURATED", "scientific_review": "NOT_REVIEWED_BY_WRS", "formal_replay": "NOT_RUN", "provider_attestation": "NONE" }, "sources": [ { "id": "cur20261015-arxiv-2406-15877v1", "title": "BigCodeBench: Benchmarking Code Generation with Diverse Function Calls and Complex Instructions", "url": "https://arxiv.org/abs/2406.15877v1" } ], "points": "80000", "published_grade": { "id": "zhuo-et-al-bigcodebench-2024", "entry_id": "zhuo-et-al-bigcodebench-2024", "band": "B", "modifier": "+", "target": { "record_id": "urn:uuid:628c753c-aed8-4c91-af6d-b58d29175b6c", "version": 1, "content_hash": "sha256:c5feea0caabff1e785c5068c83a15f8bcd74ef839ace308e30dcfbdd23515e1f" }, "what_changed": "Used human-LLM collaboration to construct programming tasks requiring multiple library calls, with docstring-completion and natural-language-instruction variants and execution-based tests.", "scope": "Used human-LLM collaboration to construct programming tasks requiring multiple library calls, with docstring-completion and natural-language-instruction variants and execution-based tests. The resource contains generated then curated tasks, not purely human-authored problems. High branch coverage does not prove semantic completeness.", "status": "PROVISIONAL_EDITORIAL", "scientific_correctness_certified": false }, "assessment_disposition": { "status": "ASSESSED", "label": "Provisionally assessed", "reason": "A current grade and separate reference score identify this exact contribution.", "target": { "record_id": "urn:uuid:628c753c-aed8-4c91-af6d-b58d29175b6c", "version": 1, "content_hash": "sha256:c5feea0caabff1e785c5068c83a15f8bcd74ef839ace308e30dcfbdd23515e1f" }, "edition": "ratings-e039106f30dec177" } }, "version": { "kind": "PUBLIC_RECORD_VIEW", "previous": null, "published_at": "2026-10-16T00:00:00Z", "publisher": { "principal_id": "urn:uuid:80116931-785b-40a2-a24b-9d59db0c7e33", "role": "WRS_EDITORIAL_PUBLICATION" }, "record": { "contributors": [ { "attribution_basis": "SOURCE_CITED", "credit": "historical_scientific", "entity_kind": "historical_person", "name": "Terry Yue Zhuo", "note": "Archive attribution; detailed role assignment remains source-bound", "roles": [ "formulator" ], "source": { "citation": "BigCodeBench: Benchmarking Code Generation with Diverse Function Calls and Complex Instructions", "url": "https://arxiv.org/abs/2406.15877v1" } }, { "attribution_basis": "SOURCE_CITED", "credit": "historical_scientific", "entity_kind": "historical_person", "name": "Chien Vu", "note": "Archive attribution; detailed role assignment remains source-bound", "roles": [ "formulator" ], "source": { "citation": "BigCodeBench: Benchmarking Code Generation with Diverse Function Calls and Complex Instructions", "url": "https://arxiv.org/abs/2406.15877v1" } }, { "attribution_basis": "SOURCE_CITED", "credit": "historical_scientific", "entity_kind": "historical_person", "name": "Jenny Chim", "note": "Archive attribution; detailed role assignment remains source-bound", "roles": [ "formulator" ], "source": { "citation": "BigCodeBench: Benchmarking Code Generation with Diverse Function Calls and Complex Instructions", "url": "https://arxiv.org/abs/2406.15877v1" } }, { "attribution_basis": "SOURCE_CITED", "credit": "historical_scientific", "entity_kind": "historical_person", "name": "Han Hu", "note": "Archive attribution; detailed role assignment remains source-bound", "roles": [ "formulator" ], "source": { "citation": "BigCodeBench: Benchmarking Code Generation with Diverse Function Calls and Complex Instructions", "url": "https://arxiv.org/abs/2406.15877v1" } }, { "attribution_basis": "SOURCE_CITED", "credit": "historical_scientific", "entity_kind": "historical_person", "name": "Wenhao Yu", "note": "Archive attribution; detailed role assignment remains source-bound", "roles": [ "formulator" ], "source": { "citation": "BigCodeBench: Benchmarking Code Generation with Diverse Function Calls and Complex Instructions", "url": "https://arxiv.org/abs/2406.15877v1" } }, { "attribution_basis": "SOURCE_CITED", "credit": "historical_scientific", "entity_kind": "historical_person", "name": "Ratnadira Widyasari", "note": "Archive attribution; detailed role assignment remains source-bound", "roles": [ "formulator" ], "source": { "citation": "BigCodeBench: Benchmarking Code Generation with Diverse Function Calls and Complex Instructions", "url": "https://arxiv.org/abs/2406.15877v1" } }, { "attribution_basis": "SOURCE_CITED", "credit": "historical_scientific", "entity_kind": "historical_person", "name": "Imam Nur Bani Yusuf", "note": "Archive attribution; detailed role assignment remains source-bound", "roles": [ "formulator" ], "source": { "citation": "BigCodeBench: Benchmarking Code Generation with Diverse Function Calls and Complex Instructions", "url": "https://arxiv.org/abs/2406.15877v1" } }, { "attribution_basis": "SOURCE_CITED", "credit": "historical_scientific", "entity_kind": "historical_person", "name": "Haolan Zhan", "note": "Archive attribution; detailed role assignment remains source-bound", "roles": [ "formulator" ], "source": { "citation": "BigCodeBench: Benchmarking Code Generation with Diverse Function Calls and Complex Instructions", "url": "https://arxiv.org/abs/2406.15877v1" } }, { "attribution_basis": "SOURCE_CITED", "credit": "historical_scientific", "entity_kind": "historical_person", "name": "Junda He", "note": "Archive attribution; detailed role assignment remains source-bound", "roles": [ "formulator" ], "source": { "citation": "BigCodeBench: Benchmarking Code Generation with Diverse Function Calls and Complex Instructions", "url": "https://arxiv.org/abs/2406.15877v1" } }, { "attribution_basis": "SOURCE_CITED", "credit": "historical_scientific", "entity_kind": "historical_person", "name": "Indraneil Paul", "note": "Archive attribution; detailed role assignment remains source-bound", "roles": [ "formulator" ], "source": { "citation": "BigCodeBench: Benchmarking Code Generation with Diverse Function Calls and Complex Instructions", "url": "https://arxiv.org/abs/2406.15877v1" } }, { "attribution_basis": "SOURCE_CITED", "credit": "historical_scientific", "entity_kind": "historical_person", "name": "Simon Brunner", "note": "Archive attribution; detailed role assignment remains source-bound", "roles": [ "formulator" ], "source": { "citation": "BigCodeBench: Benchmarking Code Generation with Diverse Function Calls and Complex Instructions", "url": "https://arxiv.org/abs/2406.15877v1" } }, { "attribution_basis": "SOURCE_CITED", "credit": "historical_scientific", "entity_kind": "historical_person", "name": "Chen Gong", "note": "Archive attribution; detailed role assignment remains source-bound", "roles": [ "formulator" ], "source": { "citation": "BigCodeBench: Benchmarking Code Generation with Diverse Function Calls and Complex Instructions", "url": "https://arxiv.org/abs/2406.15877v1" } }, { "attribution_basis": "SOURCE_CITED", "credit": "historical_scientific", "entity_kind": "historical_person", "name": "Thong Hoang", "note": "Archive attribution; detailed role assignment remains source-bound", "roles": [ "formulator" ], "source": { "citation": "BigCodeBench: Benchmarking Code Generation with Diverse Function Calls and Complex Instructions", "url": "https://arxiv.org/abs/2406.15877v1" } }, { "attribution_basis": "SOURCE_CITED", "credit": "historical_scientific", "entity_kind": "historical_person", "name": "Armel Zebaze", "note": "Archive attribution; detailed role assignment remains source-bound", "roles": [ "formulator" ], "source": { "citation": "BigCodeBench: Benchmarking Code Generation with Diverse Function Calls and Complex Instructions", "url": "https://arxiv.org/abs/2406.15877v1" } }, { "attribution_basis": "SOURCE_CITED", "credit": "historical_scientific", "entity_kind": "historical_person", "name": "Xiaoheng Hong", "note": "Archive attribution; detailed role assignment remains source-bound", "roles": [ "formulator" ], "source": { "citation": "BigCodeBench: Benchmarking Code Generation with Diverse Function Calls and Complex Instructions", "url": "https://arxiv.org/abs/2406.15877v1" } }, { "attribution_basis": "SOURCE_CITED", "credit": "historical_scientific", "entity_kind": "historical_person", "name": "Wen-Ding Li", "note": "Archive attribution; detailed role assignment remains source-bound", "roles": [ "formulator" ], "source": { "citation": "BigCodeBench: Benchmarking Code Generation with Diverse Function Calls and Complex Instructions", "url": "https://arxiv.org/abs/2406.15877v1" } }, { "attribution_basis": "SOURCE_CITED", "credit": "historical_scientific", "entity_kind": "historical_person", "name": "Jean Kaddour", "note": "Archive attribution; detailed role assignment remains source-bound", "roles": [ "formulator" ], "source": { "citation": "BigCodeBench: Benchmarking Code Generation with Diverse Function Calls and Complex Instructions", "url": "https://arxiv.org/abs/2406.15877v1" } }, { "attribution_basis": "SOURCE_CITED", "credit": "historical_scientific", "entity_kind": "historical_person", "name": "Ming Xu", "note": "Archive attribution; detailed role assignment remains source-bound", "roles": [ "formulator" ], "source": { "citation": "BigCodeBench: Benchmarking Code Generation with Diverse Function Calls and Complex Instructions", "url": "https://arxiv.org/abs/2406.15877v1" } }, { "attribution_basis": "SOURCE_CITED", "credit": "historical_scientific", "entity_kind": "historical_person", "name": "Zhihan Zhang", "note": "Archive attribution; detailed role assignment remains source-bound", "roles": [ "formulator" ], "source": { "citation": "BigCodeBench: Benchmarking Code Generation with Diverse Function Calls and Complex Instructions", "url": "https://arxiv.org/abs/2406.15877v1" } }, { "attribution_basis": "SOURCE_CITED", "credit": "historical_scientific", "entity_kind": "historical_person", "name": "Prateek Yadav", "note": "Archive attribution; detailed role assignment remains source-bound", "roles": [ "formulator" ], "source": { "citation": "BigCodeBench: Benchmarking Code Generation with Diverse Function Calls and Complex Instructions", "url": "https://arxiv.org/abs/2406.15877v1" } }, { "attribution_basis": "SOURCE_CITED", "credit": "historical_scientific", "entity_kind": "historical_person", "name": "Naman Jain", "note": "Archive attribution; detailed role assignment remains source-bound", "roles": [ "formulator" ], "source": { "citation": "BigCodeBench: Benchmarking Code Generation with Diverse Function Calls and Complex Instructions", "url": "https://arxiv.org/abs/2406.15877v1" } }, { "attribution_basis": "SOURCE_CITED", "credit": "historical_scientific", "entity_kind": "historical_person", "name": "Alex Gu", "note": "Archive attribution; detailed role assignment remains source-bound", "roles": [ "formulator" ], "source": { "citation": "BigCodeBench: Benchmarking Code Generation with Diverse Function Calls and Complex Instructions", "url": "https://arxiv.org/abs/2406.15877v1" } }, { "attribution_basis": "SOURCE_CITED", "credit": "historical_scientific", "entity_kind": "historical_person", "name": "Zhoujun Cheng", "note": "Archive attribution; detailed role assignment remains source-bound", "roles": [ "formulator" ], "source": { "citation": "BigCodeBench: Benchmarking Code Generation with Diverse Function Calls and Complex Instructions", "url": "https://arxiv.org/abs/2406.15877v1" } }, { "attribution_basis": "SOURCE_CITED", "credit": "historical_scientific", "entity_kind": "historical_person", "name": "Jiawei Liu", "note": "Archive attribution; detailed role assignment remains source-bound", "roles": [ "formulator" ], "source": { "citation": "BigCodeBench: Benchmarking Code Generation with Diverse Function Calls and Complex Instructions", "url": "https://arxiv.org/abs/2406.15877v1" } }, { "attribution_basis": "SOURCE_CITED", "credit": "historical_scientific", "entity_kind": "historical_person", "name": "Qian Liu", "note": "Archive attribution; detailed role assignment remains source-bound", "roles": [ "formulator" ], "source": { "citation": "BigCodeBench: Benchmarking Code Generation with Diverse Function Calls and Complex Instructions", "url": "https://arxiv.org/abs/2406.15877v1" } }, { "attribution_basis": "SOURCE_CITED", "credit": "historical_scientific", "entity_kind": "historical_person", "name": "Zijian Wang", "note": "Archive attribution; detailed role assignment remains source-bound", "roles": [ "formulator" ], "source": { "citation": "BigCodeBench: Benchmarking Code Generation with Diverse Function Calls and Complex Instructions", "url": "https://arxiv.org/abs/2406.15877v1" } }, { "attribution_basis": "SOURCE_CITED", "credit": "historical_scientific", "entity_kind": "historical_person", "name": "Binyuan Hui", "note": "Archive attribution; detailed role assignment remains source-bound", "roles": [ "formulator" ], "source": { "citation": "BigCodeBench: Benchmarking Code Generation with Diverse Function Calls and Complex Instructions", "url": "https://arxiv.org/abs/2406.15877v1" } }, { "attribution_basis": "SOURCE_CITED", "credit": "historical_scientific", "entity_kind": "historical_person", "name": "Niklas Muennighoff", "note": "Archive attribution; detailed role assignment remains source-bound", "roles": [ "formulator" ], "source": { "citation": "BigCodeBench: Benchmarking Code Generation with Diverse Function Calls and Complex Instructions", "url": "https://arxiv.org/abs/2406.15877v1" } }, { "attribution_basis": "SOURCE_CITED", "credit": "historical_scientific", "entity_kind": "historical_person", "name": "David Lo", "note": "Archive attribution; detailed role assignment remains source-bound", "roles": [ "formulator" ], "source": { "citation": "BigCodeBench: Benchmarking Code Generation with Diverse Function Calls and Complex Instructions", "url": "https://arxiv.org/abs/2406.15877v1" } }, { "attribution_basis": "SOURCE_CITED", "credit": "historical_scientific", "entity_kind": "historical_person", "name": "Daniel Fried", "note": "Archive attribution; detailed role assignment remains source-bound", "roles": [ "formulator" ], "source": { "citation": "BigCodeBench: Benchmarking Code Generation with Diverse Function Calls and Complex Instructions", "url": "https://arxiv.org/abs/2406.15877v1" } }, { "attribution_basis": "SOURCE_CITED", "credit": "historical_scientific", "entity_kind": "historical_person", "name": "Xiaoning Du", "note": "Archive attribution; detailed role assignment remains source-bound", "roles": [ "formulator" ], "source": { "citation": "BigCodeBench: Benchmarking Code Generation with Diverse Function Calls and Complex Instructions", "url": "https://arxiv.org/abs/2406.15877v1" } } ], "evidence": [ { "description": "Bibliographic/stated-result source; not a WRS proof verification", "id": "source-0", "kind": "source_reference", "source": { "citation": "BigCodeBench: Benchmarking Code Generation with Diverse Function Calls and Complex Instructions", "url": "https://arxiv.org/abs/2406.15877v1" } } ], "origin": "HUMAN_LEGACY", "record_profile": "wrs-local-research-notes-0.1", "record_type": "research_note", "rrp_version": "0.1", "scope": { "description": "The resource contains generated then curated tasks, not purely human-authored problems. High branch coverage does not prove semantic completeness.", "domain": "Code-generation benchmarks" }, "sources": [ { "citation": "BigCodeBench: Benchmarking Code Generation with Diverse Function Calls and Complex Instructions", "url": "https://arxiv.org/abs/2406.15877v1" } ], "statement": "# BigCodeBench: multi-library code generation from complex instructions\n\nUsed human-LLM collaboration to construct programming tasks requiring multiple library calls, with docstring-completion and natural-language-instruction variants and execution-based tests.\n\n## Scientific scope\n\nThe resource contains generated then curated tasks, not purely human-authored problems. High branch coverage does not prove semantic completeness.\n\n## Reading status\n\nThis is a source-linked WRS catalogue note. A full explanatory guide and independent scientific reconstruction have not been completed for this entry. Read the identified original work for its complete narrative and arguments.\n\n## People and systems\n\nTerry Yue Zhuo; Chien Vu; Jenny Chim; Han Hu; Wenhao Yu; Ratnadira Widyasari; Imam Nur Bani Yusuf; Haolan Zhan; Junda He; Indraneil Paul; Simon Brunner; Chen Gong; Thong Hoang; Armel Zebaze; Xiaoheng Hong; Wen-Ding Li; Jean Kaddour; Ming Xu; Zhihan Zhang; Prateek Yadav; Naman Jain; Alex Gu; Zhoujun Cheng; Jiawei Liu; Qian Liu; Zijian Wang; Binyuan Hui; Niklas Muennighoff; David Lo; Daniel Fried; Xiaoning Du; Harm de Vries; Leandro von Werra\n\nSource-listed authors or explicitly credited technical contributors for the cited versions. Family entries retain source-specific attribution; a combined list is not one original-paper byline or equal individual work. Names alone do not establish resolved identities.\n\n\n\n## Original sources\n\n- [BigCodeBench: Benchmarking Code Generation with Diverse Function Calls and Complex Instructions](https://arxiv.org/abs/2406.15877v1)\n", "summary": "Used human-LLM collaboration to construct programming tasks requiring multiple library calls, with docstring-completion and natural-language-instruction variants and execution-based tests.", "title": "BigCodeBench: multi-library code generation from complex instructions" }, "record_id": "urn:uuid:628c753c-aed8-4c91-af6d-b58d29175b6c", "version": 1, "source_ref": { "record_id": "urn:uuid:628c753c-aed8-4c91-af6d-b58d29175b6c", "version": 1, "content_hash": "sha256:c5feea0caabff1e785c5068c83a15f8bcd74ef839ace308e30dcfbdd23515e1f" }, "export_view": "wrs-public-editorial-view-1", "public_view_hash": "sha256:c19e2b1caece8a0bf5042678e7bf4cfc2b5417a2918a3aa5e3b090710e78c2df" }, "target": { "record_id": "urn:uuid:628c753c-aed8-4c91-af6d-b58d29175b6c", "version": 1, "content_hash": "sha256:c5feea0caabff1e785c5068c83a15f8bcd74ef839ace308e30dcfbdd23515e1f" }, "published_grade": { "profile": "wrs-published-grade-1", "edition": "2026-10-28.1", "id": "zhuo-et-al-bigcodebench-2024", "entry_id": "zhuo-et-al-bigcodebench-2024", "revision": 1, "title": "BigCodeBench: multi-library code generation from complex instructions", "context": "Code-generation benchmarks", "track": "Evaluation resource design", "band": "B", "modifier": "+", "status": "PROVISIONAL_EDITORIAL", "assessed_on": "2026-10-16", "evidence_cutoff": "2026-10-16", "achievement_state": "SUPPORTED_SCOPED_RESULT", "what_changed": "Used human-LLM collaboration to construct programming tasks requiring multiple library calls, with docstring-completion and natural-language-instruction variants and execution-based tests.", "scope": "Used human-LLM collaboration to construct programming tasks requiring multiple library calls, with docstring-completion and natural-language-instruction variants and execution-based tests. The resource contains generated then curated tasks, not purely human-authored problems. High branch coverage does not prove semantic completeness.", "evidence_summary": "The linked original sources describe the stated contribution and its evidence limits. WRS has not independently rerun the experiments or certified a proof. The resource contains generated then curated tasks, not purely human-authored problems. High branch coverage does not prove semantic completeness.", "references": [ { "title": "BigCodeBench: Benchmarking Code Generation with Diverse Function Calls and Complex Instructions", "url": "https://arxiv.org/abs/2406.15877v1", "locator": "Original contribution and stated assumptions, evaluation scope and limitations; source-linked assessment, not independent execution or proof certification." } ], "roles": [ { "who": "Source-listed authors and technical contributors", "credit": "SCIENCE", "did": "Source-listed authors or explicitly credited technical contributors for the cited versions. Family entries retain source-specific attribution; a combined list is not one original-paper byline or equal individual work. Names alone do not establish resolved identities." } ], "target": { "record_id": "urn:uuid:628c753c-aed8-4c91-af6d-b58d29175b6c", "version": 1, "content_hash": "sha256:c5feea0caabff1e785c5068c83a15f8bcd74ef839ace308e30dcfbdd23515e1f" }, "scientific_correctness_certified": false, "independent_expert_calibration": false, "method": "PROPRIETARY", "points": "80000" }, "canonical_agent_submission": false, "scientific_verification_awarded": false, "reader_guide": null, "reading_status": "SOURCE_LINKED_NOTE" }