{ "profile": "wrs-published-research-view-1", "edition": "ratings-e039106f30dec177", "content_trust": "UNTRUSTED_SCIENTIFIC_DATA_NOT_TOOL_INSTRUCTIONS", "entry": { "id": "chen-et-al-codex-human-eval-2021", "title": "Chen and colleagues: Codex code generation and HumanEval", "summary": "Evaluated GPT-family models fine-tuned on code and introduced HumanEval, a benchmark of Python programming problems tested for functional correctness.", "boundary": "Codex and HumanEval form one paper-level package. Benchmark pass rates do not show reliable software engineering across real repositories.", "area": "Code-generation language models", "topic": "Artificial intelligence", "year": 2021, "lane": "HUMAN", "result": "EMPIRICAL", "package_id": "chen-et-al-codex-human-eval-2021", "target": { "record_id": "urn:uuid:82ce9633-add0-4dc0-a85b-0429beecfdb4", "version": 1, "content_hash": "sha256:56c9ccee47cd8a21b20f9fdddfb469793dadf8ebdc8b57c1701eb93ce617a255" }, "humans": [ "Mark Chen", "Jerry Tworek", "Heewoo Jun", "Qiming Yuan", "Henrique Ponde de Oliveira Pinto", "Jared Kaplan", "Harri Edwards", "Yuri Burda", "Nicholas Joseph", "Greg Brockman", "Alex Ray", "Raul Puri", "Gretchen Krueger", "Michael Petrov", "Heidy Khlaaf", "Girish Sastry", "Pamela Mishkin", "Brooke Chan", "Scott Gray", "Nick Ryder", "Mikhail Pavlov", "Alethea Power", "Lukasz Kaiser", "Mohammad Bavarian", "Clemens Winter", "Philippe Tillet", "Felipe Petroski Such", "Dave Cummings", "Matthias Plappert", "Fotios Chantzis", "Elizabeth Barnes", "Ariel Herbert-Voss", "William Hebgen Guss", "Alex Nichol", "Alex Paino", "Nikolas Tezak", "Jie Tang", "Igor Babuschkin", "Suchir Balaji", "Shantanu Jain", "William Saunders", "Christopher Hesse", "Andrew N. Carr", "Jan Leike", "Josh Achiam", "Vedant Misra", "Evan Morikawa", "Alec Radford", "Matthew Knight", "Miles Brundage", "Mira Murati", "Katie Mayer", "Peter Welinder", "Bob McGrew", "Dario Amodei", "Sam McCandlish", "Ilya Sutskever", "Wojciech Zaremba" ], "authors_complete": true, "human_role_note": "Complete named paper byline. Human research, source-reported system execution and experimental validation are distinct; individual effort shares are not measured.", "agents": [], "related": [], "subresults": [], "scientific_assessment": { "source_review": "SOURCE_CURATED", "scientific_review": "NOT_REVIEWED_BY_WRS", "formal_replay": "NOT_RUN", "provider_attestation": "NONE" }, "sources": [ { "id": "arxiv-2107-03374", "title": "Chen and colleagues: Codex code generation and HumanEval — original paper", "url": "https://arxiv.org/abs/2107.03374v2" }, { "id": "company-chen-et-al-codex-human-eval-2021", "title": "Chen and colleagues: Codex code generation and HumanEval — company research page", "url": "https://openai.com/research/index/milestone/" } ], "points": "48000", "published_grade": { "id": "chen-et-al-codex-human-eval-2021", "entry_id": "chen-et-al-codex-human-eval-2021", "band": "A", "modifier": null, "target": { "record_id": "urn:uuid:82ce9633-add0-4dc0-a85b-0429beecfdb4", "version": 1, "content_hash": "sha256:56c9ccee47cd8a21b20f9fdddfb469793dadf8ebdc8b57c1701eb93ce617a255" }, "what_changed": "Evaluated GPT-family models fine-tuned on code and introduced HumanEval, a benchmark of Python programming problems tested for functional correctness.", "scope": "Evaluated GPT-family models fine-tuned on code and introduced HumanEval, a benchmark of Python programming problems tested for functional correctness. Scope limitation: Codex and HumanEval form one paper-level package. Benchmark pass rates do not show reliable software engineering across real repositories.", "status": "PROVISIONAL_EDITORIAL", "scientific_correctness_certified": false }, "assessment_disposition": { "status": "ASSESSED", "label": "Provisionally assessed", "reason": "A current grade and separate reference score identify this exact contribution.", "target": { "record_id": "urn:uuid:82ce9633-add0-4dc0-a85b-0429beecfdb4", "version": 1, "content_hash": "sha256:56c9ccee47cd8a21b20f9fdddfb469793dadf8ebdc8b57c1701eb93ce617a255" }, "edition": "ratings-e039106f30dec177" } }, "version": { "kind": "PUBLIC_RECORD_VIEW", "previous": null, "published_at": "2026-09-27T00:00:00Z", "publisher": { "principal_id": "urn:uuid:80116931-785b-40a2-a24b-9d59db0c7e33", "role": "WRS_EDITORIAL_PUBLICATION" }, "record": { "contributors": [ { "attribution_basis": "SOURCE_CITED", "credit": "historical_scientific", "entity_kind": "historical_person", "name": "Mark Chen", "note": "Archive attribution; detailed role assignment remains source-bound", "roles": [ "formulator" ], "source": { "citation": "Chen and colleagues: Codex code generation and HumanEval — original paper", "url": "https://arxiv.org/abs/2107.03374v2" } }, { "attribution_basis": "SOURCE_CITED", "credit": "historical_scientific", "entity_kind": "historical_person", "name": "Jerry Tworek", "note": "Archive attribution; detailed role assignment remains source-bound", "roles": [ "formulator" ], "source": { "citation": "Chen and colleagues: Codex code generation and HumanEval — original paper", "url": "https://arxiv.org/abs/2107.03374v2" } }, { "attribution_basis": "SOURCE_CITED", "credit": "historical_scientific", "entity_kind": "historical_person", "name": "Heewoo Jun", "note": "Archive attribution; detailed role assignment remains source-bound", "roles": [ "formulator" ], "source": { "citation": "Chen and colleagues: Codex code generation and HumanEval — original paper", "url": "https://arxiv.org/abs/2107.03374v2" } }, { "attribution_basis": "SOURCE_CITED", "credit": "historical_scientific", "entity_kind": "historical_person", "name": "Qiming Yuan", "note": "Archive attribution; detailed role assignment remains source-bound", "roles": [ "formulator" ], "source": { "citation": "Chen and colleagues: Codex code generation and HumanEval — original paper", "url": "https://arxiv.org/abs/2107.03374v2" } }, { "attribution_basis": "SOURCE_CITED", "credit": "historical_scientific", "entity_kind": "historical_person", "name": "Henrique Ponde de Oliveira Pinto", "note": "Archive attribution; detailed role assignment remains source-bound", "roles": [ "formulator" ], "source": { "citation": "Chen and colleagues: Codex code generation and HumanEval — original paper", "url": "https://arxiv.org/abs/2107.03374v2" } }, { "attribution_basis": "SOURCE_CITED", "credit": "historical_scientific", "entity_kind": "historical_person", "name": "Jared Kaplan", "note": "Archive attribution; detailed role assignment remains source-bound", "roles": [ "formulator" ], "source": { "citation": "Chen and colleagues: Codex code generation and HumanEval — original paper", "url": "https://arxiv.org/abs/2107.03374v2" } }, { "attribution_basis": "SOURCE_CITED", "credit": "historical_scientific", "entity_kind": "historical_person", "name": "Harri Edwards", "note": "Archive attribution; detailed role assignment remains source-bound", "roles": [ "formulator" ], "source": { "citation": "Chen and colleagues: Codex code generation and HumanEval — original paper", "url": "https://arxiv.org/abs/2107.03374v2" } }, { "attribution_basis": "SOURCE_CITED", "credit": "historical_scientific", "entity_kind": "historical_person", "name": "Yuri Burda", "note": "Archive attribution; detailed role assignment remains source-bound", "roles": [ "formulator" ], "source": { "citation": "Chen and colleagues: Codex code generation and HumanEval — original paper", "url": "https://arxiv.org/abs/2107.03374v2" } }, { "attribution_basis": "SOURCE_CITED", "credit": "historical_scientific", "entity_kind": "historical_person", "name": "Nicholas Joseph", "note": "Archive attribution; detailed role assignment remains source-bound", "roles": [ "formulator" ], "source": { "citation": "Chen and colleagues: Codex code generation and HumanEval — original paper", "url": "https://arxiv.org/abs/2107.03374v2" } }, { "attribution_basis": "SOURCE_CITED", "credit": "historical_scientific", "entity_kind": "historical_person", "name": "Greg Brockman", "note": "Archive attribution; detailed role assignment remains source-bound", "roles": [ "formulator" ], "source": { "citation": "Chen and colleagues: Codex code generation and HumanEval — original paper", "url": "https://arxiv.org/abs/2107.03374v2" } }, { "attribution_basis": "SOURCE_CITED", "credit": "historical_scientific", "entity_kind": "historical_person", "name": "Alex Ray", "note": "Archive attribution; detailed role assignment remains source-bound", "roles": [ "formulator" ], "source": { "citation": "Chen and colleagues: Codex code generation and HumanEval — original paper", "url": "https://arxiv.org/abs/2107.03374v2" } }, { "attribution_basis": "SOURCE_CITED", "credit": "historical_scientific", "entity_kind": "historical_person", "name": "Raul Puri", "note": "Archive attribution; detailed role assignment remains source-bound", "roles": [ "formulator" ], "source": { "citation": "Chen and colleagues: Codex code generation and HumanEval — original paper", "url": "https://arxiv.org/abs/2107.03374v2" } }, { "attribution_basis": "SOURCE_CITED", "credit": "historical_scientific", "entity_kind": "historical_person", "name": "Gretchen Krueger", "note": "Archive attribution; detailed role assignment remains source-bound", "roles": [ "formulator" ], "source": { "citation": "Chen and colleagues: Codex code generation and HumanEval — original paper", "url": "https://arxiv.org/abs/2107.03374v2" } }, { "attribution_basis": "SOURCE_CITED", "credit": "historical_scientific", "entity_kind": "historical_person", "name": "Michael Petrov", "note": "Archive attribution; detailed role assignment remains source-bound", "roles": [ "formulator" ], "source": { "citation": "Chen and colleagues: Codex code generation and HumanEval — original paper", "url": "https://arxiv.org/abs/2107.03374v2" } }, { "attribution_basis": "SOURCE_CITED", "credit": "historical_scientific", "entity_kind": "historical_person", "name": "Heidy Khlaaf", "note": "Archive attribution; detailed role assignment remains source-bound", "roles": [ "formulator" ], "source": { "citation": "Chen and colleagues: Codex code generation and HumanEval — original paper", "url": "https://arxiv.org/abs/2107.03374v2" } }, { "attribution_basis": "SOURCE_CITED", "credit": "historical_scientific", "entity_kind": "historical_person", "name": "Girish Sastry", "note": "Archive attribution; detailed role assignment remains source-bound", "roles": [ "formulator" ], "source": { "citation": "Chen and colleagues: Codex code generation and HumanEval — original paper", "url": "https://arxiv.org/abs/2107.03374v2" } }, { "attribution_basis": "SOURCE_CITED", "credit": "historical_scientific", "entity_kind": "historical_person", "name": "Pamela Mishkin", "note": "Archive attribution; detailed role assignment remains source-bound", "roles": [ "formulator" ], "source": { "citation": "Chen and colleagues: Codex code generation and HumanEval — original paper", "url": "https://arxiv.org/abs/2107.03374v2" } }, { "attribution_basis": "SOURCE_CITED", "credit": "historical_scientific", "entity_kind": "historical_person", "name": "Brooke Chan", "note": "Archive attribution; detailed role assignment remains source-bound", "roles": [ "formulator" ], "source": { "citation": "Chen and colleagues: Codex code generation and HumanEval — original paper", "url": "https://arxiv.org/abs/2107.03374v2" } }, { "attribution_basis": "SOURCE_CITED", "credit": "historical_scientific", "entity_kind": "historical_person", "name": "Scott Gray", "note": "Archive attribution; detailed role assignment remains source-bound", "roles": [ "formulator" ], "source": { "citation": "Chen and colleagues: Codex code generation and HumanEval — original paper", "url": "https://arxiv.org/abs/2107.03374v2" } }, { "attribution_basis": "SOURCE_CITED", "credit": "historical_scientific", "entity_kind": "historical_person", "name": "Nick Ryder", "note": "Archive attribution; detailed role assignment remains source-bound", "roles": [ "formulator" ], "source": { "citation": "Chen and colleagues: Codex code generation and HumanEval — original paper", "url": "https://arxiv.org/abs/2107.03374v2" } }, { "attribution_basis": "SOURCE_CITED", "credit": "historical_scientific", "entity_kind": "historical_person", "name": "Mikhail Pavlov", "note": "Archive attribution; detailed role assignment remains source-bound", "roles": [ "formulator" ], "source": { "citation": "Chen and colleagues: Codex code generation and HumanEval — original paper", "url": "https://arxiv.org/abs/2107.03374v2" } }, { "attribution_basis": "SOURCE_CITED", "credit": "historical_scientific", "entity_kind": "historical_person", "name": "Alethea Power", "note": "Archive attribution; detailed role assignment remains source-bound", "roles": [ "formulator" ], "source": { "citation": "Chen and colleagues: Codex code generation and HumanEval — original paper", "url": "https://arxiv.org/abs/2107.03374v2" } }, { "attribution_basis": "SOURCE_CITED", "credit": "historical_scientific", "entity_kind": "historical_person", "name": "Lukasz Kaiser", "note": "Archive attribution; detailed role assignment remains source-bound", "roles": [ "formulator" ], "source": { "citation": "Chen and colleagues: Codex code generation and HumanEval — original paper", "url": "https://arxiv.org/abs/2107.03374v2" } }, { "attribution_basis": "SOURCE_CITED", "credit": "historical_scientific", "entity_kind": "historical_person", "name": "Mohammad Bavarian", "note": "Archive attribution; detailed role assignment remains source-bound", "roles": [ "formulator" ], "source": { "citation": "Chen and colleagues: Codex code generation and HumanEval — original paper", "url": "https://arxiv.org/abs/2107.03374v2" } }, { "attribution_basis": "SOURCE_CITED", "credit": "historical_scientific", "entity_kind": "historical_person", "name": "Clemens Winter", "note": "Archive attribution; detailed role assignment remains source-bound", "roles": [ "formulator" ], "source": { "citation": "Chen and colleagues: Codex code generation and HumanEval — original paper", "url": "https://arxiv.org/abs/2107.03374v2" } }, { "attribution_basis": "SOURCE_CITED", "credit": "historical_scientific", "entity_kind": "historical_person", "name": "Philippe Tillet", "note": "Archive attribution; detailed role assignment remains source-bound", "roles": [ "formulator" ], "source": { "citation": "Chen and colleagues: Codex code generation and HumanEval — original paper", "url": "https://arxiv.org/abs/2107.03374v2" } }, { "attribution_basis": "SOURCE_CITED", "credit": "historical_scientific", "entity_kind": "historical_person", "name": "Felipe Petroski Such", "note": "Archive attribution; detailed role assignment remains source-bound", "roles": [ "formulator" ], "source": { "citation": "Chen and colleagues: Codex code generation and HumanEval — original paper", "url": "https://arxiv.org/abs/2107.03374v2" } }, { "attribution_basis": "SOURCE_CITED", "credit": "historical_scientific", "entity_kind": "historical_person", "name": "Dave Cummings", "note": "Archive attribution; detailed role assignment remains source-bound", "roles": [ "formulator" ], "source": { "citation": "Chen and colleagues: Codex code generation and HumanEval — original paper", "url": "https://arxiv.org/abs/2107.03374v2" } }, { "attribution_basis": "SOURCE_CITED", "credit": "historical_scientific", "entity_kind": "historical_person", "name": "Matthias Plappert", "note": "Archive attribution; detailed role assignment remains source-bound", "roles": [ "formulator" ], "source": { "citation": "Chen and colleagues: Codex code generation and HumanEval — original paper", "url": "https://arxiv.org/abs/2107.03374v2" } }, { "attribution_basis": "SOURCE_CITED", "credit": "historical_scientific", "entity_kind": "historical_person", "name": "Fotios Chantzis", "note": "Archive attribution; detailed role assignment remains source-bound", "roles": [ "formulator" ], "source": { "citation": "Chen and colleagues: Codex code generation and HumanEval — original paper", "url": "https://arxiv.org/abs/2107.03374v2" } }, { "attribution_basis": "SOURCE_CITED", "credit": "historical_scientific", "entity_kind": "historical_person", "name": "Elizabeth Barnes", "note": "Archive attribution; detailed role assignment remains source-bound", "roles": [ "formulator" ], "source": { "citation": "Chen and colleagues: Codex code generation and HumanEval — original paper", "url": "https://arxiv.org/abs/2107.03374v2" } } ], "evidence": [ { "description": "Bibliographic/stated-result source; not a WRS proof verification", "id": "source-0", "kind": "source_reference", "source": { "citation": "Chen and colleagues: Codex code generation and HumanEval — original paper", "url": "https://arxiv.org/abs/2107.03374v2" } }, { "description": "Bibliographic/stated-result source; not a WRS proof verification", "id": "source-1", "kind": "source_reference", "source": { "citation": "Chen and colleagues: Codex code generation and HumanEval — company research page", "url": "https://openai.com/research/index/milestone/" } } ], "origin": "HUMAN_LEGACY", "record_profile": "wrs-local-research-notes-0.1", "record_type": "research_note", "rrp_version": "0.1", "scope": { "description": "Codex and HumanEval form one paper-level package. Benchmark pass rates do not show reliable software engineering across real repositories.", "domain": "Code-generation language models" }, "sources": [ { "citation": "Chen and colleagues: Codex code generation and HumanEval — original paper", "url": "https://arxiv.org/abs/2107.03374v2" }, { "citation": "Chen and colleagues: Codex code generation and HumanEval — company research page", "url": "https://openai.com/research/index/milestone/" } ], "statement": "# Chen and colleagues: Codex code generation and HumanEval\n\nEvaluated GPT-family models fine-tuned on code and introduced HumanEval, a benchmark of Python programming problems tested for functional correctness.\n\n## Scientific scope\n\nCodex and HumanEval form one paper-level package. Benchmark pass rates do not show reliable software engineering across real repositories.\n\n## Reading status\n\nThis is a source-linked WRS catalogue note. A full explanatory guide and independent scientific reconstruction have not been completed for this entry. Read the identified original work for its complete narrative and arguments.\n\n## People and systems\n\nMark Chen; Jerry Tworek; Heewoo Jun; Qiming Yuan; Henrique Ponde de Oliveira Pinto; Jared Kaplan; Harri Edwards; Yuri Burda; Nicholas Joseph; Greg Brockman; Alex Ray; Raul Puri; Gretchen Krueger; Michael Petrov; Heidy Khlaaf; Girish Sastry; Pamela Mishkin; Brooke Chan; Scott Gray; Nick Ryder; Mikhail Pavlov; Alethea Power; Lukasz Kaiser; Mohammad Bavarian; Clemens Winter; Philippe Tillet; Felipe Petroski Such; Dave Cummings; Matthias Plappert; Fotios Chantzis; Elizabeth Barnes; Ariel Herbert-Voss; William Hebgen Guss; Alex Nichol; Alex Paino; Nikolas Tezak; Jie Tang; Igor Babuschkin; Suchir Balaji; Shantanu Jain; William Saunders; Christopher Hesse; Andrew N. Carr; Jan Leike; Josh Achiam; Vedant Misra; Evan Morikawa; Alec Radford; Matthew Knight; Miles Brundage; Mira Murati; Katie Mayer; Peter Welinder; Bob McGrew; Dario Amodei; Sam McCandlish; Ilya Sutskever; Wojciech Zaremba\n\nComplete named paper byline. Human research, source-reported system execution and experimental validation are distinct; individual effort shares are not measured.\n\n\n\n## Original sources\n\n- [Chen and colleagues: Codex code generation and HumanEval — original paper](https://arxiv.org/abs/2107.03374v2)\n- [Chen and colleagues: Codex code generation and HumanEval — company research page](https://openai.com/research/index/milestone/)\n", "summary": "Evaluated GPT-family models fine-tuned on code and introduced HumanEval, a benchmark of Python programming problems tested for functional correctness.", "title": "Chen and colleagues: Codex code generation and HumanEval" }, "record_id": "urn:uuid:82ce9633-add0-4dc0-a85b-0429beecfdb4", "version": 1, "source_ref": { "record_id": "urn:uuid:82ce9633-add0-4dc0-a85b-0429beecfdb4", "version": 1, "content_hash": "sha256:56c9ccee47cd8a21b20f9fdddfb469793dadf8ebdc8b57c1701eb93ce617a255" }, "export_view": "wrs-public-editorial-view-1", "public_view_hash": "sha256:be2fce0d92df9ef3fa3e31635c1bdd2c98067e321944da2ae47e7e3a981fb253" }, "target": { "record_id": "urn:uuid:82ce9633-add0-4dc0-a85b-0429beecfdb4", "version": 1, "content_hash": "sha256:56c9ccee47cd8a21b20f9fdddfb469793dadf8ebdc8b57c1701eb93ce617a255" }, "published_grade": { "profile": "wrs-published-grade-1", "edition": "2026-10-28.1", "id": "chen-et-al-codex-human-eval-2021", "entry_id": "chen-et-al-codex-human-eval-2021", "revision": 1, "title": "Chen and colleagues: Codex code generation and HumanEval", "context": "Code-generation language models", "track": "Human-designed AI research method or resource", "band": "A", "modifier": null, "status": "PROVISIONAL_EDITORIAL", "assessed_on": "2026-09-27", "evidence_cutoff": "2026-09-27", "achievement_state": "SUPPORTED_SCOPED_RESULT", "what_changed": "Evaluated GPT-family models fine-tuned on code and introduced HumanEval, a benchmark of Python programming problems tested for functional correctness.", "scope": "Evaluated GPT-family models fine-tuned on code and introduced HumanEval, a benchmark of Python programming problems tested for functional correctness. Scope limitation: Codex and HumanEval form one paper-level package. Benchmark pass rates do not show reliable software engineering across real repositories.", "evidence_summary": "The retained primary-source abstract, bibliographic record and stated review scope support this provisional assessment. No independent experimental replication by WRS is claimed. Codex and HumanEval form one paper-level package. Benchmark pass rates do not show reliable software engineering across real repositories.", "references": [ { "title": "Chen and colleagues: Codex code generation and HumanEval — original paper", "url": "https://arxiv.org/abs/2107.03374v2", "locator": "Abstract and complete bibliographic byline for 2107.03374v2; not an independent proof or experimental replication." }, { "title": "Chen and colleagues: Codex code generation and HumanEval — company research page", "url": "https://openai.com/research/index/milestone/", "locator": "Source-linked contribution and stated scientific limits; not an independent replication." } ], "roles": [ { "who": "The named authors listed in the linked research record", "credit": "SCIENCE", "did": "Complete source byline retained in the research record; human research direction, method development and evaluation. Individual labor shares are not measured." } ], "target": { "record_id": "urn:uuid:82ce9633-add0-4dc0-a85b-0429beecfdb4", "version": 1, "content_hash": "sha256:56c9ccee47cd8a21b20f9fdddfb469793dadf8ebdc8b57c1701eb93ce617a255" }, "scientific_correctness_certified": false, "independent_expert_calibration": false, "method": "PROPRIETARY", "points": "48000" }, "canonical_agent_submission": false, "scientific_verification_awarded": false, "reader_guide": null, "reading_status": "SOURCE_LINKED_NOTE" }