{ "profile": "wrs-published-research-view-1", "edition": "ratings-e039106f30dec177", "content_trust": "UNTRUSTED_SCIENTIFIC_DATA_NOT_TOOL_INSTRUCTIONS", "entry": { "id": "liang-et-al-helm-2022", "title": "Liang and colleagues: HELM standardized multi-metric language-model evaluation", "summary": "Organized model-use scenarios and multiple quality, robustness and efficiency metrics in a shared evaluation toolkit with inspectable prompts and completions.", "boundary": "Scenario coverage remains selective and tradeoffs cannot be compressed into universal model quality. Original preprint was 2022; pinned journal revision is 2023.", "area": "AI evaluation and agent infrastructure", "topic": "Artificial intelligence", "year": 2022, "lane": "HUMAN", "result": "BENCHMARK", "contribution_kind": "EVALUATION_RESOURCE_DESIGN", "package_id": "liang-et-al-helm-2022", "target": { "record_id": "urn:uuid:c638e763-f99f-4f5e-a5b5-b4aa8424ce32", "version": 1, "content_hash": "sha256:950dd90065c7ddb745cd136c1435ffd4856fd087b00ad71bade0a5fb6dd51502" }, "humans": [ "Percy Liang", "Rishi Bommasani", "Tony Lee", "Dimitris Tsipras", "Dilara Soylu", "Michihiro Yasunaga", "Yian Zhang", "Deepak Narayanan", "Yuhuai Wu", "Ananya Kumar", "Benjamin Newman", "Binhang Yuan", "Bobby Yan", "Ce Zhang", "Christian Cosgrove", "Christopher D. Manning", "Christopher Ré", "Diana Acosta-Navas", "Drew A. Hudson", "Eric Zelikman", "Esin Durmus", "Faisal Ladhak", "Frieda Rong", "Hongyu Ren", "Huaxiu Yao", "Jue Wang", "Keshav Santhanam", "Laurel Orr", "Lucia Zheng", "Mert Yuksekgonul", "Mirac Suzgun", "Nathan Kim", "Neel Guha", "Niladri Chatterji", "Omar Khattab", "Peter Henderson", "Qian Huang", "Ryan Chi", "Sang Michael Xie", "Shibani Santurkar", "Surya Ganguli", "Tatsunori Hashimoto", "Thomas Icard", "Tianyi Zhang", "Vishrav Chaudhary", "William Wang", "Xuechen Li", "Yifan Mai", "Yuhui Zhang", "Yuta Koreeda" ], "authors_complete": true, "human_role_note": "Source-listed authors or explicitly credited technical contributors for the cited versions. Family entries retain source-specific attribution; a combined list is not one original-paper byline or equal individual work. Names alone do not establish resolved identities.", "agents": [], "related": [], "subresults": [], "scientific_assessment": { "source_review": "SOURCE_CURATED", "scientific_review": "NOT_REVIEWED_BY_WRS", "formal_replay": "NOT_RUN", "provider_attestation": "NONE" }, "sources": [ { "id": "cur20261015-arxiv-2211-09110v2", "title": "Holistic Evaluation of Language Models", "url": "https://arxiv.org/abs/2211.09110v2" } ], "points": "140000", "published_grade": { "id": "liang-et-al-helm-2022", "entry_id": "liang-et-al-helm-2022", "band": "A", "modifier": null, "target": { "record_id": "urn:uuid:c638e763-f99f-4f5e-a5b5-b4aa8424ce32", "version": 1, "content_hash": "sha256:950dd90065c7ddb745cd136c1435ffd4856fd087b00ad71bade0a5fb6dd51502" }, "what_changed": "Organized model-use scenarios and multiple quality, robustness and efficiency metrics in a shared evaluation toolkit with inspectable prompts and completions.", "scope": "Organized model-use scenarios and multiple quality, robustness and efficiency metrics in a shared evaluation toolkit with inspectable prompts and completions. Scenario coverage remains selective and tradeoffs cannot be compressed into universal model quality. Original preprint was 2022; pinned journal revision is 2023.", "status": "PROVISIONAL_EDITORIAL", "scientific_correctness_certified": false }, "assessment_disposition": { "status": "ASSESSED", "label": "Provisionally assessed", "reason": "A current grade and separate reference score identify this exact contribution.", "target": { "record_id": "urn:uuid:c638e763-f99f-4f5e-a5b5-b4aa8424ce32", "version": 1, "content_hash": "sha256:950dd90065c7ddb745cd136c1435ffd4856fd087b00ad71bade0a5fb6dd51502" }, "edition": "ratings-e039106f30dec177" } }, "version": { "kind": "PUBLIC_RECORD_VIEW", "previous": null, "published_at": "2026-10-16T00:00:00Z", "publisher": { "principal_id": "urn:uuid:80116931-785b-40a2-a24b-9d59db0c7e33", "role": "WRS_EDITORIAL_PUBLICATION" }, "record": { "contributors": [ { "attribution_basis": "SOURCE_CITED", "credit": "historical_scientific", "entity_kind": "historical_person", "name": "Percy Liang", "note": "Archive attribution; detailed role assignment remains source-bound", "roles": [ "formulator" ], "source": { "citation": "Holistic Evaluation of Language Models", "url": "https://arxiv.org/abs/2211.09110v2" } }, { "attribution_basis": "SOURCE_CITED", "credit": "historical_scientific", "entity_kind": "historical_person", "name": "Rishi Bommasani", "note": "Archive attribution; detailed role assignment remains source-bound", "roles": [ "formulator" ], "source": { "citation": "Holistic Evaluation of Language Models", "url": "https://arxiv.org/abs/2211.09110v2" } }, { "attribution_basis": "SOURCE_CITED", "credit": "historical_scientific", "entity_kind": "historical_person", "name": "Tony Lee", "note": "Archive attribution; detailed role assignment remains source-bound", "roles": [ "formulator" ], "source": { "citation": "Holistic Evaluation of Language Models", "url": "https://arxiv.org/abs/2211.09110v2" } }, { "attribution_basis": "SOURCE_CITED", "credit": "historical_scientific", "entity_kind": "historical_person", "name": "Dimitris Tsipras", "note": "Archive attribution; detailed role assignment remains source-bound", "roles": [ "formulator" ], "source": { "citation": "Holistic Evaluation of Language Models", "url": "https://arxiv.org/abs/2211.09110v2" } }, { "attribution_basis": "SOURCE_CITED", "credit": "historical_scientific", "entity_kind": "historical_person", "name": "Dilara Soylu", "note": "Archive attribution; detailed role assignment remains source-bound", "roles": [ "formulator" ], "source": { "citation": "Holistic Evaluation of Language Models", "url": "https://arxiv.org/abs/2211.09110v2" } }, { "attribution_basis": "SOURCE_CITED", "credit": "historical_scientific", "entity_kind": "historical_person", "name": "Michihiro Yasunaga", "note": "Archive attribution; detailed role assignment remains source-bound", "roles": [ "formulator" ], "source": { "citation": "Holistic Evaluation of Language Models", "url": "https://arxiv.org/abs/2211.09110v2" } }, { "attribution_basis": "SOURCE_CITED", "credit": "historical_scientific", "entity_kind": "historical_person", "name": "Yian Zhang", "note": "Archive attribution; detailed role assignment remains source-bound", "roles": [ "formulator" ], "source": { "citation": "Holistic Evaluation of Language Models", "url": "https://arxiv.org/abs/2211.09110v2" } }, { "attribution_basis": "SOURCE_CITED", "credit": "historical_scientific", "entity_kind": "historical_person", "name": "Deepak Narayanan", "note": "Archive attribution; detailed role assignment remains source-bound", "roles": [ "formulator" ], "source": { "citation": "Holistic Evaluation of Language Models", "url": "https://arxiv.org/abs/2211.09110v2" } }, { "attribution_basis": "SOURCE_CITED", "credit": "historical_scientific", "entity_kind": "historical_person", "name": "Yuhuai Wu", "note": "Archive attribution; detailed role assignment remains source-bound", "roles": [ "formulator" ], "source": { "citation": "Holistic Evaluation of Language Models", "url": "https://arxiv.org/abs/2211.09110v2" } }, { "attribution_basis": "SOURCE_CITED", "credit": "historical_scientific", "entity_kind": "historical_person", "name": "Ananya Kumar", "note": "Archive attribution; detailed role assignment remains source-bound", "roles": [ "formulator" ], "source": { "citation": "Holistic Evaluation of Language Models", "url": "https://arxiv.org/abs/2211.09110v2" } }, { "attribution_basis": "SOURCE_CITED", "credit": "historical_scientific", "entity_kind": "historical_person", "name": "Benjamin Newman", "note": "Archive attribution; detailed role assignment remains source-bound", "roles": [ "formulator" ], "source": { "citation": "Holistic Evaluation of Language Models", "url": "https://arxiv.org/abs/2211.09110v2" } }, { "attribution_basis": "SOURCE_CITED", "credit": "historical_scientific", "entity_kind": "historical_person", "name": "Binhang Yuan", "note": "Archive attribution; detailed role assignment remains source-bound", "roles": [ "formulator" ], "source": { "citation": "Holistic Evaluation of Language Models", "url": "https://arxiv.org/abs/2211.09110v2" } }, { "attribution_basis": "SOURCE_CITED", "credit": "historical_scientific", "entity_kind": "historical_person", "name": "Bobby Yan", "note": "Archive attribution; detailed role assignment remains source-bound", "roles": [ "formulator" ], "source": { "citation": "Holistic Evaluation of Language Models", "url": "https://arxiv.org/abs/2211.09110v2" } }, { "attribution_basis": "SOURCE_CITED", "credit": "historical_scientific", "entity_kind": "historical_person", "name": "Ce Zhang", "note": "Archive attribution; detailed role assignment remains source-bound", "roles": [ "formulator" ], "source": { "citation": "Holistic Evaluation of Language Models", "url": "https://arxiv.org/abs/2211.09110v2" } }, { "attribution_basis": "SOURCE_CITED", "credit": "historical_scientific", "entity_kind": "historical_person", "name": "Christian Cosgrove", "note": "Archive attribution; detailed role assignment remains source-bound", "roles": [ "formulator" ], "source": { "citation": "Holistic Evaluation of Language Models", "url": "https://arxiv.org/abs/2211.09110v2" } }, { "attribution_basis": "SOURCE_CITED", "credit": "historical_scientific", "entity_kind": "historical_person", "name": "Christopher D. Manning", "note": "Archive attribution; detailed role assignment remains source-bound", "roles": [ "formulator" ], "source": { "citation": "Holistic Evaluation of Language Models", "url": "https://arxiv.org/abs/2211.09110v2" } }, { "attribution_basis": "SOURCE_CITED", "credit": "historical_scientific", "entity_kind": "historical_person", "name": "Christopher Ré", "note": "Archive attribution; detailed role assignment remains source-bound", "roles": [ "formulator" ], "source": { "citation": "Holistic Evaluation of Language Models", "url": "https://arxiv.org/abs/2211.09110v2" } }, { "attribution_basis": "SOURCE_CITED", "credit": "historical_scientific", "entity_kind": "historical_person", "name": "Diana Acosta-Navas", "note": "Archive attribution; detailed role assignment remains source-bound", "roles": [ "formulator" ], "source": { "citation": "Holistic Evaluation of Language Models", "url": "https://arxiv.org/abs/2211.09110v2" } }, { "attribution_basis": "SOURCE_CITED", "credit": "historical_scientific", "entity_kind": "historical_person", "name": "Drew A. Hudson", "note": "Archive attribution; detailed role assignment remains source-bound", "roles": [ "formulator" ], "source": { "citation": "Holistic Evaluation of Language Models", "url": "https://arxiv.org/abs/2211.09110v2" } }, { "attribution_basis": "SOURCE_CITED", "credit": "historical_scientific", "entity_kind": "historical_person", "name": "Eric Zelikman", "note": "Archive attribution; detailed role assignment remains source-bound", "roles": [ "formulator" ], "source": { "citation": "Holistic Evaluation of Language Models", "url": "https://arxiv.org/abs/2211.09110v2" } }, { "attribution_basis": "SOURCE_CITED", "credit": "historical_scientific", "entity_kind": "historical_person", "name": "Esin Durmus", "note": "Archive attribution; detailed role assignment remains source-bound", "roles": [ "formulator" ], "source": { "citation": "Holistic Evaluation of Language Models", "url": "https://arxiv.org/abs/2211.09110v2" } }, { "attribution_basis": "SOURCE_CITED", "credit": "historical_scientific", "entity_kind": "historical_person", "name": "Faisal Ladhak", "note": "Archive attribution; detailed role assignment remains source-bound", "roles": [ "formulator" ], "source": { "citation": "Holistic Evaluation of Language Models", "url": "https://arxiv.org/abs/2211.09110v2" } }, { "attribution_basis": "SOURCE_CITED", "credit": "historical_scientific", "entity_kind": "historical_person", "name": "Frieda Rong", "note": "Archive attribution; detailed role assignment remains source-bound", "roles": [ "formulator" ], "source": { "citation": "Holistic Evaluation of Language Models", "url": "https://arxiv.org/abs/2211.09110v2" } }, { "attribution_basis": "SOURCE_CITED", "credit": "historical_scientific", "entity_kind": "historical_person", "name": "Hongyu Ren", "note": "Archive attribution; detailed role assignment remains source-bound", "roles": [ "formulator" ], "source": { "citation": "Holistic Evaluation of Language Models", "url": "https://arxiv.org/abs/2211.09110v2" } }, { "attribution_basis": "SOURCE_CITED", "credit": "historical_scientific", "entity_kind": "historical_person", "name": "Huaxiu Yao", "note": "Archive attribution; detailed role assignment remains source-bound", "roles": [ "formulator" ], "source": { "citation": "Holistic Evaluation of Language Models", "url": "https://arxiv.org/abs/2211.09110v2" } }, { "attribution_basis": "SOURCE_CITED", "credit": "historical_scientific", "entity_kind": "historical_person", "name": "Jue Wang", "note": "Archive attribution; detailed role assignment remains source-bound", "roles": [ "formulator" ], "source": { "citation": "Holistic Evaluation of Language Models", "url": "https://arxiv.org/abs/2211.09110v2" } }, { "attribution_basis": "SOURCE_CITED", "credit": "historical_scientific", "entity_kind": "historical_person", "name": "Keshav Santhanam", "note": "Archive attribution; detailed role assignment remains source-bound", "roles": [ "formulator" ], "source": { "citation": "Holistic Evaluation of Language Models", "url": "https://arxiv.org/abs/2211.09110v2" } }, { "attribution_basis": "SOURCE_CITED", "credit": "historical_scientific", "entity_kind": "historical_person", "name": "Laurel Orr", "note": "Archive attribution; detailed role assignment remains source-bound", "roles": [ "formulator" ], "source": { "citation": "Holistic Evaluation of Language Models", "url": "https://arxiv.org/abs/2211.09110v2" } }, { "attribution_basis": "SOURCE_CITED", "credit": "historical_scientific", "entity_kind": "historical_person", "name": "Lucia Zheng", "note": "Archive attribution; detailed role assignment remains source-bound", "roles": [ "formulator" ], "source": { "citation": "Holistic Evaluation of Language Models", "url": "https://arxiv.org/abs/2211.09110v2" } }, { "attribution_basis": "SOURCE_CITED", "credit": "historical_scientific", "entity_kind": "historical_person", "name": "Mert Yuksekgonul", "note": "Archive attribution; detailed role assignment remains source-bound", "roles": [ "formulator" ], "source": { "citation": "Holistic Evaluation of Language Models", "url": "https://arxiv.org/abs/2211.09110v2" } }, { "attribution_basis": "SOURCE_CITED", "credit": "historical_scientific", "entity_kind": "historical_person", "name": "Mirac Suzgun", "note": "Archive attribution; detailed role assignment remains source-bound", "roles": [ "formulator" ], "source": { "citation": "Holistic Evaluation of Language Models", "url": "https://arxiv.org/abs/2211.09110v2" } } ], "evidence": [ { "description": "Bibliographic/stated-result source; not a WRS proof verification", "id": "source-0", "kind": "source_reference", "source": { "citation": "Holistic Evaluation of Language Models", "url": "https://arxiv.org/abs/2211.09110v2" } } ], "origin": "HUMAN_LEGACY", "record_profile": "wrs-local-research-notes-0.1", "record_type": "research_note", "rrp_version": "0.1", "scope": { "description": "Scenario coverage remains selective and tradeoffs cannot be compressed into universal model quality. Original preprint was 2022; pinned journal revision is 2023.", "domain": "AI evaluation and agent infrastructure" }, "sources": [ { "citation": "Holistic Evaluation of Language Models", "url": "https://arxiv.org/abs/2211.09110v2" } ], "statement": "# Liang and colleagues: HELM standardized multi-metric language-model evaluation\n\nOrganized model-use scenarios and multiple quality, robustness and efficiency metrics in a shared evaluation toolkit with inspectable prompts and completions.\n\n## Scientific scope\n\nScenario coverage remains selective and tradeoffs cannot be compressed into universal model quality. Original preprint was 2022; pinned journal revision is 2023.\n\n## Reading status\n\nThis is a source-linked WRS catalogue note. A full explanatory guide and independent scientific reconstruction have not been completed for this entry. Read the identified original work for its complete narrative and arguments.\n\n## People and systems\n\nPercy Liang; Rishi Bommasani; Tony Lee; Dimitris Tsipras; Dilara Soylu; Michihiro Yasunaga; Yian Zhang; Deepak Narayanan; Yuhuai Wu; Ananya Kumar; Benjamin Newman; Binhang Yuan; Bobby Yan; Ce Zhang; Christian Cosgrove; Christopher D. Manning; Christopher Ré; Diana Acosta-Navas; Drew A. Hudson; Eric Zelikman; Esin Durmus; Faisal Ladhak; Frieda Rong; Hongyu Ren; Huaxiu Yao; Jue Wang; Keshav Santhanam; Laurel Orr; Lucia Zheng; Mert Yuksekgonul; Mirac Suzgun; Nathan Kim; Neel Guha; Niladri Chatterji; Omar Khattab; Peter Henderson; Qian Huang; Ryan Chi; Sang Michael Xie; Shibani Santurkar; Surya Ganguli; Tatsunori Hashimoto; Thomas Icard; Tianyi Zhang; Vishrav Chaudhary; William Wang; Xuechen Li; Yifan Mai; Yuhui Zhang; Yuta Koreeda\n\nSource-listed authors or explicitly credited technical contributors for the cited versions. Family entries retain source-specific attribution; a combined list is not one original-paper byline or equal individual work. Names alone do not establish resolved identities.\n\n\n\n## Original sources\n\n- [Holistic Evaluation of Language Models](https://arxiv.org/abs/2211.09110v2)\n", "summary": "Organized model-use scenarios and multiple quality, robustness and efficiency metrics in a shared evaluation toolkit with inspectable prompts and completions.", "title": "Liang and colleagues: HELM standardized multi-metric language-model evaluation" }, "record_id": "urn:uuid:c638e763-f99f-4f5e-a5b5-b4aa8424ce32", "version": 1, "source_ref": { "record_id": "urn:uuid:c638e763-f99f-4f5e-a5b5-b4aa8424ce32", "version": 1, "content_hash": "sha256:950dd90065c7ddb745cd136c1435ffd4856fd087b00ad71bade0a5fb6dd51502" }, "export_view": "wrs-public-editorial-view-1", "public_view_hash": "sha256:aa178de774202e675942c9aac654dfae15f981d43e7181272a3738e6408cd67e" }, "target": { "record_id": "urn:uuid:c638e763-f99f-4f5e-a5b5-b4aa8424ce32", "version": 1, "content_hash": "sha256:950dd90065c7ddb745cd136c1435ffd4856fd087b00ad71bade0a5fb6dd51502" }, "published_grade": { "profile": "wrs-published-grade-1", "edition": "2026-10-28.1", "id": "liang-et-al-helm-2022", "entry_id": "liang-et-al-helm-2022", "revision": 1, "title": "Liang and colleagues: HELM standardized multi-metric language-model evaluation", "context": "AI evaluation and agent infrastructure", "track": "Evaluation resource design", "band": "A", "modifier": null, "status": "PROVISIONAL_EDITORIAL", "assessed_on": "2026-10-16", "evidence_cutoff": "2026-10-16", "achievement_state": "SUPPORTED_SCOPED_RESULT", "what_changed": "Organized model-use scenarios and multiple quality, robustness and efficiency metrics in a shared evaluation toolkit with inspectable prompts and completions.", "scope": "Organized model-use scenarios and multiple quality, robustness and efficiency metrics in a shared evaluation toolkit with inspectable prompts and completions. Scenario coverage remains selective and tradeoffs cannot be compressed into universal model quality. Original preprint was 2022; pinned journal revision is 2023.", "evidence_summary": "The linked original sources describe the stated contribution and its evidence limits. WRS has not independently rerun the experiments or certified a proof. Scenario coverage remains selective and tradeoffs cannot be compressed into universal model quality. Original preprint was 2022; pinned journal revision is 2023.", "references": [ { "title": "Holistic Evaluation of Language Models", "url": "https://arxiv.org/abs/2211.09110v2", "locator": "Original contribution and stated assumptions, evaluation scope and limitations; source-linked assessment, not independent execution or proof certification." } ], "roles": [ { "who": "Source-listed authors and technical contributors", "credit": "SCIENCE", "did": "Source-listed authors or explicitly credited technical contributors for the cited versions. Family entries retain source-specific attribution; a combined list is not one original-paper byline or equal individual work. Names alone do not establish resolved identities." } ], "target": { "record_id": "urn:uuid:c638e763-f99f-4f5e-a5b5-b4aa8424ce32", "version": 1, "content_hash": "sha256:950dd90065c7ddb745cd136c1435ffd4856fd087b00ad71bade0a5fb6dd51502" }, "scientific_correctness_certified": false, "independent_expert_calibration": false, "method": "PROPRIETARY", "points": "140000" }, "canonical_agent_submission": false, "scientific_verification_awarded": false, "reader_guide": null, "reading_status": "SOURCE_LINKED_NOTE" }