{ "profile": "wrs-published-research-view-1", "edition": "ratings-e039106f30dec177", "content_trust": "UNTRUSTED_SCIENTIFIC_DATA_NOT_TOOL_INSTRUCTIONS", "entry": { "id": "mazeika-et-al-harmbench-2024", "title": "Mazeika and colleagues: HarmBench standardized safeguard evaluation", "summary": "Built a standardized framework combining curated harmful-behavior test cases, automated red-team method comparisons and response classifiers to evaluate robust refusal across language models under a common evaluation procedure.", "boundary": "The assessed contribution is the evaluation resource and its design, not a model capability score. Results depend on held-out behaviors, classifiers, generated length and attack budgets. For non-copyright cases, the original success criterion includes attempted behavior even when the output is incorrect. It is not exhaustive harm coverage, verified harmful capability or a deployment safety certificate.", "area": "Language-model safety evaluation", "topic": "Artificial intelligence", "year": 2024, "lane": "HUMAN", "result": "BENCHMARK", "contribution_kind": "EVALUATION_RESOURCE_DESIGN", "package_id": "mazeika-et-al-harmbench-2024", "target": { "record_id": "urn:uuid:fd3b59b0-0bb3-4a98-abea-b73b20bbba37", "version": 1, "content_hash": "sha256:a8e24a297623ee03b3e2db19835dc925cb7c1fe84631ed72e2ac761554227a0e" }, "humans": [ "Mantas Mazeika", "Long Phan", "Xuwang Yin", "Andy Zou", "Zifan Wang", "Norman Mu", "Elham Sakhaee", "Nathaniel Li", "Steven Basart", "Bo Li", "David Forsyth", "Dan Hendrycks" ], "authors_complete": true, "human_role_note": "Source-listed authors for the exact contribution; experimental subjects are not autonomous scientific authors. Source support and predecessor work retain separate roles. Individual effort shares and unresolved identities are not inferred.", "agents": [], "related": [ "souly-et-al-strongreject-2024" ], "subresults": [], "scientific_assessment": { "source_review": "SOURCE_CURATED", "scientific_review": "NOT_REVIEWED_BY_WRS", "formal_replay": "NOT_RUN", "provider_attestation": "NONE" }, "sources": [ { "id": "cur20261017-arxiv-2402-04249v1", "title": "HarmBench: A Standardized Evaluation Framework for Automated Red Teaming and Robust Refusal", "url": "https://arxiv.org/abs/2402.04249v1", "date": "2024-02-06" } ], "points": "95000", "published_grade": { "id": "mazeika-et-al-harmbench-2024", "entry_id": "mazeika-et-al-harmbench-2024", "band": "A", "modifier": "-", "target": { "record_id": "urn:uuid:fd3b59b0-0bb3-4a98-abea-b73b20bbba37", "version": 1, "content_hash": "sha256:a8e24a297623ee03b3e2db19835dc925cb7c1fe84631ed72e2ac761554227a0e" }, "what_changed": "Built a standardized framework combining curated harmful-behavior test cases, automated red-team method comparisons and response classifiers to evaluate robust refusal across language models under a common evaluation procedure.", "scope": "Built a standardized framework combining curated harmful-behavior test cases, automated red-team method comparisons and response classifiers to evaluate robust refusal across language models under a common evaluation procedure. The assessed contribution is the evaluation resource and its design, not a model capability score. Results depend on held-out behaviors, classifiers, generated length and attack budgets. For non-copyright cases, the original success criterion includes attempted behavior even when the output is incorrect. It is not exhaustive harm coverage, verified harmful capability or a deployment safety certificate.", "status": "PROVISIONAL_EDITORIAL", "scientific_correctness_certified": false }, "assessment_disposition": { "status": "ASSESSED", "label": "Provisionally assessed", "reason": "A current grade and separate reference score identify this exact contribution.", "target": { "record_id": "urn:uuid:fd3b59b0-0bb3-4a98-abea-b73b20bbba37", "version": 1, "content_hash": "sha256:a8e24a297623ee03b3e2db19835dc925cb7c1fe84631ed72e2ac761554227a0e" }, "edition": "ratings-e039106f30dec177" } }, "version": { "kind": "PUBLIC_RECORD_VIEW", "previous": null, "published_at": "2026-10-23T00:00:00Z", "publisher": { "principal_id": "urn:uuid:80116931-785b-40a2-a24b-9d59db0c7e33", "role": "WRS_EDITORIAL_PUBLICATION" }, "record": { "contributors": [ { "attribution_basis": "SOURCE_CITED", "credit": "historical_scientific", "entity_kind": "historical_person", "name": "Mantas Mazeika", "note": "Archive attribution; detailed role assignment remains source-bound", "roles": [ "formulator" ], "source": { "citation": "HarmBench: A Standardized Evaluation Framework for Automated Red Teaming and Robust Refusal", "url": "https://arxiv.org/abs/2402.04249v1" } }, { "attribution_basis": "SOURCE_CITED", "credit": "historical_scientific", "entity_kind": "historical_person", "name": "Long Phan", "note": "Archive attribution; detailed role assignment remains source-bound", "roles": [ "formulator" ], "source": { "citation": "HarmBench: A Standardized Evaluation Framework for Automated Red Teaming and Robust Refusal", "url": "https://arxiv.org/abs/2402.04249v1" } }, { "attribution_basis": "SOURCE_CITED", "credit": "historical_scientific", "entity_kind": "historical_person", "name": "Xuwang Yin", "note": "Archive attribution; detailed role assignment remains source-bound", "roles": [ "formulator" ], "source": { "citation": "HarmBench: A Standardized Evaluation Framework for Automated Red Teaming and Robust Refusal", "url": "https://arxiv.org/abs/2402.04249v1" } }, { "attribution_basis": "SOURCE_CITED", "credit": "historical_scientific", "entity_kind": "historical_person", "name": "Andy Zou", "note": "Archive attribution; detailed role assignment remains source-bound", "roles": [ "formulator" ], "source": { "citation": "HarmBench: A Standardized Evaluation Framework for Automated Red Teaming and Robust Refusal", "url": "https://arxiv.org/abs/2402.04249v1" } }, { "attribution_basis": "SOURCE_CITED", "credit": "historical_scientific", "entity_kind": "historical_person", "name": "Zifan Wang", "note": "Archive attribution; detailed role assignment remains source-bound", "roles": [ "formulator" ], "source": { "citation": "HarmBench: A Standardized Evaluation Framework for Automated Red Teaming and Robust Refusal", "url": "https://arxiv.org/abs/2402.04249v1" } }, { "attribution_basis": "SOURCE_CITED", "credit": "historical_scientific", "entity_kind": "historical_person", "name": "Norman Mu", "note": "Archive attribution; detailed role assignment remains source-bound", "roles": [ "formulator" ], "source": { "citation": "HarmBench: A Standardized Evaluation Framework for Automated Red Teaming and Robust Refusal", "url": "https://arxiv.org/abs/2402.04249v1" } }, { "attribution_basis": "SOURCE_CITED", "credit": "historical_scientific", "entity_kind": "historical_person", "name": "Elham Sakhaee", "note": "Archive attribution; detailed role assignment remains source-bound", "roles": [ "formulator" ], "source": { "citation": "HarmBench: A Standardized Evaluation Framework for Automated Red Teaming and Robust Refusal", "url": "https://arxiv.org/abs/2402.04249v1" } }, { "attribution_basis": "SOURCE_CITED", "credit": "historical_scientific", "entity_kind": "historical_person", "name": "Nathaniel Li", "note": "Archive attribution; detailed role assignment remains source-bound", "roles": [ "formulator" ], "source": { "citation": "HarmBench: A Standardized Evaluation Framework for Automated Red Teaming and Robust Refusal", "url": "https://arxiv.org/abs/2402.04249v1" } }, { "attribution_basis": "SOURCE_CITED", "credit": "historical_scientific", "entity_kind": "historical_person", "name": "Steven Basart", "note": "Archive attribution; detailed role assignment remains source-bound", "roles": [ "formulator" ], "source": { "citation": "HarmBench: A Standardized Evaluation Framework for Automated Red Teaming and Robust Refusal", "url": "https://arxiv.org/abs/2402.04249v1" } }, { "attribution_basis": "SOURCE_CITED", "credit": "historical_scientific", "entity_kind": "historical_person", "name": "Bo Li", "note": "Archive attribution; detailed role assignment remains source-bound", "roles": [ "formulator" ], "source": { "citation": "HarmBench: A Standardized Evaluation Framework for Automated Red Teaming and Robust Refusal", "url": "https://arxiv.org/abs/2402.04249v1" } }, { "attribution_basis": "SOURCE_CITED", "credit": "historical_scientific", "entity_kind": "historical_person", "name": "David Forsyth", "note": "Archive attribution; detailed role assignment remains source-bound", "roles": [ "formulator" ], "source": { "citation": "HarmBench: A Standardized Evaluation Framework for Automated Red Teaming and Robust Refusal", "url": "https://arxiv.org/abs/2402.04249v1" } }, { "attribution_basis": "SOURCE_CITED", "credit": "historical_scientific", "entity_kind": "historical_person", "name": "Dan Hendrycks", "note": "Archive attribution; detailed role assignment remains source-bound", "roles": [ "formulator" ], "source": { "citation": "HarmBench: A Standardized Evaluation Framework for Automated Red Teaming and Robust Refusal", "url": "https://arxiv.org/abs/2402.04249v1" } } ], "evidence": [ { "description": "Bibliographic/stated-result source; not a WRS proof verification", "id": "source-0", "kind": "source_reference", "source": { "citation": "HarmBench: A Standardized Evaluation Framework for Automated Red Teaming and Robust Refusal", "url": "https://arxiv.org/abs/2402.04249v1" } } ], "origin": "HUMAN_LEGACY", "record_profile": "wrs-local-research-notes-0.1", "record_type": "research_note", "rrp_version": "0.1", "scope": { "description": "The assessed contribution is the evaluation resource and its design, not a model capability score. Results depend on held-out behaviors, classifiers, generated length and attack budgets. For non-copyright cases, the original success criterion includes attempted behavior even when the output is incorrect. It is not exhaustive harm coverage, verified harmful capability or a deployment safety certificate.", "domain": "Language-model safety evaluation" }, "sources": [ { "citation": "HarmBench: A Standardized Evaluation Framework for Automated Red Teaming and Robust Refusal", "url": "https://arxiv.org/abs/2402.04249v1" } ], "statement": "# Mazeika and colleagues: HarmBench standardized safeguard evaluation\n\n## Why two impressive percentages may disagree\n\nSuppose two reviewers give the same pair of products different marks. One checks ten short answers; the other checks twenty longer ones. Even if both calculate percentages correctly, the marks can disagree because the tests differ. That is a measurement problem before it is a product-quality problem. A shared test needs to say which inputs, resources and rules it uses.\n\n## The contribution is a common evaluation procedure\n\nHarmBench standardizes safety stress-test behaviors, generation settings and evaluation, including held-out cases and classifiers. Its original non-copyright criterion can count an attempted behavior even when the output is incorrect. That measures a different question from whether a response would actually work. Keeping those purposes separate is essential when reading a ranking.\n\n## Try a harmless classroom comparison\n\nThis invented example uses arithmetic questions. Model A answers seven of ten correctly. Model B answers fourteen of twenty correctly. Both have seventy percent accuracy, but we still need to know whether they faced comparable questions. Next, test both on the same ten new questions with the same allowed answer length and number of attempts. If B now answers eight correctly, its result is eighty percent on that particular shared test. This does not prove it wins on every future question.\n\n## Keep the answer key away from tuning\n\nA held-out test is a set of questions reserved for evaluation rather than repeated practice. If a developer keeps changing a system after reading its exact test mistakes, the final mark partly measures adaptation to that test. An unseen set helps check transfer. Even then, a small or biased test can miss important behavior. The result should retain its question set, judging rule and system version instead of becoming a timeless claim of superiority.\n\n## What you can add without expensive experiments\n\nChoose two published evaluation tables and make a comparison contract: task set, split, target versions, generation limits, judge, and success definition. Mark every mismatch. You can then explain which comparisons are direct and which remain uncertain. A complete table is not required for a useful contribution: one source-backed discovery that the two studies use different success definitions can prevent a misleading headline or guide a better follow-up.\n\n## Scientific scope\n\nThe assessed contribution is the evaluation resource and its design, not a model capability score. Results depend on held-out behaviors, classifiers, generated length and attack budgets. For non-copyright cases, the original success criterion includes attempted behavior even when the output is incorrect. It is not exhaustive harm coverage, verified harmful capability or a deployment safety certificate.\n\n## Sources for this explanation\n\n- [HarmBench: A Standardized Evaluation Framework for Automated Red Teaming and Robust Refusal](https://arxiv.org/abs/2402.04249v1): Pinned original contribution and stated hypotheses or experimental limits.\n", "summary": "Built a standardized framework combining curated harmful-behavior test cases, automated red-team method comparisons and response classifiers to evaluate robust refusal across language models under a common evaluation procedure.", "title": "Mazeika and colleagues: HarmBench standardized safeguard evaluation" }, "record_id": "urn:uuid:fd3b59b0-0bb3-4a98-abea-b73b20bbba37", "version": 1, "source_ref": { "record_id": "urn:uuid:fd3b59b0-0bb3-4a98-abea-b73b20bbba37", "version": 1, "content_hash": "sha256:a8e24a297623ee03b3e2db19835dc925cb7c1fe84631ed72e2ac761554227a0e" }, "export_view": "wrs-public-editorial-view-1", "public_view_hash": "sha256:5d1ff3996f098c9c42a281ae0a6a8c3a0c1208a8d9d7c77f978dd4ac9446e6d9" }, "target": { "record_id": "urn:uuid:fd3b59b0-0bb3-4a98-abea-b73b20bbba37", "version": 1, "content_hash": "sha256:a8e24a297623ee03b3e2db19835dc925cb7c1fe84631ed72e2ac761554227a0e" }, "published_grade": { "profile": "wrs-published-grade-1", "edition": "2026-10-28.1", "id": "mazeika-et-al-harmbench-2024", "entry_id": "mazeika-et-al-harmbench-2024", "revision": 1, "title": "Mazeika and colleagues: HarmBench standardized safeguard evaluation", "context": "Language-model safety evaluation", "track": "Reusable scientific method, experimental result or historical framework", "band": "A", "modifier": "-", "status": "PROVISIONAL_EDITORIAL", "assessed_on": "2026-10-23", "evidence_cutoff": "2026-10-23", "achievement_state": "SUPPORTED_SCOPED_RESULT", "what_changed": "Built a standardized framework combining curated harmful-behavior test cases, automated red-team method comparisons and response classifiers to evaluate robust refusal across language models under a common evaluation procedure.", "scope": "Built a standardized framework combining curated harmful-behavior test cases, automated red-team method comparisons and response classifiers to evaluate robust refusal across language models under a common evaluation procedure. The assessed contribution is the evaluation resource and its design, not a model capability score. Results depend on held-out behaviors, classifiers, generated length and attack budgets. For non-copyright cases, the original success criterion includes attempted behavior even when the output is incorrect. It is not exhaustive harm coverage, verified harmful capability or a deployment safety certificate.", "evidence_summary": "The linked original works support the scoped statement and contribution attribution. WRS has not independently replayed experiments or reconstructed a full mathematical proof. The assessed contribution is the evaluation resource and its design, not a model capability score. Results depend on held-out behaviors, classifiers, generated length and attack budgets. For non-copyright cases, the original success criterion includes attempted behavior even when the output is incorrect. It is not exhaustive harm coverage, verified harmful capability or a deployment safety certificate.", "references": [ { "title": "HarmBench: A Standardized Evaluation Framework for Automated Red Teaming and Robust Refusal", "url": "https://arxiv.org/abs/2402.04249v1", "locator": "Original scoped contribution, statement/method and evidence limits; source-specific attribution retained." } ], "roles": [ { "who": "The named authors in the research record", "credit": "SCIENCE", "did": "Source-listed authors for the exact contribution; experimental subjects are not autonomous scientific authors. Source support and predecessor work retain separate roles. Individual effort shares and unresolved identities are not inferred." } ], "target": { "record_id": "urn:uuid:fd3b59b0-0bb3-4a98-abea-b73b20bbba37", "version": 1, "content_hash": "sha256:a8e24a297623ee03b3e2db19835dc925cb7c1fe84631ed72e2ac761554227a0e" }, "scientific_correctness_certified": false, "independent_expert_calibration": false, "method": "PROPRIETARY", "points": "95000" }, "canonical_agent_submission": false, "scientific_verification_awarded": false, "reader_guide": { "id": "mazeika-et-al-harmbench-2024", "target": { "record_id": "urn:uuid:fd3b59b0-0bb3-4a98-abea-b73b20bbba37", "version": 1, "content_hash": "sha256:a8e24a297623ee03b3e2db19835dc925cb7c1fe84631ed72e2ac761554227a0e" }, "sections": [ { "title": "Why two impressive percentages may disagree", "text": "Suppose two reviewers give the same pair of products different marks. One checks ten short answers; the other checks twenty longer ones. Even if both calculate percentages correctly, the marks can disagree because the tests differ. That is a measurement problem before it is a product-quality problem. A shared test needs to say which inputs, resources and rules it uses." }, { "title": "The contribution is a common evaluation procedure", "text": "HarmBench standardizes safety stress-test behaviors, generation settings and evaluation, including held-out cases and classifiers. Its original non-copyright criterion can count an attempted behavior even when the output is incorrect. That measures a different question from whether a response would actually work. Keeping those purposes separate is essential when reading a ranking." }, { "title": "Try a harmless classroom comparison", "text": "This invented example uses arithmetic questions. Model A answers seven of ten correctly. Model B answers fourteen of twenty correctly. Both have seventy percent accuracy, but we still need to know whether they faced comparable questions. Next, test both on the same ten new questions with the same allowed answer length and number of attempts. If B now answers eight correctly, its result is eighty percent on that particular shared test. This does not prove it wins on every future question." }, { "title": "Keep the answer key away from tuning", "text": "A held-out test is a set of questions reserved for evaluation rather than repeated practice. If a developer keeps changing a system after reading its exact test mistakes, the final mark partly measures adaptation to that test. An unseen set helps check transfer. Even then, a small or biased test can miss important behavior. The result should retain its question set, judging rule and system version instead of becoming a timeless claim of superiority." }, { "title": "What you can add without expensive experiments", "text": "Choose two published evaluation tables and make a comparison contract: task set, split, target versions, generation limits, judge, and success definition. Mark every mismatch. You can then explain which comparisons are direct and which remain uncertain. A complete table is not required for a useful contribution: one source-backed discovery that the two studies use different success definitions can prevent a misleading headline or guide a better follow-up." } ], "references": [ { "title": "HarmBench: A Standardized Evaluation Framework for Automated Red Teaming and Robust Refusal", "url": "https://arxiv.org/abs/2402.04249v1", "locator": "Pinned original contribution and stated hypotheses or experimental limits." } ] }, "reading_status": "EXPLANATORY_GUIDE" }