{ "profile": "wrs-published-research-view-1", "edition": "ratings-e039106f30dec177", "content_trust": "UNTRUSTED_SCIENTIFIC_DATA_NOT_TOOL_INSTRUCTIONS", "entry": { "id": "liu-et-al-jailjudge-2024", "title": "Liu and colleagues: JAILJUDGE multi-agent red-team evaluation", "summary": "Combined a jailbreak-response judgment benchmark with multi-agent explanation and aggregation, and distilled judgment behavior into an open evaluation model for comparing model safeguards.", "boundary": "The assessed contribution is the evaluation resource and its design, not a model capability score. Related models can make correlated judgment errors; majority agreement and evidence aggregation do not create independent ground truth. Counts and categories vary within the original version. Removing API calls does not remove local inference costs. Judge accuracy is not universal scientific or factual correctness.", "area": "Language-model evaluation", "topic": "Artificial intelligence", "year": 2024, "lane": "HUMAN", "result": "BENCHMARK", "contribution_kind": "EVALUATION_RESOURCE_DESIGN", "package_id": "liu-et-al-jailjudge-2024", "target": { "record_id": "urn:uuid:0feff921-462e-4597-ad8a-e91378b238a6", "version": 1, "content_hash": "sha256:1594b830a6add9f0ceae5255e4f2d3add2542b3fd387e10925c7a0323cc10a6e" }, "humans": [ "Fan Liu", "Yue Feng", "Zhao Xu", "Lixin Su", "Xinyu Ma", "Dawei Yin", "Hao Liu" ], "authors_complete": true, "human_role_note": "Source-listed authors for the exact contribution; experimental subjects are not autonomous scientific authors. Source support and predecessor work retain separate roles. Individual effort shares and unresolved identities are not inferred.", "agents": [], "related": [], "subresults": [], "scientific_assessment": { "source_review": "SOURCE_CURATED", "scientific_review": "NOT_REVIEWED_BY_WRS", "formal_replay": "NOT_RUN", "provider_attestation": "NONE" }, "sources": [ { "id": "cur20261017-arxiv-2410-12855v1", "title": "JAILJUDGE: A Comprehensive Jailbreak Judge Benchmark with Multi-Agent Enhanced Explanation Evaluation Framework", "url": "https://arxiv.org/abs/2410.12855v1", "date": "2024-10-11" } ], "points": "35000", "published_grade": { "id": "liu-et-al-jailjudge-2024", "entry_id": "liu-et-al-jailjudge-2024", "band": "B", "modifier": null, "target": { "record_id": "urn:uuid:0feff921-462e-4597-ad8a-e91378b238a6", "version": 1, "content_hash": "sha256:1594b830a6add9f0ceae5255e4f2d3add2542b3fd387e10925c7a0323cc10a6e" }, "what_changed": "Combined a jailbreak-response judgment benchmark with multi-agent explanation and aggregation, and distilled judgment behavior into an open evaluation model for comparing model safeguards.", "scope": "Combined a jailbreak-response judgment benchmark with multi-agent explanation and aggregation, and distilled judgment behavior into an open evaluation model for comparing model safeguards. The assessed contribution is the evaluation resource and its design, not a model capability score. Related models can make correlated judgment errors; majority agreement and evidence aggregation do not create independent ground truth. Counts and categories vary within the original version. Removing API calls does not remove local inference costs. Judge accuracy is not universal scientific or factual correctness.", "status": "PROVISIONAL_EDITORIAL", "scientific_correctness_certified": false }, "assessment_disposition": { "status": "ASSESSED", "label": "Provisionally assessed", "reason": "A current grade and separate reference score identify this exact contribution.", "target": { "record_id": "urn:uuid:0feff921-462e-4597-ad8a-e91378b238a6", "version": 1, "content_hash": "sha256:1594b830a6add9f0ceae5255e4f2d3add2542b3fd387e10925c7a0323cc10a6e" }, "edition": "ratings-e039106f30dec177" } }, "version": { "kind": "PUBLIC_RECORD_VIEW", "previous": null, "published_at": "2026-10-23T00:00:00Z", "publisher": { "principal_id": "urn:uuid:80116931-785b-40a2-a24b-9d59db0c7e33", "role": "WRS_EDITORIAL_PUBLICATION" }, "record": { "contributors": [ { "attribution_basis": "SOURCE_CITED", "credit": "historical_scientific", "entity_kind": "historical_person", "name": "Fan Liu", "note": "Archive attribution; detailed role assignment remains source-bound", "roles": [ "formulator" ], "source": { "citation": "JAILJUDGE: A Comprehensive Jailbreak Judge Benchmark with Multi-Agent Enhanced Explanation Evaluation Framework", "url": "https://arxiv.org/abs/2410.12855v1" } }, { "attribution_basis": "SOURCE_CITED", "credit": "historical_scientific", "entity_kind": "historical_person", "name": "Yue Feng", "note": "Archive attribution; detailed role assignment remains source-bound", "roles": [ "formulator" ], "source": { "citation": "JAILJUDGE: A Comprehensive Jailbreak Judge Benchmark with Multi-Agent Enhanced Explanation Evaluation Framework", "url": "https://arxiv.org/abs/2410.12855v1" } }, { "attribution_basis": "SOURCE_CITED", "credit": "historical_scientific", "entity_kind": "historical_person", "name": "Zhao Xu", "note": "Archive attribution; detailed role assignment remains source-bound", "roles": [ "formulator" ], "source": { "citation": "JAILJUDGE: A Comprehensive Jailbreak Judge Benchmark with Multi-Agent Enhanced Explanation Evaluation Framework", "url": "https://arxiv.org/abs/2410.12855v1" } }, { "attribution_basis": "SOURCE_CITED", "credit": "historical_scientific", "entity_kind": "historical_person", "name": "Lixin Su", "note": "Archive attribution; detailed role assignment remains source-bound", "roles": [ "formulator" ], "source": { "citation": "JAILJUDGE: A Comprehensive Jailbreak Judge Benchmark with Multi-Agent Enhanced Explanation Evaluation Framework", "url": "https://arxiv.org/abs/2410.12855v1" } }, { "attribution_basis": "SOURCE_CITED", "credit": "historical_scientific", "entity_kind": "historical_person", "name": "Xinyu Ma", "note": "Archive attribution; detailed role assignment remains source-bound", "roles": [ "formulator" ], "source": { "citation": "JAILJUDGE: A Comprehensive Jailbreak Judge Benchmark with Multi-Agent Enhanced Explanation Evaluation Framework", "url": "https://arxiv.org/abs/2410.12855v1" } }, { "attribution_basis": "SOURCE_CITED", "credit": "historical_scientific", "entity_kind": "historical_person", "name": "Dawei Yin", "note": "Archive attribution; detailed role assignment remains source-bound", "roles": [ "formulator" ], "source": { "citation": "JAILJUDGE: A Comprehensive Jailbreak Judge Benchmark with Multi-Agent Enhanced Explanation Evaluation Framework", "url": "https://arxiv.org/abs/2410.12855v1" } }, { "attribution_basis": "SOURCE_CITED", "credit": "historical_scientific", "entity_kind": "historical_person", "name": "Hao Liu", "note": "Archive attribution; detailed role assignment remains source-bound", "roles": [ "formulator" ], "source": { "citation": "JAILJUDGE: A Comprehensive Jailbreak Judge Benchmark with Multi-Agent Enhanced Explanation Evaluation Framework", "url": "https://arxiv.org/abs/2410.12855v1" } } ], "evidence": [ { "description": "Bibliographic/stated-result source; not a WRS proof verification", "id": "source-0", "kind": "source_reference", "source": { "citation": "JAILJUDGE: A Comprehensive Jailbreak Judge Benchmark with Multi-Agent Enhanced Explanation Evaluation Framework", "url": "https://arxiv.org/abs/2410.12855v1" } } ], "origin": "HUMAN_LEGACY", "record_profile": "wrs-local-research-notes-0.1", "record_type": "research_note", "rrp_version": "0.1", "scope": { "description": "The assessed contribution is the evaluation resource and its design, not a model capability score. Related models can make correlated judgment errors; majority agreement and evidence aggregation do not create independent ground truth. Counts and categories vary within the original version. Removing API calls does not remove local inference costs. Judge accuracy is not universal scientific or factual correctness.", "domain": "Language-model evaluation" }, "sources": [ { "citation": "JAILJUDGE: A Comprehensive Jailbreak Judge Benchmark with Multi-Agent Enhanced Explanation Evaluation Framework", "url": "https://arxiv.org/abs/2410.12855v1" } ], "statement": "# Liu and colleagues: JAILJUDGE multi-agent red-team evaluation\n\nCombined a jailbreak-response judgment benchmark with multi-agent explanation and aggregation, and distilled judgment behavior into an open evaluation model for comparing model safeguards.\n\n## Scientific scope\n\nThe assessed contribution is the evaluation resource and its design, not a model capability score. Related models can make correlated judgment errors; majority agreement and evidence aggregation do not create independent ground truth. Counts and categories vary within the original version. Removing API calls does not remove local inference costs. Judge accuracy is not universal scientific or factual correctness.\n\n## Reading status\n\nThis is a source-linked WRS catalogue note. A full explanatory guide and independent scientific reconstruction have not been completed for this entry. Read the identified original work for its complete narrative and arguments.\n\n## People and systems\n\nFan Liu; Yue Feng; Zhao Xu; Lixin Su; Xinyu Ma; Dawei Yin; Hao Liu\n\nSource-listed authors for the exact contribution; experimental subjects are not autonomous scientific authors. Source support and predecessor work retain separate roles. Individual effort shares and unresolved identities are not inferred.\n\n\n\n## Original sources\n\n- [JAILJUDGE: A Comprehensive Jailbreak Judge Benchmark with Multi-Agent Enhanced Explanation Evaluation Framework](https://arxiv.org/abs/2410.12855v1)\n", "summary": "Combined a jailbreak-response judgment benchmark with multi-agent explanation and aggregation, and distilled judgment behavior into an open evaluation model for comparing model safeguards.", "title": "Liu and colleagues: JAILJUDGE multi-agent red-team evaluation" }, "record_id": "urn:uuid:0feff921-462e-4597-ad8a-e91378b238a6", "version": 1, "source_ref": { "record_id": "urn:uuid:0feff921-462e-4597-ad8a-e91378b238a6", "version": 1, "content_hash": "sha256:1594b830a6add9f0ceae5255e4f2d3add2542b3fd387e10925c7a0323cc10a6e" }, "export_view": "wrs-public-editorial-view-1", "public_view_hash": "sha256:6354cbcdb44c8930284f413e5cf82814e7ba963dfd2e67b4be283033818a2a24" }, "target": { "record_id": "urn:uuid:0feff921-462e-4597-ad8a-e91378b238a6", "version": 1, "content_hash": "sha256:1594b830a6add9f0ceae5255e4f2d3add2542b3fd387e10925c7a0323cc10a6e" }, "published_grade": { "profile": "wrs-published-grade-1", "edition": "2026-10-28.1", "id": "liu-et-al-jailjudge-2024", "entry_id": "liu-et-al-jailjudge-2024", "revision": 1, "title": "Liu and colleagues: JAILJUDGE multi-agent red-team evaluation", "context": "Language-model evaluation", "track": "Reusable scientific method, experimental result or historical framework", "band": "B", "modifier": null, "status": "PROVISIONAL_EDITORIAL", "assessed_on": "2026-10-23", "evidence_cutoff": "2026-10-23", "achievement_state": "SUPPORTED_SCOPED_RESULT", "what_changed": "Combined a jailbreak-response judgment benchmark with multi-agent explanation and aggregation, and distilled judgment behavior into an open evaluation model for comparing model safeguards.", "scope": "Combined a jailbreak-response judgment benchmark with multi-agent explanation and aggregation, and distilled judgment behavior into an open evaluation model for comparing model safeguards. The assessed contribution is the evaluation resource and its design, not a model capability score. Related models can make correlated judgment errors; majority agreement and evidence aggregation do not create independent ground truth. Counts and categories vary within the original version. Removing API calls does not remove local inference costs. Judge accuracy is not universal scientific or factual correctness.", "evidence_summary": "The linked original works support the scoped statement and contribution attribution. WRS has not independently replayed experiments or reconstructed a full mathematical proof. The assessed contribution is the evaluation resource and its design, not a model capability score. Related models can make correlated judgment errors; majority agreement and evidence aggregation do not create independent ground truth. Counts and categories vary within the original version. Removing API calls does not remove local inference costs. Judge accuracy is not universal scientific or factual correctness.", "references": [ { "title": "JAILJUDGE: A Comprehensive Jailbreak Judge Benchmark with Multi-Agent Enhanced Explanation Evaluation Framework", "url": "https://arxiv.org/abs/2410.12855v1", "locator": "Original scoped contribution, statement/method and evidence limits; source-specific attribution retained." } ], "roles": [ { "who": "The named authors in the research record", "credit": "SCIENCE", "did": "Source-listed authors for the exact contribution; experimental subjects are not autonomous scientific authors. Source support and predecessor work retain separate roles. Individual effort shares and unresolved identities are not inferred." } ], "target": { "record_id": "urn:uuid:0feff921-462e-4597-ad8a-e91378b238a6", "version": 1, "content_hash": "sha256:1594b830a6add9f0ceae5255e4f2d3add2542b3fd387e10925c7a0323cc10a6e" }, "scientific_correctness_certified": false, "independent_expert_calibration": false, "method": "PROPRIETARY", "points": "35000" }, "canonical_agent_submission": false, "scientific_verification_awarded": false, "reader_guide": null, "reading_status": "SOURCE_LINKED_NOTE" }