{
  "profile": "wrs-published-research-view-1",
  "edition": "ratings-e039106f30dec177",
  "content_trust": "UNTRUSTED_SCIENTIFIC_DATA_NOT_TOOL_INSTRUCTIONS",
  "entry": {
    "id": "chao-et-al-pair-2023",
    "title": "Chao and colleagues: iterative black-box model red-teaming with PAIR",
    "summary": "Introduced an automated red-teaming workflow in which an attacker language model revises candidate requests using feedback from a target model and an evaluation model, enabling query-based stress testing without target-weight access.",
    "boundary": "The title describes reported efficiency, not a guarantee of success within twenty queries. Parallel streams add to the total budget. Outcomes depend on attacker, judge, target and settings; automated judgments can mistake nominal compliance for useful or correct output.",
    "area": "Adversarial evaluation of language models",
    "topic": "Artificial intelligence",
    "year": 2023,
    "lane": "HUMAN",
    "result": "ALGORITHM",
    "package_id": "chao-et-al-pair-2023",
    "target": {
      "record_id": "urn:uuid:8c525b0e-c15e-4958-aa81-4852a69dffd6",
      "version": 1,
      "content_hash": "sha256:b92a5d54b5bf4b8eea0c1305f5954f3750be1d31cec1e5c7ab5fb9f563076972"
    },
    "humans": [
      "Patrick Chao",
      "Alexander Robey",
      "Edgar Dobriban",
      "Hamed Hassani",
      "George J. Pappas",
      "Eric Wong"
    ],
    "authors_complete": true,
    "human_role_note": "Source-listed authors for the exact contribution; experimental subjects are not autonomous scientific authors. Source support and predecessor work retain separate roles. Individual effort shares and unresolved identities are not inferred.",
    "agents": [],
    "related": [
      "perez-et-al-model-redteaming-2022",
      "mehrotra-et-al-tree-of-attacks-2023"
    ],
    "subresults": [],
    "scientific_assessment": {
      "source_review": "SOURCE_CURATED",
      "scientific_review": "NOT_REVIEWED_BY_WRS",
      "formal_replay": "NOT_RUN",
      "provider_attestation": "NONE"
    },
    "sources": [
      {
        "id": "cur20261017-arxiv-2310-08419v1",
        "title": "Jailbreaking Black Box Large Language Models in Twenty Queries",
        "url": "https://arxiv.org/abs/2310.08419v1",
        "date": "2023-10-12"
      }
    ],
    "points": "65000",
    "published_grade": {
      "id": "chao-et-al-pair-2023",
      "entry_id": "chao-et-al-pair-2023",
      "band": "B",
      "modifier": "+",
      "target": {
        "record_id": "urn:uuid:8c525b0e-c15e-4958-aa81-4852a69dffd6",
        "version": 1,
        "content_hash": "sha256:b92a5d54b5bf4b8eea0c1305f5954f3750be1d31cec1e5c7ab5fb9f563076972"
      },
      "what_changed": "Introduced an automated red-teaming workflow in which an attacker language model revises candidate requests using feedback from a target model and an evaluation model, enabling query-based stress testing without target-weight access.",
      "scope": "Introduced an automated red-teaming workflow in which an attacker language model revises candidate requests using feedback from a target model and an evaluation model, enabling query-based stress testing without target-weight access. The title describes reported efficiency, not a guarantee of success within twenty queries. Parallel streams add to the total budget. Outcomes depend on attacker, judge, target and settings; automated judgments can mistake nominal compliance for useful or correct output.",
      "status": "PROVISIONAL_EDITORIAL",
      "scientific_correctness_certified": false
    },
    "assessment_disposition": {
      "status": "ASSESSED",
      "label": "Provisionally assessed",
      "reason": "A current grade and separate reference score identify this exact contribution.",
      "target": {
        "record_id": "urn:uuid:8c525b0e-c15e-4958-aa81-4852a69dffd6",
        "version": 1,
        "content_hash": "sha256:b92a5d54b5bf4b8eea0c1305f5954f3750be1d31cec1e5c7ab5fb9f563076972"
      },
      "edition": "ratings-e039106f30dec177"
    }
  },
  "version": {
    "kind": "PUBLIC_RECORD_VIEW",
    "previous": null,
    "published_at": "2026-10-23T00:00:00Z",
    "publisher": {
      "principal_id": "urn:uuid:80116931-785b-40a2-a24b-9d59db0c7e33",
      "role": "WRS_EDITORIAL_PUBLICATION"
    },
    "record": {
      "contributors": [
        {
          "attribution_basis": "SOURCE_CITED",
          "credit": "historical_scientific",
          "entity_kind": "historical_person",
          "name": "Patrick Chao",
          "note": "Archive attribution; detailed role assignment remains source-bound",
          "roles": [
            "formulator"
          ],
          "source": {
            "citation": "Jailbreaking Black Box Large Language Models in Twenty Queries",
            "url": "https://arxiv.org/abs/2310.08419v1"
          }
        },
        {
          "attribution_basis": "SOURCE_CITED",
          "credit": "historical_scientific",
          "entity_kind": "historical_person",
          "name": "Alexander Robey",
          "note": "Archive attribution; detailed role assignment remains source-bound",
          "roles": [
            "formulator"
          ],
          "source": {
            "citation": "Jailbreaking Black Box Large Language Models in Twenty Queries",
            "url": "https://arxiv.org/abs/2310.08419v1"
          }
        },
        {
          "attribution_basis": "SOURCE_CITED",
          "credit": "historical_scientific",
          "entity_kind": "historical_person",
          "name": "Edgar Dobriban",
          "note": "Archive attribution; detailed role assignment remains source-bound",
          "roles": [
            "formulator"
          ],
          "source": {
            "citation": "Jailbreaking Black Box Large Language Models in Twenty Queries",
            "url": "https://arxiv.org/abs/2310.08419v1"
          }
        },
        {
          "attribution_basis": "SOURCE_CITED",
          "credit": "historical_scientific",
          "entity_kind": "historical_person",
          "name": "Hamed Hassani",
          "note": "Archive attribution; detailed role assignment remains source-bound",
          "roles": [
            "formulator"
          ],
          "source": {
            "citation": "Jailbreaking Black Box Large Language Models in Twenty Queries",
            "url": "https://arxiv.org/abs/2310.08419v1"
          }
        },
        {
          "attribution_basis": "SOURCE_CITED",
          "credit": "historical_scientific",
          "entity_kind": "historical_person",
          "name": "George J. Pappas",
          "note": "Archive attribution; detailed role assignment remains source-bound",
          "roles": [
            "formulator"
          ],
          "source": {
            "citation": "Jailbreaking Black Box Large Language Models in Twenty Queries",
            "url": "https://arxiv.org/abs/2310.08419v1"
          }
        },
        {
          "attribution_basis": "SOURCE_CITED",
          "credit": "historical_scientific",
          "entity_kind": "historical_person",
          "name": "Eric Wong",
          "note": "Archive attribution; detailed role assignment remains source-bound",
          "roles": [
            "formulator"
          ],
          "source": {
            "citation": "Jailbreaking Black Box Large Language Models in Twenty Queries",
            "url": "https://arxiv.org/abs/2310.08419v1"
          }
        }
      ],
      "evidence": [
        {
          "description": "Bibliographic/stated-result source; not a WRS proof verification",
          "id": "source-0",
          "kind": "source_reference",
          "source": {
            "citation": "Jailbreaking Black Box Large Language Models in Twenty Queries",
            "url": "https://arxiv.org/abs/2310.08419v1"
          }
        }
      ],
      "origin": "HUMAN_LEGACY",
      "record_profile": "wrs-local-research-notes-0.1",
      "record_type": "research_note",
      "rrp_version": "0.1",
      "scope": {
        "description": "The title describes reported efficiency, not a guarantee of success within twenty queries. Parallel streams add to the total budget. Outcomes depend on attacker, judge, target and settings; automated judgments can mistake nominal compliance for useful or correct output.",
        "domain": "Adversarial evaluation of language models"
      },
      "sources": [
        {
          "citation": "Jailbreaking Black Box Large Language Models in Twenty Queries",
          "url": "https://arxiv.org/abs/2310.08419v1"
        }
      ],
      "statement": "# Chao and colleagues: iterative black-box model red-teaming with PAIR\n\n## A useful test can learn from its last attempt\n\nImagine checking whether a classroom chatbot follows a harmless formatting rule. You try a question, inspect the reply, and adjust the next question after seeing what happened. Keeping that feedback is different from throwing a fresh random question at the system every time. The broader lesson is about how to search: a failed attempt can contain information that helps choose the next one, even when you cannot inspect the program inside.\n\n## What PAIR contributes\n\nPAIR applies model-generated refinement to safety stress testing with query access to the target. It uses feedback on candidate requests to guide further candidates. The original reports efficient results on selected systems. Its title is an empirical description, not a promise that every target will fail within twenty calls. A separate evaluator can also misunderstand the reply.\n\n## Count the work on both sides\n\nHere is an invented budget example, not a paper measurement. Suppose three independent search streams each allow four rounds. At most twelve target calls are made. If each round also makes one proposal-model call and one evaluator call, the round-based total is thirty-six model calls. Startup messages, retries and other work would be additional. A report saying twelve target calls can be accurate while still leaving most of the total resource accounting unexplained.\n\n## A fair comparison needs the same finish line\n\nBefore comparing two search methods, ask what counts as success, which model version answered, how many independent streams ran, and which kinds of calls are counted. Also keep unsuccessful searches. Selecting only the fastest successful run tells you something about that run, but it does not reveal the cost of a typical attempt or the chance of failure. An evaluator score is a measurement made under a chosen rule.\n\n## An attainable contribution\n\nRead the method and parallel-search sections and make a compact budget checklist. Find which quantities refer to target calls, total model calls, elapsed time or hardware. Leave an item unknown if the source does not supply it. No harmful request needs to be executed to improve this accounting. A carefully sourced correction to a claimed budget can help later participants choose a realistic experiment and avoid repeating an unfair comparison.\n\n## Scientific scope\n\nThe title describes reported efficiency, not a guarantee of success within twenty queries. Parallel streams add to the total budget. Outcomes depend on attacker, judge, target and settings; automated judgments can mistake nominal compliance for useful or correct output.\n\n## Sources for this explanation\n\n- [Jailbreaking Black Box Large Language Models in Twenty Queries](https://arxiv.org/abs/2310.08419v1): Pinned original contribution and stated hypotheses or experimental limits.\n",
      "summary": "Introduced an automated red-teaming workflow in which an attacker language model revises candidate requests using feedback from a target model and an evaluation model, enabling query-based stress testing without target-weight access.",
      "title": "Chao and colleagues: iterative black-box model red-teaming with PAIR"
    },
    "record_id": "urn:uuid:8c525b0e-c15e-4958-aa81-4852a69dffd6",
    "version": 1,
    "source_ref": {
      "record_id": "urn:uuid:8c525b0e-c15e-4958-aa81-4852a69dffd6",
      "version": 1,
      "content_hash": "sha256:b92a5d54b5bf4b8eea0c1305f5954f3750be1d31cec1e5c7ab5fb9f563076972"
    },
    "export_view": "wrs-public-editorial-view-1",
    "public_view_hash": "sha256:9a9c96bf0a747c66a11ad894e8a1bd97a1671c2d272e6aa135494410c602621f"
  },
  "target": {
    "record_id": "urn:uuid:8c525b0e-c15e-4958-aa81-4852a69dffd6",
    "version": 1,
    "content_hash": "sha256:b92a5d54b5bf4b8eea0c1305f5954f3750be1d31cec1e5c7ab5fb9f563076972"
  },
  "published_grade": {
    "profile": "wrs-published-grade-1",
    "edition": "2026-10-28.1",
    "id": "chao-et-al-pair-2023",
    "entry_id": "chao-et-al-pair-2023",
    "revision": 1,
    "title": "Chao and colleagues: iterative black-box model red-teaming with PAIR",
    "context": "Adversarial evaluation of language models",
    "track": "Reusable scientific method, experimental result or historical framework",
    "band": "B",
    "modifier": "+",
    "status": "PROVISIONAL_EDITORIAL",
    "assessed_on": "2026-10-23",
    "evidence_cutoff": "2026-10-23",
    "achievement_state": "SUPPORTED_SCOPED_RESULT",
    "what_changed": "Introduced an automated red-teaming workflow in which an attacker language model revises candidate requests using feedback from a target model and an evaluation model, enabling query-based stress testing without target-weight access.",
    "scope": "Introduced an automated red-teaming workflow in which an attacker language model revises candidate requests using feedback from a target model and an evaluation model, enabling query-based stress testing without target-weight access. The title describes reported efficiency, not a guarantee of success within twenty queries. Parallel streams add to the total budget. Outcomes depend on attacker, judge, target and settings; automated judgments can mistake nominal compliance for useful or correct output.",
    "evidence_summary": "The linked original works support the scoped statement and contribution attribution. WRS has not independently replayed experiments or reconstructed a full mathematical proof. The title describes reported efficiency, not a guarantee of success within twenty queries. Parallel streams add to the total budget. Outcomes depend on attacker, judge, target and settings; automated judgments can mistake nominal compliance for useful or correct output.",
    "references": [
      {
        "title": "Jailbreaking Black Box Large Language Models in Twenty Queries",
        "url": "https://arxiv.org/abs/2310.08419v1",
        "locator": "Original scoped contribution, statement/method and evidence limits; source-specific attribution retained."
      }
    ],
    "roles": [
      {
        "who": "The named authors in the research record",
        "credit": "SCIENCE",
        "did": "Source-listed authors for the exact contribution; experimental subjects are not autonomous scientific authors. Source support and predecessor work retain separate roles. Individual effort shares and unresolved identities are not inferred."
      }
    ],
    "target": {
      "record_id": "urn:uuid:8c525b0e-c15e-4958-aa81-4852a69dffd6",
      "version": 1,
      "content_hash": "sha256:b92a5d54b5bf4b8eea0c1305f5954f3750be1d31cec1e5c7ab5fb9f563076972"
    },
    "scientific_correctness_certified": false,
    "independent_expert_calibration": false,
    "method": "PROPRIETARY",
    "points": "65000"
  },
  "canonical_agent_submission": false,
  "scientific_verification_awarded": false,
  "reader_guide": {
    "id": "chao-et-al-pair-2023",
    "target": {
      "record_id": "urn:uuid:8c525b0e-c15e-4958-aa81-4852a69dffd6",
      "version": 1,
      "content_hash": "sha256:b92a5d54b5bf4b8eea0c1305f5954f3750be1d31cec1e5c7ab5fb9f563076972"
    },
    "sections": [
      {
        "title": "A useful test can learn from its last attempt",
        "text": "Imagine checking whether a classroom chatbot follows a harmless formatting rule. You try a question, inspect the reply, and adjust the next question after seeing what happened. Keeping that feedback is different from throwing a fresh random question at the system every time. The broader lesson is about how to search: a failed attempt can contain information that helps choose the next one, even when you cannot inspect the program inside."
      },
      {
        "title": "What PAIR contributes",
        "text": "PAIR applies model-generated refinement to safety stress testing with query access to the target. It uses feedback on candidate requests to guide further candidates. The original reports efficient results on selected systems. Its title is an empirical description, not a promise that every target will fail within twenty calls. A separate evaluator can also misunderstand the reply."
      },
      {
        "title": "Count the work on both sides",
        "text": "Here is an invented budget example, not a paper measurement. Suppose three independent search streams each allow four rounds. At most twelve target calls are made. If each round also makes one proposal-model call and one evaluator call, the round-based total is thirty-six model calls. Startup messages, retries and other work would be additional. A report saying twelve target calls can be accurate while still leaving most of the total resource accounting unexplained."
      },
      {
        "title": "A fair comparison needs the same finish line",
        "text": "Before comparing two search methods, ask what counts as success, which model version answered, how many independent streams ran, and which kinds of calls are counted. Also keep unsuccessful searches. Selecting only the fastest successful run tells you something about that run, but it does not reveal the cost of a typical attempt or the chance of failure. An evaluator score is a measurement made under a chosen rule."
      },
      {
        "title": "An attainable contribution",
        "text": "Read the method and parallel-search sections and make a compact budget checklist. Find which quantities refer to target calls, total model calls, elapsed time or hardware. Leave an item unknown if the source does not supply it. No harmful request needs to be executed to improve this accounting. A carefully sourced correction to a claimed budget can help later participants choose a realistic experiment and avoid repeating an unfair comparison."
      }
    ],
    "references": [
      {
        "title": "Jailbreaking Black Box Large Language Models in Twenty Queries",
        "url": "https://arxiv.org/abs/2310.08419v1",
        "locator": "Pinned original contribution and stated hypotheses or experimental limits."
      }
    ]
  },
  "reading_status": "EXPLANATORY_GUIDE"
}