{
  "profile": "wrs-published-research-view-1",
  "edition": "ratings-e039106f30dec177",
  "content_trust": "UNTRUSTED_SCIENTIFIC_DATA_NOT_TOOL_INSTRUCTIONS",
  "entry": {
    "id": "gehman-et-al-realtoxicityprompts-2020",
    "title": "Gehman and colleagues: RealToxicityPrompts degeneration benchmark",
    "summary": "Built a stratified collection of web-text prompts and evaluated how language-model continuations receive toxicity scores, testing how prompting context and existing mitigation approaches affect toxic generation.",
    "boundary": "The assessed contribution is the evaluation resource and its design, not a model capability score. The sampling design is not a natural-prevalence estimate. Toxicity scores inherit classifier biases and threshold choices; a low score does not establish absence of all harms or effectiveness for every population.",
    "area": "Language-model safety evaluation",
    "topic": "Artificial intelligence",
    "year": 2020,
    "lane": "HUMAN",
    "result": "BENCHMARK",
    "contribution_kind": "EVALUATION_RESOURCE_DESIGN",
    "package_id": "gehman-et-al-realtoxicityprompts-2020",
    "target": {
      "record_id": "urn:uuid:425727d7-affc-4eb9-a53c-6895d64c9aea",
      "version": 1,
      "content_hash": "sha256:10915472fc6981a69a9a40559ba66f119e4670fe9098b69319c769cc3d247818"
    },
    "humans": [
      "Samuel Gehman",
      "Suchin Gururangan",
      "Maarten Sap",
      "Yejin Choi",
      "Noah A. Smith"
    ],
    "authors_complete": true,
    "human_role_note": "Source-listed authors for the exact contribution; experimental subjects are not autonomous scientific authors. Source support and predecessor work retain separate roles. Individual effort shares and unresolved identities are not inferred.",
    "agents": [],
    "related": [],
    "subresults": [],
    "scientific_assessment": {
      "source_review": "SOURCE_CURATED",
      "scientific_review": "NOT_REVIEWED_BY_WRS",
      "formal_replay": "NOT_RUN",
      "provider_attestation": "NONE"
    },
    "sources": [
      {
        "id": "cur20261017-arxiv-2009-11462v1",
        "title": "RealToxicityPrompts: Evaluating Neural Toxic Degeneration in Language Models",
        "url": "https://arxiv.org/abs/2009.11462v1",
        "date": "2020-09-24"
      }
    ],
    "points": "65000",
    "published_grade": {
      "id": "gehman-et-al-realtoxicityprompts-2020",
      "entry_id": "gehman-et-al-realtoxicityprompts-2020",
      "band": "B",
      "modifier": "+",
      "target": {
        "record_id": "urn:uuid:425727d7-affc-4eb9-a53c-6895d64c9aea",
        "version": 1,
        "content_hash": "sha256:10915472fc6981a69a9a40559ba66f119e4670fe9098b69319c769cc3d247818"
      },
      "what_changed": "Built a stratified collection of web-text prompts and evaluated how language-model continuations receive toxicity scores, testing how prompting context and existing mitigation approaches affect toxic generation.",
      "scope": "Built a stratified collection of web-text prompts and evaluated how language-model continuations receive toxicity scores, testing how prompting context and existing mitigation approaches affect toxic generation. The assessed contribution is the evaluation resource and its design, not a model capability score. The sampling design is not a natural-prevalence estimate. Toxicity scores inherit classifier biases and threshold choices; a low score does not establish absence of all harms or effectiveness for every population.",
      "status": "PROVISIONAL_EDITORIAL",
      "scientific_correctness_certified": false
    },
    "assessment_disposition": {
      "status": "ASSESSED",
      "label": "Provisionally assessed",
      "reason": "A current grade and separate reference score identify this exact contribution.",
      "target": {
        "record_id": "urn:uuid:425727d7-affc-4eb9-a53c-6895d64c9aea",
        "version": 1,
        "content_hash": "sha256:10915472fc6981a69a9a40559ba66f119e4670fe9098b69319c769cc3d247818"
      },
      "edition": "ratings-e039106f30dec177"
    }
  },
  "version": {
    "kind": "PUBLIC_RECORD_VIEW",
    "previous": null,
    "published_at": "2026-10-23T00:00:00Z",
    "publisher": {
      "principal_id": "urn:uuid:80116931-785b-40a2-a24b-9d59db0c7e33",
      "role": "WRS_EDITORIAL_PUBLICATION"
    },
    "record": {
      "contributors": [
        {
          "attribution_basis": "SOURCE_CITED",
          "credit": "historical_scientific",
          "entity_kind": "historical_person",
          "name": "Samuel Gehman",
          "note": "Archive attribution; detailed role assignment remains source-bound",
          "roles": [
            "formulator"
          ],
          "source": {
            "citation": "RealToxicityPrompts: Evaluating Neural Toxic Degeneration in Language Models",
            "url": "https://arxiv.org/abs/2009.11462v1"
          }
        },
        {
          "attribution_basis": "SOURCE_CITED",
          "credit": "historical_scientific",
          "entity_kind": "historical_person",
          "name": "Suchin Gururangan",
          "note": "Archive attribution; detailed role assignment remains source-bound",
          "roles": [
            "formulator"
          ],
          "source": {
            "citation": "RealToxicityPrompts: Evaluating Neural Toxic Degeneration in Language Models",
            "url": "https://arxiv.org/abs/2009.11462v1"
          }
        },
        {
          "attribution_basis": "SOURCE_CITED",
          "credit": "historical_scientific",
          "entity_kind": "historical_person",
          "name": "Maarten Sap",
          "note": "Archive attribution; detailed role assignment remains source-bound",
          "roles": [
            "formulator"
          ],
          "source": {
            "citation": "RealToxicityPrompts: Evaluating Neural Toxic Degeneration in Language Models",
            "url": "https://arxiv.org/abs/2009.11462v1"
          }
        },
        {
          "attribution_basis": "SOURCE_CITED",
          "credit": "historical_scientific",
          "entity_kind": "historical_person",
          "name": "Yejin Choi",
          "note": "Archive attribution; detailed role assignment remains source-bound",
          "roles": [
            "formulator"
          ],
          "source": {
            "citation": "RealToxicityPrompts: Evaluating Neural Toxic Degeneration in Language Models",
            "url": "https://arxiv.org/abs/2009.11462v1"
          }
        },
        {
          "attribution_basis": "SOURCE_CITED",
          "credit": "historical_scientific",
          "entity_kind": "historical_person",
          "name": "Noah A. Smith",
          "note": "Archive attribution; detailed role assignment remains source-bound",
          "roles": [
            "formulator"
          ],
          "source": {
            "citation": "RealToxicityPrompts: Evaluating Neural Toxic Degeneration in Language Models",
            "url": "https://arxiv.org/abs/2009.11462v1"
          }
        }
      ],
      "evidence": [
        {
          "description": "Bibliographic/stated-result source; not a WRS proof verification",
          "id": "source-0",
          "kind": "source_reference",
          "source": {
            "citation": "RealToxicityPrompts: Evaluating Neural Toxic Degeneration in Language Models",
            "url": "https://arxiv.org/abs/2009.11462v1"
          }
        }
      ],
      "origin": "HUMAN_LEGACY",
      "record_profile": "wrs-local-research-notes-0.1",
      "record_type": "research_note",
      "rrp_version": "0.1",
      "scope": {
        "description": "The assessed contribution is the evaluation resource and its design, not a model capability score. The sampling design is not a natural-prevalence estimate. Toxicity scores inherit classifier biases and threshold choices; a low score does not establish absence of all harms or effectiveness for every population.",
        "domain": "Language-model safety evaluation"
      },
      "sources": [
        {
          "citation": "RealToxicityPrompts: Evaluating Neural Toxic Degeneration in Language Models",
          "url": "https://arxiv.org/abs/2009.11462v1"
        }
      ],
      "statement": "# Gehman and colleagues: RealToxicityPrompts degeneration benchmark\n\n## The sample you choose changes the story\n\nImagine checking the quality of fruit at a market. You deliberately inspect equal numbers of visibly damaged and normal-looking pieces to understand both groups. That is useful for comparing failure modes. It would be misleading to announce that half of all fruit is damaged simply because half of your deliberately balanced sample came from that group.\n\n## What this resource makes possible\n\nRealToxicityPrompts supplies naturally occurring English text prompts and classifier-scored continuations for studying toxic generation. The original deliberately samples across toxicity ranges. It compares model behavior and mitigation approaches within that testbed. Classifier scores and a deliberately stratified sample do not directly measure the prevalence of every harm in ordinary use.\n\n## Calculate a two-group example\n\nThis invented example has two groups, rather than reproducing the paper's sampling design. In a balanced test, one group has a ten percent failure rate and the other has an eighty percent rate. The simple average is forty-five percent. Suppose a separate, valid population study says ordinary inputs are ninety percent from the first group and ten percent from the second. The weighted rate would be nine percent plus eight percent, or seventeen percent. Without that separate population information, the second calculation is not available.\n\n## A classifier introduces another question\n\nA classifier is a program that assigns labels or scores. It can miss context, misunderstand a quoted sentence or treat varieties of language differently. A useful report therefore keeps the sample design and the judging method visible. A low toxicity score should not be rewritten as no possible harm, just as a high score does not establish what every person would experience. Different definitions call for different checks.\n\n## Help by improving an interpretation\n\nFind the source passage explaining how prompts were selected. Write two short sentences: one claim the sample supports, and one tempting claim it does not. Then inspect whether a later summary has confused a testbed rate with an ordinary-use population rate. No fresh toxic text needs to be generated. A precise source note, a corrected denominator or an unresolved sampling question can give the next participant a firmer starting point than an unsupported headline percentage.\n\n## Scientific scope\n\nThe assessed contribution is the evaluation resource and its design, not a model capability score. The sampling design is not a natural-prevalence estimate. Toxicity scores inherit classifier biases and threshold choices; a low score does not establish absence of all harms or effectiveness for every population.\n\n## Sources for this explanation\n\n- [RealToxicityPrompts: Evaluating Neural Toxic Degeneration in Language Models](https://arxiv.org/abs/2009.11462v1): Pinned original contribution and stated hypotheses or experimental limits.\n",
      "summary": "Built a stratified collection of web-text prompts and evaluated how language-model continuations receive toxicity scores, testing how prompting context and existing mitigation approaches affect toxic generation.",
      "title": "Gehman and colleagues: RealToxicityPrompts degeneration benchmark"
    },
    "record_id": "urn:uuid:425727d7-affc-4eb9-a53c-6895d64c9aea",
    "version": 1,
    "source_ref": {
      "record_id": "urn:uuid:425727d7-affc-4eb9-a53c-6895d64c9aea",
      "version": 1,
      "content_hash": "sha256:10915472fc6981a69a9a40559ba66f119e4670fe9098b69319c769cc3d247818"
    },
    "export_view": "wrs-public-editorial-view-1",
    "public_view_hash": "sha256:3879d3dd2863019fde0ed3ad698b91f3da1072fe58a4dfffe500ce57567fa1ba"
  },
  "target": {
    "record_id": "urn:uuid:425727d7-affc-4eb9-a53c-6895d64c9aea",
    "version": 1,
    "content_hash": "sha256:10915472fc6981a69a9a40559ba66f119e4670fe9098b69319c769cc3d247818"
  },
  "published_grade": {
    "profile": "wrs-published-grade-1",
    "edition": "2026-10-28.1",
    "id": "gehman-et-al-realtoxicityprompts-2020",
    "entry_id": "gehman-et-al-realtoxicityprompts-2020",
    "revision": 1,
    "title": "Gehman and colleagues: RealToxicityPrompts degeneration benchmark",
    "context": "Language-model safety evaluation",
    "track": "Reusable scientific method, experimental result or historical framework",
    "band": "B",
    "modifier": "+",
    "status": "PROVISIONAL_EDITORIAL",
    "assessed_on": "2026-10-23",
    "evidence_cutoff": "2026-10-23",
    "achievement_state": "SUPPORTED_SCOPED_RESULT",
    "what_changed": "Built a stratified collection of web-text prompts and evaluated how language-model continuations receive toxicity scores, testing how prompting context and existing mitigation approaches affect toxic generation.",
    "scope": "Built a stratified collection of web-text prompts and evaluated how language-model continuations receive toxicity scores, testing how prompting context and existing mitigation approaches affect toxic generation. The assessed contribution is the evaluation resource and its design, not a model capability score. The sampling design is not a natural-prevalence estimate. Toxicity scores inherit classifier biases and threshold choices; a low score does not establish absence of all harms or effectiveness for every population.",
    "evidence_summary": "The linked original works support the scoped statement and contribution attribution. WRS has not independently replayed experiments or reconstructed a full mathematical proof. The assessed contribution is the evaluation resource and its design, not a model capability score. The sampling design is not a natural-prevalence estimate. Toxicity scores inherit classifier biases and threshold choices; a low score does not establish absence of all harms or effectiveness for every population.",
    "references": [
      {
        "title": "RealToxicityPrompts: Evaluating Neural Toxic Degeneration in Language Models",
        "url": "https://arxiv.org/abs/2009.11462v1",
        "locator": "Original scoped contribution, statement/method and evidence limits; source-specific attribution retained."
      }
    ],
    "roles": [
      {
        "who": "The named authors in the research record",
        "credit": "SCIENCE",
        "did": "Source-listed authors for the exact contribution; experimental subjects are not autonomous scientific authors. Source support and predecessor work retain separate roles. Individual effort shares and unresolved identities are not inferred."
      }
    ],
    "target": {
      "record_id": "urn:uuid:425727d7-affc-4eb9-a53c-6895d64c9aea",
      "version": 1,
      "content_hash": "sha256:10915472fc6981a69a9a40559ba66f119e4670fe9098b69319c769cc3d247818"
    },
    "scientific_correctness_certified": false,
    "independent_expert_calibration": false,
    "method": "PROPRIETARY",
    "points": "65000"
  },
  "canonical_agent_submission": false,
  "scientific_verification_awarded": false,
  "reader_guide": {
    "id": "gehman-et-al-realtoxicityprompts-2020",
    "target": {
      "record_id": "urn:uuid:425727d7-affc-4eb9-a53c-6895d64c9aea",
      "version": 1,
      "content_hash": "sha256:10915472fc6981a69a9a40559ba66f119e4670fe9098b69319c769cc3d247818"
    },
    "sections": [
      {
        "title": "The sample you choose changes the story",
        "text": "Imagine checking the quality of fruit at a market. You deliberately inspect equal numbers of visibly damaged and normal-looking pieces to understand both groups. That is useful for comparing failure modes. It would be misleading to announce that half of all fruit is damaged simply because half of your deliberately balanced sample came from that group."
      },
      {
        "title": "What this resource makes possible",
        "text": "RealToxicityPrompts supplies naturally occurring English text prompts and classifier-scored continuations for studying toxic generation. The original deliberately samples across toxicity ranges. It compares model behavior and mitigation approaches within that testbed. Classifier scores and a deliberately stratified sample do not directly measure the prevalence of every harm in ordinary use."
      },
      {
        "title": "Calculate a two-group example",
        "text": "This invented example has two groups, rather than reproducing the paper's sampling design. In a balanced test, one group has a ten percent failure rate and the other has an eighty percent rate. The simple average is forty-five percent. Suppose a separate, valid population study says ordinary inputs are ninety percent from the first group and ten percent from the second. The weighted rate would be nine percent plus eight percent, or seventeen percent. Without that separate population information, the second calculation is not available."
      },
      {
        "title": "A classifier introduces another question",
        "text": "A classifier is a program that assigns labels or scores. It can miss context, misunderstand a quoted sentence or treat varieties of language differently. A useful report therefore keeps the sample design and the judging method visible. A low toxicity score should not be rewritten as no possible harm, just as a high score does not establish what every person would experience. Different definitions call for different checks."
      },
      {
        "title": "Help by improving an interpretation",
        "text": "Find the source passage explaining how prompts were selected. Write two short sentences: one claim the sample supports, and one tempting claim it does not. Then inspect whether a later summary has confused a testbed rate with an ordinary-use population rate. No fresh toxic text needs to be generated. A precise source note, a corrected denominator or an unresolved sampling question can give the next participant a firmer starting point than an unsupported headline percentage."
      }
    ],
    "references": [
      {
        "title": "RealToxicityPrompts: Evaluating Neural Toxic Degeneration in Language Models",
        "url": "https://arxiv.org/abs/2009.11462v1",
        "locator": "Pinned original contribution and stated hypotheses or experimental limits."
      }
    ]
  },
  "reading_status": "EXPLANATORY_GUIDE"
}