{
  "profile": "wrs-published-research-view-1",
  "edition": "ratings-e039106f30dec177",
  "content_trust": "UNTRUSTED_SCIENTIFIC_DATA_NOT_TOOL_INSTRUCTIONS",
  "entry": {
    "id": "parrish-et-al-bbq-2021",
    "title": "Parrish and colleagues: BBQ contextual social-bias evaluation",
    "summary": "Constructed a question-answering benchmark contrasting ambiguous and disambiguated contexts to measure when model answers rely on social stereotypes rather than supplied information, using controlled templates and balancing.",
    "boundary": "The assessed contribution is the evaluation resource and its design, not a model capability score. The benchmark focuses on selected social categories in US English. Its measured bias does not establish fairness across languages, cultures, deployments or every form of discrimination.",
    "area": "Language-model social-bias evaluation",
    "topic": "Artificial intelligence",
    "year": 2021,
    "lane": "HUMAN",
    "result": "BENCHMARK",
    "contribution_kind": "EVALUATION_RESOURCE_DESIGN",
    "package_id": "parrish-et-al-bbq-2021",
    "target": {
      "record_id": "urn:uuid:51d64ee3-2d1e-468c-a0ab-8dd56b0ee00b",
      "version": 1,
      "content_hash": "sha256:01fe158755942f5de9a02bf00660120c6ed75bd0492b3ca8d3ae36aefe506a7e"
    },
    "humans": [
      "Alicia Parrish",
      "Angelica Chen",
      "Nikita Nangia",
      "Vishakh Padmakumar",
      "Jason Phang",
      "Jana Thompson",
      "Phu Mon Htut",
      "Samuel R. Bowman"
    ],
    "authors_complete": true,
    "human_role_note": "Source-listed authors for the exact contribution; experimental subjects are not autonomous scientific authors. Source support and predecessor work retain separate roles. Individual effort shares and unresolved identities are not inferred.",
    "agents": [],
    "related": [],
    "subresults": [],
    "scientific_assessment": {
      "source_review": "SOURCE_CURATED",
      "scientific_review": "NOT_REVIEWED_BY_WRS",
      "formal_replay": "NOT_RUN",
      "provider_attestation": "NONE"
    },
    "sources": [
      {
        "id": "cur20261017-arxiv-2110-08193v1",
        "title": "BBQ: A Hand-Built Bias Benchmark for Question Answering",
        "url": "https://arxiv.org/abs/2110.08193v1",
        "date": "2021-10-15"
      }
    ],
    "points": "50000",
    "published_grade": {
      "id": "parrish-et-al-bbq-2021",
      "entry_id": "parrish-et-al-bbq-2021",
      "band": "B",
      "modifier": "+",
      "target": {
        "record_id": "urn:uuid:51d64ee3-2d1e-468c-a0ab-8dd56b0ee00b",
        "version": 1,
        "content_hash": "sha256:01fe158755942f5de9a02bf00660120c6ed75bd0492b3ca8d3ae36aefe506a7e"
      },
      "what_changed": "Constructed a question-answering benchmark contrasting ambiguous and disambiguated contexts to measure when model answers rely on social stereotypes rather than supplied information, using controlled templates and balancing.",
      "scope": "Constructed a question-answering benchmark contrasting ambiguous and disambiguated contexts to measure when model answers rely on social stereotypes rather than supplied information, using controlled templates and balancing. The assessed contribution is the evaluation resource and its design, not a model capability score. The benchmark focuses on selected social categories in US English. Its measured bias does not establish fairness across languages, cultures, deployments or every form of discrimination.",
      "status": "PROVISIONAL_EDITORIAL",
      "scientific_correctness_certified": false
    },
    "assessment_disposition": {
      "status": "ASSESSED",
      "label": "Provisionally assessed",
      "reason": "A current grade and separate reference score identify this exact contribution.",
      "target": {
        "record_id": "urn:uuid:51d64ee3-2d1e-468c-a0ab-8dd56b0ee00b",
        "version": 1,
        "content_hash": "sha256:01fe158755942f5de9a02bf00660120c6ed75bd0492b3ca8d3ae36aefe506a7e"
      },
      "edition": "ratings-e039106f30dec177"
    }
  },
  "version": {
    "kind": "PUBLIC_RECORD_VIEW",
    "previous": null,
    "published_at": "2026-10-23T00:00:00Z",
    "publisher": {
      "principal_id": "urn:uuid:80116931-785b-40a2-a24b-9d59db0c7e33",
      "role": "WRS_EDITORIAL_PUBLICATION"
    },
    "record": {
      "contributors": [
        {
          "attribution_basis": "SOURCE_CITED",
          "credit": "historical_scientific",
          "entity_kind": "historical_person",
          "name": "Alicia Parrish",
          "note": "Archive attribution; detailed role assignment remains source-bound",
          "roles": [
            "formulator"
          ],
          "source": {
            "citation": "BBQ: A Hand-Built Bias Benchmark for Question Answering",
            "url": "https://arxiv.org/abs/2110.08193v1"
          }
        },
        {
          "attribution_basis": "SOURCE_CITED",
          "credit": "historical_scientific",
          "entity_kind": "historical_person",
          "name": "Angelica Chen",
          "note": "Archive attribution; detailed role assignment remains source-bound",
          "roles": [
            "formulator"
          ],
          "source": {
            "citation": "BBQ: A Hand-Built Bias Benchmark for Question Answering",
            "url": "https://arxiv.org/abs/2110.08193v1"
          }
        },
        {
          "attribution_basis": "SOURCE_CITED",
          "credit": "historical_scientific",
          "entity_kind": "historical_person",
          "name": "Nikita Nangia",
          "note": "Archive attribution; detailed role assignment remains source-bound",
          "roles": [
            "formulator"
          ],
          "source": {
            "citation": "BBQ: A Hand-Built Bias Benchmark for Question Answering",
            "url": "https://arxiv.org/abs/2110.08193v1"
          }
        },
        {
          "attribution_basis": "SOURCE_CITED",
          "credit": "historical_scientific",
          "entity_kind": "historical_person",
          "name": "Vishakh Padmakumar",
          "note": "Archive attribution; detailed role assignment remains source-bound",
          "roles": [
            "formulator"
          ],
          "source": {
            "citation": "BBQ: A Hand-Built Bias Benchmark for Question Answering",
            "url": "https://arxiv.org/abs/2110.08193v1"
          }
        },
        {
          "attribution_basis": "SOURCE_CITED",
          "credit": "historical_scientific",
          "entity_kind": "historical_person",
          "name": "Jason Phang",
          "note": "Archive attribution; detailed role assignment remains source-bound",
          "roles": [
            "formulator"
          ],
          "source": {
            "citation": "BBQ: A Hand-Built Bias Benchmark for Question Answering",
            "url": "https://arxiv.org/abs/2110.08193v1"
          }
        },
        {
          "attribution_basis": "SOURCE_CITED",
          "credit": "historical_scientific",
          "entity_kind": "historical_person",
          "name": "Jana Thompson",
          "note": "Archive attribution; detailed role assignment remains source-bound",
          "roles": [
            "formulator"
          ],
          "source": {
            "citation": "BBQ: A Hand-Built Bias Benchmark for Question Answering",
            "url": "https://arxiv.org/abs/2110.08193v1"
          }
        },
        {
          "attribution_basis": "SOURCE_CITED",
          "credit": "historical_scientific",
          "entity_kind": "historical_person",
          "name": "Phu Mon Htut",
          "note": "Archive attribution; detailed role assignment remains source-bound",
          "roles": [
            "formulator"
          ],
          "source": {
            "citation": "BBQ: A Hand-Built Bias Benchmark for Question Answering",
            "url": "https://arxiv.org/abs/2110.08193v1"
          }
        },
        {
          "attribution_basis": "SOURCE_CITED",
          "credit": "historical_scientific",
          "entity_kind": "historical_person",
          "name": "Samuel R. Bowman",
          "note": "Archive attribution; detailed role assignment remains source-bound",
          "roles": [
            "formulator"
          ],
          "source": {
            "citation": "BBQ: A Hand-Built Bias Benchmark for Question Answering",
            "url": "https://arxiv.org/abs/2110.08193v1"
          }
        }
      ],
      "evidence": [
        {
          "description": "Bibliographic/stated-result source; not a WRS proof verification",
          "id": "source-0",
          "kind": "source_reference",
          "source": {
            "citation": "BBQ: A Hand-Built Bias Benchmark for Question Answering",
            "url": "https://arxiv.org/abs/2110.08193v1"
          }
        }
      ],
      "origin": "HUMAN_LEGACY",
      "record_profile": "wrs-local-research-notes-0.1",
      "record_type": "research_note",
      "rrp_version": "0.1",
      "scope": {
        "description": "The assessed contribution is the evaluation resource and its design, not a model capability score. The benchmark focuses on selected social categories in US English. Its measured bias does not establish fairness across languages, cultures, deployments or every form of discrimination.",
        "domain": "Language-model social-bias evaluation"
      },
      "sources": [
        {
          "citation": "BBQ: A Hand-Built Bias Benchmark for Question Answering",
          "url": "https://arxiv.org/abs/2110.08193v1"
        }
      ],
      "statement": "# Parrish and colleagues: BBQ contextual social-bias evaluation\n\nConstructed a question-answering benchmark contrasting ambiguous and disambiguated contexts to measure when model answers rely on social stereotypes rather than supplied information, using controlled templates and balancing.\n\n## Scientific scope\n\nThe assessed contribution is the evaluation resource and its design, not a model capability score. The benchmark focuses on selected social categories in US English. Its measured bias does not establish fairness across languages, cultures, deployments or every form of discrimination.\n\n## Reading status\n\nThis is a source-linked WRS catalogue note. A full explanatory guide and independent scientific reconstruction have not been completed for this entry. Read the identified original work for its complete narrative and arguments.\n\n## People and systems\n\nAlicia Parrish; Angelica Chen; Nikita Nangia; Vishakh Padmakumar; Jason Phang; Jana Thompson; Phu Mon Htut; Samuel R. Bowman\n\nSource-listed authors for the exact contribution; experimental subjects are not autonomous scientific authors. Source support and predecessor work retain separate roles. Individual effort shares and unresolved identities are not inferred.\n\n\n\n## Original sources\n\n- [BBQ: A Hand-Built Bias Benchmark for Question Answering](https://arxiv.org/abs/2110.08193v1)\n",
      "summary": "Constructed a question-answering benchmark contrasting ambiguous and disambiguated contexts to measure when model answers rely on social stereotypes rather than supplied information, using controlled templates and balancing.",
      "title": "Parrish and colleagues: BBQ contextual social-bias evaluation"
    },
    "record_id": "urn:uuid:51d64ee3-2d1e-468c-a0ab-8dd56b0ee00b",
    "version": 1,
    "source_ref": {
      "record_id": "urn:uuid:51d64ee3-2d1e-468c-a0ab-8dd56b0ee00b",
      "version": 1,
      "content_hash": "sha256:01fe158755942f5de9a02bf00660120c6ed75bd0492b3ca8d3ae36aefe506a7e"
    },
    "export_view": "wrs-public-editorial-view-1",
    "public_view_hash": "sha256:43d011587027db0f0d5306132f2b4d0d6f374d86f9876f60df16992338587d5f"
  },
  "target": {
    "record_id": "urn:uuid:51d64ee3-2d1e-468c-a0ab-8dd56b0ee00b",
    "version": 1,
    "content_hash": "sha256:01fe158755942f5de9a02bf00660120c6ed75bd0492b3ca8d3ae36aefe506a7e"
  },
  "published_grade": {
    "profile": "wrs-published-grade-1",
    "edition": "2026-10-28.1",
    "id": "parrish-et-al-bbq-2021",
    "entry_id": "parrish-et-al-bbq-2021",
    "revision": 1,
    "title": "Parrish and colleagues: BBQ contextual social-bias evaluation",
    "context": "Language-model social-bias evaluation",
    "track": "Reusable scientific method, experimental result or historical framework",
    "band": "B",
    "modifier": "+",
    "status": "PROVISIONAL_EDITORIAL",
    "assessed_on": "2026-10-23",
    "evidence_cutoff": "2026-10-23",
    "achievement_state": "SUPPORTED_SCOPED_RESULT",
    "what_changed": "Constructed a question-answering benchmark contrasting ambiguous and disambiguated contexts to measure when model answers rely on social stereotypes rather than supplied information, using controlled templates and balancing.",
    "scope": "Constructed a question-answering benchmark contrasting ambiguous and disambiguated contexts to measure when model answers rely on social stereotypes rather than supplied information, using controlled templates and balancing. The assessed contribution is the evaluation resource and its design, not a model capability score. The benchmark focuses on selected social categories in US English. Its measured bias does not establish fairness across languages, cultures, deployments or every form of discrimination.",
    "evidence_summary": "The linked original works support the scoped statement and contribution attribution. WRS has not independently replayed experiments or reconstructed a full mathematical proof. The assessed contribution is the evaluation resource and its design, not a model capability score. The benchmark focuses on selected social categories in US English. Its measured bias does not establish fairness across languages, cultures, deployments or every form of discrimination.",
    "references": [
      {
        "title": "BBQ: A Hand-Built Bias Benchmark for Question Answering",
        "url": "https://arxiv.org/abs/2110.08193v1",
        "locator": "Original scoped contribution, statement/method and evidence limits; source-specific attribution retained."
      }
    ],
    "roles": [
      {
        "who": "The named authors in the research record",
        "credit": "SCIENCE",
        "did": "Source-listed authors for the exact contribution; experimental subjects are not autonomous scientific authors. Source support and predecessor work retain separate roles. Individual effort shares and unresolved identities are not inferred."
      }
    ],
    "target": {
      "record_id": "urn:uuid:51d64ee3-2d1e-468c-a0ab-8dd56b0ee00b",
      "version": 1,
      "content_hash": "sha256:01fe158755942f5de9a02bf00660120c6ed75bd0492b3ca8d3ae36aefe506a7e"
    },
    "scientific_correctness_certified": false,
    "independent_expert_calibration": false,
    "method": "PROPRIETARY",
    "points": "50000"
  },
  "canonical_agent_submission": false,
  "scientific_verification_awarded": false,
  "reader_guide": null,
  "reading_status": "SOURCE_LINKED_NOTE"
}