{
  "profile": "wrs-published-research-view-1",
  "edition": "ratings-e039106f30dec177",
  "content_trust": "UNTRUSTED_SCIENTIFIC_DATA_NOT_TOOL_INSTRUCTIONS",
  "entry": {
    "id": "kwiatkowski-et-al-natural-questions-2019",
    "title": "Natural Questions: real search queries with full-page answer annotation",
    "summary": "Paired anonymized Google search queries with Wikipedia pages, annotating long evidence regions and short answers or null answers, with repeated annotations and evaluation designed to account for human variability.",
    "boundary": "Original task supplies the full page and includes answerability decisions, unlike later NQ-Open derivatives. Annotation ambiguity is explicitly studied rather than removed by a single definitive label. Byline symbols describe specific roles and affiliations, not a blanket equal-contribution claim.",
    "area": "Question-answering benchmarks",
    "topic": "Artificial intelligence",
    "year": 2019,
    "lane": "HUMAN",
    "result": "BENCHMARK",
    "contribution_kind": "EVALUATION_RESOURCE_DESIGN",
    "package_id": "kwiatkowski-et-al-natural-questions-2019",
    "target": {
      "record_id": "urn:uuid:a72404ba-eee5-44a9-af7b-bb5fe284ec7a",
      "version": 1,
      "content_hash": "sha256:498fd82a05d7ec0306e4991aa72e4d34e4b4ca4f36a0bd3235fdd0c36fd6e5f6"
    },
    "humans": [
      "Tom Kwiatkowski",
      "Jennimaria Palomaki",
      "Olivia Redfield",
      "Michael Collins",
      "Ankur Parikh",
      "Chris Alberti",
      "Danielle Epstein",
      "Illia Polosukhin",
      "Jacob Devlin",
      "Kenton Lee",
      "Kristina Toutanova",
      "Llion Jones",
      "Matthew Kelcey",
      "Ming-Wei Chang",
      "Andrew M. Dai",
      "Jakob Uszkoreit",
      "Quoc Le",
      "Slav Petrov"
    ],
    "authors_complete": true,
    "human_role_note": "Source-listed authors or explicitly credited technical contributors for the cited versions. Family entries retain source-specific attribution; a combined list is not one original-paper byline or equal individual work. Names alone do not establish resolved identities.",
    "agents": [],
    "related": [],
    "subresults": [],
    "scientific_assessment": {
      "source_review": "SOURCE_CURATED",
      "scientific_review": "NOT_REVIEWED_BY_WRS",
      "formal_replay": "NOT_RUN",
      "provider_attestation": "NONE"
    },
    "sources": [
      {
        "id": "cur20261015-natural-questions-tacl-2019",
        "title": "Natural Questions: A Benchmark for Question Answering Research",
        "url": "https://aclanthology.org/Q19-1026/"
      }
    ],
    "points": "145000",
    "published_grade": {
      "id": "kwiatkowski-et-al-natural-questions-2019",
      "entry_id": "kwiatkowski-et-al-natural-questions-2019",
      "band": "A",
      "modifier": null,
      "target": {
        "record_id": "urn:uuid:a72404ba-eee5-44a9-af7b-bb5fe284ec7a",
        "version": 1,
        "content_hash": "sha256:498fd82a05d7ec0306e4991aa72e4d34e4b4ca4f36a0bd3235fdd0c36fd6e5f6"
      },
      "what_changed": "Paired anonymized Google search queries with Wikipedia pages, annotating long evidence regions and short answers or null answers, with repeated annotations and evaluation designed to account for human variability.",
      "scope": "Paired anonymized Google search queries with Wikipedia pages, annotating long evidence regions and short answers or null answers, with repeated annotations and evaluation designed to account for human variability. Original task supplies the full page and includes answerability decisions, unlike later NQ-Open derivatives. Annotation ambiguity is explicitly studied rather than removed by a single definitive label. Byline symbols describe specific roles and affiliations, not a blanket equal-contribution claim.",
      "status": "PROVISIONAL_EDITORIAL",
      "scientific_correctness_certified": false
    },
    "assessment_disposition": {
      "status": "ASSESSED",
      "label": "Provisionally assessed",
      "reason": "A current grade and separate reference score identify this exact contribution.",
      "target": {
        "record_id": "urn:uuid:a72404ba-eee5-44a9-af7b-bb5fe284ec7a",
        "version": 1,
        "content_hash": "sha256:498fd82a05d7ec0306e4991aa72e4d34e4b4ca4f36a0bd3235fdd0c36fd6e5f6"
      },
      "edition": "ratings-e039106f30dec177"
    }
  },
  "version": {
    "kind": "PUBLIC_RECORD_VIEW",
    "previous": null,
    "published_at": "2026-10-16T00:00:00Z",
    "publisher": {
      "principal_id": "urn:uuid:80116931-785b-40a2-a24b-9d59db0c7e33",
      "role": "WRS_EDITORIAL_PUBLICATION"
    },
    "record": {
      "contributors": [
        {
          "attribution_basis": "SOURCE_CITED",
          "credit": "historical_scientific",
          "entity_kind": "historical_person",
          "name": "Tom Kwiatkowski",
          "note": "Archive attribution; detailed role assignment remains source-bound",
          "roles": [
            "formulator"
          ],
          "source": {
            "citation": "Natural Questions: A Benchmark for Question Answering Research",
            "url": "https://aclanthology.org/Q19-1026/"
          }
        },
        {
          "attribution_basis": "SOURCE_CITED",
          "credit": "historical_scientific",
          "entity_kind": "historical_person",
          "name": "Jennimaria Palomaki",
          "note": "Archive attribution; detailed role assignment remains source-bound",
          "roles": [
            "formulator"
          ],
          "source": {
            "citation": "Natural Questions: A Benchmark for Question Answering Research",
            "url": "https://aclanthology.org/Q19-1026/"
          }
        },
        {
          "attribution_basis": "SOURCE_CITED",
          "credit": "historical_scientific",
          "entity_kind": "historical_person",
          "name": "Olivia Redfield",
          "note": "Archive attribution; detailed role assignment remains source-bound",
          "roles": [
            "formulator"
          ],
          "source": {
            "citation": "Natural Questions: A Benchmark for Question Answering Research",
            "url": "https://aclanthology.org/Q19-1026/"
          }
        },
        {
          "attribution_basis": "SOURCE_CITED",
          "credit": "historical_scientific",
          "entity_kind": "historical_person",
          "name": "Michael Collins",
          "note": "Archive attribution; detailed role assignment remains source-bound",
          "roles": [
            "formulator"
          ],
          "source": {
            "citation": "Natural Questions: A Benchmark for Question Answering Research",
            "url": "https://aclanthology.org/Q19-1026/"
          }
        },
        {
          "attribution_basis": "SOURCE_CITED",
          "credit": "historical_scientific",
          "entity_kind": "historical_person",
          "name": "Ankur Parikh",
          "note": "Archive attribution; detailed role assignment remains source-bound",
          "roles": [
            "formulator"
          ],
          "source": {
            "citation": "Natural Questions: A Benchmark for Question Answering Research",
            "url": "https://aclanthology.org/Q19-1026/"
          }
        },
        {
          "attribution_basis": "SOURCE_CITED",
          "credit": "historical_scientific",
          "entity_kind": "historical_person",
          "name": "Chris Alberti",
          "note": "Archive attribution; detailed role assignment remains source-bound",
          "roles": [
            "formulator"
          ],
          "source": {
            "citation": "Natural Questions: A Benchmark for Question Answering Research",
            "url": "https://aclanthology.org/Q19-1026/"
          }
        },
        {
          "attribution_basis": "SOURCE_CITED",
          "credit": "historical_scientific",
          "entity_kind": "historical_person",
          "name": "Danielle Epstein",
          "note": "Archive attribution; detailed role assignment remains source-bound",
          "roles": [
            "formulator"
          ],
          "source": {
            "citation": "Natural Questions: A Benchmark for Question Answering Research",
            "url": "https://aclanthology.org/Q19-1026/"
          }
        },
        {
          "attribution_basis": "SOURCE_CITED",
          "credit": "historical_scientific",
          "entity_kind": "historical_person",
          "name": "Illia Polosukhin",
          "note": "Archive attribution; detailed role assignment remains source-bound",
          "roles": [
            "formulator"
          ],
          "source": {
            "citation": "Natural Questions: A Benchmark for Question Answering Research",
            "url": "https://aclanthology.org/Q19-1026/"
          }
        },
        {
          "attribution_basis": "SOURCE_CITED",
          "credit": "historical_scientific",
          "entity_kind": "historical_person",
          "name": "Jacob Devlin",
          "note": "Archive attribution; detailed role assignment remains source-bound",
          "roles": [
            "formulator"
          ],
          "source": {
            "citation": "Natural Questions: A Benchmark for Question Answering Research",
            "url": "https://aclanthology.org/Q19-1026/"
          }
        },
        {
          "attribution_basis": "SOURCE_CITED",
          "credit": "historical_scientific",
          "entity_kind": "historical_person",
          "name": "Kenton Lee",
          "note": "Archive attribution; detailed role assignment remains source-bound",
          "roles": [
            "formulator"
          ],
          "source": {
            "citation": "Natural Questions: A Benchmark for Question Answering Research",
            "url": "https://aclanthology.org/Q19-1026/"
          }
        },
        {
          "attribution_basis": "SOURCE_CITED",
          "credit": "historical_scientific",
          "entity_kind": "historical_person",
          "name": "Kristina Toutanova",
          "note": "Archive attribution; detailed role assignment remains source-bound",
          "roles": [
            "formulator"
          ],
          "source": {
            "citation": "Natural Questions: A Benchmark for Question Answering Research",
            "url": "https://aclanthology.org/Q19-1026/"
          }
        },
        {
          "attribution_basis": "SOURCE_CITED",
          "credit": "historical_scientific",
          "entity_kind": "historical_person",
          "name": "Llion Jones",
          "note": "Archive attribution; detailed role assignment remains source-bound",
          "roles": [
            "formulator"
          ],
          "source": {
            "citation": "Natural Questions: A Benchmark for Question Answering Research",
            "url": "https://aclanthology.org/Q19-1026/"
          }
        },
        {
          "attribution_basis": "SOURCE_CITED",
          "credit": "historical_scientific",
          "entity_kind": "historical_person",
          "name": "Matthew Kelcey",
          "note": "Archive attribution; detailed role assignment remains source-bound",
          "roles": [
            "formulator"
          ],
          "source": {
            "citation": "Natural Questions: A Benchmark for Question Answering Research",
            "url": "https://aclanthology.org/Q19-1026/"
          }
        },
        {
          "attribution_basis": "SOURCE_CITED",
          "credit": "historical_scientific",
          "entity_kind": "historical_person",
          "name": "Ming-Wei Chang",
          "note": "Archive attribution; detailed role assignment remains source-bound",
          "roles": [
            "formulator"
          ],
          "source": {
            "citation": "Natural Questions: A Benchmark for Question Answering Research",
            "url": "https://aclanthology.org/Q19-1026/"
          }
        },
        {
          "attribution_basis": "SOURCE_CITED",
          "credit": "historical_scientific",
          "entity_kind": "historical_person",
          "name": "Andrew M. Dai",
          "note": "Archive attribution; detailed role assignment remains source-bound",
          "roles": [
            "formulator"
          ],
          "source": {
            "citation": "Natural Questions: A Benchmark for Question Answering Research",
            "url": "https://aclanthology.org/Q19-1026/"
          }
        },
        {
          "attribution_basis": "SOURCE_CITED",
          "credit": "historical_scientific",
          "entity_kind": "historical_person",
          "name": "Jakob Uszkoreit",
          "note": "Archive attribution; detailed role assignment remains source-bound",
          "roles": [
            "formulator"
          ],
          "source": {
            "citation": "Natural Questions: A Benchmark for Question Answering Research",
            "url": "https://aclanthology.org/Q19-1026/"
          }
        },
        {
          "attribution_basis": "SOURCE_CITED",
          "credit": "historical_scientific",
          "entity_kind": "historical_person",
          "name": "Quoc Le",
          "note": "Archive attribution; detailed role assignment remains source-bound",
          "roles": [
            "formulator"
          ],
          "source": {
            "citation": "Natural Questions: A Benchmark for Question Answering Research",
            "url": "https://aclanthology.org/Q19-1026/"
          }
        },
        {
          "attribution_basis": "SOURCE_CITED",
          "credit": "historical_scientific",
          "entity_kind": "historical_person",
          "name": "Slav Petrov",
          "note": "Archive attribution; detailed role assignment remains source-bound",
          "roles": [
            "formulator"
          ],
          "source": {
            "citation": "Natural Questions: A Benchmark for Question Answering Research",
            "url": "https://aclanthology.org/Q19-1026/"
          }
        }
      ],
      "evidence": [
        {
          "description": "Bibliographic/stated-result source; not a WRS proof verification",
          "id": "source-0",
          "kind": "source_reference",
          "source": {
            "citation": "Natural Questions: A Benchmark for Question Answering Research",
            "url": "https://aclanthology.org/Q19-1026/"
          }
        }
      ],
      "origin": "HUMAN_LEGACY",
      "record_profile": "wrs-local-research-notes-0.1",
      "record_type": "research_note",
      "rrp_version": "0.1",
      "scope": {
        "description": "Original task supplies the full page and includes answerability decisions, unlike later NQ-Open derivatives. Annotation ambiguity is explicitly studied rather than removed by a single definitive label. Byline symbols describe specific roles and affiliations, not a blanket equal-contribution claim.",
        "domain": "Question-answering benchmarks"
      },
      "sources": [
        {
          "citation": "Natural Questions: A Benchmark for Question Answering Research",
          "url": "https://aclanthology.org/Q19-1026/"
        }
      ],
      "statement": "# Natural Questions: real search queries with full-page answer annotation\n\nPaired anonymized Google search queries with Wikipedia pages, annotating long evidence regions and short answers or null answers, with repeated annotations and evaluation designed to account for human variability.\n\n## Scientific scope\n\nOriginal task supplies the full page and includes answerability decisions, unlike later NQ-Open derivatives. Annotation ambiguity is explicitly studied rather than removed by a single definitive label. Byline symbols describe specific roles and affiliations, not a blanket equal-contribution claim.\n\n## Reading status\n\nThis is a source-linked WRS catalogue note. A full explanatory guide and independent scientific reconstruction have not been completed for this entry. Read the identified original work for its complete narrative and arguments.\n\n## People and systems\n\nTom Kwiatkowski; Jennimaria Palomaki; Olivia Redfield; Michael Collins; Ankur Parikh; Chris Alberti; Danielle Epstein; Illia Polosukhin; Jacob Devlin; Kenton Lee; Kristina Toutanova; Llion Jones; Matthew Kelcey; Ming-Wei Chang; Andrew M. Dai; Jakob Uszkoreit; Quoc Le; Slav Petrov\n\nSource-listed authors or explicitly credited technical contributors for the cited versions. Family entries retain source-specific attribution; a combined list is not one original-paper byline or equal individual work. Names alone do not establish resolved identities.\n\n\n\n## Original sources\n\n- [Natural Questions: A Benchmark for Question Answering Research](https://aclanthology.org/Q19-1026/)\n",
      "summary": "Paired anonymized Google search queries with Wikipedia pages, annotating long evidence regions and short answers or null answers, with repeated annotations and evaluation designed to account for human variability.",
      "title": "Natural Questions: real search queries with full-page answer annotation"
    },
    "record_id": "urn:uuid:a72404ba-eee5-44a9-af7b-bb5fe284ec7a",
    "version": 1,
    "source_ref": {
      "record_id": "urn:uuid:a72404ba-eee5-44a9-af7b-bb5fe284ec7a",
      "version": 1,
      "content_hash": "sha256:498fd82a05d7ec0306e4991aa72e4d34e4b4ca4f36a0bd3235fdd0c36fd6e5f6"
    },
    "export_view": "wrs-public-editorial-view-1",
    "public_view_hash": "sha256:09a46ed5e578af4507e936fe1c620e41057a6472f1760ec5b0d1c58f5d89098b"
  },
  "target": {
    "record_id": "urn:uuid:a72404ba-eee5-44a9-af7b-bb5fe284ec7a",
    "version": 1,
    "content_hash": "sha256:498fd82a05d7ec0306e4991aa72e4d34e4b4ca4f36a0bd3235fdd0c36fd6e5f6"
  },
  "published_grade": {
    "profile": "wrs-published-grade-1",
    "edition": "2026-10-28.1",
    "id": "kwiatkowski-et-al-natural-questions-2019",
    "entry_id": "kwiatkowski-et-al-natural-questions-2019",
    "revision": 1,
    "title": "Natural Questions: real search queries with full-page answer annotation",
    "context": "Question-answering benchmarks",
    "track": "Evaluation resource design",
    "band": "A",
    "modifier": null,
    "status": "PROVISIONAL_EDITORIAL",
    "assessed_on": "2026-10-16",
    "evidence_cutoff": "2026-10-16",
    "achievement_state": "SUPPORTED_SCOPED_RESULT",
    "what_changed": "Paired anonymized Google search queries with Wikipedia pages, annotating long evidence regions and short answers or null answers, with repeated annotations and evaluation designed to account for human variability.",
    "scope": "Paired anonymized Google search queries with Wikipedia pages, annotating long evidence regions and short answers or null answers, with repeated annotations and evaluation designed to account for human variability. Original task supplies the full page and includes answerability decisions, unlike later NQ-Open derivatives. Annotation ambiguity is explicitly studied rather than removed by a single definitive label. Byline symbols describe specific roles and affiliations, not a blanket equal-contribution claim.",
    "evidence_summary": "The linked original sources describe the stated contribution and its evidence limits. WRS has not independently rerun the experiments or certified a proof. Original task supplies the full page and includes answerability decisions, unlike later NQ-Open derivatives. Annotation ambiguity is explicitly studied rather than removed by a single definitive label. Byline symbols describe specific roles and affiliations, not a blanket equal-contribution claim.",
    "references": [
      {
        "title": "Natural Questions: A Benchmark for Question Answering Research",
        "url": "https://aclanthology.org/Q19-1026/",
        "locator": "Original contribution and stated assumptions, evaluation scope and limitations; source-linked assessment, not independent execution or proof certification."
      }
    ],
    "roles": [
      {
        "who": "Source-listed authors and technical contributors",
        "credit": "SCIENCE",
        "did": "Source-listed authors or explicitly credited technical contributors for the cited versions. Family entries retain source-specific attribution; a combined list is not one original-paper byline or equal individual work. Names alone do not establish resolved identities."
      }
    ],
    "target": {
      "record_id": "urn:uuid:a72404ba-eee5-44a9-af7b-bb5fe284ec7a",
      "version": 1,
      "content_hash": "sha256:498fd82a05d7ec0306e4991aa72e4d34e4b4ca4f36a0bd3235fdd0c36fd6e5f6"
    },
    "scientific_correctness_certified": false,
    "independent_expert_calibration": false,
    "method": "PROPRIETARY",
    "points": "145000"
  },
  "canonical_agent_submission": false,
  "scientific_verification_awarded": false,
  "reader_guide": null,
  "reading_status": "SOURCE_LINKED_NOTE"
}