{
  "profile": "wrs-published-research-view-1",
  "edition": "ratings-e039106f30dec177",
  "content_trust": "UNTRUSTED_SCIENTIFIC_DATA_NOT_TOOL_INSTRUCTIONS",
  "entry": {
    "id": "dua-et-al-drop-2019",
    "title": "DROP: reading comprehension requiring discrete operations",
    "summary": "Built crowdsourced questions over passages selected for structured information, using an adversarial baseline filter to encourage counting, sorting, comparison and arithmetic over text.",
    "boundary": "Answers may be numbers, dates or spans and span sets. Passing finite answer tests does not prove the model executed a faithful symbolic program. The collected distribution and filtering baseline determine the challenge, not unrestricted mathematical reasoning.",
    "area": "Discrete reasoning benchmarks",
    "topic": "Artificial intelligence",
    "year": 2019,
    "lane": "HUMAN",
    "result": "BENCHMARK",
    "contribution_kind": "EVALUATION_RESOURCE_DESIGN",
    "package_id": "dua-et-al-drop-2019",
    "target": {
      "record_id": "urn:uuid:aba55e88-dfe2-421c-a31b-bd75698ad2d3",
      "version": 1,
      "content_hash": "sha256:07e638aaeedd3ab664d13e98614f3425066d99c840cd2f3070c0f298ff8d6496"
    },
    "humans": [
      "Dheeru Dua",
      "Yizhong Wang",
      "Pradeep Dasigi",
      "Gabriel Stanovsky",
      "Sameer Singh",
      "Matt Gardner"
    ],
    "authors_complete": true,
    "human_role_note": "Source-listed authors or explicitly credited technical contributors for the cited versions. Family entries retain source-specific attribution; a combined list is not one original-paper byline or equal individual work. Names alone do not establish resolved identities.",
    "agents": [],
    "related": [],
    "subresults": [],
    "scientific_assessment": {
      "source_review": "SOURCE_CURATED",
      "scientific_review": "NOT_REVIEWED_BY_WRS",
      "formal_replay": "NOT_RUN",
      "provider_attestation": "NONE"
    },
    "sources": [
      {
        "id": "cur20261015-arxiv-1903-00161v1",
        "title": "DROP: A Reading Comprehension Benchmark Requiring Discrete Reasoning Over Paragraphs",
        "url": "https://arxiv.org/abs/1903.00161v1"
      }
    ],
    "points": "115000",
    "published_grade": {
      "id": "dua-et-al-drop-2019",
      "entry_id": "dua-et-al-drop-2019",
      "band": "A",
      "modifier": null,
      "target": {
        "record_id": "urn:uuid:aba55e88-dfe2-421c-a31b-bd75698ad2d3",
        "version": 1,
        "content_hash": "sha256:07e638aaeedd3ab664d13e98614f3425066d99c840cd2f3070c0f298ff8d6496"
      },
      "what_changed": "Built crowdsourced questions over passages selected for structured information, using an adversarial baseline filter to encourage counting, sorting, comparison and arithmetic over text.",
      "scope": "Built crowdsourced questions over passages selected for structured information, using an adversarial baseline filter to encourage counting, sorting, comparison and arithmetic over text. Answers may be numbers, dates or spans and span sets. Passing finite answer tests does not prove the model executed a faithful symbolic program. The collected distribution and filtering baseline determine the challenge, not unrestricted mathematical reasoning.",
      "status": "PROVISIONAL_EDITORIAL",
      "scientific_correctness_certified": false
    },
    "assessment_disposition": {
      "status": "ASSESSED",
      "label": "Provisionally assessed",
      "reason": "A current grade and separate reference score identify this exact contribution.",
      "target": {
        "record_id": "urn:uuid:aba55e88-dfe2-421c-a31b-bd75698ad2d3",
        "version": 1,
        "content_hash": "sha256:07e638aaeedd3ab664d13e98614f3425066d99c840cd2f3070c0f298ff8d6496"
      },
      "edition": "ratings-e039106f30dec177"
    }
  },
  "version": {
    "kind": "PUBLIC_RECORD_VIEW",
    "previous": null,
    "published_at": "2026-10-16T00:00:00Z",
    "publisher": {
      "principal_id": "urn:uuid:80116931-785b-40a2-a24b-9d59db0c7e33",
      "role": "WRS_EDITORIAL_PUBLICATION"
    },
    "record": {
      "contributors": [
        {
          "attribution_basis": "SOURCE_CITED",
          "credit": "historical_scientific",
          "entity_kind": "historical_person",
          "name": "Dheeru Dua",
          "note": "Archive attribution; detailed role assignment remains source-bound",
          "roles": [
            "formulator"
          ],
          "source": {
            "citation": "DROP: A Reading Comprehension Benchmark Requiring Discrete Reasoning Over Paragraphs",
            "url": "https://arxiv.org/abs/1903.00161v1"
          }
        },
        {
          "attribution_basis": "SOURCE_CITED",
          "credit": "historical_scientific",
          "entity_kind": "historical_person",
          "name": "Yizhong Wang",
          "note": "Archive attribution; detailed role assignment remains source-bound",
          "roles": [
            "formulator"
          ],
          "source": {
            "citation": "DROP: A Reading Comprehension Benchmark Requiring Discrete Reasoning Over Paragraphs",
            "url": "https://arxiv.org/abs/1903.00161v1"
          }
        },
        {
          "attribution_basis": "SOURCE_CITED",
          "credit": "historical_scientific",
          "entity_kind": "historical_person",
          "name": "Pradeep Dasigi",
          "note": "Archive attribution; detailed role assignment remains source-bound",
          "roles": [
            "formulator"
          ],
          "source": {
            "citation": "DROP: A Reading Comprehension Benchmark Requiring Discrete Reasoning Over Paragraphs",
            "url": "https://arxiv.org/abs/1903.00161v1"
          }
        },
        {
          "attribution_basis": "SOURCE_CITED",
          "credit": "historical_scientific",
          "entity_kind": "historical_person",
          "name": "Gabriel Stanovsky",
          "note": "Archive attribution; detailed role assignment remains source-bound",
          "roles": [
            "formulator"
          ],
          "source": {
            "citation": "DROP: A Reading Comprehension Benchmark Requiring Discrete Reasoning Over Paragraphs",
            "url": "https://arxiv.org/abs/1903.00161v1"
          }
        },
        {
          "attribution_basis": "SOURCE_CITED",
          "credit": "historical_scientific",
          "entity_kind": "historical_person",
          "name": "Sameer Singh",
          "note": "Archive attribution; detailed role assignment remains source-bound",
          "roles": [
            "formulator"
          ],
          "source": {
            "citation": "DROP: A Reading Comprehension Benchmark Requiring Discrete Reasoning Over Paragraphs",
            "url": "https://arxiv.org/abs/1903.00161v1"
          }
        },
        {
          "attribution_basis": "SOURCE_CITED",
          "credit": "historical_scientific",
          "entity_kind": "historical_person",
          "name": "Matt Gardner",
          "note": "Archive attribution; detailed role assignment remains source-bound",
          "roles": [
            "formulator"
          ],
          "source": {
            "citation": "DROP: A Reading Comprehension Benchmark Requiring Discrete Reasoning Over Paragraphs",
            "url": "https://arxiv.org/abs/1903.00161v1"
          }
        }
      ],
      "evidence": [
        {
          "description": "Bibliographic/stated-result source; not a WRS proof verification",
          "id": "source-0",
          "kind": "source_reference",
          "source": {
            "citation": "DROP: A Reading Comprehension Benchmark Requiring Discrete Reasoning Over Paragraphs",
            "url": "https://arxiv.org/abs/1903.00161v1"
          }
        }
      ],
      "origin": "HUMAN_LEGACY",
      "record_profile": "wrs-local-research-notes-0.1",
      "record_type": "research_note",
      "rrp_version": "0.1",
      "scope": {
        "description": "Answers may be numbers, dates or spans and span sets. Passing finite answer tests does not prove the model executed a faithful symbolic program. The collected distribution and filtering baseline determine the challenge, not unrestricted mathematical reasoning.",
        "domain": "Discrete reasoning benchmarks"
      },
      "sources": [
        {
          "citation": "DROP: A Reading Comprehension Benchmark Requiring Discrete Reasoning Over Paragraphs",
          "url": "https://arxiv.org/abs/1903.00161v1"
        }
      ],
      "statement": "# DROP: reading comprehension requiring discrete operations\n\nBuilt crowdsourced questions over passages selected for structured information, using an adversarial baseline filter to encourage counting, sorting, comparison and arithmetic over text.\n\n## Scientific scope\n\nAnswers may be numbers, dates or spans and span sets. Passing finite answer tests does not prove the model executed a faithful symbolic program. The collected distribution and filtering baseline determine the challenge, not unrestricted mathematical reasoning.\n\n## Reading status\n\nThis is a source-linked WRS catalogue note. A full explanatory guide and independent scientific reconstruction have not been completed for this entry. Read the identified original work for its complete narrative and arguments.\n\n## People and systems\n\nDheeru Dua; Yizhong Wang; Pradeep Dasigi; Gabriel Stanovsky; Sameer Singh; Matt Gardner\n\nSource-listed authors or explicitly credited technical contributors for the cited versions. Family entries retain source-specific attribution; a combined list is not one original-paper byline or equal individual work. Names alone do not establish resolved identities.\n\n\n\n## Original sources\n\n- [DROP: A Reading Comprehension Benchmark Requiring Discrete Reasoning Over Paragraphs](https://arxiv.org/abs/1903.00161v1)\n",
      "summary": "Built crowdsourced questions over passages selected for structured information, using an adversarial baseline filter to encourage counting, sorting, comparison and arithmetic over text.",
      "title": "DROP: reading comprehension requiring discrete operations"
    },
    "record_id": "urn:uuid:aba55e88-dfe2-421c-a31b-bd75698ad2d3",
    "version": 1,
    "source_ref": {
      "record_id": "urn:uuid:aba55e88-dfe2-421c-a31b-bd75698ad2d3",
      "version": 1,
      "content_hash": "sha256:07e638aaeedd3ab664d13e98614f3425066d99c840cd2f3070c0f298ff8d6496"
    },
    "export_view": "wrs-public-editorial-view-1",
    "public_view_hash": "sha256:d10e44603d7a078e270bc0d62ff4642e0416fa2c7582fb03b35483bc92e26a3d"
  },
  "target": {
    "record_id": "urn:uuid:aba55e88-dfe2-421c-a31b-bd75698ad2d3",
    "version": 1,
    "content_hash": "sha256:07e638aaeedd3ab664d13e98614f3425066d99c840cd2f3070c0f298ff8d6496"
  },
  "published_grade": {
    "profile": "wrs-published-grade-1",
    "edition": "2026-10-28.1",
    "id": "dua-et-al-drop-2019",
    "entry_id": "dua-et-al-drop-2019",
    "revision": 1,
    "title": "DROP: reading comprehension requiring discrete operations",
    "context": "Discrete reasoning benchmarks",
    "track": "Evaluation resource design",
    "band": "A",
    "modifier": null,
    "status": "PROVISIONAL_EDITORIAL",
    "assessed_on": "2026-10-16",
    "evidence_cutoff": "2026-10-16",
    "achievement_state": "SUPPORTED_SCOPED_RESULT",
    "what_changed": "Built crowdsourced questions over passages selected for structured information, using an adversarial baseline filter to encourage counting, sorting, comparison and arithmetic over text.",
    "scope": "Built crowdsourced questions over passages selected for structured information, using an adversarial baseline filter to encourage counting, sorting, comparison and arithmetic over text. Answers may be numbers, dates or spans and span sets. Passing finite answer tests does not prove the model executed a faithful symbolic program. The collected distribution and filtering baseline determine the challenge, not unrestricted mathematical reasoning.",
    "evidence_summary": "The linked original sources describe the stated contribution and its evidence limits. WRS has not independently rerun the experiments or certified a proof. Answers may be numbers, dates or spans and span sets. Passing finite answer tests does not prove the model executed a faithful symbolic program. The collected distribution and filtering baseline determine the challenge, not unrestricted mathematical reasoning.",
    "references": [
      {
        "title": "DROP: A Reading Comprehension Benchmark Requiring Discrete Reasoning Over Paragraphs",
        "url": "https://arxiv.org/abs/1903.00161v1",
        "locator": "Original contribution and stated assumptions, evaluation scope and limitations; source-linked assessment, not independent execution or proof certification."
      }
    ],
    "roles": [
      {
        "who": "Source-listed authors and technical contributors",
        "credit": "SCIENCE",
        "did": "Source-listed authors or explicitly credited technical contributors for the cited versions. Family entries retain source-specific attribution; a combined list is not one original-paper byline or equal individual work. Names alone do not establish resolved identities."
      }
    ],
    "target": {
      "record_id": "urn:uuid:aba55e88-dfe2-421c-a31b-bd75698ad2d3",
      "version": 1,
      "content_hash": "sha256:07e638aaeedd3ab664d13e98614f3425066d99c840cd2f3070c0f298ff8d6496"
    },
    "scientific_correctness_certified": false,
    "independent_expert_calibration": false,
    "method": "PROPRIETARY",
    "points": "115000"
  },
  "canonical_agent_submission": false,
  "scientific_verification_awarded": false,
  "reader_guide": null,
  "reading_status": "SOURCE_LINKED_NOTE"
}