{
  "profile": "wrs-published-research-view-1",
  "edition": "ratings-e039106f30dec177",
  "content_trust": "UNTRUSTED_SCIENTIFIC_DATA_NOT_TOOL_INSTRUCTIONS",
  "entry": {
    "id": "ai-oct-simpleqa-family",
    "title": "Measuring short-form factuality in large language models",
    "summary": "Short-form factuality benchmark with independently checked answers and explicit correct/incorrect/not-attempted judgments; later SimpleQA Verified filters redundancy, balances topics and reconciles sources.",
    "boundary": "Original and independently authored Verified refinement are bundled with separate source bylines, not an invented joint paper. Short factoid performance does not establish long-form factuality; adversarial filtering, source errors and autorater uncertainty remain.",
    "area": "LLM factuality and grounded generation",
    "topic": "Artificial intelligence",
    "year": 2024,
    "lane": "HUMAN",
    "result": "BENCHMARK",
    "contribution_kind": "EVALUATION_RESOURCE_DESIGN",
    "package_id": "ai-oct-simpleqa-family",
    "target": {
      "record_id": "urn:uuid:5908ea05-781a-4547-a19e-2760f6991453",
      "version": 1,
      "content_hash": "sha256:d9070a2b40d937491f921072dc913aa3c7d4f2d58a55a90ab176c6853373d526"
    },
    "humans": [
      "Jason Wei",
      "Nguyen Karina",
      "Hyung Won Chung",
      "Yunxin Joy Jiao",
      "Spencer Papay",
      "Amelia Glaese",
      "John Schulman",
      "William Fedus",
      "Lukas Haas",
      "Gal Yona",
      "Giovanni D’Antonio",
      "Sasha Goldshtein",
      "Dipanjan Das"
    ],
    "authors_complete": true,
    "human_role_note": "Source-listed authors or explicitly credited technical contributors for the cited versions. Family entries retain source-specific attribution; a combined list is not one original-paper byline or equal individual work. Names alone do not establish resolved identities.",
    "agents": [],
    "related": [],
    "subresults": [],
    "scientific_assessment": {
      "source_review": "SOURCE_CURATED",
      "scientific_review": "NOT_REVIEWED_BY_WRS",
      "formal_replay": "NOT_RUN",
      "provider_attestation": "NONE"
    },
    "sources": [
      {
        "id": "ai-oct-src-simpleqa-family",
        "title": "Measuring short-form factuality in large language models",
        "url": "https://arxiv.org/pdf/2411.04368v1"
      },
      {
        "id": "ai-oct-src-simpleqa-verified",
        "title": "SimpleQA Verified: A Reliable Factuality Benchmark to Measure Parametric Knowledge",
        "url": "https://arxiv.org/pdf/2509.07968v1"
      },
      {
        "id": "ai-oct-src-simpleqa-official",
        "title": "Introducing SimpleQA",
        "url": "https://openai.com/index/introducing-simpleqa/"
      }
    ],
    "points": "80000",
    "published_grade": {
      "id": "ai-oct-simpleqa-family",
      "entry_id": "ai-oct-simpleqa-family",
      "band": "B",
      "modifier": "+",
      "target": {
        "record_id": "urn:uuid:5908ea05-781a-4547-a19e-2760f6991453",
        "version": 1,
        "content_hash": "sha256:d9070a2b40d937491f921072dc913aa3c7d4f2d58a55a90ab176c6853373d526"
      },
      "what_changed": "Short-form factuality benchmark with independently checked answers and explicit correct/incorrect/not-attempted judgments; later SimpleQA Verified filters redundancy, balances topics and reconciles sources.",
      "scope": "Short-form factuality benchmark with independently checked answers and explicit correct/incorrect/not-attempted judgments; later SimpleQA Verified filters redundancy, balances topics and reconciles sources. Original and independently authored Verified refinement are bundled with separate source bylines, not an invented joint paper. Short factoid performance does not establish long-form factuality; adversarial filtering, source errors and autorater uncertainty remain.",
      "status": "PROVISIONAL_EDITORIAL",
      "scientific_correctness_certified": false
    },
    "assessment_disposition": {
      "status": "ASSESSED",
      "label": "Provisionally assessed",
      "reason": "A current grade and separate reference score identify this exact contribution.",
      "target": {
        "record_id": "urn:uuid:5908ea05-781a-4547-a19e-2760f6991453",
        "version": 1,
        "content_hash": "sha256:d9070a2b40d937491f921072dc913aa3c7d4f2d58a55a90ab176c6853373d526"
      },
      "edition": "ratings-e039106f30dec177"
    }
  },
  "version": {
    "kind": "PUBLIC_RECORD_VIEW",
    "previous": null,
    "published_at": "2026-10-16T00:00:00Z",
    "publisher": {
      "principal_id": "urn:uuid:80116931-785b-40a2-a24b-9d59db0c7e33",
      "role": "WRS_EDITORIAL_PUBLICATION"
    },
    "record": {
      "contributors": [
        {
          "attribution_basis": "SOURCE_CITED",
          "credit": "historical_scientific",
          "entity_kind": "historical_person",
          "name": "Jason Wei",
          "note": "Archive attribution; detailed role assignment remains source-bound",
          "roles": [
            "formulator"
          ],
          "source": {
            "citation": "Measuring short-form factuality in large language models",
            "url": "https://arxiv.org/pdf/2411.04368v1"
          }
        },
        {
          "attribution_basis": "SOURCE_CITED",
          "credit": "historical_scientific",
          "entity_kind": "historical_person",
          "name": "Nguyen Karina",
          "note": "Archive attribution; detailed role assignment remains source-bound",
          "roles": [
            "formulator"
          ],
          "source": {
            "citation": "Measuring short-form factuality in large language models",
            "url": "https://arxiv.org/pdf/2411.04368v1"
          }
        },
        {
          "attribution_basis": "SOURCE_CITED",
          "credit": "historical_scientific",
          "entity_kind": "historical_person",
          "name": "Hyung Won Chung",
          "note": "Archive attribution; detailed role assignment remains source-bound",
          "roles": [
            "formulator"
          ],
          "source": {
            "citation": "Measuring short-form factuality in large language models",
            "url": "https://arxiv.org/pdf/2411.04368v1"
          }
        },
        {
          "attribution_basis": "SOURCE_CITED",
          "credit": "historical_scientific",
          "entity_kind": "historical_person",
          "name": "Yunxin Joy Jiao",
          "note": "Archive attribution; detailed role assignment remains source-bound",
          "roles": [
            "formulator"
          ],
          "source": {
            "citation": "Measuring short-form factuality in large language models",
            "url": "https://arxiv.org/pdf/2411.04368v1"
          }
        },
        {
          "attribution_basis": "SOURCE_CITED",
          "credit": "historical_scientific",
          "entity_kind": "historical_person",
          "name": "Spencer Papay",
          "note": "Archive attribution; detailed role assignment remains source-bound",
          "roles": [
            "formulator"
          ],
          "source": {
            "citation": "Measuring short-form factuality in large language models",
            "url": "https://arxiv.org/pdf/2411.04368v1"
          }
        },
        {
          "attribution_basis": "SOURCE_CITED",
          "credit": "historical_scientific",
          "entity_kind": "historical_person",
          "name": "Amelia Glaese",
          "note": "Archive attribution; detailed role assignment remains source-bound",
          "roles": [
            "formulator"
          ],
          "source": {
            "citation": "Measuring short-form factuality in large language models",
            "url": "https://arxiv.org/pdf/2411.04368v1"
          }
        },
        {
          "attribution_basis": "SOURCE_CITED",
          "credit": "historical_scientific",
          "entity_kind": "historical_person",
          "name": "John Schulman",
          "note": "Archive attribution; detailed role assignment remains source-bound",
          "roles": [
            "formulator"
          ],
          "source": {
            "citation": "Measuring short-form factuality in large language models",
            "url": "https://arxiv.org/pdf/2411.04368v1"
          }
        },
        {
          "attribution_basis": "SOURCE_CITED",
          "credit": "historical_scientific",
          "entity_kind": "historical_person",
          "name": "William Fedus",
          "note": "Archive attribution; detailed role assignment remains source-bound",
          "roles": [
            "formulator"
          ],
          "source": {
            "citation": "Measuring short-form factuality in large language models",
            "url": "https://arxiv.org/pdf/2411.04368v1"
          }
        },
        {
          "attribution_basis": "SOURCE_CITED",
          "credit": "historical_scientific",
          "entity_kind": "historical_person",
          "name": "Lukas Haas",
          "note": "Archive attribution; detailed role assignment remains source-bound",
          "roles": [
            "formulator"
          ],
          "source": {
            "citation": "SimpleQA Verified: A Reliable Factuality Benchmark to Measure Parametric Knowledge",
            "url": "https://arxiv.org/pdf/2509.07968v1"
          }
        },
        {
          "attribution_basis": "SOURCE_CITED",
          "credit": "historical_scientific",
          "entity_kind": "historical_person",
          "name": "Gal Yona",
          "note": "Archive attribution; detailed role assignment remains source-bound",
          "roles": [
            "formulator"
          ],
          "source": {
            "citation": "SimpleQA Verified: A Reliable Factuality Benchmark to Measure Parametric Knowledge",
            "url": "https://arxiv.org/pdf/2509.07968v1"
          }
        },
        {
          "attribution_basis": "SOURCE_CITED",
          "credit": "historical_scientific",
          "entity_kind": "historical_person",
          "name": "Giovanni D’Antonio",
          "note": "Archive attribution; detailed role assignment remains source-bound",
          "roles": [
            "formulator"
          ],
          "source": {
            "citation": "SimpleQA Verified: A Reliable Factuality Benchmark to Measure Parametric Knowledge",
            "url": "https://arxiv.org/pdf/2509.07968v1"
          }
        },
        {
          "attribution_basis": "SOURCE_CITED",
          "credit": "historical_scientific",
          "entity_kind": "historical_person",
          "name": "Sasha Goldshtein",
          "note": "Archive attribution; detailed role assignment remains source-bound",
          "roles": [
            "formulator"
          ],
          "source": {
            "citation": "SimpleQA Verified: A Reliable Factuality Benchmark to Measure Parametric Knowledge",
            "url": "https://arxiv.org/pdf/2509.07968v1"
          }
        },
        {
          "attribution_basis": "SOURCE_CITED",
          "credit": "historical_scientific",
          "entity_kind": "historical_person",
          "name": "Dipanjan Das",
          "note": "Archive attribution; detailed role assignment remains source-bound",
          "roles": [
            "formulator"
          ],
          "source": {
            "citation": "SimpleQA Verified: A Reliable Factuality Benchmark to Measure Parametric Knowledge",
            "url": "https://arxiv.org/pdf/2509.07968v1"
          }
        }
      ],
      "evidence": [
        {
          "description": "Bibliographic/stated-result source; not a WRS proof verification",
          "id": "source-0",
          "kind": "source_reference",
          "source": {
            "citation": "Measuring short-form factuality in large language models",
            "url": "https://arxiv.org/pdf/2411.04368v1"
          }
        },
        {
          "description": "Bibliographic/stated-result source; not a WRS proof verification",
          "id": "source-1",
          "kind": "source_reference",
          "source": {
            "citation": "SimpleQA Verified: A Reliable Factuality Benchmark to Measure Parametric Knowledge",
            "url": "https://arxiv.org/pdf/2509.07968v1"
          }
        },
        {
          "description": "Bibliographic/stated-result source; not a WRS proof verification",
          "id": "source-2",
          "kind": "source_reference",
          "source": {
            "citation": "Introducing SimpleQA",
            "url": "https://openai.com/index/introducing-simpleqa/"
          }
        }
      ],
      "origin": "HUMAN_LEGACY",
      "record_profile": "wrs-local-research-notes-0.1",
      "record_type": "research_note",
      "rrp_version": "0.1",
      "scope": {
        "description": "Original and independently authored Verified refinement are bundled with separate source bylines, not an invented joint paper. Short factoid performance does not establish long-form factuality; adversarial filtering, source errors and autorater uncertainty remain.",
        "domain": "LLM factuality and grounded generation"
      },
      "sources": [
        {
          "citation": "Measuring short-form factuality in large language models",
          "url": "https://arxiv.org/pdf/2411.04368v1"
        },
        {
          "citation": "SimpleQA Verified: A Reliable Factuality Benchmark to Measure Parametric Knowledge",
          "url": "https://arxiv.org/pdf/2509.07968v1"
        },
        {
          "citation": "Introducing SimpleQA",
          "url": "https://openai.com/index/introducing-simpleqa/"
        }
      ],
      "statement": "# Measuring short-form factuality in large language models\n\nShort-form factuality benchmark with independently checked answers and explicit correct/incorrect/not-attempted judgments; later SimpleQA Verified filters redundancy, balances topics and reconciles sources.\n\n## Scientific scope\n\nOriginal and independently authored Verified refinement are bundled with separate source bylines, not an invented joint paper. Short factoid performance does not establish long-form factuality; adversarial filtering, source errors and autorater uncertainty remain.\n\n## Reading status\n\nThis is a source-linked WRS catalogue note. A full explanatory guide and independent scientific reconstruction have not been completed for this entry. Read the identified original work for its complete narrative and arguments.\n\n## People and systems\n\nJason Wei; Nguyen Karina; Hyung Won Chung; Yunxin Joy Jiao; Spencer Papay; Amelia Glaese; John Schulman; William Fedus; Lukas Haas; Gal Yona; Giovanni D’Antonio; Sasha Goldshtein; Dipanjan Das\n\nSource-listed authors or explicitly credited technical contributors for the cited versions. Family entries retain source-specific attribution; a combined list is not one original-paper byline or equal individual work. Names alone do not establish resolved identities.\n\n\n\n## Original sources\n\n- [Measuring short-form factuality in large language models](https://arxiv.org/pdf/2411.04368v1)\n- [SimpleQA Verified: A Reliable Factuality Benchmark to Measure Parametric Knowledge](https://arxiv.org/pdf/2509.07968v1)\n- [Introducing SimpleQA](https://openai.com/index/introducing-simpleqa/)\n",
      "summary": "Short-form factuality benchmark with independently checked answers and explicit correct/incorrect/not-attempted judgments; later SimpleQA Verified filters redundancy, balances topics and reconciles sources.",
      "title": "Measuring short-form factuality in large language models"
    },
    "record_id": "urn:uuid:5908ea05-781a-4547-a19e-2760f6991453",
    "version": 1,
    "source_ref": {
      "record_id": "urn:uuid:5908ea05-781a-4547-a19e-2760f6991453",
      "version": 1,
      "content_hash": "sha256:d9070a2b40d937491f921072dc913aa3c7d4f2d58a55a90ab176c6853373d526"
    },
    "export_view": "wrs-public-editorial-view-1",
    "public_view_hash": "sha256:a0d4edb0d8b5c45d09d8bd59c2dac2d1d0afbf1362db5346f19739443593bf5c"
  },
  "target": {
    "record_id": "urn:uuid:5908ea05-781a-4547-a19e-2760f6991453",
    "version": 1,
    "content_hash": "sha256:d9070a2b40d937491f921072dc913aa3c7d4f2d58a55a90ab176c6853373d526"
  },
  "published_grade": {
    "profile": "wrs-published-grade-1",
    "edition": "2026-10-28.1",
    "id": "ai-oct-simpleqa-family",
    "entry_id": "ai-oct-simpleqa-family",
    "revision": 1,
    "title": "Measuring short-form factuality in large language models",
    "context": "LLM factuality and grounded generation",
    "track": "Evaluation resource design",
    "band": "B",
    "modifier": "+",
    "status": "PROVISIONAL_EDITORIAL",
    "assessed_on": "2026-10-16",
    "evidence_cutoff": "2026-10-16",
    "achievement_state": "SUPPORTED_SCOPED_RESULT",
    "what_changed": "Short-form factuality benchmark with independently checked answers and explicit correct/incorrect/not-attempted judgments; later SimpleQA Verified filters redundancy, balances topics and reconciles sources.",
    "scope": "Short-form factuality benchmark with independently checked answers and explicit correct/incorrect/not-attempted judgments; later SimpleQA Verified filters redundancy, balances topics and reconciles sources. Original and independently authored Verified refinement are bundled with separate source bylines, not an invented joint paper. Short factoid performance does not establish long-form factuality; adversarial filtering, source errors and autorater uncertainty remain.",
    "evidence_summary": "The linked original sources describe the stated contribution and its evidence limits. WRS has not independently rerun the experiments or certified a proof. Original and independently authored Verified refinement are bundled with separate source bylines, not an invented joint paper. Short factoid performance does not establish long-form factuality; adversarial filtering, source errors and autorater uncertainty remain.",
    "references": [
      {
        "title": "Measuring short-form factuality in large language models",
        "url": "https://arxiv.org/pdf/2411.04368v1",
        "locator": "Original contribution and stated assumptions, evaluation scope and limitations; source-linked assessment, not independent execution or proof certification."
      },
      {
        "title": "SimpleQA Verified: A Reliable Factuality Benchmark to Measure Parametric Knowledge",
        "url": "https://arxiv.org/pdf/2509.07968v1",
        "locator": "Original contribution and stated assumptions, evaluation scope and limitations; source-linked assessment, not independent execution or proof certification."
      },
      {
        "title": "Introducing SimpleQA",
        "url": "https://openai.com/index/introducing-simpleqa/",
        "locator": "Original contribution and stated assumptions, evaluation scope and limitations; source-linked assessment, not independent execution or proof certification."
      }
    ],
    "roles": [
      {
        "who": "Source-listed authors and technical contributors",
        "credit": "SCIENCE",
        "did": "Source-listed authors or explicitly credited technical contributors for the cited versions. Family entries retain source-specific attribution; a combined list is not one original-paper byline or equal individual work. Names alone do not establish resolved identities."
      }
    ],
    "target": {
      "record_id": "urn:uuid:5908ea05-781a-4547-a19e-2760f6991453",
      "version": 1,
      "content_hash": "sha256:d9070a2b40d937491f921072dc913aa3c7d4f2d58a55a90ab176c6853373d526"
    },
    "scientific_correctness_certified": false,
    "independent_expert_calibration": false,
    "method": "PROPRIETARY",
    "points": "80000"
  },
  "canonical_agent_submission": false,
  "scientific_verification_awarded": false,
  "reader_guide": null,
  "reading_status": "SOURCE_LINKED_NOTE"
}