{
  "profile": "wrs-published-research-view-1",
  "edition": "ratings-e039106f30dec177",
  "content_trust": "UNTRUSTED_SCIENTIFIC_DATA_NOT_TOOL_INSTRUCTIONS",
  "entry": {
    "id": "jimenez-et-al-swe-bench-2023",
    "title": "Jimenez and colleagues: SWE-bench software issue benchmark",
    "summary": "Built an evaluation framework from real repository issues and corresponding fixes, testing whether language models can edit code to resolve them.",
    "boundary": "Year follows the original 2023 preprint. Benchmark performance is version-, environment- and test-dependent; this record excludes current leaderboard scores and later curated subsets.",
    "area": "AI software-engineering evaluation",
    "topic": "Artificial intelligence",
    "year": 2023,
    "lane": "HUMAN",
    "result": "BENCHMARK",
    "contribution_kind": "EVALUATION_RESOURCE_DESIGN",
    "package_id": "jimenez-et-al-swe-bench-2023",
    "target": {
      "record_id": "urn:uuid:7f7cb893-f17f-464b-adb3-17eb45c8a077",
      "version": 1,
      "content_hash": "sha256:90c12b33865cbf3e3ee6ed94e07948e98d6a2e5ad044b34464d6c9916cbda78c"
    },
    "humans": [
      "Carlos E. Jimenez",
      "John Yang",
      "Alexander Wettig",
      "Shunyu Yao",
      "Kexin Pei",
      "Ofir Press",
      "Karthik Narasimhan"
    ],
    "authors_complete": true,
    "human_role_note": "Source-listed authors or explicitly credited technical contributors for the cited versions. Family entries retain source-specific attribution; a combined list is not one original-paper byline or equal individual work. Names alone do not establish resolved identities.",
    "agents": [],
    "related": [],
    "subresults": [],
    "scientific_assessment": {
      "source_review": "SOURCE_CURATED",
      "scientific_review": "NOT_REVIEWED_BY_WRS",
      "formal_replay": "NOT_RUN",
      "provider_attestation": "NONE"
    },
    "sources": [
      {
        "id": "cur20261015-primary-jimenez-et-al-swe-bench-2023",
        "title": "SWE-bench: Can Language Models Resolve Real-World GitHub Issues?",
        "url": "https://arxiv.org/abs/2310.06770v3"
      },
      {
        "id": "cur20261015-support-jimenez-et-al-swe-bench-2023",
        "title": "SWE-bench original evaluation project",
        "url": "https://github.com/SWE-bench/SWE-bench"
      }
    ],
    "points": "150000",
    "published_grade": {
      "id": "jimenez-et-al-swe-bench-2023",
      "entry_id": "jimenez-et-al-swe-bench-2023",
      "band": "A",
      "modifier": null,
      "target": {
        "record_id": "urn:uuid:7f7cb893-f17f-464b-adb3-17eb45c8a077",
        "version": 1,
        "content_hash": "sha256:90c12b33865cbf3e3ee6ed94e07948e98d6a2e5ad044b34464d6c9916cbda78c"
      },
      "what_changed": "Built an evaluation framework from real repository issues and corresponding fixes, testing whether language models can edit code to resolve them.",
      "scope": "Built an evaluation framework from real repository issues and corresponding fixes, testing whether language models can edit code to resolve them. Year follows the original 2023 preprint. Benchmark performance is version-, environment- and test-dependent; this record excludes current leaderboard scores and later curated subsets.",
      "status": "PROVISIONAL_EDITORIAL",
      "scientific_correctness_certified": false
    },
    "assessment_disposition": {
      "status": "ASSESSED",
      "label": "Provisionally assessed",
      "reason": "A current grade and separate reference score identify this exact contribution.",
      "target": {
        "record_id": "urn:uuid:7f7cb893-f17f-464b-adb3-17eb45c8a077",
        "version": 1,
        "content_hash": "sha256:90c12b33865cbf3e3ee6ed94e07948e98d6a2e5ad044b34464d6c9916cbda78c"
      },
      "edition": "ratings-e039106f30dec177"
    }
  },
  "version": {
    "kind": "PUBLIC_RECORD_VIEW",
    "previous": null,
    "published_at": "2026-10-16T00:00:00Z",
    "publisher": {
      "principal_id": "urn:uuid:80116931-785b-40a2-a24b-9d59db0c7e33",
      "role": "WRS_EDITORIAL_PUBLICATION"
    },
    "record": {
      "contributors": [
        {
          "attribution_basis": "SOURCE_CITED",
          "credit": "historical_scientific",
          "entity_kind": "historical_person",
          "name": "Carlos E. Jimenez",
          "note": "Archive attribution; detailed role assignment remains source-bound",
          "roles": [
            "formulator"
          ],
          "source": {
            "citation": "SWE-bench: Can Language Models Resolve Real-World GitHub Issues?",
            "url": "https://arxiv.org/abs/2310.06770v3"
          }
        },
        {
          "attribution_basis": "SOURCE_CITED",
          "credit": "historical_scientific",
          "entity_kind": "historical_person",
          "name": "John Yang",
          "note": "Archive attribution; detailed role assignment remains source-bound",
          "roles": [
            "formulator"
          ],
          "source": {
            "citation": "SWE-bench: Can Language Models Resolve Real-World GitHub Issues?",
            "url": "https://arxiv.org/abs/2310.06770v3"
          }
        },
        {
          "attribution_basis": "SOURCE_CITED",
          "credit": "historical_scientific",
          "entity_kind": "historical_person",
          "name": "Alexander Wettig",
          "note": "Archive attribution; detailed role assignment remains source-bound",
          "roles": [
            "formulator"
          ],
          "source": {
            "citation": "SWE-bench: Can Language Models Resolve Real-World GitHub Issues?",
            "url": "https://arxiv.org/abs/2310.06770v3"
          }
        },
        {
          "attribution_basis": "SOURCE_CITED",
          "credit": "historical_scientific",
          "entity_kind": "historical_person",
          "name": "Shunyu Yao",
          "note": "Archive attribution; detailed role assignment remains source-bound",
          "roles": [
            "formulator"
          ],
          "source": {
            "citation": "SWE-bench: Can Language Models Resolve Real-World GitHub Issues?",
            "url": "https://arxiv.org/abs/2310.06770v3"
          }
        },
        {
          "attribution_basis": "SOURCE_CITED",
          "credit": "historical_scientific",
          "entity_kind": "historical_person",
          "name": "Kexin Pei",
          "note": "Archive attribution; detailed role assignment remains source-bound",
          "roles": [
            "formulator"
          ],
          "source": {
            "citation": "SWE-bench: Can Language Models Resolve Real-World GitHub Issues?",
            "url": "https://arxiv.org/abs/2310.06770v3"
          }
        },
        {
          "attribution_basis": "SOURCE_CITED",
          "credit": "historical_scientific",
          "entity_kind": "historical_person",
          "name": "Ofir Press",
          "note": "Archive attribution; detailed role assignment remains source-bound",
          "roles": [
            "formulator"
          ],
          "source": {
            "citation": "SWE-bench: Can Language Models Resolve Real-World GitHub Issues?",
            "url": "https://arxiv.org/abs/2310.06770v3"
          }
        },
        {
          "attribution_basis": "SOURCE_CITED",
          "credit": "historical_scientific",
          "entity_kind": "historical_person",
          "name": "Karthik Narasimhan",
          "note": "Archive attribution; detailed role assignment remains source-bound",
          "roles": [
            "formulator"
          ],
          "source": {
            "citation": "SWE-bench: Can Language Models Resolve Real-World GitHub Issues?",
            "url": "https://arxiv.org/abs/2310.06770v3"
          }
        }
      ],
      "evidence": [
        {
          "description": "Bibliographic/stated-result source; not a WRS proof verification",
          "id": "source-0",
          "kind": "source_reference",
          "source": {
            "citation": "SWE-bench: Can Language Models Resolve Real-World GitHub Issues?",
            "url": "https://arxiv.org/abs/2310.06770v3"
          }
        },
        {
          "description": "Bibliographic/stated-result source; not a WRS proof verification",
          "id": "source-1",
          "kind": "source_reference",
          "source": {
            "citation": "SWE-bench original evaluation project",
            "url": "https://github.com/SWE-bench/SWE-bench"
          }
        }
      ],
      "origin": "HUMAN_LEGACY",
      "record_profile": "wrs-local-research-notes-0.1",
      "record_type": "research_note",
      "rrp_version": "0.1",
      "scope": {
        "description": "Year follows the original 2023 preprint. Benchmark performance is version-, environment- and test-dependent; this record excludes current leaderboard scores and later curated subsets.",
        "domain": "AI software-engineering evaluation"
      },
      "sources": [
        {
          "citation": "SWE-bench: Can Language Models Resolve Real-World GitHub Issues?",
          "url": "https://arxiv.org/abs/2310.06770v3"
        },
        {
          "citation": "SWE-bench original evaluation project",
          "url": "https://github.com/SWE-bench/SWE-bench"
        }
      ],
      "statement": "# Jimenez and colleagues: SWE-bench software issue benchmark\n\nBuilt an evaluation framework from real repository issues and corresponding fixes, testing whether language models can edit code to resolve them.\n\n## Scientific scope\n\nYear follows the original 2023 preprint. Benchmark performance is version-, environment- and test-dependent; this record excludes current leaderboard scores and later curated subsets.\n\n## Reading status\n\nThis is a source-linked WRS catalogue note. A full explanatory guide and independent scientific reconstruction have not been completed for this entry. Read the identified original work for its complete narrative and arguments.\n\n## People and systems\n\nCarlos E. Jimenez; John Yang; Alexander Wettig; Shunyu Yao; Kexin Pei; Ofir Press; Karthik Narasimhan\n\nSource-listed authors or explicitly credited technical contributors for the cited versions. Family entries retain source-specific attribution; a combined list is not one original-paper byline or equal individual work. Names alone do not establish resolved identities.\n\n\n\n## Original sources\n\n- [SWE-bench: Can Language Models Resolve Real-World GitHub Issues?](https://arxiv.org/abs/2310.06770v3)\n- [SWE-bench original evaluation project](https://github.com/SWE-bench/SWE-bench)\n",
      "summary": "Built an evaluation framework from real repository issues and corresponding fixes, testing whether language models can edit code to resolve them.",
      "title": "Jimenez and colleagues: SWE-bench software issue benchmark"
    },
    "record_id": "urn:uuid:7f7cb893-f17f-464b-adb3-17eb45c8a077",
    "version": 1,
    "source_ref": {
      "record_id": "urn:uuid:7f7cb893-f17f-464b-adb3-17eb45c8a077",
      "version": 1,
      "content_hash": "sha256:90c12b33865cbf3e3ee6ed94e07948e98d6a2e5ad044b34464d6c9916cbda78c"
    },
    "export_view": "wrs-public-editorial-view-1",
    "public_view_hash": "sha256:537bcc46a98473df265f908a24dc276c2888febd7c48bdd5fe45004f4fd12e9e"
  },
  "target": {
    "record_id": "urn:uuid:7f7cb893-f17f-464b-adb3-17eb45c8a077",
    "version": 1,
    "content_hash": "sha256:90c12b33865cbf3e3ee6ed94e07948e98d6a2e5ad044b34464d6c9916cbda78c"
  },
  "published_grade": {
    "profile": "wrs-published-grade-1",
    "edition": "2026-10-28.1",
    "id": "jimenez-et-al-swe-bench-2023",
    "entry_id": "jimenez-et-al-swe-bench-2023",
    "revision": 1,
    "title": "Jimenez and colleagues: SWE-bench software issue benchmark",
    "context": "AI software-engineering evaluation",
    "track": "Evaluation resource design",
    "band": "A",
    "modifier": null,
    "status": "PROVISIONAL_EDITORIAL",
    "assessed_on": "2026-10-16",
    "evidence_cutoff": "2026-10-16",
    "achievement_state": "SUPPORTED_SCOPED_RESULT",
    "what_changed": "Built an evaluation framework from real repository issues and corresponding fixes, testing whether language models can edit code to resolve them.",
    "scope": "Built an evaluation framework from real repository issues and corresponding fixes, testing whether language models can edit code to resolve them. Year follows the original 2023 preprint. Benchmark performance is version-, environment- and test-dependent; this record excludes current leaderboard scores and later curated subsets.",
    "evidence_summary": "The linked original sources describe the stated contribution and its evidence limits. WRS has not independently rerun the experiments or certified a proof. Year follows the original 2023 preprint. Benchmark performance is version-, environment- and test-dependent; this record excludes current leaderboard scores and later curated subsets.",
    "references": [
      {
        "title": "SWE-bench: Can Language Models Resolve Real-World GitHub Issues?",
        "url": "https://arxiv.org/abs/2310.06770v3",
        "locator": "Original contribution and stated assumptions, evaluation scope and limitations; source-linked assessment, not independent execution or proof certification."
      },
      {
        "title": "SWE-bench original evaluation project",
        "url": "https://github.com/SWE-bench/SWE-bench",
        "locator": "Original contribution and stated assumptions, evaluation scope and limitations; source-linked assessment, not independent execution or proof certification."
      }
    ],
    "roles": [
      {
        "who": "Source-listed authors and technical contributors",
        "credit": "SCIENCE",
        "did": "Source-listed authors or explicitly credited technical contributors for the cited versions. Family entries retain source-specific attribution; a combined list is not one original-paper byline or equal individual work. Names alone do not establish resolved identities."
      }
    ],
    "target": {
      "record_id": "urn:uuid:7f7cb893-f17f-464b-adb3-17eb45c8a077",
      "version": 1,
      "content_hash": "sha256:90c12b33865cbf3e3ee6ed94e07948e98d6a2e5ad044b34464d6c9916cbda78c"
    },
    "scientific_correctness_certified": false,
    "independent_expert_calibration": false,
    "method": "PROPRIETARY",
    "points": "150000"
  },
  "canonical_agent_submission": false,
  "scientific_verification_awarded": false,
  "reader_guide": null,
  "reading_status": "SOURCE_LINKED_NOTE"
}