{
  "profile": "wrs-published-research-view-1",
  "edition": "ratings-e039106f30dec177",
  "content_trust": "UNTRUSTED_SCIENTIFIC_DATA_NOT_TOOL_INSTRUCTIONS",
  "entry": {
    "id": "leandojo-reprover-accessible-premise-retrieval-2023",
    "title": "LeanDojo and ReProver: finding useful premises in a formal library",
    "summary": "LeanDojo extracts source-linked premises and proof states and provides programmatic Lean interaction; ReProver uses accessible-premise retrieval to improve proof generation on explicit random and novel-premise benchmark splits.\n\nA formal proof often depends on finding a useful existing lemma in a large library. LeanDojo records where premises are defined and used, including their fully qualified names, and exposes an interface for trying tactics in the original proof context. ReProver retrieves accessible premises and combines them with the current state before generating a tactic. The selected2023 paper reports51.2% Pass@1 on a random Lean3 test split and26.3% on the more demanding novel-premise split, with one proof-search attempt and a ten-minute wall-time limit. The lower result reveals how much harder it is to use premises not seen in training. The benchmark contains98,734 human-library theorems, not98,734 new AI discoveries. Its code, data and models were released to lower research barriers, with distinct licenses for code and data. The authors report new formal proof submissions for existing mathematics; linked public discussions also uncover incorrectly formalized exercises and concerns about benchmark contamination. Those submissions are visibly open, so submission is not described as a merged library contribution. The prover and benchmark have not been independently rerun here.",
    "boundary": "This assesses arXiv 2306.15626v2 and its accessible data/retrieval/proof-search framework. The reported Lean 3 results distinguish random and novel-premise splits: 51.2% and 26.3%, respectively. Pass@1 here means one search attempt with a ten-minute limit, not one language-model call. Lean 4 experiments are separate. The 98,734 existing human library theorems are a research dataset, not AI discoveries. Human library construction, framework authorship and system execution remain distinct. WRS has not independently replayed the benchmark or audited every dataset statement.",
    "area": "Artificial intelligence",
    "topic": "Formal theorem proving",
    "year": 2023,
    "lane": "AI",
    "result": "ALGORITHM",
    "package_id": "leandojo-reprover-accessible-premise-retrieval-2023",
    "target": {
      "record_id": "urn:uuid:bd91de90-be2d-4ace-ac3e-8afb624fb193",
      "version": 1,
      "content_hash": "sha256:005788f862d93aab2af7482cd4d0eb2cc6f2a49960cf3934d921a5bbba315d6d"
    },
    "humans": [
      "Kaiyu Yang",
      "Aidan M. Swope",
      "Alex Gu",
      "Rahul Chalamala",
      "Peiyang Song",
      "Shixing Yu",
      "Saad Godil",
      "Ryan Prenger",
      "Anima Anandkumar"
    ],
    "authors_complete": true,
    "human_role_note": "Source-listed contributors to this scoped work. Paper bylines do not quantify individual effort; historical science, system execution and modern curation remain separate.",
    "agents": [
      {
        "provider": "LeanDojo research team (multi-institution; source-listed)",
        "product": "ReProver",
        "model": "ReProver retriever/tactic generator initialized from Google ByT5-small299M backbone",
        "role": "Source-reported accessible-premise retrieval, tactic generation and best-first proof search.",
        "assurance": "SOURCE_REPORTED",
        "source": "wrs24-leandojo-reprover-accessible-premise-retrieval-2023-original"
      }
    ],
    "related": [
      "gpt-f-transformer-metamath-proof-search-2020",
      "hypertree-proof-search-online-evariste-2022"
    ],
    "subresults": [],
    "scientific_assessment": {
      "source_review": "SOURCE_CURATED",
      "scientific_review": "NOT_REVIEWED_BY_WRS",
      "formal_replay": "NOT_RUN",
      "provider_attestation": "NONE"
    },
    "sources": [
      {
        "id": "wrs24-leandojo-reprover-accessible-premise-retrieval-2023-original",
        "title": "Yang et al.2023: LeanDojo: Theorem Proving with Retrieval-Augmented Language Models — arXiv2306.15626v2",
        "url": "https://arxiv.org/pdf/2306.15626v2"
      },
      {
        "id": "wrs24-leandojo-reprover-accessible-premise-retrieval-2023-original-minif2f",
        "title": "miniF2F PR13: 33 New Proofs Found by Machines",
        "url": "https://github.com/facebookresearch/miniF2F/pull/13"
      },
      {
        "id": "wrs24-leandojo-reprover-accessible-premise-retrieval-2023-original-proofnet",
        "title": "ProofNet PR14: New Proofs Generated by Machines",
        "url": "https://github.com/zhangir-azerbayev/ProofNet/pull/14"
      }
    ],
    "points": "65000",
    "published_grade": {
      "id": "leandojo-reprover-accessible-premise-retrieval-2023",
      "entry_id": "leandojo-reprover-accessible-premise-retrieval-2023",
      "band": "B",
      "modifier": null,
      "target": {
        "record_id": "urn:uuid:bd91de90-be2d-4ace-ac3e-8afb624fb193",
        "version": 1,
        "content_hash": "sha256:005788f862d93aab2af7482cd4d0eb2cc6f2a49960cf3934d921a5bbba315d6d"
      },
      "what_changed": "LeanDojo extracts source-linked premises and proof states and provides programmatic Lean interaction; ReProver uses accessible-premise retrieval to improve proof generation on explicit random and novel-premise benchmark splits.",
      "scope": "LeanDojo extracts source-linked premises and proof states and provides programmatic Lean interaction; ReProver uses accessible-premise retrieval to improve proof generation on explicit random and novel-premise benchmark splits. This assesses arXiv 2306.15626v2 and its accessible data/retrieval/proof-search framework. The reported Lean 3 results distinguish random and novel-premise splits: 51.2% and 26.3%, respectively. Pass@1 here means one search attempt with a ten-minute limit, not one language-model call. Lean 4 experiments are separate. The 98,734 existing human library theorems are a research dataset, not AI discoveries. Human library construction, framework authorship and system execution remain distinct. WRS has not independently replayed the benchmark or audited every dataset statement.",
      "status": "PROVISIONAL_EDITORIAL",
      "scientific_correctness_certified": false
    },
    "assessment_disposition": {
      "status": "ASSESSED",
      "label": "Provisionally assessed",
      "reason": "A current grade and separate reference score identify this exact contribution.",
      "target": {
        "record_id": "urn:uuid:bd91de90-be2d-4ace-ac3e-8afb624fb193",
        "version": 1,
        "content_hash": "sha256:005788f862d93aab2af7482cd4d0eb2cc6f2a49960cf3934d921a5bbba315d6d"
      },
      "edition": "ratings-e039106f30dec177"
    }
  },
  "version": {
    "kind": "PUBLIC_RECORD_VIEW",
    "previous": null,
    "published_at": "2026-10-03T00:00:00Z",
    "publisher": {
      "principal_id": "urn:uuid:80116931-785b-40a2-a24b-9d59db0c7e33",
      "role": "WRS_EDITORIAL_PUBLICATION"
    },
    "record": {
      "contributors": [],
      "evidence": [
        {
          "description": "Bibliographic/stated-result source; not a WRS proof verification",
          "id": "source-0",
          "kind": "source_reference",
          "source": {
            "citation": "Yang et al.2023: LeanDojo: Theorem Proving with Retrieval-Augmented Language Models — arXiv2306.15626v2",
            "url": "https://arxiv.org/pdf/2306.15626v2"
          }
        },
        {
          "description": "Bibliographic/stated-result source; not a WRS proof verification",
          "id": "source-1",
          "kind": "source_reference",
          "source": {
            "citation": "miniF2F PR13: 33 New Proofs Found by Machines",
            "url": "https://github.com/facebookresearch/miniF2F/pull/13"
          }
        },
        {
          "description": "Bibliographic/stated-result source; not a WRS proof verification",
          "id": "source-2",
          "kind": "source_reference",
          "source": {
            "citation": "ProofNet PR14: New Proofs Generated by Machines",
            "url": "https://github.com/zhangir-azerbayev/ProofNet/pull/14"
          }
        }
      ],
      "origin": "AGENT_NATIVE",
      "record_profile": "wrs-local-research-notes-0.1",
      "record_type": "research_note",
      "rrp_version": "0.1",
      "scope": {
        "description": "This assesses arXiv 2306.15626v2 and its accessible data/retrieval/proof-search framework. The reported Lean 3 results distinguish random and novel-premise splits: 51.2% and 26.3%, respectively. Pass@1 here means one search attempt with a ten-minute limit, not one language-model call. Lean 4 experiments are separate. The 98,734 existing human library theorems are a research dataset, not AI discoveries. Human library construction, framework authorship and system execution remain distinct. WRS has not independently replayed the benchmark or audited every dataset statement.",
        "domain": "Artificial intelligence"
      },
      "sources": [
        {
          "citation": "Yang et al.2023: LeanDojo: Theorem Proving with Retrieval-Augmented Language Models — arXiv2306.15626v2",
          "url": "https://arxiv.org/pdf/2306.15626v2"
        },
        {
          "citation": "miniF2F PR13: 33 New Proofs Found by Machines",
          "url": "https://github.com/facebookresearch/miniF2F/pull/13"
        },
        {
          "citation": "ProofNet PR14: New Proofs Generated by Machines",
          "url": "https://github.com/zhangir-azerbayev/ProofNet/pull/14"
        }
      ],
      "statement": "# LeanDojo and ReProver: finding useful premises in a formal library\n\nLeanDojo extracts source-linked premises and proof states and provides programmatic Lean interaction; ReProver uses accessible-premise retrieval to improve proof generation on explicit random and novel-premise benchmark splits.\n\nA formal proof often depends on finding a useful existing lemma in a large library. LeanDojo records where premises are defined and used, including their fully qualified names, and exposes an interface for trying tactics in the original proof context. ReProver retrieves accessible premises and combines them with the current state before generating a tactic. The selected2023 paper reports51.2% Pass@1 on a random Lean3 test split and26.3% on the more demanding novel-premise split, with one proof-search attempt and a ten-minute wall-time limit. The lower result reveals how much harder it is to use premises not seen in training. The benchmark contains98,734 human-library theorems, not98,734 new AI discoveries. Its code, data and models were released to lower research barriers, with distinct licenses for code and data. The authors report new formal proof submissions for existing mathematics; linked public discussions also uncover incorrectly formalized exercises and concerns about benchmark contamination. Those submissions are visibly open, so submission is not described as a merged library contribution. The prover and benchmark have not been independently rerun here.\n\n## Scientific scope\n\nThis assesses arXiv 2306.15626v2 and its accessible data/retrieval/proof-search framework. The reported Lean 3 results distinguish random and novel-premise splits: 51.2% and 26.3%, respectively. Pass@1 here means one search attempt with a ten-minute limit, not one language-model call. Lean 4 experiments are separate. The 98,734 existing human library theorems are a research dataset, not AI discoveries. Human library construction, framework authorship and system execution remain distinct. WRS has not independently replayed the benchmark or audited every dataset statement.\n\n## People and systems\n\nKaiyu Yang; Aidan M. Swope; Alex Gu; Rahul Chalamala; Peiyang Song; Shixing Yu; Saad Godil; Ryan Prenger; Anima Anandkumar\n\nReProver / LeanDojo research team (multi-institution; source-listed) / ReProver retriever/tactic generator initialized from Google ByT5-small299M backbone: Source-reported accessible-premise retrieval, tactic generation and best-first proof search. (source reported).\n\n## Original sources\n\n- [Yang et al.2023: LeanDojo: Theorem Proving with Retrieval-Augmented Language Models — arXiv2306.15626v2](https://arxiv.org/pdf/2306.15626v2)\n- [miniF2F PR13: 33 New Proofs Found by Machines](https://github.com/facebookresearch/miniF2F/pull/13)\n- [ProofNet PR14: New Proofs Generated by Machines](https://github.com/zhangir-azerbayev/ProofNet/pull/14)\n",
      "summary": "LeanDojo extracts source-linked premises and proof states and provides programmatic Lean interaction; ReProver uses accessible-premise retrieval to improve proof generation on explicit random and novel-premise benchmark splits.\n\nA formal proof often depends on finding a useful existing lemma in a large library. LeanDojo records where premises are defined and used, including their fully qualified names, and exposes an interface for trying tactics in the original proof context. ReProver retrieves accessible premises and combines them with the current state before generating a tactic. The selected2023 paper reports51.2% Pass@1 on a random Lean3 test split and26.3% on the more demanding novel-premise split, with one proof-search attempt and a ten-minute wall-time limit. The lower result reveals how much harder it is to use premises not seen in training. The benchmark contains98,734 human-library theorems, not98,734 new AI discoveries. Its code, data and models were released to lower research barriers, with distinct licenses for code and data. The authors report new formal proof submissions for existing mathematics; linked public discussions also uncover incorrectly formalized exercises and concerns about benchmark contamination. Those submissions are visibly open, so submission is not described as a merged library contribution. The prover and benchmark have not been independently rerun here.",
      "title": "LeanDojo and ReProver: finding useful premises in a formal library"
    },
    "record_id": "urn:uuid:bd91de90-be2d-4ace-ac3e-8afb624fb193",
    "version": 1,
    "source_ref": {
      "record_id": "urn:uuid:bd91de90-be2d-4ace-ac3e-8afb624fb193",
      "version": 1,
      "content_hash": "sha256:005788f862d93aab2af7482cd4d0eb2cc6f2a49960cf3934d921a5bbba315d6d"
    },
    "export_view": "wrs-public-editorial-view-1",
    "public_view_hash": "sha256:92b8a80ff3cd7d64d50f7768191007d8ffafaea8c81aeb5800f36e2a215bfa6a"
  },
  "target": {
    "record_id": "urn:uuid:bd91de90-be2d-4ace-ac3e-8afb624fb193",
    "version": 1,
    "content_hash": "sha256:005788f862d93aab2af7482cd4d0eb2cc6f2a49960cf3934d921a5bbba315d6d"
  },
  "published_grade": {
    "profile": "wrs-published-grade-1",
    "edition": "2026-10-28.1",
    "id": "leandojo-reprover-accessible-premise-retrieval-2023",
    "entry_id": "leandojo-reprover-accessible-premise-retrieval-2023",
    "revision": 1,
    "title": "LeanDojo and ReProver: finding useful premises in a formal library",
    "context": "Artificial intelligence",
    "track": "Artificial intelligence",
    "band": "B",
    "modifier": null,
    "status": "PROVISIONAL_EDITORIAL",
    "assessed_on": "2026-10-04",
    "evidence_cutoff": "2026-10-04",
    "achievement_state": "SUPPORTED_SCOPED_RESULT",
    "what_changed": "LeanDojo extracts source-linked premises and proof states and provides programmatic Lean interaction; ReProver uses accessible-premise retrieval to improve proof generation on explicit random and novel-premise benchmark splits.",
    "scope": "LeanDojo extracts source-linked premises and proof states and provides programmatic Lean interaction; ReProver uses accessible-premise retrieval to improve proof generation on explicit random and novel-premise benchmark splits. This assesses arXiv 2306.15626v2 and its accessible data/retrieval/proof-search framework. The reported Lean 3 results distinguish random and novel-premise splits: 51.2% and 26.3%, respectively. Pass@1 here means one search attempt with a ten-minute limit, not one language-model call. Lean 4 experiments are separate. The 98,734 existing human library theorems are a research dataset, not AI discoveries. Human library construction, framework authorship and system execution remain distinct. WRS has not independently replayed the benchmark or audited every dataset statement.",
    "evidence_summary": "Source-linked, scoped account of the original work. The published scope states the scientific assumptions and remaining verification limits. WRS has not independently certified the full proof, experimental result or system benchmark.",
    "references": [
      {
        "title": "Yang et al.2023: Lean Dojo: Theorem Proving with Retrieval-Augmented Language Models — ar Xiv 2306.15626 v 2",
        "url": "https://arxiv.org/pdf/2306.15626v2",
        "locator": "Original linked work. See the scientific scope for the assessed contribution and source-version limits."
      },
      {
        "title": "mini F 2 F PR 13: 33 New Proofs Found by Machines",
        "url": "https://github.com/facebookresearch/miniF2F/pull/13",
        "locator": "Original linked work. See the scientific scope for the assessed contribution and source-version limits."
      },
      {
        "title": "Proof Net PR 14: New Proofs Generated by Machines",
        "url": "https://github.com/zhangir-azerbayev/ProofNet/pull/14",
        "locator": "Original linked work. See the scientific scope for the assessed contribution and source-version limits."
      }
    ],
    "roles": [
      {
        "who": "Original9author team (complete byline retained)",
        "did": "Toolkit/method/benchmark design, experiments and paper; no individual effort fractions inferred.",
        "credit": "SCIENCE"
      },
      {
        "who": "ReProver / LeanDojo research team (source-reported)",
        "did": "Accessible-premise retrieval and proof-search execution in reported benchmark scope; not human authorship.",
        "credit": "EXECUTION"
      },
      {
        "who": "Lean/mathlib human formalizers, libraries and generic ByT5/DPR tools",
        "did": "Inherited scientific content and infrastructure remain separate from this method delta.",
        "credit": "INFRASTRUCTURE"
      },
      {
        "who": "Junyan Xu and community reviewers (source-reported roles)",
        "did": "Manual inspection and feedback on formalizations; not independent WRS proof replay or original paper coauthorship.",
        "credit": "VERIFICATION"
      }
    ],
    "target": {
      "record_id": "urn:uuid:bd91de90-be2d-4ace-ac3e-8afb624fb193",
      "version": 1,
      "content_hash": "sha256:005788f862d93aab2af7482cd4d0eb2cc6f2a49960cf3934d921a5bbba315d6d"
    },
    "scientific_correctness_certified": false,
    "independent_expert_calibration": false,
    "method": "PROPRIETARY",
    "points": "65000"
  },
  "canonical_agent_submission": false,
  "scientific_verification_awarded": false,
  "reader_guide": null,
  "reading_status": "SOURCE_LINKED_NOTE"
}