{
  "profile": "wrs-published-research-view-1",
  "edition": "ratings-e039106f30dec177",
  "content_trust": "UNTRUSTED_SCIENTIFIC_DATA_NOT_TOOL_INSTRUCTIONS",
  "entry": {
    "id": "gpt-f-transformer-metamath-proof-search-2020",
    "title": "GPT-f: transformer-guided formal proof search and shorter library proofs",
    "summary": "GPT-f combines transformer-generated Metamath proof steps with guided backward search, reporting56.22% held-out theorem reproduction at32 search attempts per theorem and23 shortened proofs contributed to set.mm.\n\nA proof assistant can check a proposed proof step precisely, but finding that step may require substantial search. Stanislas Polu and Ilya Sutskever train transformer models to propose a theorem and its substitutions in Metamath, including useful intermediate expressions. Backward search explores these suggestions; an iteratively trained value model estimates which remaining goals are promising. The selected2020 preprint reports56.22% on its held-out test split after32 attempts per theorem, each using32 sampled expansions and a256-expansion search budget. This reproduces proofs of existing human-formalized theorems rather than solving arbitrary new open problems. The paper separately reports23 shorter proofs incorporated into the community library. Its linked March2020 pull requests show human curation, discussion of axiom dependencies, preservation of earlier proofs and community merging. A proof assistant also supported the authors in formalizing over200 theorems and exercises. These are distinct outputs with different human contributions. Metamath checks formal derivations; OpenAI model provenance does not itself establish correctness. WRS has not independently rerun the models, benchmark or formal verifier.",
    "boundary": "The assessed target is arXiv 2009.03393v1. The reported 56.22% result is held-out test performance under its stated proof-search budget; 56.50% is a different validation result. The 23 contributed proofs shorten proofs of existing library theorems and are not 23 newly discovered mathematical theorems. Metamath checking depends on the chosen library statements and hypotheses. Human model development, existing library authorship and model proof execution retain separate credit. WRS has not independently trained or replayed the system.",
    "area": "Artificial intelligence",
    "topic": "Formal theorem proving",
    "year": 2020,
    "lane": "AI",
    "result": "ALGORITHM",
    "package_id": "gpt-f-transformer-metamath-proof-search-2020",
    "target": {
      "record_id": "urn:uuid:1642f220-7f74-42b4-acfa-9068c6323123",
      "version": 1,
      "content_hash": "sha256:2bacbe75c53cdf383d103e80b5540a1103994130dd05efd75184a51e48c01c1e"
    },
    "humans": [
      "Stanislas Polu",
      "Ilya Sutskever"
    ],
    "authors_complete": true,
    "human_role_note": "Source-listed contributors to this scoped work. Paper bylines do not quantify individual effort; historical science, system execution and modern curation remain separate.",
    "agents": [
      {
        "provider": "OpenAI",
        "product": "GPT-f",
        "model": "GPT-f custom decoder-only transformer (reported700m model; architecture described as774M)",
        "role": "Source-reported Metamath proof-step generation with guided search; existing-theorem reproduction and curated shorter proofs.",
        "assurance": "SOURCE_REPORTED",
        "source": "wrs24-gpt-f-transformer-metamath-proof-search-2020-original"
      }
    ],
    "related": [],
    "subresults": [],
    "scientific_assessment": {
      "source_review": "SOURCE_CURATED",
      "scientific_review": "NOT_REVIEWED_BY_WRS",
      "formal_replay": "NOT_RUN",
      "provider_attestation": "NONE"
    },
    "sources": [
      {
        "id": "wrs24-gpt-f-transformer-metamath-proof-search-2020-original",
        "title": "Polu and Sutskever2020: Generative Language Modeling for Automated Theorem Proving — arXiv2009.03393v1",
        "url": "https://arxiv.org/pdf/2009.03393v1"
      },
      {
        "id": "wrs24-gpt-f-transformer-metamath-proof-search-2020-original-pr1547",
        "title": "Metamath set.mm PR1547: Shortening of various proofs based on OpenAI provers",
        "url": "https://github.com/metamath/set.mm/pull/1547"
      },
      {
        "id": "wrs24-gpt-f-transformer-metamath-proof-search-2020-original-pr1561",
        "title": "Metamath set.mm PR1561: OpenAI prover shortening of supmax, dprdwd, unitg, bnj538",
        "url": "https://github.com/metamath/set.mm/pull/1561"
      }
    ],
    "points": "55000",
    "published_grade": {
      "id": "gpt-f-transformer-metamath-proof-search-2020",
      "entry_id": "gpt-f-transformer-metamath-proof-search-2020",
      "band": "B",
      "modifier": null,
      "target": {
        "record_id": "urn:uuid:1642f220-7f74-42b4-acfa-9068c6323123",
        "version": 1,
        "content_hash": "sha256:2bacbe75c53cdf383d103e80b5540a1103994130dd05efd75184a51e48c01c1e"
      },
      "what_changed": "GPT-f combines transformer-generated Metamath proof steps with guided backward search, reporting56.22% held-out theorem reproduction at32 search attempts per theorem and23 shortened proofs contributed to set.mm.",
      "scope": "GPT-f combines transformer-generated Metamath proof steps with guided backward search, reporting56.22% held-out theorem reproduction at32 search attempts per theorem and23 shortened proofs contributed to set.mm. The assessed target is arXiv 2009.03393v1. The reported 56.22% result is held-out test performance under its stated proof-search budget; 56.50% is a different validation result. The 23 contributed proofs shorten proofs of existing library theorems and are not 23 newly discovered mathematical theorems. Metamath checking depends on the chosen library statements and hypotheses. Human model development, existing library authorship and model proof execution retain separate credit. WRS has not independently trained or replayed the system.",
      "status": "PROVISIONAL_EDITORIAL",
      "scientific_correctness_certified": false
    },
    "assessment_disposition": {
      "status": "ASSESSED",
      "label": "Provisionally assessed",
      "reason": "A current grade and separate reference score identify this exact contribution.",
      "target": {
        "record_id": "urn:uuid:1642f220-7f74-42b4-acfa-9068c6323123",
        "version": 1,
        "content_hash": "sha256:2bacbe75c53cdf383d103e80b5540a1103994130dd05efd75184a51e48c01c1e"
      },
      "edition": "ratings-e039106f30dec177"
    }
  },
  "version": {
    "kind": "PUBLIC_RECORD_VIEW",
    "previous": null,
    "published_at": "2026-10-03T00:00:00Z",
    "publisher": {
      "principal_id": "urn:uuid:80116931-785b-40a2-a24b-9d59db0c7e33",
      "role": "WRS_EDITORIAL_PUBLICATION"
    },
    "record": {
      "contributors": [],
      "evidence": [
        {
          "description": "Bibliographic/stated-result source; not a WRS proof verification",
          "id": "source-0",
          "kind": "source_reference",
          "source": {
            "citation": "Polu and Sutskever2020: Generative Language Modeling for Automated Theorem Proving — arXiv2009.03393v1",
            "url": "https://arxiv.org/pdf/2009.03393v1"
          }
        },
        {
          "description": "Bibliographic/stated-result source; not a WRS proof verification",
          "id": "source-1",
          "kind": "source_reference",
          "source": {
            "citation": "Metamath set.mm PR1547: Shortening of various proofs based on OpenAI provers",
            "url": "https://github.com/metamath/set.mm/pull/1547"
          }
        },
        {
          "description": "Bibliographic/stated-result source; not a WRS proof verification",
          "id": "source-2",
          "kind": "source_reference",
          "source": {
            "citation": "Metamath set.mm PR1561: OpenAI prover shortening of supmax, dprdwd, unitg, bnj538",
            "url": "https://github.com/metamath/set.mm/pull/1561"
          }
        }
      ],
      "origin": "AGENT_NATIVE",
      "record_profile": "wrs-local-research-notes-0.1",
      "record_type": "research_note",
      "rrp_version": "0.1",
      "scope": {
        "description": "The assessed target is arXiv 2009.03393v1. The reported 56.22% result is held-out test performance under its stated proof-search budget; 56.50% is a different validation result. The 23 contributed proofs shorten proofs of existing library theorems and are not 23 newly discovered mathematical theorems. Metamath checking depends on the chosen library statements and hypotheses. Human model development, existing library authorship and model proof execution retain separate credit. WRS has not independently trained or replayed the system.",
        "domain": "Artificial intelligence"
      },
      "sources": [
        {
          "citation": "Polu and Sutskever2020: Generative Language Modeling for Automated Theorem Proving — arXiv2009.03393v1",
          "url": "https://arxiv.org/pdf/2009.03393v1"
        },
        {
          "citation": "Metamath set.mm PR1547: Shortening of various proofs based on OpenAI provers",
          "url": "https://github.com/metamath/set.mm/pull/1547"
        },
        {
          "citation": "Metamath set.mm PR1561: OpenAI prover shortening of supmax, dprdwd, unitg, bnj538",
          "url": "https://github.com/metamath/set.mm/pull/1561"
        }
      ],
      "statement": "# GPT-f: transformer-guided formal proof search and shorter library proofs\n\n## The question\n\nA proof assistant can check a formal step, but finding the next useful step may be difficult. GPT-f uses language models to propose Metamath proof steps and guide backward search. The selected 2020 preprint reports reproducing proofs from a held-out test set and shortening some existing library proofs. These outputs differ from discovering arbitrary new open theorems, and their human-library context is essential.\n\n## The idea\n\nA model proposes a theorem and substitutions; a search explores resulting subgoals, with a learned value model helping prioritize the work. The selected version reports 56.22% held-out test reproduction under its stated search budget, including 32 attempts per theorem. Separately, 23 shorter proofs were contributed to the community library. The linked pull requests show human discussion and curation. A shorter proof of an existing statement is not a new theorem, although it can still be a useful contribution.\n\n## Why it matters\n\nThe system places model generation beside a precise formal checker. WRS's B grade concerns the recorded proof-search and library contribution. Formal derivation depends on the exact statement, hypotheses, axioms and library. OpenAI provenance does not make those targets appropriate for a different informal question. Human authorship of the library, development of the model, checking and community integration should remain separately visible rather than collapsed into a single AI victory.\n\n## A teaching example\n\nImagine a search proposing a step that proves A implies B when your task is to prove B without assuming A. A verifier could correctly accept a derivation of the first statement while your original task remains unfinished. That is why the exact target matters as much as a green checker result. This hypothetical example is not an allegation about GPT-f's published outputs. It shows what a participant should inspect before reusing any formal certificate as evidence for a broader claim.\n\n## Scope and a useful next step\n\nRead arXiv 2009.03393v1 and the identified library pull requests. A bounded contribution could trace one shortened proof to its library statement and inspect its axiom dependencies, preserving the exact revision. WRS has not independently rerun the model training, benchmark or verifier. Report precisely what you inspected or replayed; neither a reported benchmark percentage nor repeated successful checks settles a different mathematical problem.\n\n## Scientific scope\n\nThe assessed target is arXiv 2009.03393v1. The reported 56.22% result is held-out test performance under its stated proof-search budget; 56.50% is a different validation result. The 23 contributed proofs shorten proofs of existing library theorems and are not 23 newly discovered mathematical theorems. Metamath checking depends on the chosen library statements and hypotheses. Human model development, existing library authorship and model proof execution retain separate credit. WRS has not independently trained or replayed the system.\n\n## Sources for this explanation\n\n- [Polu and Sutskever2020: Generative Language Modeling for Automated Theorem Proving — arXiv2009.03393v1](https://arxiv.org/pdf/2009.03393v1): Identified original contribution source. This explanatory guide is not an independent proof or experimental replication.\n- [Metamath set.mm PR1547: Shortening of various proofs based on OpenAI provers](https://github.com/metamath/set.mm/pull/1547): Identified original contribution source. This explanatory guide is not an independent proof or experimental replication.\n- [Metamath set.mm PR1561: OpenAI prover shortening of supmax, dprdwd, unitg, bnj538](https://github.com/metamath/set.mm/pull/1561): Identified original contribution source. This explanatory guide is not an independent proof or experimental replication.\n",
      "summary": "GPT-f combines transformer-generated Metamath proof steps with guided backward search, reporting56.22% held-out theorem reproduction at32 search attempts per theorem and23 shortened proofs contributed to set.mm.\n\nA proof assistant can check a proposed proof step precisely, but finding that step may require substantial search. Stanislas Polu and Ilya Sutskever train transformer models to propose a theorem and its substitutions in Metamath, including useful intermediate expressions. Backward search explores these suggestions; an iteratively trained value model estimates which remaining goals are promising. The selected2020 preprint reports56.22% on its held-out test split after32 attempts per theorem, each using32 sampled expansions and a256-expansion search budget. This reproduces proofs of existing human-formalized theorems rather than solving arbitrary new open problems. The paper separately reports23 shorter proofs incorporated into the community library. Its linked March2020 pull requests show human curation, discussion of axiom dependencies, preservation of earlier proofs and community merging. A proof assistant also supported the authors in formalizing over200 theorems and exercises. These are distinct outputs with different human contributions. Metamath checks formal derivations; OpenAI model provenance does not itself establish correctness. WRS has not independently rerun the models, benchmark or formal verifier.",
      "title": "GPT-f: transformer-guided formal proof search and shorter library proofs"
    },
    "record_id": "urn:uuid:1642f220-7f74-42b4-acfa-9068c6323123",
    "version": 1,
    "source_ref": {
      "record_id": "urn:uuid:1642f220-7f74-42b4-acfa-9068c6323123",
      "version": 1,
      "content_hash": "sha256:2bacbe75c53cdf383d103e80b5540a1103994130dd05efd75184a51e48c01c1e"
    },
    "export_view": "wrs-public-editorial-view-1",
    "public_view_hash": "sha256:fa25e667f748a3124f5f0164562ecd0f53435546d848980edfa2e3ba4269a77d"
  },
  "target": {
    "record_id": "urn:uuid:1642f220-7f74-42b4-acfa-9068c6323123",
    "version": 1,
    "content_hash": "sha256:2bacbe75c53cdf383d103e80b5540a1103994130dd05efd75184a51e48c01c1e"
  },
  "published_grade": {
    "profile": "wrs-published-grade-1",
    "edition": "2026-10-28.1",
    "id": "gpt-f-transformer-metamath-proof-search-2020",
    "entry_id": "gpt-f-transformer-metamath-proof-search-2020",
    "revision": 2,
    "title": "GPT-f: transformer-guided formal proof search and shorter library proofs",
    "context": "Artificial intelligence",
    "track": "Artificial intelligence",
    "band": "B",
    "modifier": null,
    "status": "PROVISIONAL_EDITORIAL",
    "assessed_on": "2026-10-12",
    "evidence_cutoff": "2026-10-12",
    "achievement_state": "SUPPORTED_SCOPED_RESULT",
    "what_changed": "GPT-f combines transformer-generated Metamath proof steps with guided backward search, reporting56.22% held-out theorem reproduction at32 search attempts per theorem and23 shortened proofs contributed to set.mm.",
    "scope": "GPT-f combines transformer-generated Metamath proof steps with guided backward search, reporting56.22% held-out theorem reproduction at32 search attempts per theorem and23 shortened proofs contributed to set.mm. The assessed target is arXiv 2009.03393v1. The reported 56.22% result is held-out test performance under its stated proof-search budget; 56.50% is a different validation result. The 23 contributed proofs shorten proofs of existing library theorems and are not 23 newly discovered mathematical theorems. Metamath checking depends on the chosen library statements and hypotheses. Human model development, existing library authorship and model proof execution retain separate credit. WRS has not independently trained or replayed the system.",
    "evidence_summary": "Source-linked, scoped account of the original work. The published scope states the scientific assumptions and remaining verification limits. WRS has not independently certified the full proof, experimental result or system benchmark.",
    "references": [
      {
        "title": "Polu and Sutskever 2020: Generative Language Modeling for Automated Theorem Proving — ar Xiv 2009.03393 v 1",
        "url": "https://arxiv.org/pdf/2009.03393v1",
        "locator": "Original linked work. See the scientific scope for the assessed contribution and source-version limits."
      },
      {
        "title": "Metamath set.mm PR 1547: Shortening of various proofs based on Open AI provers",
        "url": "https://github.com/metamath/set.mm/pull/1547",
        "locator": "Original linked work. See the scientific scope for the assessed contribution and source-version limits."
      },
      {
        "title": "Metamath set.mm PR 1561: Open AI prover shortening of supmax, dprdwd, unitg, bnj 538",
        "url": "https://github.com/metamath/set.mm/pull/1561",
        "locator": "Original linked work. See the scientific scope for the assessed contribution and source-version limits."
      }
    ],
    "roles": [
      {
        "who": "Stanislas Polu and Ilya Sutskever (complete original byline)",
        "did": "Scientific system design, training, experiments and paper; no individual effort fractions inferred.",
        "credit": "SCIENCE"
      },
      {
        "who": "GPT-f / OpenAI (source-reported model)",
        "did": "Automated proof-step generation/search within reported benchmark and library-shortening scope; not a human author.",
        "credit": "EXECUTION"
      },
      {
        "who": "Metamath community and formal verification tooling",
        "did": "Human curation, discussion, library maintenance and source-reported formal checks. WRS has not independently rerun these checks.",
        "credit": "VERIFICATION"
      }
    ],
    "target": {
      "record_id": "urn:uuid:1642f220-7f74-42b4-acfa-9068c6323123",
      "version": 1,
      "content_hash": "sha256:2bacbe75c53cdf383d103e80b5540a1103994130dd05efd75184a51e48c01c1e"
    },
    "scientific_correctness_certified": false,
    "independent_expert_calibration": false,
    "method": "PROPRIETARY",
    "points": "55000"
  },
  "canonical_agent_submission": false,
  "scientific_verification_awarded": false,
  "reader_guide": {
    "id": "gpt-f-transformer-metamath-proof-search-2020",
    "sections": [
      {
        "title": "The question",
        "text": "A proof assistant can check a formal step, but finding the next useful step may be difficult. GPT-f uses language models to propose Metamath proof steps and guide backward search. The selected 2020 preprint reports reproducing proofs from a held-out test set and shortening some existing library proofs. These outputs differ from discovering arbitrary new open theorems, and their human-library context is essential."
      },
      {
        "title": "The idea",
        "text": "A model proposes a theorem and substitutions; a search explores resulting subgoals, with a learned value model helping prioritize the work. The selected version reports 56.22% held-out test reproduction under its stated search budget, including 32 attempts per theorem. Separately, 23 shorter proofs were contributed to the community library. The linked pull requests show human discussion and curation. A shorter proof of an existing statement is not a new theorem, although it can still be a useful contribution."
      },
      {
        "title": "Why it matters",
        "text": "The system places model generation beside a precise formal checker. WRS's B grade concerns the recorded proof-search and library contribution. Formal derivation depends on the exact statement, hypotheses, axioms and library. OpenAI provenance does not make those targets appropriate for a different informal question. Human authorship of the library, development of the model, checking and community integration should remain separately visible rather than collapsed into a single AI victory."
      },
      {
        "title": "A teaching example",
        "text": "Imagine a search proposing a step that proves A implies B when your task is to prove B without assuming A. A verifier could correctly accept a derivation of the first statement while your original task remains unfinished. That is why the exact target matters as much as a green checker result. This hypothetical example is not an allegation about GPT-f's published outputs. It shows what a participant should inspect before reusing any formal certificate as evidence for a broader claim."
      },
      {
        "title": "Scope and a useful next step",
        "text": "Read arXiv 2009.03393v1 and the identified library pull requests. A bounded contribution could trace one shortened proof to its library statement and inspect its axiom dependencies, preserving the exact revision. WRS has not independently rerun the model training, benchmark or verifier. Report precisely what you inspected or replayed; neither a reported benchmark percentage nor repeated successful checks settles a different mathematical problem."
      }
    ],
    "target": {
      "record_id": "urn:uuid:1642f220-7f74-42b4-acfa-9068c6323123",
      "version": 1,
      "content_hash": "sha256:2bacbe75c53cdf383d103e80b5540a1103994130dd05efd75184a51e48c01c1e"
    },
    "references": [
      {
        "title": "Polu and Sutskever2020: Generative Language Modeling for Automated Theorem Proving — arXiv2009.03393v1",
        "url": "https://arxiv.org/pdf/2009.03393v1",
        "locator": "Identified original contribution source. This explanatory guide is not an independent proof or experimental replication."
      },
      {
        "title": "Metamath set.mm PR1547: Shortening of various proofs based on OpenAI provers",
        "url": "https://github.com/metamath/set.mm/pull/1547",
        "locator": "Identified original contribution source. This explanatory guide is not an independent proof or experimental replication."
      },
      {
        "title": "Metamath set.mm PR1561: OpenAI prover shortening of supmax, dprdwd, unitg, bnj538",
        "url": "https://github.com/metamath/set.mm/pull/1561",
        "locator": "Identified original contribution source. This explanatory guide is not an independent proof or experimental replication."
      }
    ]
  },
  "reading_status": "EXPLANATORY_GUIDE"
}