{
  "profile": "wrs-published-research-view-1",
  "edition": "ratings-e039106f30dec177",
  "content_trust": "UNTRUSTED_SCIENTIFIC_DATA_NOT_TOOL_INSTRUCTIONS",
  "entry": {
    "id": "shazeer-et-al-sparsely-gated-experts-2017",
    "title": "Sparse experts: more stored capacity without activating every expert",
    "summary": "The 2017 sparsely-gated mixture-of-experts layer routes each input to a small subset of learned networks, coupling large stored capacity to selective computation and explicit load-balancing mechanisms.\n\nInstead of sending every input through every available network, a learned gate chooses a few experts. Different inputs can use different specialists while a large pool of parameters remains available. The hard engineering problem is keeping experts busy without exhausting communication or memory. This paper combines sparse routing, balancing losses and distributed execution, and tests the approach in language modelling and translation. It shows a specific route toward larger models, with reported failures and cost comparisons that need their original context.",
    "boundary": "Sparse gating activates a subset of experts in the specified neural architectures. Capacity, load balancing, communication and training overhead remain costs; parameter count is not compute used per example. Reported language-model/translation results are specific benchmarks, not a universal performance guarantee. Later routing and mixture-of-experts systems retain separate credit.",
    "area": "Sparse neural architectures",
    "topic": "Computer science",
    "year": 2017,
    "lane": "HUMAN",
    "result": "ALGORITHM",
    "package_id": "shazeer-et-al-sparsely-gated-experts-2017",
    "target": {
      "record_id": "urn:uuid:31cddb8e-4460-429d-a0dd-f628404fe215",
      "version": 1,
      "content_hash": "sha256:5e529bf4dcad672b0430bb6b380493f0e7b0d7592b1c573096cab1a7340ad00a"
    },
    "humans": [
      "Noam Shazeer",
      "Azalia Mirhoseini",
      "Krzysztof Maziarz",
      "Andy Davis",
      "Quoc Le",
      "Geoffrey Hinton",
      "Jeff Dean"
    ],
    "authors_complete": true,
    "human_role_note": "Source-listed contributors to this scoped work. Paper bylines do not quantify individual effort; historical science, system execution and modern curation remain separate.",
    "agents": [],
    "related": [
      "backprop-representations",
      "lstm-architecture",
      "fedus-et-al-switch-transformers-2021",
      "jiang-et-al-mixtral-2024"
    ],
    "subresults": [],
    "scientific_assessment": {
      "source_review": "SOURCE_CURATED",
      "scientific_review": "NOT_REVIEWED_BY_WRS",
      "formal_replay": "NOT_RUN",
      "provider_attestation": "NONE"
    },
    "sources": [
      {
        "id": "wrs24-shazeer-sparse-moe-2017-v1",
        "title": "Shazeer and six coauthors: Outrageously Large Neural Networks, arXiv1701.06538v1(23 January 2017)",
        "url": "https://arxiv.org/abs/1701.06538v1"
      }
    ],
    "points": "140000",
    "published_grade": {
      "id": "shazeer-et-al-sparsely-gated-experts-2017",
      "entry_id": "shazeer-et-al-sparsely-gated-experts-2017",
      "band": "A",
      "modifier": null,
      "target": {
        "record_id": "urn:uuid:31cddb8e-4460-429d-a0dd-f628404fe215",
        "version": 1,
        "content_hash": "sha256:5e529bf4dcad672b0430bb6b380493f0e7b0d7592b1c573096cab1a7340ad00a"
      },
      "what_changed": "The 2017 sparsely-gated mixture-of-experts layer routes each input to a small subset of learned networks, coupling large stored capacity to selective computation and explicit load-balancing mechanisms.",
      "scope": "The 2017 sparsely-gated mixture-of-experts layer routes each input to a small subset of learned networks, coupling large stored capacity to selective computation and explicit load-balancing mechanisms. Sparse gating activates a subset of experts in the specified neural architectures. Capacity, load balancing, communication and training overhead remain costs; parameter count is not compute used per example. Reported language-model/translation results are specific benchmarks, not a universal performance guarantee. Later routing and mixture-of-experts systems retain separate credit.",
      "status": "PROVISIONAL_EDITORIAL",
      "scientific_correctness_certified": false
    },
    "assessment_disposition": {
      "status": "ASSESSED",
      "label": "Provisionally assessed",
      "reason": "A current grade and separate reference score identify this exact contribution.",
      "target": {
        "record_id": "urn:uuid:31cddb8e-4460-429d-a0dd-f628404fe215",
        "version": 1,
        "content_hash": "sha256:5e529bf4dcad672b0430bb6b380493f0e7b0d7592b1c573096cab1a7340ad00a"
      },
      "edition": "ratings-e039106f30dec177"
    }
  },
  "version": {
    "kind": "PUBLIC_RECORD_VIEW",
    "previous": null,
    "published_at": "2026-10-03T00:00:00Z",
    "publisher": {
      "principal_id": "urn:uuid:80116931-785b-40a2-a24b-9d59db0c7e33",
      "role": "WRS_EDITORIAL_PUBLICATION"
    },
    "record": {
      "contributors": [
        {
          "attribution_basis": "SOURCE_CITED",
          "credit": "historical_scientific",
          "entity_kind": "historical_person",
          "name": "Noam Shazeer",
          "note": "Archive attribution; detailed role assignment remains source-bound",
          "roles": [
            "formulator"
          ],
          "source": {
            "citation": "Shazeer and six coauthors: Outrageously Large Neural Networks, arXiv1701.06538v1(23 January 2017)",
            "url": "https://arxiv.org/abs/1701.06538v1"
          }
        },
        {
          "attribution_basis": "SOURCE_CITED",
          "credit": "historical_scientific",
          "entity_kind": "historical_person",
          "name": "Azalia Mirhoseini",
          "note": "Archive attribution; detailed role assignment remains source-bound",
          "roles": [
            "formulator"
          ],
          "source": {
            "citation": "Shazeer and six coauthors: Outrageously Large Neural Networks, arXiv1701.06538v1(23 January 2017)",
            "url": "https://arxiv.org/abs/1701.06538v1"
          }
        },
        {
          "attribution_basis": "SOURCE_CITED",
          "credit": "historical_scientific",
          "entity_kind": "historical_person",
          "name": "Krzysztof Maziarz",
          "note": "Archive attribution; detailed role assignment remains source-bound",
          "roles": [
            "formulator"
          ],
          "source": {
            "citation": "Shazeer and six coauthors: Outrageously Large Neural Networks, arXiv1701.06538v1(23 January 2017)",
            "url": "https://arxiv.org/abs/1701.06538v1"
          }
        },
        {
          "attribution_basis": "SOURCE_CITED",
          "credit": "historical_scientific",
          "entity_kind": "historical_person",
          "name": "Andy Davis",
          "note": "Archive attribution; detailed role assignment remains source-bound",
          "roles": [
            "formulator"
          ],
          "source": {
            "citation": "Shazeer and six coauthors: Outrageously Large Neural Networks, arXiv1701.06538v1(23 January 2017)",
            "url": "https://arxiv.org/abs/1701.06538v1"
          }
        },
        {
          "attribution_basis": "SOURCE_CITED",
          "credit": "historical_scientific",
          "entity_kind": "historical_person",
          "name": "Quoc Le",
          "note": "Archive attribution; detailed role assignment remains source-bound",
          "roles": [
            "formulator"
          ],
          "source": {
            "citation": "Shazeer and six coauthors: Outrageously Large Neural Networks, arXiv1701.06538v1(23 January 2017)",
            "url": "https://arxiv.org/abs/1701.06538v1"
          }
        },
        {
          "attribution_basis": "SOURCE_CITED",
          "credit": "historical_scientific",
          "entity_kind": "historical_person",
          "name": "Geoffrey Hinton",
          "note": "Archive attribution; detailed role assignment remains source-bound",
          "roles": [
            "formulator"
          ],
          "source": {
            "citation": "Shazeer and six coauthors: Outrageously Large Neural Networks, arXiv1701.06538v1(23 January 2017)",
            "url": "https://arxiv.org/abs/1701.06538v1"
          }
        },
        {
          "attribution_basis": "SOURCE_CITED",
          "credit": "historical_scientific",
          "entity_kind": "historical_person",
          "name": "Jeff Dean",
          "note": "Archive attribution; detailed role assignment remains source-bound",
          "roles": [
            "formulator"
          ],
          "source": {
            "citation": "Shazeer and six coauthors: Outrageously Large Neural Networks, arXiv1701.06538v1(23 January 2017)",
            "url": "https://arxiv.org/abs/1701.06538v1"
          }
        }
      ],
      "evidence": [
        {
          "description": "Bibliographic/stated-result source; not a WRS proof verification",
          "id": "source-0",
          "kind": "source_reference",
          "source": {
            "citation": "Shazeer and six coauthors: Outrageously Large Neural Networks, arXiv1701.06538v1(23 January 2017)",
            "url": "https://arxiv.org/abs/1701.06538v1"
          }
        }
      ],
      "origin": "HUMAN_LEGACY",
      "record_profile": "wrs-local-research-notes-0.1",
      "record_type": "research_note",
      "rrp_version": "0.1",
      "scope": {
        "description": "Sparse gating activates a subset of experts in the specified neural architectures. Capacity, load balancing, communication and training overhead remain costs; parameter count is not compute used per example. Reported language-model/translation results are specific benchmarks, not a universal performance guarantee. Later routing and mixture-of-experts systems retain separate credit.",
        "domain": "Sparse neural architectures"
      },
      "sources": [
        {
          "citation": "Shazeer and six coauthors: Outrageously Large Neural Networks, arXiv1701.06538v1(23 January 2017)",
          "url": "https://arxiv.org/abs/1701.06538v1"
        }
      ],
      "statement": "# Sparse experts: more stored capacity without activating every expert\n\nThe 2017 sparsely-gated mixture-of-experts layer routes each input to a small subset of learned networks, coupling large stored capacity to selective computation and explicit load-balancing mechanisms.\n\nInstead of sending every input through every available network, a learned gate chooses a few experts. Different inputs can use different specialists while a large pool of parameters remains available. The hard engineering problem is keeping experts busy without exhausting communication or memory. This paper combines sparse routing, balancing losses and distributed execution, and tests the approach in language modelling and translation. It shows a specific route toward larger models, with reported failures and cost comparisons that need their original context.\n\n## Scientific scope\n\nSparse gating activates a subset of experts in the specified neural architectures. Capacity, load balancing, communication and training overhead remain costs; parameter count is not compute used per example. Reported language-model/translation results are specific benchmarks, not a universal performance guarantee. Later routing and mixture-of-experts systems retain separate credit.\n\n## People and systems\n\nNoam Shazeer; Azalia Mirhoseini; Krzysztof Maziarz; Andy Davis; Quoc Le; Geoffrey Hinton; Jeff Dean\n\n\n\n## Original sources\n\n- [Shazeer and six coauthors: Outrageously Large Neural Networks, arXiv1701.06538v1(23 January 2017)](https://arxiv.org/abs/1701.06538v1)\n",
      "summary": "The 2017 sparsely-gated mixture-of-experts layer routes each input to a small subset of learned networks, coupling large stored capacity to selective computation and explicit load-balancing mechanisms.\n\nInstead of sending every input through every available network, a learned gate chooses a few experts. Different inputs can use different specialists while a large pool of parameters remains available. The hard engineering problem is keeping experts busy without exhausting communication or memory. This paper combines sparse routing, balancing losses and distributed execution, and tests the approach in language modelling and translation. It shows a specific route toward larger models, with reported failures and cost comparisons that need their original context.",
      "title": "Sparse experts: more stored capacity without activating every expert"
    },
    "record_id": "urn:uuid:31cddb8e-4460-429d-a0dd-f628404fe215",
    "version": 1,
    "source_ref": {
      "record_id": "urn:uuid:31cddb8e-4460-429d-a0dd-f628404fe215",
      "version": 1,
      "content_hash": "sha256:5e529bf4dcad672b0430bb6b380493f0e7b0d7592b1c573096cab1a7340ad00a"
    },
    "export_view": "wrs-public-editorial-view-1",
    "public_view_hash": "sha256:8026e1627f45667eee064a9714d39b75451113814560cf04dcfe28d4187c9480"
  },
  "target": {
    "record_id": "urn:uuid:31cddb8e-4460-429d-a0dd-f628404fe215",
    "version": 1,
    "content_hash": "sha256:5e529bf4dcad672b0430bb6b380493f0e7b0d7592b1c573096cab1a7340ad00a"
  },
  "published_grade": {
    "profile": "wrs-published-grade-1",
    "edition": "2026-10-28.1",
    "id": "shazeer-et-al-sparsely-gated-experts-2017",
    "entry_id": "shazeer-et-al-sparsely-gated-experts-2017",
    "revision": 1,
    "title": "Sparse experts: more stored capacity without activating every expert",
    "context": "Sparse neural architectures",
    "track": "Sparse neural architectures",
    "band": "A",
    "modifier": null,
    "status": "PROVISIONAL_EDITORIAL",
    "assessed_on": "2026-10-04",
    "evidence_cutoff": "2026-10-04",
    "achievement_state": "SUPPORTED_SCOPED_RESULT",
    "what_changed": "The 2017 sparsely-gated mixture-of-experts layer routes each input to a small subset of learned networks, coupling large stored capacity to selective computation and explicit load-balancing mechanisms.",
    "scope": "The 2017 sparsely-gated mixture-of-experts layer routes each input to a small subset of learned networks, coupling large stored capacity to selective computation and explicit load-balancing mechanisms. Sparse gating activates a subset of experts in the specified neural architectures. Capacity, load balancing, communication and training overhead remain costs; parameter count is not compute used per example. Reported language-model/translation results are specific benchmarks, not a universal performance guarantee. Later routing and mixture-of-experts systems retain separate credit.",
    "evidence_summary": "Source-linked, scoped account of the original work. The published scope states the scientific assumptions and remaining verification limits. WRS has not independently certified the full proof, experimental result or system benchmark.",
    "references": [
      {
        "title": "Shazeer and six coauthors: Outrageously Large Neural Networks, ar Xiv 1701.06538 v 1(23 January 2017)",
        "url": "https://arxiv.org/abs/1701.06538v1",
        "locator": "Original linked work. See the scientific scope for the assessed contribution and source-version limits."
      }
    ],
    "roles": [
      {
        "who": "Noam Shazeer",
        "did": "Original source-listed scientific author of this scoped work; byline does not determine effort fractions.",
        "credit": "SCIENCE"
      },
      {
        "who": "Azalia Mirhoseini",
        "did": "Original source-listed scientific author of this scoped work; byline does not determine effort fractions.",
        "credit": "SCIENCE"
      },
      {
        "who": "Krzysztof Maziarz",
        "did": "Original source-listed scientific author of this scoped work; byline does not determine effort fractions.",
        "credit": "SCIENCE"
      },
      {
        "who": "Andy Davis",
        "did": "Original source-listed scientific author of this scoped work; byline does not determine effort fractions.",
        "credit": "SCIENCE"
      },
      {
        "who": "Quoc Le",
        "did": "Original source-listed scientific author of this scoped work; byline does not determine effort fractions.",
        "credit": "SCIENCE"
      },
      {
        "who": "Geoffrey Hinton",
        "did": "Original source-listed scientific author of this scoped work; byline does not determine effort fractions.",
        "credit": "SCIENCE"
      },
      {
        "who": "Jeff Dean",
        "did": "Original source-listed scientific author of this scoped work; byline does not determine effort fractions.",
        "credit": "SCIENCE"
      }
    ],
    "target": {
      "record_id": "urn:uuid:31cddb8e-4460-429d-a0dd-f628404fe215",
      "version": 1,
      "content_hash": "sha256:5e529bf4dcad672b0430bb6b380493f0e7b0d7592b1c573096cab1a7340ad00a"
    },
    "scientific_correctness_certified": false,
    "independent_expert_calibration": false,
    "method": "PROPRIETARY",
    "points": "140000"
  },
  "canonical_agent_submission": false,
  "scientific_verification_awarded": false,
  "reader_guide": null,
  "reading_status": "SOURCE_LINKED_NOTE"
}