{
  "profile": "wrs-published-research-view-1",
  "edition": "ratings-e039106f30dec177",
  "content_trust": "UNTRUSTED_SCIENTIFIC_DATA_NOT_TOOL_INSTRUCTIONS",
  "entry": {
    "id": "rafailov-et-al-direct-preference-optimization-2023",
    "title": "DPO: learn response preferences without a separate reward-training loop",
    "summary": "Rafailov and colleagues reparameterize a KL-regularized preference-learning objective as a direct policy classification loss, removing the separate learned-reward/PPO optimization stage and reporting results on sentiment, summarization and single-turn dialogue.\n\nA model can learn from pairs of responses that people rank without first building a separate reward model and then running a reinforcement-learning loop. DPO uses the model's own response probabilities, compared with a fixed reference model, to fit those preferences directly. A mathematical change of variables connects this loss to a familiar regularized reward objective under specified preference and support assumptions. The paper then tests the resulting training method on several text tasks. It simplifies a particular preference-training pipeline; it does not turn popular answers into scientific truth, eliminate the need for preference data or prove that every future model is aligned.",
    "boundary": "The reformulation uses a specified preference/regularization model, reference policy and pairwise objective. Finite preference data, model misspecification and optimization limits remain. Avoiding an explicit reward-model/RL loop does not imply objectively correct preferences, universal alignment or zero training cost. Reported comparisons are setup-dependent and have not been independently replayed by WRS.",
    "area": "Direct preference learning",
    "topic": "Computing",
    "year": 2023,
    "lane": "HUMAN",
    "result": "ALGORITHM",
    "package_id": "rafailov-et-al-direct-preference-optimization-2023",
    "target": {
      "record_id": "urn:uuid:baeaf450-e457-4dbd-abfa-2c2095f0a6a5",
      "version": 1,
      "content_hash": "sha256:2d547550a5c9df63e7b5e4823946bab912e01612138e8848646389e5b929c5f9"
    },
    "humans": [
      "Rafael Rafailov",
      "Archit Sharma",
      "Eric Mitchell",
      "Stefano Ermon",
      "Christopher D. Manning",
      "Chelsea Finn"
    ],
    "authors_complete": true,
    "human_role_note": "Source-listed contributors to this scoped work. Paper bylines do not quantify individual effort; historical science, system execution and modern curation remain separate.",
    "agents": [],
    "related": [
      "ppo-2017",
      "instructgpt-2022"
    ],
    "subresults": [],
    "scientific_assessment": {
      "source_review": "SOURCE_CURATED",
      "scientific_review": "NOT_REVIEWED_BY_WRS",
      "formal_replay": "NOT_RUN",
      "provider_attestation": "NONE"
    },
    "sources": [
      {
        "id": "wrs24-dpo2023-v1-primary",
        "title": "Rafailov et al., Direct Preference Optimization, arXiv2305.18290v1 (2023)",
        "url": "https://arxiv.org/pdf/2305.18290v1"
      },
      {
        "id": "wrs24-dpo-version-byline-arxiv",
        "title": "arXiv DPO version history and complete byline",
        "url": "https://arxiv.org/abs/2305.18290v1"
      }
    ],
    "points": "140000",
    "published_grade": {
      "id": "rafailov-et-al-direct-preference-optimization-2023",
      "entry_id": "rafailov-et-al-direct-preference-optimization-2023",
      "band": "A",
      "modifier": null,
      "target": {
        "record_id": "urn:uuid:baeaf450-e457-4dbd-abfa-2c2095f0a6a5",
        "version": 1,
        "content_hash": "sha256:2d547550a5c9df63e7b5e4823946bab912e01612138e8848646389e5b929c5f9"
      },
      "what_changed": "Rafailov and colleagues reparameterize a KL-regularized preference-learning objective as a direct policy classification loss, removing the separate learned-reward/PPO optimization stage and reporting results on sentiment, summarization and single-turn dialogue.",
      "scope": "Rafailov and colleagues reparameterize a KL-regularized preference-learning objective as a direct policy classification loss, removing the separate learned-reward/PPO optimization stage and reporting results on sentiment, summarization and single-turn dialogue. The reformulation uses a specified preference/regularization model, reference policy and pairwise objective. Finite preference data, model misspecification and optimization limits remain. Avoiding an explicit reward-model/RL loop does not imply objectively correct preferences, universal alignment or zero training cost. Reported comparisons are setup-dependent and have not been independently replayed by WRS.",
      "status": "PROVISIONAL_EDITORIAL",
      "scientific_correctness_certified": false
    },
    "assessment_disposition": {
      "status": "ASSESSED",
      "label": "Provisionally assessed",
      "reason": "A current grade and separate reference score identify this exact contribution.",
      "target": {
        "record_id": "urn:uuid:baeaf450-e457-4dbd-abfa-2c2095f0a6a5",
        "version": 1,
        "content_hash": "sha256:2d547550a5c9df63e7b5e4823946bab912e01612138e8848646389e5b929c5f9"
      },
      "edition": "ratings-e039106f30dec177"
    }
  },
  "version": {
    "kind": "PUBLIC_RECORD_VIEW",
    "previous": null,
    "published_at": "2026-10-03T00:00:00Z",
    "publisher": {
      "principal_id": "urn:uuid:80116931-785b-40a2-a24b-9d59db0c7e33",
      "role": "WRS_EDITORIAL_PUBLICATION"
    },
    "record": {
      "contributors": [
        {
          "attribution_basis": "SOURCE_CITED",
          "credit": "historical_scientific",
          "entity_kind": "historical_person",
          "name": "Rafael Rafailov",
          "note": "Archive attribution; detailed role assignment remains source-bound",
          "roles": [
            "formulator"
          ],
          "source": {
            "citation": "Rafailov et al., Direct Preference Optimization, arXiv2305.18290v1 (2023)",
            "url": "https://arxiv.org/pdf/2305.18290v1"
          }
        },
        {
          "attribution_basis": "SOURCE_CITED",
          "credit": "historical_scientific",
          "entity_kind": "historical_person",
          "name": "Archit Sharma",
          "note": "Archive attribution; detailed role assignment remains source-bound",
          "roles": [
            "formulator"
          ],
          "source": {
            "citation": "Rafailov et al., Direct Preference Optimization, arXiv2305.18290v1 (2023)",
            "url": "https://arxiv.org/pdf/2305.18290v1"
          }
        },
        {
          "attribution_basis": "SOURCE_CITED",
          "credit": "historical_scientific",
          "entity_kind": "historical_person",
          "name": "Eric Mitchell",
          "note": "Archive attribution; detailed role assignment remains source-bound",
          "roles": [
            "formulator"
          ],
          "source": {
            "citation": "Rafailov et al., Direct Preference Optimization, arXiv2305.18290v1 (2023)",
            "url": "https://arxiv.org/pdf/2305.18290v1"
          }
        },
        {
          "attribution_basis": "SOURCE_CITED",
          "credit": "historical_scientific",
          "entity_kind": "historical_person",
          "name": "Stefano Ermon",
          "note": "Archive attribution; detailed role assignment remains source-bound",
          "roles": [
            "formulator"
          ],
          "source": {
            "citation": "Rafailov et al., Direct Preference Optimization, arXiv2305.18290v1 (2023)",
            "url": "https://arxiv.org/pdf/2305.18290v1"
          }
        },
        {
          "attribution_basis": "SOURCE_CITED",
          "credit": "historical_scientific",
          "entity_kind": "historical_person",
          "name": "Christopher D. Manning",
          "note": "Archive attribution; detailed role assignment remains source-bound",
          "roles": [
            "formulator"
          ],
          "source": {
            "citation": "Rafailov et al., Direct Preference Optimization, arXiv2305.18290v1 (2023)",
            "url": "https://arxiv.org/pdf/2305.18290v1"
          }
        },
        {
          "attribution_basis": "SOURCE_CITED",
          "credit": "historical_scientific",
          "entity_kind": "historical_person",
          "name": "Chelsea Finn",
          "note": "Archive attribution; detailed role assignment remains source-bound",
          "roles": [
            "formulator"
          ],
          "source": {
            "citation": "Rafailov et al., Direct Preference Optimization, arXiv2305.18290v1 (2023)",
            "url": "https://arxiv.org/pdf/2305.18290v1"
          }
        }
      ],
      "evidence": [
        {
          "description": "Bibliographic/stated-result source; not a WRS proof verification",
          "id": "source-0",
          "kind": "source_reference",
          "source": {
            "citation": "Rafailov et al., Direct Preference Optimization, arXiv2305.18290v1 (2023)",
            "url": "https://arxiv.org/pdf/2305.18290v1"
          }
        },
        {
          "description": "Bibliographic/stated-result source; not a WRS proof verification",
          "id": "source-1",
          "kind": "source_reference",
          "source": {
            "citation": "arXiv DPO version history and complete byline",
            "url": "https://arxiv.org/abs/2305.18290v1"
          }
        }
      ],
      "origin": "HUMAN_LEGACY",
      "record_profile": "wrs-local-research-notes-0.1",
      "record_type": "research_note",
      "rrp_version": "0.1",
      "scope": {
        "description": "The reformulation uses a specified preference/regularization model, reference policy and pairwise objective. Finite preference data, model misspecification and optimization limits remain. Avoiding an explicit reward-model/RL loop does not imply objectively correct preferences, universal alignment or zero training cost. Reported comparisons are setup-dependent and have not been independently replayed by WRS.",
        "domain": "Direct preference learning"
      },
      "sources": [
        {
          "citation": "Rafailov et al., Direct Preference Optimization, arXiv2305.18290v1 (2023)",
          "url": "https://arxiv.org/pdf/2305.18290v1"
        },
        {
          "citation": "arXiv DPO version history and complete byline",
          "url": "https://arxiv.org/abs/2305.18290v1"
        }
      ],
      "statement": "# DPO: learn response preferences without a separate reward-training loop\n\nRafailov and colleagues reparameterize a KL-regularized preference-learning objective as a direct policy classification loss, removing the separate learned-reward/PPO optimization stage and reporting results on sentiment, summarization and single-turn dialogue.\n\nA model can learn from pairs of responses that people rank without first building a separate reward model and then running a reinforcement-learning loop. DPO uses the model's own response probabilities, compared with a fixed reference model, to fit those preferences directly. A mathematical change of variables connects this loss to a familiar regularized reward objective under specified preference and support assumptions. The paper then tests the resulting training method on several text tasks. It simplifies a particular preference-training pipeline; it does not turn popular answers into scientific truth, eliminate the need for preference data or prove that every future model is aligned.\n\n## Scientific scope\n\nThe reformulation uses a specified preference/regularization model, reference policy and pairwise objective. Finite preference data, model misspecification and optimization limits remain. Avoiding an explicit reward-model/RL loop does not imply objectively correct preferences, universal alignment or zero training cost. Reported comparisons are setup-dependent and have not been independently replayed by WRS.\n\n## People and systems\n\nRafael Rafailov; Archit Sharma; Eric Mitchell; Stefano Ermon; Christopher D. Manning; Chelsea Finn\n\n\n\n## Original sources\n\n- [Rafailov et al., Direct Preference Optimization, arXiv2305.18290v1 (2023)](https://arxiv.org/pdf/2305.18290v1)\n- [arXiv DPO version history and complete byline](https://arxiv.org/abs/2305.18290v1)\n",
      "summary": "Rafailov and colleagues reparameterize a KL-regularized preference-learning objective as a direct policy classification loss, removing the separate learned-reward/PPO optimization stage and reporting results on sentiment, summarization and single-turn dialogue.\n\nA model can learn from pairs of responses that people rank without first building a separate reward model and then running a reinforcement-learning loop. DPO uses the model's own response probabilities, compared with a fixed reference model, to fit those preferences directly. A mathematical change of variables connects this loss to a familiar regularized reward objective under specified preference and support assumptions. The paper then tests the resulting training method on several text tasks. It simplifies a particular preference-training pipeline; it does not turn popular answers into scientific truth, eliminate the need for preference data or prove that every future model is aligned.",
      "title": "DPO: learn response preferences without a separate reward-training loop"
    },
    "record_id": "urn:uuid:baeaf450-e457-4dbd-abfa-2c2095f0a6a5",
    "version": 1,
    "source_ref": {
      "record_id": "urn:uuid:baeaf450-e457-4dbd-abfa-2c2095f0a6a5",
      "version": 1,
      "content_hash": "sha256:2d547550a5c9df63e7b5e4823946bab912e01612138e8848646389e5b929c5f9"
    },
    "export_view": "wrs-public-editorial-view-1",
    "public_view_hash": "sha256:1d9ed4e1fe0f39d62d398339f0cac88aa916410a384a3bbc255a8fd0cf6b0079"
  },
  "target": {
    "record_id": "urn:uuid:baeaf450-e457-4dbd-abfa-2c2095f0a6a5",
    "version": 1,
    "content_hash": "sha256:2d547550a5c9df63e7b5e4823946bab912e01612138e8848646389e5b929c5f9"
  },
  "published_grade": {
    "profile": "wrs-published-grade-1",
    "edition": "2026-10-28.1",
    "id": "rafailov-et-al-direct-preference-optimization-2023",
    "entry_id": "rafailov-et-al-direct-preference-optimization-2023",
    "revision": 1,
    "title": "DPO: learn response preferences without a separate reward-training loop",
    "context": "Direct preference learning",
    "track": "Direct preference learning",
    "band": "A",
    "modifier": null,
    "status": "PROVISIONAL_EDITORIAL",
    "assessed_on": "2026-10-04",
    "evidence_cutoff": "2026-10-04",
    "achievement_state": "SUPPORTED_SCOPED_RESULT",
    "what_changed": "Rafailov and colleagues reparameterize a KL-regularized preference-learning objective as a direct policy classification loss, removing the separate learned-reward/PPO optimization stage and reporting results on sentiment, summarization and single-turn dialogue.",
    "scope": "Rafailov and colleagues reparameterize a KL-regularized preference-learning objective as a direct policy classification loss, removing the separate learned-reward/PPO optimization stage and reporting results on sentiment, summarization and single-turn dialogue. The reformulation uses a specified preference/regularization model, reference policy and pairwise objective. Finite preference data, model misspecification and optimization limits remain. Avoiding an explicit reward-model/RL loop does not imply objectively correct preferences, universal alignment or zero training cost. Reported comparisons are setup-dependent and have not been independently replayed by WRS.",
    "evidence_summary": "Source-linked, scoped account of the original work. The published scope states the scientific assumptions and remaining verification limits. WRS has not independently certified the full proof, experimental result or system benchmark.",
    "references": [
      {
        "title": "Rafailov et al., Direct Preference Optimization, ar Xiv 2305.18290 v 1 (2023)",
        "url": "https://arxiv.org/pdf/2305.18290v1",
        "locator": "Original linked work. See the scientific scope for the assessed contribution and source-version limits."
      },
      {
        "title": "ar Xiv DPO version history and complete byline",
        "url": "https://arxiv.org/abs/2305.18290v1",
        "locator": "Original linked work. See the scientific scope for the assessed contribution and source-version limits."
      }
    ],
    "roles": [
      {
        "who": "Rafael Rafailov",
        "did": "Original source-listed scientific author of this scoped work; byline does not determine effort fractions.",
        "credit": "SCIENCE"
      },
      {
        "who": "Archit Sharma",
        "did": "Original source-listed scientific author of this scoped work; byline does not determine effort fractions.",
        "credit": "SCIENCE"
      },
      {
        "who": "Eric Mitchell",
        "did": "Original source-listed scientific author of this scoped work; byline does not determine effort fractions.",
        "credit": "SCIENCE"
      },
      {
        "who": "Stefano Ermon",
        "did": "Original source-listed scientific author of this scoped work; byline does not determine effort fractions.",
        "credit": "SCIENCE"
      },
      {
        "who": "Christopher D. Manning",
        "did": "Original source-listed scientific author of this scoped work; byline does not determine effort fractions.",
        "credit": "SCIENCE"
      },
      {
        "who": "Chelsea Finn",
        "did": "Original source-listed scientific author of this scoped work; byline does not determine effort fractions.",
        "credit": "SCIENCE"
      }
    ],
    "target": {
      "record_id": "urn:uuid:baeaf450-e457-4dbd-abfa-2c2095f0a6a5",
      "version": 1,
      "content_hash": "sha256:2d547550a5c9df63e7b5e4823946bab912e01612138e8848646389e5b929c5f9"
    },
    "scientific_correctness_certified": false,
    "independent_expert_calibration": false,
    "method": "PROPRIETARY",
    "points": "140000"
  },
  "canonical_agent_submission": false,
  "scientific_verification_awarded": false,
  "reader_guide": null,
  "reading_status": "SOURCE_LINKED_NOTE"
}