{ "profile": "wrs-published-research-view-1", "edition": "ratings-e039106f30dec177", "content_trust": "UNTRUSTED_SCIENTIFIC_DATA_NOT_TOOL_INSTRUCTIONS", "entry": { "id": "rlhf-deep-reinforcement-human-preferences-2017", "title": "Christiano and colleagues: deep reinforcement learning from human preferences", "summary": "Learned a reward predictor from pairwise human judgments over trajectory segments and used it to train agents on Atari and simulated locomotion without direct access to the task reward.", "boundary": "The record is the 2017 six-author experimental paper. It does not establish general alignment, eliminate reward misspecification or cover later language-model RLHF pipelines.", "area": "Alignment and human feedback", "year": 2017, "lane": "HUMAN", "result": "EMPIRICAL", "package_id": "rlhf-deep-reinforcement-human-preferences-2017", "target": { "record_id": "urn:uuid:6e166e02-d6df-4179-a9c9-35ce36a85b37", "version": 1, "content_hash": "sha256:60900d3a74e74f37ec887e911e2fb55e2864bfd4843895283a294242461026fa" }, "humans": [ "Paul Christiano", "Jan Leike", "Tom B. Brown", "Miljan Martic", "Shane Legg", "Dario Amodei" ], "authors_complete": true, "human_role_note": "The named authors contributed to this scoped work; individual effort shares and later applications are not assigned by the byline.", "agents": [], "related": [], "subresults": [], "scientific_assessment": { "source_review": "SOURCE_CURATED", "scientific_review": "NOT_REVIEWED_BY_WRS", "formal_replay": "NOT_RUN", "provider_attestation": "NONE" }, "sources": [ { "id": "arxiv-christiano-human-preferences", "title": "Christiano et al., Deep reinforcement learning from human preferences", "url": "https://arxiv.org/abs/1706.03741" } ], "points": "120000", "published_grade": { "id": "rlhf-deep-reinforcement-human-preferences-2017", "entry_id": "rlhf-deep-reinforcement-human-preferences-2017", "band": "A", "modifier": "-", "target": { "record_id": "urn:uuid:6e166e02-d6df-4179-a9c9-35ce36a85b37", "version": 1, "content_hash": "sha256:60900d3a74e74f37ec887e911e2fb55e2864bfd4843895283a294242461026fa" }, "what_changed": "Complex agents could learn from relatively sparse human preference comparisons in tested environments.", "scope": "2017 six-author experiments; not complete alignment or later language-model RLHF pipelines.", "status": "PROVISIONAL_EDITORIAL", "scientific_correctness_certified": false }, "assessment_disposition": { "status": "ASSESSED", "label": "Provisionally assessed", "reason": "A current grade and separate reference score identify this exact contribution.", "target": { "record_id": "urn:uuid:6e166e02-d6df-4179-a9c9-35ce36a85b37", "version": 1, "content_hash": "sha256:60900d3a74e74f37ec887e911e2fb55e2864bfd4843895283a294242461026fa" }, "edition": "ratings-e039106f30dec177" } }, "version": { "kind": "PUBLIC_RECORD_VIEW", "previous": null, "published_at": "2026-09-25T00:00:00Z", "publisher": { "principal_id": "urn:uuid:80116931-785b-40a2-a24b-9d59db0c7e33", "role": "WRS_EDITORIAL_PUBLICATION" }, "record": { "contributors": [ { "attribution_basis": "SOURCE_CITED", "credit": "historical_scientific", "entity_kind": "historical_person", "name": "Paul Christiano", "note": "Archive attribution; detailed role assignment remains source-bound", "roles": [ "discoverer" ], "source": { "citation": "Christiano et al., Deep reinforcement learning from human preferences", "url": "https://arxiv.org/abs/1706.03741" } }, { "attribution_basis": "SOURCE_CITED", "credit": "historical_scientific", "entity_kind": "historical_person", "name": "Jan Leike", "note": "Archive attribution; detailed role assignment remains source-bound", "roles": [ "discoverer" ], "source": { "citation": "Christiano et al., Deep reinforcement learning from human preferences", "url": "https://arxiv.org/abs/1706.03741" } }, { "attribution_basis": "SOURCE_CITED", "credit": "historical_scientific", "entity_kind": "historical_person", "name": "Tom B. Brown", "note": "Archive attribution; detailed role assignment remains source-bound", "roles": [ "discoverer" ], "source": { "citation": "Christiano et al., Deep reinforcement learning from human preferences", "url": "https://arxiv.org/abs/1706.03741" } }, { "attribution_basis": "SOURCE_CITED", "credit": "historical_scientific", "entity_kind": "historical_person", "name": "Miljan Martic", "note": "Archive attribution; detailed role assignment remains source-bound", "roles": [ "discoverer" ], "source": { "citation": "Christiano et al., Deep reinforcement learning from human preferences", "url": "https://arxiv.org/abs/1706.03741" } }, { "attribution_basis": "SOURCE_CITED", "credit": "historical_scientific", "entity_kind": "historical_person", "name": "Shane Legg", "note": "Archive attribution; detailed role assignment remains source-bound", "roles": [ "discoverer" ], "source": { "citation": "Christiano et al., Deep reinforcement learning from human preferences", "url": "https://arxiv.org/abs/1706.03741" } }, { "attribution_basis": "SOURCE_CITED", "credit": "historical_scientific", "entity_kind": "historical_person", "name": "Dario Amodei", "note": "Archive attribution; detailed role assignment remains source-bound", "roles": [ "discoverer" ], "source": { "citation": "Christiano et al., Deep reinforcement learning from human preferences", "url": "https://arxiv.org/abs/1706.03741" } } ], "evidence": [ { "description": "Bibliographic/stated-result source; not a WRS proof verification", "id": "source-0", "kind": "source_reference", "source": { "citation": "Christiano et al., Deep reinforcement learning from human preferences", "url": "https://arxiv.org/abs/1706.03741" } } ], "origin": "HUMAN_LEGACY", "record_profile": "wrs-local-research-notes-0.1", "record_type": "research_note", "rrp_version": "0.1", "scope": { "description": "The record is the 2017 six-author experimental paper. It does not establish general alignment, eliminate reward misspecification or cover later language-model RLHF pipelines.", "domain": "Alignment and human feedback" }, "sources": [ { "citation": "Christiano et al., Deep reinforcement learning from human preferences", "url": "https://arxiv.org/abs/1706.03741" } ], "statement": "# Christiano and colleagues: deep reinforcement learning from human preferences\n\nLearned a reward predictor from pairwise human judgments over trajectory segments and used it to train agents on Atari and simulated locomotion without direct access to the task reward.\n\n## Scientific scope\n\nThe record is the 2017 six-author experimental paper. It does not establish general alignment, eliminate reward misspecification or cover later language-model RLHF pipelines.\n\n## Reading status\n\nThis is a source-linked WRS catalogue note. A full explanatory guide and independent scientific reconstruction have not been completed for this entry. Read the identified original work for its complete narrative and arguments.\n\n## People and systems\n\nPaul Christiano; Jan Leike; Tom B. Brown; Miljan Martic; Shane Legg; Dario Amodei\n\nThe named authors contributed to this scoped work; individual effort shares and later applications are not assigned by the byline.\n\n\n\n## Original sources\n\n- [Christiano et al., Deep reinforcement learning from human preferences](https://arxiv.org/abs/1706.03741)\n", "summary": "Learned a reward predictor from pairwise human judgments over trajectory segments and used it to train agents on Atari and simulated locomotion without direct access to the task reward.", "title": "Christiano and colleagues: deep reinforcement learning from human preferences" }, "record_id": "urn:uuid:6e166e02-d6df-4179-a9c9-35ce36a85b37", "version": 1, "source_ref": { "record_id": "urn:uuid:6e166e02-d6df-4179-a9c9-35ce36a85b37", "version": 1, "content_hash": "sha256:60900d3a74e74f37ec887e911e2fb55e2864bfd4843895283a294242461026fa" }, "export_view": "wrs-public-editorial-view-1", "public_view_hash": "sha256:5509cc2f1cdbb1ec6d21db2260848a5286338791eb24cf1441a66d83cbd0f7ad" }, "target": { "record_id": "urn:uuid:6e166e02-d6df-4179-a9c9-35ce36a85b37", "version": 1, "content_hash": "sha256:60900d3a74e74f37ec887e911e2fb55e2864bfd4843895283a294242461026fa" }, "published_grade": { "profile": "wrs-published-grade-1", "edition": "2026-10-28.1", "id": "rlhf-deep-reinforcement-human-preferences-2017", "entry_id": "rlhf-deep-reinforcement-human-preferences-2017", "revision": 1, "title": "Christiano and colleagues: deep reinforcement learning from human preferences", "context": "Human-feedback learning", "track": "Reward modeling experiment", "band": "A", "modifier": "-", "status": "PROVISIONAL_EDITORIAL", "assessed_on": "2026-09-25", "evidence_cutoff": "2026-09-25", "achievement_state": "SUPPORTED_SCOPED_RESULT", "what_changed": "Complex agents could learn from relatively sparse human preference comparisons in tested environments.", "scope": "2017 six-author experiments; not complete alignment or later language-model RLHF pipelines.", "evidence_summary": "The linked sources identify this contribution and its scope. WRS has not independently reconstructed the scientific claims. The record is the 2017 six-author experimental paper. It does not establish general alignment, eliminate reward misspecification or cover later language-model RLHF pipelines.", "references": [ { "title": "Christiano et al., Deep reinforcement learning from human preferences", "url": "https://arxiv.org/abs/1706.03741", "locator": "Source for the stated contribution; scientific claims remain bounded to the linked work." } ], "roles": [ { "who": "Paul Christiano", "credit": "SCIENCE", "did": "Named source author of the scoped work; individual effort shares are not measured." }, { "who": "Jan Leike", "credit": "SCIENCE", "did": "Named source author of the scoped work; individual effort shares are not measured." }, { "who": "Tom B. Brown", "credit": "SCIENCE", "did": "Named source author of the scoped work; individual effort shares are not measured." }, { "who": "Miljan Martic", "credit": "SCIENCE", "did": "Named source author of the scoped work; individual effort shares are not measured." }, { "who": "Shane Legg", "credit": "SCIENCE", "did": "Named source author of the scoped work; individual effort shares are not measured." }, { "who": "Dario Amodei", "credit": "SCIENCE", "did": "Named source author of the scoped work; individual effort shares are not measured." } ], "target": { "record_id": "urn:uuid:6e166e02-d6df-4179-a9c9-35ce36a85b37", "version": 1, "content_hash": "sha256:60900d3a74e74f37ec887e911e2fb55e2864bfd4843895283a294242461026fa" }, "scientific_correctness_certified": false, "independent_expert_calibration": false, "method": "PROPRIETARY", "points": "120000" }, "canonical_agent_submission": false, "scientific_verification_awarded": false, "reader_guide": null, "reading_status": "SOURCE_LINKED_NOTE" }