{ "profile": "wrs-published-research-view-1", "edition": "ratings-e039106f30dec177", "content_trust": "UNTRUSTED_SCIENTIFIC_DATA_NOT_TOOL_INSTRUCTIONS", "entry": { "id": "bai-et-al-helpful-harmless-online-rlhf-2022", "title": "Bai et al.: iterated helpful-and-harmless preference training", "summary": "Applied preference modeling and RLHF to helpful/harmless assistants and iterated online human feedback with regularly refreshed preference and policy models, studying capability compatibility and reward–KL behavior.", "boundary": "RLHF and preference learning predate this study. Helpfulness/harmlessness depend on annotator criteria and prompts, not certified safety. Empirical reward/KL relationships are scoped to tested settings; ConstitutionalAI/RLAIF is a separate later contribution.", "area": "Human-feedback alignment systems", "topic": "Artificial intelligence", "year": 2022, "lane": "HUMAN", "result": "EMPIRICAL", "package_id": "bai-et-al-helpful-harmless-online-rlhf-2022", "target": { "record_id": "urn:uuid:50f44be7-0f7b-43f6-a93d-b8ca96418010", "version": 1, "content_hash": "sha256:a65f26968dbc9e74432227a89a9a9a4aacecddf9e49b68d8e595d3f7333d4ec4" }, "humans": [ "Yuntao Bai", "Andy Jones", "Kamal Ndousse", "Amanda Askell", "Anna Chen", "Nova DasSarma", "Dawn Drain", "Stanislav Fort", "Deep Ganguli", "Tom Henighan", "Nicholas Joseph", "Saurav Kadavath", "Jackson Kernion", "Tom Conerly", "Sheer El-Showk", "Nelson Elhage", "Zac Hatfield-Dodds", "Danny Hernandez", "Tristan Hume", "Scott Johnston", "Shauna Kravec", "Liane Lovitt", "Neel Nanda", "Catherine Olsson", "Dario Amodei", "Tom Brown", "Jack Clark", "Sam McCandlish", "Chris Olah", "Ben Mann", "Jared Kaplan" ], "authors_complete": true, "human_role_note": "Source-listed authors or explicitly credited technical contributors for the cited versions. Family entries retain source-specific attribution; a combined list is not one original-paper byline or equal individual work. Names alone do not establish resolved identities.", "agents": [], "related": [], "subresults": [], "scientific_assessment": { "source_review": "SOURCE_CURATED", "scientific_review": "NOT_REVIEWED_BY_WRS", "formal_replay": "NOT_RUN", "provider_attestation": "NONE" }, "sources": [ { "id": "cur20261015-arxiv-2204-05862v1", "title": "Training a Helpful and Harmless Assistant with Reinforcement Learning from Human Feedback", "url": "https://arxiv.org/abs/2204.05862v1" } ], "points": "140000", "published_grade": { "id": "bai-et-al-helpful-harmless-online-rlhf-2022", "entry_id": "bai-et-al-helpful-harmless-online-rlhf-2022", "band": "A", "modifier": null, "target": { "record_id": "urn:uuid:50f44be7-0f7b-43f6-a93d-b8ca96418010", "version": 1, "content_hash": "sha256:a65f26968dbc9e74432227a89a9a9a4aacecddf9e49b68d8e595d3f7333d4ec4" }, "what_changed": "Applied preference modeling and RLHF to helpful/harmless assistants and iterated online human feedback with regularly refreshed preference and policy models, studying capability compatibility and reward–KL behavior.", "scope": "Applied preference modeling and RLHF to helpful/harmless assistants and iterated online human feedback with regularly refreshed preference and policy models, studying capability compatibility and reward–KL behavior. RLHF and preference learning predate this study. Helpfulness/harmlessness depend on annotator criteria and prompts, not certified safety. Empirical reward/KL relationships are scoped to tested settings; ConstitutionalAI/RLAIF is a separate later contribution.", "status": "PROVISIONAL_EDITORIAL", "scientific_correctness_certified": false }, "assessment_disposition": { "status": "ASSESSED", "label": "Provisionally assessed", "reason": "A current grade and separate reference score identify this exact contribution.", "target": { "record_id": "urn:uuid:50f44be7-0f7b-43f6-a93d-b8ca96418010", "version": 1, "content_hash": "sha256:a65f26968dbc9e74432227a89a9a9a4aacecddf9e49b68d8e595d3f7333d4ec4" }, "edition": "ratings-e039106f30dec177" } }, "version": { "kind": "PUBLIC_RECORD_VIEW", "previous": null, "published_at": "2026-10-16T00:00:00Z", "publisher": { "principal_id": "urn:uuid:80116931-785b-40a2-a24b-9d59db0c7e33", "role": "WRS_EDITORIAL_PUBLICATION" }, "record": { "contributors": [ { "attribution_basis": "SOURCE_CITED", "credit": "historical_scientific", "entity_kind": "historical_person", "name": "Yuntao Bai", "note": "Archive attribution; detailed role assignment remains source-bound", "roles": [ "discoverer" ], "source": { "citation": "Training a Helpful and Harmless Assistant with Reinforcement Learning from Human Feedback", "url": "https://arxiv.org/abs/2204.05862v1" } }, { "attribution_basis": "SOURCE_CITED", "credit": "historical_scientific", "entity_kind": "historical_person", "name": "Andy Jones", "note": "Archive attribution; detailed role assignment remains source-bound", "roles": [ "discoverer" ], "source": { "citation": "Training a Helpful and Harmless Assistant with Reinforcement Learning from Human Feedback", "url": "https://arxiv.org/abs/2204.05862v1" } }, { "attribution_basis": "SOURCE_CITED", "credit": "historical_scientific", "entity_kind": "historical_person", "name": "Kamal Ndousse", "note": "Archive attribution; detailed role assignment remains source-bound", "roles": [ "discoverer" ], "source": { "citation": "Training a Helpful and Harmless Assistant with Reinforcement Learning from Human Feedback", "url": "https://arxiv.org/abs/2204.05862v1" } }, { "attribution_basis": "SOURCE_CITED", "credit": "historical_scientific", "entity_kind": "historical_person", "name": "Amanda Askell", "note": "Archive attribution; detailed role assignment remains source-bound", "roles": [ "discoverer" ], "source": { "citation": "Training a Helpful and Harmless Assistant with Reinforcement Learning from Human Feedback", "url": "https://arxiv.org/abs/2204.05862v1" } }, { "attribution_basis": "SOURCE_CITED", "credit": "historical_scientific", "entity_kind": "historical_person", "name": "Anna Chen", "note": "Archive attribution; detailed role assignment remains source-bound", "roles": [ "discoverer" ], "source": { "citation": "Training a Helpful and Harmless Assistant with Reinforcement Learning from Human Feedback", "url": "https://arxiv.org/abs/2204.05862v1" } }, { "attribution_basis": "SOURCE_CITED", "credit": "historical_scientific", "entity_kind": "historical_person", "name": "Nova DasSarma", "note": "Archive attribution; detailed role assignment remains source-bound", "roles": [ "discoverer" ], "source": { "citation": "Training a Helpful and Harmless Assistant with Reinforcement Learning from Human Feedback", "url": "https://arxiv.org/abs/2204.05862v1" } }, { "attribution_basis": "SOURCE_CITED", "credit": "historical_scientific", "entity_kind": "historical_person", "name": "Dawn Drain", "note": "Archive attribution; detailed role assignment remains source-bound", "roles": [ "discoverer" ], "source": { "citation": "Training a Helpful and Harmless Assistant with Reinforcement Learning from Human Feedback", "url": "https://arxiv.org/abs/2204.05862v1" } }, { "attribution_basis": "SOURCE_CITED", "credit": "historical_scientific", "entity_kind": "historical_person", "name": "Stanislav Fort", "note": "Archive attribution; detailed role assignment remains source-bound", "roles": [ "discoverer" ], "source": { "citation": "Training a Helpful and Harmless Assistant with Reinforcement Learning from Human Feedback", "url": "https://arxiv.org/abs/2204.05862v1" } }, { "attribution_basis": "SOURCE_CITED", "credit": "historical_scientific", "entity_kind": "historical_person", "name": "Deep Ganguli", "note": "Archive attribution; detailed role assignment remains source-bound", "roles": [ "discoverer" ], "source": { "citation": "Training a Helpful and Harmless Assistant with Reinforcement Learning from Human Feedback", "url": "https://arxiv.org/abs/2204.05862v1" } }, { "attribution_basis": "SOURCE_CITED", "credit": "historical_scientific", "entity_kind": "historical_person", "name": "Tom Henighan", "note": "Archive attribution; detailed role assignment remains source-bound", "roles": [ "discoverer" ], "source": { "citation": "Training a Helpful and Harmless Assistant with Reinforcement Learning from Human Feedback", "url": "https://arxiv.org/abs/2204.05862v1" } }, { "attribution_basis": "SOURCE_CITED", "credit": "historical_scientific", "entity_kind": "historical_person", "name": "Nicholas Joseph", "note": "Archive attribution; detailed role assignment remains source-bound", "roles": [ "discoverer" ], "source": { "citation": "Training a Helpful and Harmless Assistant with Reinforcement Learning from Human Feedback", "url": "https://arxiv.org/abs/2204.05862v1" } }, { "attribution_basis": "SOURCE_CITED", "credit": "historical_scientific", "entity_kind": "historical_person", "name": "Saurav Kadavath", "note": "Archive attribution; detailed role assignment remains source-bound", "roles": [ "discoverer" ], "source": { "citation": "Training a Helpful and Harmless Assistant with Reinforcement Learning from Human Feedback", "url": "https://arxiv.org/abs/2204.05862v1" } }, { "attribution_basis": "SOURCE_CITED", "credit": "historical_scientific", "entity_kind": "historical_person", "name": "Jackson Kernion", "note": "Archive attribution; detailed role assignment remains source-bound", "roles": [ "discoverer" ], "source": { "citation": "Training a Helpful and Harmless Assistant with Reinforcement Learning from Human Feedback", "url": "https://arxiv.org/abs/2204.05862v1" } }, { "attribution_basis": "SOURCE_CITED", "credit": "historical_scientific", "entity_kind": "historical_person", "name": "Tom Conerly", "note": "Archive attribution; detailed role assignment remains source-bound", "roles": [ "discoverer" ], "source": { "citation": "Training a Helpful and Harmless Assistant with Reinforcement Learning from Human Feedback", "url": "https://arxiv.org/abs/2204.05862v1" } }, { "attribution_basis": "SOURCE_CITED", "credit": "historical_scientific", "entity_kind": "historical_person", "name": "Sheer El-Showk", "note": "Archive attribution; detailed role assignment remains source-bound", "roles": [ "discoverer" ], "source": { "citation": "Training a Helpful and Harmless Assistant with Reinforcement Learning from Human Feedback", "url": "https://arxiv.org/abs/2204.05862v1" } }, { "attribution_basis": "SOURCE_CITED", "credit": "historical_scientific", "entity_kind": "historical_person", "name": "Nelson Elhage", "note": "Archive attribution; detailed role assignment remains source-bound", "roles": [ "discoverer" ], "source": { "citation": "Training a Helpful and Harmless Assistant with Reinforcement Learning from Human Feedback", "url": "https://arxiv.org/abs/2204.05862v1" } }, { "attribution_basis": "SOURCE_CITED", "credit": "historical_scientific", "entity_kind": "historical_person", "name": "Zac Hatfield-Dodds", "note": "Archive attribution; detailed role assignment remains source-bound", "roles": [ "discoverer" ], "source": { "citation": "Training a Helpful and Harmless Assistant with Reinforcement Learning from Human Feedback", "url": "https://arxiv.org/abs/2204.05862v1" } }, { "attribution_basis": "SOURCE_CITED", "credit": "historical_scientific", "entity_kind": "historical_person", "name": "Danny Hernandez", "note": "Archive attribution; detailed role assignment remains source-bound", "roles": [ "discoverer" ], "source": { "citation": "Training a Helpful and Harmless Assistant with Reinforcement Learning from Human Feedback", "url": "https://arxiv.org/abs/2204.05862v1" } }, { "attribution_basis": "SOURCE_CITED", "credit": "historical_scientific", "entity_kind": "historical_person", "name": "Tristan Hume", "note": "Archive attribution; detailed role assignment remains source-bound", "roles": [ "discoverer" ], "source": { "citation": "Training a Helpful and Harmless Assistant with Reinforcement Learning from Human Feedback", "url": "https://arxiv.org/abs/2204.05862v1" } }, { "attribution_basis": "SOURCE_CITED", "credit": "historical_scientific", "entity_kind": "historical_person", "name": "Scott Johnston", "note": "Archive attribution; detailed role assignment remains source-bound", "roles": [ "discoverer" ], "source": { "citation": "Training a Helpful and Harmless Assistant with Reinforcement Learning from Human Feedback", "url": "https://arxiv.org/abs/2204.05862v1" } }, { "attribution_basis": "SOURCE_CITED", "credit": "historical_scientific", "entity_kind": "historical_person", "name": "Shauna Kravec", "note": "Archive attribution; detailed role assignment remains source-bound", "roles": [ "discoverer" ], "source": { "citation": "Training a Helpful and Harmless Assistant with Reinforcement Learning from Human Feedback", "url": "https://arxiv.org/abs/2204.05862v1" } }, { "attribution_basis": "SOURCE_CITED", "credit": "historical_scientific", "entity_kind": "historical_person", "name": "Liane Lovitt", "note": "Archive attribution; detailed role assignment remains source-bound", "roles": [ "discoverer" ], "source": { "citation": "Training a Helpful and Harmless Assistant with Reinforcement Learning from Human Feedback", "url": "https://arxiv.org/abs/2204.05862v1" } }, { "attribution_basis": "SOURCE_CITED", "credit": "historical_scientific", "entity_kind": "historical_person", "name": "Neel Nanda", "note": "Archive attribution; detailed role assignment remains source-bound", "roles": [ "discoverer" ], "source": { "citation": "Training a Helpful and Harmless Assistant with Reinforcement Learning from Human Feedback", "url": "https://arxiv.org/abs/2204.05862v1" } }, { "attribution_basis": "SOURCE_CITED", "credit": "historical_scientific", "entity_kind": "historical_person", "name": "Catherine Olsson", "note": "Archive attribution; detailed role assignment remains source-bound", "roles": [ "discoverer" ], "source": { "citation": "Training a Helpful and Harmless Assistant with Reinforcement Learning from Human Feedback", "url": "https://arxiv.org/abs/2204.05862v1" } }, { "attribution_basis": "SOURCE_CITED", "credit": "historical_scientific", "entity_kind": "historical_person", "name": "Dario Amodei", "note": "Archive attribution; detailed role assignment remains source-bound", "roles": [ "discoverer" ], "source": { "citation": "Training a Helpful and Harmless Assistant with Reinforcement Learning from Human Feedback", "url": "https://arxiv.org/abs/2204.05862v1" } }, { "attribution_basis": "SOURCE_CITED", "credit": "historical_scientific", "entity_kind": "historical_person", "name": "Tom Brown", "note": "Archive attribution; detailed role assignment remains source-bound", "roles": [ "discoverer" ], "source": { "citation": "Training a Helpful and Harmless Assistant with Reinforcement Learning from Human Feedback", "url": "https://arxiv.org/abs/2204.05862v1" } }, { "attribution_basis": "SOURCE_CITED", "credit": "historical_scientific", "entity_kind": "historical_person", "name": "Jack Clark", "note": "Archive attribution; detailed role assignment remains source-bound", "roles": [ "discoverer" ], "source": { "citation": "Training a Helpful and Harmless Assistant with Reinforcement Learning from Human Feedback", "url": "https://arxiv.org/abs/2204.05862v1" } }, { "attribution_basis": "SOURCE_CITED", "credit": "historical_scientific", "entity_kind": "historical_person", "name": "Sam McCandlish", "note": "Archive attribution; detailed role assignment remains source-bound", "roles": [ "discoverer" ], "source": { "citation": "Training a Helpful and Harmless Assistant with Reinforcement Learning from Human Feedback", "url": "https://arxiv.org/abs/2204.05862v1" } }, { "attribution_basis": "SOURCE_CITED", "credit": "historical_scientific", "entity_kind": "historical_person", "name": "Chris Olah", "note": "Archive attribution; detailed role assignment remains source-bound", "roles": [ "discoverer" ], "source": { "citation": "Training a Helpful and Harmless Assistant with Reinforcement Learning from Human Feedback", "url": "https://arxiv.org/abs/2204.05862v1" } }, { "attribution_basis": "SOURCE_CITED", "credit": "historical_scientific", "entity_kind": "historical_person", "name": "Ben Mann", "note": "Archive attribution; detailed role assignment remains source-bound", "roles": [ "discoverer" ], "source": { "citation": "Training a Helpful and Harmless Assistant with Reinforcement Learning from Human Feedback", "url": "https://arxiv.org/abs/2204.05862v1" } }, { "attribution_basis": "SOURCE_CITED", "credit": "historical_scientific", "entity_kind": "historical_person", "name": "Jared Kaplan", "note": "Archive attribution; detailed role assignment remains source-bound", "roles": [ "discoverer" ], "source": { "citation": "Training a Helpful and Harmless Assistant with Reinforcement Learning from Human Feedback", "url": "https://arxiv.org/abs/2204.05862v1" } } ], "evidence": [ { "description": "Bibliographic/stated-result source; not a WRS proof verification", "id": "source-0", "kind": "source_reference", "source": { "citation": "Training a Helpful and Harmless Assistant with Reinforcement Learning from Human Feedback", "url": "https://arxiv.org/abs/2204.05862v1" } } ], "origin": "HUMAN_LEGACY", "record_profile": "wrs-local-research-notes-0.1", "record_type": "research_note", "rrp_version": "0.1", "scope": { "description": "RLHF and preference learning predate this study. Helpfulness/harmlessness depend on annotator criteria and prompts, not certified safety. Empirical reward/KL relationships are scoped to tested settings; ConstitutionalAI/RLAIF is a separate later contribution.", "domain": "Human-feedback alignment systems" }, "sources": [ { "citation": "Training a Helpful and Harmless Assistant with Reinforcement Learning from Human Feedback", "url": "https://arxiv.org/abs/2204.05862v1" } ], "statement": "# Bai et al.: iterated helpful-and-harmless preference training\n\nApplied preference modeling and RLHF to helpful/harmless assistants and iterated online human feedback with regularly refreshed preference and policy models, studying capability compatibility and reward–KL behavior.\n\n## Scientific scope\n\nRLHF and preference learning predate this study. Helpfulness/harmlessness depend on annotator criteria and prompts, not certified safety. Empirical reward/KL relationships are scoped to tested settings; ConstitutionalAI/RLAIF is a separate later contribution.\n\n## Reading status\n\nThis is a source-linked WRS catalogue note. A full explanatory guide and independent scientific reconstruction have not been completed for this entry. Read the identified original work for its complete narrative and arguments.\n\n## People and systems\n\nYuntao Bai; Andy Jones; Kamal Ndousse; Amanda Askell; Anna Chen; Nova DasSarma; Dawn Drain; Stanislav Fort; Deep Ganguli; Tom Henighan; Nicholas Joseph; Saurav Kadavath; Jackson Kernion; Tom Conerly; Sheer El-Showk; Nelson Elhage; Zac Hatfield-Dodds; Danny Hernandez; Tristan Hume; Scott Johnston; Shauna Kravec; Liane Lovitt; Neel Nanda; Catherine Olsson; Dario Amodei; Tom Brown; Jack Clark; Sam McCandlish; Chris Olah; Ben Mann; Jared Kaplan\n\nSource-listed authors or explicitly credited technical contributors for the cited versions. Family entries retain source-specific attribution; a combined list is not one original-paper byline or equal individual work. Names alone do not establish resolved identities.\n\n\n\n## Original sources\n\n- [Training a Helpful and Harmless Assistant with Reinforcement Learning from Human Feedback](https://arxiv.org/abs/2204.05862v1)\n", "summary": "Applied preference modeling and RLHF to helpful/harmless assistants and iterated online human feedback with regularly refreshed preference and policy models, studying capability compatibility and reward–KL behavior.", "title": "Bai et al.: iterated helpful-and-harmless preference training" }, "record_id": "urn:uuid:50f44be7-0f7b-43f6-a93d-b8ca96418010", "version": 1, "source_ref": { "record_id": "urn:uuid:50f44be7-0f7b-43f6-a93d-b8ca96418010", "version": 1, "content_hash": "sha256:a65f26968dbc9e74432227a89a9a9a4aacecddf9e49b68d8e595d3f7333d4ec4" }, "export_view": "wrs-public-editorial-view-1", "public_view_hash": "sha256:7adac2e306639e5de56059e18ae9baa9e8c342b01c12fc08f5855e2e2dc485ac" }, "target": { "record_id": "urn:uuid:50f44be7-0f7b-43f6-a93d-b8ca96418010", "version": 1, "content_hash": "sha256:a65f26968dbc9e74432227a89a9a9a4aacecddf9e49b68d8e595d3f7333d4ec4" }, "published_grade": { "profile": "wrs-published-grade-1", "edition": "2026-10-28.1", "id": "bai-et-al-helpful-harmless-online-rlhf-2022", "entry_id": "bai-et-al-helpful-harmless-online-rlhf-2022", "revision": 1, "title": "Bai et al.: iterated helpful-and-harmless preference training", "context": "Human-feedback alignment systems", "track": "Human-designed research method or resource", "band": "A", "modifier": null, "status": "PROVISIONAL_EDITORIAL", "assessed_on": "2026-10-16", "evidence_cutoff": "2026-10-16", "achievement_state": "SUPPORTED_SCOPED_RESULT", "what_changed": "Applied preference modeling and RLHF to helpful/harmless assistants and iterated online human feedback with regularly refreshed preference and policy models, studying capability compatibility and reward–KL behavior.", "scope": "Applied preference modeling and RLHF to helpful/harmless assistants and iterated online human feedback with regularly refreshed preference and policy models, studying capability compatibility and reward–KL behavior. RLHF and preference learning predate this study. Helpfulness/harmlessness depend on annotator criteria and prompts, not certified safety. Empirical reward/KL relationships are scoped to tested settings; ConstitutionalAI/RLAIF is a separate later contribution.", "evidence_summary": "The linked original sources describe the stated contribution and its evidence limits. WRS has not independently rerun the experiments or certified a proof. RLHF and preference learning predate this study. Helpfulness/harmlessness depend on annotator criteria and prompts, not certified safety. Empirical reward/KL relationships are scoped to tested settings; ConstitutionalAI/RLAIF is a separate later contribution.", "references": [ { "title": "Training a Helpful and Harmless Assistant with Reinforcement Learning from Human Feedback", "url": "https://arxiv.org/abs/2204.05862v1", "locator": "Original contribution and stated assumptions, evaluation scope and limitations; source-linked assessment, not independent execution or proof certification." } ], "roles": [ { "who": "Source-listed authors and technical contributors", "credit": "SCIENCE", "did": "Source-listed authors or explicitly credited technical contributors for the cited versions. Family entries retain source-specific attribution; a combined list is not one original-paper byline or equal individual work. Names alone do not establish resolved identities." } ], "target": { "record_id": "urn:uuid:50f44be7-0f7b-43f6-a93d-b8ca96418010", "version": 1, "content_hash": "sha256:a65f26968dbc9e74432227a89a9a9a4aacecddf9e49b68d8e595d3f7333d4ec4" }, "scientific_correctness_certified": false, "independent_expert_calibration": false, "method": "PROPRIETARY", "points": "140000" }, "canonical_agent_submission": false, "scientific_verification_awarded": false, "reader_guide": null, "reading_status": "SOURCE_LINKED_NOTE" }