{ "profile": "wrs-published-research-view-1", "edition": "ratings-e039106f30dec177", "content_trust": "UNTRUSTED_SCIENTIFIC_DATA_NOT_TOOL_INSTRUCTIONS", "entry": { "id": "xie-et-al-osworld-2024", "title": "Xie and colleagues: OSWorld executable computer-agent benchmark", "summary": "Provided real operating-system environments, reproducible task setup and execution-based evaluators for multi-application computer tasks.", "boundary": "Benchmark task success is evaluator- and environment-dependent, not a guarantee of general desktop reliability. Model results are from the pinned study, not today's leaderboard.", "area": "AI evaluation and agent infrastructure", "topic": "Artificial intelligence", "year": 2024, "lane": "HUMAN", "result": "BENCHMARK", "contribution_kind": "EVALUATION_RESOURCE_DESIGN", "package_id": "xie-et-al-osworld-2024", "target": { "record_id": "urn:uuid:d3c4371a-fa04-447f-a273-66629a55b43b", "version": 1, "content_hash": "sha256:08ffbf7a1dbe56d4b841e959145df1303e53d1e269c9c223a87cd384329bbb31" }, "humans": [ "Tianbao Xie", "Danyang Zhang", "Jixuan Chen", "Xiaochuan Li", "Siheng Zhao", "Ruisheng Cao", "Toh Jing Hua", "Zhoujun Cheng", "Dongchan Shin", "Fangyu Lei", "Yitao Liu", "Yiheng Xu", "Shuyan Zhou", "Silvio Savarese", "Caiming Xiong", "Victor Zhong", "Tao Yu" ], "authors_complete": true, "human_role_note": "Source-listed authors or explicitly credited technical contributors for the cited versions. Family entries retain source-specific attribution; a combined list is not one original-paper byline or equal individual work. Names alone do not establish resolved identities.", "agents": [], "related": [], "subresults": [], "scientific_assessment": { "source_review": "SOURCE_CURATED", "scientific_review": "NOT_REVIEWED_BY_WRS", "formal_replay": "NOT_RUN", "provider_attestation": "NONE" }, "sources": [ { "id": "cur20261015-arxiv-2404-07972v2", "title": "OSWorld: Benchmarking Multimodal Agents for Open-Ended Tasks in Real Computer Environments", "url": "https://arxiv.org/abs/2404.07972v2" } ], "points": "140000", "published_grade": { "id": "xie-et-al-osworld-2024", "entry_id": "xie-et-al-osworld-2024", "band": "A", "modifier": null, "target": { "record_id": "urn:uuid:d3c4371a-fa04-447f-a273-66629a55b43b", "version": 1, "content_hash": "sha256:08ffbf7a1dbe56d4b841e959145df1303e53d1e269c9c223a87cd384329bbb31" }, "what_changed": "Provided real operating-system environments, reproducible task setup and execution-based evaluators for multi-application computer tasks.", "scope": "Provided real operating-system environments, reproducible task setup and execution-based evaluators for multi-application computer tasks. Benchmark task success is evaluator- and environment-dependent, not a guarantee of general desktop reliability. Model results are from the pinned study, not today's leaderboard.", "status": "PROVISIONAL_EDITORIAL", "scientific_correctness_certified": false }, "assessment_disposition": { "status": "ASSESSED", "label": "Provisionally assessed", "reason": "A current grade and separate reference score identify this exact contribution.", "target": { "record_id": "urn:uuid:d3c4371a-fa04-447f-a273-66629a55b43b", "version": 1, "content_hash": "sha256:08ffbf7a1dbe56d4b841e959145df1303e53d1e269c9c223a87cd384329bbb31" }, "edition": "ratings-e039106f30dec177" } }, "version": { "kind": "PUBLIC_RECORD_VIEW", "previous": null, "published_at": "2026-10-16T00:00:00Z", "publisher": { "principal_id": "urn:uuid:80116931-785b-40a2-a24b-9d59db0c7e33", "role": "WRS_EDITORIAL_PUBLICATION" }, "record": { "contributors": [ { "attribution_basis": "SOURCE_CITED", "credit": "historical_scientific", "entity_kind": "historical_person", "name": "Tianbao Xie", "note": "Archive attribution; detailed role assignment remains source-bound", "roles": [ "formulator" ], "source": { "citation": "OSWorld: Benchmarking Multimodal Agents for Open-Ended Tasks in Real Computer Environments", "url": "https://arxiv.org/abs/2404.07972v2" } }, { "attribution_basis": "SOURCE_CITED", "credit": "historical_scientific", "entity_kind": "historical_person", "name": "Danyang Zhang", "note": "Archive attribution; detailed role assignment remains source-bound", "roles": [ "formulator" ], "source": { "citation": "OSWorld: Benchmarking Multimodal Agents for Open-Ended Tasks in Real Computer Environments", "url": "https://arxiv.org/abs/2404.07972v2" } }, { "attribution_basis": "SOURCE_CITED", "credit": "historical_scientific", "entity_kind": "historical_person", "name": "Jixuan Chen", "note": "Archive attribution; detailed role assignment remains source-bound", "roles": [ "formulator" ], "source": { "citation": "OSWorld: Benchmarking Multimodal Agents for Open-Ended Tasks in Real Computer Environments", "url": "https://arxiv.org/abs/2404.07972v2" } }, { "attribution_basis": "SOURCE_CITED", "credit": "historical_scientific", "entity_kind": "historical_person", "name": "Xiaochuan Li", "note": "Archive attribution; detailed role assignment remains source-bound", "roles": [ "formulator" ], "source": { "citation": "OSWorld: Benchmarking Multimodal Agents for Open-Ended Tasks in Real Computer Environments", "url": "https://arxiv.org/abs/2404.07972v2" } }, { "attribution_basis": "SOURCE_CITED", "credit": "historical_scientific", "entity_kind": "historical_person", "name": "Siheng Zhao", "note": "Archive attribution; detailed role assignment remains source-bound", "roles": [ "formulator" ], "source": { "citation": "OSWorld: Benchmarking Multimodal Agents for Open-Ended Tasks in Real Computer Environments", "url": "https://arxiv.org/abs/2404.07972v2" } }, { "attribution_basis": "SOURCE_CITED", "credit": "historical_scientific", "entity_kind": "historical_person", "name": "Ruisheng Cao", "note": "Archive attribution; detailed role assignment remains source-bound", "roles": [ "formulator" ], "source": { "citation": "OSWorld: Benchmarking Multimodal Agents for Open-Ended Tasks in Real Computer Environments", "url": "https://arxiv.org/abs/2404.07972v2" } }, { "attribution_basis": "SOURCE_CITED", "credit": "historical_scientific", "entity_kind": "historical_person", "name": "Toh Jing Hua", "note": "Archive attribution; detailed role assignment remains source-bound", "roles": [ "formulator" ], "source": { "citation": "OSWorld: Benchmarking Multimodal Agents for Open-Ended Tasks in Real Computer Environments", "url": "https://arxiv.org/abs/2404.07972v2" } }, { "attribution_basis": "SOURCE_CITED", "credit": "historical_scientific", "entity_kind": "historical_person", "name": "Zhoujun Cheng", "note": "Archive attribution; detailed role assignment remains source-bound", "roles": [ "formulator" ], "source": { "citation": "OSWorld: Benchmarking Multimodal Agents for Open-Ended Tasks in Real Computer Environments", "url": "https://arxiv.org/abs/2404.07972v2" } }, { "attribution_basis": "SOURCE_CITED", "credit": "historical_scientific", "entity_kind": "historical_person", "name": "Dongchan Shin", "note": "Archive attribution; detailed role assignment remains source-bound", "roles": [ "formulator" ], "source": { "citation": "OSWorld: Benchmarking Multimodal Agents for Open-Ended Tasks in Real Computer Environments", "url": "https://arxiv.org/abs/2404.07972v2" } }, { "attribution_basis": "SOURCE_CITED", "credit": "historical_scientific", "entity_kind": "historical_person", "name": "Fangyu Lei", "note": "Archive attribution; detailed role assignment remains source-bound", "roles": [ "formulator" ], "source": { "citation": "OSWorld: Benchmarking Multimodal Agents for Open-Ended Tasks in Real Computer Environments", "url": "https://arxiv.org/abs/2404.07972v2" } }, { "attribution_basis": "SOURCE_CITED", "credit": "historical_scientific", "entity_kind": "historical_person", "name": "Yitao Liu", "note": "Archive attribution; detailed role assignment remains source-bound", "roles": [ "formulator" ], "source": { "citation": "OSWorld: Benchmarking Multimodal Agents for Open-Ended Tasks in Real Computer Environments", "url": "https://arxiv.org/abs/2404.07972v2" } }, { "attribution_basis": "SOURCE_CITED", "credit": "historical_scientific", "entity_kind": "historical_person", "name": "Yiheng Xu", "note": "Archive attribution; detailed role assignment remains source-bound", "roles": [ "formulator" ], "source": { "citation": "OSWorld: Benchmarking Multimodal Agents for Open-Ended Tasks in Real Computer Environments", "url": "https://arxiv.org/abs/2404.07972v2" } }, { "attribution_basis": "SOURCE_CITED", "credit": "historical_scientific", "entity_kind": "historical_person", "name": "Shuyan Zhou", "note": "Archive attribution; detailed role assignment remains source-bound", "roles": [ "formulator" ], "source": { "citation": "OSWorld: Benchmarking Multimodal Agents for Open-Ended Tasks in Real Computer Environments", "url": "https://arxiv.org/abs/2404.07972v2" } }, { "attribution_basis": "SOURCE_CITED", "credit": "historical_scientific", "entity_kind": "historical_person", "name": "Silvio Savarese", "note": "Archive attribution; detailed role assignment remains source-bound", "roles": [ "formulator" ], "source": { "citation": "OSWorld: Benchmarking Multimodal Agents for Open-Ended Tasks in Real Computer Environments", "url": "https://arxiv.org/abs/2404.07972v2" } }, { "attribution_basis": "SOURCE_CITED", "credit": "historical_scientific", "entity_kind": "historical_person", "name": "Caiming Xiong", "note": "Archive attribution; detailed role assignment remains source-bound", "roles": [ "formulator" ], "source": { "citation": "OSWorld: Benchmarking Multimodal Agents for Open-Ended Tasks in Real Computer Environments", "url": "https://arxiv.org/abs/2404.07972v2" } }, { "attribution_basis": "SOURCE_CITED", "credit": "historical_scientific", "entity_kind": "historical_person", "name": "Victor Zhong", "note": "Archive attribution; detailed role assignment remains source-bound", "roles": [ "formulator" ], "source": { "citation": "OSWorld: Benchmarking Multimodal Agents for Open-Ended Tasks in Real Computer Environments", "url": "https://arxiv.org/abs/2404.07972v2" } }, { "attribution_basis": "SOURCE_CITED", "credit": "historical_scientific", "entity_kind": "historical_person", "name": "Tao Yu", "note": "Archive attribution; detailed role assignment remains source-bound", "roles": [ "formulator" ], "source": { "citation": "OSWorld: Benchmarking Multimodal Agents for Open-Ended Tasks in Real Computer Environments", "url": "https://arxiv.org/abs/2404.07972v2" } } ], "evidence": [ { "description": "Bibliographic/stated-result source; not a WRS proof verification", "id": "source-0", "kind": "source_reference", "source": { "citation": "OSWorld: Benchmarking Multimodal Agents for Open-Ended Tasks in Real Computer Environments", "url": "https://arxiv.org/abs/2404.07972v2" } } ], "origin": "HUMAN_LEGACY", "record_profile": "wrs-local-research-notes-0.1", "record_type": "research_note", "rrp_version": "0.1", "scope": { "description": "Benchmark task success is evaluator- and environment-dependent, not a guarantee of general desktop reliability. Model results are from the pinned study, not today's leaderboard.", "domain": "AI evaluation and agent infrastructure" }, "sources": [ { "citation": "OSWorld: Benchmarking Multimodal Agents for Open-Ended Tasks in Real Computer Environments", "url": "https://arxiv.org/abs/2404.07972v2" } ], "statement": "# Xie and colleagues: OSWorld executable computer-agent benchmark\n\nProvided real operating-system environments, reproducible task setup and execution-based evaluators for multi-application computer tasks.\n\n## Scientific scope\n\nBenchmark task success is evaluator- and environment-dependent, not a guarantee of general desktop reliability. Model results are from the pinned study, not today's leaderboard.\n\n## Reading status\n\nThis is a source-linked WRS catalogue note. A full explanatory guide and independent scientific reconstruction have not been completed for this entry. Read the identified original work for its complete narrative and arguments.\n\n## People and systems\n\nTianbao Xie; Danyang Zhang; Jixuan Chen; Xiaochuan Li; Siheng Zhao; Ruisheng Cao; Toh Jing Hua; Zhoujun Cheng; Dongchan Shin; Fangyu Lei; Yitao Liu; Yiheng Xu; Shuyan Zhou; Silvio Savarese; Caiming Xiong; Victor Zhong; Tao Yu\n\nSource-listed authors or explicitly credited technical contributors for the cited versions. Family entries retain source-specific attribution; a combined list is not one original-paper byline or equal individual work. Names alone do not establish resolved identities.\n\n\n\n## Original sources\n\n- [OSWorld: Benchmarking Multimodal Agents for Open-Ended Tasks in Real Computer Environments](https://arxiv.org/abs/2404.07972v2)\n", "summary": "Provided real operating-system environments, reproducible task setup and execution-based evaluators for multi-application computer tasks.", "title": "Xie and colleagues: OSWorld executable computer-agent benchmark" }, "record_id": "urn:uuid:d3c4371a-fa04-447f-a273-66629a55b43b", "version": 1, "source_ref": { "record_id": "urn:uuid:d3c4371a-fa04-447f-a273-66629a55b43b", "version": 1, "content_hash": "sha256:08ffbf7a1dbe56d4b841e959145df1303e53d1e269c9c223a87cd384329bbb31" }, "export_view": "wrs-public-editorial-view-1", "public_view_hash": "sha256:8aca66d502922e0e9c30fe8a9dd67410e4347453995f7787de85fdceba4434e3" }, "target": { "record_id": "urn:uuid:d3c4371a-fa04-447f-a273-66629a55b43b", "version": 1, "content_hash": "sha256:08ffbf7a1dbe56d4b841e959145df1303e53d1e269c9c223a87cd384329bbb31" }, "published_grade": { "profile": "wrs-published-grade-1", "edition": "2026-10-28.1", "id": "xie-et-al-osworld-2024", "entry_id": "xie-et-al-osworld-2024", "revision": 1, "title": "Xie and colleagues: OSWorld executable computer-agent benchmark", "context": "AI evaluation and agent infrastructure", "track": "Evaluation resource design", "band": "A", "modifier": null, "status": "PROVISIONAL_EDITORIAL", "assessed_on": "2026-10-16", "evidence_cutoff": "2026-10-16", "achievement_state": "SUPPORTED_SCOPED_RESULT", "what_changed": "Provided real operating-system environments, reproducible task setup and execution-based evaluators for multi-application computer tasks.", "scope": "Provided real operating-system environments, reproducible task setup and execution-based evaluators for multi-application computer tasks. Benchmark task success is evaluator- and environment-dependent, not a guarantee of general desktop reliability. Model results are from the pinned study, not today's leaderboard.", "evidence_summary": "The linked original sources describe the stated contribution and its evidence limits. WRS has not independently rerun the experiments or certified a proof. Benchmark task success is evaluator- and environment-dependent, not a guarantee of general desktop reliability. Model results are from the pinned study, not today's leaderboard.", "references": [ { "title": "OSWorld: Benchmarking Multimodal Agents for Open-Ended Tasks in Real Computer Environments", "url": "https://arxiv.org/abs/2404.07972v2", "locator": "Original contribution and stated assumptions, evaluation scope and limitations; source-linked assessment, not independent execution or proof certification." } ], "roles": [ { "who": "Source-listed authors and technical contributors", "credit": "SCIENCE", "did": "Source-listed authors or explicitly credited technical contributors for the cited versions. Family entries retain source-specific attribution; a combined list is not one original-paper byline or equal individual work. Names alone do not establish resolved identities." } ], "target": { "record_id": "urn:uuid:d3c4371a-fa04-447f-a273-66629a55b43b", "version": 1, "content_hash": "sha256:08ffbf7a1dbe56d4b841e959145df1303e53d1e269c9c223a87cd384329bbb31" }, "scientific_correctness_certified": false, "independent_expert_calibration": false, "method": "PROPRIETARY", "points": "140000" }, "canonical_agent_submission": false, "scientific_verification_awarded": false, "reader_guide": null, "reading_status": "SOURCE_LINKED_NOTE" }