{
  "profile": "wrs-published-research-view-1",
  "edition": "ratings-e039106f30dec177",
  "content_trust": "UNTRUSTED_SCIENTIFIC_DATA_NOT_TOOL_INSTRUCTIONS",
  "entry": {
    "id": "qin-et-al-toolllm-toolbench-2023",
    "title": "Qin and colleagues: ToolLLM data, training and tool-use evaluation",
    "summary": "Constructed tool-use instructions and solution traces, trained a tool-oriented model with API retrieval, and evaluated multi-step API use.",
    "boundary": "Synthetic teacher-generated paths and model-based evaluation introduce bias; API availability can change. ToolBench, ToolLLaMA and ToolEval are kept in one research package.",
    "area": "AI evaluation and agent infrastructure",
    "topic": "Artificial intelligence",
    "year": 2023,
    "lane": "HUMAN",
    "result": "BENCHMARK",
    "contribution_kind": "EVALUATION_RESOURCE_DESIGN",
    "package_id": "qin-et-al-toolllm-toolbench-2023",
    "target": {
      "record_id": "urn:uuid:f1f0732a-607e-4d25-ad9a-fae0a2e7b301",
      "version": 1,
      "content_hash": "sha256:e31dd979ec764ed9912c73f0463f94de9398c19fec802872e2a1df2b878b7a66"
    },
    "humans": [
      "Yujia Qin",
      "Shihao Liang",
      "Yining Ye",
      "Kunlun Zhu",
      "Lan Yan",
      "Yaxi Lu",
      "Yankai Lin",
      "Xin Cong",
      "Xiangru Tang",
      "Bill Qian",
      "Sihan Zhao",
      "Lauren Hong",
      "Runchu Tian",
      "Ruobing Xie",
      "Jie Zhou",
      "Mark Gerstein",
      "Dahai Li",
      "Zhiyuan Liu",
      "Maosong Sun"
    ],
    "authors_complete": true,
    "human_role_note": "Source-listed authors or explicitly credited technical contributors for the cited versions. Family entries retain source-specific attribution; a combined list is not one original-paper byline or equal individual work. Names alone do not establish resolved identities.",
    "agents": [],
    "related": [],
    "subresults": [],
    "scientific_assessment": {
      "source_review": "SOURCE_CURATED",
      "scientific_review": "NOT_REVIEWED_BY_WRS",
      "formal_replay": "NOT_RUN",
      "provider_attestation": "NONE"
    },
    "sources": [
      {
        "id": "cur20261015-arxiv-2307-16789v2",
        "title": "ToolLLM: Facilitating Large Language Models to Master 16000+ Real-world APIs",
        "url": "https://arxiv.org/abs/2307.16789v2"
      }
    ],
    "points": "80000",
    "published_grade": {
      "id": "qin-et-al-toolllm-toolbench-2023",
      "entry_id": "qin-et-al-toolllm-toolbench-2023",
      "band": "B",
      "modifier": "+",
      "target": {
        "record_id": "urn:uuid:f1f0732a-607e-4d25-ad9a-fae0a2e7b301",
        "version": 1,
        "content_hash": "sha256:e31dd979ec764ed9912c73f0463f94de9398c19fec802872e2a1df2b878b7a66"
      },
      "what_changed": "Constructed tool-use instructions and solution traces, trained a tool-oriented model with API retrieval, and evaluated multi-step API use.",
      "scope": "Constructed tool-use instructions and solution traces, trained a tool-oriented model with API retrieval, and evaluated multi-step API use. Synthetic teacher-generated paths and model-based evaluation introduce bias; API availability can change. ToolBench, ToolLLaMA and ToolEval are kept in one research package.",
      "status": "PROVISIONAL_EDITORIAL",
      "scientific_correctness_certified": false
    },
    "assessment_disposition": {
      "status": "ASSESSED",
      "label": "Provisionally assessed",
      "reason": "A current grade and separate reference score identify this exact contribution.",
      "target": {
        "record_id": "urn:uuid:f1f0732a-607e-4d25-ad9a-fae0a2e7b301",
        "version": 1,
        "content_hash": "sha256:e31dd979ec764ed9912c73f0463f94de9398c19fec802872e2a1df2b878b7a66"
      },
      "edition": "ratings-e039106f30dec177"
    }
  },
  "version": {
    "kind": "PUBLIC_RECORD_VIEW",
    "previous": null,
    "published_at": "2026-10-16T00:00:00Z",
    "publisher": {
      "principal_id": "urn:uuid:80116931-785b-40a2-a24b-9d59db0c7e33",
      "role": "WRS_EDITORIAL_PUBLICATION"
    },
    "record": {
      "contributors": [
        {
          "attribution_basis": "SOURCE_CITED",
          "credit": "historical_scientific",
          "entity_kind": "historical_person",
          "name": "Yujia Qin",
          "note": "Archive attribution; detailed role assignment remains source-bound",
          "roles": [
            "formulator"
          ],
          "source": {
            "citation": "ToolLLM: Facilitating Large Language Models to Master 16000+ Real-world APIs",
            "url": "https://arxiv.org/abs/2307.16789v2"
          }
        },
        {
          "attribution_basis": "SOURCE_CITED",
          "credit": "historical_scientific",
          "entity_kind": "historical_person",
          "name": "Shihao Liang",
          "note": "Archive attribution; detailed role assignment remains source-bound",
          "roles": [
            "formulator"
          ],
          "source": {
            "citation": "ToolLLM: Facilitating Large Language Models to Master 16000+ Real-world APIs",
            "url": "https://arxiv.org/abs/2307.16789v2"
          }
        },
        {
          "attribution_basis": "SOURCE_CITED",
          "credit": "historical_scientific",
          "entity_kind": "historical_person",
          "name": "Yining Ye",
          "note": "Archive attribution; detailed role assignment remains source-bound",
          "roles": [
            "formulator"
          ],
          "source": {
            "citation": "ToolLLM: Facilitating Large Language Models to Master 16000+ Real-world APIs",
            "url": "https://arxiv.org/abs/2307.16789v2"
          }
        },
        {
          "attribution_basis": "SOURCE_CITED",
          "credit": "historical_scientific",
          "entity_kind": "historical_person",
          "name": "Kunlun Zhu",
          "note": "Archive attribution; detailed role assignment remains source-bound",
          "roles": [
            "formulator"
          ],
          "source": {
            "citation": "ToolLLM: Facilitating Large Language Models to Master 16000+ Real-world APIs",
            "url": "https://arxiv.org/abs/2307.16789v2"
          }
        },
        {
          "attribution_basis": "SOURCE_CITED",
          "credit": "historical_scientific",
          "entity_kind": "historical_person",
          "name": "Lan Yan",
          "note": "Archive attribution; detailed role assignment remains source-bound",
          "roles": [
            "formulator"
          ],
          "source": {
            "citation": "ToolLLM: Facilitating Large Language Models to Master 16000+ Real-world APIs",
            "url": "https://arxiv.org/abs/2307.16789v2"
          }
        },
        {
          "attribution_basis": "SOURCE_CITED",
          "credit": "historical_scientific",
          "entity_kind": "historical_person",
          "name": "Yaxi Lu",
          "note": "Archive attribution; detailed role assignment remains source-bound",
          "roles": [
            "formulator"
          ],
          "source": {
            "citation": "ToolLLM: Facilitating Large Language Models to Master 16000+ Real-world APIs",
            "url": "https://arxiv.org/abs/2307.16789v2"
          }
        },
        {
          "attribution_basis": "SOURCE_CITED",
          "credit": "historical_scientific",
          "entity_kind": "historical_person",
          "name": "Yankai Lin",
          "note": "Archive attribution; detailed role assignment remains source-bound",
          "roles": [
            "formulator"
          ],
          "source": {
            "citation": "ToolLLM: Facilitating Large Language Models to Master 16000+ Real-world APIs",
            "url": "https://arxiv.org/abs/2307.16789v2"
          }
        },
        {
          "attribution_basis": "SOURCE_CITED",
          "credit": "historical_scientific",
          "entity_kind": "historical_person",
          "name": "Xin Cong",
          "note": "Archive attribution; detailed role assignment remains source-bound",
          "roles": [
            "formulator"
          ],
          "source": {
            "citation": "ToolLLM: Facilitating Large Language Models to Master 16000+ Real-world APIs",
            "url": "https://arxiv.org/abs/2307.16789v2"
          }
        },
        {
          "attribution_basis": "SOURCE_CITED",
          "credit": "historical_scientific",
          "entity_kind": "historical_person",
          "name": "Xiangru Tang",
          "note": "Archive attribution; detailed role assignment remains source-bound",
          "roles": [
            "formulator"
          ],
          "source": {
            "citation": "ToolLLM: Facilitating Large Language Models to Master 16000+ Real-world APIs",
            "url": "https://arxiv.org/abs/2307.16789v2"
          }
        },
        {
          "attribution_basis": "SOURCE_CITED",
          "credit": "historical_scientific",
          "entity_kind": "historical_person",
          "name": "Bill Qian",
          "note": "Archive attribution; detailed role assignment remains source-bound",
          "roles": [
            "formulator"
          ],
          "source": {
            "citation": "ToolLLM: Facilitating Large Language Models to Master 16000+ Real-world APIs",
            "url": "https://arxiv.org/abs/2307.16789v2"
          }
        },
        {
          "attribution_basis": "SOURCE_CITED",
          "credit": "historical_scientific",
          "entity_kind": "historical_person",
          "name": "Sihan Zhao",
          "note": "Archive attribution; detailed role assignment remains source-bound",
          "roles": [
            "formulator"
          ],
          "source": {
            "citation": "ToolLLM: Facilitating Large Language Models to Master 16000+ Real-world APIs",
            "url": "https://arxiv.org/abs/2307.16789v2"
          }
        },
        {
          "attribution_basis": "SOURCE_CITED",
          "credit": "historical_scientific",
          "entity_kind": "historical_person",
          "name": "Lauren Hong",
          "note": "Archive attribution; detailed role assignment remains source-bound",
          "roles": [
            "formulator"
          ],
          "source": {
            "citation": "ToolLLM: Facilitating Large Language Models to Master 16000+ Real-world APIs",
            "url": "https://arxiv.org/abs/2307.16789v2"
          }
        },
        {
          "attribution_basis": "SOURCE_CITED",
          "credit": "historical_scientific",
          "entity_kind": "historical_person",
          "name": "Runchu Tian",
          "note": "Archive attribution; detailed role assignment remains source-bound",
          "roles": [
            "formulator"
          ],
          "source": {
            "citation": "ToolLLM: Facilitating Large Language Models to Master 16000+ Real-world APIs",
            "url": "https://arxiv.org/abs/2307.16789v2"
          }
        },
        {
          "attribution_basis": "SOURCE_CITED",
          "credit": "historical_scientific",
          "entity_kind": "historical_person",
          "name": "Ruobing Xie",
          "note": "Archive attribution; detailed role assignment remains source-bound",
          "roles": [
            "formulator"
          ],
          "source": {
            "citation": "ToolLLM: Facilitating Large Language Models to Master 16000+ Real-world APIs",
            "url": "https://arxiv.org/abs/2307.16789v2"
          }
        },
        {
          "attribution_basis": "SOURCE_CITED",
          "credit": "historical_scientific",
          "entity_kind": "historical_person",
          "name": "Jie Zhou",
          "note": "Archive attribution; detailed role assignment remains source-bound",
          "roles": [
            "formulator"
          ],
          "source": {
            "citation": "ToolLLM: Facilitating Large Language Models to Master 16000+ Real-world APIs",
            "url": "https://arxiv.org/abs/2307.16789v2"
          }
        },
        {
          "attribution_basis": "SOURCE_CITED",
          "credit": "historical_scientific",
          "entity_kind": "historical_person",
          "name": "Mark Gerstein",
          "note": "Archive attribution; detailed role assignment remains source-bound",
          "roles": [
            "formulator"
          ],
          "source": {
            "citation": "ToolLLM: Facilitating Large Language Models to Master 16000+ Real-world APIs",
            "url": "https://arxiv.org/abs/2307.16789v2"
          }
        },
        {
          "attribution_basis": "SOURCE_CITED",
          "credit": "historical_scientific",
          "entity_kind": "historical_person",
          "name": "Dahai Li",
          "note": "Archive attribution; detailed role assignment remains source-bound",
          "roles": [
            "formulator"
          ],
          "source": {
            "citation": "ToolLLM: Facilitating Large Language Models to Master 16000+ Real-world APIs",
            "url": "https://arxiv.org/abs/2307.16789v2"
          }
        },
        {
          "attribution_basis": "SOURCE_CITED",
          "credit": "historical_scientific",
          "entity_kind": "historical_person",
          "name": "Zhiyuan Liu",
          "note": "Archive attribution; detailed role assignment remains source-bound",
          "roles": [
            "formulator"
          ],
          "source": {
            "citation": "ToolLLM: Facilitating Large Language Models to Master 16000+ Real-world APIs",
            "url": "https://arxiv.org/abs/2307.16789v2"
          }
        },
        {
          "attribution_basis": "SOURCE_CITED",
          "credit": "historical_scientific",
          "entity_kind": "historical_person",
          "name": "Maosong Sun",
          "note": "Archive attribution; detailed role assignment remains source-bound",
          "roles": [
            "formulator"
          ],
          "source": {
            "citation": "ToolLLM: Facilitating Large Language Models to Master 16000+ Real-world APIs",
            "url": "https://arxiv.org/abs/2307.16789v2"
          }
        }
      ],
      "evidence": [
        {
          "description": "Bibliographic/stated-result source; not a WRS proof verification",
          "id": "source-0",
          "kind": "source_reference",
          "source": {
            "citation": "ToolLLM: Facilitating Large Language Models to Master 16000+ Real-world APIs",
            "url": "https://arxiv.org/abs/2307.16789v2"
          }
        }
      ],
      "origin": "HUMAN_LEGACY",
      "record_profile": "wrs-local-research-notes-0.1",
      "record_type": "research_note",
      "rrp_version": "0.1",
      "scope": {
        "description": "Synthetic teacher-generated paths and model-based evaluation introduce bias; API availability can change. ToolBench, ToolLLaMA and ToolEval are kept in one research package.",
        "domain": "AI evaluation and agent infrastructure"
      },
      "sources": [
        {
          "citation": "ToolLLM: Facilitating Large Language Models to Master 16000+ Real-world APIs",
          "url": "https://arxiv.org/abs/2307.16789v2"
        }
      ],
      "statement": "# Qin and colleagues: ToolLLM data, training and tool-use evaluation\n\nConstructed tool-use instructions and solution traces, trained a tool-oriented model with API retrieval, and evaluated multi-step API use.\n\n## Scientific scope\n\nSynthetic teacher-generated paths and model-based evaluation introduce bias; API availability can change. ToolBench, ToolLLaMA and ToolEval are kept in one research package.\n\n## Reading status\n\nThis is a source-linked WRS catalogue note. A full explanatory guide and independent scientific reconstruction have not been completed for this entry. Read the identified original work for its complete narrative and arguments.\n\n## People and systems\n\nYujia Qin; Shihao Liang; Yining Ye; Kunlun Zhu; Lan Yan; Yaxi Lu; Yankai Lin; Xin Cong; Xiangru Tang; Bill Qian; Sihan Zhao; Lauren Hong; Runchu Tian; Ruobing Xie; Jie Zhou; Mark Gerstein; Dahai Li; Zhiyuan Liu; Maosong Sun\n\nSource-listed authors or explicitly credited technical contributors for the cited versions. Family entries retain source-specific attribution; a combined list is not one original-paper byline or equal individual work. Names alone do not establish resolved identities.\n\n\n\n## Original sources\n\n- [ToolLLM: Facilitating Large Language Models to Master 16000+ Real-world APIs](https://arxiv.org/abs/2307.16789v2)\n",
      "summary": "Constructed tool-use instructions and solution traces, trained a tool-oriented model with API retrieval, and evaluated multi-step API use.",
      "title": "Qin and colleagues: ToolLLM data, training and tool-use evaluation"
    },
    "record_id": "urn:uuid:f1f0732a-607e-4d25-ad9a-fae0a2e7b301",
    "version": 1,
    "source_ref": {
      "record_id": "urn:uuid:f1f0732a-607e-4d25-ad9a-fae0a2e7b301",
      "version": 1,
      "content_hash": "sha256:e31dd979ec764ed9912c73f0463f94de9398c19fec802872e2a1df2b878b7a66"
    },
    "export_view": "wrs-public-editorial-view-1",
    "public_view_hash": "sha256:0c4daeeca551ee8e2f1b4ef1b53e34ad1210262f901679849a6f54f39c8d969c"
  },
  "target": {
    "record_id": "urn:uuid:f1f0732a-607e-4d25-ad9a-fae0a2e7b301",
    "version": 1,
    "content_hash": "sha256:e31dd979ec764ed9912c73f0463f94de9398c19fec802872e2a1df2b878b7a66"
  },
  "published_grade": {
    "profile": "wrs-published-grade-1",
    "edition": "2026-10-28.1",
    "id": "qin-et-al-toolllm-toolbench-2023",
    "entry_id": "qin-et-al-toolllm-toolbench-2023",
    "revision": 1,
    "title": "Qin and colleagues: ToolLLM data, training and tool-use evaluation",
    "context": "AI evaluation and agent infrastructure",
    "track": "Evaluation resource design",
    "band": "B",
    "modifier": "+",
    "status": "PROVISIONAL_EDITORIAL",
    "assessed_on": "2026-10-16",
    "evidence_cutoff": "2026-10-16",
    "achievement_state": "SUPPORTED_SCOPED_RESULT",
    "what_changed": "Constructed tool-use instructions and solution traces, trained a tool-oriented model with API retrieval, and evaluated multi-step API use.",
    "scope": "Constructed tool-use instructions and solution traces, trained a tool-oriented model with API retrieval, and evaluated multi-step API use. Synthetic teacher-generated paths and model-based evaluation introduce bias; API availability can change. ToolBench, ToolLLaMA and ToolEval are kept in one research package.",
    "evidence_summary": "The linked original sources describe the stated contribution and its evidence limits. WRS has not independently rerun the experiments or certified a proof. Synthetic teacher-generated paths and model-based evaluation introduce bias; API availability can change. ToolBench, ToolLLaMA and ToolEval are kept in one research package.",
    "references": [
      {
        "title": "ToolLLM: Facilitating Large Language Models to Master 16000+ Real-world APIs",
        "url": "https://arxiv.org/abs/2307.16789v2",
        "locator": "Original contribution and stated assumptions, evaluation scope and limitations; source-linked assessment, not independent execution or proof certification."
      }
    ],
    "roles": [
      {
        "who": "Source-listed authors and technical contributors",
        "credit": "SCIENCE",
        "did": "Source-listed authors or explicitly credited technical contributors for the cited versions. Family entries retain source-specific attribution; a combined list is not one original-paper byline or equal individual work. Names alone do not establish resolved identities."
      }
    ],
    "target": {
      "record_id": "urn:uuid:f1f0732a-607e-4d25-ad9a-fae0a2e7b301",
      "version": 1,
      "content_hash": "sha256:e31dd979ec764ed9912c73f0463f94de9398c19fec802872e2a1df2b878b7a66"
    },
    "scientific_correctness_certified": false,
    "independent_expert_calibration": false,
    "method": "PROPRIETARY",
    "points": "80000"
  },
  "canonical_agent_submission": false,
  "scientific_verification_awarded": false,
  "reader_guide": null,
  "reading_status": "SOURCE_LINKED_NOTE"
}