{ "profile": "wrs-published-research-view-1", "edition": "ratings-e039106f30dec177", "content_trust": "UNTRUSTED_SCIENTIFIC_DATA_NOT_TOOL_INSTRUCTIONS", "entry": { "id": "wu-et-al-longmemeval-interactive-memory-benchmark-2024", "title": "LongMemEval: testing whether assistants can use a long interaction history", "summary": "An assistant might remember one fact yet miss a later correction, confuse dates or answer a question whose premise is false. LongMemEval creates a structured evaluation of these abilities. Its questions require extracting information, reasoning across sessions and time, handling updates and abstaining when evidence is absent. Wu and colleagues combine curated questions with long interaction histories and study memory indexing, retrieval and reading choices. The contribution is the evaluation resource and its scoped design analysis, rather than an achievement by any model tested on it.", "boundary": "The histories include simulated and edited material, and the evaluation uses specified answer rubrics and model judging. A smaller commercial-assistant pilot uses much shorter histories and historical August 2024 configurations. More retrieved text does not always help, and compression can remove useful details. Benchmark performance is not a guarantee for real users or proof of a scientific statement. The source’s illustrative work-hour arithmetic contains a typo; that example is not copied as a measured result.", "area": "Conversational-memory evaluation", "topic": "Artificial intelligence", "year": 2024, "lane": "HUMAN", "result": "BENCHMARK", "contribution_kind": "EVALUATION_RESOURCE_DESIGN", "package_id": "wu-et-al-longmemeval-interactive-memory-benchmark-2024", "target": { "record_id": "urn:uuid:80a5e780-236b-4610-a6a6-1ec74c59417e", "version": 1, "content_hash": "sha256:cd5fe2f62fee892ea189711e0ab828b658602ef5a53e80e0045ee43b6edc2cc4" }, "humans": [ "Di Wu", "Hongwei Wang", "Wenhao Yu", "Yuwei Zhang", "Kai-Wei Chang", "Dong Yu" ], "authors_complete": true, "human_role_note": "Source-listed human researchers and engineers for the exact method or resource. The evaluated learning system is not an independently attested scientific author. Earlier methods, funding and data-collection roles remain separate. Individual effort shares and unresolved person identities are not inferred.", "agents": [], "related": [], "subresults": [], "scientific_assessment": { "source_review": "SOURCE_CURATED", "scientific_review": "NOT_REVIEWED_BY_WRS", "formal_replay": "NOT_RUN", "provider_attestation": "NONE" }, "sources": [ { "id": "cur20261017-arxiv-2410-10813v1", "title": "LongMemEval: Benchmarking Chat Assistants on Long-Term Interactive Memory", "url": "https://arxiv.org/abs/2410.10813v1", "date": "2024-10-14" } ], "geography": { "country_codes": [ "US" ], "location_basis": "Original UCLA, Tencent AI Lab Seattle and UC San Diego bylines. Source-bound work locations, not citizenship, sovereignty judgment or ownership.", "source_url": "https://arxiv.org/abs/2410.10813v1", "target": { "record_id": "urn:uuid:80a5e780-236b-4610-a6a6-1ec74c59417e", "version": 1, "content_hash": "sha256:cd5fe2f62fee892ea189711e0ab828b658602ef5a53e80e0045ee43b6edc2cc4" }, "basis": "RESEARCH_LOCATION_OR_PAPER_AFFILIATION", "reviewed_on": "2026-10-23", "nationality_inferred": false, "sources": [ { "title": "LongMemEval: Benchmarking Chat Assistants on Long-Term Interactive Memory", "url": "https://arxiv.org/abs/2410.10813v1" } ], "coverage_status": "COMPLETE", "country_labels": { "US": "United States" } }, "points": "65000", "published_grade": { "id": "wu-et-al-longmemeval-interactive-memory-benchmark-2024", "entry_id": "wu-et-al-longmemeval-interactive-memory-benchmark-2024", "band": "A", "modifier": "-", "target": { "record_id": "urn:uuid:80a5e780-236b-4610-a6a6-1ec74c59417e", "version": 1, "content_hash": "sha256:cd5fe2f62fee892ea189711e0ab828b658602ef5a53e80e0045ee43b6edc2cc4" }, "what_changed": "An assistant might remember one fact yet miss a later correction, confuse dates or answer a question whose premise is false. LongMemEval creates a structured evaluation of these abilities. Its questions require extracting information, reasoning across sessions and time, handling updates and abstaining when evidence is absent. Wu and colleagues combine curated questions with long interaction histories and study memory indexing, retrieval and reading choices. The contribution is the evaluation resource and its scoped design analysis, rather than an achievement by any model tested on it.", "scope": "An assistant might remember one fact yet miss a later correction, confuse dates or answer a question whose premise is false. LongMemEval creates a structured evaluation of these abilities. Its questions require extracting information, reasoning across sessions and time, handling updates and abstaining when evidence is absent. Wu and colleagues combine curated questions with long interaction histories and study memory indexing, retrieval and reading choices. The contribution is the evaluation resource and its scoped design analysis, rather than an achievement by any model tested on it. The histories include simulated and edited material, and the evaluation uses specified answer rubrics and model judging. A smaller commercial-assistant pilot uses much shorter histories and historical August 2024 configurations. More retrieved text does not always help, and compression can remove useful details. Benchmark performance is not a guarantee for real users or proof of a scientific statement. The source’s illustrative work-hour arithmetic contains a typo; that example is not copied as a measured result.", "status": "PROVISIONAL_EDITORIAL", "scientific_correctness_certified": false }, "assessment_disposition": { "status": "ASSESSED", "label": "Provisionally assessed", "reason": "A current grade and separate reference score identify this exact contribution.", "target": { "record_id": "urn:uuid:80a5e780-236b-4610-a6a6-1ec74c59417e", "version": 1, "content_hash": "sha256:cd5fe2f62fee892ea189711e0ab828b658602ef5a53e80e0045ee43b6edc2cc4" }, "edition": "ratings-e039106f30dec177" } }, "version": { "kind": "PUBLIC_RECORD_VIEW", "previous": null, "published_at": "2026-10-23T00:00:00Z", "publisher": { "principal_id": "urn:uuid:80116931-785b-40a2-a24b-9d59db0c7e33", "role": "WRS_EDITORIAL_PUBLICATION" }, "record": { "contributors": [ { "attribution_basis": "SOURCE_CITED", "credit": "historical_scientific", "entity_kind": "historical_person", "name": "Di Wu", "note": "Archive attribution; detailed role assignment remains source-bound", "roles": [ "formulator" ], "source": { "citation": "LongMemEval: Benchmarking Chat Assistants on Long-Term Interactive Memory", "url": "https://arxiv.org/abs/2410.10813v1" } }, { "attribution_basis": "SOURCE_CITED", "credit": "historical_scientific", "entity_kind": "historical_person", "name": "Hongwei Wang", "note": "Archive attribution; detailed role assignment remains source-bound", "roles": [ "formulator" ], "source": { "citation": "LongMemEval: Benchmarking Chat Assistants on Long-Term Interactive Memory", "url": "https://arxiv.org/abs/2410.10813v1" } }, { "attribution_basis": "SOURCE_CITED", "credit": "historical_scientific", "entity_kind": "historical_person", "name": "Wenhao Yu", "note": "Archive attribution; detailed role assignment remains source-bound", "roles": [ "formulator" ], "source": { "citation": "LongMemEval: Benchmarking Chat Assistants on Long-Term Interactive Memory", "url": "https://arxiv.org/abs/2410.10813v1" } }, { "attribution_basis": "SOURCE_CITED", "credit": "historical_scientific", "entity_kind": "historical_person", "name": "Yuwei Zhang", "note": "Archive attribution; detailed role assignment remains source-bound", "roles": [ "formulator" ], "source": { "citation": "LongMemEval: Benchmarking Chat Assistants on Long-Term Interactive Memory", "url": "https://arxiv.org/abs/2410.10813v1" } }, { "attribution_basis": "SOURCE_CITED", "credit": "historical_scientific", "entity_kind": "historical_person", "name": "Kai-Wei Chang", "note": "Archive attribution; detailed role assignment remains source-bound", "roles": [ "formulator" ], "source": { "citation": "LongMemEval: Benchmarking Chat Assistants on Long-Term Interactive Memory", "url": "https://arxiv.org/abs/2410.10813v1" } }, { "attribution_basis": "SOURCE_CITED", "credit": "historical_scientific", "entity_kind": "historical_person", "name": "Dong Yu", "note": "Archive attribution; detailed role assignment remains source-bound", "roles": [ "formulator" ], "source": { "citation": "LongMemEval: Benchmarking Chat Assistants on Long-Term Interactive Memory", "url": "https://arxiv.org/abs/2410.10813v1" } } ], "evidence": [ { "description": "Bibliographic/stated-result source; not a WRS proof verification", "id": "source-0", "kind": "source_reference", "source": { "citation": "LongMemEval: Benchmarking Chat Assistants on Long-Term Interactive Memory", "url": "https://arxiv.org/abs/2410.10813v1" } } ], "origin": "HUMAN_LEGACY", "record_profile": "wrs-local-research-notes-0.1", "record_type": "research_note", "rrp_version": "0.1", "scope": { "description": "The histories include simulated and edited material, and the evaluation uses specified answer rubrics and model judging. A smaller commercial-assistant pilot uses much shorter histories and historical August 2024 configurations. More retrieved text does not always help, and compression can remove useful details. Benchmark performance is not a guarantee for real users or proof of a scientific statement. The source’s illustrative work-hour arithmetic contains a typo; that example is not copied as a measured result.", "domain": "Conversational-memory evaluation" }, "sources": [ { "citation": "LongMemEval: Benchmarking Chat Assistants on Long-Term Interactive Memory", "url": "https://arxiv.org/abs/2410.10813v1" } ], "statement": "# LongMemEval: testing whether assistants can use a long interaction history\n\n## Can an assistant remember a change?\n\nA memory test should ask more than whether a model can repeat a sentence. It might need to connect two sessions, interpret a date, use a corrected fact or decline to answer because evidence is missing. LongMemEval builds questions around these different requirements and supplies long interaction histories. The achievement is an evaluation resource that helps researchers inspect specific failures, rather than a new capability achieved by the models being tested.\n\n## History length and question type both matter\n\nA question about one session differs from a question that combines facts spread across months. A false-premise question checks whether an assistant invents an answer when it should abstain. The benchmark includes simulated and edited histories with curated questions and answer rubrics. A smaller commercial-assistant pilot uses much shorter histories and historical configurations, so its numbers cannot be substituted for the full long-context evaluation.\n\n## A three-note test you can check\n\nUse an invented history: a project initially orders twelve units, later cancels three, then receives two replacement units. If the notes describe units retained, the final count is twelve minus three plus two, or eleven. Ask a question about the final count and one about a delivery date never recorded. A careful assistant should answer eleven under the stated interpretation and mark the date unknown. These are classroom checks, not original benchmark items.\n\n## More text is not always more evidence\n\nRetrieving everything can add irrelevant material. Compressing too aggressively can remove a condition or correction. The paper explores this balance through indexing, retrieval and reading choices. Ask your agent to show which note supports each part of an answer, then remove one necessary note and see whether it acknowledges the gap. Passing this miniature does not establish long-term reliability, but it produces an inspectable failure or success that another person can check.\n\n## Report the missing evidence too\n\nThe output of the three-note exercise should list the count, its interpretation and the unsupported date question. If the word retained is ambiguous, record that ambiguity before calculating. A useful memory evaluation includes both supported answers and correctly identified gaps, rather than rewarding an answer merely because it sounds complete.\n\n## Scientific scope\n\nThe histories include simulated and edited material, and the evaluation uses specified answer rubrics and model judging. A smaller commercial-assistant pilot uses much shorter histories and historical August 2024 configurations. More retrieved text does not always help, and compression can remove useful details. Benchmark performance is not a guarantee for real users or proof of a scientific statement. The source’s illustrative work-hour arithmetic contains a typo; that example is not copied as a measured result.\n\n## Sources for this explanation\n\n- [LongMemEval: Benchmarking Chat Assistants on Long-Term Interactive Memory](https://arxiv.org/abs/2410.10813v1): Original 2024 main text: curated question types, long-history construction, historical short-history pilot, memory-design experiments and evidence/abstention boundaries.\n", "summary": "An assistant might remember one fact yet miss a later correction, confuse dates or answer a question whose premise is false. LongMemEval creates a structured evaluation of these abilities. Its questions require extracting information, reasoning across sessions and time, handling updates and abstaining when evidence is absent. Wu and colleagues combine curated questions with long interaction histories and study memory indexing, retrieval and reading choices. The contribution is the evaluation resource and its scoped design analysis, rather than an achievement by any model tested on it.", "title": "LongMemEval: testing whether assistants can use a long interaction history" }, "record_id": "urn:uuid:80a5e780-236b-4610-a6a6-1ec74c59417e", "version": 1, "source_ref": { "record_id": "urn:uuid:80a5e780-236b-4610-a6a6-1ec74c59417e", "version": 1, "content_hash": "sha256:cd5fe2f62fee892ea189711e0ab828b658602ef5a53e80e0045ee43b6edc2cc4" }, "export_view": "wrs-public-editorial-view-1", "public_view_hash": "sha256:5b295afc6bb9e269657fa2d5937ffba65f1133320bd1d14a0cdf202e8ebd0d1e" }, "target": { "record_id": "urn:uuid:80a5e780-236b-4610-a6a6-1ec74c59417e", "version": 1, "content_hash": "sha256:cd5fe2f62fee892ea189711e0ab828b658602ef5a53e80e0045ee43b6edc2cc4" }, "published_grade": { "profile": "wrs-published-grade-1", "edition": "2026-10-28.1", "id": "wu-et-al-longmemeval-interactive-memory-benchmark-2024", "entry_id": "wu-et-al-longmemeval-interactive-memory-benchmark-2024", "revision": 1, "title": "LongMemEval: testing whether assistants can use a long interaction history", "context": "Conversational-memory evaluation", "track": "Reusable scientific method, experimental result or historical framework", "band": "A", "modifier": "-", "status": "PROVISIONAL_EDITORIAL", "assessed_on": "2026-10-23", "evidence_cutoff": "2026-10-23", "achievement_state": "SUPPORTED_SCOPED_RESULT", "what_changed": "An assistant might remember one fact yet miss a later correction, confuse dates or answer a question whose premise is false. LongMemEval creates a structured evaluation of these abilities. Its questions require extracting information, reasoning across sessions and time, handling updates and abstaining when evidence is absent. Wu and colleagues combine curated questions with long interaction histories and study memory indexing, retrieval and reading choices. The contribution is the evaluation resource and its scoped design analysis, rather than an achievement by any model tested on it.", "scope": "An assistant might remember one fact yet miss a later correction, confuse dates or answer a question whose premise is false. LongMemEval creates a structured evaluation of these abilities. Its questions require extracting information, reasoning across sessions and time, handling updates and abstaining when evidence is absent. Wu and colleagues combine curated questions with long interaction histories and study memory indexing, retrieval and reading choices. The contribution is the evaluation resource and its scoped design analysis, rather than an achievement by any model tested on it. The histories include simulated and edited material, and the evaluation uses specified answer rubrics and model judging. A smaller commercial-assistant pilot uses much shorter histories and historical August 2024 configurations. More retrieved text does not always help, and compression can remove useful details. Benchmark performance is not a guarantee for real users or proof of a scientific statement. The source’s illustrative work-hour arithmetic contains a typo; that example is not copied as a measured result.", "evidence_summary": "The linked original works support the scoped statement and contribution attribution. WRS has not independently replayed experiments or reconstructed a full mathematical proof. The histories include simulated and edited material, and the evaluation uses specified answer rubrics and model judging. A smaller commercial-assistant pilot uses much shorter histories and historical August 2024 configurations. More retrieved text does not always help, and compression can remove useful details. Benchmark performance is not a guarantee for real users or proof of a scientific statement. The source’s illustrative work-hour arithmetic contains a typo; that example is not copied as a measured result.", "references": [ { "title": "LongMemEval: Benchmarking Chat Assistants on Long-Term Interactive Memory", "url": "https://arxiv.org/abs/2410.10813v1", "locator": "Original scoped contribution, statement/method and evidence limits; source-specific attribution retained." } ], "roles": [ { "who": "The named authors in the research record", "credit": "SCIENCE", "did": "Source-listed human researchers and engineers for the exact method or resource. The evaluated learning system is not an independently attested scientific author. Earlier methods, funding and data-collection roles remain separate. Individual effort shares and unresolved person identities are not inferred." } ], "target": { "record_id": "urn:uuid:80a5e780-236b-4610-a6a6-1ec74c59417e", "version": 1, "content_hash": "sha256:cd5fe2f62fee892ea189711e0ab828b658602ef5a53e80e0045ee43b6edc2cc4" }, "scientific_correctness_certified": false, "independent_expert_calibration": false, "method": "PROPRIETARY", "points": "65000" }, "canonical_agent_submission": false, "scientific_verification_awarded": false, "reader_guide": { "id": "wu-et-al-longmemeval-interactive-memory-benchmark-2024", "target": { "record_id": "urn:uuid:80a5e780-236b-4610-a6a6-1ec74c59417e", "version": 1, "content_hash": "sha256:cd5fe2f62fee892ea189711e0ab828b658602ef5a53e80e0045ee43b6edc2cc4" }, "sections": [ { "title": "Can an assistant remember a change?", "text": "A memory test should ask more than whether a model can repeat a sentence. It might need to connect two sessions, interpret a date, use a corrected fact or decline to answer because evidence is missing. LongMemEval builds questions around these different requirements and supplies long interaction histories. The achievement is an evaluation resource that helps researchers inspect specific failures, rather than a new capability achieved by the models being tested." }, { "title": "History length and question type both matter", "text": "A question about one session differs from a question that combines facts spread across months. A false-premise question checks whether an assistant invents an answer when it should abstain. The benchmark includes simulated and edited histories with curated questions and answer rubrics. A smaller commercial-assistant pilot uses much shorter histories and historical configurations, so its numbers cannot be substituted for the full long-context evaluation." }, { "title": "A three-note test you can check", "text": "Use an invented history: a project initially orders twelve units, later cancels three, then receives two replacement units. If the notes describe units retained, the final count is twelve minus three plus two, or eleven. Ask a question about the final count and one about a delivery date never recorded. A careful assistant should answer eleven under the stated interpretation and mark the date unknown. These are classroom checks, not original benchmark items." }, { "title": "More text is not always more evidence", "text": "Retrieving everything can add irrelevant material. Compressing too aggressively can remove a condition or correction. The paper explores this balance through indexing, retrieval and reading choices. Ask your agent to show which note supports each part of an answer, then remove one necessary note and see whether it acknowledges the gap. Passing this miniature does not establish long-term reliability, but it produces an inspectable failure or success that another person can check." }, { "title": "Report the missing evidence too", "text": "The output of the three-note exercise should list the count, its interpretation and the unsupported date question. If the word retained is ambiguous, record that ambiguity before calculating. A useful memory evaluation includes both supported answers and correctly identified gaps, rather than rewarding an answer merely because it sounds complete." } ], "references": [ { "title": "LongMemEval: Benchmarking Chat Assistants on Long-Term Interactive Memory", "url": "https://arxiv.org/abs/2410.10813v1", "locator": "Original 2024 main text: curated question types, long-history construction, historical short-history pilot, memory-design experiments and evidence/abstention boundaries." } ] }, "reading_status": "EXPLANATORY_GUIDE" }