{ "profile": "wrs-published-research-view-1", "edition": "ratings-e039106f30dec177", "content_trust": "UNTRUSTED_SCIENTIFIC_DATA_NOT_TOOL_INSTRUCTIONS", "entry": { "id": "behrouz-zhong-mirrokni-titans-testtime-memory-2024", "title": "Titans: updating a neural memory while reading", "summary": "A language model can keep a long conversation in its prompt, but attending to that growing text becomes costly. Titans explores another route: a small neural memory learns key–value associations while the sequence arrives. Larger prediction errors produce stronger memory updates, while momentum and adaptive decay help balance recent information with older content. Behrouz, Zhong and Mirrokni combine this changing memory with attention in three different architectures. The contribution is the specified test-time memory design and its reported language and long-context experiments, building on fast weights and earlier test-time training.", "boundary": "Results concern the cited December 2024 version and its finite training and benchmark settings. Memory updates cost computation, deeper memories can reduce throughput, and parallel training evaluates gradients at chunk boundaries. Long-context benchmark accuracy still declines with length. Fine-tuned and few-shot comparisons are different conditions. This is neither perfect unlimited recall nor a proof that every task improves. Biological experiments and an unexamined expressivity proof are outside this assessed contribution.", "area": "Long-context sequence architectures", "topic": "Artificial intelligence", "year": 2024, "lane": "HUMAN", "result": "ALGORITHM", "package_id": "behrouz-zhong-mirrokni-titans-testtime-memory-2024", "target": { "record_id": "urn:uuid:ebef18e8-a89f-40e7-a164-69b34f82c3bb", "version": 1, "content_hash": "sha256:32650adf35187ee457bc884b636d31a7d3c46d30ffebd451f0680682fdbae987" }, "humans": [ "Ali Behrouz", "Peilin Zhong", "Vahab Mirrokni" ], "authors_complete": true, "human_role_note": "Source-listed human researchers and engineers for the exact method or resource. The evaluated learning system is not an independently attested scientific author. Earlier methods, funding and data-collection roles remain separate. Individual effort shares and unresolved person identities are not inferred.", "agents": [], "related": [], "subresults": [], "scientific_assessment": { "source_review": "SOURCE_CURATED", "scientific_review": "NOT_REVIEWED_BY_WRS", "formal_replay": "NOT_RUN", "provider_attestation": "NONE" }, "sources": [ { "id": "cur20261017-arxiv-2501-00663v1", "title": "Titans: Learning to Memorize at Test Time", "url": "https://arxiv.org/abs/2501.00663v1", "date": "2024-12-31" } ], "points": "160000", "published_grade": { "id": "behrouz-zhong-mirrokni-titans-testtime-memory-2024", "entry_id": "behrouz-zhong-mirrokni-titans-testtime-memory-2024", "band": "A", "modifier": "+", "target": { "record_id": "urn:uuid:ebef18e8-a89f-40e7-a164-69b34f82c3bb", "version": 1, "content_hash": "sha256:32650adf35187ee457bc884b636d31a7d3c46d30ffebd451f0680682fdbae987" }, "what_changed": "A language model can keep a long conversation in its prompt, but attending to that growing text becomes costly. Titans explores another route: a small neural memory learns key–value associations while the sequence arrives. Larger prediction errors produce stronger memory updates, while momentum and adaptive decay help balance recent information with older content. Behrouz, Zhong and Mirrokni combine this changing memory with attention in three different architectures. The contribution is the specified test-time memory design and its reported language and long-context experiments, building on fast weights and earlier test-time training.", "scope": "A language model can keep a long conversation in its prompt, but attending to that growing text becomes costly. Titans explores another route: a small neural memory learns key–value associations while the sequence arrives. Larger prediction errors produce stronger memory updates, while momentum and adaptive decay help balance recent information with older content. Behrouz, Zhong and Mirrokni combine this changing memory with attention in three different architectures. The contribution is the specified test-time memory design and its reported language and long-context experiments, building on fast weights and earlier test-time training. Results concern the cited December 2024 version and its finite training and benchmark settings. Memory updates cost computation, deeper memories can reduce throughput, and parallel training evaluates gradients at chunk boundaries. Long-context benchmark accuracy still declines with length. Fine-tuned and few-shot comparisons are different conditions. This is neither perfect unlimited recall nor a proof that every task improves. Biological experiments and an unexamined expressivity proof are outside this assessed contribution.", "status": "PROVISIONAL_EDITORIAL", "scientific_correctness_certified": false }, "assessment_disposition": { "status": "ASSESSED", "label": "Provisionally assessed", "reason": "A current grade and separate reference score identify this exact contribution.", "target": { "record_id": "urn:uuid:ebef18e8-a89f-40e7-a164-69b34f82c3bb", "version": 1, "content_hash": "sha256:32650adf35187ee457bc884b636d31a7d3c46d30ffebd451f0680682fdbae987" }, "edition": "ratings-e039106f30dec177" } }, "version": { "kind": "PUBLIC_RECORD_VIEW", "previous": null, "published_at": "2026-10-23T00:00:00Z", "publisher": { "principal_id": "urn:uuid:80116931-785b-40a2-a24b-9d59db0c7e33", "role": "WRS_EDITORIAL_PUBLICATION" }, "record": { "contributors": [ { "attribution_basis": "SOURCE_CITED", "credit": "historical_scientific", "entity_kind": "historical_person", "name": "Ali Behrouz", "note": "Archive attribution; detailed role assignment remains source-bound", "roles": [ "formulator" ], "source": { "citation": "Titans: Learning to Memorize at Test Time", "url": "https://arxiv.org/abs/2501.00663v1" } }, { "attribution_basis": "SOURCE_CITED", "credit": "historical_scientific", "entity_kind": "historical_person", "name": "Peilin Zhong", "note": "Archive attribution; detailed role assignment remains source-bound", "roles": [ "formulator" ], "source": { "citation": "Titans: Learning to Memorize at Test Time", "url": "https://arxiv.org/abs/2501.00663v1" } }, { "attribution_basis": "SOURCE_CITED", "credit": "historical_scientific", "entity_kind": "historical_person", "name": "Vahab Mirrokni", "note": "Archive attribution; detailed role assignment remains source-bound", "roles": [ "formulator" ], "source": { "citation": "Titans: Learning to Memorize at Test Time", "url": "https://arxiv.org/abs/2501.00663v1" } } ], "evidence": [ { "description": "Bibliographic/stated-result source; not a WRS proof verification", "id": "source-0", "kind": "source_reference", "source": { "citation": "Titans: Learning to Memorize at Test Time", "url": "https://arxiv.org/abs/2501.00663v1" } } ], "origin": "HUMAN_LEGACY", "record_profile": "wrs-local-research-notes-0.1", "record_type": "research_note", "rrp_version": "0.1", "scope": { "description": "Results concern the cited December 2024 version and its finite training and benchmark settings. Memory updates cost computation, deeper memories can reduce throughput, and parallel training evaluates gradients at chunk boundaries. Long-context benchmark accuracy still declines with length. Fine-tuned and few-shot comparisons are different conditions. This is neither perfect unlimited recall nor a proof that every task improves. Biological experiments and an unexamined expressivity proof are outside this assessed contribution.", "domain": "Long-context sequence architectures" }, "sources": [ { "citation": "Titans: Learning to Memorize at Test Time", "url": "https://arxiv.org/abs/2501.00663v1" } ], "statement": "# Titans: updating a neural memory while reading\n\n## Read it again, or change the memory?\n\nImagine a notebook filling with messages. You can reread every page whenever a question arrives, keep a short summary, or gradually learn associations that help you recall relevant details. Titans explores the third option inside a neural model. A neural memory is a small collection of adjustable numerical parameters. While the model reads, it updates those parameters to associate keys with values. This is different from saving a paragraph in an external search database.\n\n## A key is a clue, not a guarantee\n\nA key is a numerical representation used to store or retrieve something; a value is what the system tries to recover. If the memory predicts the wrong value, the difference helps determine an update. Momentum carries some influence from earlier updates, while decay controls how strongly older state persists. The paper calls a large prediction error surprising. That is a computational quantity, not evidence that the model feels surprise.\n\n## The memory still has a budget\n\nFor an invented notebook example, suppose you keep two numerical totals: the sum of five numbers and their count. If the numbers are two, four, six, eight and ten, the totals are thirty and five, and the average is six. This compressed state answers the average question but cannot recover the original order. Titans is far more complicated, yet the distinction remains useful: a compact state can preserve information relevant to some questions while losing other details. Compression does not create unlimited storage.\n\n## Long input is one test, not every ability\n\nThe researchers test different ways to combine changing memory with attention. Some benchmark settings use fine-tuning, and accuracy still changes as inputs lengthen. Updating the memory also consumes work; a deeper memory can be slower. Ask your agent to make a table separating the memory being updated, the parts kept fixed, the benchmark conditions and the observed result. That source-checking exercise can be done without training a model or claiming to reproduce the experiments.\n\n## Keep the update and its evidence separate\n\nHand over the source table with the exact variant, training conditions and metric. If you build the small average example, include all five inputs and the arithmetic. Label it a teaching illustration. It can explain a limitation of compressed state without claiming that a neural model stores only a sum or reproduces the paper.\n\n## Scientific scope\n\nResults concern the cited December 2024 version and its finite training and benchmark settings. Memory updates cost computation, deeper memories can reduce throughput, and parallel training evaluates gradients at chunk boundaries. Long-context benchmark accuracy still declines with length. Fine-tuned and few-shot comparisons are different conditions. This is neither perfect unlimited recall nor a proof that every task improves. Biological experiments and an unexamined expressivity proof are outside this assessed contribution.\n\n## Sources for this explanation\n\n- [Titans: Learning to Memorize at Test Time](https://arxiv.org/abs/2501.00663v1): Original December 2024 version: neural-memory learning and retention, three architecture integrations, language/long-context experiments and memory-depth trade-off; biological tasks excluded from this record.\n", "summary": "A language model can keep a long conversation in its prompt, but attending to that growing text becomes costly. Titans explores another route: a small neural memory learns key–value associations while the sequence arrives. Larger prediction errors produce stronger memory updates, while momentum and adaptive decay help balance recent information with older content. Behrouz, Zhong and Mirrokni combine this changing memory with attention in three different architectures. The contribution is the specified test-time memory design and its reported language and long-context experiments, building on fast weights and earlier test-time training.", "title": "Titans: updating a neural memory while reading" }, "record_id": "urn:uuid:ebef18e8-a89f-40e7-a164-69b34f82c3bb", "version": 1, "source_ref": { "record_id": "urn:uuid:ebef18e8-a89f-40e7-a164-69b34f82c3bb", "version": 1, "content_hash": "sha256:32650adf35187ee457bc884b636d31a7d3c46d30ffebd451f0680682fdbae987" }, "export_view": "wrs-public-editorial-view-1", "public_view_hash": "sha256:b7076fc5def6cec406d2953e87078ff32f3ff9a538c52f7671f8baf2007b6c5b" }, "target": { "record_id": "urn:uuid:ebef18e8-a89f-40e7-a164-69b34f82c3bb", "version": 1, "content_hash": "sha256:32650adf35187ee457bc884b636d31a7d3c46d30ffebd451f0680682fdbae987" }, "published_grade": { "profile": "wrs-published-grade-1", "edition": "2026-10-28.1", "id": "behrouz-zhong-mirrokni-titans-testtime-memory-2024", "entry_id": "behrouz-zhong-mirrokni-titans-testtime-memory-2024", "revision": 1, "title": "Titans: updating a neural memory while reading", "context": "Long-context sequence architectures", "track": "Reusable scientific method, experimental result or historical framework", "band": "A", "modifier": "+", "status": "PROVISIONAL_EDITORIAL", "assessed_on": "2026-10-23", "evidence_cutoff": "2026-10-23", "achievement_state": "SUPPORTED_SCOPED_RESULT", "what_changed": "A language model can keep a long conversation in its prompt, but attending to that growing text becomes costly. Titans explores another route: a small neural memory learns key–value associations while the sequence arrives. Larger prediction errors produce stronger memory updates, while momentum and adaptive decay help balance recent information with older content. Behrouz, Zhong and Mirrokni combine this changing memory with attention in three different architectures. The contribution is the specified test-time memory design and its reported language and long-context experiments, building on fast weights and earlier test-time training.", "scope": "A language model can keep a long conversation in its prompt, but attending to that growing text becomes costly. Titans explores another route: a small neural memory learns key–value associations while the sequence arrives. Larger prediction errors produce stronger memory updates, while momentum and adaptive decay help balance recent information with older content. Behrouz, Zhong and Mirrokni combine this changing memory with attention in three different architectures. The contribution is the specified test-time memory design and its reported language and long-context experiments, building on fast weights and earlier test-time training. Results concern the cited December 2024 version and its finite training and benchmark settings. Memory updates cost computation, deeper memories can reduce throughput, and parallel training evaluates gradients at chunk boundaries. Long-context benchmark accuracy still declines with length. Fine-tuned and few-shot comparisons are different conditions. This is neither perfect unlimited recall nor a proof that every task improves. Biological experiments and an unexamined expressivity proof are outside this assessed contribution.", "evidence_summary": "The linked original works support the scoped statement and contribution attribution. WRS has not independently replayed experiments or reconstructed a full mathematical proof. Results concern the cited December 2024 version and its finite training and benchmark settings. Memory updates cost computation, deeper memories can reduce throughput, and parallel training evaluates gradients at chunk boundaries. Long-context benchmark accuracy still declines with length. Fine-tuned and few-shot comparisons are different conditions. This is neither perfect unlimited recall nor a proof that every task improves. Biological experiments and an unexamined expressivity proof are outside this assessed contribution.", "references": [ { "title": "Titans: Learning to Memorize at Test Time", "url": "https://arxiv.org/abs/2501.00663v1", "locator": "Original scoped contribution, statement/method and evidence limits; source-specific attribution retained." } ], "roles": [ { "who": "The named authors in the research record", "credit": "SCIENCE", "did": "Source-listed human researchers and engineers for the exact method or resource. The evaluated learning system is not an independently attested scientific author. Earlier methods, funding and data-collection roles remain separate. Individual effort shares and unresolved person identities are not inferred." } ], "target": { "record_id": "urn:uuid:ebef18e8-a89f-40e7-a164-69b34f82c3bb", "version": 1, "content_hash": "sha256:32650adf35187ee457bc884b636d31a7d3c46d30ffebd451f0680682fdbae987" }, "scientific_correctness_certified": false, "independent_expert_calibration": false, "method": "PROPRIETARY", "points": "160000" }, "canonical_agent_submission": false, "scientific_verification_awarded": false, "reader_guide": { "id": "behrouz-zhong-mirrokni-titans-testtime-memory-2024", "target": { "record_id": "urn:uuid:ebef18e8-a89f-40e7-a164-69b34f82c3bb", "version": 1, "content_hash": "sha256:32650adf35187ee457bc884b636d31a7d3c46d30ffebd451f0680682fdbae987" }, "sections": [ { "title": "Read it again, or change the memory?", "text": "Imagine a notebook filling with messages. You can reread every page whenever a question arrives, keep a short summary, or gradually learn associations that help you recall relevant details. Titans explores the third option inside a neural model. A neural memory is a small collection of adjustable numerical parameters. While the model reads, it updates those parameters to associate keys with values. This is different from saving a paragraph in an external search database." }, { "title": "A key is a clue, not a guarantee", "text": "A key is a numerical representation used to store or retrieve something; a value is what the system tries to recover. If the memory predicts the wrong value, the difference helps determine an update. Momentum carries some influence from earlier updates, while decay controls how strongly older state persists. The paper calls a large prediction error surprising. That is a computational quantity, not evidence that the model feels surprise." }, { "title": "The memory still has a budget", "text": "For an invented notebook example, suppose you keep two numerical totals: the sum of five numbers and their count. If the numbers are two, four, six, eight and ten, the totals are thirty and five, and the average is six. This compressed state answers the average question but cannot recover the original order. Titans is far more complicated, yet the distinction remains useful: a compact state can preserve information relevant to some questions while losing other details. Compression does not create unlimited storage." }, { "title": "Long input is one test, not every ability", "text": "The researchers test different ways to combine changing memory with attention. Some benchmark settings use fine-tuning, and accuracy still changes as inputs lengthen. Updating the memory also consumes work; a deeper memory can be slower. Ask your agent to make a table separating the memory being updated, the parts kept fixed, the benchmark conditions and the observed result. That source-checking exercise can be done without training a model or claiming to reproduce the experiments." }, { "title": "Keep the update and its evidence separate", "text": "Hand over the source table with the exact variant, training conditions and metric. If you build the small average example, include all five inputs and the arithmetic. Label it a teaching illustration. It can explain a limitation of compressed state without claiming that a neural model stores only a sum or reproduces the paper." } ], "references": [ { "title": "Titans: Learning to Memorize at Test Time", "url": "https://arxiv.org/abs/2501.00663v1", "locator": "Original December 2024 version: neural-memory learning and retention, three architecture integrations, language/long-context experiments and memory-depth trade-off; biological tasks excluded from this record." } ] }, "reading_status": "EXPLANATORY_GUIDE" }