Skip to content

Self-evolving Context Database for AI Agents. Unify Agent Memory, Knowledge RAG and Skills.

fix(session): retry memory extraction after safe read failures (#6301)

Wenxi Zhuaa47c878f4b6
diff --git a/openviking/session/compressor_v3.py b/openviking/session/compressor_v3.py
index 6433c220d..698f8c200 100644
--- a/openviking/session/compressor_v3.py
+++ b/openviking/session/compressor_v3.py
@@ -40,7 +40,11 @@ from openviking.session.memory.dataclass import (
)
from openviking.session.memory.memory_isolation_handler import MemoryIsolationHandler
from openviking.session.memory.memory_type_registry import get_default_registry
-from openviking.session.memory.memory_updater import ExtractContext, write_stored_links
+from openviking.session.memory.memory_updater import (
+ ExtractContext,
+ MemoryUpdateError,
+ write_stored_links,
+)
from openviking.session.memory.session_extract_context_provider import (
SessionExtractContextProvider,
)
@@ -782,6 +786,8 @@ class SessionCompressorV3:
result = update_result.apply_result
patch_operations = update_result.operations
_report_extraction_telemetry(result, patch_operations)
+ if strict_extract_errors and result.errors:
+ raise MemoryUpdateError(result, has_link_updates=bool(patch_operations.resolved_links))
memory_diff = None
if archive_uri and viking_fs and result is not None:
diff --git a/openviking/session/memory/memory_updater.py b/openviking/session/memory/memory_updater.py
index 389184a59..9aac1d6b1 100644
--- a/openviking/session/memory/memory_updater.py
+++ b/openviking/session/memory/memory_updater.py
@@ -9,6 +9,7 @@ to the storage system.
from __future__ import annotations
+import errno
import hashlib
import re
from dataclasses import dataclass
@@ -20,6 +21,7 @@ if TYPE_CHECKING:
from openviking.core.context import ContextLevel
from openviking.message import Message
from openviking.message.part import TextPart
+from openviking.pyagfs.exceptions import AGFSConnectionError, AGFSIoError, AGFSTimeoutError
from openviking.server.identity import RequestContext
from openviking.session.memory.dataclass import (
MemoryFile,
@@ -757,6 +759,43 @@ class MemoryUpdateResult:
)
+class MemoryReadError(Exception):
+ """A fresh upsert read failed before this target could be written."""
+
+ def __init__(self, uri: str, cause: Exception, *, write_started: bool = False):
+ super().__init__(f"Failed to refresh memory {uri}: {cause}")
+ self.retryable = not write_started and (
+ isinstance(
+ cause,
+ (TimeoutError, ConnectionError, AGFSConnectionError, AGFSTimeoutError, AGFSIoError),
+ )
+ or (
+ isinstance(cause, OSError)
+ and cause.errno in {errno.EAGAIN, errno.EBUSY, errno.EINTR, errno.EIO}
+ )
+ )
+
+
+class MemoryUpdateError(Exception):
+ """Report apply failures without treating a partial write as safe to replay."""
+
+ def __init__(self, result: MemoryUpdateResult, *, has_link_updates: bool = False):
+ super().__init__(
+ f"Memory update failed ({result.summary()}): "
+ + "; ".join(f"{uri}: {error}" for uri, error in result.errors)
+ )
+ self.retryable = (
+ bool(result.errors)
+ and not (result.written_uris or result.edited_uris or result.deleted_uris)
+ # Link writes are best-effort and absent from the URI result lists.
+ and not has_link_updates
+ and all(
+ isinstance(error, MemoryReadError) and error.retryable
+ for _uri, error in result.errors
+ )
+ )
+
+
def _same_batch_delete_conflict_key(uri: str) -> str:
"""Return a conservative key for detecting same-batch upsert/delete URI conflicts.
@@ -1378,6 +1417,7 @@ class MemoryUpdater:
memory_type = resolved_op.memory_type
schema = self._registry.get(memory_type)
+ write_started = False
# Process each URI independently
for uri in resolved_op.uris:
source_content = resolved_op.old_memory_file_content
@@ -1401,8 +1441,10 @@ class MemoryUpdater:
except (NotFoundError, FileNotFoundError):
# File doesn't exist yet, that's okay
pass
- # Any other read error propagates: the prefetched snapshot may be older
- # than the file, so writing a result built on it could lose a newer update.
+ except Exception as exc:
+ # Preserve the read stage so callers can regenerate from fresh
+ # inputs, but never replay an operation that already wrote a URI.
+ raise MemoryReadError(uri, exc, write_started=write_started) from exc
if is_uri_migration:
try:
source_raw = await viking_fs.read_file(source_content.uri, ctx=ctx)
@@ -1526,6 +1568,7 @@ class MemoryUpdater:
),
extract_context=extract_context,
)
+ write_started = True
await viking_fs.write_file(
uri,
new_full_content,
diff --git a/openviking/session/session.py b/openviking/session/session.py
index b8eb62284..c1c411424 100644
--- a/openviking/session/session.py
+++ b/openviking/session/session.py
@@ -59,6 +59,7 @@ from openviking.session.extraction_batch import (
resolve_extraction_batch_limits,
)
from openviking.session.memory.constants import AGENT_EVOLUTION_MEMORY_TYPES
+from openviking.session.memory.memory_updater import MemoryUpdateError
from openviking.session.memory.utils.language import resolve_output_language_from_conversation
from openviking.session.memory_policy import MemoryPolicy
from openviking.session.retention import (
@@ -2273,7 +2274,11 @@ class Session:
max_retries=_MEMORY_EXTRACTION_MAX_RETRIES,
base_delay=_MEMORY_EXTRACTION_RETRY_BASE_DELAY_SECONDS,
max_delay=_MEMORY_EXTRACTION_RETRY_MAX_DELAY_SECONDS,
- is_retryable=is_retryable_api_error,
+ is_retryable=lambda error: (
+ error.retryable
+ if isinstance(error, MemoryUpdateError)
+ else is_retryable_api_error(error)
+ ),
logger=logger,
operation_name=operation_name,
)
diff --git a/tests/session/memory/test_memory_updater.py b/tests/session/memory/test_memory_updater.py
index 636e2a990..205133539 100644
--- a/tests/session/memory/test_memory_updater.py
+++ b/tests/session/memory/test_memory_updater.py
@@ -4,6 +4,7 @@
Tests for MemoryUpdater.
"""
+import errno
from copy import deepcopy
from unittest.mock import AsyncMock, MagicMock, patch
@@ -12,6 +13,7 @@ import pytest
from openviking.message import Message
from openviking.message.part import TextPart, ToolPart
from openviking.prompts.manager import PromptManager
+from openviking.pyagfs.exceptions import AGFSIoError
from openviking.server.identity import RequestContext, Role
from openviking.session.memory.dataclass import (
MemoryField,
@@ -27,6 +29,7 @@ from openviking.session.memory.dataclass import (
from openviking.session.memory.memory_type_registry import MemoryTypeRegistry
from openviking.session.memory.memory_updater import (
ExtractContext,
+ MemoryReadError,
MemoryUpdater,
MemoryUpdateResult,
)
@@ -1316,16 +1319,28 @@ class TestMemoryUpdater:
updater._apply_delete.assert_not_awaited()
@pytest.mark.asyncio
- async def test_apply_operations_preserves_memories_when_fresh_reads_fail(self):
+ @pytest.mark.parametrize(
+ ("read_error", "retryable", "write_first"),
+ [
+ (OSError(errno.EIO, "transient read failure"), True, False),
+ (TimeoutError("read timeout"), True, False),
+ (AGFSIoError("storage I/O failure"), True, False),
+ (PermissionError("read denied"), False, False),
+ (OSError("unclassified read failure"), False, False),
+ (TimeoutError("read after earlier write"), False, True),
+ ],
+ )
+ async def test_apply_operations_preserves_memories_when_fresh_reads_fail(
+ self, read_error, retryable, write_first
+ ):
source_uri = "viking://user/u/memories/entities/person/阿珍.md"
target_uri = "viking://user/u/memories/entities/person/陈静娴.md"
update_uri = "viking://user/u/memories/entities/person/小王.md"
- read_error = OSError("read failed")
old_file = MemoryFile(uri=source_uri, memory_type="entities", content="source")
mock_viking_fs = MagicMock()
- mock_viking_fs.read_file = AsyncMock(
- side_effect=[NotFoundError(target_uri, "file"), read_error, read_error]
- )
+ reads = [NotFoundError(target_uri, "file"), read_error]
+ reads.extend([FileNotFoundError(), read_error] if write_first else [read_error])
+ mock_viking_fs.read_file = AsyncMock(side_effect=reads)
mock_viking_fs.write_file = AsyncMock()
registry = MagicMock()
registry.get.return_value = MemoryTypeSchema(memory_type="entities")
@@ -1344,12 +1359,14 @@ class TestMemoryUpdater:
uris=[target_uri],
),
ResolvedOperation(
- old_memory_file_content=MemoryFile(
+ old_memory_file_content=None
+ if write_first
+ else MemoryFile(
uri=update_uri, memory_type="entities", content="stale snapshot"
),
memory_fields={"content": "updated"},
memory_type="entities",
- uris=[update_uri],
+ uris=[update_uri, update_uri + ".second"] if write_first else [update_uri],
),
],
delete_file_contents=[],
@@ -1358,13 +1375,18 @@ class TestMemoryUpdater:
result = await updater.apply_operations(operations, MagicMock())
- assert {uri for uri, error in result.errors if error is read_error} == {
+ read_errors = {
+ uri: error for uri, error in result.errors if isinstance(error, MemoryReadError)
+ }
+ assert set(read_errors) == {
target_uri,
- update_uri,
+ *operations.upsert_operations[1].uris,
}
+ assert all(error.__cause__ is read_error for error in read_errors.values())
+ assert read_errors[update_uri].retryable is retryable
assert result.written_uris == []
assert result.edited_uris == []
- mock_viking_fs.write_file.assert_not_awaited()
+ assert mock_viking_fs.write_file.await_count == int(write_first)
assert result.deleted_uris == []
updater._apply_delete.assert_not_awaited()
diff --git a/tests/session/test_compressor_v3.py b/tests/session/test_compressor_v3.py
index 7f8da7ca5..81f5ff378 100644
--- a/tests/session/test_compressor_v3.py
+++ b/tests/session/test_compressor_v3.py
@@ -29,7 +29,11 @@ from openviking.session.memory.dataclass import (
SkippedMemoryOperation,
StoredLink,
)
-from openviking.session.memory.memory_updater import MemoryUpdateResult
+from openviking.session.memory.memory_updater import (
+ MemoryReadError,
+ MemoryUpdateError,
+ MemoryUpdateResult,
+)
from openviking.session.memory.utils.memory_file_utils import MemoryFileUtils
from openviking.session.train import (
Case,
@@ -795,9 +799,12 @@ async def test_train_from_extracted_multiple_case_memories_analyzes_bound_rollou
@pytest.mark.asyncio
-async def test_v3_extract_uses_patch_merge_without_directory_lock(monkeypatch):
[email protected]("apply_error", [False, True, "links"])
+async def test_v3_extract_uses_patch_merge_without_directory_lock(monkeypatch, apply_error):
applied_operations = []
trained_cases = []
+ uri = _case_operation().uris[0]
+ link = StoredLink(from_uri=uri, to_uri=uri + ".other.md", link_type="related_to")
class DummyRegistry:
async def initialize_memory_files(self, ctx, allowed_memory_types=None):
@@ -811,6 +818,7 @@ async def test_v3_extract_uses_patch_merge_without_directory_lock(monkeypatch):
upsert_operations=[_case_operation()],
delete_file_contents=[],
errors=[],
+ resolved_links=[link] if apply_error == "links" else [],
),
[],
)
@@ -819,10 +827,17 @@ async def test_v3_extract_uses_patch_merge_without_directory_lock(monkeypatch):
async def submit(self, request):
applied_operations.append(request.operations)
result = MemoryUpdateResult()
- result.add_written(_case_operation().uris[0])
+ if apply_error:
+ result.add_error(uri, MemoryReadError(uri, TimeoutError("read timeout")))
+ else:
+ result.add_written(uri)
return SimpleNamespace(operations=request.operations, apply_result=result)
- compressor = SessionCompressorV3(vikingdb=None)
+ compressor = SessionCompressorV3(
+ vikingdb=None,
+ rollout_analyzer=SimpleNamespace(),
+ vlm_resolver=SimpleNamespace(get_vlm=AsyncMock(return_value=SimpleNamespace())),
+ )
compressor._get_or_create_react = lambda **kwargs: DummyOrchestrator()
async def fake_train_from_extracted_cases(**kwargs):
@@ -839,16 +854,28 @@ async def test_v3_extract_uses_patch_merge_without_directory_lock(monkeypatch):
"openviking.session.compressor_v3.get_default_registry",
lambda: DummyRegistry(),
)
+ monkeypatch.setattr(
+ "openviking.session.memory.account_templates.resolve_account_memory_registry",
+ AsyncMock(return_value=DummyRegistry()),
+ )
monkeypatch.setattr(
"openviking.session.compressor_v3.get_streaming_memory_updater",
AsyncMock(return_value=FakeStreamingUpdater()),
)
- contexts = await compressor.extract_long_term_memories(
+ extraction = compressor.extract_long_term_memories(
messages=_messages(),
ctx=_ctx(),
allowed_memory_types={"cases", "profile", "trajectories", "experiences"},
- )
+ strict_extract_errors=True,
+ )
+ if apply_error:
+ with pytest.raises(MemoryUpdateError) as raised:
+ await extraction
+ assert raised.value.retryable is (apply_error != "links")
+ assert trained_cases == []
+ return
+ contexts = await extraction
assert len(applied_operations) == 1
assert applied_operations[0].upsert_operations[0].memory_type == "cases"
diff --git a/tests/session/test_session_commit.py b/tests/session/test_session_commit.py
index a9b90cc71..a3da165ce 100644
--- a/tests/session/test_session_commit.py
+++ b/tests/session/test_session_commit.py
@@ -8,6 +8,8 @@ import json
from types import SimpleNamespace
from unittest.mock import AsyncMock, MagicMock
+import pytest
+
from openviking.message import TextPart
from openviking.scheduling.expiry import (
CallbackRequest,
@@ -19,6 +21,11 @@ from openviking.server.identity import RequestContext
from openviking.service.core import OpenVikingService
from openviking.service.task_tracker import get_task_tracker
from openviking.session import Session
+from openviking.session.memory.memory_updater import (
+ MemoryReadError,
+ MemoryUpdateError,
+ MemoryUpdateResult,
+)
from openviking.utils.time_utils import parse_iso_datetime
@@ -44,6 +51,35 @@ async def _marker_exists(session, archive_uri: str, name: str) -> bool:
class TestCommit:
"""Test commit"""
+ @pytest.mark.parametrize(
+ ("failure", "attempts"),
+ [("transient", 2), ("exhausted", 4), ("permission", 1), ("partial", 1), ("write", 1)],
+ )
+ async def test_memory_apply_failure_retries_only_before_any_write(
+ self, session_with_messages, monkeypatch, failure, attempts
+ ):
+ import openviking.session.session as session_module
+
+ monkeypatch.setattr(session_module, "_MEMORY_EXTRACTION_RETRY_BASE_DELAY_SECONDS", 0)
+ session = session_with_messages
+ uri = "viking://user/default/memories/preferences/style.md"
+ cause = PermissionError("denied") if failure == "permission" else TimeoutError("timeout")
+ error = cause if failure == "write" else MemoryReadError(uri, cause)
+ result = MemoryUpdateResult()
+ result.add_error(uri, error)
+ if failure == "partial":
+ result.add_written("viking://user/default/memories/preferences/other.md")
+ error = MemoryUpdateError(result)
+ extract = AsyncMock(side_effect=[error, []] if failure == "transient" else error)
+ session._session_compressor.extract_long_term_memories = extract
+ committed = await session.commit_async()
+ task = await _wait_for_task(committed["task_id"])
+ assert extract.await_count == attempts
+ succeeded = failure == "transient"
+ assert task["status"] == ("completed" if succeeded else "failed")
+ meta = await session._archives.read_meta(committed["archive_uri"])
+ assert bool(meta.get("completed_memory_steps", {}).get("long_term")) is succeeded
+
async def test_commit_preserves_unicode_separators_and_accepts_later_messages(
self, wm_session_with_messages: Session
):
diff --git a/tests/session/train/test_train_components.py b/tests/session/train/test_train_components.py
index f12858b70..68307fcae 100644
--- a/tests/session/train/test_train_components.py
+++ b/tests/session/train/test_train_components.py
@@ -90,6 +90,8 @@ class FakeVikingFS:
]
async def read_file(self, uri: str, ctx=None):
+ if uri not in self.files:
+ raise FileNotFoundError(uri)
return self.files[uri]
async def write_file(self, uri: str, content: str, ctx=None, lease_ref=None):
diff --git a/tests/session/train/test_trajectory_analyzer_component.py b/tests/session/train/test_trajectory_analyzer_component.py
index fe9e13842..7deb471e2 100644
--- a/tests/session/train/test_trajectory_analyzer_component.py
+++ b/tests/session/train/test_trajectory_analyzer_component.py
@@ -131,6 +131,8 @@ class FakeVikingFS:
self.writes = []
async def read_file(self, uri, ctx=None):
+ if uri not in self.files:
+ raise FileNotFoundError(uri)
return self.files[uri]
async def write_file(self, uri, content, ctx=None, lock_handle=None, lease_ref=None):