Self-evolving Context Database for AI Agents. Unify Agent Memory, Knowledge RAG and Skills.
Python agent-contextagent-memoryagent-pluginsagentic-ragcontext-databasedsh-pluginknowledge-basememorymemory-managementself-evolvingvector-database
fix(session): resume commits after restart (#3254)
Qin Haojie4a54dbeccfcf
diff --git a/openviking/service/core.py b/openviking/service/core.pyindex 0f7bd1147..7a35a14b8 100644--- a/openviking/service/core.py+++ b/openviking/service/core.py@@ -6,6 +6,7 @@ OpenViking Service Core.Main service class that composes all sub-services and manages infrastructure lifecycle."""+import asyncioimport osfrom typing import TYPE_CHECKING, Any, Optional@@ -28,6 +29,7 @@ from openviking.storage import VikingDBManagerfrom openviking.storage.collection_schemas import init_context_collectionfrom openviking.storage.index_consistency import check_index_consistencyfrom openviking.storage.queuefs.queue_manager import QueueManager, init_queue_manager+from openviking.storage.queuefs.session_commit_processor import SessionCommitProcessorfrom openviking.storage.queuefs.understanding_parse_processor import UnderstandingParseProcessorfrom openviking.storage.transaction import LockManager, init_lock_managerfrom openviking.storage.viking_fs import VikingFS, init_viking_fs@@ -423,6 +425,14 @@ class OpenVikingService:dequeue_handler=external_parse_processor,allow_create=True,)+ self._queue_manager.get_queue(+ self._queue_manager.SESSION_COMMIT,+ dequeue_handler=SessionCommitProcessor(+ self._session_service,+ asyncio.get_running_loop(),+ ),+ allow_create=True,+ )self._queue_manager.start()logger.info("QueueManager workers started")@@ -446,6 +456,11 @@ class OpenVikingService:self._watch_scheduler = Nonelogger.info("WatchScheduler stopped")+ if self._queue_manager:+ await asyncio.to_thread(self._queue_manager.stop)+ self._queue_manager = None+ logger.info("Queue manager stopped")+if self._lock_manager:await self._lock_manager.stop()self._lock_manager = None@@ -453,11 +468,6 @@ class OpenVikingService:if self._vikingdb_manager:self._vikingdb_manager.mark_closing()- if self._queue_manager:- self._queue_manager.stop()- self._queue_manager = None- logger.info("Queue manager stopped")-if self._vikingdb_manager:await self._vikingdb_manager.close()self._vikingdb_manager = Nonediff --git a/openviking/service/task_tracker.py b/openviking/service/task_tracker.pyindex cc3c53f48..8e235184e 100644--- a/openviking/service/task_tracker.py+++ b/openviking/service/task_tracker.py@@ -246,17 +246,30 @@ class TaskTracker:*,account_id: str,user_id: str,+ task_id: Optional[str] = None,) -> TaskRecord:"""Register a new pending task. Returns a snapshot copy."""self._validate_owner(account_id, user_id)task = TaskRecord(- task_id=str(uuid4()),+ task_id=task_id or str(uuid4()),task_type=task_type,resource_id=resource_id,account_id=account_id,user_id=user_id,)async with self._async_lock:+ if task_id:+ with self._lock:+ existing = self._tasks.get(task_id)+ if existing is not None:+ if not self._matches_owner(existing, account_id, user_id):+ raise ValueError(f"Task ID already belongs to another owner: {task_id}")+ return self._copy(existing)+ existing = await self._load_from_store(task_id, account_id, user_id)+ if existing is not None:+ with self._lock:+ self._tasks[task_id] = existing+ return self._copy(existing)await self._store.create(task)with self._lock:self._tasks[task.task_id] = taskdiff --git a/openviking/session/session.py b/openviking/session/session.pyindex aa63f47dd..61fbadc2d 100644--- a/openviking/session/session.py+++ b/openviking/session/session.py@@ -45,6 +45,7 @@ from openviking_cli.utils.config import get_openviking_configif TYPE_CHECKING:from openviking.session.compressor_v2 import SessionCompressorV2 as SessionCompressorfrom openviking.storage import VikingDBManager+ from openviking.storage.queuefs.session_commit_msg import SessionCommitMsgfrom openviking.storage.viking_fs import VikingFSlogger = get_logger(__name__)@@ -1089,14 +1090,13 @@ class Session:*,memory_policy: Optional[Dict[str, Any]] = None,) -> Dict[str, Any]:- """Async commit session: archive immediately, extract memories in background.+ """Archive immediately and enqueue restart-safe Phase 2 processing.Phase 1 (Archive prep, path-lock protected): Split messages intoarchive/retain parts, write the retained tail back to messages.jsonl,then persist the archive. Uses a distributed filesystem lockso this works across workers and processes.- Phase 2 (Memory extraction): Always runs in background via- asyncio.create_task().+ Phase 2 (Memory extraction): Runs through the persistent QueueFS queue.Args:keep_recent_count: Number of most-recent messages to keep in the@@ -1108,6 +1108,8 @@ class Session:Returns a task_id for tracking Phase 2 progress."""from openviking.service.task_tracker import get_task_tracker+ from openviking.storage.queuefs import QueueManager, get_queue_manager+ from openviking.storage.queuefs.session_commit_msg import SessionCommitMsgfrom openviking.storage.transaction import LockContext, get_lock_managertrace_id = tracer.get_trace_id()@@ -1230,39 +1232,106 @@ class Session:f"history/archive_{self._compression.compression_index:03d}/")- # Snapshot mutable state for Phase 2+ # Snapshot mutable state for Phase 2.usage_snapshot = self._usage_records.copy()-- # Create TaskRecord for tracking Phase 2- tracker = get_task_tracker()- task = await tracker.create(+ task_id = str(uuid4())+ queue_msg = SessionCommitMsg(+ task_id=task_id,+ session_id=self.session_id,+ session_uri=self._session_uri,+ archive_uri=archive_uri,+ user=self.ctx.user.to_dict(),+ actor_peer_id=self.ctx.actor_peer_id,+ memory_policy=effective_memory_policy,+ usage_uris=list(dict.fromkeys(u.uri for u in usage_snapshot if u.uri)),+ )+ await get_queue_manager().enqueue(QueueManager.SESSION_COMMIT, queue_msg.to_dict())+ await get_task_tracker().create("session_commit",resource_id=self.session_id,account_id=self.ctx.account_id,user_id=self.ctx.user.user_id,- )-- asyncio.create_task(- self._run_memory_extraction(- task_id=task.task_id,- archive_uri=archive_uri,- messages=messages_to_archive,- usage_records=usage_snapshot,- first_message_id=messages_to_archive[0].id if messages_to_archive else "",- last_message_id=messages_to_archive[-1].id if messages_to_archive else "",- memory_policy=effective_memory_policy,- )+ task_id=task_id,)return {"session_id": self.session_id,"status": "accepted",- "task_id": task.task_id,+ "task_id": task_id,"archive_uri": archive_uri,"archived": True,"trace_id": trace_id,}+ async def resume_queued_commit(self, msg: "SessionCommitMsg") -> None:+ """Run one durable Phase 2 job from its archived messages."""+ from openviking.service.task_tracker import get_task_tracker++ tracker = get_task_tracker()+ task = await tracker.create(+ "session_commit",+ resource_id=self.session_id,+ account_id=self.ctx.account_id,+ user_id=self.ctx.user.user_id,+ task_id=msg.task_id,+ )++ try:+ await self._viking_fs.read_file(f"{msg.archive_uri}/.done", ctx=self.ctx)+ except Exception:+ pass+ else:+ if task.status.value == "completed":+ return+ await tracker.complete(+ msg.task_id,+ {"session_id": self.session_id, "archive_uri": msg.archive_uri},+ account_id=self.ctx.account_id,+ user_id=self.ctx.user.user_id,+ )+ return++ try:+ failed = json.loads(+ await self._viking_fs.read_file(f"{msg.archive_uri}/.failed.json", ctx=self.ctx)+ )+ except Exception:+ pass+ else:+ await tracker.fail(+ msg.task_id,+ str(failed.get("error") or "session commit failed"),+ account_id=self.ctx.account_id,+ user_id=self.ctx.user.user_id,+ )+ return++ archive_messages = await self._read_archive_messages(msg.archive_uri)+ if not archive_messages:+ error = "session commit archive has no messages"+ await self._write_failed_marker(+ msg.archive_uri,+ stage="archive_read",+ error=error,+ )+ await tracker.fail(+ msg.task_id,+ error,+ account_id=self.ctx.account_id,+ user_id=self.ctx.user.user_id,+ )+ return++ await self._run_memory_extraction(+ task_id=msg.task_id,+ archive_uri=msg.archive_uri,+ messages=archive_messages,+ usage_records=[Usage(uri=uri, type="context") for uri in msg.usage_uris],+ first_message_id=archive_messages[0].id,+ last_message_id=archive_messages[-1].id,+ memory_policy=msg.memory_policy,+ )+@tracer("session.commit.phase2", ignore_result=True, ignore_args=True)async def _run_memory_extraction(self,@@ -1275,10 +1344,7 @@ class Session:memory_policy: Optional[Dict[str, Any]],) -> None:"""Phase 2: Extract memories, write relations, enqueue — runs in background."""- import uuid-from openviking.service.task_tracker import get_task_tracker- from openviking.storage.transaction import get_lock_managerfrom openviking.telemetry import OperationTelemetry, bind_telemetryfrom openviking.telemetry.registry import register_telemetry, unregister_telemetry@@ -1291,10 +1357,6 @@ class Session:memory_diff_uri: Optional[str] = Nonetelemetry = OperationTelemetry(operation="session_commit_phase2", enabled=True)archive_index = self._archive_index_from_uri(archive_uri)- redo_task_id: Optional[str] = None- lock_manager = get_lock_manager()- redo_enabled = lock_manager.redo_recovery_enabled- redo_log = lock_manager.redo_logtry:await self._wait_for_previous_archive_done(archive_index)@@ -1308,20 +1370,6 @@ class Session:register_telemetry(telemetry)try:with bind_telemetry(telemetry):- # redo-log protection- if redo_enabled:- redo_task_id = str(uuid.uuid4())- await redo_log.write_pending_async(- redo_task_id,- {- "archive_uri": archive_uri,- "session_uri": self._session_uri,- "account_id": self.ctx.account_id,- "user_id": self.ctx.user.user_id,- "role": str(self.ctx.role),- },- )-ov_config = get_openviking_config()effective_policy = MemoryPolicy.from_dict(memory_policy)working_memory_enabled = effective_policy.working_memory_enabled@@ -1575,9 +1623,6 @@ class Session:except Exception as e:logger.warning(f"Failed to create relation to {usage.uri}: {e}")- if redo_enabled and redo_task_id:- await redo_log.mark_done_async(redo_task_id)-# Update active_count (using snapshot, not self._usage_records)if self._vikingdb_manager:uris = [u.uri for u in usage_records if u.uri]@@ -1621,9 +1666,7 @@ class Session:telemetry_snapshot=snapshot,)- # Write .done file last — signals that all state is finalized. We- # only reach here when every Phase 2 step succeeded; any failure- # would have raised above and marked the archive .failed.json.+ # Write .done last so a recovered queue item can skip completed work.await self._write_done_file(archive_uri, first_message_id, last_message_id)result_payload = {@@ -1658,28 +1701,9 @@ class Session:user_id=self.ctx.user.user_id,)logger.info(f"Session {self.session_id} memory extraction completed")- except asyncio.CancelledError as e:- if redo_enabled and redo_task_id:- await redo_log.mark_done_async(redo_task_id)- try:- await self._write_failed_marker(- archive_uri,- stage="memory_extraction",- error=f"cancelled: {e}",- )- except Exception:- logger.debug("Failed to write cancelled marker for session %s", self.session_id)- await tracker.fail(- task_id,- f"cancelled: {e}",- account_id=self.ctx.account_id,- user_id=self.ctx.user.user_id,- )- logger.warning("Memory extraction cancelled for session %s", self.session_id)+ except asyncio.CancelledError:raiseexcept Exception as e:- if redo_enabled and redo_task_id:- await redo_log.mark_done_async(redo_task_id)await self._write_failed_marker(archive_uri,stage="memory_extraction",diff --git a/openviking/storage/queuefs/__init__.py b/openviking/storage/queuefs/__init__.pyindex f98e0b93c..7302db2df 100644--- a/openviking/storage/queuefs/__init__.py+++ b/openviking/storage/queuefs/__init__.py@@ -9,6 +9,7 @@ from .semantic_dag import SemanticDagExecutorfrom .semantic_msg import SemanticMsgfrom .semantic_processor import SemanticProcessorfrom .semantic_queue import SemanticQueue+from .session_commit_msg import SessionCommitMsg__all__ = ["QueueManager",@@ -24,4 +25,5 @@ __all__ = ["SemanticDagExecutor","SemanticMsg","SemanticProcessor",+ "SessionCommitMsg",]diff --git a/openviking/storage/queuefs/queue_manager.py b/openviking/storage/queuefs/queue_manager.pyindex 3c40b5b58..1cfd6d4c9 100644--- a/openviking/storage/queuefs/queue_manager.py+++ b/openviking/storage/queuefs/queue_manager.py@@ -70,6 +70,7 @@ class QueueManager:EMBEDDING = "Embedding"SEMANTIC = "Semantic"EXTERNAL_PARSE = "ExternalParse"+ SESSION_COMMIT = "SessionCommit"def __init__(self,@@ -156,7 +157,7 @@ class QueueManager:if queue.name == self.EMBEDDING:max_concurrent = self._max_concurrent_embedding- elif queue.name == self.EXTERNAL_PARSE:+ elif queue.name in {self.EXTERNAL_PARSE, self.SESSION_COMMIT}:max_concurrent = self._max_concurrent_external_parseelse:max_concurrent = self._max_concurrent_semanticdiff --git a/openviking/storage/queuefs/session_commit_msg.py b/openviking/storage/queuefs/session_commit_msg.pynew file mode 100644index 000000000..7f6ba0da1--- /dev/null+++ b/openviking/storage/queuefs/session_commit_msg.py@@ -0,0 +1,21 @@+# Copyright (c) 2026 Beijing Volcano Engine Technology Co., Ltd.+# SPDX-License-Identifier: AGPL-3.0+"""Persistent Session Phase 2 queue message."""++from dataclasses import asdict, dataclass, field+from typing import Any, Dict, List, Optional+++@dataclass+class SessionCommitMsg:+ task_id: str+ session_id: str+ session_uri: str+ archive_uri: str+ user: Dict[str, str]+ actor_peer_id: Optional[str] = None+ memory_policy: Dict[str, Any] = field(default_factory=dict)+ usage_uris: List[str] = field(default_factory=list)++ def to_dict(self) -> Dict[str, Any]:+ return asdict(self)diff --git a/openviking/storage/queuefs/session_commit_processor.py b/openviking/storage/queuefs/session_commit_processor.pynew file mode 100644index 000000000..2d070a0f7--- /dev/null+++ b/openviking/storage/queuefs/session_commit_processor.py@@ -0,0 +1,84 @@+# Copyright (c) 2026 Beijing Volcano Engine Technology Co., Ltd.+# SPDX-License-Identifier: AGPL-3.0+"""Queue consumer for restart-safe Session Phase 2 work."""++import asyncio+import concurrent.futures+import json+from typing import TYPE_CHECKING, Any, Dict, Optional++from openviking.server.identity import RequestContext, Role+from openviking.service.task_tracker import get_task_tracker+from openviking.storage.queuefs.named_queue import DequeueHandlerBase+from openviking.storage.queuefs.session_commit_msg import SessionCommitMsg+from openviking_cli.session.user_id import UserIdentifier++if TYPE_CHECKING:+ from openviking.service.session_service import SessionService+++class SessionCommitProcessor(DequeueHandlerBase):+ def __init__(+ self,+ session_service: "SessionService",+ service_loop: asyncio.AbstractEventLoop,+ ) -> None:+ self._session_service = session_service+ self._service_loop = service_loop++ async def _process(self, msg: SessionCommitMsg, ctx: RequestContext) -> None:+ session = self._session_service.session(+ ctx,+ msg.session_id,+ session_uri=msg.session_uri,+ )+ if not await session.exists():+ error = f"Session '{msg.session_id}' no longer exists"+ tracker = get_task_tracker()+ await tracker.create(+ "session_commit",+ resource_id=msg.session_id,+ account_id=ctx.account_id,+ user_id=ctx.user.user_id,+ task_id=msg.task_id,+ )+ await tracker.fail(+ msg.task_id,+ error,+ account_id=ctx.account_id,+ user_id=ctx.user.user_id,+ )+ return+ await session.load()+ await session.resume_queued_commit(msg)++ async def on_dequeue(+ self, data: Optional[Dict[str, Any]]+ ) -> Optional[Dict[str, Any]]:+ if not data:+ return None++ try:+ payload = data.get("data", data)+ if isinstance(payload, str):+ payload = json.loads(payload)+ msg = SessionCommitMsg(**payload)+ ctx = RequestContext(+ user=UserIdentifier.from_dict(msg.user),+ role=Role.USER,+ actor_peer_id=msg.actor_peer_id,+ )+ except (json.JSONDecodeError, KeyError, TypeError, ValueError) as exc:+ self.report_error(str(exc), data)+ return None+ future: concurrent.futures.Future[None] = asyncio.run_coroutine_threadsafe(+ self._process(msg, ctx),+ self._service_loop,+ )+ try:+ await asyncio.wrap_future(future)+ self.report_success()+ except asyncio.CancelledError:+ future.cancel()+ raise+ return Nonediff --git a/tests/unit/session/test_session_commit_resume.py b/tests/unit/session/test_session_commit_resume.pynew file mode 100644index 000000000..646d5f780--- /dev/null+++ b/tests/unit/session/test_session_commit_resume.py@@ -0,0 +1,86 @@+# Copyright (c) 2026 Beijing Volcano Engine Technology Co., Ltd.+# SPDX-License-Identifier: AGPL-3.0++import json+from unittest.mock import AsyncMock++import pytest++from openviking.message import Message, TextPart+from openviking.service.task_tracker import TaskTracker, reset_task_tracker, set_task_tracker+from openviking.session.session import Session+from openviking.storage.queuefs.session_commit_msg import SessionCommitMsg+++class _TaskStore:+ def __init__(self):+ self.tasks = {}++ async def create(self, task):+ self.tasks[task.task_id] = task++ async def update(self, task):+ self.tasks[task.task_id] = task++ async def get(self, task_id, *, account_id=None, user_id=None):+ return None++ async def list(self, account_id, *, user_id=None):+ return []++ async def delete(self, task_id, *, account_id, user_id=None):+ self.tasks.pop(task_id, None)+++class _MemoryVikingFS:+ def __init__(self, files):+ self.files = files++ def _uri_to_path(self, uri, ctx=None):+ return "/local/session-1"++ async def read_file(self, uri, ctx=None):+ if uri not in self.files:+ raise FileNotFoundError(uri)+ return self.files[uri]++ async def write_file(self, uri, content, ctx=None):+ self.files[uri] = content+++async def test_resume_queued_commit_continues_phase2(monkeypatch):+ session_uri = "viking://user/sessions/session-1"+ archive_uri = f"{session_uri}/history/archive_001"+ archived = Message(id="archived", role="user", parts=[TextPart("old")])+ retained = Message(id="retained", role="assistant", parts=[TextPart("new")])+ files = {+ f"{session_uri}/messages.jsonl": f"{retained.to_jsonl()}\n",+ f"{session_uri}/.meta.json": json.dumps(+ {"session_id": "session-1", "message_count": 1, "commit_count": 1}+ ),+ f"{archive_uri}/messages.jsonl": f"{archived.to_jsonl()}\n",+ }+ viking_fs = _MemoryVikingFS(files)+ session = Session(viking_fs=viking_fs, session_id="session-1", session_uri=session_uri)+ tracker = TaskTracker(_TaskStore())+ set_task_tracker(tracker)+ monkeypatch.setattr(session, "_run_memory_extraction", AsyncMock())+ message = SessionCommitMsg(+ task_id="task-1",+ session_id="session-1",+ session_uri=session_uri,+ archive_uri=archive_uri,+ user={"account_id": "default", "user_id": "default"},+ )++ try:+ await session.resume_queued_commit(message)+ finally:+ reset_task_tracker()++ session._run_memory_extraction.assert_awaited_once()+ assert session._run_memory_extraction.await_args.kwargs["task_id"] == "task-1"+ assert [+ item.id for item in session._run_memory_extraction.await_args.kwargs["messages"]+ ] == ["archived"]