Self-evolving Context Database for AI Agents. Unify Agent Memory, Knowledge RAG and Skills.
Python agent-contextagent-memoryagent-pluginsagentic-ragcontext-databasedsh-pluginknowledge-basememorymemory-managementself-evolvingvector-database
fix(server): preserve graceful SIGTERM shutdown (#3278)
huangruiteng2ea129e63bed
diff --git a/openviking/service/core.py b/openviking/service/core.pyindex cc5a770bf..98a13f81d 100644--- a/openviking/service/core.py+++ b/openviking/service/core.py@@ -94,6 +94,7 @@ class OpenVikingService:self._encryptor: Optional[Any] = Noneself._privacy_config_service: Optional[UserPrivacyConfigService] = Noneself._data_dir_lock_acquired = False+ self._data_dir_lock_path: Optional[str] = None# Sub-servicesself._fs_service = FSService()@@ -198,7 +199,7 @@ class OpenVikingService:if not self._config.storage.skip_process_lock:from openviking.utils.process_lock import acquire_data_dir_lock- acquire_data_dir_lock(self._config.storage.workspace)+ self._data_dir_lock_path = acquire_data_dir_lock(self._config.storage.workspace)else:logger.warning("Skipping workspace process lock for '%s'; multi-process access may corrupt data",@@ -206,6 +207,16 @@ class OpenVikingService:)self._data_dir_lock_acquired = True+ def _release_data_dir_lock(self) -> None:+ """Release this service instance's process-level workspace lock."""+ lock_path = getattr(self, "_data_dir_lock_path", None)+ if lock_path:+ from openviking.utils.process_lock import release_data_dir_lock++ release_data_dir_lock(lock_path)+ self._data_dir_lock_path = None+ self._data_dir_lock_acquired = False+@propertydef _agfs(self) -> Any:"""Internal access to AGFS client for APIKeyManager."""@@ -491,6 +502,11 @@ class OpenVikingService:if get_service_or_none() is self:set_service(None)+ # The PID lock protects every live workspace resource above. If any+ # cleanup step failed or was cancelled, keep the lock so another+ # process cannot enter while this service may still own storage state.+ self._release_data_dir_lock()+logger.info("OpenVikingService closed")async def reindex(diff --git a/openviking/utils/process_lock.py b/openviking/utils/process_lock.pyindex 333b77bff..db2799545 100644--- a/openviking/utils/process_lock.py+++ b/openviking/utils/process_lock.py@@ -8,8 +8,8 @@ directory, which causes silent failures in AGFS and VectorDB.import atexitimport os-import signalimport sys+import threadingfrom openviking_cli.utils import get_logger@@ -17,11 +17,24 @@ logger = get_logger(__name__)LOCK_FILENAME = ".openviking.pid"+# A PID file protects the whole process, while multiple embedded services may+# legitimately share that process and workspace. Keep process-local ownership+# counts so closing one service cannot expose another live service to a second+# process. The file remains the cross-process source of truth.+_LOCK_STATE_GUARD = threading.Lock()+_LOCK_REF_COUNTS: dict[str, int] = {}+_ATEXIT_REGISTERED: set[str] = set()+class DataDirectoryLocked(RuntimeError):"""Raised when another OpenViking process holds the data directory lock."""+def _normalize_lock_path(lock_path: str) -> str:+ """Return one process-local identity for aliases of the same PID file."""+ return os.path.realpath(os.path.abspath(lock_path))++def _read_pid_file(lock_path: str) -> int:"""Read PID from lock file. Returns 0 if unreadable."""try:@@ -69,6 +82,37 @@ def _is_pid_alive(pid: int) -> bool:return True+def _remove_owned_lock(lock_path: str, owner_pid: int) -> None:+ """Remove *lock_path* only while it still belongs to *owner_pid*."""+ try:+ if os.path.isfile(lock_path) and _read_pid_file(lock_path) == owner_pid:+ os.remove(lock_path)+ except OSError:+ pass+++def release_data_dir_lock(lock_path: str, *, pid: int | None = None) -> None:+ """Release a data-directory lock when it is still owned by *pid*.++ The ownership check keeps a delayed cleanup callback from removing a lock+ that a replacement process has already acquired.+ """+ owner_pid = os.getpid() if pid is None else pid+ normalized_path = _normalize_lock_path(lock_path)++ with _LOCK_STATE_GUARD:+ holder_count = _LOCK_REF_COUNTS.get(normalized_path, 0)+ if holder_count > 1:+ _LOCK_REF_COUNTS[normalized_path] = holder_count - 1+ return+ if holder_count == 1:+ _LOCK_REF_COUNTS.pop(normalized_path, None)++ # A zero count is retained as a compatibility path for callers that+ # acquired the lock before this module state was initialized/reloaded.+ _remove_owned_lock(normalized_path, owner_pid)++def acquire_data_dir_lock(data_dir: str) -> str:"""Acquire an advisory PID lock on *data_dir*.@@ -76,50 +120,53 @@ def acquire_data_dir_lock(data_dir: str) -> str:Raises ``DataDirectoryLocked`` if another live process already holds thelock, with a message that explains the situation and suggests HTTP mode.+ Raises ``OSError`` when the PID file cannot be created or updated; callers+ must never continue as if an exclusivity lock had been acquired."""- lock_path = os.path.join(data_dir, LOCK_FILENAME)+ normalized_data_dir = os.path.realpath(os.path.abspath(data_dir))+ lock_path = _normalize_lock_path(os.path.join(normalized_data_dir, LOCK_FILENAME))my_pid = os.getpid()- existing_pid = _read_pid_file(lock_path)- if existing_pid and existing_pid != my_pid and _is_pid_alive(existing_pid):- raise DataDirectoryLocked(- f"Another OpenViking process (PID {existing_pid}) is already using "- f"the data directory '{data_dir}'. Running multiple OpenViking "- f"instances on the same data directory causes silent storage "- f"contention and data corruption.\n\n"- f"To fix this, use one of these approaches:\n"- f" 1. Use HTTP mode: start a single openviking-server and connect "- f"via --transport http (recommended for multi-session hosts)\n"- f" 2. Use separate data directories for each instance\n"- f" 3. Stop the other process (PID {existing_pid}) first"- )-- # Write our PID (overwrites stale lock from a dead process).- try:- os.makedirs(data_dir, exist_ok=True)- with open(lock_path, "w") as f:- f.write(str(my_pid))- except OSError as exc:- logger.warning("Could not write PID lock %s: %s", lock_path, exc)- return lock_path-- # Schedule cleanup on exit.- def _cleanup(*_args: object) -> None:+ with _LOCK_STATE_GUARD:+ existing_pid = _read_pid_file(lock_path)+ if existing_pid and existing_pid != my_pid and _is_pid_alive(existing_pid):+ raise DataDirectoryLocked(+ f"Another OpenViking process (PID {existing_pid}) is already using "+ f"the data directory '{data_dir}'. Running multiple OpenViking "+ f"instances on the same data directory causes silent storage "+ f"contention and data corruption.\n\n"+ f"To fix this, use one of these approaches:\n"+ f" 1. Use HTTP mode: start a single openviking-server and connect "+ f"via --transport http (recommended for multi-session hosts)\n"+ f" 2. Use separate data directories for each instance\n"+ f" 3. Stop the other process (PID {existing_pid}) first"+ )++ # Write our PID (overwrites stale lock from a dead process). Reentrant+ # acquisitions still refresh the file in case an external actor+ # removed it while this process remained alive.try:- if os.path.isfile(lock_path):- stored = _read_pid_file(lock_path)- if stored == my_pid:- os.remove(lock_path)- except OSError:- pass-- atexit.register(_cleanup)- # Also try to clean up on SIGTERM (graceful shutdown).- try:- signal.signal(signal.SIGTERM, lambda sig, frame: (_cleanup(), sys.exit(0)))- except (OSError, ValueError):- # signal.signal() can fail in non-main threads.- pass+ os.makedirs(normalized_data_dir, exist_ok=True)+ with open(lock_path, "w") as f:+ f.write(str(my_pid))+ except OSError as exc:+ logger.warning("Could not write PID lock %s: %s", lock_path, exc)+ raise++ _LOCK_REF_COUNTS[lock_path] = _LOCK_REF_COUNTS.get(lock_path, 0) + 1++ # One force-cleanup callback per path is enough. At interpreter exit+ # every in-process holder is terminal, so refcounts must not prevent+ # cleanup.+ if lock_path not in _ATEXIT_REGISTERED:++ def _cleanup(*_args: object) -> None:+ with _LOCK_STATE_GUARD:+ _LOCK_REF_COUNTS.pop(lock_path, None)+ _remove_owned_lock(lock_path, my_pid)++ atexit.register(_cleanup)+ _ATEXIT_REGISTERED.add(lock_path)logger.debug("Acquired data directory lock: %s (PID %d)", lock_path, my_pid)return lock_pathdiff --git a/tests/unit/service/test_core_encryption_startup.py b/tests/unit/service/test_core_encryption_startup.pyindex c6ac2bb9c..f1cf7a7d8 100644--- a/tests/unit/service/test_core_encryption_startup.py+++ b/tests/unit/service/test_core_encryption_startup.py@@ -4,6 +4,7 @@from __future__ import annotations+import asynciofrom types import SimpleNamespaceimport pytest@@ -95,11 +96,13 @@ def test_ensure_data_dir_lock_acquired_once(monkeypatch, tmp_path):storage=SimpleNamespace(workspace=str(tmp_path), skip_process_lock=False))service._data_dir_lock_acquired = False+ service._data_dir_lock_path = Noneservice._ensure_data_dir_lock_acquired()service._ensure_data_dir_lock_acquired()assert calls == [str(tmp_path)]+ assert service._data_dir_lock_path == str(tmp_path / ".openviking.pid")def test_ensure_data_dir_lock_respects_skip_process_lock(monkeypatch, tmp_path):@@ -117,8 +120,81 @@ def test_ensure_data_dir_lock_respects_skip_process_lock(monkeypatch, tmp_path):storage=SimpleNamespace(workspace=str(tmp_path), skip_process_lock=True))service._data_dir_lock_acquired = False+ service._data_dir_lock_path = Noneservice._ensure_data_dir_lock_acquired()assert calls == []assert service._data_dir_lock_acquired is True+ assert service._data_dir_lock_path is None+++def test_release_data_dir_lock_resets_service_state(monkeypatch, tmp_path):+ calls = []+ lock_path = str(tmp_path / ".openviking.pid")+ monkeypatch.setattr("openviking.utils.process_lock.release_data_dir_lock", calls.append)+ service = OpenVikingService.__new__(OpenVikingService)+ service._data_dir_lock_acquired = True+ service._data_dir_lock_path = lock_path++ service._release_data_dir_lock()++ assert calls == [lock_path]+ assert service._data_dir_lock_acquired is False+ assert service._data_dir_lock_path is None+++async def test_close_preserves_data_dir_lock_when_resource_cleanup_fails(monkeypatch):+ class _FailingResourceService:+ async def close_background_tasks(self) -> None:+ raise RuntimeError("cleanup failed")++ released = []+ service = OpenVikingService.__new__(OpenVikingService)+ service._resource_service = _FailingResourceService()+ monkeypatch.setattr(service, "_release_data_dir_lock", lambda: released.append(True))++ with pytest.raises(RuntimeError, match="cleanup failed"):+ await service.close()++ assert released == []+++async def test_close_preserves_data_dir_lock_when_resource_cleanup_is_cancelled(monkeypatch):+ class _CancelledResourceService:+ async def close_background_tasks(self) -> None:+ raise asyncio.CancelledError++ released = []+ service = OpenVikingService.__new__(OpenVikingService)+ service._resource_service = _CancelledResourceService()+ monkeypatch.setattr(service, "_release_data_dir_lock", lambda: released.append(True))++ with pytest.raises(asyncio.CancelledError):+ await service.close()++ assert released == []+++async def test_close_releases_data_dir_lock_after_successful_cleanup(monkeypatch):+ class _ResourceService:+ async def close_background_tasks(self) -> None:+ return None++ released = []+ service = OpenVikingService.__new__(OpenVikingService)+ service._resource_service = _ResourceService()+ service._watch_scheduler = None+ service._queue_manager = None+ service._lock_manager = None+ service._vikingdb_manager = None+ service._initialized = True+ monkeypatch.setattr(service, "_release_data_dir_lock", lambda: released.append(True))++ await service.close()++ assert service._initialized is False+ assert released == [True]diff --git a/tests/unit/test_process_lock.py b/tests/unit/test_process_lock.pyindex 836100031..7e3228d36 100644--- a/tests/unit/test_process_lock.py+++ b/tests/unit/test_process_lock.py@@ -15,6 +15,7 @@ from openviking.utils.process_lock import (_is_pid_alive,_read_pid_file,acquire_data_dir_lock,+ release_data_dir_lock,)@@ -173,6 +174,20 @@ class TestAcquireDataDirLock:assert new_dir.exists()assert (new_dir / LOCK_FILENAME).exists()+ def test_acquire_write_failure_is_not_reported_as_success(self, tmp_path: Path, monkeypatch):+ """A failed PID-file write must fail closed instead of claiming exclusivity."""++ def _deny_makedirs(*_args, **_kwargs) -> None:+ raise PermissionError("read-only workspace")++ workspace = tmp_path / "missing"+ monkeypatch.setattr(process_lock_module.os, "makedirs", _deny_makedirs)++ with pytest.raises(PermissionError, match="read-only workspace"):+ acquire_data_dir_lock(str(workspace))++ assert not (workspace / LOCK_FILENAME).exists()+def test_error_message_suggests_http_mode(self, tmp_path: Path):"""Test error message suggests HTTP mode."""(tmp_path / LOCK_FILENAME).write_text("1")@@ -260,6 +275,64 @@ class TestAcquireDataDirLockEdgeCases:assert lock_path.endswith(LOCK_FILENAME)+class TestReleaseDataDirLock:+ """Test explicit process-lock release used by service shutdown."""++ def test_release_removes_owned_lock(self, tmp_path: Path):+ lock_path = acquire_data_dir_lock(str(tmp_path))++ release_data_dir_lock(lock_path)++ assert not (tmp_path / LOCK_FILENAME).exists()++ def test_release_preserves_replacement_owner(self, tmp_path: Path):+ lock_path = acquire_data_dir_lock(str(tmp_path))+ (tmp_path / LOCK_FILENAME).write_text("999999")++ release_data_dir_lock(lock_path)++ assert (tmp_path / LOCK_FILENAME).read_text() == "999999"++ def test_release_is_idempotent(self, tmp_path: Path):+ lock_path = acquire_data_dir_lock(str(tmp_path))++ release_data_dir_lock(lock_path)+ release_data_dir_lock(lock_path)++ assert not (tmp_path / LOCK_FILENAME).exists()++ def test_release_keeps_lock_until_last_same_process_holder(self, tmp_path: Path):+ lock_path = acquire_data_dir_lock(str(tmp_path))+ assert acquire_data_dir_lock(str(tmp_path)) == lock_path++ release_data_dir_lock(lock_path)++ assert (tmp_path / LOCK_FILENAME).read_text() == str(os.getpid())++ release_data_dir_lock(lock_path)++ assert not (tmp_path / LOCK_FILENAME).exists()++ def test_symlink_alias_shares_the_same_process_local_refcount(self, tmp_path: Path):+ """Workspace aliases must not let one service release another's PID lock."""+ workspace = tmp_path / "workspace"+ workspace.mkdir()+ alias = tmp_path / "workspace-alias"+ try:+ alias.symlink_to(workspace, target_is_directory=True)+ except OSError as exc:+ pytest.skip(f"directory symlinks are unavailable: {exc}")++ lock_path = acquire_data_dir_lock(str(workspace))+ assert acquire_data_dir_lock(str(alias)) == lock_path++ release_data_dir_lock(lock_path)+ assert (workspace / LOCK_FILENAME).read_text() == str(os.getpid())++ release_data_dir_lock(lock_path)+ assert not (workspace / LOCK_FILENAME).exists()++class TestProcessLockIntegration:"""Integration tests for process lock."""diff --git a/tests/utils/test_process_lock.py b/tests/utils/test_process_lock.pyindex 80ba75c1c..8e7f2bb42 100644--- a/tests/utils/test_process_lock.py+++ b/tests/utils/test_process_lock.py@@ -1,53 +1,92 @@# Copyright (c) 2026 Beijing Volcano Engine Technology Co., Ltd.# SPDX-License-Identifier: AGPL-3.0-"""Tests for process_lock.py signal handling."""+"""Tests for process-lock interaction with host signal handling."""+import osimport signal---def test_sigterm_handler_calls_sys_exit():- """SIGTERM handler 应该调用 sys.exit(0) 干净退出。-- 验证修复后的 lambda 调用 sys.exit(0) 而非 signal.default_int_handler。- signal.default_int_handler 会抛出 KeyboardInterrupt(SIGINT handler),- 语义错误。sys.exit(0) 触发 atexit cleanup 并正常退出。- """- exit_calls = []-- def fake_exit(code=0):- exit_calls.append(code)-- # 模拟修复后的 handler(与 process_lock.py:119-120 一致)- def handler(sig, frame):- (None, fake_exit(0))-- handler(signal.SIGTERM, None)-- assert exit_calls == [0]---def test_sigterm_handler_calls_cleanup_before_exit():- """SIGTERM handler 应该在退出前调用 _cleanup。-- _cleanup 负责移除 PID 锁文件,即使在 SIGTERM 强制关闭时也应执行,- 避免残留锁文件阻止后续进程启动。lambda 中的元组按顺序执行:- 先 _cleanup(),再 sys.exit(0)。- """- cleanup_called = []- exit_calls = []-- def fake_cleanup():- cleanup_called.append(True)-- def fake_exit(code=0):- exit_calls.append(code)-- # 模拟修复后的 handler(与 process_lock.py:119-120 一致)- def handler(sig, frame):- (fake_cleanup(), fake_exit(0))-- handler(signal.SIGTERM, None)-- assert cleanup_called == [True], "_cleanup 应在 exit 前被调用"- assert exit_calls == [0]+import subprocess+import sys+import textwrap+import time++import pytest++from openviking.utils.process_lock import acquire_data_dir_lock+++def test_acquire_does_not_override_sigterm_handler(tmp_path):+ """Uvicorn must retain ownership of SIGTERM for graceful lifespan shutdown."""+ original_handler = signal.getsignal(signal.SIGTERM)++ acquire_data_dir_lock(str(tmp_path))++ assert signal.getsignal(signal.SIGTERM) is original_handler++[email protected](sys.platform == "win32", reason="SIGTERM lifecycle is POSIX-specific")+def test_uvicorn_sigterm_runs_lifespan_and_releases_process_lock(tmp_path):+ """A real Uvicorn SIGTERM must reach lifespan cleanup without tracebacks."""+ workspace = tmp_path / "workspace"+ ready_file = tmp_path / "ready"+ script = textwrap.dedent(+ """+ from contextlib import asynccontextmanager+ from pathlib import Path+ import sys++ import uvicorn+ from fastapi import FastAPI++ from openviking.utils.process_lock import (+ acquire_data_dir_lock,+ release_data_dir_lock,+ )++ workspace = sys.argv[1]+ ready_file = Path(sys.argv[2])++ @asynccontextmanager+ async def lifespan(_app):+ lock_path = acquire_data_dir_lock(workspace)+ ready_file.write_text("ready")+ try:+ yield+ finally:+ release_data_dir_lock(lock_path)++ app = FastAPI(lifespan=lifespan)+ uvicorn.run(app, host="127.0.0.1", port=0, log_level="info")+ """+ )+ process = subprocess.Popen(+ [sys.executable, "-c", script, str(workspace), str(ready_file)],+ cwd=os.getcwd(),+ stdout=subprocess.PIPE,+ stderr=subprocess.STDOUT,+ text=True,+ )+ try:+ deadline = time.monotonic() + 20+ while not ready_file.exists() and process.poll() is None and time.monotonic() < deadline:+ time.sleep(0.05)+ assert ready_file.exists(), "Uvicorn child did not enter application lifespan"++ process.send_signal(signal.SIGTERM)+ output, _ = process.communicate(timeout=20)+ finally:+ if process.poll() is None:+ process.kill()+ process.wait(timeout=5)++ # Current Uvicorn restores and re-raises the captured POSIX signal after+ # its graceful shutdown completes, so both a clean return and -SIGTERM are+ # valid host-level terminal statuses. The lifecycle evidence below is the+ # regression contract: lifespan completed and no exception traceback was+ # emitted by a process-lock handler.+ assert process.returncode in {0, -signal.SIGTERM}, output+ assert "Application shutdown complete" in output+ assert "Finished server process" in output+ assert not (workspace / ".openviking.pid").exists()+ assert "SystemExit" not in output+ assert "CancelledError" not in output