Skip to content

Self-evolving Context Database for AI Agents. Unify Agent Memory, Knowledge RAG and Skills.

fix(server): preserve graceful SIGTERM shutdown (#3278)

huangruiteng2ea129e63bed
diff --git a/openviking/service/core.py b/openviking/service/core.py
index cc5a770bf..98a13f81d 100644
--- a/openviking/service/core.py
+++ b/openviking/service/core.py
@@ -94,6 +94,7 @@ class OpenVikingService:
self._encryptor: Optional[Any] = None
self._privacy_config_service: Optional[UserPrivacyConfigService] = None
self._data_dir_lock_acquired = False
+ self._data_dir_lock_path: Optional[str] = None
# Sub-services
self._fs_service = FSService()
@@ -198,7 +199,7 @@ class OpenVikingService:
if not self._config.storage.skip_process_lock:
from openviking.utils.process_lock import acquire_data_dir_lock
- acquire_data_dir_lock(self._config.storage.workspace)
+ self._data_dir_lock_path = acquire_data_dir_lock(self._config.storage.workspace)
else:
logger.warning(
"Skipping workspace process lock for '%s'; multi-process access may corrupt data",
@@ -206,6 +207,16 @@ class OpenVikingService:
)
self._data_dir_lock_acquired = True
+ def _release_data_dir_lock(self) -> None:
+ """Release this service instance's process-level workspace lock."""
+ lock_path = getattr(self, "_data_dir_lock_path", None)
+ if lock_path:
+ from openviking.utils.process_lock import release_data_dir_lock
+
+ release_data_dir_lock(lock_path)
+ self._data_dir_lock_path = None
+ self._data_dir_lock_acquired = False
+
@property
def _agfs(self) -> Any:
"""Internal access to AGFS client for APIKeyManager."""
@@ -491,6 +502,11 @@ class OpenVikingService:
if get_service_or_none() is self:
set_service(None)
+ # The PID lock protects every live workspace resource above. If any
+ # cleanup step failed or was cancelled, keep the lock so another
+ # process cannot enter while this service may still own storage state.
+ self._release_data_dir_lock()
+
logger.info("OpenVikingService closed")
async def reindex(
diff --git a/openviking/utils/process_lock.py b/openviking/utils/process_lock.py
index 333b77bff..db2799545 100644
--- a/openviking/utils/process_lock.py
+++ b/openviking/utils/process_lock.py
@@ -8,8 +8,8 @@ directory, which causes silent failures in AGFS and VectorDB.
import atexit
import os
-import signal
import sys
+import threading
from openviking_cli.utils import get_logger
@@ -17,11 +17,24 @@ logger = get_logger(__name__)
LOCK_FILENAME = ".openviking.pid"
+# A PID file protects the whole process, while multiple embedded services may
+# legitimately share that process and workspace. Keep process-local ownership
+# counts so closing one service cannot expose another live service to a second
+# process. The file remains the cross-process source of truth.
+_LOCK_STATE_GUARD = threading.Lock()
+_LOCK_REF_COUNTS: dict[str, int] = {}
+_ATEXIT_REGISTERED: set[str] = set()
+
class DataDirectoryLocked(RuntimeError):
"""Raised when another OpenViking process holds the data directory lock."""
+def _normalize_lock_path(lock_path: str) -> str:
+ """Return one process-local identity for aliases of the same PID file."""
+ return os.path.realpath(os.path.abspath(lock_path))
+
+
def _read_pid_file(lock_path: str) -> int:
"""Read PID from lock file. Returns 0 if unreadable."""
try:
@@ -69,6 +82,37 @@ def _is_pid_alive(pid: int) -> bool:
return True
+def _remove_owned_lock(lock_path: str, owner_pid: int) -> None:
+ """Remove *lock_path* only while it still belongs to *owner_pid*."""
+ try:
+ if os.path.isfile(lock_path) and _read_pid_file(lock_path) == owner_pid:
+ os.remove(lock_path)
+ except OSError:
+ pass
+
+
+def release_data_dir_lock(lock_path: str, *, pid: int | None = None) -> None:
+ """Release a data-directory lock when it is still owned by *pid*.
+
+ The ownership check keeps a delayed cleanup callback from removing a lock
+ that a replacement process has already acquired.
+ """
+ owner_pid = os.getpid() if pid is None else pid
+ normalized_path = _normalize_lock_path(lock_path)
+
+ with _LOCK_STATE_GUARD:
+ holder_count = _LOCK_REF_COUNTS.get(normalized_path, 0)
+ if holder_count > 1:
+ _LOCK_REF_COUNTS[normalized_path] = holder_count - 1
+ return
+ if holder_count == 1:
+ _LOCK_REF_COUNTS.pop(normalized_path, None)
+
+ # A zero count is retained as a compatibility path for callers that
+ # acquired the lock before this module state was initialized/reloaded.
+ _remove_owned_lock(normalized_path, owner_pid)
+
+
def acquire_data_dir_lock(data_dir: str) -> str:
"""Acquire an advisory PID lock on *data_dir*.
@@ -76,50 +120,53 @@ def acquire_data_dir_lock(data_dir: str) -> str:
Raises ``DataDirectoryLocked`` if another live process already holds the
lock, with a message that explains the situation and suggests HTTP mode.
+ Raises ``OSError`` when the PID file cannot be created or updated; callers
+ must never continue as if an exclusivity lock had been acquired.
"""
- lock_path = os.path.join(data_dir, LOCK_FILENAME)
+ normalized_data_dir = os.path.realpath(os.path.abspath(data_dir))
+ lock_path = _normalize_lock_path(os.path.join(normalized_data_dir, LOCK_FILENAME))
my_pid = os.getpid()
- existing_pid = _read_pid_file(lock_path)
- if existing_pid and existing_pid != my_pid and _is_pid_alive(existing_pid):
- raise DataDirectoryLocked(
- f"Another OpenViking process (PID {existing_pid}) is already using "
- f"the data directory '{data_dir}'. Running multiple OpenViking "
- f"instances on the same data directory causes silent storage "
- f"contention and data corruption.\n\n"
- f"To fix this, use one of these approaches:\n"
- f" 1. Use HTTP mode: start a single openviking-server and connect "
- f"via --transport http (recommended for multi-session hosts)\n"
- f" 2. Use separate data directories for each instance\n"
- f" 3. Stop the other process (PID {existing_pid}) first"
- )
-
- # Write our PID (overwrites stale lock from a dead process).
- try:
- os.makedirs(data_dir, exist_ok=True)
- with open(lock_path, "w") as f:
- f.write(str(my_pid))
- except OSError as exc:
- logger.warning("Could not write PID lock %s: %s", lock_path, exc)
- return lock_path
-
- # Schedule cleanup on exit.
- def _cleanup(*_args: object) -> None:
+ with _LOCK_STATE_GUARD:
+ existing_pid = _read_pid_file(lock_path)
+ if existing_pid and existing_pid != my_pid and _is_pid_alive(existing_pid):
+ raise DataDirectoryLocked(
+ f"Another OpenViking process (PID {existing_pid}) is already using "
+ f"the data directory '{data_dir}'. Running multiple OpenViking "
+ f"instances on the same data directory causes silent storage "
+ f"contention and data corruption.\n\n"
+ f"To fix this, use one of these approaches:\n"
+ f" 1. Use HTTP mode: start a single openviking-server and connect "
+ f"via --transport http (recommended for multi-session hosts)\n"
+ f" 2. Use separate data directories for each instance\n"
+ f" 3. Stop the other process (PID {existing_pid}) first"
+ )
+
+ # Write our PID (overwrites stale lock from a dead process). Reentrant
+ # acquisitions still refresh the file in case an external actor
+ # removed it while this process remained alive.
try:
- if os.path.isfile(lock_path):
- stored = _read_pid_file(lock_path)
- if stored == my_pid:
- os.remove(lock_path)
- except OSError:
- pass
-
- atexit.register(_cleanup)
- # Also try to clean up on SIGTERM (graceful shutdown).
- try:
- signal.signal(signal.SIGTERM, lambda sig, frame: (_cleanup(), sys.exit(0)))
- except (OSError, ValueError):
- # signal.signal() can fail in non-main threads.
- pass
+ os.makedirs(normalized_data_dir, exist_ok=True)
+ with open(lock_path, "w") as f:
+ f.write(str(my_pid))
+ except OSError as exc:
+ logger.warning("Could not write PID lock %s: %s", lock_path, exc)
+ raise
+
+ _LOCK_REF_COUNTS[lock_path] = _LOCK_REF_COUNTS.get(lock_path, 0) + 1
+
+ # One force-cleanup callback per path is enough. At interpreter exit
+ # every in-process holder is terminal, so refcounts must not prevent
+ # cleanup.
+ if lock_path not in _ATEXIT_REGISTERED:
+
+ def _cleanup(*_args: object) -> None:
+ with _LOCK_STATE_GUARD:
+ _LOCK_REF_COUNTS.pop(lock_path, None)
+ _remove_owned_lock(lock_path, my_pid)
+
+ atexit.register(_cleanup)
+ _ATEXIT_REGISTERED.add(lock_path)
logger.debug("Acquired data directory lock: %s (PID %d)", lock_path, my_pid)
return lock_path
diff --git a/tests/unit/service/test_core_encryption_startup.py b/tests/unit/service/test_core_encryption_startup.py
index c6ac2bb9c..f1cf7a7d8 100644
--- a/tests/unit/service/test_core_encryption_startup.py
+++ b/tests/unit/service/test_core_encryption_startup.py
@@ -4,6 +4,7 @@
from __future__ import annotations
+import asyncio
from types import SimpleNamespace
import pytest
@@ -95,11 +96,13 @@ def test_ensure_data_dir_lock_acquired_once(monkeypatch, tmp_path):
storage=SimpleNamespace(workspace=str(tmp_path), skip_process_lock=False)
)
service._data_dir_lock_acquired = False
+ service._data_dir_lock_path = None
service._ensure_data_dir_lock_acquired()
service._ensure_data_dir_lock_acquired()
assert calls == [str(tmp_path)]
+ assert service._data_dir_lock_path == str(tmp_path / ".openviking.pid")
def test_ensure_data_dir_lock_respects_skip_process_lock(monkeypatch, tmp_path):
@@ -117,8 +120,81 @@ def test_ensure_data_dir_lock_respects_skip_process_lock(monkeypatch, tmp_path):
storage=SimpleNamespace(workspace=str(tmp_path), skip_process_lock=True)
)
service._data_dir_lock_acquired = False
+ service._data_dir_lock_path = None
service._ensure_data_dir_lock_acquired()
assert calls == []
assert service._data_dir_lock_acquired is True
+ assert service._data_dir_lock_path is None
+
+
+def test_release_data_dir_lock_resets_service_state(monkeypatch, tmp_path):
+ calls = []
+ lock_path = str(tmp_path / ".openviking.pid")
+ monkeypatch.setattr("openviking.utils.process_lock.release_data_dir_lock", calls.append)
+ service = OpenVikingService.__new__(OpenVikingService)
+ service._data_dir_lock_acquired = True
+ service._data_dir_lock_path = lock_path
+
+ service._release_data_dir_lock()
+
+ assert calls == [lock_path]
+ assert service._data_dir_lock_acquired is False
+ assert service._data_dir_lock_path is None
+
+
+async def test_close_preserves_data_dir_lock_when_resource_cleanup_fails(monkeypatch):
+ class _FailingResourceService:
+ async def close_background_tasks(self) -> None:
+ raise RuntimeError("cleanup failed")
+
+ released = []
+ service = OpenVikingService.__new__(OpenVikingService)
+ service._resource_service = _FailingResourceService()
+ monkeypatch.setattr(service, "_release_data_dir_lock", lambda: released.append(True))
+
+ with pytest.raises(RuntimeError, match="cleanup failed"):
+ await service.close()
+
+ assert released == []
+
+
+async def test_close_preserves_data_dir_lock_when_resource_cleanup_is_cancelled(monkeypatch):
+ class _CancelledResourceService:
+ async def close_background_tasks(self) -> None:
+ raise asyncio.CancelledError
+
+ released = []
+ service = OpenVikingService.__new__(OpenVikingService)
+ service._resource_service = _CancelledResourceService()
+ monkeypatch.setattr(service, "_release_data_dir_lock", lambda: released.append(True))
+
+ with pytest.raises(asyncio.CancelledError):
+ await service.close()
+
+ assert released == []
+
+
+async def test_close_releases_data_dir_lock_after_successful_cleanup(monkeypatch):
+ class _ResourceService:
+ async def close_background_tasks(self) -> None:
+ return None
+
+ released = []
+ service = OpenVikingService.__new__(OpenVikingService)
+ service._resource_service = _ResourceService()
+ service._watch_scheduler = None
+ service._queue_manager = None
+ service._lock_manager = None
+ service._vikingdb_manager = None
+ service._initialized = True
+ monkeypatch.setattr(service, "_release_data_dir_lock", lambda: released.append(True))
+
+ await service.close()
+
+ assert service._initialized is False
+ assert released == [True]
diff --git a/tests/unit/test_process_lock.py b/tests/unit/test_process_lock.py
index 836100031..7e3228d36 100644
--- a/tests/unit/test_process_lock.py
+++ b/tests/unit/test_process_lock.py
@@ -15,6 +15,7 @@ from openviking.utils.process_lock import (
_is_pid_alive,
_read_pid_file,
acquire_data_dir_lock,
+ release_data_dir_lock,
)
@@ -173,6 +174,20 @@ class TestAcquireDataDirLock:
assert new_dir.exists()
assert (new_dir / LOCK_FILENAME).exists()
+ def test_acquire_write_failure_is_not_reported_as_success(self, tmp_path: Path, monkeypatch):
+ """A failed PID-file write must fail closed instead of claiming exclusivity."""
+
+ def _deny_makedirs(*_args, **_kwargs) -> None:
+ raise PermissionError("read-only workspace")
+
+ workspace = tmp_path / "missing"
+ monkeypatch.setattr(process_lock_module.os, "makedirs", _deny_makedirs)
+
+ with pytest.raises(PermissionError, match="read-only workspace"):
+ acquire_data_dir_lock(str(workspace))
+
+ assert not (workspace / LOCK_FILENAME).exists()
+
def test_error_message_suggests_http_mode(self, tmp_path: Path):
"""Test error message suggests HTTP mode."""
(tmp_path / LOCK_FILENAME).write_text("1")
@@ -260,6 +275,64 @@ class TestAcquireDataDirLockEdgeCases:
assert lock_path.endswith(LOCK_FILENAME)
+class TestReleaseDataDirLock:
+ """Test explicit process-lock release used by service shutdown."""
+
+ def test_release_removes_owned_lock(self, tmp_path: Path):
+ lock_path = acquire_data_dir_lock(str(tmp_path))
+
+ release_data_dir_lock(lock_path)
+
+ assert not (tmp_path / LOCK_FILENAME).exists()
+
+ def test_release_preserves_replacement_owner(self, tmp_path: Path):
+ lock_path = acquire_data_dir_lock(str(tmp_path))
+ (tmp_path / LOCK_FILENAME).write_text("999999")
+
+ release_data_dir_lock(lock_path)
+
+ assert (tmp_path / LOCK_FILENAME).read_text() == "999999"
+
+ def test_release_is_idempotent(self, tmp_path: Path):
+ lock_path = acquire_data_dir_lock(str(tmp_path))
+
+ release_data_dir_lock(lock_path)
+ release_data_dir_lock(lock_path)
+
+ assert not (tmp_path / LOCK_FILENAME).exists()
+
+ def test_release_keeps_lock_until_last_same_process_holder(self, tmp_path: Path):
+ lock_path = acquire_data_dir_lock(str(tmp_path))
+ assert acquire_data_dir_lock(str(tmp_path)) == lock_path
+
+ release_data_dir_lock(lock_path)
+
+ assert (tmp_path / LOCK_FILENAME).read_text() == str(os.getpid())
+
+ release_data_dir_lock(lock_path)
+
+ assert not (tmp_path / LOCK_FILENAME).exists()
+
+ def test_symlink_alias_shares_the_same_process_local_refcount(self, tmp_path: Path):
+ """Workspace aliases must not let one service release another's PID lock."""
+ workspace = tmp_path / "workspace"
+ workspace.mkdir()
+ alias = tmp_path / "workspace-alias"
+ try:
+ alias.symlink_to(workspace, target_is_directory=True)
+ except OSError as exc:
+ pytest.skip(f"directory symlinks are unavailable: {exc}")
+
+ lock_path = acquire_data_dir_lock(str(workspace))
+ assert acquire_data_dir_lock(str(alias)) == lock_path
+
+ release_data_dir_lock(lock_path)
+ assert (workspace / LOCK_FILENAME).read_text() == str(os.getpid())
+
+ release_data_dir_lock(lock_path)
+ assert not (workspace / LOCK_FILENAME).exists()
+
+
class TestProcessLockIntegration:
"""Integration tests for process lock."""
diff --git a/tests/utils/test_process_lock.py b/tests/utils/test_process_lock.py
index 80ba75c1c..8e7f2bb42 100644
--- a/tests/utils/test_process_lock.py
+++ b/tests/utils/test_process_lock.py
@@ -1,53 +1,92 @@
# Copyright (c) 2026 Beijing Volcano Engine Technology Co., Ltd.
# SPDX-License-Identifier: AGPL-3.0
-"""Tests for process_lock.py signal handling."""
+"""Tests for process-lock interaction with host signal handling."""
+import os
import signal
-
-
-def test_sigterm_handler_calls_sys_exit():
- """SIGTERM handler 应该调用 sys.exit(0) 干净退出。
-
- 验证修复后的 lambda 调用 sys.exit(0) 而非 signal.default_int_handler。
- signal.default_int_handler 会抛出 KeyboardInterrupt(SIGINT handler),
- 语义错误。sys.exit(0) 触发 atexit cleanup 并正常退出。
- """
- exit_calls = []
-
- def fake_exit(code=0):
- exit_calls.append(code)
-
- # 模拟修复后的 handler(与 process_lock.py:119-120 一致)
- def handler(sig, frame):
- (None, fake_exit(0))
-
- handler(signal.SIGTERM, None)
-
- assert exit_calls == [0]
-
-
-def test_sigterm_handler_calls_cleanup_before_exit():
- """SIGTERM handler 应该在退出前调用 _cleanup。
-
- _cleanup 负责移除 PID 锁文件,即使在 SIGTERM 强制关闭时也应执行,
- 避免残留锁文件阻止后续进程启动。lambda 中的元组按顺序执行:
- 先 _cleanup(),再 sys.exit(0)。
- """
- cleanup_called = []
- exit_calls = []
-
- def fake_cleanup():
- cleanup_called.append(True)
-
- def fake_exit(code=0):
- exit_calls.append(code)
-
- # 模拟修复后的 handler(与 process_lock.py:119-120 一致)
- def handler(sig, frame):
- (fake_cleanup(), fake_exit(0))
-
- handler(signal.SIGTERM, None)
-
- assert cleanup_called == [True], "_cleanup 应在 exit 前被调用"
- assert exit_calls == [0]
+import subprocess
+import sys
+import textwrap
+import time
+
+import pytest
+
+from openviking.utils.process_lock import acquire_data_dir_lock
+
+
+def test_acquire_does_not_override_sigterm_handler(tmp_path):
+ """Uvicorn must retain ownership of SIGTERM for graceful lifespan shutdown."""
+ original_handler = signal.getsignal(signal.SIGTERM)
+
+ acquire_data_dir_lock(str(tmp_path))
+
+ assert signal.getsignal(signal.SIGTERM) is original_handler
+
+
[email protected](sys.platform == "win32", reason="SIGTERM lifecycle is POSIX-specific")
+def test_uvicorn_sigterm_runs_lifespan_and_releases_process_lock(tmp_path):
+ """A real Uvicorn SIGTERM must reach lifespan cleanup without tracebacks."""
+ workspace = tmp_path / "workspace"
+ ready_file = tmp_path / "ready"
+ script = textwrap.dedent(
+ """
+ from contextlib import asynccontextmanager
+ from pathlib import Path
+ import sys
+
+ import uvicorn
+ from fastapi import FastAPI
+
+ from openviking.utils.process_lock import (
+ acquire_data_dir_lock,
+ release_data_dir_lock,
+ )
+
+ workspace = sys.argv[1]
+ ready_file = Path(sys.argv[2])
+
+ @asynccontextmanager
+ async def lifespan(_app):
+ lock_path = acquire_data_dir_lock(workspace)
+ ready_file.write_text("ready")
+ try:
+ yield
+ finally:
+ release_data_dir_lock(lock_path)
+
+ app = FastAPI(lifespan=lifespan)
+ uvicorn.run(app, host="127.0.0.1", port=0, log_level="info")
+ """
+ )
+ process = subprocess.Popen(
+ [sys.executable, "-c", script, str(workspace), str(ready_file)],
+ cwd=os.getcwd(),
+ stdout=subprocess.PIPE,
+ stderr=subprocess.STDOUT,
+ text=True,
+ )
+ try:
+ deadline = time.monotonic() + 20
+ while not ready_file.exists() and process.poll() is None and time.monotonic() < deadline:
+ time.sleep(0.05)
+ assert ready_file.exists(), "Uvicorn child did not enter application lifespan"
+
+ process.send_signal(signal.SIGTERM)
+ output, _ = process.communicate(timeout=20)
+ finally:
+ if process.poll() is None:
+ process.kill()
+ process.wait(timeout=5)
+
+ # Current Uvicorn restores and re-raises the captured POSIX signal after
+ # its graceful shutdown completes, so both a clean return and -SIGTERM are
+ # valid host-level terminal statuses. The lifecycle evidence below is the
+ # regression contract: lifespan completed and no exception traceback was
+ # emitted by a process-lock handler.
+ assert process.returncode in {0, -signal.SIGTERM}, output
+ assert "Application shutdown complete" in output
+ assert "Finished server process" in output
+ assert not (workspace / ".openviking.pid").exists()
+ assert "SystemExit" not in output
+ assert "CancelledError" not in output