Skip to content

Self-evolving Context Database for AI Agents. Unify Agent Memory, Knowledge RAG and Skills.

fix(storage): recover stale semantic lock handoffs (#2214)

Qin Haojiec99623feee10
diff --git a/openviking/storage/queuefs/semantic_lock.py b/openviking/storage/queuefs/semantic_lock.py
index 9e00165e5..88ca0b616 100644
--- a/openviking/storage/queuefs/semantic_lock.py
+++ b/openviking/storage/queuefs/semantic_lock.py
@@ -5,9 +5,33 @@
from __future__ import annotations
from dataclasses import dataclass
-from typing import Optional
+from typing import Iterable, Optional
-from openviking.storage.transaction import NO_LOCK, LockHandoffRef, LockLease, OwnedLockLease
+from openviking.storage.errors import LockAcquisitionError
+from openviking.storage.transaction import (
+ NO_LOCK,
+ LockHandoffRef,
+ LockLease,
+ OwnedLockLease,
+ get_lock_manager,
+)
+from openviking.storage.transaction.path_lock import LOCK_FILE_NAME
+from openviking_cli.utils.logger import get_logger
+
+logger = get_logger(__name__)
+
+_TREE_LOCK_SUFFIX = f"/{LOCK_FILE_NAME}"
+
+
+def _tree_paths_from_handoff(lock_paths: Iterable[str]) -> list[str]:
+ tree_paths: list[str] = []
+ for lock_path in lock_paths:
+ if not lock_path.endswith(_TREE_LOCK_SUFFIX):
+ continue
+ tree_path = lock_path[: -len(_TREE_LOCK_SUFFIX)] or "/"
+ if tree_path not in tree_paths:
+ tree_paths.append(tree_path)
+ return tree_paths
@dataclass
@@ -30,7 +54,31 @@ class SemanticLockScope:
if caller_lock.active:
return cls(caller_lock.as_borrowed())
if lock_handoff:
- return cls(await OwnedLockLease.from_handoff(lock_handoff))
+ manager = get_lock_manager()
+ try:
+ return cls(await OwnedLockLease.from_handoff(lock_handoff, manager=manager))
+ except LockAcquisitionError as exc:
+ tree_paths = _tree_paths_from_handoff(lock_handoff.lock_paths)
+ if not tree_paths:
+ raise
+
+ handle = manager.create_handle()
+ if len(tree_paths) == 1:
+ acquired = await manager.acquire_tree(handle, tree_paths[0])
+ else:
+ acquired = await manager.acquire_tree_batch(handle, tree_paths)
+ if not acquired:
+ await manager.release(handle)
+ raise LockAcquisitionError(
+ f"Failed to reacquire semantic lock for {tree_paths}"
+ ) from exc
+
+ logger.info(
+ "Recovered semantic lock handoff %s by reacquiring %s",
+ lock_handoff.handle_id,
+ tree_paths,
+ )
+ return cls(OwnedLockLease.from_handle(manager, handle))
return cls(NO_LOCK)
async def close(self) -> None:
diff --git a/openviking/storage/queuefs/semantic_processor.py b/openviking/storage/queuefs/semantic_processor.py
index 9ed9f97b7..6b5d0ee29 100644
--- a/openviking/storage/queuefs/semantic_processor.py
+++ b/openviking/storage/queuefs/semantic_processor.py
@@ -27,6 +27,7 @@ from openviking.parse.parsers.media.utils import (
)
from openviking.prompts import render_prompt
from openviking.server.identity import RequestContext, Role
+from openviking.storage.errors import LockAcquisitionError
from openviking.storage.queuefs.named_queue import DequeueHandlerBase
from openviking.storage.queuefs.semantic_dag import DagStats, SemanticDagExecutor
from openviking.storage.queuefs.semantic_lock import SemanticLockScope
@@ -239,6 +240,25 @@ class SemanticProcessor(DequeueHandlerBase):
else:
logger.warning(f"No queue manager available, cannot re-enqueue: {msg.uri}")
+ async def _requeue_semantic_msg_after_error(
+ self,
+ msg: SemanticMsg,
+ data: Optional[Dict[str, Any]],
+ error: Exception,
+ ) -> None:
+ try:
+ await self._reenqueue_semantic_msg(msg)
+ self._merge_request_stats(msg.telemetry_id, requeue_count=1)
+ get_request_wait_tracker().record_semantic_requeue(msg.telemetry_id)
+ self.report_requeue()
+ except Exception as requeue_err:
+ logger.error(f"Failed to re-enqueue semantic message: {requeue_err}")
+ self._merge_request_stats(msg.telemetry_id, error_count=1)
+ get_request_wait_tracker().mark_semantic_failed(msg.telemetry_id, msg.id, str(error))
+ self.report_error(str(error), data)
+ return
+ self.report_success()
+
async def _enqueue_parent_refresh(self, msg: SemanticMsg, uri: str) -> None:
if msg.context_type not in {"resource", "skill"}:
return
@@ -440,6 +460,19 @@ class SemanticProcessor(DequeueHandlerBase):
reset_root_observability_context(root_context_token)
except Exception as e:
+ if isinstance(e, LockAcquisitionError):
+ logger.warning(
+ "Lock error processing semantic message, re-enqueueing without "
+ "tripping API circuit breaker: %s",
+ e,
+ exc_info=True,
+ )
+ if msg is not None:
+ await self._requeue_semantic_msg_after_error(msg, data, e)
+ else:
+ self.report_error(str(e), data)
+ return None
+
error_class = classify_api_error(e)
if error_class == ERROR_CLASS_INPUT_TOO_LARGE:
logger.error(
@@ -472,20 +505,7 @@ class SemanticProcessor(DequeueHandlerBase):
)
self._circuit_breaker.record_failure(e)
if msg is not None:
- try:
- await self._reenqueue_semantic_msg(msg)
- self._merge_request_stats(msg.telemetry_id, requeue_count=1)
- get_request_wait_tracker().record_semantic_requeue(msg.telemetry_id)
- self.report_requeue()
- except Exception as requeue_err:
- logger.error(f"Failed to re-enqueue semantic message: {requeue_err}")
- self._merge_request_stats(msg.telemetry_id, error_count=1)
- get_request_wait_tracker().mark_semantic_failed(
- msg.telemetry_id, msg.id, str(e)
- )
- self.report_error(str(e), data)
- return None
- self.report_success()
+ await self._requeue_semantic_msg_after_error(msg, data, e)
else:
self.report_error(str(e), data)
return None
diff --git a/openviking/storage/transaction/__init__.py b/openviking/storage/transaction/__init__.py
index c635ce6b1..cdcd30756 100644
--- a/openviking/storage/transaction/__init__.py
+++ b/openviking/storage/transaction/__init__.py
@@ -16,6 +16,7 @@ from openviking.storage.transaction.lock_lease import (
OwnedLockLease,
)
from openviking.storage.transaction.lock_manager import (
+ LOCK_TIMEOUT_DEFAULT,
LockManager,
get_lock_handle_async,
get_lock_manager,
@@ -32,6 +33,7 @@ __all__ = [
"LockHandle",
"LockHandoffRef",
"LockLease",
+ "LOCK_TIMEOUT_DEFAULT",
"LockManager",
"LockOwner",
"NO_LOCK",
diff --git a/openviking/storage/transaction/lock_lease.py b/openviking/storage/transaction/lock_lease.py
index 3b91b6b1e..3cbf26ddb 100644
--- a/openviking/storage/transaction/lock_lease.py
+++ b/openviking/storage/transaction/lock_lease.py
@@ -11,7 +11,11 @@ from typing import Any, Iterable, Optional
from openviking.storage.errors import LockAcquisitionError
from openviking.storage.transaction.lock_handle import LockHandle
-from openviking.storage.transaction.lock_manager import LockManager, get_lock_manager
+from openviking.storage.transaction.lock_manager import (
+ LOCK_TIMEOUT_DEFAULT,
+ LockManager,
+ get_lock_manager,
+)
from openviking_cli.utils.logger import get_logger
logger = get_logger(__name__)
@@ -139,7 +143,7 @@ class OwnedLockLease(LockLease):
manager: LockManager,
path: str,
*,
- timeout: Optional[float] = None,
+ timeout: Any = LOCK_TIMEOUT_DEFAULT,
) -> "OwnedLockLease":
handle = manager.create_handle()
if await manager.acquire_tree(handle, path, timeout=timeout):
@@ -153,7 +157,7 @@ class OwnedLockLease(LockLease):
manager: LockManager,
paths: Iterable[str],
*,
- timeout: Optional[float] = None,
+ timeout: Any = LOCK_TIMEOUT_DEFAULT,
) -> "OwnedLockLease":
handle = manager.create_handle()
path_list = list(paths)
diff --git a/openviking/storage/transaction/lock_manager.py b/openviking/storage/transaction/lock_manager.py
index 3d9bfc217..89e0b269c 100644
--- a/openviking/storage/transaction/lock_manager.py
+++ b/openviking/storage/transaction/lock_manager.py
@@ -16,7 +16,7 @@ from openviking_cli.utils.logger import get_logger
logger = get_logger(__name__)
_HANDLE_CLEANUP_INTERVAL_SECONDS = 60.0
-_USE_MANAGER_DEFAULT_TIMEOUT = object()
+LOCK_TIMEOUT_DEFAULT = object()
class _LockManagerLike(Protocol):
@@ -61,8 +61,8 @@ class LockManager:
def redo_recovery_enabled(self) -> bool:
return self._redo_recovery_enabled
- def _resolve_timeout(self, timeout: Optional[float]) -> Optional[float]:
- return self._lock_timeout if timeout is _USE_MANAGER_DEFAULT_TIMEOUT else timeout
+ def _resolve_timeout(self, timeout: Any) -> Optional[float]:
+ return self._lock_timeout if timeout is LOCK_TIMEOUT_DEFAULT else timeout
def _mark_handle_active(self, handle: LockHandle) -> None:
handle.last_active_at = time.time()
@@ -123,7 +123,7 @@ class LockManager:
self,
handle: LockHandle,
path: str,
- timeout: Optional[float] = _USE_MANAGER_DEFAULT_TIMEOUT,
+ timeout: Any = LOCK_TIMEOUT_DEFAULT,
) -> bool:
acquired = await self._path_lock.acquire_exact_path(
path, handle, timeout=self._resolve_timeout(timeout)
@@ -136,7 +136,7 @@ class LockManager:
self,
handle: LockHandle,
path: str,
- timeout: Optional[float] = _USE_MANAGER_DEFAULT_TIMEOUT,
+ timeout: Any = LOCK_TIMEOUT_DEFAULT,
) -> bool:
acquired = await self._path_lock.acquire_tree(
path, handle, timeout=self._resolve_timeout(timeout)
@@ -149,7 +149,7 @@ class LockManager:
self,
handle: LockHandle,
paths: List[str],
- timeout: Optional[float] = _USE_MANAGER_DEFAULT_TIMEOUT,
+ timeout: Any = LOCK_TIMEOUT_DEFAULT,
) -> bool:
"""
一次性对多个路径进行树锁加锁,使用有序加锁法防止死锁
@@ -203,7 +203,7 @@ class LockManager:
self,
handle: LockHandle,
paths: List[str],
- timeout: Optional[float] = _USE_MANAGER_DEFAULT_TIMEOUT,
+ timeout: Any = LOCK_TIMEOUT_DEFAULT,
) -> bool:
if not paths:
self._mark_handle_active(handle)
@@ -239,7 +239,7 @@ class LockManager:
handle: LockHandle,
exact_paths: List[str],
tree_paths: List[str],
- timeout: Optional[float] = _USE_MANAGER_DEFAULT_TIMEOUT,
+ timeout: Any = LOCK_TIMEOUT_DEFAULT,
) -> bool:
exact_set = set(exact_paths)
tree_set = set(tree_paths)
@@ -288,7 +288,7 @@ class LockManager:
src: str,
dst: str,
src_is_dir: bool = True,
- timeout: Optional[float] = _USE_MANAGER_DEFAULT_TIMEOUT,
+ timeout: Any = LOCK_TIMEOUT_DEFAULT,
) -> bool:
acquired = await self._path_lock.acquire_mv(
src,
diff --git a/openviking/utils/resource_processor.py b/openviking/utils/resource_processor.py
index 29793ed31..fdfa3edee 100644
--- a/openviking/utils/resource_processor.py
+++ b/openviking/utils/resource_processor.py
@@ -14,7 +14,12 @@ from openviking.parse.tree_builder import TreeBuilder
from openviking.server.identity import RequestContext
from openviking.storage import VikingDBManager
from openviking.storage.errors import LockAcquisitionError
-from openviking.storage.transaction import NO_LOCK, LockLease, OwnedLockLease
+from openviking.storage.transaction import (
+ LOCK_TIMEOUT_DEFAULT,
+ NO_LOCK,
+ LockLease,
+ OwnedLockLease,
+)
from openviking.storage.viking_fs import get_viking_fs
from openviking.telemetry import get_current_telemetry
from openviking.utils.embedding_utils import index_resource
@@ -390,7 +395,7 @@ class ResourceProcessor:
path: str,
*,
uri: str = "",
- timeout: Optional[float] = None,
+ timeout: Any = LOCK_TIMEOUT_DEFAULT,
) -> OwnedLockLease:
"""Acquire the per-resource TreeLock or raise a structured conflict."""
from openviking.storage.errors import ResourceBusyError
diff --git a/tests/storage/test_semantic_processor_lock_ownership.py b/tests/storage/test_semantic_processor_lock_ownership.py
index 95c2a652b..b5ff173d3 100644
--- a/tests/storage/test_semantic_processor_lock_ownership.py
+++ b/tests/storage/test_semantic_processor_lock_ownership.py
@@ -5,10 +5,12 @@ from __future__ import annotations
import pytest
+from openviking.storage.errors import LockAcquisitionError
from openviking.storage.queuefs.semantic_dag import DagStats
+from openviking.storage.queuefs.semantic_lock import SemanticLockScope
from openviking.storage.queuefs.semantic_msg import SemanticMsg
from openviking.storage.queuefs.semantic_processor import SemanticProcessor
-from openviking.storage.transaction import BorrowedLockLease
+from openviking.storage.transaction import BorrowedLockLease, LockHandoffRef
class _FakeHandle:
@@ -88,3 +90,116 @@ async def test_semantic_processor_borrows_caller_owned_lock(monkeypatch):
)
assert lock_manager.release_calls == []
+
+
+async def test_semantic_lock_scope_reacquires_tree_lock_when_handoff_handle_is_stale(
+ monkeypatch,
+):
+ class _RecoveringLockManager:
+ def __init__(self):
+ self._handles = {}
+ self.acquire_tree_calls = []
+ self.release_calls = []
+
+ async def get_handle_async(self, handle_id):
+ return self._handles.get(handle_id)
+
+ async def adopt_handle_async(self, handle_id, lock_paths):
+ del handle_id, lock_paths
+ return None
+
+ def get_handle(self, handle_id):
+ return self._handles.get(handle_id)
+
+ def create_handle(self):
+ handle = _FakeHandle("new-lock")
+ handle.locks = []
+ self._handles[handle.id] = handle
+ return handle
+
+ async def acquire_tree(self, handle, lock_path):
+ self.acquire_tree_calls.append(lock_path)
+ handle.locks.append(f"{lock_path}/.path.ovlock")
+ return True
+
+ async def release(self, handle):
+ self.release_calls.append(handle.id)
+ self._handles.pop(handle.id, None)
+
+ lock_manager = _RecoveringLockManager()
+ monkeypatch.setattr(
+ "openviking.storage.queuefs.semantic_lock.get_lock_manager",
+ lambda: lock_manager,
+ )
+
+ scope = await SemanticLockScope.resolve(
+ LockHandoffRef(
+ handle_id="stale-lock",
+ lock_paths=("/local/default/resources/CONTRIBUTING_CN_3/.path.ovlock",),
+ )
+ )
+
+ try:
+ assert scope.lock.handle_id == "new-lock"
+ assert lock_manager.acquire_tree_calls == ["/local/default/resources/CONTRIBUTING_CN_3"]
+ finally:
+ await scope.close()
+
+ assert lock_manager.release_calls == ["new-lock"]
+
+
+async def test_semantic_processor_lock_error_requeues_without_circuit_breaker(monkeypatch):
+ processor = SemanticProcessor()
+ reenqueue_calls = []
+ success_called = False
+ requeue_called = False
+ error_called = False
+
+ async def _reenqueue(msg):
+ reenqueue_calls.append(msg.uri)
+
+ def on_success():
+ nonlocal success_called
+ success_called = True
+
+ def on_requeue():
+ nonlocal requeue_called
+ requeue_called = True
+
+ def on_error(error_msg, error_data=None):
+ del error_msg, error_data
+ nonlocal error_called
+ error_called = True
+
+ def _record_failure(error):
+ raise AssertionError(f"lock errors must not trip circuit breaker: {error}")
+
+ processor.set_callbacks(on_success, on_requeue, on_error)
+ processor._circuit_breaker.record_failure = _record_failure
+
+ monkeypatch.setattr(processor, "_reenqueue_semantic_msg", _reenqueue)
+ monkeypatch.setattr(
+ "openviking.storage.queuefs.semantic_processor.SemanticLockScope.resolve",
+ lambda *args, **kwargs: (_ for _ in ()).throw(
+ LockAcquisitionError("lock handle is no longer active")
+ ),
+ )
+
+ msg = SemanticMsg(
+ uri="viking://resources/CONTRIBUTING_CN_3",
+ context_type="resource",
+ recursive=True,
+ lock_handoff=LockHandoffRef(
+ handle_id="stale-lock",
+ lock_paths=("/local/default/resources/CONTRIBUTING_CN_3/.path.ovlock",),
+ ),
+ )
+
+ await processor.on_dequeue(msg.to_dict())
+
+ assert reenqueue_calls == ["viking://resources/CONTRIBUTING_CN_3"]
+ assert requeue_called is True
+ assert success_called is True
+ assert error_called is False