Self-evolving Context Database for AI Agents. Unify Agent Memory, Knowledge RAG and Skills.
Python agent-contextagent-memoryagent-pluginsagentic-ragcontext-databasedsh-pluginknowledge-basememorymemory-managementself-evolvingvector-database
fix(storage): recover stale semantic lock handoffs (#2214)
Qin Haojiec99623feee10
diff --git a/openviking/storage/queuefs/semantic_lock.py b/openviking/storage/queuefs/semantic_lock.pyindex 9e00165e5..88ca0b616 100644--- a/openviking/storage/queuefs/semantic_lock.py+++ b/openviking/storage/queuefs/semantic_lock.py@@ -5,9 +5,33 @@from __future__ import annotationsfrom dataclasses import dataclass-from typing import Optional+from typing import Iterable, Optional-from openviking.storage.transaction import NO_LOCK, LockHandoffRef, LockLease, OwnedLockLease+from openviking.storage.errors import LockAcquisitionError+from openviking.storage.transaction import (+ NO_LOCK,+ LockHandoffRef,+ LockLease,+ OwnedLockLease,+ get_lock_manager,+)+from openviking.storage.transaction.path_lock import LOCK_FILE_NAME+from openviking_cli.utils.logger import get_logger++logger = get_logger(__name__)++_TREE_LOCK_SUFFIX = f"/{LOCK_FILE_NAME}"+++def _tree_paths_from_handoff(lock_paths: Iterable[str]) -> list[str]:+ tree_paths: list[str] = []+ for lock_path in lock_paths:+ if not lock_path.endswith(_TREE_LOCK_SUFFIX):+ continue+ tree_path = lock_path[: -len(_TREE_LOCK_SUFFIX)] or "/"+ if tree_path not in tree_paths:+ tree_paths.append(tree_path)+ return tree_paths@dataclass@@ -30,7 +54,31 @@ class SemanticLockScope:if caller_lock.active:return cls(caller_lock.as_borrowed())if lock_handoff:- return cls(await OwnedLockLease.from_handoff(lock_handoff))+ manager = get_lock_manager()+ try:+ return cls(await OwnedLockLease.from_handoff(lock_handoff, manager=manager))+ except LockAcquisitionError as exc:+ tree_paths = _tree_paths_from_handoff(lock_handoff.lock_paths)+ if not tree_paths:+ raise++ handle = manager.create_handle()+ if len(tree_paths) == 1:+ acquired = await manager.acquire_tree(handle, tree_paths[0])+ else:+ acquired = await manager.acquire_tree_batch(handle, tree_paths)+ if not acquired:+ await manager.release(handle)+ raise LockAcquisitionError(+ f"Failed to reacquire semantic lock for {tree_paths}"+ ) from exc++ logger.info(+ "Recovered semantic lock handoff %s by reacquiring %s",+ lock_handoff.handle_id,+ tree_paths,+ )+ return cls(OwnedLockLease.from_handle(manager, handle))return cls(NO_LOCK)async def close(self) -> None:diff --git a/openviking/storage/queuefs/semantic_processor.py b/openviking/storage/queuefs/semantic_processor.pyindex 9ed9f97b7..6b5d0ee29 100644--- a/openviking/storage/queuefs/semantic_processor.py+++ b/openviking/storage/queuefs/semantic_processor.py@@ -27,6 +27,7 @@ from openviking.parse.parsers.media.utils import ()from openviking.prompts import render_promptfrom openviking.server.identity import RequestContext, Role+from openviking.storage.errors import LockAcquisitionErrorfrom openviking.storage.queuefs.named_queue import DequeueHandlerBasefrom openviking.storage.queuefs.semantic_dag import DagStats, SemanticDagExecutorfrom openviking.storage.queuefs.semantic_lock import SemanticLockScope@@ -239,6 +240,25 @@ class SemanticProcessor(DequeueHandlerBase):else:logger.warning(f"No queue manager available, cannot re-enqueue: {msg.uri}")+ async def _requeue_semantic_msg_after_error(+ self,+ msg: SemanticMsg,+ data: Optional[Dict[str, Any]],+ error: Exception,+ ) -> None:+ try:+ await self._reenqueue_semantic_msg(msg)+ self._merge_request_stats(msg.telemetry_id, requeue_count=1)+ get_request_wait_tracker().record_semantic_requeue(msg.telemetry_id)+ self.report_requeue()+ except Exception as requeue_err:+ logger.error(f"Failed to re-enqueue semantic message: {requeue_err}")+ self._merge_request_stats(msg.telemetry_id, error_count=1)+ get_request_wait_tracker().mark_semantic_failed(msg.telemetry_id, msg.id, str(error))+ self.report_error(str(error), data)+ return+ self.report_success()+async def _enqueue_parent_refresh(self, msg: SemanticMsg, uri: str) -> None:if msg.context_type not in {"resource", "skill"}:return@@ -440,6 +460,19 @@ class SemanticProcessor(DequeueHandlerBase):reset_root_observability_context(root_context_token)except Exception as e:+ if isinstance(e, LockAcquisitionError):+ logger.warning(+ "Lock error processing semantic message, re-enqueueing without "+ "tripping API circuit breaker: %s",+ e,+ exc_info=True,+ )+ if msg is not None:+ await self._requeue_semantic_msg_after_error(msg, data, e)+ else:+ self.report_error(str(e), data)+ return None+error_class = classify_api_error(e)if error_class == ERROR_CLASS_INPUT_TOO_LARGE:logger.error(@@ -472,20 +505,7 @@ class SemanticProcessor(DequeueHandlerBase):)self._circuit_breaker.record_failure(e)if msg is not None:- try:- await self._reenqueue_semantic_msg(msg)- self._merge_request_stats(msg.telemetry_id, requeue_count=1)- get_request_wait_tracker().record_semantic_requeue(msg.telemetry_id)- self.report_requeue()- except Exception as requeue_err:- logger.error(f"Failed to re-enqueue semantic message: {requeue_err}")- self._merge_request_stats(msg.telemetry_id, error_count=1)- get_request_wait_tracker().mark_semantic_failed(- msg.telemetry_id, msg.id, str(e)- )- self.report_error(str(e), data)- return None- self.report_success()+ await self._requeue_semantic_msg_after_error(msg, data, e)else:self.report_error(str(e), data)return Nonediff --git a/openviking/storage/transaction/__init__.py b/openviking/storage/transaction/__init__.pyindex c635ce6b1..cdcd30756 100644--- a/openviking/storage/transaction/__init__.py+++ b/openviking/storage/transaction/__init__.py@@ -16,6 +16,7 @@ from openviking.storage.transaction.lock_lease import (OwnedLockLease,)from openviking.storage.transaction.lock_manager import (+ LOCK_TIMEOUT_DEFAULT,LockManager,get_lock_handle_async,get_lock_manager,@@ -32,6 +33,7 @@ __all__ = ["LockHandle","LockHandoffRef","LockLease",+ "LOCK_TIMEOUT_DEFAULT","LockManager","LockOwner","NO_LOCK",diff --git a/openviking/storage/transaction/lock_lease.py b/openviking/storage/transaction/lock_lease.pyindex 3b91b6b1e..3cbf26ddb 100644--- a/openviking/storage/transaction/lock_lease.py+++ b/openviking/storage/transaction/lock_lease.py@@ -11,7 +11,11 @@ from typing import Any, Iterable, Optionalfrom openviking.storage.errors import LockAcquisitionErrorfrom openviking.storage.transaction.lock_handle import LockHandle-from openviking.storage.transaction.lock_manager import LockManager, get_lock_manager+from openviking.storage.transaction.lock_manager import (+ LOCK_TIMEOUT_DEFAULT,+ LockManager,+ get_lock_manager,+)from openviking_cli.utils.logger import get_loggerlogger = get_logger(__name__)@@ -139,7 +143,7 @@ class OwnedLockLease(LockLease):manager: LockManager,path: str,*,- timeout: Optional[float] = None,+ timeout: Any = LOCK_TIMEOUT_DEFAULT,) -> "OwnedLockLease":handle = manager.create_handle()if await manager.acquire_tree(handle, path, timeout=timeout):@@ -153,7 +157,7 @@ class OwnedLockLease(LockLease):manager: LockManager,paths: Iterable[str],*,- timeout: Optional[float] = None,+ timeout: Any = LOCK_TIMEOUT_DEFAULT,) -> "OwnedLockLease":handle = manager.create_handle()path_list = list(paths)diff --git a/openviking/storage/transaction/lock_manager.py b/openviking/storage/transaction/lock_manager.pyindex 3d9bfc217..89e0b269c 100644--- a/openviking/storage/transaction/lock_manager.py+++ b/openviking/storage/transaction/lock_manager.py@@ -16,7 +16,7 @@ from openviking_cli.utils.logger import get_loggerlogger = get_logger(__name__)_HANDLE_CLEANUP_INTERVAL_SECONDS = 60.0-_USE_MANAGER_DEFAULT_TIMEOUT = object()+LOCK_TIMEOUT_DEFAULT = object()class _LockManagerLike(Protocol):@@ -61,8 +61,8 @@ class LockManager:def redo_recovery_enabled(self) -> bool:return self._redo_recovery_enabled- def _resolve_timeout(self, timeout: Optional[float]) -> Optional[float]:- return self._lock_timeout if timeout is _USE_MANAGER_DEFAULT_TIMEOUT else timeout+ def _resolve_timeout(self, timeout: Any) -> Optional[float]:+ return self._lock_timeout if timeout is LOCK_TIMEOUT_DEFAULT else timeoutdef _mark_handle_active(self, handle: LockHandle) -> None:handle.last_active_at = time.time()@@ -123,7 +123,7 @@ class LockManager:self,handle: LockHandle,path: str,- timeout: Optional[float] = _USE_MANAGER_DEFAULT_TIMEOUT,+ timeout: Any = LOCK_TIMEOUT_DEFAULT,) -> bool:acquired = await self._path_lock.acquire_exact_path(path, handle, timeout=self._resolve_timeout(timeout)@@ -136,7 +136,7 @@ class LockManager:self,handle: LockHandle,path: str,- timeout: Optional[float] = _USE_MANAGER_DEFAULT_TIMEOUT,+ timeout: Any = LOCK_TIMEOUT_DEFAULT,) -> bool:acquired = await self._path_lock.acquire_tree(path, handle, timeout=self._resolve_timeout(timeout)@@ -149,7 +149,7 @@ class LockManager:self,handle: LockHandle,paths: List[str],- timeout: Optional[float] = _USE_MANAGER_DEFAULT_TIMEOUT,+ timeout: Any = LOCK_TIMEOUT_DEFAULT,) -> bool:"""一次性对多个路径进行树锁加锁,使用有序加锁法防止死锁@@ -203,7 +203,7 @@ class LockManager:self,handle: LockHandle,paths: List[str],- timeout: Optional[float] = _USE_MANAGER_DEFAULT_TIMEOUT,+ timeout: Any = LOCK_TIMEOUT_DEFAULT,) -> bool:if not paths:self._mark_handle_active(handle)@@ -239,7 +239,7 @@ class LockManager:handle: LockHandle,exact_paths: List[str],tree_paths: List[str],- timeout: Optional[float] = _USE_MANAGER_DEFAULT_TIMEOUT,+ timeout: Any = LOCK_TIMEOUT_DEFAULT,) -> bool:exact_set = set(exact_paths)tree_set = set(tree_paths)@@ -288,7 +288,7 @@ class LockManager:src: str,dst: str,src_is_dir: bool = True,- timeout: Optional[float] = _USE_MANAGER_DEFAULT_TIMEOUT,+ timeout: Any = LOCK_TIMEOUT_DEFAULT,) -> bool:acquired = await self._path_lock.acquire_mv(src,diff --git a/openviking/utils/resource_processor.py b/openviking/utils/resource_processor.pyindex 29793ed31..fdfa3edee 100644--- a/openviking/utils/resource_processor.py+++ b/openviking/utils/resource_processor.py@@ -14,7 +14,12 @@ from openviking.parse.tree_builder import TreeBuilderfrom openviking.server.identity import RequestContextfrom openviking.storage import VikingDBManagerfrom openviking.storage.errors import LockAcquisitionError-from openviking.storage.transaction import NO_LOCK, LockLease, OwnedLockLease+from openviking.storage.transaction import (+ LOCK_TIMEOUT_DEFAULT,+ NO_LOCK,+ LockLease,+ OwnedLockLease,+)from openviking.storage.viking_fs import get_viking_fsfrom openviking.telemetry import get_current_telemetryfrom openviking.utils.embedding_utils import index_resource@@ -390,7 +395,7 @@ class ResourceProcessor:path: str,*,uri: str = "",- timeout: Optional[float] = None,+ timeout: Any = LOCK_TIMEOUT_DEFAULT,) -> OwnedLockLease:"""Acquire the per-resource TreeLock or raise a structured conflict."""from openviking.storage.errors import ResourceBusyErrordiff --git a/tests/storage/test_semantic_processor_lock_ownership.py b/tests/storage/test_semantic_processor_lock_ownership.pyindex 95c2a652b..b5ff173d3 100644--- a/tests/storage/test_semantic_processor_lock_ownership.py+++ b/tests/storage/test_semantic_processor_lock_ownership.py@@ -5,10 +5,12 @@ from __future__ import annotationsimport pytest+from openviking.storage.errors import LockAcquisitionErrorfrom openviking.storage.queuefs.semantic_dag import DagStats+from openviking.storage.queuefs.semantic_lock import SemanticLockScopefrom openviking.storage.queuefs.semantic_msg import SemanticMsgfrom openviking.storage.queuefs.semantic_processor import SemanticProcessor-from openviking.storage.transaction import BorrowedLockLease+from openviking.storage.transaction import BorrowedLockLease, LockHandoffRefclass _FakeHandle:@@ -88,3 +90,116 @@ async def test_semantic_processor_borrows_caller_owned_lock(monkeypatch):)assert lock_manager.release_calls == []+++async def test_semantic_lock_scope_reacquires_tree_lock_when_handoff_handle_is_stale(+ monkeypatch,+):+ class _RecoveringLockManager:+ def __init__(self):+ self._handles = {}+ self.acquire_tree_calls = []+ self.release_calls = []++ async def get_handle_async(self, handle_id):+ return self._handles.get(handle_id)++ async def adopt_handle_async(self, handle_id, lock_paths):+ del handle_id, lock_paths+ return None++ def get_handle(self, handle_id):+ return self._handles.get(handle_id)++ def create_handle(self):+ handle = _FakeHandle("new-lock")+ handle.locks = []+ self._handles[handle.id] = handle+ return handle++ async def acquire_tree(self, handle, lock_path):+ self.acquire_tree_calls.append(lock_path)+ handle.locks.append(f"{lock_path}/.path.ovlock")+ return True++ async def release(self, handle):+ self.release_calls.append(handle.id)+ self._handles.pop(handle.id, None)++ lock_manager = _RecoveringLockManager()+ monkeypatch.setattr(+ "openviking.storage.queuefs.semantic_lock.get_lock_manager",+ lambda: lock_manager,+ )++ scope = await SemanticLockScope.resolve(+ LockHandoffRef(+ handle_id="stale-lock",+ lock_paths=("/local/default/resources/CONTRIBUTING_CN_3/.path.ovlock",),+ )+ )++ try:+ assert scope.lock.handle_id == "new-lock"+ assert lock_manager.acquire_tree_calls == ["/local/default/resources/CONTRIBUTING_CN_3"]+ finally:+ await scope.close()++ assert lock_manager.release_calls == ["new-lock"]+++async def test_semantic_processor_lock_error_requeues_without_circuit_breaker(monkeypatch):+ processor = SemanticProcessor()+ reenqueue_calls = []+ success_called = False+ requeue_called = False+ error_called = False++ async def _reenqueue(msg):+ reenqueue_calls.append(msg.uri)++ def on_success():+ nonlocal success_called+ success_called = True++ def on_requeue():+ nonlocal requeue_called+ requeue_called = True++ def on_error(error_msg, error_data=None):+ del error_msg, error_data+ nonlocal error_called+ error_called = True++ def _record_failure(error):+ raise AssertionError(f"lock errors must not trip circuit breaker: {error}")++ processor.set_callbacks(on_success, on_requeue, on_error)+ processor._circuit_breaker.record_failure = _record_failure++ monkeypatch.setattr(processor, "_reenqueue_semantic_msg", _reenqueue)+ monkeypatch.setattr(+ "openviking.storage.queuefs.semantic_processor.SemanticLockScope.resolve",+ lambda *args, **kwargs: (_ for _ in ()).throw(+ LockAcquisitionError("lock handle is no longer active")+ ),+ )++ msg = SemanticMsg(+ uri="viking://resources/CONTRIBUTING_CN_3",+ context_type="resource",+ recursive=True,+ lock_handoff=LockHandoffRef(+ handle_id="stale-lock",+ lock_paths=("/local/default/resources/CONTRIBUTING_CN_3/.path.ovlock",),+ ),+ )++ await processor.on_dequeue(msg.to_dict())++ assert reenqueue_calls == ["viking://resources/CONTRIBUTING_CN_3"]+ assert requeue_called is True+ assert success_called is True+ assert error_called is False