Self-evolving Context Database for AI Agents. Unify Agent Memory, Knowledge RAG and Skills.
Python agent-contextagent-memoryagent-pluginsagentic-ragcontext-databasedsh-pluginknowledge-basememorymemory-managementself-evolvingvector-database
fix(storage): preserve semantic lock ownership (#2140)
Qin Haojiefd9ada92959c
diff --git a/openviking/service/reindex_executor.py b/openviking/service/reindex_executor.pyindex 6de627e76..42ebd1de8 100644--- a/openviking/service/reindex_executor.py+++ b/openviking/service/reindex_executor.py@@ -25,12 +25,19 @@ from openviking.server.dependencies import get_servicefrom openviking.server.identity import RequestContextfrom openviking.service.task_tracker import get_task_trackerfrom openviking.session.memory.utils.messages import parse_memory_file_with_fields-from openviking.storage.collection_schemas import TextEmbeddingHandlerfrom openviking.storage.queuefs.embedding_msg_converter import EmbeddingMsgConverterfrom openviking.storage.queuefs.semantic_msg import SemanticMsgfrom openviking.storage.queuefs.semantic_processor import SemanticProcessor-from openviking.storage.transaction import LockContext, get_lock_manager+from openviking.storage.transaction import (+ NO_LOCK,+ BorrowedLockLease,+ LockContext,+ LockLease,+ get_lock_manager,+)from openviking.storage.viking_fs import get_viking_fs+from openviking.telemetry import get_current_telemetry+from openviking.telemetry.request_wait_tracker import get_request_wait_trackerfrom openviking.utils.embedding_utils import get_resource_content_typefrom openviking.utils.skill_processor import SkillProcessorfrom openviking_cli.exceptions import NotFoundError, OpenVikingError@@ -60,6 +67,13 @@ class _ReindexCounters:warnings: list[str] = field(default_factory=list)+@dataclass+class _ReindexRunContext:+ ctx: RequestContext+ counters: _ReindexCounters+ lock: LockLease = NO_LOCK++class ReindexExecutor:"""Non-destructive reindex orchestration for admin maintenance flows."""@@ -196,9 +210,10 @@ class ReindexExecutor:target_root: str,directories: list[str],files: list[str],- counters: _ReindexCounters,- ctx: RequestContext,+ run: _ReindexRunContext,) -> tuple[list[str], list[str]]:+ counters = run.counters+ ctx = run.ctxprefix = self._child_prefix(target_root)semantic_roots = sorted({@@ -235,6 +250,7 @@ class ReindexExecutor:uri=semantic_root,context_type="resource",ctx=ctx,+ lock=run.lock,)return filtered_directories, filtered_files@@ -244,6 +260,22 @@ class ReindexExecutor:return "viking://"return root.rstrip("/") + "/"+ @staticmethod+ def _apply_embedding_wait_status(+ counters: _ReindexCounters,+ queue_status: dict[str, Any],+ ) -> None:+ embedding_status = queue_status.get("Embedding") or {}+ error_count = int(embedding_status.get("error_count", 0) or 0)+ if error_count <= 0:+ return+ counters.failed_records += error_count+ counters.rebuilt_records = max(0, counters.rebuilt_records - error_count)+ for error in embedding_status.get("errors", []) or []:+ message = error.get("message") if isinstance(error, dict) else str(error)+ if message:+ counters.warnings.append(f"Embedding queue failed during reindex: {message}")+def _is_resource_entry_for_namespace(self, uri: str, target_root: str) -> bool:if not uri.startswith(self._child_prefix(target_root)):return False@@ -260,9 +292,10 @@ class ReindexExecutor:*,uri: str,mode: str,- counters: _ReindexCounters,- ctx: RequestContext,+ run: _ReindexRunContext,) -> None:+ counters = run.counters+ ctx = run.ctxviking_fs = get_viking_fs()try:entries = await self._tree_all(viking_fs, uri, show_all_hidden=True, ctx=ctx)@@ -276,7 +309,11 @@ class ReindexExecutor:skill_roots.append(entry_uri)for skill_root in sorted(set(skill_roots)):- await self._reindex_skill(uri=skill_root, mode=mode, counters=counters, ctx=ctx)+ await self._reindex_skill(+ uri=skill_root,+ mode=mode,+ run=run,+ )if not skill_roots:counters.unsupported_records += 1@@ -306,52 +343,86 @@ class ReindexExecutor:service = get_service()if service.viking_fs is None or service.vikingdb_manager is None:raise RuntimeError("OpenVikingService not initialized")+ if not service.vikingdb_manager.has_queue_manager:+ raise OpenVikingError(+ "Reindex requires embedding queue",+ code="FAILED_PRECONDITION",+ details={"uri": uri},+ )path = service.viking_fs._uri_to_path(uri, ctx=ctx)started_at = time.perf_counter()counters = _ReindexCounters()+ telemetry_id = get_current_telemetry().telemetry_id+ wait_tracker = get_request_wait_tracker()+ if telemetry_id:+ wait_tracker.register_request(telemetry_id)- async with LockContext(get_lock_manager(), [path], lock_mode="tree"):- if object_type == "global_namespace":- await self._reindex_global_namespace(- uri=uri,- mode=mode,- counters=counters,- ctx=ctx,- )- elif object_type == "agent_namespace":- await self._reindex_agent_namespace(- uri=uri,- mode=mode,- counters=counters,- ctx=ctx,- )- elif object_type == "user_namespace":- await self._reindex_user_namespace(- uri=uri,- mode=mode,- counters=counters,+ try:+ async with LockContext(get_lock_manager(), [path], lock_mode="tree") as lock_handle:+ run = _ReindexRunContext(ctx=ctx,- )- elif object_type == "skill_namespace":- await self._reindex_skill_namespace(- uri=uri,- mode=mode,counters=counters,- ctx=ctx,- )- elif object_type == "resource":- await self._reindex_resource(uri=uri, mode=mode, counters=counters, ctx=ctx)- elif object_type == "skill":- await self._reindex_skill(uri=uri, mode=mode, counters=counters, ctx=ctx)- elif object_type == "memory":- await self._reindex_memory(uri=uri, mode=mode, counters=counters, ctx=ctx)- else:- raise OpenVikingError(- f"Unsupported reindex type: {object_type}",- code="UNSUPPORTED_URI",- details={"uri": uri},+ lock=BorrowedLockLease.from_handle(get_lock_manager(), lock_handle),)+ if object_type == "global_namespace":+ await self._reindex_global_namespace(+ uri=uri,+ mode=mode,+ run=run,+ )+ elif object_type == "agent_namespace":+ await self._reindex_agent_namespace(+ uri=uri,+ mode=mode,+ run=run,+ )+ elif object_type == "user_namespace":+ await self._reindex_user_namespace(+ uri=uri,+ mode=mode,+ run=run,+ )+ elif object_type == "skill_namespace":+ await self._reindex_skill_namespace(+ uri=uri,+ mode=mode,+ run=run,+ )+ elif object_type == "resource":+ await self._reindex_resource(+ uri=uri,+ mode=mode,+ run=run,+ )+ elif object_type == "skill":+ await self._reindex_skill(+ uri=uri,+ mode=mode,+ run=run,+ )+ elif object_type == "memory":+ await self._reindex_memory(+ uri=uri,+ mode=mode,+ run=run,+ )+ else:+ raise OpenVikingError(+ f"Unsupported reindex type: {object_type}",+ code="UNSUPPORTED_URI",+ details={"uri": uri},+ )++ if telemetry_id:+ await wait_tracker.wait_for_request(telemetry_id)+ self._apply_embedding_wait_status(+ counters,+ wait_tracker.build_queue_status(telemetry_id),+ )+ finally:+ if telemetry_id:+ wait_tracker.cleanup(telemetry_id)return {"status": "completed",@@ -393,11 +464,17 @@ class ReindexExecutor:*,uri: str,mode: str,- counters: _ReindexCounters,- ctx: RequestContext,+ run: _ReindexRunContext,) -> None:+ counters = run.counters+ ctx = run.ctxif mode == "semantic_and_vectors":- await self._run_semantic_processor(uri=uri, context_type="resource", ctx=ctx)+ await self._run_semantic_processor(+ uri=uri,+ context_type="resource",+ ctx=ctx,+ lock=run.lock,+ )await self._reindex_resource_vectors(uri=uri, counters=counters, ctx=ctx)returnawait self._reindex_resource_vectors(uri=uri, counters=counters, ctx=ctx)@@ -407,9 +484,10 @@ class ReindexExecutor:*,uri: str,mode: str,- counters: _ReindexCounters,- ctx: RequestContext,+ run: _ReindexRunContext,) -> None:+ counters = run.counters+ ctx = run.ctxif mode == "semantic_and_vectors":await self._regenerate_skill_semantics(uri=uri, ctx=ctx)await self._reindex_skill_vectors(uri=uri, counters=counters, ctx=ctx)@@ -419,17 +497,28 @@ class ReindexExecutor:*,uri: str,mode: str,- counters: _ReindexCounters,- ctx: RequestContext,+ run: _ReindexRunContext,) -> None:+ counters = run.counters+ ctx = run.ctxif mode == "semantic_and_vectors":- await self._run_semantic_processor(uri=uri, context_type="memory", ctx=ctx)+ await self._run_semantic_processor(+ uri=uri,+ context_type="memory",+ ctx=ctx,+ lock=run.lock,+ )await self._reindex_memory_vectors(uri=uri, counters=counters, ctx=ctx)returnawait self._reindex_memory_vectors(uri=uri, counters=counters, ctx=ctx)async def _run_semantic_processor(- self, *, uri: str, context_type: str, ctx: RequestContext+ self,+ *,+ uri: str,+ context_type: str,+ ctx: RequestContext,+ lock: LockLease = NO_LOCK,) -> None:processor = SemanticProcessor()msg = SemanticMsg(@@ -442,7 +531,7 @@ class ReindexExecutor:role=ctx.role.value,skip_vectorization=True,)- await processor.on_dequeue({"data": msg.to_json()})+ await processor.on_dequeue({"data": msg.to_json()}, lock=lock.as_borrowed())async def _reindex_resource_vectors(self,@@ -586,9 +675,10 @@ class ReindexExecutor:*,uri: str,mode: str,- counters: _ReindexCounters,- ctx: RequestContext,+ run: _ReindexRunContext,) -> None:+ counters = run.counters+ ctx = run.ctxnormalized_uri = uri.rstrip("/")target_root = normalized_uri if normalized_uri else uriviking_fs = get_viking_fs()@@ -608,8 +698,7 @@ class ReindexExecutor:await self._reindex_user_namespace(uri=user_root,mode=mode,- counters=counters,- ctx=ctx,+ run=run,)return@@ -638,7 +727,9 @@ class ReindexExecutor:"semantic_and_vectors" if mode == "semantic_and_vectors" else "vectors_only")await self._reindex_memory(- uri=memory_root, mode=memory_mode, counters=counters, ctx=ctx+ uri=memory_root,+ mode=memory_mode,+ run=run,)if mode == "semantic_and_vectors":@@ -649,8 +740,7 @@ class ReindexExecutor:target_root=target_root,directories=resource_directories,files=resource_files,- counters=counters,- ctx=ctx,+ run=run,)await self._reindex_resource_vectors_from_entries(@@ -666,9 +756,10 @@ class ReindexExecutor:*,uri: str,mode: str,- counters: _ReindexCounters,- ctx: RequestContext,+ run: _ReindexRunContext,) -> None:+ counters = run.counters+ ctx = run.ctxnormalized_uri = uri.rstrip("/")target_root = normalized_uri if normalized_uri else uriviking_fs = get_viking_fs()@@ -688,8 +779,7 @@ class ReindexExecutor:await self._reindex_agent_namespace(uri=agent_root,mode=mode,- counters=counters,- ctx=ctx,+ run=run,)return@@ -723,14 +813,20 @@ class ReindexExecutor:"semantic_and_vectors" if mode == "semantic_and_vectors" else "vectors_only")await self._reindex_memory(- uri=memory_root, mode=memory_mode, counters=counters, ctx=ctx+ uri=memory_root,+ mode=memory_mode,+ run=run,)for skill_root in sorted(set(skill_roots)):skill_mode = ("semantic_and_vectors" if mode == "semantic_and_vectors" else "vectors_only")- await self._reindex_skill(uri=skill_root, mode=skill_mode, counters=counters, ctx=ctx)+ await self._reindex_skill(+ uri=skill_root,+ mode=skill_mode,+ run=run,+ )if mode == "semantic_and_vectors":(@@ -740,8 +836,7 @@ class ReindexExecutor:target_root=target_root,directories=resource_directories,files=resource_files,- counters=counters,- ctx=ctx,+ run=run,)await self._reindex_resource_vectors_from_entries(@@ -757,9 +852,10 @@ class ReindexExecutor:*,uri: str,mode: str,- counters: _ReindexCounters,- ctx: RequestContext,+ run: _ReindexRunContext,) -> None:+ counters = run.counters+ ctx = run.ctxtarget_root = "viking://"viking_fs = get_viking_fs()try:@@ -801,16 +897,14 @@ class ReindexExecutor:await self._reindex_user_namespace(uri=user_root,mode=mode,- counters=counters,- ctx=ctx,+ run=run,)for agent_root in sorted(set(agent_roots)):await self._reindex_agent_namespace(uri=agent_root,mode=mode,- counters=counters,- ctx=ctx,+ run=run,)if mode == "semantic_and_vectors":@@ -821,8 +915,7 @@ class ReindexExecutor:target_root=target_root,directories=resource_directories,files=resource_files,- counters=counters,- ctx=ctx,+ run=run,)await self._reindex_resource_vectors_from_entries(@@ -1229,17 +1322,20 @@ class ReindexExecutor:code="FAILED_PRECONDITION",details={"uri": uri},)-- result = await TextEmbeddingHandler(service.vikingdb_manager).on_dequeue(- {"data": msg.to_json()}- )- if result is None:+ wait_tracker = get_request_wait_tracker()+ wait_tracker.register_embedding_root(msg.telemetry_id, msg.id)+ enqueued = await service.vikingdb_manager.enqueue_embedding_msg(msg)+ if not enqueued:+ wait_tracker.mark_embedding_failed(+ msg.telemetry_id,+ msg.id,+ f"Failed to enqueue reindex vector for {uri}",+ )raise OpenVikingError(- f"Failed to reindex vector for {uri}",+ f"Failed to enqueue reindex vector for {uri}",code="PROCESSING_ERROR",details={"uri": uri, "level": int(level)},)- logger.debug("Reindexed vector for %s level=%s", uri, int(level))async def _fetch_existing_record(self,diff --git a/openviking/storage/content_write.py b/openviking/storage/content_write.pyindex 82b2a79d0..53074ed64 100644--- a/openviking/storage/content_write.py+++ b/openviking/storage/content_write.py@@ -201,7 +201,6 @@ class ContentWriteCoordinator:changed_uri=uri,context_type=context_type,ctx=ctx,- lifecycle_lock_handle_id="",change_type="added" if mode == "create" else "modified",)semantic_enqueued = True@@ -386,7 +385,6 @@ class ContentWriteCoordinator:changed_uri: str,context_type: str,ctx: RequestContext,- lifecycle_lock_handle_id: str,change_type: str = "modified",target_uri: str = "",) -> None:@@ -403,7 +401,6 @@ class ContentWriteCoordinator:role=ctx.role.value,skip_vectorization=False,telemetry_id=telemetry.telemetry_id,- lifecycle_lock_handle_id=lifecycle_lock_handle_id,coalesce_key=(build_semantic_coalesce_key(context_type=context_type,@@ -432,7 +429,6 @@ class ContentWriteCoordinator:root_uri: str,modified_uri: str,ctx: RequestContext,- lifecycle_lock_handle_id: str,) -> None:queue_manager = get_queue_manager()semantic_queue = queue_manager.get_queue(queue_manager.SEMANTIC, allow_create=True)@@ -446,7 +442,6 @@ class ContentWriteCoordinator:role=ctx.role.value,skip_vectorization=False,telemetry_id=telemetry.telemetry_id,- lifecycle_lock_handle_id=lifecycle_lock_handle_id,coalesce_key=build_semantic_coalesce_key(context_type="memory",uri=root_uri,@@ -563,7 +558,6 @@ class ContentWriteCoordinator:root_uri=root_uri,modified_uri=uri,ctx=ctx,- lifecycle_lock_handle_id="",)await lock_manager.release(handle)released = Truediff --git a/openviking/storage/queuefs/semantic_dag.py b/openviking/storage/queuefs/semantic_dag.pyindex d849c5208..8452d9b5b 100644--- a/openviking/storage/queuefs/semantic_dag.py+++ b/openviking/storage/queuefs/semantic_dag.py@@ -8,6 +8,7 @@ from typing import Awaitable, Callable, Dict, List, Optionalfrom openviking.server.identity import RequestContextfrom openviking.storage.queuefs.semantic_sidecar import write_semantic_sidecars+from openviking.storage.transaction import NO_LOCK, LockLeasefrom openviking.storage.viking_fs import get_viking_fsfrom openviking.telemetry.request_wait_tracker import get_request_wait_trackerfrom openviking_cli.utils import VikingURI@@ -79,7 +80,7 @@ class SemanticDagExecutor:semantic_msg_id: Optional[str] = None,telemetry_id: str = "",recursive: bool = True,- lifecycle_lock_handle_id: str = "",+ lock: LockLease = NO_LOCK,is_code_repo: bool = False,changes: Optional[Dict[str, List[str]]] = None,skip_vectorization: bool = False,@@ -95,7 +96,7 @@ class SemanticDagExecutor:self._semantic_msg_id = semantic_msg_idself._telemetry_id = telemetry_idself._recursive = recursive- self._lifecycle_lock_handle_id = lifecycle_lock_handle_id+ self._lock = lockself._is_code_repo = is_code_repoself._changes = changes or {}self._skip_vectorization = skip_vectorization@@ -119,7 +120,6 @@ class SemanticDagExecutor:self._dir_change_status: Dict[str, bool] = {}self._overview_cache: Dict[str, Dict[str, str]] = {}self._overview_cache_lock = asyncio.Lock()- self._refresh_task: Optional[asyncio.Task] = Nonedef _create_on_complete_callback(self) -> Callable[[], Awaitable[None]]:"""Create on_complete callback for incremental update or full update."""@@ -144,7 +144,7 @@ class SemanticDagExecutor:self._target_uri,ctx=self._ctx,file_change_status=self._file_change_status,- lifecycle_lock_handle_id=self._lifecycle_lock_handle_id,+ lock=self._lock,)logger.info(f"[SyncDiff] Diff computed: "@@ -169,20 +169,16 @@ class SemanticDagExecutor:self._root_uri = root_uriself._root_done = asyncio.Event()- # Start lifecycle lock refresh loop if we hold a lock- if self._lifecycle_lock_handle_id:- self._refresh_task = asyncio.create_task(self._lock_refresh_loop())-try:await self._dispatch_dir(root_uri, parent_uri=None)await self._root_done.wait()except Exception:- await self._release_lifecycle_lock()+ await self._lock.close()raiseoriginal_on_complete = self._create_on_complete_callback()- # Wrap on_complete to release lifecycle lock after all processing+ # Release owned semantic locks after downstream vectorization finishes.async def wrapped_on_complete() -> None:try:if original_on_complete:@@ -192,7 +188,7 @@ class SemanticDagExecutor:self._telemetry_id, self._semantic_msg_id)finally:- await self._release_lifecycle_lock()+ await self._lock.close()async with self._vectorize_lock:task_count = self._vectorize_task_count@@ -611,7 +607,7 @@ class SemanticDagExecutor:abstract=abstract,ctx=self._ctx,is_stale=self._is_stale,- lifecycle_lock_handle_id=self._lifecycle_lock_handle_id,+ lock=self._lock,log_prefix="[SemanticDag]",)if not wrote:@@ -647,7 +643,7 @@ class SemanticDagExecutor:abstract = self._processor._extract_abstract_from_overview(overview)overview, abstract = self._processor._enforce_size_limits(overview, abstract)- # Write directly — protected by the outer lifecycle tree lock+ # Write directly, protected by the outer semantic lock.try:wrote = await self._write_directory_semantics(dir_uri, overview, abstract)if not wrote:@@ -701,46 +697,6 @@ class SemanticDagExecutor:else: # directoryself._vectorize_task_count += 2- async def _lock_refresh_loop(self) -> None:- """Periodically refresh lifecycle lock to prevent stale expiry."""- from openviking.storage.transaction import get_lock_manager-- try:- interval = get_lock_manager()._path_lock._lock_expire / 2- except Exception:- interval = 150.0-- while True:- try:- await asyncio.sleep(interval)- handle = get_lock_manager().get_handle(self._lifecycle_lock_handle_id)- if handle:- await get_lock_manager().refresh_lock(handle)- else:- break- except asyncio.CancelledError:- break- except Exception as e:- logger.warning(f"[SemanticDag] Lock refresh failed: {e}")-- async def _release_lifecycle_lock(self) -> None:- """Stop refresh loop and release lifecycle lock."""- if self._refresh_task and not self._refresh_task.done():- self._refresh_task.cancel()- self._refresh_task = None- if not self._lifecycle_lock_handle_id:- return- handle_id = self._lifecycle_lock_handle_id- self._lifecycle_lock_handle_id = ""- try:- from openviking.storage.transaction import get_lock_manager-- handle = get_lock_manager().get_handle(handle_id)- if handle:- await get_lock_manager().release(handle)- except Exception as e:- logger.warning(f"[SemanticDag] Failed to release lifecycle lock {handle_id}: {e}")-def get_stats(self) -> DagStats:return DagStats(total_nodes=self._stats.total_nodes,diff --git a/openviking/storage/queuefs/semantic_lock.py b/openviking/storage/queuefs/semantic_lock.pynew file mode 100644index 000000000..9e00165e5--- /dev/null+++ b/openviking/storage/queuefs/semantic_lock.py@@ -0,0 +1,37 @@+# Copyright (c) 2026 Beijing Volcano Engine Technology Co., Ltd.+# SPDX-License-Identifier: AGPL-3.0+"""Semantic queue lock resolution."""++from __future__ import annotations++from dataclasses import dataclass+from typing import Optional++from openviking.storage.transaction import NO_LOCK, LockHandoffRef, LockLease, OwnedLockLease+++@dataclass+class SemanticLockScope:+ """Resolved lock scope for one semantic message."""++ lock: LockLease++ @classmethod+ async def resolve(+ cls,+ lock_handoff: Optional[LockHandoffRef],+ *,+ caller_lock: LockLease = NO_LOCK,+ ) -> "SemanticLockScope":+ if lock_handoff and caller_lock.active:+ raise ValueError("semantic lock must come from either message or caller, not both")+ if caller_lock is not NO_LOCK and not caller_lock.active:+ raise ValueError("caller semantic lock is inactive")+ if caller_lock.active:+ return cls(caller_lock.as_borrowed())+ if lock_handoff:+ return cls(await OwnedLockLease.from_handoff(lock_handoff))+ return cls(NO_LOCK)++ async def close(self) -> None:+ await self.lock.close()diff --git a/openviking/storage/queuefs/semantic_msg.py b/openviking/storage/queuefs/semantic_msg.pyindex bc46dc7d3..aecb0bbf7 100644--- a/openviking/storage/queuefs/semantic_msg.py+++ b/openviking/storage/queuefs/semantic_msg.py@@ -8,6 +8,8 @@ from datetime import datetimefrom typing import Any, Dict, List, Optionalfrom uuid import uuid4+from openviking.storage.transaction import LockHandoffRef+def build_semantic_coalesce_key(*,@@ -50,7 +52,7 @@ class SemanticMsg:skip_vectorization: bool = Falsetelemetry_id: str = ""target_uri: str = ""- lifecycle_lock_handle_id: str = ""+ lock_handoff: Optional[LockHandoffRef] = Noneis_code_repo: bool = Falsecoalesce_key: str = ""coalesce_version: int = 0@@ -70,7 +72,7 @@ class SemanticMsg:skip_vectorization: bool = False,telemetry_id: str = "",target_uri: str = "",- lifecycle_lock_handle_id: str = "",+ lock_handoff: Optional[LockHandoffRef] = None,is_code_repo: bool = False,coalesce_key: str = "",coalesce_version: int = 0,@@ -87,7 +89,7 @@ class SemanticMsg:self.skip_vectorization = skip_vectorizationself.telemetry_id = telemetry_idself.target_uri = target_uri- self.lifecycle_lock_handle_id = lifecycle_lock_handle_id+ self.lock_handoff = lock_handoffself.is_code_repo = is_code_repoself.coalesce_key = coalesce_keyself.coalesce_version = coalesce_version@@ -129,7 +131,7 @@ class SemanticMsg:skip_vectorization=data.get("skip_vectorization", False),telemetry_id=data.get("telemetry_id", ""),target_uri=data.get("target_uri", ""),- lifecycle_lock_handle_id=data.get("lifecycle_lock_handle_id", ""),+ lock_handoff=LockHandoffRef.from_value(data.get("lock_handoff")),is_code_repo=data.get("is_code_repo", False),coalesce_key=data.get("coalesce_key", ""),coalesce_version=data.get("coalesce_version", 0),diff --git a/openviking/storage/queuefs/semantic_processor.py b/openviking/storage/queuefs/semantic_processor.pyindex 6057405ab..f41bbb158 100644--- a/openviking/storage/queuefs/semantic_processor.py+++ b/openviking/storage/queuefs/semantic_processor.py@@ -31,7 +31,9 @@ from openviking.storage.queuefs.named_queue import DequeueHandlerBasefrom openviking.storage.queuefs.semantic_dag import DagStats, SemanticDagExecutorfrom openviking.storage.queuefs.semantic_msg import SemanticMsg, build_semantic_coalesce_keyfrom openviking.storage.queuefs.semantic_queue import is_semantic_msg_stale+from openviking.storage.queuefs.semantic_lock import SemanticLockScopefrom openviking.storage.queuefs.semantic_sidecar import write_semantic_sidecars+from openviking.storage.transaction import NO_LOCK, LockLeasefrom openviking.storage.viking_fs import get_viking_fsfrom openviking.telemetry import bind_telemetry, bind_telemetry_stage, resolve_telemetryfrom openviking.telemetry.request_wait_tracker import get_request_wait_tracker@@ -271,11 +273,14 @@ class SemanticProcessor(DequeueHandlerBase):await semantic_queue.enqueue(parent_msg)logger.info("Enqueued parent semantic refresh: %s", parent_uri)- async def on_dequeue(self, data: Optional[Dict[str, Any]]) -> Optional[Dict[str, Any]]:+ async def on_dequeue(+ self,+ data: Optional[Dict[str, Any]],+ lock: LockLease = NO_LOCK,+ ) -> Optional[Dict[str, Any]]:"""Process dequeued SemanticMsg, recursively process all subdirectories."""msg: Optional[SemanticMsg] = Nonecollector = None- release_lock_in_finally = Truetry:import json@@ -332,69 +337,74 @@ class SemanticProcessor(DequeueHandlerBase):logger.info(f"Processing semantic generation for: {msg})")- if msg.context_type == "memory":- await self._process_memory_directory(msg)- else:- is_incremental = False- target_uri = msg.target_uri- viking_fs = get_viking_fs()- if msg.target_uri:- target_exists = await viking_fs.exists(- msg.target_uri, ctx=self._current_ctx+ semantic_lock = await SemanticLockScope.resolve(+ msg.lock_handoff,+ caller_lock=lock,+ )+ lock_transferred = False+ try:+ if msg.context_type == "memory":+ lock_transferred = True+ await self._process_memory_directory(+ msg,+ lock=semantic_lock.lock,)- # Check if target URI exists and is not the same as the source URI(避免重复处理)- if target_exists and msg.uri != msg.target_uri:- is_incremental = True- logger.info(- f"Target URI exists, using incremental update: {msg.target_uri}"+ else:+ is_incremental = False+ target_uri = msg.target_uri+ viking_fs = get_viking_fs()+ if msg.target_uri:+ target_exists = await viking_fs.exists(+ msg.target_uri, ctx=self._current_ctx)- elif target_exists and msg.changes and msg.uri == msg.target_uri:+ # Check if target URI exists and is not the same as the source URI(避免重复处理)+ if target_exists and msg.uri != msg.target_uri:+ is_incremental = True+ logger.info(+ f"Target URI exists, using incremental update: {msg.target_uri}"+ )+ elif target_exists and msg.changes and msg.uri == msg.target_uri:+ is_incremental = True+ logger.info(+ f"Using direct incremental semantic update for: {msg.uri}"+ )+ elif msg.changes:is_incremental = True+ target_uri = msg.urilogger.info(f"Using direct incremental semantic update for: {msg.uri}")- elif msg.changes:- is_incremental = True- target_uri = msg.uri- logger.info(f"Using direct incremental semantic update for: {msg.uri}")-- # Re-acquire lifecycle lock if handle was lost (e.g. server restart)- if msg.lifecycle_lock_handle_id:- lock_uri = target_uri or msg.uri- msg.lifecycle_lock_handle_id = await self._ensure_lifecycle_lock(- msg.lifecycle_lock_handle_id,- viking_fs._uri_to_path(lock_uri, ctx=self._current_ctx),- )- executor = SemanticDagExecutor(- processor=self,- context_type=msg.context_type,- max_concurrent_llm=self.max_concurrent_llm,- ctx=self._current_ctx,- incremental_update=is_incremental,- target_uri=target_uri,- semantic_msg_id=msg.id,- telemetry_id=msg.telemetry_id,- recursive=msg.recursive,- lifecycle_lock_handle_id=msg.lifecycle_lock_handle_id,- is_code_repo=msg.is_code_repo,- changes=msg.changes,- skip_vectorization=msg.skip_vectorization,- coalesce_key=msg.coalesce_key,- coalesce_version=msg.coalesce_version,- )- self._dag_executor = executor- if msg.lifecycle_lock_handle_id:- # The DAG owns lifecycle lock release after this point.- release_lock_in_finally = False- await executor.run(msg.uri)- self._cache_dag_stats(- msg.telemetry_id,- msg.uri,- executor.get_stats(),- )- if not executor.stale:- await self._enqueue_parent_refresh(msg, target_uri or msg.uri)+ executor = SemanticDagExecutor(+ processor=self,+ context_type=msg.context_type,+ max_concurrent_llm=self.max_concurrent_llm,+ ctx=self._current_ctx,+ incremental_update=is_incremental,+ target_uri=target_uri,+ semantic_msg_id=msg.id,+ telemetry_id=msg.telemetry_id,+ recursive=msg.recursive,+ lock=semantic_lock.lock,+ is_code_repo=msg.is_code_repo,+ changes=msg.changes,+ skip_vectorization=msg.skip_vectorization,+ coalesce_key=msg.coalesce_key,+ coalesce_version=msg.coalesce_version,+ )+ self._dag_executor = executor+ lock_transferred = True+ await executor.run(msg.uri)+ self._cache_dag_stats(+ msg.telemetry_id,+ msg.uri,+ executor.get_stats(),+ )+ if not executor.stale:+ await self._enqueue_parent_refresh(msg, target_uri or msg.uri)+ finally:+ if not lock_transferred:+ await semantic_lock.close()self._merge_request_stats(msg.telemetry_id, processed=1)logger.info(f"Completed semantic generation for: {msg.uri}")self.report_success()@@ -443,22 +453,6 @@ class SemanticProcessor(DequeueHandlerBase):self.report_error(str(e), data)return Nonefinally:- # Safety net: release lifecycle lock if still held (e.g. on exception- # before the DAG executor took ownership)- if release_lock_in_finally and msg and msg.lifecycle_lock_handle_id:- try:- from openviking.storage.transaction import get_lock_manager-- lm = get_lock_manager()- handle = lm.get_handle(msg.lifecycle_lock_handle_id)- if handle:- await lm.release(handle)- logger.info(- f"[SemanticProcessor] Safety-net released lifecycle lock "- f"{msg.lifecycle_lock_handle_id}"- )- except Exception:- passself._current_msg = Noneself._current_ctx = None@@ -467,36 +461,10 @@ class SemanticProcessor(DequeueHandlerBase):return Nonereturn self._dag_executor.get_stats()- @staticmethod- async def _ensure_lifecycle_lock(handle_id: str, lock_path: str) -> str:- """If the handle is missing (server restart), re-acquire a TreeLock.-- Returns the (possibly new) handle ID, or "" on failure.- """- from openviking.storage.transaction import get_lock_manager-- lm = get_lock_manager()- if lm.get_handle(handle_id):- return handle_id- new_handle = lm.create_handle()- if await lm.acquire_tree(new_handle, lock_path):- logger.info(f"Re-acquired lifecycle lock on {lock_path} (handle {new_handle.id})")- return new_handle.id- logger.warning(f"Failed to re-acquire lifecycle lock on {lock_path}")- await lm.release(new_handle)- return ""-- async def _process_memory_directory(self, msg: SemanticMsg) -> None:- """Process a memory directory with special handling.-- For memory directories:- - Memory files are already vectorized via embedding queue- - Only generate abstract.md and overview.md- - Vectorize the generated abstract.md and overview.md-- Args:- msg: The semantic message containing directory info and changes- """+ async def _process_memory_directory(+ self, msg: SemanticMsg, lock: LockLease = NO_LOCK+ ) -> None:+ """Process a memory directory with special handling."""viking_fs = get_viking_fs()dir_uri = msg.urictx = self._current_ctx@@ -508,140 +476,133 @@ class SemanticProcessor(DequeueHandlerBase):request_wait_tracker.mark_semantic_done(msg.telemetry_id, msg.id)try:- entries = await viking_fs.ls(dir_uri, ctx=ctx)- except Exception as e:- if msg.lifecycle_lock_handle_id:- await self._release_memory_lifecycle_lock(msg.lifecycle_lock_handle_id)- raise RuntimeError(f"Failed to list memory directory {dir_uri}: {e}") from e-- file_paths: List[str] = []- for entry in entries:- name = entry.get("name", "")- if not name or name.startswith(".") or name in [".", ".."]:- continue- if not entry.get("isDir", False):- item_uri = VikingURI(dir_uri).join(name).uri- file_paths.append(item_uri)-- if not file_paths:- logger.info(f"No memory files found in {dir_uri}")- _mark_done()- if msg.lifecycle_lock_handle_id:- await self._release_memory_lifecycle_lock(msg.lifecycle_lock_handle_id)- return-- file_summaries: List[Dict[str, str]] = []- existing_summaries: Dict[str, str] = {}-- if msg.changes:try:- old_overview = await viking_fs.read_file(f"{dir_uri}/.overview.md", ctx=ctx)- if old_overview:- existing_summaries = self._parse_overview_md(old_overview)- logger.info(- f"Parsed {len(existing_summaries)} existing summaries from overview.md"- )+ entries = await viking_fs.ls(dir_uri, ctx=ctx)except Exception as e:- logger.debug(f"No existing overview.md found for {dir_uri}: {e}")+ raise RuntimeError(f"Failed to list memory directory {dir_uri}: {e}") from e- changed_files: Set[str] = set()- if msg.changes:- changed_files = set(msg.changes.get("added", []) + msg.changes.get("modified", []))- deleted_files = set(msg.changes.get("deleted", []))- logger.info(- f"Processing memory directory {dir_uri} with changes: "- f"added={len(msg.changes.get('added', []))}, "- f"modified={len(msg.changes.get('modified', []))}, "- f"deleted={len(deleted_files)}"- )+ file_paths: List[str] = []+ for entry in entries:+ name = entry.get("name", "")+ if not name or name.startswith(".") or name in [".", ".."]:+ continue+ if not entry.get("isDir", False):+ item_uri = VikingURI(dir_uri).join(name).uri+ file_paths.append(item_uri)- # Separate cached from changed files to allow concurrent VLM calls- pending_indices: List[Tuple[int, str]] = []- file_summaries: List[Optional[Dict[str, str]]] = [None] * len(file_paths)+ if not file_paths:+ logger.info(f"No memory files found in {dir_uri}")+ _mark_done()+ return++ existing_summaries: Dict[str, str] = {}+ if msg.changes:+ try:+ old_overview = await viking_fs.read_file(f"{dir_uri}/.overview.md", ctx=ctx)+ if old_overview:+ existing_summaries = self._parse_overview_md(old_overview)+ logger.info(+ f"Parsed {len(existing_summaries)} existing summaries from overview.md"+ )+ except Exception as e:+ logger.debug(f"No existing overview.md found for {dir_uri}: {e}")- for idx, file_path in enumerate(file_paths):- file_name = file_path.split("/")[-1]+ changed_files: Set[str] = set()+ if msg.changes:+ changed_files = set(msg.changes.get("added", []) + msg.changes.get("modified", []))+ deleted_files = set(msg.changes.get("deleted", []))+ logger.info(+ f"Processing memory directory {dir_uri} with changes: "+ f"added={len(msg.changes.get('added', []))}, "+ f"modified={len(msg.changes.get('modified', []))}, "+ f"deleted={len(deleted_files)}"+ )- if file_path not in changed_files and file_name in existing_summaries:- file_summaries[idx] = {"name": file_name, "summary": existing_summaries[file_name]}- logger.debug(f"Reused existing summary for {file_name}")- else:- pending_indices.append((idx, file_path))+ pending_indices: List[Tuple[int, str]] = []+ file_summaries: List[Optional[Dict[str, str]]] = [None] * len(file_paths)- if file_paths and not pending_indices:- try:- from openviking.metrics.datasources.cache import CacheEventDataSource+ for idx, file_path in enumerate(file_paths):+ file_name = file_path.split("/")[-1]+ if file_path not in changed_files and file_name in existing_summaries:+ file_summaries[idx] = {+ "name": file_name,+ "summary": existing_summaries[file_name],+ }+ logger.debug(f"Reused existing summary for {file_name}")+ else:+ pending_indices.append((idx, file_path))- CacheEventDataSource.record_hit("L1")- except Exception:- pass- elif file_paths and pending_indices:- try:- from openviking.metrics.datasources.cache import CacheEventDataSource+ if file_paths and not pending_indices:+ try:+ from openviking.metrics.datasources.cache import CacheEventDataSource- if len(file_paths) > len(pending_indices):CacheEventDataSource.record_hit("L1")- CacheEventDataSource.record_miss("L1")- except Exception:- pass+ except Exception:+ pass+ elif file_paths and pending_indices:+ try:+ from openviking.metrics.datasources.cache import CacheEventDataSource- if pending_indices:- logger.info(- f"Generating summaries for {len(pending_indices)} changed files "- f"(reused {len(file_paths) - len(pending_indices)} cached)"- )+ if len(file_paths) > len(pending_indices):+ CacheEventDataSource.record_hit("L1")+ CacheEventDataSource.record_miss("L1")+ except Exception:+ pass- async def _gen(idx: int, file_path: str) -> None:- file_name = file_path.split("/")[-1]- try:- summary_dict = await self._generate_single_file_summary(- file_path, llm_sem=llm_sem, ctx=ctx- )- file_summaries[idx] = summary_dict- logger.debug(f"Generated summary for {file_name}")- except Exception as e:- logger.warning(f"Failed to generate summary for {file_path}: {e}")- file_summaries[idx] = {"name": file_name, "summary": ""}-- # Fix for Issue #1245: Batch processing to prevent coroutine scheduling storms- # Use a reasonable batch size (min of semaphore and 10) to keep event loop responsive- batch_size = max(1, min(self.max_concurrent_llm, 10))- for batch_start in range(0, len(pending_indices), batch_size):- batch = pending_indices[batch_start : batch_start + batch_size]+ if pending_indices:logger.info(- f"[MemorySemantic] Processing batch {batch_start // batch_size + 1}/"- f"{(len(pending_indices) + batch_size - 1) // batch_size} "- f"({len(batch)} files)"+ f"Generating summaries for {len(pending_indices)} changed files "+ f"(reused {len(file_paths) - len(pending_indices)} cached)")- await asyncio.gather(*[_gen(i, fp) for i, fp in batch])- file_summaries = [s for s in file_summaries if s is not None]+ async def _gen(idx: int, file_path: str) -> None:+ file_name = file_path.split("/")[-1]+ try:+ summary_dict = await self._generate_single_file_summary(+ file_path, llm_sem=llm_sem, ctx=ctx+ )+ file_summaries[idx] = summary_dict+ logger.debug(f"Generated summary for {file_name}")+ except Exception as e:+ logger.warning(f"Failed to generate summary for {file_path}: {e}")+ file_summaries[idx] = {"name": file_name, "summary": ""}- overview = await self._generate_overview(dir_uri, file_summaries, [], llm_sem=llm_sem)- abstract = self._extract_abstract_from_overview(overview)- overview, abstract = self._enforce_size_limits(overview, abstract)+ batch_size = max(1, min(self.max_concurrent_llm, 10))+ for batch_start in range(0, len(pending_indices), batch_size):+ batch = pending_indices[batch_start : batch_start + batch_size]+ logger.info(+ f"[MemorySemantic] Processing batch {batch_start // batch_size + 1}/"+ f"{(len(pending_indices) + batch_size - 1) // batch_size} "+ f"({len(batch)} files)"+ )+ await asyncio.gather(*[_gen(i, fp) for i, fp in batch])- try:- wrote_semantics = await self._write_memory_directory_semantics(- msg=msg,- viking_fs=viking_fs,- dir_uri=dir_uri,- overview=overview,- abstract=abstract,- ctx=ctx,+ completed_summaries = [s for s in file_summaries if s is not None]+ overview = await self._generate_overview(+ dir_uri, completed_summaries, [], llm_sem=llm_sem)- except Exception as e:- if msg.lifecycle_lock_handle_id:- await self._release_memory_lifecycle_lock(msg.lifecycle_lock_handle_id)- raise RuntimeError(f"Failed to write abstract/overview for {dir_uri}: {e}") from e- if not wrote_semantics:- _mark_done()- if msg.lifecycle_lock_handle_id:- await self._release_memory_lifecycle_lock(msg.lifecycle_lock_handle_id)- return- logger.info(f"Generated abstract.md and overview.md for {dir_uri}")+ abstract = self._extract_abstract_from_overview(overview)+ overview, abstract = self._enforce_size_limits(overview, abstract)++ try:+ wrote_semantics = await self._write_memory_directory_semantics(+ msg=msg,+ viking_fs=viking_fs,+ dir_uri=dir_uri,+ overview=overview,+ abstract=abstract,+ ctx=ctx,+ lock=lock,+ )+ except Exception as e:+ raise RuntimeError(+ f"Failed to write abstract/overview for {dir_uri}: {e}"+ ) from e+ if not wrote_semantics:+ _mark_done()+ return+ logger.info(f"Generated abstract.md and overview.md for {dir_uri}")- try:if msg.skip_vectorization:logger.info(f"Skipping vectorization for {dir_uri} (requested via SemanticMsg)")_mark_done()@@ -669,8 +630,7 @@ class SemanticProcessor(DequeueHandlerBase):)logger.info(f"Vectorized abstract.md and overview.md for {dir_uri}")finally:- if msg.lifecycle_lock_handle_id:- await self._release_memory_lifecycle_lock(msg.lifecycle_lock_handle_id)+ await lock.close()async def _write_memory_directory_semantics(self,@@ -681,6 +641,7 @@ class SemanticProcessor(DequeueHandlerBase):overview: str,abstract: str,ctx: Optional[RequestContext],+ lock: LockLease = NO_LOCK,) -> bool:return await write_semantic_sidecars(viking_fs=viking_fs,@@ -689,36 +650,21 @@ class SemanticProcessor(DequeueHandlerBase):abstract=abstract,ctx=ctx,is_stale=lambda: is_semantic_msg_stale(msg),- lifecycle_lock_handle_id=msg.lifecycle_lock_handle_id,+ lock=lock,log_prefix="[MemorySemantic]",)- async def _release_memory_lifecycle_lock(self, handle_id: str) -> None:- """Release a lifecycle lock held by in-place memory refresh."""- try:- from openviking.storage.transaction import get_lock_manager-- handle = get_lock_manager().get_handle(handle_id)- if handle:- await get_lock_manager().release(handle)- except Exception as e:- logger.warning(f"[SemanticProcessor] Failed to release memory lifecycle lock: {e}")-async def _sync_topdown_recursive(self,root_uri: str,target_uri: str,ctx: Optional[RequestContext] = None,file_change_status: Optional[Dict[str, bool]] = None,- lifecycle_lock_handle_id: str = "",+ lock: LockLease = NO_LOCK,) -> DiffResult:viking_fs = get_viking_fs()diff = DiffResult()- lock_handle = None- if lifecycle_lock_handle_id:- from openviking.storage.transaction import get_lock_manager-- lock_handle = get_lock_manager().get_handle(lifecycle_lock_handle_id)+ lock_handle = lock.handleasync def list_children(dir_uri: str) -> Tuple[Dict[str, str], Dict[str, str]]:files: Dict[str, str] = {}diff --git a/openviking/storage/queuefs/semantic_sidecar.py b/openviking/storage/queuefs/semantic_sidecar.pyindex 58db8bb58..7e512f821 100644--- a/openviking/storage/queuefs/semantic_sidecar.py+++ b/openviking/storage/queuefs/semantic_sidecar.py@@ -5,6 +5,7 @@from typing import Any, Callable, Optionalfrom openviking.server.identity import RequestContext+from openviking.storage.transaction import NO_LOCK, LockLeasefrom openviking_cli.utils.logger import get_loggerlogger = get_logger(__name__)@@ -18,7 +19,7 @@ async def write_semantic_sidecars(abstract: str,ctx: Optional[RequestContext],is_stale: Callable[[], bool],- lifecycle_lock_handle_id: str = "",+ lock: LockLease = NO_LOCK,log_prefix: str = "[Semantic]",) -> bool:if is_stale():@@ -33,28 +34,16 @@ async def write_semantic_sidecars(await _write_sidecars(viking_fs, dir_uri, overview, abstract, ctx)return True- handle = None- owns_handle = False- if lifecycle_lock_handle_id:- handle = lock_manager.get_handle(lifecycle_lock_handle_id)- if handle is None:- handle = lock_manager.create_handle()- owns_handle = True-lock_paths = [viking_fs._uri_to_path(f"{dir_uri}/.overview.md", ctx=ctx),viking_fs._uri_to_path(f"{dir_uri}/.abstract.md", ctx=ctx),]- try:- async with LockContext(lock_manager, lock_paths, lock_mode="exact", handle=handle):- if is_stale():- logger.info("%s Skipping stale semantic write for %s", log_prefix, dir_uri)- return False- await _write_sidecars(viking_fs, dir_uri, overview, abstract, ctx)- return True- finally:- if owns_handle:- await lock_manager.release(handle)+ async with LockContext(lock_manager, lock_paths, lock_mode="exact", handle=lock.handle):+ if is_stale():+ logger.info("%s Skipping stale semantic write for %s", log_prefix, dir_uri)+ return False+ await _write_sidecars(viking_fs, dir_uri, overview, abstract, ctx)+ return Trueasync def _write_sidecars(diff --git a/openviking/storage/transaction/__init__.py b/openviking/storage/transaction/__init__.pyindex 6d16e6e37..2aa5a40dc 100644--- a/openviking/storage/transaction/__init__.py+++ b/openviking/storage/transaction/__init__.py@@ -8,6 +8,13 @@ Provides path-lock management and redo-log crash recovery.from openviking.storage.transaction.lock_context import LockContextfrom openviking.storage.transaction.lock_handle import LockHandle, LockOwner+from openviking.storage.transaction.lock_lease import (+ NO_LOCK,+ BorrowedLockLease,+ LockHandoffRef,+ LockLease,+ OwnedLockLease,+)from openviking.storage.transaction.lock_manager import (LockManager,get_lock_manager,@@ -19,10 +26,15 @@ from openviking.storage.transaction.path_lock import PathLockEnginefrom openviking.storage.transaction.redo_log import RedoLog__all__ = [+ "BorrowedLockLease","LockContext","LockHandle",+ "LockHandoffRef",+ "LockLease","LockManager","LockOwner",+ "NO_LOCK",+ "OwnedLockLease","PathLockEngine","RedoLog","get_lock_manager",diff --git a/openviking/storage/transaction/lock_context.py b/openviking/storage/transaction/lock_context.pyindex ec4a561fa..05212ddf0 100644--- a/openviking/storage/transaction/lock_context.py+++ b/openviking/storage/transaction/lock_context.py@@ -6,6 +6,7 @@ from typing import Optionalfrom openviking.storage.errors import LockAcquisitionErrorfrom openviking.storage.transaction.lock_handle import LockHandle+from openviking.storage.transaction.lock_lease import OwnedLockLeasefrom openviking.storage.transaction.lock_manager import LockManager@@ -34,6 +35,7 @@ class LockContext:self._owns_handle = handle is Noneself._locks_before: list[str] = []self._acquired_lock_paths: list[str] = []+ self._owned_lease: Optional[OwnedLockLease] = Noneasync def __aenter__(self) -> LockHandle:if self._handle is None:@@ -72,12 +74,17 @@ class LockContext:raise LockAcquisitionError(f"Failed to acquire {self._lock_mode} lock for {self._paths}")+ if self._owns_handle and self._handle.locks:+ self._owned_lease = OwnedLockLease.from_handle(self._manager, self._handle)return self._handleasync def __aexit__(self, exc_type, exc_val, exc_tb):if self._handle:if self._owns_handle:- await self._manager.release(self._handle)+ if self._owned_lease:+ await self._owned_lease.close()+ else:+ await self._manager.release(self._handle)else:await self._manager.release_selected(self._handle, self._acquired_lock_paths)return Falsediff --git a/openviking/storage/transaction/lock_lease.py b/openviking/storage/transaction/lock_lease.pynew file mode 100644index 000000000..021ba9af5--- /dev/null+++ b/openviking/storage/transaction/lock_lease.py@@ -0,0 +1,225 @@+# Copyright (c) 2026 Beijing Volcano Engine Technology Co., Ltd.+# SPDX-License-Identifier: AGPL-3.0+"""Typed lock ownership helpers for path-lock users."""++from __future__ import annotations++import asyncio+from contextlib import suppress+from dataclasses import dataclass+from typing import Any, Iterable, Optional++from openviking.storage.errors import LockAcquisitionError+from openviking.storage.transaction.lock_handle import LockHandle+from openviking.storage.transaction.lock_manager import LockManager, get_lock_manager+from openviking_cli.utils.logger import get_logger++logger = get_logger(__name__)+++@dataclass(frozen=True)+class LockHandoffRef:+ """Serializable reference used to hand lock ownership through a queue."""++ handle_id: str+ lock_paths: tuple[str, ...] = ()++ @classmethod+ def from_value(cls, value: Any) -> Optional["LockHandoffRef"]:+ if value is None:+ return None+ if isinstance(value, LockHandoffRef):+ return value+ if not isinstance(value, dict):+ raise ValueError("lock_handoff must be an object")++ handle_id = value.get("handle_id")+ if not handle_id:+ return None+ lock_paths = tuple(str(path) for path in value.get("lock_paths", []) if path)+ return cls(handle_id=str(handle_id), lock_paths=lock_paths)++ def to_dict(self) -> dict[str, Any]:+ return {"handle_id": self.handle_id, "lock_paths": list(self.lock_paths)}+++class LockLease:+ """Base lock lease interface."""++ @property+ def handle(self) -> Optional[LockHandle]:+ return None++ @property+ def handle_id(self) -> str:+ handle = self.handle+ return handle.id if handle else ""++ @property+ def active(self) -> bool:+ handle = self.handle+ return bool(handle and handle.locks)++ def as_borrowed(self) -> "LockLease":+ return self++ def to_handoff(self) -> Optional[LockHandoffRef]:+ return None++ async def close(self) -> None:+ return++ async def handoff(self) -> None:+ return+++class _NoLockLease(LockLease):+ pass+++NO_LOCK: LockLease = _NoLockLease()+++@dataclass+class BorrowedLockLease(LockLease):+ """A lock lease borrowed from an outer owner."""++ manager: LockManager+ _handle: LockHandle++ @classmethod+ def from_handle(cls, manager: LockManager, handle: LockHandle) -> "BorrowedLockLease":+ return cls(manager=manager, _handle=handle)++ @property+ def handle(self) -> Optional[LockHandle]:+ return self.manager.get_handle(self._handle.id)++ def as_borrowed(self) -> "LockLease":+ return self+++class OwnedLockLease(LockLease):+ """An owned lock lease that refreshes and releases its handle."""++ def __init__(+ self,+ manager: LockManager,+ handle: LockHandle,+ *,+ start_refresh: bool = True,+ ):+ self._manager = manager+ self._handle: Optional[LockHandle] = handle+ self._refresh_task: Optional[asyncio.Task] = None+ if start_refresh and handle.locks:+ self._start_refresh()++ @classmethod+ def from_handle(cls, manager: LockManager, handle: LockHandle) -> "OwnedLockLease":+ return cls(manager, handle)++ @classmethod+ async def from_handoff(+ cls,+ ref: LockHandoffRef,+ manager: Optional[LockManager] = None,+ ) -> "OwnedLockLease":+ manager = manager or get_lock_manager()+ handle = manager.get_handle(ref.handle_id)+ if handle is None:+ handle = manager.adopt_handle(ref.handle_id, ref.lock_paths)+ if handle is None:+ raise LockAcquisitionError(f"Lock handle is no longer active: {ref.handle_id}")+ return cls(manager, handle)++ @classmethod+ async def acquire_tree(+ cls,+ manager: LockManager,+ path: str,+ *,+ timeout: Optional[float] = None,+ ) -> "OwnedLockLease":+ handle = manager.create_handle()+ if await manager.acquire_tree(handle, path, timeout=timeout):+ return cls(manager, handle)+ await manager.release(handle)+ raise LockAcquisitionError(f"Failed to acquire tree lock for {path}")++ @classmethod+ async def acquire_exact_paths(+ cls,+ manager: LockManager,+ paths: Iterable[str],+ *,+ timeout: Optional[float] = None,+ ) -> "OwnedLockLease":+ handle = manager.create_handle()+ path_list = list(paths)+ if await manager.acquire_exact_path_batch(handle, path_list, timeout=timeout):+ return cls(manager, handle)+ await manager.release(handle)+ raise LockAcquisitionError(f"Failed to acquire exact lock for {path_list}")++ @property+ def handle(self) -> Optional[LockHandle]:+ if self._handle is None:+ return None+ return self._manager.get_handle(self._handle.id)++ def as_borrowed(self) -> LockLease:+ handle = self.handle+ if handle is None:+ return NO_LOCK+ return BorrowedLockLease.from_handle(self._manager, handle)++ def to_handoff(self) -> Optional[LockHandoffRef]:+ handle = self.handle+ if handle is None:+ return None+ return LockHandoffRef(handle_id=handle.id, lock_paths=tuple(handle.locks))++ async def close(self) -> None:+ await self._stop_refresh()+ handle = self.handle or self._handle+ self._handle = None+ if handle is not None:+ await self._manager.release(handle)++ async def handoff(self) -> None:+ """Stop managing this lease after another worker has received its handle."""+ await self._stop_refresh()+ self._handle = None++ def _start_refresh(self) -> None:+ if self._refresh_task is not None:+ return+ self._refresh_task = asyncio.create_task(self._refresh_loop())++ async def _refresh_loop(self) -> None:+ try:+ expire = self._manager._path_lock._lock_expire+ interval = max(0.1, expire / 2)+ except Exception:+ interval = 150.0++ while True:+ try:+ await asyncio.sleep(interval)+ handle = self.handle+ if handle is None:+ return+ await self._manager.refresh_lock(handle)+ except asyncio.CancelledError:+ raise+ except Exception as exc:+ logger.warning("Failed to refresh lock handle %s: %s", self.handle_id, exc)++ async def _stop_refresh(self) -> None:+ if self._refresh_task is None:+ return+ self._refresh_task.cancel()+ with suppress(asyncio.CancelledError):+ await self._refresh_task+ self._refresh_task = Nonediff --git a/openviking/storage/transaction/lock_manager.py b/openviking/storage/transaction/lock_manager.pyindex f8338bab6..8eccd14ab 100644--- a/openviking/storage/transaction/lock_manager.py+++ b/openviking/storage/transaction/lock_manager.py@@ -278,6 +278,22 @@ class LockManager:return Nonereturn current+ def adopt_handle(self, handle_id: str, lock_paths: List[str]) -> Optional[LockHandle]:+ handle = self.get_handle(handle_id)+ if handle is not None:+ return handle++ adopted = LockHandle(id=handle_id)+ for lock_path in dict.fromkeys(lock_paths):+ if self._path_lock.is_lock_owned_by(lock_path, handle_id):+ adopted.add_lock(lock_path)+ if not adopted.locks:+ return None++ self._handles[adopted.id] = adopted+ self._mark_handle_active(adopted)+ return adopted+def is_path_locked(self, path: str, ignore_stale: bool = True) -> bool:"""Check whether *path* is currently locked.diff --git a/openviking/utils/resource_processor.py b/openviking/utils/resource_processor.pyindex d92431826..2023e2d2b 100644--- a/openviking/utils/resource_processor.py+++ b/openviking/utils/resource_processor.py@@ -13,7 +13,9 @@ from typing import TYPE_CHECKING, Any, Dict, List, Optionalfrom openviking.parse.tree_builder import TreeBuilderfrom openviking.server.identity import RequestContext+from openviking.storage.errors import LockAcquisitionErrorfrom openviking.storage import VikingDBManager+from openviking.storage.transaction import NO_LOCK, LockLease, OwnedLockLeasefrom openviking.storage.viking_fs import get_viking_fsfrom openviking.telemetry import get_current_telemetryfrom openviking.utils.embedding_utils import index_resource@@ -247,12 +249,12 @@ class ResourceProcessor:except Exception:pass- # ============ Phase 3.5: Source commit + lifecycle lock ============+ # ============ Phase 3.5: Source commit + resource lock ============root_uri = result.get("root_uri")temp_uri = result.get("temp_uri") # temp_doc_urioriginal_temp_uri = temp_uri # 保存原始 temp_uri 用于最终输出candidate_uri = getattr(context_tree, "_candidate_uri", None) if context_tree else None- lifecycle_lock_handle_id = ""+ resource_lock: LockLease = NO_LOCKif root_uri and temp_uri:from openviking.storage.transaction import get_lock_manager@@ -263,22 +265,16 @@ class ResourceProcessor:lock_manager = get_lock_manager()try:if candidate_uri:- root_uri, lifecycle_lock_handle_id = await self._commit_unique_candidate(+ root_uri, resource_lock = await self._commit_unique_candidate(candidate_uri=candidate_uri,ctx=ctx,)result["root_uri"] = root_urielse:dst_path = viking_fs._uri_to_path(root_uri, ctx=ctx)- handle = lock_manager.create_handle()- try:- lifecycle_lock_handle_id = await self._acquire_lifecycle_lock(- lock_manager, dst_path, uri=root_uri, handle=handle- )- except Exception:- await lock_manager.release(handle)- lifecycle_lock_handle_id = ""- raise+ resource_lock = await self._acquire_resource_lock(+ lock_manager, dst_path, uri=root_uri+ )except Exception:stage_status = "error"raise@@ -304,15 +300,22 @@ class ResourceProcessor:stage_start = time.perf_counter()stage_status = "ok"with telemetry.measure("resource.summarize"):- await self._get_summarizer().summarize(+ summary_result = await self._get_summarizer().summarize(resource_uris=[result["root_uri"]],ctx=ctx,skip_vectorization=skip_vec,- lifecycle_lock_handle_id=lifecycle_lock_handle_id,+ lock=resource_lock,temp_uris=[temp_uri_for_summarize],is_code_repo=is_code_repo,**kwargs,)+ if (+ resource_lock.active+ and summary_result.get("status") == "success"+ and summary_result.get("enqueued_count", 0) > 0+ ):+ await resource_lock.handoff()+ resource_lock = NO_LOCKexcept Exception as e:logger.error(f"Summarization failed: {e}")result["warnings"] = result.get("warnings", []) + [f"Summarization failed: {e}"]@@ -327,28 +330,23 @@ class ResourceProcessor:)except Exception:pass- elif lifecycle_lock_handle_id:- # No downstream worker will sync temp content or release the lifecycle lock.- from openviking.storage.transaction import get_lock_manager- lock_manager = get_lock_manager()- handle = lock_manager.get_handle(lifecycle_lock_handle_id)- if handle:- if temp_uri:- from openviking.pyagfs.helpers import cp as agfs_cp-- viking_fs = get_viking_fs()- src_path = viking_fs._uri_to_path(temp_uri, ctx=ctx)- dst_path = viking_fs._uri_to_path(root_uri, ctx=ctx)- await asyncio.to_thread(- agfs_cp,- viking_fs.agfs,- src_path,- dst_path,- recursive=True,- )- await viking_fs.delete_temp(parse_result.temp_dir_path, ctx=ctx)- await lock_manager.release(handle)+ if resource_lock.active:+ if not should_summarize and temp_uri:+ from openviking.pyagfs.helpers import cp as agfs_cp++ viking_fs = get_viking_fs()+ src_path = viking_fs._uri_to_path(temp_uri, ctx=ctx)+ dst_path = viking_fs._uri_to_path(root_uri, ctx=ctx)+ await asyncio.to_thread(+ agfs_cp,+ viking_fs.agfs,+ src_path,+ dst_path,+ recursive=True,+ )+ await viking_fs.delete_temp(parse_result.temp_dir_path, ctx=ctx)+ await resource_lock.close()# 恢复原始 temp_uri 用于输出if original_temp_uri is not None:@@ -362,8 +360,8 @@ class ResourceProcessor:candidate_uri: str,ctx: RequestContext,max_attempts: int = 100,- ) -> tuple[str, str]:- """Pick the first free candidate URI and reserve it with a lifecycle TreeLock."""+ ) -> tuple[str, OwnedLockLease]:+ """Pick the first free candidate URI and reserve it with a resource TreeLock."""from openviking.storage.errors import ResourceBusyErrorfrom openviking.storage.transaction import get_lock_manager@@ -376,46 +374,36 @@ class ResourceProcessor:continuedst_path = viking_fs._uri_to_path(root_uri, ctx=ctx)- handle = lock_manager.create_handle()try:- lifecycle_lock_handle_id = await self._acquire_lifecycle_lock(- lock_manager, dst_path, uri=root_uri, handle=handle, timeout=0.0+ resource_lock = await self._acquire_resource_lock(+ lock_manager, dst_path, uri=root_uri, timeout=0.0)- return root_uri, lifecycle_lock_handle_id+ return root_uri, resource_lockexcept ResourceBusyError:- await lock_manager.release(handle)continue- except Exception:- await lock_manager.release(handle)- raiseraise FileExistsError(f"Cannot resolve unique name for {candidate_uri} after {max_attempts} attempts")@staticmethod- async def _acquire_lifecycle_lock(+ async def _acquire_resource_lock(lock_manager,path: str,*,uri: str = "",- handle=None,timeout: Optional[float] = None,- ) -> str:- """Acquire per-resource TreeLock lifecycle lock or raise a structured conflict."""+ ) -> OwnedLockLease:+ """Acquire the per-resource TreeLock or raise a structured conflict."""from openviking.storage.errors import ResourceBusyError- owns_handle = handle is None- if handle is None:- handle = lock_manager.create_handle()- if await lock_manager.acquire_tree(handle, path, timeout=timeout):- return handle.id- if owns_handle:- await lock_manager.release(handle)- logger.warning(f"[ResourceProcessor] Failed to acquire lifecycle lock on {path}")- raise ResourceBusyError(- f"Resource is busy: {uri or path}",- uri=uri or path,- conflict_type="path_busy",- retryable=True,- )+ try:+ return await OwnedLockLease.acquire_tree(lock_manager, path, timeout=timeout)+ except LockAcquisitionError as exc:+ logger.warning(f"[ResourceProcessor] Failed to acquire resource lock on {path}")+ raise ResourceBusyError(+ f"Resource is busy: {uri or path}",+ uri=uri or path,+ conflict_type="path_busy",+ retryable=True,+ ) from excdiff --git a/openviking/utils/summarizer.py b/openviking/utils/summarizer.pyindex 72b47787e..2a073588b 100644--- a/openviking/utils/summarizer.py+++ b/openviking/utils/summarizer.py@@ -9,6 +9,7 @@ from typing import TYPE_CHECKING, Any, Dict, List, Tuplefrom openviking.core.namespace import context_type_for_urifrom openviking.storage.queuefs import SemanticMsg, get_queue_manager+from openviking.storage.transaction import NO_LOCK, LockLeasefrom openviking.storage.viking_fs import get_viking_fsfrom openviking.telemetry import get_current_telemetryfrom openviking.telemetry.request_wait_tracker import get_request_wait_tracker@@ -35,7 +36,7 @@ class Summarizer:resource_uris: List[str],ctx: "RequestContext",skip_vectorization: bool = False,- lifecycle_lock_handle_id: str = "",+ lock: LockLease = NO_LOCK,**kwargs,) -> Dict[str, Any]:"""@@ -59,6 +60,7 @@ class Summarizer:enqueued_count = 0telemetry = get_current_telemetry()+ lock_handoff = lock.to_handoff()def is_resources_root(uri: str) -> bool:return (uri or "").rstrip("/") == "viking://resources"@@ -80,7 +82,7 @@ class Summarizer:context_type = context_type_for_uri(uri)enqueue_units: List[Tuple[str, str]] = []- if is_resources_root(uri) and uri != temp_uri:+ if is_resources_root(uri) and uri != temp_uri and lock_handoff is None:children = await list_top_children(temp_uri)if not children:return {@@ -104,7 +106,7 @@ class Summarizer:skip_vectorization=skip_vectorization,telemetry_id=telemetry.telemetry_id,target_uri=target_uri if target_uri != source_uri else None,- lifecycle_lock_handle_id=lifecycle_lock_handle_id,+ lock_handoff=lock_handoff,is_code_repo=kwargs.get("is_code_repo", False),)if msg.telemetry_id:diff --git a/tests/client/test_file_operations.py b/tests/client/test_file_operations.pyindex 316f460ce..d292911ea 100644--- a/tests/client/test_file_operations.py+++ b/tests/client/test_file_operations.py@@ -37,7 +37,7 @@ class TestRm:for f in sample_directory.glob("**/*.txt"):await client.add_resource(path=str(f), reason="Test rm dir")- # Release lifecycle locks held by add_resource before rm+ # Release resource locks held by add_resource before rmawait release_all_locks()entries = await client.ls("viking://resources/")for data in entries:diff --git a/tests/server/test_admin_rebuild_api.py b/tests/server/test_admin_rebuild_api.pyindex 313830eb5..d1a46a911 100644--- a/tests/server/test_admin_rebuild_api.py+++ b/tests/server/test_admin_rebuild_api.py@@ -21,6 +21,12 @@ ROOT_ACCOUNT_HEADERS = {}+def _make_reindex_run(ctx, counters):+ from openviking.service.reindex_executor import _ReindexRunContext++ return _ReindexRunContext(ctx=ctx, counters=counters)++async def test_reindex_requires_admin_role(admin_client: httpx.AsyncClient):resp = await admin_client.post("/api/v1/content/reindex",@@ -167,7 +173,7 @@ async def test_reindex_memory_semantic_and_vectors_rebuilds_full_subtree(monkeypseen = {"semantic": [], "vectors": []}- async def fake_run_semantic_processor(self, *, uri, context_type, ctx):+ async def fake_run_semantic_processor(self, *, uri, context_type, ctx, lock=None):seen["semantic"].append((uri, context_type))async def fake_reindex_memory_vectors(self, *, uri, counters, ctx):@@ -186,8 +192,7 @@ async def test_reindex_memory_semantic_and_vectors_rebuilds_full_subtree(monkeypawait service._reindex_memory(uri="viking://user/default/memories",mode="semantic_and_vectors",- counters=counters,- ctx=ctx,+ run=_make_reindex_run(ctx, counters),)assert seen["semantic"] == [("viking://user/default/memories", "memory")]@@ -291,10 +296,10 @@ async def test_reindex_user_namespace_semantic_and_vectors_promotes_memory_mode(seen = {"memory_modes": [], "semantic_calls": [], "resource_calls": []}- async def fake_reindex_memory(self, *, uri, mode, counters, ctx):+ async def fake_reindex_memory(self, *, uri, mode, run):seen["memory_modes"].append((uri, mode))- async def fake_run_semantic_processor(self, *, uri, context_type, ctx):+ async def fake_run_semantic_processor(self, *, uri, context_type, ctx, lock=None):seen["semantic_calls"].append((uri, context_type))async def fake_reindex_resource_vectors_from_entries(@@ -321,8 +326,7 @@ async def test_reindex_user_namespace_semantic_and_vectors_promotes_memory_mode(await service._reindex_user_namespace(uri="viking://user/default",mode="semantic_and_vectors",- counters=counters,- ctx=ctx,+ run=_make_reindex_run(ctx, counters),)assert seen["memory_modes"] == [("viking://user/default/memories", "semantic_and_vectors")]@@ -354,10 +358,10 @@ async def test_reindex_user_namespace_semantic_and_vectors_does_not_reprocess_meseen = {"memory_modes": [], "semantic_calls": []}- async def fake_reindex_memory(self, *, uri, mode, counters, ctx):+ async def fake_reindex_memory(self, *, uri, mode, run):seen["memory_modes"].append((uri, mode))- async def fake_run_semantic_processor(self, *, uri, context_type, ctx):+ async def fake_run_semantic_processor(self, *, uri, context_type, ctx, lock=None):seen["semantic_calls"].append((uri, context_type))async def fake_reindex_resource_vectors_from_entries(@@ -384,8 +388,7 @@ async def test_reindex_user_namespace_semantic_and_vectors_does_not_reprocess_meawait service._reindex_user_namespace(uri="viking://user/default",mode="semantic_and_vectors",- counters=counters,- ctx=ctx,+ run=_make_reindex_run(ctx, counters),)assert seen["memory_modes"] == [("viking://user/default/memories", "semantic_and_vectors")]@@ -416,7 +419,7 @@ async def test_reindex_user_namespace_semantic_and_vectors_skips_uncovered_root_seen = {"semantic_calls": [], "resource_files": []}- async def fake_run_semantic_processor(self, *, uri, context_type, ctx):+ async def fake_run_semantic_processor(self, *, uri, context_type, ctx, lock=None):seen["semantic_calls"].append((uri, context_type))async def fake_reindex_resource_vectors_from_entries(@@ -442,8 +445,7 @@ async def test_reindex_user_namespace_semantic_and_vectors_skips_uncovered_root_await service._reindex_user_namespace(uri="viking://user/default",mode="semantic_and_vectors",- counters=counters,- ctx=ctx,+ run=_make_reindex_run(ctx, counters),)assert seen["semantic_calls"] == [("viking://user/default/resources", "resource")]@@ -476,13 +478,13 @@ async def test_reindex_agent_namespace_semantic_and_vectors_promotes_memory_and_seen = {"memory_modes": [], "skill_modes": [], "semantic_calls": [], "resource_calls": []}- async def fake_reindex_memory(self, *, uri, mode, counters, ctx):+ async def fake_reindex_memory(self, *, uri, mode, run):seen["memory_modes"].append((uri, mode))- async def fake_reindex_skill(self, *, uri, mode, counters, ctx):+ async def fake_reindex_skill(self, *, uri, mode, run):seen["skill_modes"].append((uri, mode))- async def fake_run_semantic_processor(self, *, uri, context_type, ctx):+ async def fake_run_semantic_processor(self, *, uri, context_type, ctx, lock=None):seen["semantic_calls"].append((uri, context_type))async def fake_reindex_resource_vectors_from_entries(@@ -510,8 +512,7 @@ async def test_reindex_agent_namespace_semantic_and_vectors_promotes_memory_and_await service._reindex_agent_namespace(uri="viking://agent/default",mode="semantic_and_vectors",- counters=counters,- ctx=ctx,+ run=_make_reindex_run(ctx, counters),)assert seen["memory_modes"] == [("viking://agent/default/memories", "semantic_and_vectors")]@@ -545,13 +546,13 @@ async def test_reindex_agent_namespace_semantic_and_vectors_does_not_reprocess_mseen = {"memory_modes": [], "skill_modes": [], "semantic_calls": []}- async def fake_reindex_memory(self, *, uri, mode, counters, ctx):+ async def fake_reindex_memory(self, *, uri, mode, run):seen["memory_modes"].append((uri, mode))- async def fake_reindex_skill(self, *, uri, mode, counters, ctx):+ async def fake_reindex_skill(self, *, uri, mode, run):seen["skill_modes"].append((uri, mode))- async def fake_run_semantic_processor(self, *, uri, context_type, ctx):+ async def fake_run_semantic_processor(self, *, uri, context_type, ctx, lock=None):seen["semantic_calls"].append((uri, context_type))async def fake_reindex_resource_vectors_from_entries(@@ -579,8 +580,7 @@ async def test_reindex_agent_namespace_semantic_and_vectors_does_not_reprocess_mawait service._reindex_agent_namespace(uri="viking://agent/default",mode="semantic_and_vectors",- counters=counters,- ctx=ctx,+ run=_make_reindex_run(ctx, counters),)assert seen["memory_modes"] == [("viking://agent/default/memories", "semantic_and_vectors")]@@ -612,7 +612,7 @@ async def test_reindex_skill_namespace_reindexes_only_skill_roots(monkeypatch):seen = []- async def fake_reindex_skill(self, *, uri, mode, counters, ctx):+ async def fake_reindex_skill(self, *, uri, mode, run):seen.append((uri, mode))monkeypatch.setattr("openviking.service.reindex_executor.get_viking_fs", lambda: FakeVikingFS())@@ -628,8 +628,7 @@ async def test_reindex_skill_namespace_reindexes_only_skill_roots(monkeypatch):await service._reindex_skill_namespace(uri="viking://agent/default/skills",mode="semantic_and_vectors",- counters=counters,- ctx=ctx,+ run=_make_reindex_run(ctx, counters),)assert seen == [("viking://agent/default/skills/my_skill", "semantic_and_vectors")]@@ -659,13 +658,13 @@ async def test_reindex_global_namespace_semantic_and_vectors_propagates_to_childseen = {"user_modes": [], "agent_modes": [], "semantic_calls": [], "resource_calls": []}- async def fake_reindex_user_namespace(self, *, uri, mode, counters, ctx):+ async def fake_reindex_user_namespace(self, *, uri, mode, run):seen["user_modes"].append((uri, mode))- async def fake_reindex_agent_namespace(self, *, uri, mode, counters, ctx):+ async def fake_reindex_agent_namespace(self, *, uri, mode, run):seen["agent_modes"].append((uri, mode))- async def fake_run_semantic_processor(self, *, uri, context_type, ctx):+ async def fake_run_semantic_processor(self, *, uri, context_type, ctx, lock=None):seen["semantic_calls"].append((uri, context_type))async def fake_reindex_resource_vectors_from_entries(@@ -693,8 +692,7 @@ async def test_reindex_global_namespace_semantic_and_vectors_propagates_to_childawait service._reindex_global_namespace(uri="viking://",mode="semantic_and_vectors",- counters=counters,- ctx=ctx,+ run=_make_reindex_run(ctx, counters),)assert seen["user_modes"] == [("viking://user/default", "semantic_and_vectors")]@@ -826,7 +824,7 @@ async def test_reindex_semantic_processor_runs_with_skip_vectorization(monkeypatseen = {}class FakeSemanticProcessor:- async def on_dequeue(self, payload):+ async def on_dequeue(self, payload, lock=None):seen["payload"] = payloadmonkeypatch.setattr(@@ -1325,7 +1323,7 @@ async def test_reindex_user_namespace_partitions_memory_and_resource(monkeypatchseen = {"memory": [], "resource_dirs": [], "resource_files": []}- async def fake_reindex_memory(self, *, uri, mode, counters, ctx):+ async def fake_reindex_memory(self, *, uri, mode, run):seen["memory"].append((uri, mode))async def fake_reindex_resource_vectors_from_entries(@@ -1358,8 +1356,7 @@ async def test_reindex_user_namespace_partitions_memory_and_resource(monkeypatchawait service._reindex_user_namespace(uri="viking://user/",mode="vectors_only",- counters=counters,- ctx=ctx,+ run=_make_reindex_run(ctx, counters),)assert seen["memory"] == [("viking://user/default/memories", "vectors_only")]@@ -1399,10 +1396,10 @@ async def test_reindex_agent_namespace_partitions_memory_skill_and_resource(monkseen = {"memory": [], "skill": [], "resource_dirs": [], "resource_files": []}- async def fake_reindex_memory(self, *, uri, mode, counters, ctx):+ async def fake_reindex_memory(self, *, uri, mode, run):seen["memory"].append((uri, mode))- async def fake_reindex_skill(self, *, uri, mode, counters, ctx):+ async def fake_reindex_skill(self, *, uri, mode, run):seen["skill"].append((uri, mode))async def fake_reindex_resource_vectors_from_entries(@@ -1436,8 +1433,7 @@ async def test_reindex_agent_namespace_partitions_memory_skill_and_resource(monkawait service._reindex_agent_namespace(uri="viking://agent/",mode="vectors_only",- counters=counters,- ctx=ctx,+ run=_make_reindex_run(ctx, counters),)assert seen["memory"] == [("viking://agent/default/memories", "vectors_only")]@@ -1482,10 +1478,10 @@ async def test_reindex_global_namespace_partitions_user_agent_and_resources(monkseen = {"user": [], "agent": [], "resource_dirs": [], "resource_files": []}- async def fake_reindex_user_namespace(self, *, uri, mode, counters, ctx):+ async def fake_reindex_user_namespace(self, *, uri, mode, run):seen["user"].append((uri, mode))- async def fake_reindex_agent_namespace(self, *, uri, mode, counters, ctx):+ async def fake_reindex_agent_namespace(self, *, uri, mode, run):seen["agent"].append((uri, mode))async def fake_reindex_resource_vectors_from_entries(@@ -1519,8 +1515,7 @@ async def test_reindex_global_namespace_partitions_user_agent_and_resources(monkawait service._reindex_global_namespace(uri="viking://",mode="vectors_only",- counters=counters,- ctx=ctx,+ run=_make_reindex_run(ctx, counters),)assert seen["user"] == [("viking://user/default", "vectors_only")]diff --git a/tests/server/test_content_write_service.py b/tests/server/test_content_write_service.pyindex 97011faaa..bb70021cd 100644--- a/tests/server/test_content_write_service.py+++ b/tests/server/test_content_write_service.py@@ -274,14 +274,13 @@ async def test_resource_write_semantic_refresh_uses_coalesce_key(monkeypatch):changed_uri=file_uri,context_type="resource",ctx=ctx,- lifecycle_lock_handle_id="lock-1",)assert len(queue.messages) == 1assert queue.messages[0].coalesce_key == ("resource|default|default|default|viking://resources/demo")- assert queue.messages[0].lifecycle_lock_handle_id == "lock-1"+ assert queue.messages[0].lock_handoff is None@pytest.mark.asynciodiff --git a/tests/storage/test_memory_semantic_stall.py b/tests/storage/test_memory_semantic_stall.pyindex 02949785a..502b086e2 100644--- a/tests/storage/test_memory_semantic_stall.py+++ b/tests/storage/test_memory_semantic_stall.py@@ -28,7 +28,6 @@ def _make_msg(uri="viking://user/memories", context_type="memory", **kwargs):"agent_id": "test-agent","telemetry_id": "","target_uri": "",- "lifecycle_lock_handle_id": "","changes": None,"is_code_repo": False,}diff --git a/tests/storage/test_semantic_dag_incremental_missing_summary.py b/tests/storage/test_semantic_dag_incremental_missing_summary.pyindex 88ec20a9f..beb989290 100644--- a/tests/storage/test_semantic_dag_incremental_missing_summary.py+++ b/tests/storage/test_semantic_dag_incremental_missing_summary.py@@ -93,7 +93,7 @@ class _FakeProcessor:return overview, abstractasync def _sync_topdown_recursive(- self, root_uri, target_uri, ctx=None, file_change_status=None, lifecycle_lock_handle_id=""+ self, root_uri, target_uri, ctx=None, file_change_status=None, lock=None):self.sync_calls.append((root_uri, target_uri))root_uri = self._fs._norm(root_uri)diff --git a/tests/storage/test_semantic_processor_lifecycle_lock.py b/tests/storage/test_semantic_processor_lock_ownership.pysimilarity index 85%rename from tests/storage/test_semantic_processor_lifecycle_lock.pyrename to tests/storage/test_semantic_processor_lock_ownership.pyindex b1cc09f79..95c2a652b 100644--- a/tests/storage/test_semantic_processor_lifecycle_lock.py+++ b/tests/storage/test_semantic_processor_lock_ownership.py@@ -8,11 +8,13 @@ import pytestfrom openviking.storage.queuefs.semantic_dag import DagStatsfrom openviking.storage.queuefs.semantic_msg import SemanticMsgfrom openviking.storage.queuefs.semantic_processor import SemanticProcessor+from openviking.storage.transaction import BorrowedLockLeaseclass _FakeHandle:def __init__(self, handle_id: str):self.id = handle_id+ self.locks = ["/fake/root/.path.ovlock"]class _FakeLockManager:@@ -48,17 +50,17 @@ class _FakeVikingFS:@pytest.mark.asyncio-async def test_semantic_processor_does_not_release_lock_owned_by_dag(monkeypatch):+async def test_semantic_processor_borrows_caller_owned_lock(monkeypatch):processor = SemanticProcessor()lock_manager = _FakeLockManager()class _FakeDagExecutor:def __init__(self, **kwargs):- self.lifecycle_lock_handle_id = kwargs.get("lifecycle_lock_handle_id", "")+ self.lock = kwargs["lock"]async def run(self, root_uri):assert root_uri == "viking://resources/demo"- assert self.lifecycle_lock_handle_id == "lock-1"+ assert self.lock.handle_id == "lock-1"def get_stats(self):return DagStats()@@ -81,8 +83,8 @@ async def test_semantic_processor_does_not_release_lock_owned_by_dag(monkeypatchuri="viking://resources/demo",context_type="resource",recursive=False,- lifecycle_lock_handle_id="lock-1",- ).to_dict()+ ).to_dict(),+ lock=BorrowedLockLease.from_handle(lock_manager, lock_manager.get_handle("lock-1")),)assert lock_manager.release_calls == []