Self-evolving Context Database for AI Agents. Unify Agent Memory, Knowledge RAG and Skills.
Python agent-contextagent-memoryagent-pluginsagentic-ragcontext-databasedsh-pluginknowledge-basememorymemory-managementself-evolvingvector-database
refactor(tasks): unify task and queue work tracking (#6043)
runyunzhouec56b9443750
diff --git a/openviking/server/routers/filesystem.py b/openviking/server/routers/filesystem.pyindex d3768cae5..064140023 100644--- a/openviking/server/routers/filesystem.py+++ b/openviking/server/routers/filesystem.py@@ -332,6 +332,8 @@ async def rm(response_result["semantic_status"] = result["semantic_status"]if isinstance(result, dict) and "queue_status" in result:response_result["queue_status"] = result["queue_status"]+ if isinstance(result, dict) and "task_id" in result:+ response_result["task_id"] = result["task_id"]return Response(status="ok", result=response_result)diff --git a/openviking/service/fs_service.py b/openviking/service/fs_service.pyindex 5b1910751..d61a7e033 100644--- a/openviking/service/fs_service.py+++ b/openviking/service/fs_service.py@@ -27,6 +27,8 @@ from openviking.privacy import (from openviking.resource.uri_mutation_coordinator import UriMutationCoordinatorfrom openviking.resource.watch_storage import is_watch_task_control_urifrom openviking.server.identity import RequestContext+from openviking.service.task_tracker import get_task_tracker, run_tracked_operation+from openviking.service.task_work_index import get_enqueued_work_ids, get_task_contextfrom openviking.session.memory.memory_updater import MemoryUpdaterfrom openviking.session.memory.utils.content_visibility import visible_contentfrom openviking.storage.abstract_overview import (@@ -45,12 +47,12 @@ from openviking.storage.vector_ids import is_vector_record_idfrom openviking.storage.viking_fs import VikingFSfrom openviking.storage.vikingdb_manager import VikingDBManagerProxyfrom openviking.telemetry import get_current_telemetry-from openviking.telemetry.request_wait_tracker import get_request_wait_trackerfrom openviking.utils.embedding_utils import vectorize_directory_metafrom openviking.utils.ingest_options import IngestOptionsfrom openviking.utils.path_safety import validate_safe_viking_uri_pathfrom openviking.utils.tags import normalize_search_tagsfrom openviking_cli.exceptions import (+ DeadlineExceededError,InvalidArgumentError,NotFoundError,NotInitializedError,@@ -454,6 +456,31 @@ class FSService:ctx: RequestContext,recursive: bool = False,wait: bool = False,+ ) -> Optional[Dict[str, Any]]:+ async def execute() -> Optional[Dict[str, Any]]:+ return await self._rm(+ uri,+ ctx,+ recursive=recursive,+ wait=wait,+ )++ result, _task_id = await run_tracked_operation(+ "fs_rm",+ account_id=ctx.account_id,+ user_id=ctx.user.user_id,+ operation=execute,+ wait=wait,+ resource_id=uri,+ )+ return result++ async def _rm(+ self,+ uri: str,+ ctx: RequestContext,+ recursive: bool = False,+ wait: bool = False,) -> Optional[Dict[str, Any]]:"""Remove resource."""viking_fs = self._ensure_initialized()@@ -470,13 +497,8 @@ class FSService:refresh_parent_uri = Nonequeue_status = Nonerefresh_action: Optional[FreshnessAction] = None- request_registered = False- telemetry_id = get_current_telemetry().telemetry_idtry:if refresh_parent_uri:- if wait and telemetry_id:- get_request_wait_tracker().register_request(telemetry_id)- request_registered = Truerefresh_action = await self._enqueue_delete_refresh(root_uri=refresh_parent_uri,deleted_uri=uri,@@ -505,12 +527,9 @@ class FSService:ctx=ctx,)if refresh_parent_uri and wait and refresh_action is not FreshnessAction.MARK_PENDING:- tracker = get_request_wait_tracker()- await tracker.wait_for_request(telemetry_id)- queue_status = tracker.build_queue_status(telemetry_id)+ queue_status = await self._wait_for_refresh()finally:- if request_registered:- get_request_wait_tracker().cleanup(telemetry_id)+ passif cleanup_result is not None and isinstance(result, dict):result["memory_cleanup"] = cleanup_resultif refresh_parent_uri and isinstance(result, dict):@@ -651,14 +670,7 @@ class FSService:changes={"deleted": [deleted_uri]},generation_trigger="content_delete",)- if telemetry_id:- get_request_wait_tracker().register_semantic_root(telemetry_id, msg.id)- try:- await semantic_queue.enqueue(msg)- except Exception as exc:- if telemetry_id:- get_request_wait_tracker().mark_semantic_failed(telemetry_id, msg.id, str(exc))- raise+ await semantic_queue.enqueue(msg)return decision.actionasync def _enqueue_copy_refresh(@@ -707,16 +719,30 @@ class FSService:generation_trigger="content_copy",copy_source_uri=source_uri,)- if telemetry_id:- get_request_wait_tracker().register_semantic_root(telemetry_id, msg.id)- try:- await semantic_queue.enqueue(msg)- except Exception as exc:- if telemetry_id:- get_request_wait_tracker().mark_semantic_failed(telemetry_id, msg.id, str(exc))- raise+ await semantic_queue.enqueue(msg)return "queued"+ async def _wait_for_refresh(self) -> Dict[str, Any]:+ context = get_task_context()+ if context is None:+ raise RuntimeError("TaskContext is required for request-scoped queue waiting")+ work_ids = get_enqueued_work_ids()+ tracker = get_task_tracker()+ try:+ if context.work_id is None:+ await tracker.wait_for_task_work(context.task_id, timeout=None)+ elif work_ids:+ await tracker.wait_for_work(+ context.task_id,+ work_ids,+ timeout=None,+ )+ except (TimeoutError, asyncio.TimeoutError) as exc:+ raise DeadlineExceededError("queue processing", None) from exc+ return tracker.build_queue_status(+ context.task_id, work_ids if context.work_id is not None else None+ )+async def cp(self,from_uri: str,diff --git a/openviking/service/reindex_executor.py b/openviking/service/reindex_executor.pyindex bd8131a67..f44ff7b5d 100644--- a/openviking/service/reindex_executor.py+++ b/openviking/service/reindex_executor.py@@ -26,7 +26,8 @@ from openviking.core.namespace import ()from openviking.server.dependencies import get_servicefrom openviking.server.identity import RequestContext-from openviking.service.task_tracker import get_task_tracker+from openviking.service.task_tracker import get_task_tracker, run_tracked_operation+from openviking.service.task_work_index import get_task_contextfrom openviking.session.memory.utils.memory_file_utils import MemoryFileUtilsfrom openviking.storage.abstract_overview import body_for_preview, embedding_text_for_bodyfrom openviking.storage.queuefs.embedding_msg_converter import EmbeddingMsgConverter@@ -34,7 +35,6 @@ from openviking.storage.queuefs.semantic_msg import SemanticMsgfrom openviking.storage.queuefs.semantic_processor import SemanticProcessorfrom openviking.storage.viking_fs import get_viking_fsfrom openviking.telemetry import get_current_telemetry-from openviking.telemetry.request_wait_tracker import get_request_wait_trackerfrom openviking.utils.embedding_input import truncate_embedding_inputfrom openviking.utils.embedding_utils import (_apply_ingest_options,@@ -218,26 +218,41 @@ class ReindexExecutor:tracker = get_task_tracker()if wait:- if await tracker.has_running(+ task = await tracker.create_if_no_running(REINDEX_TASK_TYPE,uri,account_id=ctx.account_id,user_id=ctx.user.user_id,- ):+ )+ if task is None:raise OpenVikingError(f"URI {uri} already has a reindex in progress",code="CONFLICT",details={"uri": uri},)- return await self._run(- uri=uri,- object_type=object_type,- mode=mode,- force=force,- recursive=recursive,- ingest_options=ingest_options,- ctx=ctx,++ async def execute_run() -> dict[str, Any]:+ return await self._run(+ uri=uri,+ object_type=object_type,+ mode=mode,+ force=force,+ recursive=recursive,+ ingest_options=ingest_options,+ ctx=ctx,+ )++ result, _task_id = await run_tracked_operation(+ REINDEX_TASK_TYPE,+ account_id=ctx.account_id,+ user_id=ctx.user.user_id,+ operation=execute_run,+ wait=True,+ resource_id=uri,+ existing_task=task,+ accept_failed_result=self._accept_partial_result,)+ return resulttask = await tracker.create_if_no_running(REINDEX_TASK_TYPE,@@ -314,6 +329,35 @@ class ReindexExecutor:"mode": mode,}+ async def execute_in_current_task(+ self,+ *,+ uri: str,+ mode: str,+ ctx: RequestContext,+ ) -> dict[str, Any]:+ """Run reindex work under an already-bound task.++ This is for composite operations such as snapshot restore. Public+ reindex requests must use execute(), which owns admin task creation and+ URI-level duplicate detection.+ """+ if get_task_context() is None:+ raise RuntimeError("TaskContext is required for nested reindex execution")+ object_type = self._infer_target_type(uri)+ self._validate_mode(object_type, mode)+ return await self._run(+ uri=uri,+ object_type=object_type,+ mode=mode,+ ctx=ctx,+ )++ @staticmethod+ def _accept_partial_result(result: Any) -> bool:+ """Keep record-level reindex failures in the structured result."""+ return isinstance(result, dict) and int(result.get("failed_records", 0) or 0) > 0+@staticmethoddef _resolve_ingest_options(*,@@ -569,10 +613,6 @@ class ReindexExecutor:started_at = time.perf_counter()counters = _ReindexCounters()telemetry_id = get_current_telemetry().telemetry_id- wait_tracker = get_request_wait_tracker()- if telemetry_id:- wait_tracker.register_request(telemetry_id)-acquire_lock = service.viking_fs._async_agfs.pathlock_acquire_treestat = await service.viking_fs.stat(uri, ctx=ctx, skip_count=True)root_is_dir = bool(stat.get("isDir", stat.get("is_dir")))@@ -633,22 +673,23 @@ class ReindexExecutor:details={"uri": uri},)- if telemetry_id:- await wait_tracker.wait_for_request(telemetry_id)+ task_context = get_task_context()+ if task_context is not None:+ await get_task_tracker().wait_for_task_work(+ task_context.task_id, exclude_work_id=task_context.work_id+ )self._apply_semantic_tree_stats(counters,SemanticProcessor.consume_tree_stats(telemetry_id=telemetry_id),)self._apply_embedding_wait_status(counters,- wait_tracker.build_queue_status(telemetry_id),+ get_task_tracker().build_queue_status(task_context.task_id),)finally:active_lease = run.lease if run is not None else leaseif active_lease is not None and (run is None or not run.lease_handed_off):await service.viking_fs._async_agfs.pathlock_release(active_lease)- if telemetry_id:- wait_tracker.cleanup(telemetry_id)return {"status": "completed",@@ -842,23 +883,10 @@ class ReindexExecutor:propagate_to_parent=recursive,plan=semantic_plan,)- tracker = get_request_wait_tracker()- if telemetry_id:- tracker.register_semantic_root(telemetry_id, msg.id)-queue_manager = get_queue_manager()semantic_queue = queue_manager.get_queue(queue_manager.SEMANTIC, allow_create=True)if run.lease is None:- try:- enqueue_id = await semantic_queue.enqueue(msg)- except Exception:- if telemetry_id:- tracker.mark_semantic_failed(- telemetry_id, msg.id, "semantic plan enqueue failed"- )- raise- if enqueue_id == "deduplicated" and telemetry_id:- tracker.mark_semantic_done(telemetry_id, msg.id, processed_delta=0)+ await semantic_queue.enqueue(msg)returnviking_fs = get_service().viking_fs@@ -895,8 +923,6 @@ class ReindexExecutor:run.lease_handed_off = Truerun.lease = Nonerun.lock = None- if telemetry_id:- tracker.mark_semantic_done(telemetry_id, msg.id, processed_delta=0)returnif not acquired_plan_lease:run.lease_handed_off = True@@ -912,8 +938,6 @@ class ReindexExecutor:run.lock = await agfs.pathlock_as_borrowed(run.lease)elif acquired_plan_lease:await agfs.pathlock_release(plan_lease)- if telemetry_id:- tracker.mark_semantic_failed(telemetry_id, msg.id, "semantic plan enqueue failed")raiseasync def _reindex_skill(@@ -1579,17 +1603,11 @@ class ReindexExecutor:code="FAILED_PRECONDITION",details={"uri": uri},)- wait_tracker = get_request_wait_tracker()- wait_tracker.register_embedding_root(msg.telemetry_id, msg.id)+ enqueue_error = f"Failed to enqueue reindex vector for {uri}"enqueued = await service.vikingdb_manager.enqueue_embedding_msg(msg)if not enqueued:- wait_tracker.mark_embedding_failed(- msg.telemetry_id,- msg.id,- f"Failed to enqueue reindex vector for {uri}",- )raise OpenVikingError(- f"Failed to enqueue reindex vector for {uri}",+ enqueue_error,code="PROCESSING_ERROR",details={"uri": uri, "level": int(level)},)diff --git a/openviking/service/resource_service.py b/openviking/service/resource_service.pyindex 308a0410f..7d72cf23e 100644--- a/openviking/service/resource_service.py+++ b/openviking/service/resource_service.py@@ -67,7 +67,6 @@ from openviking.storage.queuefs.add_resource_msg import AddResourcePhasefrom openviking.storage.viking_fs import LS_ALL_NODES, VikingFSfrom openviking.storage.vikingdb_manager import VikingDBManagerfrom openviking.telemetry import get_current_telemetry, register_telemetry, unregister_telemetry-from openviking.telemetry.request_wait_tracker import get_request_wait_trackerfrom openviking.utils import is_git_repo_url, is_github_url, parse_code_hosting_urlfrom openviking.utils.git_auth import (GitHttpAuthConfig,@@ -84,6 +83,7 @@ from openviking.utils.resource_processor import ResourceProcessorfrom openviking.utils.skill_processor import SkillProcessingPreparation, SkillProcessorfrom openviking_cli.exceptions import (ConflictError,+ DeadlineExceededError,FailedPreconditionError,InternalError,InvalidArgumentError,@@ -497,7 +497,9 @@ class ResourceService:self._runtime_config_manager,ctx.account_id,)- app_credentials = self._load_feishu_credentials_for_watch(app_id, app_secret, feishu_config)+ app_credentials = self._load_feishu_credentials_for_watch(+ app_id, app_secret, feishu_config+ )watch_auth_state = create_feishu_auth_state(token,refresh_token.strip(),@@ -657,6 +659,30 @@ class ResourceService:await asyncio.gather(*tasks, return_exceptions=True)self._background_tasks.clear()+ def _unregister_telemetry_after_task(+ self,+ task_id: str,+ telemetry_id: str,+ *,+ account_id: str,+ user_id: str,+ ) -> None:+ async def wait_and_unregister() -> None:+ from openviking.service.task_tracker import get_task_tracker++ try:+ await get_task_tracker().wait(+ task_id,+ account_id=account_id,+ user_id=user_id,+ )+ finally:+ unregister_telemetry(telemetry_id)++ background = asyncio.create_task(wait_and_unregister())+ self._background_tasks.add(background)+ background.add_done_callback(self._background_tasks.discard)+async def _enqueue_add_resource_job(self,msg: Any,@@ -2256,42 +2282,6 @@ class ResourceService:logger.warning("[ResourceService] Failed to link resource reason memory: %s", exc)result.setdefault("warnings", []).append(f"Memory linking failed: {exc}")- async def _monitor_queue_processing(- self,- task_id: str,- telemetry_id: str,- account_id: str,- user_id: str,- ) -> None:- from openviking.service.task_tracker import get_task_tracker-- task_tracker = get_task_tracker()- request_wait_tracker = get_request_wait_tracker()- await task_tracker.start(task_id, account_id=account_id, user_id=user_id)- try:- await request_wait_tracker.wait_for_request(telemetry_id)- status = request_wait_tracker.build_queue_status(telemetry_id)- errors = sum(int(group.get("error_count", 0) or 0) for group in status.values())- if errors:- await task_tracker.fail(- task_id,- f"queue processing failed: {status}",- account_id=account_id,- user_id=user_id,- )- else:- await task_tracker.complete(- task_id,- {"queue_status": status},- account_id=account_id,- user_id=user_id,- )- except Exception as exc:- await task_tracker.fail(task_id, str(exc), account_id=account_id, user_id=user_id)- finally:- request_wait_tracker.cleanup(telemetry_id)- unregister_telemetry(telemetry_id)-@staticmethoddef _raise_queue_status_errors(status: Dict[str, Any]) -> None:failed = {@@ -2440,18 +2430,15 @@ class ResourceService:ctx=ctx,server_config=get_server_config(),)- telemetry_id = get_current_telemetry().telemetry_id- request_wait_tracker = get_request_wait_tracker()- monitor_started = False- processing_started = Falsefrom openviking.service.task_tracker import get_task_trackerfrom openviking.service.task_tracker_concurrency import run_to_completionfrom openviking.service.task_work_index import bind_task_contexttask_tracker = get_task_tracker()task = None- if telemetry_id:- request_wait_tracker.register_request(telemetry_id)+ outcome_recorded = False+ telemetry_id = get_current_telemetry().telemetry_id+ telemetry_cleanup_scheduled = Falsetry:# Establish ownership before any queue work can start. Descendant@@ -2468,6 +2455,7 @@ class ResourceService:# Keep the returned record before propagating cancellation, so a# committed task can still be settled by the exception handler.await run_to_completion(create_skill_task)+ assert task is not Noneawait task_tracker.start(task.task_id, account_id=ctx.account_id, user_id=ctx.user.user_id)@@ -2498,73 +2486,84 @@ class ResourceService:source_metadata=source_metadata,owner_lease_ref=owner_lease_ref,)- finally:- await task_tracker.unregister_running_task(task.task_id)- processing_started = True- if not wait:- monitor_started = True- asyncio.create_task(- self._monitor_queue_processing(- task.task_id, telemetry_id, ctx.account_id, ctx.user.user_id+ if isinstance(result, dict) and "root_uri" not in result and result.get("uri"):+ result["root_uri"] = result["uri"]++ result["task_id"] = task.task_id+ if telemetry_id:+ self._unregister_telemetry_after_task(+ task.task_id,+ telemetry_id,+ account_id=ctx.account_id,+ user_id=ctx.user.user_id,)+ telemetry_cleanup_scheduled = True+ await task_tracker.complete(+ task.task_id,+ result,+ account_id=ctx.account_id,+ user_id=ctx.user.user_id,)- if isinstance(result, dict) and "root_uri" not in result and result.get("uri"):- result["root_uri"] = result["uri"]-+ outcome_recorded = True+ except BaseException as exc:+ failure_message = str(exc) or "Skill processing cancelled"+ await run_to_completion(+ lambda: task_tracker.fail(+ task.task_id,+ failure_message,+ account_id=ctx.account_id,+ user_id=ctx.user.user_id,+ )+ )+ outcome_recorded = True+ raise+ finally:+ await task_tracker.unregister_running_task(task.task_id)if wait:wait_start = time.perf_counter()- await request_wait_tracker.wait_for_request(telemetry_id)- status = request_wait_tracker.build_queue_status(telemetry_id)+ try:+ completed = await task_tracker.wait(+ task.task_id,+ account_id=ctx.account_id,+ user_id=ctx.user.user_id,+ timeout=None,+ )+ except (TimeoutError, asyncio.TimeoutError) as exc:+ get_current_telemetry().set_error(+ "resource_service.wait_complete",+ "DEADLINE_EXCEEDED",+ str(exc),+ )+ raise DeadlineExceededError("queue processing", None) from excget_current_telemetry().set("queue.wait.duration_ms",round((time.perf_counter() - wait_start) * 1000, 3),)+ status = task_tracker.build_queue_status(task.task_id)result["queue_status"] = statusself._raise_queue_status_errors(status)- await task_tracker.complete(- task.task_id,- {"queue_status": status},- account_id=ctx.account_id,- user_id=ctx.user.user_id,- )# Cancellation settles queue entries without counting errors.- # complete() preserves cancellation, so check it before the- # update caller commits the package and discards its backup.- if task_tracker.is_cancellation_requested(task.task_id):+ if completed.status.value == "cancelled":raise OpenVikingError("Skill processing was cancelled",code="PROCESSING_ERROR",details={"task_id": task.task_id, "status": "cancelled"},)- else:- result["task_id"] = task.task_id-return resultexcept BaseException as exc:- if task is not None and not monitor_started:- if processing_started and not request_wait_tracker.is_complete(telemetry_id):- # A cancelled standalone request leaves processing in the background.- # An update's caller separately cancels before restoring.- monitor_started = True- asyncio.create_task(- self._monitor_queue_processing(- task.task_id, telemetry_id, ctx.account_id, ctx.user.user_id- )- )- else:- failure_message = str(exc) or "Skill processing cancelled"- await run_to_completion(- lambda: task_tracker.fail(- task.task_id,- failure_message,- account_id=ctx.account_id,- user_id=ctx.user.user_id,- )+ if task is not None and not outcome_recorded:+ failure_message = str(exc) or "Skill processing cancelled"+ await run_to_completion(+ lambda: task_tracker.fail(+ task.task_id,+ failure_message,+ account_id=ctx.account_id,+ user_id=ctx.user.user_id,)+ )raisefinally:- if not monitor_started:- request_wait_tracker.cleanup(telemetry_id)+ if not telemetry_cleanup_scheduled:unregister_telemetry(telemetry_id)async def cancel_skill_processing(self, task_id: str, ctx: RequestContext) -> None:diff --git a/openviking/service/task_queue_middleware.py b/openviking/service/task_queue_middleware.pyindex 64006c570..c542cae9a 100644--- a/openviking/service/task_queue_middleware.py+++ b/openviking/service/task_queue_middleware.py@@ -10,6 +10,7 @@ from openviking.service.task_work_index import (bind_task_context,extract_task_metadata,prepare_task_payload,+ record_enqueued_work,)from openviking.storage.queuefs.process_result import ProcessOutcome, ProcessResultfrom openviking.storage.queuefs.queue_middleware import (@@ -35,15 +36,38 @@ class TaskWorkQueueMiddleware(QueueMiddleware):if not isinstance(ctx.payload, dict):return await call_next(ctx)ctx.payload, metadata = prepare_task_payload(ctx.payload)+ ctx.work_id = metadata.work_id if metadata is not None else Noneif not self._index.register(ctx.queue, metadata):raise TaskWorkRejected(f"Task {metadata.task_id} is cancelling; rejected work for {ctx.queue}")try:- return await call_next(ctx)- finally:- if metadata is not None and not ctx.committed:+ result = await call_next(ctx)+ except asyncio.CancelledError:+ if metadata is not None:+ if ctx.committed:+ record_enqueued_work(ctx.queue, metadata.work_id)+ else:+ await self._index.discard(ctx.queue, metadata)+ raise+ except BaseException as exc:+ if metadata is not None:+ if ctx.committed:+ record_enqueued_work(ctx.queue, metadata.work_id)+ else:+ await self._index.fail_enqueue(+ ctx.queue,+ metadata,+ str(exc),+ )+ record_enqueued_work(ctx.queue, metadata.work_id)+ raise+ if metadata is not None:+ if ctx.committed:+ record_enqueued_work(ctx.queue, metadata.work_id)+ else:await self._index.discard(ctx.queue, metadata)+ return resultasync def process(self, ctx: ProcessContext, call_next: ProcessNext) -> ProcessResult:metadata = extract_task_metadata(ctx.message)@@ -51,11 +75,17 @@ class TaskWorkQueueMiddleware(QueueMiddleware):return await call_next(ctx)active_task = asyncio.current_task()- with bind_task_context(metadata.task_id, metadata.account_id, metadata.user_id):+ enqueued_work = []+ with bind_task_context(+ metadata.task_id,+ metadata.account_id,+ metadata.user_id,+ enqueued_work=enqueued_work,+ work_id=metadata.work_id,+ ):if self._index.cancellation_requested(metadata.task_id):result = await ctx.cancel()- if result.outcome is ProcessOutcome.FAILED:- self._index.record_failure(metadata.task_id, result.error)+ self._record_result(metadata, result)return resultif active_task is not None:self._index.register_active(@@ -66,20 +96,33 @@ class TaskWorkQueueMiddleware(QueueMiddleware):try:result = await call_next(ctx)except asyncio.CancelledError:- if self._index.cancellation_requested(metadata.task_id):- # Handler unwinding must finish before this delivery is settled.- return ProcessResult.cancelled()- raise+ if not self._index.cancellation_requested(metadata.task_id):+ raise+ # Handler unwinding finished; settle user cancellation before ACK.+ result = ProcessResult.cancelled()except Exception as exc:self._index.record_failure(metadata.task_id, str(exc))raisefinally:if active_task is not None:self._index.unregister_active(metadata.task_id, active_task)- if result.outcome is ProcessOutcome.FAILED:- self._index.record_failure(metadata.task_id, result.error)+ self._record_result(metadata, result)return result+ def _record_result(self, metadata, result: ProcessResult) -> None:+ outcome = {+ ProcessOutcome.SUCCESS: "completed",+ ProcessOutcome.FAILED: "failed",+ ProcessOutcome.REQUEUED: "requeued",+ ProcessOutcome.CANCELLED: "cancelled",+ }[result.outcome]+ self._index.record_result(+ metadata,+ outcome,+ error=result.error,+ errors=result.errors,+ )+async def ack(self, ctx: AckContext, call_next: AckNext) -> None:prepared = Nonetry:diff --git a/openviking/service/task_tracker.py b/openviking/service/task_tracker.pyindex 90bb232e6..201e34366 100644--- a/openviking/service/task_tracker.py+++ b/openviking/service/task_tracker.py@@ -18,11 +18,13 @@ import mathimport reimport threadingimport time+from contextlib import contextmanager+from contextvars import ContextVarfrom copy import deepcopyfrom dataclasses import dataclass, field, fieldsfrom datetime import datetime, timezonefrom enum import Enum-from typing import Any, Dict, List, Optional+from typing import Any, Awaitable, Callable, Dict, Iterator, List, Optional, TypeVarfrom uuid import uuid4from openviking.service.task_events import (@@ -37,9 +39,35 @@ from openviking.service.task_tracker_concurrency import (run_to_completion,)from openviking.service.task_work_index import QueueTaskMetadata, TaskWorkIndex+from openviking_cli.exceptions import DeadlineExceededError, OpenVikingErrorfrom openviking_cli.utils.logger import get_loggerlogger = get_logger(__name__)+_OperationResult = TypeVar("_OperationResult")+++@dataclass+class _WaitBudget:+ task_id: str+ remaining: float+++_wait_budget: ContextVar[Optional[_WaitBudget]] = ContextVar("task_wait_budget", default=None)+++@contextmanager+def _consume_wait_budget(task_id: str, timeout: Optional[float]) -> Iterator[Optional[float]]:+ """Charge only time spent waiting, shared across an inline operation."""+ budget = _wait_budget.get()+ if budget is None or budget.task_id != task_id:+ yield timeout+ return+ allowed = budget.remaining if timeout is None else min(timeout, budget.remaining)+ started = time.monotonic()+ try:+ yield max(0.0, allowed)+ finally:+ budget.remaining = max(0.0, budget.remaining - (time.monotonic() - started))class _CommittedMutationCancelled(asyncio.CancelledError):@@ -59,6 +87,7 @@ class TaskStatus(str, Enum):_TERMINAL_STATUSES = (TaskStatus.COMPLETED, TaskStatus.FAILED, TaskStatus.CANCELLED)_ACTIVE_STATUSES = (TaskStatus.PENDING, TaskStatus.RUNNING, TaskStatus.CANCELLING)+_PUBLIC_QUEUE_NAMES = ("Semantic", "Embedding")_CANCELLABLE_TASK_TYPES = {"add_resource",@@ -146,6 +175,138 @@ def set_task_tracker(tracker: "TaskTracker") -> None:_instance = tracker+async def run_tracked_operation(+ task_type: str,+ *,+ account_id: str,+ user_id: str,+ operation: Callable[[], Awaitable[_OperationResult]],+ wait: bool,+ timeout: Optional[float] = None,+ resource_id: Optional[str] = None,+ existing_task: Optional[TaskRecord] = None,+ accept_failed_result: Callable[[_OperationResult], bool] | None = None,+) -> tuple[_OperationResult, str]:+ """Run a top-level operation under a Task, reusing an existing Task context.++ ``accept_failed_result`` is reserved for operations whose result explicitly+ reports record-level failures while the Task retains its failed status.+ """+ from openviking.service.task_work_index import bind_task_context, get_task_context++ current = get_task_context()+ if current is not None:+ return await operation(), current.task_id++ tracker = get_task_tracker()+ created_task: Optional[TaskRecord] = existing_task++ async def create_task() -> None:+ nonlocal created_task+ created_task = await tracker.create(+ task_type,+ resource_id=resource_id,+ account_id=account_id,+ user_id=user_id,+ )++ try:+ # Keep the committed record reachable even if the caller is cancelled+ # while the store write is still running.+ if created_task is None:+ await run_to_completion(create_task)+ assert created_task is not None+ task = created_task+ await run_to_completion(+ lambda: tracker.start(task.task_id, account_id=account_id, user_id=user_id)+ )+ except BaseException as exc:+ if created_task is not None:+ failed_task_id = created_task.task_id+ failure_message = str(exc) or type(exc).__name__+ await run_to_completion(+ lambda: tracker.fail(+ failed_task_id,+ failure_message,+ account_id=account_id,+ user_id=user_id,+ )+ )+ raise+ enqueued_work: list[tuple[str, str]] = []++ async def execute_operation() -> _OperationResult:+ tracker.register_running_task(task.task_id)+ try:+ with bind_task_context(+ task.task_id,+ account_id,+ user_id,+ enqueued_work=enqueued_work,+ ):+ result = await operation()+ task_result = deepcopy(result) if isinstance(result, dict) else {"result": result}+ task_result.setdefault("task_id", task.task_id)+ await tracker.complete(+ task.task_id,+ task_result,+ account_id=account_id,+ user_id=user_id,+ resource_id=resource_id,+ )+ if isinstance(result, dict) and not wait:+ result.setdefault("task_id", task.task_id)+ return result+ except _CommittedMutationCancelled:+ raise+ except BaseException as exc:+ failure_message = str(exc) or type(exc).__name__+ await run_to_completion(+ lambda: tracker.fail(+ task.task_id,+ failure_message,+ account_id=account_id,+ user_id=user_id,+ )+ )+ raise+ finally:+ await tracker.unregister_running_task(task.task_id)++ token = _wait_budget.set(+ _WaitBudget(task.task_id, timeout) if wait and timeout is not None else None+ )+ try:+ result = await execute_operation()+ if wait:+ try:+ terminal = await tracker.wait(+ task.task_id,+ account_id=account_id,+ user_id=user_id,+ timeout=timeout,+ )+ if terminal.status == TaskStatus.FAILED and not (+ accept_failed_result is not None and accept_failed_result(result)+ ):+ raise OpenVikingError(+ terminal.error or "Task processing failed",+ code="PROCESSING_ERROR",+ details={"task_id": task.task_id},+ )+ if terminal.status == TaskStatus.CANCELLED:+ raise OpenVikingError(+ "Task processing was cancelled",+ code="PROCESSING_ERROR",+ details={"task_id": task.task_id, "status": "cancelled"},+ )+ except (TimeoutError, asyncio.TimeoutError) as exc:+ raise DeadlineExceededError("queue processing", timeout) from exc+ return result, task.task_id+ finally:+ _wait_budget.reset(token)++# ── Sanitization ──_SENSITIVE_PATTERNS = re.compile(@@ -289,17 +450,20 @@ class TaskTracker:for task_id in expired_ids:evicted_count += await self._delete_expired_task(task_id, now)- capacity_evicted = []+ capacity_evicted: list[tuple[str, bool]] = []with self._lock:if len(self._tasks) > self.MAX_TASKS:sorted_tasks = sorted(self._tasks.items(), key=lambda x: x[1].created_at)excess = len(self._tasks) - self.MAX_TASKS- for tid, _ in sorted_tasks[:excess]:+ for tid, task in sorted_tasks[:excess]:self._tasks.pop(tid, None)- capacity_evicted.append(tid)+ capacity_evicted.append((tid, task.status in _TERMINAL_STATUSES))- for tid in capacity_evicted:- self._work_index.forget_processing(tid)+ for tid, terminal in capacity_evicted:+ if terminal:+ self._work_index.forget_task(tid)+ else:+ self._work_index.forget_processing(tid)if evicted_count:logger.debug("[TaskTracker] Evicted %d expired tasks", evicted_count)@@ -344,7 +508,7 @@ class TaskTracker:with self._lock:self._tasks.pop(task_id, None)- self._work_index.forget_processing(task_id)+ self._work_index.forget_task(task_id)return True@staticmethod@@ -632,7 +796,12 @@ class TaskTracker:updated.status = terminal_statusupdated.stage = terminal_status.valueupdated.updated_at = self._next_updated_at(task)- updated.auth = {}+ enqueue_markers = updated.auth.get("enqueue_complete_work_ids")+ updated.auth = (+ {"enqueue_complete_work_ids": deepcopy(enqueue_markers)}+ if enqueue_markers+ else {}+ )try:await self._persist_and_publish("update", updated, previous=task)except _CommittedMutationCancelled as exc:@@ -759,7 +928,11 @@ class TaskTracker:elif updated.result is not None:updated.status = TaskStatus.COMPLETEDelse:- return+ updated.error = "Task operation interrupted before its result was persisted"+ updated.status = TaskStatus.FAILED+ work_status = self._work_index.queue_status(task_id)+ if updated.result is not None and work_status:+ updated.result["queue_status"] = self._public_queue_status(work_status)updated.stage = updated.status.valueupdated.updated_at = self._next_updated_at(task)updated.auth = {}@@ -777,18 +950,24 @@ class TaskTracker:) -> TaskRecord:"""Wait for one task's terminal state without changing its lifecycle."""- async def _poll() -> TaskRecord:+ with _consume_wait_budget(task_id, timeout) as remaining:+ deadline = None if remaining is None else time.monotonic() + remainingwhile True:- task = await self.get(task_id, account_id=account_id, user_id=user_id)+ lookup = self.get(task_id, account_id=account_id, user_id=user_id)+ if remaining is not None and self._cached_task(task_id) is None:+ task = await asyncio.wait_for(lookup, max(0.0, remaining))+ else:+ task = await lookupif task is None:raise KeyError(f"Task not found: {task_id}")if task.status in _TERMINAL_STATUSES:return task- await asyncio.sleep(poll_interval)-- if timeout is None:- return await _poll()- return await asyncio.wait_for(_poll(), timeout)+ remaining = None if deadline is None else deadline - time.monotonic()+ if remaining is not None and remaining <= 0:+ raise TimeoutError(f"Timed out waiting for Task {task_id}")+ await asyncio.sleep(+ poll_interval if remaining is None else min(poll_interval, remaining)+ )async def get_task_auth(self,@@ -818,8 +997,7 @@ class TaskTracker:for task_id in task_ids:del self._tasks[task_id]for task_id in task_ids:- self._work_index.clear_failure(task_id)- self._work_index.forget_processing(task_id)+ self._work_index.forget_task(task_id)async def delete_user_tasks(self, account_id: str, user_id: str) -> int:"""Delete terminal task records for one user from storage and cache."""@@ -859,25 +1037,126 @@ class TaskTracker:)with self._lock:self._tasks.pop(task.task_id, None)- self._work_index.forget_processing(task.task_id)- self._work_index.clear_failure(task.task_id)+ self._work_index.forget_task(task.task_id)deleted += 1return deleted- async def wait_for_descendants(self, task_id: str, current_work_id: str) -> None:- """Wait on the same durable work index used by completion and cancellation."""- if self._work_index.has_work(task_id, exclude_work_id=current_work_id):- task = self._cached_task(task_id)- if task and task.account_id and task.user_id:- await self.record_event(- task_id,- "waiting_for_descendants",- operation=current_work_id,- account_id=task.account_id,- user_id=task.user_id,+ async def wait_for_work(+ self,+ task_id: str,+ work_ids: List[str],+ *,+ timeout: Optional[float] = None,+ stop_waiting: Optional[Any] = None,+ ) -> None:+ """Wait only for the supplied logical Work IDs to become terminal."""+ with _consume_wait_budget(task_id, timeout) as remaining:+ await self._work_index.wait_for_work(+ task_id,+ work_ids,+ timeout=remaining,+ stop_waiting=stop_waiting,+ )++ async def wait_for_task_work(+ self,+ task_id: str,+ *,+ exclude_work_id: Optional[str] = None,+ timeout: Optional[float] = None,+ ) -> None:+ """Let a root coordinator drain its Task's other queue deliveries."""+ with _consume_wait_budget(task_id, timeout) as remaining:+ await self._work_index.wait_for_task_work(+ task_id, exclude_work_id=exclude_work_id, timeout=remaining+ )++ async def mark_enqueue_complete(+ self,+ task_id: str,+ work_id: str,+ *,+ account_id: str,+ user_id: str,+ ) -> None:+ """Persist that a coordinator's complete child Work set is durable."""+ async with self._task_locks.acquire(task_id):+ task = await self._load_for_update(task_id, account_id, user_id)+ if task is None or task.status not in _ACTIVE_STATUSES:+ return+ updated = deepcopy(task)+ completed = set(updated.auth.get("enqueue_complete_work_ids") or ())+ completed.add(work_id)+ updated.auth["enqueue_complete_work_ids"] = sorted(completed)+ updated.updated_at = self._next_updated_at(task)+ await self._persist_and_publish("update", updated, previous=task)++ async def is_enqueue_complete(+ self,+ task_id: str,+ work_id: str,+ *,+ account_id: str,+ user_id: str,+ ) -> bool:+ """Check the durable producer marker used by a lock coordinator."""+ auth = await self.get_task_auth(+ task_id,+ account_id=account_id,+ user_id=user_id,+ )+ return work_id in set(auth.get("enqueue_complete_work_ids") or ())++ def build_queue_status(+ self,+ task_id: str,+ work_ids: Optional[List[str]] = None,+ ) -> Dict[str, Dict[str, object]]:+ """Project Task-scoped queue outcomes from logical Work."""+ return self._public_queue_status(self._work_index.queue_status(task_id, work_ids))++ @staticmethod+ def _public_queue_status(+ status: Dict[str, Dict[str, object]],+ ) -> Dict[str, Dict[str, object]]:+ result: Dict[str, Dict[str, object]] = {}+ for queue_name in _PUBLIC_QUEUE_NAMES:+ queue_status = deepcopy(+ status.get(+ queue_name,+ {+ "processed": 0,+ "requeue_count": 0,+ "error_count": 0,+ "errors": [],+ },)- while self._work_index.has_work(task_id, exclude_work_id=current_work_id):- await asyncio.sleep(0.05)+ )+ errors = queue_status.get("errors")+ if isinstance(errors, list):+ queue_status["errors"] = [+ {+ **error,+ "message": _sanitize_error(str(error.get("message") or "")),+ }+ if isinstance(error, dict)+ else {"message": _sanitize_error(str(error))}+ for error in errors[:100]+ ]+ result[queue_name] = queue_status+ return result++ def get_queue_timing(self, task_id: str) -> Dict[str, Dict[str, float]]:+ """Return Task-scoped queue wait and execution durations."""+ return self._work_index.queue_timing(task_id)++ def get_context_count(self, task_id: str) -> int:+ """Return successful context writes recorded by Task Work."""+ return self._work_index.context_count(task_id)++ def record_context_count(self, task_id: str, work_id: str, delta: int = 1) -> None:+ """Record successful context writes against one logical Work."""+ self._work_index.record_context_count(task_id, work_id, delta)async def record_event(self,@@ -914,9 +1193,13 @@ class TaskTracker:updated.updated_at = self._next_updated_at(task)await self._persist_and_publish("update", updated, previous=task)- def has_work(self, task_id: str) -> bool:+ def has_work(self, task_id: str, *, exclude_work_id: Optional[str] = None) -> bool:"""Return whether a task still owns durable or active queue work."""- return self._work_index.has_work(task_id)+ return self._work_index.has_work(task_id, exclude_work_id=exclude_work_id)++ def has_active_producer(self, task_id: str) -> bool:+ """Return whether another active handler can persist an enqueue marker."""+ return self._work_index.has_other_active(task_id, asyncio.current_task())def register_running_task(self, task_id: str) -> None:"""Register the current asyncio task so cancellation can interrupt it."""@@ -985,7 +1268,7 @@ class TaskTracker:)with self._lock:self._tasks.pop(task.task_id, None)- self._work_index.forget_processing(task.task_id)+ self._work_index.forget_task(task.task_id)return Trueasync def list_page(diff --git a/openviking/service/task_work_index.py b/openviking/service/task_work_index.pyindex cb034afa0..dfb6ff93d 100644--- a/openviking/service/task_work_index.py+++ b/openviking/service/task_work_index.py@@ -1,10 +1,10 @@# Copyright (c) 2026 Beijing Volcano Engine Technology Co., Ltd.# SPDX-License-Identifier: AGPL-3.0-"""Runtime index of durable queue work owned by a tracked task.+"""Runtime index of logical queue Work owned by tracked Tasks.-QueueFS remains the durable source of truth. This index is rebuilt from all-unacknowledged queue messages during startup and then maintained by enqueue/ACK-hooks. It deliberately does not persist a second counter beside QueueFS.+QueueFS remains the durable delivery source of truth. This index separates+logical Work state from delivery counts so a replacement delivery can retain+the same Work ID without allowing the old delivery's ACK to finish the Task."""from __future__ import annotations@@ -16,10 +16,11 @@ import timefrom contextlib import contextmanagerfrom contextvars import ContextVarfrom dataclasses import dataclass-from typing import Any, Awaitable, Callable, Dict, Iterable, Iterator, Mapping, Optional+from typing import Any, Awaitable, Callable, Dict, Generator, Iterable, Mapping, Optionalfrom uuid import uuid4TASK_WORK_ID_FIELD = "_task_work_id"+_TERMINAL_WORK_STATUSES = frozenset({"completed", "failed", "cancelled"})class TaskWorkRejected(Exception):@@ -31,6 +32,8 @@ class TaskExecutionContext:task_id: straccount_id: struser_id: str+ enqueued_work: Optional[list[tuple[str, str]]] = None+ work_id: Optional[str] = None@dataclass(frozen=True)@@ -43,13 +46,33 @@ class QueueTaskMetadata:@dataclassclass WorkTiming:- """Local timestamps matching the future persisted Work timing contract."""+ """Timestamps for one logical Work across all of its deliveries."""enqueued_at: floatstarted_at: Optional[float] = Nonefinished_at: Optional[float] = None+@dataclass+class WorkRecord:+ """Mutable process-local state for one logical Work."""++ task_id: str+ work_id: str+ queue_name: str+ status: str = "pending"+ delivery_count: int = 0+ requeue_count: int = 0+ error: Optional[str] = None+ errors: Optional[list[str]] = None+ context_count: int = 0+ timing: Optional[WorkTiming] = None++ @property+ def terminal(self) -> bool:+ return self.status in _TERMINAL_WORK_STATUSES++_current_task_context: ContextVar[Optional[TaskExecutionContext]] = ContextVar("openviking_task_execution_context",default=None,@@ -61,13 +84,23 @@ def bind_task_context(task_id: str,account_id: str,user_id: str,-) -> Iterator[None]:- """Propagate task ownership to queue messages produced by this coroutine."""+ *,+ enqueued_work: Optional[list[tuple[str, str]]] = None,+ work_id: Optional[str] = None,+) -> Generator[None, None, None]:+ """Propagate Task ownership to queue messages produced by this coroutine."""+ current = _current_task_context.get()+ if enqueued_work is None and current is not None and current.task_id == str(task_id):+ enqueued_work = current.enqueued_work+ if work_id is None and current is not None and current.task_id == str(task_id):+ work_id = current.work_idtoken = _current_task_context.set(TaskExecutionContext(task_id=str(task_id),account_id=str(account_id),user_id=str(user_id),+ enqueued_work=enqueued_work,+ work_id=work_id,))try:@@ -77,8 +110,8 @@ def bind_task_context(@contextmanager-def detach_task_context() -> Iterator[None]:- """Keep independently scheduled work outside the current task lifecycle."""+def detach_task_context() -> Generator[None, None, None]:+ """Keep independently scheduled work outside the current Task lifecycle."""token = _current_task_context.set(None)try:yield@@ -90,6 +123,25 @@ def get_task_context() -> Optional[TaskExecutionContext]:return _current_task_context.get()+def record_enqueued_work(queue_name: str, work_id: str) -> None:+ """Record a committed child enqueue for the currently executing handler."""+ current = _current_task_context.get()+ if current is not None and current.enqueued_work is not None:+ current.enqueued_work.append((queue_name, work_id))+++def get_enqueued_work_ids(queue_name: Optional[str] = None) -> list[str]:+ """Return committed child Work IDs produced by the current handler."""+ current = _current_task_context.get()+ if current is None or current.enqueued_work is None:+ return []+ return [+ work_id+ for work_queue, work_id in current.enqueued_work+ if queue_name is None or work_queue == queue_name+ ]++def _payload_dict(message: Any) -> Optional[Dict[str, Any]]:if not isinstance(message, dict):return None@@ -115,13 +167,19 @@ def _owner_from_payload(payload: Mapping[str, Any]) -> tuple[str, str]:if isinstance(context_data, dict):account_id = account_id or context_data.get("account_id")user_id = user_id or context_data.get("owner_user_id")+ payload_data = payload.get("payload")+ if isinstance(payload_data, dict):+ context_data = payload_data.get("context_data")+ if isinstance(context_data, dict):+ account_id = account_id or context_data.get("account_id")+ user_id = user_id or context_data.get("owner_user_id")return str(account_id or ""), str(user_id or "")def prepare_task_payload(data: Dict[str, Any],) -> tuple[Dict[str, Any], Optional[QueueTaskMetadata]]:- """Copy a payload and attach durable task metadata when it belongs to a task."""+ """Copy a payload and attach durable Task metadata when it belongs to a Task."""payload = dict(data)current = _current_task_context.get()task_id = payload.get("task_id") or (current.task_id if current is not None else "")@@ -144,15 +202,13 @@ def prepare_task_payload(def extract_task_metadata(message: Any) -> Optional[QueueTaskMetadata]:- """Read task metadata from either a QueueFS envelope or its inner payload."""+ """Read Task metadata from either a QueueFS envelope or its inner payload."""payload = _payload_dict(message)if payload is None:return Nonetask_id = payload.get("task_id")work_id = payload.get(TASK_WORK_ID_FIELD)if not work_id and isinstance(message, dict) and "data" in message and message.get("id"):- # Messages produced before task work IDs were introduced can still be- # rebuilt from their stable QueueFS envelope ID.work_id = f"queuefs:{message['id']}"if not task_id or not work_id:return None@@ -161,11 +217,12 @@ def extract_task_metadata(message: Any) -> Optional[QueueTaskMetadata]:class TaskWorkIndex:- """Thread-safe, rebuildable index of persistent and currently active task work."""+ """Thread-safe, rebuildable index of logical Work and active handlers."""def __init__(self) -> None:self._lock = threading.Lock()- self._work: Dict[str, set[tuple[str, str]]] = {}+ self._work: Dict[str, Dict[str, WorkRecord]] = {}+ self._delivery_counts: Dict[str, int] = {}self._active: Dict[str, set[asyncio.Task[Any]]] = {}self._failures: Dict[str, str] = {}self._work_timings: Dict[str, Dict[str, WorkTiming]] = {}@@ -186,67 +243,119 @@ class TaskWorkIndex:self,snapshots: Mapping[str, Iterable[Any]],) -> Dict[str, tuple[str, str]]:- """Rebuild work from QueueFS and return owners needed to restore task records."""- work: Dict[str, set[tuple[str, str]]] = {}+ """Rebuild unacknowledged deliveries and return Task owners to restore."""+ work: Dict[str, Dict[str, WorkRecord]] = {}+ delivery_counts: Dict[str, int] = {}owners: Dict[str, tuple[str, str]] = {}+ now = time.time()for queue_name, messages in snapshots.items():for message in messages:metadata = extract_task_metadata(message)if metadata is None:continue- work.setdefault(metadata.task_id, set()).add((queue_name, metadata.work_id))+ records = work.setdefault(metadata.task_id, {})+ record = records.setdefault(+ metadata.work_id,+ WorkRecord(+ task_id=metadata.task_id,+ work_id=metadata.work_id,+ queue_name=queue_name,+ timing=WorkTiming(enqueued_at=now),+ ),+ )+ record.delivery_count += 1+ delivery_counts[metadata.task_id] = delivery_counts.get(metadata.task_id, 0) + 1if metadata.account_id and metadata.user_id:owners[metadata.task_id] = (metadata.account_id, metadata.user_id)with self._lock:self._work = work+ self._delivery_counts = delivery_countsself._failures = {}return owners+ def _add_delivery_locked(self, task_id: str) -> None:+ self._delivery_counts[task_id] = self._delivery_counts.get(task_id, 0) + 1++ def _remove_delivery_locked(self, task_id: str) -> None:+ remaining = self._delivery_counts.get(task_id, 0) - 1+ if remaining > 0:+ self._delivery_counts[task_id] = remaining+ else:+ self._delivery_counts.pop(task_id, None)+def register(self, queue_name: str, metadata: Optional[QueueTaskMetadata]) -> bool:- """Atomically reject cancelled work or add it to the runtime index."""+ """Reject cancelled Work or provisionally register one delivery."""if metadata is None:return Truewith self._lock:if self.cancellation_requested(metadata.task_id):return False- self._work.setdefault(metadata.task_id, set()).add((queue_name, metadata.work_id))+ records = self._work.setdefault(metadata.task_id, {})+ record = records.get(metadata.work_id)+ now = time.time()+ if record is None:+ record = WorkRecord(+ task_id=metadata.task_id,+ work_id=metadata.work_id,+ queue_name=queue_name,+ timing=WorkTiming(enqueued_at=now),+ )+ records[metadata.work_id] = record+ elif record.queue_name != queue_name:+ raise ValueError(+ f"Work {metadata.work_id} already belongs to queue {record.queue_name}"+ )+ record.delivery_count += 1+ self._add_delivery_locked(metadata.task_id)timings = self._work_timings.get(metadata.task_id)if timings is not None:- timings.setdefault(metadata.work_id, WorkTiming(enqueued_at=time.time()))+ record.timing = timings.setdefault(metadata.work_id, record.timing)return True- def _remove_work(+ def _remove_delivery(self,queue_name: str,metadata: QueueTaskMetadata,) -> tuple[bool, bool]:- """Remove one work item and report whether it existed and made its task idle."""- removed = False+ """Remove one delivery and report whether its Task became idle."""with self._lock:- entries = self._work.get(metadata.task_id)- if entries is not None:- entry = (queue_name, metadata.work_id)- removed = entry in entries- entries.discard(entry)- if not entries:- self._work.pop(metadata.task_id, None)- became_idle = (- removed- and not self._work.get(metadata.task_id)- and not self._active.get(metadata.task_id)+ record = self._work.get(metadata.task_id, {}).get(metadata.work_id)+ removed = bool(+ record is not None and record.queue_name == queue_name and record.delivery_count > 0)+ if removed:+ record.delivery_count -= 1+ self._remove_delivery_locked(metadata.task_id)+ became_idle = removed and not self._has_work_locked(metadata.task_id)return removed, became_idle+ def _has_delivery_locked(self, task_id: str, exclude_work_id: Optional[str] = None) -> bool:+ if exclude_work_id is not None:+ record = self._work.get(task_id, {}).get(exclude_work_id)+ excluded = record.delivery_count if record is not None else 0+ return self._delivery_counts.get(task_id, 0) > excluded+ return self._delivery_counts.get(task_id, 0) > 0++ def _has_work_locked(self, task_id: str, exclude_work_id: Optional[str] = None) -> bool:+ if self._has_delivery_locked(task_id, exclude_work_id):+ return True+ if exclude_work_id is not None:+ return False+ return bool(self._active.get(task_id))+async def discard(self, queue_name: str, message: Any) -> None:- """Discard work that was never durably enqueued and finalize if it was last."""+ """Discard a provisionally registered delivery that did not commit."""metadata = (message if isinstance(message, QueueTaskMetadata) else extract_task_metadata(message))if metadata is None:return- removed, became_idle = self._remove_work(queue_name, metadata)+ removed, became_idle = self._remove_delivery(queue_name, metadata)if removed:with self._lock:+ record = self._work.get(metadata.task_id, {}).get(metadata.work_id)+ if record is not None and record.delivery_count == 0 and not record.terminal:+ self._work[metadata.task_id].pop(metadata.work_id, None)timings = self._work_timings.get(metadata.task_id)if timings is not None:timings.pop(metadata.work_id, None)@@ -254,19 +363,34 @@ class TaskWorkIndex:if became_idle and callback is not None:await callback(metadata)+ async def fail_enqueue(+ self,+ queue_name: str,+ metadata: QueueTaskMetadata,+ error: str,+ ) -> None:+ """Keep a failed logical Work visible after its delivery failed to persist."""+ self.record_result(metadata, "failed", error=error)+ removed, became_idle = self._remove_delivery(queue_name, metadata)+ if not removed:+ return+ callback = self._finalize_before_ack+ if became_idle and callback is not None:+ await callback(metadata)+async def prepare_ack(self,queue_name: str,message: Any,) -> Optional[QueueTaskMetadata]:- """Remove work provisionally and finalize its task before the last durable ACK."""+ """Provisionally remove one delivery and finalize before the last ACK."""metadata = (message if isinstance(message, QueueTaskMetadata) else extract_task_metadata(message))if metadata is None:return None- removed, became_idle = self._remove_work(queue_name, metadata)+ removed, became_idle = self._remove_delivery(queue_name, metadata)if not removed:return Noneif not became_idle:@@ -283,25 +407,80 @@ class TaskWorkIndex:return metadatadef rollback_ack(self, queue_name: str, metadata: QueueTaskMetadata) -> None:- """Restore provisionally removed work when finalization or ACK fails."""+ """Restore a provisionally removed delivery when finalization or ACK fails."""with self._lock:- self._work.setdefault(metadata.task_id, set()).add((queue_name, metadata.work_id))+ record = self._work.get(metadata.task_id, {}).get(metadata.work_id)+ if record is None:+ record = WorkRecord(+ task_id=metadata.task_id,+ work_id=metadata.work_id,+ queue_name=queue_name,+ timing=WorkTiming(enqueued_at=time.time()),+ )+ self._work.setdefault(metadata.task_id, {})[metadata.work_id] = record+ record.delivery_count += 1+ self._add_delivery_locked(metadata.task_id)def has_work(self, task_id: str, exclude_work_id: Optional[str] = None) -> bool:with self._lock:- if exclude_work_id is not None:- return any(- work_id != exclude_work_id- for _queue_name, work_id in self._work.get(task_id, ())- )- return bool(self._work.get(task_id) or self._active.get(task_id))+ return self._has_work_locked(task_id, exclude_work_id)++ def has_other_active(+ self,+ task_id: str,+ current_task: Optional[asyncio.Task[Any]],+ ) -> bool:+ """Return whether another in-process handler can finish producing Work."""+ with self._lock:+ return any(task is not current_task for task in self._active.get(task_id, ()))def cancellation_requested(self, task_id: str) -> bool:callback = self._is_cancellation_requestedreturn bool(callback is not None and callback(task_id))+ def record_result(+ self,+ metadata: QueueTaskMetadata,+ outcome: str,+ *,+ error: Optional[str] = None,+ errors: Optional[Iterable[str]] = None,+ ) -> None:+ """Apply one delivery result to its logical Work monotonically."""+ with self._lock:+ record = self._work.get(metadata.task_id, {}).get(metadata.work_id)+ if record is None or record.terminal:+ return+ if outcome == "requeued":+ record.requeue_count += 1+ record.status = "pending"+ return+ if outcome not in _TERMINAL_WORK_STATUSES:+ raise ValueError(f"Unsupported Work outcome: {outcome}")+ record.status = outcome+ record.error = error if outcome == "failed" and error else None+ if outcome == "failed":+ details = [str(value) for value in errors or () if str(value)]+ record.errors = details or ([error] if error else [])+ if record.timing is not None:+ record.timing.finished_at = max(+ record.timing.started_at or record.timing.enqueued_at,+ time.time(),+ )+ if outcome == "failed" and error:+ self._failures.setdefault(metadata.task_id, error)++ def record_context_count(self, task_id: str, work_id: str, delta: int = 1) -> None:+ """Add a result count to one Work without changing its lifecycle."""+ if delta <= 0:+ return+ with self._lock:+ record = self._work.get(task_id, {}).get(work_id)+ if record is not None:+ record.context_count += delta+def record_failure(self, task_id: str, error: str) -> None:- """Record the first failed work item owned by a task."""+ """Record the first failed Work owned by a Task."""with self._lock:self._failures.setdefault(task_id, error)@@ -313,9 +492,130 @@ class TaskWorkIndex:with self._lock:self._failures.pop(task_id, None)+ async def wait_for_work(+ self,+ task_id: str,+ work_ids: Iterable[str],+ *,+ timeout: Optional[float] = None,+ poll_interval: float = 0.05,+ stop_waiting: Optional[Callable[[], bool]] = None,+ ) -> None:+ """Wait until exactly the supplied logical Work IDs are terminal."""+ requested = frozenset(str(work_id) for work_id in work_ids)+ if not requested:+ return++ deadline = None if timeout is None else time.monotonic() + timeout+ while True:+ with self._lock:+ records = self._work.get(task_id, {})+ missing = requested.difference(records)+ done = not missing and all(records[work_id].terminal for work_id in requested)+ if done:+ return+ if missing:+ raise KeyError(f"Unknown Work for Task {task_id}: {sorted(missing)}")+ if stop_waiting is not None and stop_waiting():+ raise asyncio.CancelledError("Worker stopped with queued Work")+ remaining = None if deadline is None else deadline - time.monotonic()+ if remaining is not None and remaining <= 0:+ raise TimeoutError(f"Timed out waiting for Work in Task {task_id}")+ await asyncio.sleep(+ poll_interval if remaining is None else min(poll_interval, remaining)+ )++ async def wait_for_task_work(+ self,+ task_id: str,+ *,+ exclude_work_id: Optional[str] = None,+ timeout: Optional[float] = None,+ poll_interval: float = 0.05,+ ) -> None:+ """Drain Task deliveries, including newly enqueued and recovered Work.++ Only the Task coordinator may use this barrier, excluding its own Work.+ Inline coordinators have no delivery to exclude. Other workers must wait+ for explicit Work IDs to avoid waiting on each other.+ """+ deadline = None if timeout is None else time.monotonic() + timeout+ while True:+ with self._lock:+ pending = self._has_delivery_locked(task_id, exclude_work_id)+ if not pending:+ return+ remaining = None if deadline is None else deadline - time.monotonic()+ if remaining is not None and remaining <= 0:+ raise TimeoutError(f"Timed out waiting for Work in Task {task_id}")+ await asyncio.sleep(+ poll_interval if remaining is None else min(poll_interval, remaining)+ )++ def queue_status(+ self,+ task_id: str,+ work_ids: Optional[Iterable[str]] = None,+ ) -> Dict[str, Dict[str, object]]:+ """Project queue counters from logical Work records."""+ selected = None if work_ids is None else frozenset(str(value) for value in work_ids)+ with self._lock:+ records = [+ record+ for work_id, record in self._work.get(task_id, {}).items()+ if selected is None or work_id in selected+ ]+ result: Dict[str, Dict[str, object]] = {}+ for record in records:+ status = result.setdefault(+ record.queue_name,+ {+ "processed": 0,+ "requeue_count": 0,+ "error_count": 0,+ "errors": [],+ },+ )+ if record.status == "completed":+ status["processed"] += 1+ status["requeue_count"] += record.requeue_count+ if record.status == "failed":+ errors = record.errors or ([record.error] if record.error else [])+ status["error_count"] += len(errors) or 1+ status["errors"].extend({"message": message} for message in errors)+ return result++ def queue_timing(self, task_id: str) -> Dict[str, Dict[str, float]]:+ """Project queue wait and execution timing in milliseconds."""+ with self._lock:+ result: Dict[str, Dict[str, float]] = {}+ for record in self._work.get(task_id, {}).values():+ timing = record.timing+ metrics = result.setdefault(+ record.queue_name.lower(),+ {"queue_wait_ms": 0.0, "execute_ms": 0.0},+ )+ if timing is None or timing.started_at is None:+ continue+ metrics["queue_wait_ms"] += max(+ 0.0, (timing.started_at - timing.enqueued_at) * 1000.0+ )+ if timing.finished_at is not None:+ metrics["execute_ms"] += max(+ 0.0, (timing.finished_at - timing.started_at) * 1000.0+ )+ return result++ def context_count(self, task_id: str) -> int:+ with self._lock:+ return sum(record.context_count for record in self._work.get(task_id, {}).values())+def init_processing(self, task_id: str) -> None:with self._lock:- self._work_timings.setdefault(task_id, {})+ timings = self._work_timings.setdefault(task_id, {})+ for work_id, record in self._work.get(task_id, {}).items():+ if record.timing is not None:+ timings.setdefault(work_id, record.timing)def processing_seconds(self, task_id: str) -> Optional[float]:with self._lock:@@ -347,6 +647,18 @@ class TaskWorkIndex:with self._lock:self._work_timings.pop(task_id, None)+ def forget_task(self, task_id: str) -> None:+ """Release all process-local state after the owning Task is removed."""+ with self._lock:+ self._work.pop(task_id, None)+ self._delivery_counts.pop(task_id, None)+ self._active.pop(task_id, None)+ self._failures.pop(task_id, None)+ self._work_timings.pop(task_id, None)+ active_keys = [key for key in self._active_work_ids if key[0] == task_id]+ for key in active_keys:+ self._active_work_ids.pop(key, None)+def register_active(self,task_id: str,@@ -358,10 +670,17 @@ class TaskWorkIndex:cancelled = self.cancellation_requested(task_id)if not cancelled:self._active.setdefault(task_id, set()).add(active_task)+ now = time.time()+ timing_id = work_id or f"inline:{id(active_task)}"+ record = self._work.get(task_id, {}).get(timing_id)+ if record is not None:+ timing = record.timing+ if timing is not None and timing.started_at is None:+ timing.started_at = now+ if not record.terminal:+ record.status = "running"timings = self._work_timings.get(task_id)if timings is not None:- now = time.time()- timing_id = work_id or f"inline:{id(active_task)}"timing = timings.setdefault(timing_id, WorkTiming(enqueued_at=now))if timing.started_at is None:timing.started_at = nowdiff --git a/openviking/session/memory/memory_updater.py b/openviking/session/memory/memory_updater.pyindex 01bba2f7e..591f42486 100644--- a/openviking/session/memory/memory_updater.py+++ b/openviking/session/memory/memory_updater.py@@ -45,7 +45,6 @@ from openviking.session.memory.utils.uri import render_templatefrom openviking.storage.abstract_overview import freshness_metadata, render_abstract_overviewfrom openviking.storage.viking_fs import get_viking_fsfrom openviking.telemetry import tracer-from openviking.telemetry.request_wait_tracker import get_request_wait_trackerfrom openviking.telemetry.tracer import get_trace_idfrom openviking.utils.ingest_options import IngestOptionsfrom openviking.utils.tags import merge_search_tags@@ -835,6 +834,7 @@ class MemoryUpdater:ctx,uri_memory_type_map={uri: memory_type} if memory_type else {},ingest_options=ingest_options,+ raise_on_enqueue_failure=True,)return attempted > 0except Exception:@@ -1749,6 +1749,7 @@ class MemoryUpdater:uri_memory_type_map: Dict[str, str] = None,search_tags_by_uri: Dict[str, List[str]] = None,ingest_options: Any = None,+ raise_on_enqueue_failure: bool = False,) -> int:"""Vectorize written and edited memory files.@@ -1768,7 +1769,6 @@ class MemoryUpdater:uri_memory_type_map = uri_memory_type_map or {}search_tags_by_uri = search_tags_by_uri or {}viking_fs = self._get_viking_fs()- request_wait_tracker = get_request_wait_tracker()attempted_count = 0# Collect all URIs to vectorize (skip .overview.md and .abstract.md - they are handled separately)@@ -1865,31 +1865,20 @@ class MemoryUpdater:embedding_msg.context_data["_upsert_options"]["extracted_memory_type"] = memory_type- if embedding_msg.telemetry_id:- request_wait_tracker.register_embedding_root(- embedding_msg.telemetry_id, embedding_msg.id- )+ enqueued = await self._vikingdb.enqueue_embedding_msg(embedding_msg)+ if not enqueued:+ message = f"Failed to enqueue memory for vectorization: {uri}"+ if raise_on_enqueue_failure:+ raise RuntimeError(message)+ tracer.error(message)+ continueattempted_count += 1- try:- enqueued = await self._vikingdb.enqueue_embedding_msg(embedding_msg)- except Exception as e:- if embedding_msg.telemetry_id:- request_wait_tracker.mark_embedding_failed(- embedding_msg.telemetry_id,- embedding_msg.id,- str(e),- )- raise- if not enqueued and embedding_msg.telemetry_id:- request_wait_tracker.mark_embedding_failed(- embedding_msg.telemetry_id,- embedding_msg.id,- "embedding enqueue returned false",- )logger.debug(f"Enqueued memory for vectorization: {uri}")except Exception as e:tracer.error(f"Failed to vectorize memory {uri}: {e}")+ if raise_on_enqueue_failure:+ raisereturn attempted_count@staticmethoddiff --git a/openviking/session/session.py b/openviking/session/session.pyindex c66cce97b..847f6999e 100644--- a/openviking/session/session.py+++ b/openviking/session/session.py@@ -64,7 +64,6 @@ from openviking.session.working_memory import ()from openviking.storage.abstract_overview import render_abstract_overviewfrom openviking.telemetry import get_current_telemetry, tracer-from openviking.telemetry.request_wait_tracker import get_request_wait_trackerfrom openviking.utils.model_retry import is_retryable_api_error, retry_asyncfrom openviking.utils.time_utils import get_current_timestampfrom openviking.utils.token_estimation import estimate_text_tokens@@ -2048,11 +2047,11 @@ class Session:) -> None:"""Phase 2: Extract memories and enqueue semantic work in the background."""from openviking.service.task_tracker import get_task_tracker+ from openviking.service.task_work_index import get_task_contextfrom openviking.telemetry import OperationTelemetry, bind_telemetryfrom openviking.telemetry.registry import register_telemetry, unregister_telemetrytracker = get_task_tracker()- request_wait_tracker = get_request_wait_tracker()memories_extracted: Dict[str, int] = {}usage_events_extracted = 0@@ -2081,7 +2080,6 @@ class Session:account_id=self.ctx.account_id,user_id=self.ctx.user.user_id,)- request_wait_tracker.register_request(telemetry.telemetry_id)register_telemetry(telemetry)try:with bind_telemetry(telemetry):@@ -2426,11 +2424,13 @@ class Session:)try:- await request_wait_tracker.wait_for_request(- telemetry.telemetry_id,+ task_context = get_task_context()+ await tracker.wait_for_task_work(+ task_id,+ exclude_work_id=task_context.work_id if task_context else None,timeout=_PHASE2_QUEUE_WAIT_TIMEOUT_SECONDS,)- except TimeoutError as exc:+ except (TimeoutError, asyncio.TimeoutError) as exc:telemetry.set_error("session.commit.phase2.wait_for_request","DEADLINE_EXCEEDED",@@ -2443,7 +2443,6 @@ class Session:_PHASE2_QUEUE_WAIT_TIMEOUT_SECONDS,)finally:- request_wait_tracker.cleanup(telemetry.telemetry_id)unregister_telemetry(telemetry.telemetry_id)# Phase 2 complete — update meta with telemetry and commit infodiff --git a/openviking/storage/collection_schemas.py b/openviking/storage/collection_schemas.pyindex 73b9b2d78..d10fea436 100644--- a/openviking/storage/collection_schemas.py+++ b/openviking/storage/collection_schemas.py@@ -21,6 +21,7 @@ from openviking.core.context import ContextType, ResourceContentTypefrom openviking.models.embedder.base import embed_compatfrom openviking.models.model_error import ModelErrorKind, model_error_kindfrom openviking.server.identity import RequestContext, Role+from openviking.service.task_tracker import get_task_trackerfrom openviking.service.task_tracker_concurrency import run_to_completionfrom openviking.storage.acl import ACL_GRANT_FIELDS, ACL_MODE_FIELD, AclModefrom openviking.storage.errors import (@@ -43,7 +44,6 @@ from openviking.storage.viking_vector_index_backend import (normalize_upsert_options,)from openviking.telemetry import bind_telemetry, resolve_telemetry-from openviking.telemetry.request_wait_tracker import get_request_wait_trackerfrom openviking.utils.circuit_breaker import CircuitBreakerOpenfrom openviking.utils.log_correlation import log_correlationfrom openviking.utils.tags import merge_search_tags@@ -761,9 +761,6 @@ class TextEmbeddingHandler(DequeueHandlerBase):await asyncio.sleep(wait)await self._reenqueue_embedding_msg(embedding_msg)self._merge_request_stats(embedding_msg.telemetry_id, requeue_count=1)- get_request_wait_tracker().record_embedding_requeue(- embedding_msg.telemetry_id- )return ProcessResult.requeued()execute_status = "error"request_failed_message = self._embedding_error_msg(@@ -857,9 +854,7 @@ class TextEmbeddingHandler(DequeueHandlerBase):# Write to vector databasetry:raw_upsert_options = inserted_data.pop("_upsert_options", {})- extracted_memory_type = raw_upsert_options.pop(- "extracted_memory_type", None- )+ extracted_memory_type = raw_upsert_options.pop("extracted_memory_type", None)# Reuse the actual vector-store ID when a semantic plan# rebuilds an existing same-level record. Only genuinely new# records derive an ID locally from (account, uri, level).@@ -1009,14 +1004,6 @@ class TextEmbeddingHandler(DequeueHandlerBase):return ProcessResult.failed(error_msg)finally:if embedding_msg is not None and execute_started_at is not None:- tracker = get_request_wait_tracker()- record_timing = getattr(tracker, "record_embedding_timing", None)- if callable(record_timing):- record_timing(- embedding_msg.telemetry_id,- queue_wait_ms=queue_wait_ms,- execute_ms=(time.perf_counter() - execute_started_at) * 1000.0,- )from openviking.metrics.datasources.resource import ResourceIngestionEventDataSourceResourceIngestionEventDataSource.record_stage(@@ -1031,8 +1018,6 @@ class TextEmbeddingHandler(DequeueHandlerBase):duration_seconds=(time.perf_counter() - execute_started_at),account_id=embedding_msg.context_data.get("account_id"),)- if embedding_msg is not None and request_failed_message is not None:- self._record_request_failure(embedding_msg, request_failed_message)async def on_cancelled(self, data: Optional[Dict[str, Any]]) -> ProcessResult:"""Settle request-scoped waiting when a queued embedding is cancelled."""@@ -1062,17 +1047,8 @@ class TextEmbeddingHandler(DequeueHandlerBase):*,vector_written: bool = False,) -> None:- tracker = get_request_wait_tracker()- tracker.mark_embedding_done(- embedding_msg.telemetry_id,- embedding_msg.id,- vector_written=vector_written,- )-- @staticmethod- def _record_request_failure(embedding_msg: EmbeddingMsg, message: str) -> None:- get_request_wait_tracker().mark_embedding_failed(- embedding_msg.telemetry_id,- embedding_msg.id,- message,- )+ if vector_written and embedding_msg.task_id and embedding_msg._task_work_id:+ get_task_tracker().record_context_count(+ embedding_msg.task_id,+ embedding_msg._task_work_id,+ )diff --git a/openviking/storage/content_write.py b/openviking/storage/content_write.pyindex 2c474fce8..9f3455c8b 100644--- a/openviking/storage/content_write.py+++ b/openviking/storage/content_write.py@@ -4,6 +4,7 @@from __future__ import annotations+import asyncioimport base64import binasciiimport hashlib@@ -26,6 +27,9 @@ from openviking.resource.processing_mode import ()from openviking.resource.watch_storage import is_watch_task_control_urifrom openviking.server.identity import RequestContext+from openviking.service.task_tracker import get_task_tracker, run_tracked_operation+from openviking.service.task_tracker_concurrency import run_to_completion+from openviking.service.task_work_index import get_enqueued_work_ids, get_task_contextfrom openviking.session.memory.memory_updater import MemoryUpdaterfrom openviking.session.memory.utils.memory_file_utils import MemoryFileUtilsfrom openviking.session.memory.utils.resource_refs import (@@ -75,7 +79,6 @@ from openviking.storage.resource_target import AgfsResourceTargetfrom openviking.storage.vector_ids import vector_record_idfrom openviking.storage.viking_fs import VikingFSfrom openviking.telemetry import get_current_telemetry-from openviking.telemetry.request_wait_tracker import get_request_wait_trackerfrom openviking.utils.content_hash import content_md5from openviking.utils.embedding_utils import vectorize_directory_metafrom openviking.utils.ingest_options import IngestOptions@@ -84,6 +87,7 @@ from openviking.utils.summarizer import FileRefreshRequest, Summarizerfrom openviking.utils.tags import normalize_search_tagsfrom openviking_cli.exceptions import (ConflictError,+ DeadlineExceededError,InvalidArgumentError,NotFoundError,OpenVikingError,@@ -208,6 +212,42 @@ class ContentWriteCoordinator:tags: list[str] | None = None,tag_mode: str = "replace",acl: AclSpec | Dict[str, Any] | None = None,+ ) -> Dict[str, Any]:+ async def execute() -> Dict[str, Any]:+ return await self._write(+ uri=uri,+ content=content,+ ctx=ctx,+ mode=mode,+ wait=wait,+ processing_mode=processing_mode,+ tags=tags,+ tag_mode=tag_mode,+ acl=acl,+ )++ result, _task_id = await run_tracked_operation(+ "content_write",+ account_id=ctx.account_id,+ user_id=ctx.user.user_id,+ operation=execute,+ wait=wait,+ resource_id=uri,+ )+ return result++ async def _write(+ self,+ *,+ uri: str,+ content: str,+ ctx: RequestContext,+ mode: str = "replace",+ wait: bool = False,+ processing_mode: ProcessingMode = DEFAULT_PROCESSING_MODE,+ tags: list[str] | None = None,+ tag_mode: str = "replace",+ acl: AclSpec | Dict[str, Any] | None = None,) -> Dict[str, Any]:self._validate_mode(mode)processing_mode = normalize_processing_mode(processing_mode)@@ -277,6 +317,36 @@ class ContentWriteCoordinator:wait: bool = True,skip_conflicts: bool = False,processing_mode: ProcessingMode = DEFAULT_PROCESSING_MODE,+ ) -> Dict[str, Any]:+ async def execute() -> Dict[str, Any]:+ return await self._batch_write(+ root_uri=root_uri,+ operations=operations,+ ctx=ctx,+ wait=wait,+ skip_conflicts=skip_conflicts,+ processing_mode=processing_mode,+ )++ result, _task_id = await run_tracked_operation(+ "content_batch_write",+ account_id=ctx.account_id,+ user_id=ctx.user.user_id,+ operation=execute,+ wait=wait,+ resource_id=root_uri,+ )+ return result++ async def _batch_write(+ self,+ *,+ root_uri: str,+ operations: list[dict[str, Any]],+ ctx: RequestContext,+ wait: bool = True,+ skip_conflicts: bool = False,+ processing_mode: ProcessingMode = DEFAULT_PROCESSING_MODE,) -> Dict[str, Any]:"""Write a bundle under one directory, then refresh it as a batch.@@ -445,25 +515,20 @@ class ContentWriteCoordinator:raisetelemetry_id = get_current_telemetry().telemetry_id- request_registered = False- enqueue_barrier_id: str | None = None+ coordinator_work_id: str | None = Nonehandoff_pending = Falserefresh_outcome: Optional[_BatchRefreshOutcome] = Nonetry:if refresh_kinds or sidecar_directories or committed_resource_plans:- if telemetry_id:- tracker = get_request_wait_tracker()- tracker.register_request(telemetry_id)- request_registered = True- lock_handoff = await self._viking_fs._async_agfs.pathlock_to_handoff(lease)- _, enqueue_barrier_id = await self._enqueue_lock_coordinator(- root_uri=normalized_root,- context_type=context_type_for_uri(normalized_root),- ctx=ctx,- telemetry_id=telemetry_id,- lock_handoff=lock_handoff,- )- handoff_pending = True+ lock_handoff = await self._viking_fs._async_agfs.pathlock_to_handoff(lease)+ _, coordinator_work_id = await self._enqueue_lock_coordinator(+ root_uri=normalized_root,+ context_type=context_type_for_uri(normalized_root),+ ctx=ctx,+ telemetry_id=telemetry_id,+ lock_handoff=lock_handoff,+ )+ handoff_pending = Truetry:for directory_uri, options in sorted(sidecar_directories.items()):await self._vectorize_semantic_directory(@@ -566,13 +631,13 @@ class ContentWriteCoordinator:},) from excfinally:- if enqueue_barrier_id is not None:- get_request_wait_tracker().mark_semantic_done(- telemetry_id,- enqueue_barrier_id,- processed_delta=0,+ if coordinator_work_id is not None:+ await run_to_completion(+ lambda: self._mark_coordinator_enqueue_complete(+ coordinator_work_id, ctx+ ))- enqueue_barrier_id = None+ coordinator_work_id = Noneif handoff_pending:try:@@ -586,7 +651,7 @@ class ContentWriteCoordinator:lock_settled = Truequeue_status = (await self._wait_for_request(telemetry_id=telemetry_id)- if wait and request_registered+ if waitelse refresh_outcome.queue_statusif refresh_outcome is not Noneelse None@@ -600,12 +665,6 @@ class ContentWriteCoordinator:await self._viking_fs._async_agfs.pathlock_release(lease)lock_settled = Truefinally:- if enqueue_barrier_id is not None:- get_request_wait_tracker().mark_semantic_done(- telemetry_id,- enqueue_barrier_id,- processed_delta=0,- )if not lock_settled:if handoff_pending:try:@@ -616,8 +675,6 @@ class ContentWriteCoordinator:else:await self._viking_fs._async_agfs.pathlock_release(lease)lock_settled = True- if request_registered:- get_request_wait_tracker().cleanup(telemetry_id)if write_error is not None:raise write_error@@ -929,14 +986,7 @@ class ContentWriteCoordinator:file_abstracts=file_abstracts,file_ingest_options=file_ingest_options,)- if msg.telemetry_id:- get_request_wait_tracker().register_semantic_root(msg.telemetry_id, msg.id)- try:- await semantic_queue.enqueue(msg)- except Exception as exc:- if msg.telemetry_id:- get_request_wait_tracker().mark_semantic_failed(msg.telemetry_id, msg.id, str(exc))- raise+ await semantic_queue.enqueue(msg)return actionasync def _enqueue_lock_coordinator(@@ -949,6 +999,9 @@ class ContentWriteCoordinator:lock_handoff: dict[str, Any],) -> tuple[SemanticMsg, str]:"""Enqueue the sole lease owner before scheduling request work."""+ context = get_task_context()+ if context is None:+ raise RuntimeError("TaskContext is required for lock-coordinated content writes")queue_manager = get_queue_manager()semantic_queue = queue_manager.get_queue(queue_manager.SEMANTIC, allow_create=True)msg = SemanticMsg(@@ -967,18 +1020,26 @@ class ContentWriteCoordinator:propagate_to_parent=False,wait_for_request_completion=True,)- enqueue_barrier_id = f"{msg.id}:enqueue"- tracker = get_request_wait_tracker()- tracker.register_semantic_root(telemetry_id, msg.id)- tracker.retain_request(telemetry_id, msg.id)- tracker.register_semantic_root(telemetry_id, enqueue_barrier_id)- try:- await semantic_queue.enqueue(msg)- except BaseException as exc:- tracker.mark_semantic_failed(telemetry_id, enqueue_barrier_id, str(exc))- tracker.mark_semantic_failed(telemetry_id, msg.id, str(exc))- raise- return msg, enqueue_barrier_id+ msg.coalesce_key = f"lock-coordinator|{msg.id}"+ receipt = await semantic_queue.enqueue(msg)+ if not receipt.work_id:+ raise RuntimeError("Lock coordinator was enqueued without Task Work identity")+ return msg, receipt.work_id++ async def _mark_coordinator_enqueue_complete(+ self,+ work_id: str,+ ctx: RequestContext,+ ) -> None:+ context = get_task_context()+ if context is None:+ raise RuntimeError("TaskContext is required for lock-coordinated content writes")+ await get_task_tracker().mark_enqueue_complete(+ context.task_id,+ work_id,+ account_id=ctx.account_id,+ user_id=ctx.user.user_id,+ )@staticmethoddef _raise_refresh_errors(queue_status: Dict[str, Any]) -> None:@@ -1482,8 +1543,7 @@ class ContentWriteCoordinator:previous_content: Optional[str] = Nonelock_settled = False- request_registered = False- enqueue_barrier_id: str | None = None+ coordinator_work_id: str | None = Nonehandoff_pending = Falsetry:target_state = await self._classify_locked_write_target(uri, ctx=ctx)@@ -1521,12 +1581,9 @@ class ContentWriteCoordinator:has_async_work = bool(prepared.plan.direct_index_actions or prepared.plan.file_refresh is not None)- if has_async_work and telemetry_id:- tracker = get_request_wait_tracker()- tracker.register_request(telemetry_id)- request_registered = True+ if has_async_work:lock_handoff = await self._viking_fs._async_agfs.pathlock_to_handoff(lease)- _, enqueue_barrier_id = await self._enqueue_lock_coordinator(+ _, coordinator_work_id = await self._enqueue_lock_coordinator(root_uri=uri,context_type=context_type,ctx=ctx,@@ -1534,7 +1591,6 @@ class ContentWriteCoordinator:lock_handoff=lock_handoff,)handoff_pending = True-work = await commit_and_enqueue_plan(prepared.plan,ctx=ctx,@@ -1554,13 +1610,11 @@ class ContentWriteCoordinator:ingest_options.acl_update, ctx)- if enqueue_barrier_id is not None:- get_request_wait_tracker().mark_semantic_done(- telemetry_id,- enqueue_barrier_id,- processed_delta=0,+ if coordinator_work_id is not None:+ await run_to_completion(+ lambda: self._mark_coordinator_enqueue_complete(coordinator_work_id, ctx))- enqueue_barrier_id = None+ coordinator_work_id = Noneif handoff_pending:try:await self._viking_fs._async_agfs.pathlock_handoff(lease)@@ -1572,11 +1626,7 @@ class ContentWriteCoordinator:await self._viking_fs._async_agfs.pathlock_release(lease)lock_settled = True- queue_status = (- await self._wait_for_request(telemetry_id=telemetry_id)- if wait- else None- )+ queue_status = await self._wait_for_request(telemetry_id=telemetry_id) if wait else Nonewritten_bytes = len(final_bytes)semantic_status, vector_status = self._plan_statuses(work, wait=wait, queue_status=queue_status@@ -1598,12 +1648,6 @@ class ContentWriteCoordinator:# when N already matches V, which makes that retry safe while V lags F.raisefinally:- if enqueue_barrier_id is not None:- get_request_wait_tracker().mark_semantic_done(- telemetry_id,- enqueue_barrier_id,- processed_delta=0,- )if not lock_settled:if handoff_pending:try:@@ -1614,8 +1658,6 @@ class ContentWriteCoordinator:else:await self._viking_fs._async_agfs.pathlock_release(lease)lock_settled = True- if request_registered:- get_request_wait_tracker().cleanup(telemetry_id)@staticmethoddef _ensure_explicit_write_content_action(@@ -1704,7 +1746,6 @@ class ContentWriteCoordinator:) from exclock_released = False- request_registered = Falsetry:# Preserve the public invariant that generated sidecars cannot be# materialized, while keeping the state check inside the exact lock.@@ -1713,19 +1754,12 @@ class ContentWriteCoordinator:await self._write_in_place(uri, content, mode=mode, ctx=ctx, lease_ref=lease, existing_raw=previous_content)- if wait and telemetry_id:- get_request_wait_tracker().register_request(telemetry_id)- request_registered = Truevector_enqueued = await self._vectorize_abstract_overview(uri=uri, ctx=ctx, ingest_options=ingest_options)await self._viking_fs._async_agfs.pathlock_release(lease)lock_released = True- queue_status = (- await self._wait_for_request(telemetry_id=telemetry_id)- if wait- else None- )+ queue_status = await self._wait_for_request(telemetry_id=telemetry_id) if wait else Nonevector_status = self._queue_work_status(requested=vector_enqueued,wait=wait,@@ -1746,9 +1780,6 @@ class ContentWriteCoordinator:if not lock_released:await self._viking_fs._async_agfs.pathlock_release(lease)raise- finally:- if request_registered:- get_request_wait_tracker().cleanup(telemetry_id)async def _vectorize_abstract_overview(self,@@ -1998,14 +2029,55 @@ class ContentWriteCoordinator:except AbstractOverviewFormatError as exc:raise InvalidArgumentError(str(exc)) from exc+ async def _enqueue_semantic_refresh(+ self,+ *,+ root_uri: str,+ changed_uri: str,+ context_type: str,+ ctx: RequestContext,+ change_type: str = "modified",+ target_uri: str = "",+ recursive: bool = False,+ force_refresh: bool = False,+ ingest_options: IngestOptions | None = None,+ file_md5: str | None = None,+ file_abstract: str = "",+ ) -> FreshnessAction:+ return await self._enqueue_semantic_refresh_changes(+ root_uri=root_uri,+ context_type=context_type,+ ctx=ctx,+ changes={change_type: [changed_uri]},+ target_uri=target_uri,+ recursive=recursive,+ force_refresh=force_refresh,+ ingest_options=ingest_options,+ file_md5s={changed_uri: file_md5} if file_md5 else None,+ file_abstracts={changed_uri: file_abstract} if file_abstract else None,+ )+async def _wait_for_request(self,*,telemetry_id: str,) -> Dict[str, Any]:- tracker = get_request_wait_tracker()- await tracker.wait_for_request(telemetry_id)- return tracker.build_queue_status(telemetry_id)+ del telemetry_id+ context = get_task_context()+ if context is None:+ raise RuntimeError("TaskContext is required for request-scoped queue waiting")+ work_ids = get_enqueued_work_ids()+ tracker = get_task_tracker()+ try:+ if context.work_id is None:+ await tracker.wait_for_task_work(context.task_id, timeout=None)+ elif work_ids:+ await tracker.wait_for_work(context.task_id, work_ids, timeout=None)+ except (TimeoutError, asyncio.TimeoutError) as exc:+ raise DeadlineExceededError("queue processing", None) from exc+ return tracker.build_queue_status(+ context.task_id, work_ids if context.work_id is not None else None+ )async def _write_memory_with_refresh(self,@@ -2030,7 +2102,6 @@ class ContentWriteCoordinator:) from excreleased = False- request_registered = Falsetry:target_state = await self._classify_locked_write_target(uri, ctx=ctx)is_new_file = target_state.is_new_file@@ -2049,9 +2120,6 @@ class ContentWriteCoordinator:written_bytes = len(final_bytes)await self._viking_fs._async_agfs.pathlock_release(lease)released = True- if wait and telemetry_id and self._vikingdb_has_queue():- get_request_wait_tracker().register_request(telemetry_id)- request_registered = Trueoverview_refreshed = await MemoryUpdater.refresh_schema_overview(viking_fs=self._viking_fs,directory_uri=root_uri,@@ -2063,11 +2131,14 @@ class ContentWriteCoordinator:uri=uri,memory_type=MemoryUpdater.memory_type_from_uri(root_uri),ctx=ctx,+ strict=True,ingest_options=ingest_options,)queue_status = Noneif embedding_requested and wait:- queue_status = await self._wait_for_request(telemetry_id=telemetry_id)+ queue_status = await self._wait_for_request(+ telemetry_id=telemetry_id,+ )vector_status = self._queue_work_status(requested=embedding_requested,wait=wait,@@ -2090,8 +2161,7 @@ class ContentWriteCoordinator:await self._viking_fs._async_agfs.pathlock_release(lease)raisefinally:- if request_registered:- get_request_wait_tracker().cleanup(telemetry_id)+ passdef _vikingdb_has_queue(self) -> bool:if not self._vikingdb:diff --git a/openviking/storage/queuefs/add_resource_msg.py b/openviking/storage/queuefs/add_resource_msg.pyindex e9b16364b..43e1a53d6 100644--- a/openviking/storage/queuefs/add_resource_msg.py+++ b/openviking/storage/queuefs/add_resource_msg.py@@ -22,6 +22,7 @@ class AddResourceMsg:user_id: strgroup_ids: list[str] = field(default_factory=list)role: str+ _task_work_id: str = ""path: str = ""source_path: str = ""telemetry_id: Optional[str] = None@@ -159,6 +160,7 @@ class AddResourceMsg:return cls(task_id=str(task_id),+ _task_work_id=str(data.get("_task_work_id") or ""),path=str(path or ""),source_path=str(data.get("source_path") or path or ""),root_uri=str(root_uri),diff --git a/openviking/storage/queuefs/add_resource_processor.py b/openviking/storage/queuefs/add_resource_processor.pyindex 214a151ef..f7b4ec865 100644--- a/openviking/storage/queuefs/add_resource_processor.py+++ b/openviking/storage/queuefs/add_resource_processor.py@@ -11,7 +11,10 @@ from typing import Any, Dict, Optionalfrom openviking.observability.context import bind_execution_contextfrom openviking.server.identity import RequestContext, Rolefrom openviking.service.task_tracker import TaskStatus, get_task_tracker-from openviking.service.task_work_index import bind_task_context, extract_task_metadata+from openviking.service.task_work_index import (+ bind_task_context,+ extract_task_metadata,+)from openviking.storage.queuefs.add_resource_msg import AddResourceMsgfrom openviking.storage.queuefs.named_queue import DequeueHandlerBasefrom openviking.storage.queuefs.process_result import ProcessResult@@ -22,7 +25,6 @@ from openviking.telemetry import (resolve_telemetry,unregister_telemetry,)-from openviking.telemetry.request_wait_tracker import get_request_wait_trackerfrom openviking.telemetry.resource_summary import record_resource_queue_metricsfrom openviking.utils.log_correlation import log_correlationfrom openviking_cli.exceptions import OpenVikingError@@ -177,7 +179,6 @@ class AddResourceProcessor(DequeueHandlerBase):unregister_telemetry(telemetry_id)return ProcessResult.success()- metadata = extract_task_metadata(data)replay_result = getattr(task, "result", None)resource_lock = Noneif replay_result is None:@@ -212,8 +213,6 @@ class AddResourceProcessor(DequeueHandlerBase):else:telemetry_id = telemetry.telemetry_idregister_telemetry(telemetry)- request_wait_tracker = get_request_wait_tracker()- request_wait_tracker.register_request(telemetry_id)current_stage = "queued"async def _set_stage(stage: str) -> None:@@ -298,18 +297,19 @@ class AddResourceProcessor(DequeueHandlerBase):)else:result = deepcopy(replay_result)- await tracker.wait_for_descendants(msg.task_id, metadata.work_id)- result.setdefault(- "queue_status", request_wait_tracker.build_queue_status(telemetry_id)+ metadata = extract_task_metadata(data)+ await tracker.wait_for_task_work(+ msg.task_id,+ exclude_work_id=metadata.work_id if metadata is not None else None,)+ result["queue_status"] = tracker.build_queue_status(msg.task_id)if replay_result is None:- result["context_count"] = request_wait_tracker.get_embedding_context_count(- telemetry_id- )+ result["context_count"] = tracker.get_context_count(msg.task_id)record_resource_queue_metrics(telemetry=telemetry,telemetry_id=telemetry_id,root_uri=result.get("root_uri"),+ queue_timing=tracker.get_queue_timing(msg.task_id),)telemetry.set("resource.total.duration_ms", telemetry.elapsed_ms())@@ -406,7 +406,6 @@ class AddResourceProcessor(DequeueHandlerBase):terminal = Truereturn ProcessResult.failed(str(exc))finally:- request_wait_tracker.cleanup(telemetry_id)unregister_telemetry(telemetry_id)with suppress(Exception):if resource_lock is not None:diff --git a/openviking/storage/queuefs/embedding_msg.py b/openviking/storage/queuefs/embedding_msg.pyindex b5f7203ca..09c7ed6d4 100644--- a/openviking/storage/queuefs/embedding_msg.py+++ b/openviking/storage/queuefs/embedding_msg.py@@ -100,6 +100,8 @@ class EmbeddingMsg:payload: EmbeddingPayloadid: str = field(default_factory=lambda: str(uuid4()))telemetry_id: str = ""+ task_id: str = ""+ _task_work_id: str = ""queue_enqueued_at: float = 0.0failure_requeue_count: int = 0@@ -108,6 +110,8 @@ class EmbeddingMsg:message: Optional[EmbeddingInput] = None,context_data: Optional[Dict[str, Any]] = None,telemetry_id: str = "",+ task_id: str = "",+ _task_work_id: str = "",action: IndexAction | str = IndexAction.UPSERT,record_ids: Optional[List[str]] = None,update_fields: Optional[Dict[str, Any]] = None,@@ -120,6 +124,8 @@ class EmbeddingMsg:) -> None:self.id = str(uuid4())self.telemetry_id = telemetry_id+ self.task_id = task_id+ self._task_work_id = _task_work_idself.queue_enqueued_at = max(float(queue_enqueued_at or 0.0), 0.0)self.failure_requeue_count = max(int(failure_requeue_count or 0), 0)self.action = IndexAction(action)@@ -325,6 +331,8 @@ class EmbeddingMsg:return {"id": self.id,"telemetry_id": self.telemetry_id,+ "task_id": self.task_id,+ "_task_work_id": self._task_work_id,"queue_enqueued_at": self.queue_enqueued_at,"failure_requeue_count": self.failure_requeue_count,"action": self.action.value,@@ -391,6 +399,8 @@ class EmbeddingMsg:action=action,payload=payload,telemetry_id=str(data.get("telemetry_id") or ""),+ task_id=str(data.get("task_id") or ""),+ _task_work_id=str(data.get("_task_work_id") or ""),queue_enqueued_at=data.get("queue_enqueued_at", 0.0),failure_requeue_count=data.get("failure_requeue_count", 0),)@@ -399,6 +409,8 @@ class EmbeddingMsg:message=data.get("message"),context_data=dict(data.get("context_data") or {}),telemetry_id=str(data.get("telemetry_id") or ""),+ task_id=str(data.get("task_id") or ""),+ _task_work_id=str(data.get("_task_work_id") or ""),action=action,record_ids=data.get("record_ids"),update_fields=data.get("update_fields"),diff --git a/openviking/storage/queuefs/named_queue.py b/openviking/storage/queuefs/named_queue.pyindex 9b17120d1..c0cb7dd0e 100644--- a/openviking/storage/queuefs/named_queue.py+++ b/openviking/storage/queuefs/named_queue.py@@ -18,6 +18,7 @@ from openviking.storage.queuefs.queue_middleware import (AckContext,ClearContext,EnqueueContext,+ EnqueueReceipt,ProcessContext,QueueMiddleware,)@@ -171,10 +172,12 @@ class NamedQueue:passself._initialized = True- async def enqueue(self, data: Union[str, Dict[str, Any]]) -> str:+ async def enqueue(self, data: Union[str, Dict[str, Any]]) -> EnqueueReceipt:"""Send message to queue (enqueue)."""await self._ensure_initialized()- return await self._run_middleware("enqueue", EnqueueContext(self.name, data), self._enqueue)+ ctx = EnqueueContext(self.name, data)+ message_id = await self._run_middleware("enqueue", ctx, self._enqueue)+ return EnqueueReceipt(message_id, ctx.work_id)async def _enqueue(self, ctx: EnqueueContext) -> str:data = json.dumps(ctx.payload) if isinstance(ctx.payload, dict) else ctx.payloaddiff --git a/openviking/storage/queuefs/process_result.py b/openviking/storage/queuefs/process_result.pyindex 455f4ab6c..c9ce2d12d 100644--- a/openviking/storage/queuefs/process_result.py+++ b/openviking/storage/queuefs/process_result.py@@ -4,7 +4,7 @@from dataclasses import dataclassfrom enum import Enum-from typing import Any, Optional+from typing import Any, Optional, Tupleclass ProcessOutcome(str, Enum):@@ -25,6 +25,7 @@ class ProcessResult:outcome: ProcessOutcomevalue: Any = Noneerror: Optional[str] = None+ errors: Optional[Tuple[str, ...]] = Nonedef __post_init__(self) -> None:if not isinstance(self.outcome, ProcessOutcome):@@ -32,16 +33,25 @@ class ProcessResult:if self.outcome is ProcessOutcome.FAILED:if not isinstance(self.error, str):raise ValueError("failed results require an error string")- elif self.error is not None:- raise ValueError("only failed results may contain an error")+ if self.errors is not None and (+ not self.errors or any(not isinstance(value, str) for value in self.errors)+ ):+ raise ValueError("failed result errors must be a non-empty tuple of strings")+ elif self.error is not None or self.errors is not None:+ raise ValueError("only failed results may contain errors")@classmethoddef success(cls, value: Any = None) -> "ProcessResult":return cls(ProcessOutcome.SUCCESS, value=value)@classmethod- def failed(cls, error: str) -> "ProcessResult":- return cls(ProcessOutcome.FAILED, error=error)+ def failed(+ cls,+ error: str,+ *,+ errors: Optional[Tuple[str, ...]] = None,+ ) -> "ProcessResult":+ return cls(ProcessOutcome.FAILED, error=error, errors=errors)@classmethoddef requeued(cls) -> "ProcessResult":diff --git a/openviking/storage/queuefs/queue_middleware.py b/openviking/storage/queuefs/queue_middleware.pyindex c105b0ddd..4ac3c7f2a 100644--- a/openviking/storage/queuefs/queue_middleware.py+++ b/openviking/storage/queuefs/queue_middleware.py@@ -16,6 +16,22 @@ class EnqueueContext:queue: strpayload: Payloadcommitted: bool = False+ work_id: Optional[str] = None+++class EnqueueReceipt(str):+ """String-compatible QueueFS message ID with optional logical Work identity."""++ work_id: Optional[str]++ def __new__(cls, message_id: str, work_id: Optional[str] = None) -> "EnqueueReceipt":+ value = super().__new__(cls, message_id)+ value.work_id = work_id+ return value++ @property+ def message_id(self) -> str:+ return str(self)@dataclassdiff --git a/openviking/storage/queuefs/reindex_msg.py b/openviking/storage/queuefs/reindex_msg.pyindex 0db0397db..420df78bc 100644--- a/openviking/storage/queuefs/reindex_msg.py+++ b/openviking/storage/queuefs/reindex_msg.py@@ -15,6 +15,7 @@ class ReindexMsg:account_id: struser_id: strrole: str+ _task_work_id: str = ""force: bool = Falserecursive: bool = Truetags: Optional[list[str]] = None@@ -41,6 +42,7 @@ class ReindexMsg:lock_handoff_retry = 0return cls(task_id=str(data["task_id"]),+ _task_work_id=str(data.get("_task_work_id") or ""),uri=str(data["uri"]),object_type=str(data["object_type"]),mode=str(data["mode"]),diff --git a/openviking/storage/queuefs/reindex_processor.py b/openviking/storage/queuefs/reindex_processor.pyindex a9c15ea03..229a1779b 100644--- a/openviking/storage/queuefs/reindex_processor.py+++ b/openviking/storage/queuefs/reindex_processor.py@@ -81,7 +81,6 @@ class ReindexProcessor(DequeueHandlerBase):msg = ReindexMsg.from_dict(payload)ctx = self._ctx(msg)tracker = get_task_tracker()- metadata = extract_task_metadata(data)task = await tracker.get(msg.task_id,account_id=ctx.account_id,@@ -151,8 +150,11 @@ class ReindexProcessor(DequeueHandlerBase):ctx=ctx,existing_lease=executor_lease,)- if metadata is not None:- await tracker.wait_for_descendants(msg.task_id, metadata.work_id)+ metadata = extract_task_metadata(data)+ await tracker.wait_for_task_work(+ msg.task_id,+ exclude_work_id=metadata.work_id if metadata is not None else None,+ )await tracker.complete(msg.task_id, result, account_id=ctx.account_id, user_id=ctx.user.user_id)diff --git a/openviking/storage/queuefs/semantic_executor.py b/openviking/storage/queuefs/semantic_executor.pyindex 2c3532143..8dd96d5e0 100644--- a/openviking/storage/queuefs/semantic_executor.py+++ b/openviking/storage/queuefs/semantic_executor.py@@ -516,6 +516,8 @@ class SemanticTreeExecutor:self._task_context.task_id,self._task_context.account_id,self._task_context.user_id,+ enqueued_work=self._task_context.enqueued_work,+ work_id=self._task_context.work_id,)if self._task_context is not Noneelse detach_task_context()diff --git a/openviking/storage/queuefs/semantic_msg.py b/openviking/storage/queuefs/semantic_msg.pyindex f4e41d297..4bbb151f6 100644--- a/openviking/storage/queuefs/semantic_msg.py+++ b/openviking/storage/queuefs/semantic_msg.py@@ -60,6 +60,8 @@ class SemanticMsg:# Additional flagsskip_vectorization: bool = Falsetelemetry_id: str = ""+ task_id: str = ""+ _task_work_id: str = ""target_uri: str = ""lock_handoff: Optional[Dict[str, Any]] = Noneis_code_repo: bool = False@@ -102,6 +104,8 @@ class SemanticMsg:role: str = "root",skip_vectorization: bool = False,telemetry_id: str = "",+ task_id: str = "",+ _task_work_id: str = "",target_uri: str = "",lock_handoff: Optional[Dict[str, Any]] = None,is_code_repo: bool = False,@@ -141,6 +145,8 @@ class SemanticMsg:self.role = roleself.skip_vectorization = skip_vectorizationself.telemetry_id = telemetry_id+ self.task_id = task_id+ self._task_work_id = _task_work_idself.target_uri = target_uriself.lock_handoff = lock_handoffself.is_code_repo = is_code_repo@@ -224,6 +230,8 @@ class SemanticMsg:role=data.get("role", "root"),skip_vectorization=data.get("skip_vectorization", False),telemetry_id=data.get("telemetry_id", ""),+ task_id=data.get("task_id", ""),+ _task_work_id=data.get("_task_work_id", ""),target_uri=data.get("target_uri", ""),lock_handoff=data.get("lock_handoff"),is_code_repo=data.get("is_code_repo", False),diff --git a/openviking/storage/queuefs/semantic_processor.py b/openviking/storage/queuefs/semantic_processor.pyindex 4163c3216..222494fdb 100644--- a/openviking/storage/queuefs/semantic_processor.py+++ b/openviking/storage/queuefs/semantic_processor.py@@ -36,8 +36,9 @@ from openviking.parse.parsers.media.utils import (from openviking.prompts import render_promptfrom openviking.pyagfs.exceptions import AGFSNotADirectoryErrorfrom openviking.server.identity import RequestContext, Role+from openviking.service.task_tracker import get_task_tracker, run_tracked_operationfrom openviking.service.task_tracker_concurrency import run_to_completion-from openviking.service.task_work_index import detach_task_context+from openviking.service.task_work_index import detach_task_context, get_task_contextfrom openviking.storage.abstract_overview import (AbstractOverviewWriteResult,body_for_preview,@@ -63,7 +64,6 @@ from openviking.telemetry import (bind_telemetry,bind_telemetry_stage,)-from openviking.telemetry.request_wait_tracker import get_request_wait_trackerfrom openviking.telemetry.span_models import create_root_span_attributesfrom openviking.utils.circuit_breaker import CircuitBreaker, CircuitBreakerOpenfrom openviking.utils.ingest_options import IngestOptions@@ -338,11 +338,9 @@ class SemanticProcessor(DequeueHandlerBase):try:await work.reenqueue_after_error()self._merge_request_stats(msg.telemetry_id, requeue_count=1)- get_request_wait_tracker().record_semantic_requeue(msg.telemetry_id)except Exception as requeue_err:logger.error(f"Failed to re-enqueue semantic message: {requeue_err}")self._merge_request_stats(msg.telemetry_id, error_count=1)- get_request_wait_tracker().mark_semantic_failed(msg.telemetry_id, msg.id, str(error))await self._cleanup_local_artifact(msg)return ProcessResult.failed(str(error))return ProcessResult.requeued()@@ -448,7 +446,55 @@ class SemanticProcessor(DequeueHandlerBase):lock: Optional[Dict[str, Any]] = None,) -> ProcessResult:"""Process dequeued SemanticMsg, recursively process all subdirectories."""- msg: Optional[SemanticMsg] = None+ import json++ if not data:+ return ProcessResult.success()+ try:+ if "data" in data and isinstance(data["data"], str):+ data = json.loads(data["data"])+ msg = SemanticMsg.from_dict(data)+ except Exception as exc:+ logger.error("Invalid semantic message: %s", exc, exc_info=True)+ return ProcessResult.failed(str(exc))++ if msg.context_type != "skill" or get_task_context() is not None:+ return await self._process_message(msg, lock)++ # Parent refreshes and recovered taskless Skill messages need their own+ # child-work collector so package ownership lasts through vector writes.+ result: Optional[ProcessResult] = None++ async def process() -> Dict[str, Any]:+ nonlocal result+ result = await self._process_message(msg, lock)+ if result.error is not None:+ context = get_task_context()+ assert context is not None+ await get_task_tracker().fail(+ context.task_id,+ result.error,+ account_id=msg.account_id,+ user_id=msg.user_id,+ )+ return {"uri": msg.uri, "outcome": result.outcome.value}++ await run_tracked_operation(+ "semantic_refresh",+ account_id=msg.account_id,+ user_id=msg.user_id,+ resource_id=msg.uri,+ operation=process,+ wait=False,+ )+ assert result is not None+ return result++ async def _process_message(+ self,+ msg: SemanticMsg,+ lock: Optional[Dict[str, Any]],+ ) -> ProcessResult:collector = Nonework: Optional[SemanticMessageWork] = Noneexecute_started_at: float | None = None@@ -456,16 +502,6 @@ class SemanticProcessor(DequeueHandlerBase):execute_status = "ok"breaker = Nonetry:- import json-- if not data:- return ProcessResult.success()-- if "data" in data and isinstance(data["data"], str):- data = json.loads(data["data"])-- assert data is not None- msg = SemanticMsg.from_dict(data)breaker = self._account_breaker(msg.account_id)work = self._message_work(msg, lock)work.start()@@ -474,8 +510,6 @@ class SemanticProcessor(DequeueHandlerBase):queue_wait_ms = max((time.time() - msg.queue_enqueued_at) * 1000.0, 0.0)if VikingURI(msg.uri).parent is None:logger.warning("Skipping semantic generation for root URI: %s", msg.uri)- if msg.telemetry_id and msg.id:- get_request_wait_tracker().mark_semantic_done(msg.telemetry_id, msg.id)await self._cleanup_local_artifact(msg)return ProcessResult.success()if is_semantic_msg_stale(msg):@@ -504,8 +538,6 @@ class SemanticProcessor(DequeueHandlerBase):msg.coalesce_version,)await work.skip()- if msg.telemetry_id and msg.id:- get_request_wait_tracker().mark_semantic_done(msg.telemetry_id, msg.id)await self._cleanup_local_artifact(msg)return ProcessResult.success()# Circuit breaker: if API is known-broken, re-enqueue and wait@@ -517,7 +549,6 @@ class SemanticProcessor(DequeueHandlerBase):)await work.reenqueue()self._merge_request_stats(msg.telemetry_id, requeue_count=1)- get_request_wait_tracker().record_semantic_requeue(msg.telemetry_id)return ProcessResult.requeued()collector = work.resolve_telemetry()telemetry_ctx = bind_telemetry(collector) if collector is not None else nullcontext()@@ -544,8 +575,6 @@ class SemanticProcessor(DequeueHandlerBase):if not await get_viking_fs().exists(msg.uri, ctx=current_ctx):logger.info("Skipping semantic message for missing root: uri=%s", msg.uri)await work.skip()- if msg.telemetry_id and msg.id:- get_request_wait_tracker().mark_semantic_done(msg.telemetry_id, msg.id)return ProcessResult.success()if (@@ -557,16 +586,32 @@ class SemanticProcessor(DequeueHandlerBase):"SemanticProcessor requires a VLM resolver for account-owned work")if not await work.acquire_lock(current_ctx):- get_request_wait_tracker().mark_semantic_done(msg.telemetry_id, msg.id)return ProcessResult.success()semantic_lock = work.scopeassert semantic_lock is not None- if (- msg.wait_for_request_completion- and not get_request_wait_tracker().is_complete_except(- msg.telemetry_id, msg.id+ task_context = get_task_context()+ if msg.wait_for_request_completion and task_context is not None:+ tracker = get_task_tracker()+ enqueue_complete = await tracker.is_enqueue_complete(+ task_context.task_id,+ task_context.work_id or "",+ account_id=task_context.account_id,+ user_id=task_context.user_id,+ )+ siblings_pending = tracker.has_work(+ task_context.task_id,+ exclude_work_id=task_context.work_id,)+ else:+ enqueue_complete = True+ siblings_pending = False+ if not enqueue_complete and not tracker.has_active_producer(+ task_context.task_id):+ return ProcessResult.failed(+ "Content write producer ended before child Work enqueue completed"+ )+ if not enqueue_complete or siblings_pending:# Keep a single-worker semantic queue moving while this# coordinator retains the producer's lease.await asyncio.sleep(0.05)@@ -755,11 +800,6 @@ class SemanticProcessor(DequeueHandlerBase):if failure is not None:return failureis_request_coordinator = msg.wait_for_request_completion- get_request_wait_tracker().mark_semantic_done(- msg.telemetry_id,- msg.id,- processed_delta=0 if is_request_coordinator else 1,- )if not is_request_coordinator:self._merge_request_stats(msg.telemetry_id, processed=1)logger.info(f"Completed semantic generation for: {msg.uri}")@@ -802,9 +842,6 @@ class SemanticProcessor(DequeueHandlerBase):)if msg is not None:self._merge_request_stats(msg.telemetry_id, error_count=1)- get_request_wait_tracker().mark_semantic_failed(- msg.telemetry_id, msg.id, str(e)- )if msg is not None:await self._cleanup_local_artifact(msg)return ProcessResult.failed(str(e))@@ -823,9 +860,6 @@ class SemanticProcessor(DequeueHandlerBase):breaker.record_failure(e)if msg is not None:self._merge_request_stats(msg.telemetry_id, error_count=1)- get_request_wait_tracker().mark_semantic_failed(- msg.telemetry_id, msg.id, str(e)- )if msg is not None:await self._cleanup_local_artifact(msg)return ProcessResult.failed(str(e))@@ -839,21 +873,10 @@ class SemanticProcessor(DequeueHandlerBase):)if msg is not None:self._merge_request_stats(msg.telemetry_id, error_count=1)- get_request_wait_tracker().mark_semantic_failed(- msg.telemetry_id, msg.id, str(e)- )await self._cleanup_local_artifact(msg)return ProcessResult.failed(str(e))finally:if msg is not None and execute_started_at is not None:- tracker = get_request_wait_tracker()- record_timing = getattr(tracker, "record_semantic_timing", None)- if callable(record_timing):- record_timing(- msg.telemetry_id,- queue_wait_ms=queue_wait_ms,- execute_ms=(time.perf_counter() - execute_started_at) * 1000.0,- )from openviking.metrics.datasources.resource import ResourceIngestionEventDataSourceResourceIngestionEventDataSource.record_stage(@@ -1991,7 +2014,6 @@ class SemanticProcessor(DequeueHandlerBase):return await _enqueue_embedding_message(embedding_queue,embedding_msg,- failure_message=f"Failed to enqueue vector scalar update for {uri}",)async def _vectorize_single_file(diff --git a/openviking/storage/queuefs/semantic_work.py b/openviking/storage/queuefs/semantic_work.pyindex 116088eda..ab8973bc2 100644--- a/openviking/storage/queuefs/semantic_work.py+++ b/openviking/storage/queuefs/semantic_work.py@@ -8,7 +8,9 @@ import asynciofrom typing import TYPE_CHECKING, Any, Awaitable, Callable, TypeVarfrom openviking.server.identity import RequestContext+from openviking.service.task_tracker import get_task_trackerfrom openviking.service.task_tracker_concurrency import run_to_completion+from openviking.service.task_work_index import get_enqueued_work_ids, get_task_contextfrom openviking.storage.queuefs.process_result import ProcessResultfrom openviking.storage.queuefs.semantic_executor import SemanticTreeStatsfrom openviking.storage.queuefs.semantic_lock import SemanticLockScope@@ -20,7 +22,6 @@ from openviking.telemetry import (resolve_telemetry,unregister_telemetry,)-from openviking.telemetry.request_wait_tracker import get_request_wait_trackerif TYPE_CHECKING:from openviking.storage.queuefs.semantic_processor import SemanticProcessor@@ -62,9 +63,14 @@ class SemanticMessageWork:async def finish_processing(self, succeeded: bool) -> None:assert self.scope is not None- if self.msg.lock_handoff is not None:- await get_request_wait_tracker().wait_for_embeddings(- self.msg.telemetry_id,+ context = get_task_context()+ if self.msg.wait_for_request_completion and context is not None:+ await get_task_tracker().wait_for_task_work(+ context.task_id,+ exclude_work_id=context.work_id,+ )+ elif self.msg.lock_handoff is not None:+ await self._wait_for_embedding_work(stop_waiting=getattr(self.processor, "_embedding_worker_stopped", None),)if succeeded:@@ -74,6 +80,20 @@ class SemanticMessageWork:def failure_result(self, stats: SemanticTreeStats | None) -> ProcessResult | None:return None+ async def _wait_for_embedding_work(+ self,+ *,+ stop_waiting: Callable[[], bool] | None = None,+ ) -> None:+ context = get_task_context()+ work_ids = get_enqueued_work_ids("Embedding")+ if context is not None and work_ids:+ await get_task_tracker().wait_for_work(+ context.task_id,+ work_ids,+ stop_waiting=stop_waiting,+ )+async def reenqueue(self) -> None:async def enqueue(queue, msg):if self.scope is not None and self.scope.lock is not None and self.scope._owned:@@ -93,11 +113,8 @@ class SemanticMessageWork:async def cancel(self) -> None:if self.scope is None:await self.skip()- get_request_wait_tracker().mark_semantic_done(self.msg.telemetry_id, self.msg.id)async def cancel_queued(self) -> None:- if self.msg.telemetry_id and self.msg.id:- get_request_wait_tracker().mark_semantic_done(self.msg.telemetry_id, self.msg.id)await self.processor._release_cancelled_semantic_lock(self.msg)async def close(self) -> None:@@ -120,7 +137,7 @@ class SkillSemanticMessageWork(SemanticMessageWork):caller_lock: dict[str, Any] | None,) -> None:super().__init__(processor, msg, caller_lock)- self._internal_tracking = False+ self._owns_telemetry = Falseself._lock_started = Falseself._lock_closed = False@@ -130,12 +147,6 @@ class SkillSemanticMessageWork(SemanticMessageWork):msg.telemetry_id = OperationTelemetry(operation="skill_index", enabled=False).telemetry_id- tracker = get_request_wait_tracker()- self._internal_tracking = not tracker.has_request(msg.telemetry_id)- tracker.retain_request(msg.telemetry_id, msg.id)- tracker.register_semantic_root(msg.telemetry_id, msg.id)- if self._internal_tracking:- tracker.cleanup(msg.telemetry_id)def resolve_telemetry(self) -> OperationTelemetry:collector = super().resolve_telemetry()@@ -143,6 +154,7 @@ class SkillSemanticMessageWork(SemanticMessageWork):collector = OperationTelemetry(operation="skill_index", enabled=False)collector.telemetry_id = self.msg.telemetry_idregister_telemetry(collector)+ self._owns_telemetry = Truereturn collectorasync def acquire_lock(self, ctx: RequestContext) -> bool:@@ -158,8 +170,7 @@ class SkillSemanticMessageWork(SemanticMessageWork):async def finish_processing(self, succeeded: bool) -> None:async def finish():if not self.msg.skip_vectorization:- await get_request_wait_tracker().wait_for_embeddings(- self.msg.telemetry_id,+ await self._wait_for_embedding_work(stop_waiting=self.processor._embedding_worker_stopped,)if succeeded:@@ -173,13 +184,12 @@ class SkillSemanticMessageWork(SemanticMessageWork):def failure_result(self, stats: SemanticTreeStats | None) -> ProcessResult | None:if stats is None or not stats.failures:return None- for failure in stats.failures:- get_request_wait_tracker().mark_semantic_failed(- self.msg.telemetry_id, self.msg.id, failure- )# Report every file error, but settle this queue item only once.self.processor._merge_request_stats(self.msg.telemetry_id, error_count=len(stats.failures))- return ProcessResult.failed("\n".join(stats.failures))+ return ProcessResult.failed(+ "\n".join(stats.failures),+ errors=tuple(stats.failures),+ )async def reenqueue(self) -> None:async def enqueue(queue, msg):@@ -201,7 +211,6 @@ class SkillSemanticMessageWork(SemanticMessageWork):await run_to_completion(lambda: self.processor._release_cancelled_semantic_lock(self.msg))- get_request_wait_tracker().mark_semantic_done(self.msg.telemetry_id, self.msg.id)raiseasync def skip(self) -> None:@@ -214,18 +223,13 @@ class SkillSemanticMessageWork(SemanticMessageWork):if not self._lock_started:# Cancellation during the circuit-breaker delay precedes adoption.await self.skip()- get_request_wait_tracker().mark_semantic_done(self.msg.telemetry_id, self.msg.id)async def close(self) -> None:if self.scope is not None and not self._lock_closed:await run_to_completion(self.scope.close)- if self._internal_tracking and get_request_wait_tracker().is_complete(- self.msg.telemetry_id- ):+ if self._owns_telemetry:unregister_telemetry(self.msg.telemetry_id)async def cancel_queued(self) -> None:# Skill rollback must observe release before this item is settled.await self.processor._release_cancelled_semantic_lock(self.msg)- if self.msg.telemetry_id and self.msg.id:- get_request_wait_tracker().mark_semantic_done(self.msg.telemetry_id, self.msg.id)diff --git a/openviking/storage/queuefs/session_commit_msg.py b/openviking/storage/queuefs/session_commit_msg.pyindex d66dce1a6..a62b67072 100644--- a/openviking/storage/queuefs/session_commit_msg.py+++ b/openviking/storage/queuefs/session_commit_msg.py@@ -13,6 +13,7 @@ class SessionCommitMsg:session_uri: strarchive_uri: struser: Dict[str, str]+ _task_work_id: str = ""memory_policy: Dict[str, Any] = field(default_factory=dict)# When True, Phase 2's final meta merge also clears the auto-commit error# fields and stamps last_auto_commit_at. Defaults keep old producers working.diff --git a/openviking/storage/viking_fs/_snapshot.py b/openviking/storage/viking_fs/_snapshot.pyindex 0b1fc7421..b88c352ce 100644--- a/openviking/storage/viking_fs/_snapshot.py+++ b/openviking/storage/viking_fs/_snapshot.py@@ -5,7 +5,7 @@import asyncioimport sysfrom dataclasses import replace-from typing import Any, Dict, List, Optional, Union+from typing import Any, Dict, List, Literal, Optional, Unionfrom openviking.pyagfs.exceptions import (AGFSInvalidOperationError,@@ -829,7 +829,14 @@ class _SnapshotMixin:)for op, uri, level in tasks:loop.create_task(- self._run_vector_rebuild(executor, op, uri, level, ctx),+ self._run_vector_rebuild(+ executor,+ op,+ uri,+ level,+ ctx,+ file_task_owner="standalone",+ ),name=f"vikingfs-git-{op}:{uri}:{int(level)}",)@@ -867,7 +874,14 @@ class _SnapshotMixin:with bind_task_context(task_id, ctx.account_id, ctx.user.user_id):await asyncio.gather(*[- self._run_vector_rebuild(executor, op, uri, level, reindex_ctx)+ self._run_vector_rebuild(+ executor,+ op,+ uri,+ level,+ reindex_ctx,+ file_task_owner="current",+ )for (op, uri, level) in tasks])@@ -897,13 +911,27 @@ class _SnapshotMixin:uri: str,level: Any,ctx: RequestContext,+ *,+ file_task_owner: Literal["current", "standalone"],) -> None:"""Wrapper coroutine: dispatch one vector task and swallow errors."""try:if op == "reindex_marker":await executor.reindex_directory_marker(dir_uri=uri, level=level, ctx=ctx)elif op == "reindex_file":- await executor.execute(uri=uri, mode="vectors_only", wait=True, ctx=ctx)+ if file_task_owner == "current":+ await executor.execute_in_current_task(+ uri=uri,+ mode="vectors_only",+ ctx=ctx,+ )+ else:+ await executor.execute(+ uri=uri,+ mode="vectors_only",+ wait=True,+ ctx=ctx,+ )elif op == "delete":await executor.delete_uri_level(uri=uri, level=level, ctx=ctx)else: # pragma: no cover - defensivediff --git a/openviking/telemetry/request_wait_tracker.py b/openviking/telemetry/request_wait_tracker.pydeleted file mode 100644index 3e2d21363..000000000--- a/openviking/telemetry/request_wait_tracker.py+++ /dev/null@@ -1,344 +0,0 @@-# Copyright (c) 2026 Beijing Volcano Engine Technology Co., Ltd.-# SPDX-License-Identifier: AGPL-3.0-"""Request-scoped wait tracker for write APIs."""--from __future__ import annotations--import asyncio-import threading-import time-from dataclasses import dataclass, field-from typing import Callable, Dict, List, Optional, Set---@dataclass-class _RequestWaitState:- pending_semantic_roots: Set[str] = field(default_factory=set)- pending_embedding_roots: Set[str] = field(default_factory=set)- semantic_processed: int = 0- semantic_requeue_count: int = 0- semantic_error_count: int = 0- semantic_errors: List[str] = field(default_factory=list)- embedding_processed: int = 0- embedding_context_count: int = 0- embedding_requeue_count: int = 0- embedding_error_count: int = 0- embedding_errors: List[str] = field(default_factory=list)- semantic_queue_wait_ms: float = 0.0- semantic_execute_ms: float = 0.0- embedding_queue_wait_ms: float = 0.0- embedding_execute_ms: float = 0.0- created_at: float = field(default_factory=time.time)- retained_roots: Set[str] = field(default_factory=set)- cleanup_requested: bool = False---class RequestWaitTracker:- """Track request-scoped queue completion using telemetry_id."""-- _instance: Optional["RequestWaitTracker"] = None-- def __new__(cls) -> "RequestWaitTracker":- if cls._instance is None:- cls._instance = super().__new__(cls)- return cls._instance-- def __init__(self) -> None:- if hasattr(self, "_lock"):- return- self._lock = threading.Lock()- self._states: Dict[str, _RequestWaitState] = {}-- @classmethod- def get_instance(cls) -> "RequestWaitTracker":- return cls()-- def _create_state(self, telemetry_id: str) -> Optional[_RequestWaitState]:- if not telemetry_id:- return None- with self._lock:- return self._states.setdefault(telemetry_id, _RequestWaitState())-- def register_request(self, telemetry_id: str) -> None:- self._create_state(telemetry_id)-- def has_request(self, telemetry_id: str) -> bool:- with self._lock:- return telemetry_id in self._states-- def retain_request(self, telemetry_id: str, root_id: str) -> None:- """Keep a Skill worker's accounting alive if its HTTP waiter times out."""- if not telemetry_id or not root_id:- return- with self._lock:- state = self._states.setdefault(telemetry_id, _RequestWaitState())- state.retained_roots.add(root_id)-- def _release_retained_root(- self, telemetry_id: str, root_id: str, state: _RequestWaitState- ) -> None:- state.retained_roots.discard(root_id)- if state.cleanup_requested and not state.retained_roots:- self._states.pop(telemetry_id, None)-- def register_semantic_root(self, telemetry_id: str, root_id: str) -> None:- if not telemetry_id or not root_id:- return- with self._lock:- state = self._states.get(telemetry_id)- if state is None:- return- state.pending_semantic_roots.add(root_id)-- def register_embedding_root(self, telemetry_id: str, root_id: str) -> None:- if not telemetry_id or not root_id:- return- with self._lock:- state = self._states.get(telemetry_id)- if state is None:- return- state.pending_embedding_roots.add(root_id)-- def record_embedding_requeue(self, telemetry_id: str, delta: int = 1) -> None:- if not telemetry_id:- return- with self._lock:- state = self._states.get(telemetry_id)- if state is None:- return- state.embedding_requeue_count += max(delta, 0)-- def record_semantic_timing(- self, telemetry_id: str, *, queue_wait_ms: float = 0.0, execute_ms: float = 0.0- ) -> None:- self._record_timing(- telemetry_id,- queue_wait_ms=queue_wait_ms,- execute_ms=execute_ms,- queue_wait_attr="semantic_queue_wait_ms",- execute_attr="semantic_execute_ms",- )-- def record_embedding_timing(- self, telemetry_id: str, *, queue_wait_ms: float = 0.0, execute_ms: float = 0.0- ) -> None:- self._record_timing(- telemetry_id,- queue_wait_ms=queue_wait_ms,- execute_ms=execute_ms,- queue_wait_attr="embedding_queue_wait_ms",- execute_attr="embedding_execute_ms",- )-- def _record_timing(- self,- telemetry_id: str,- *,- queue_wait_ms: float,- execute_ms: float,- queue_wait_attr: str,- execute_attr: str,- ) -> None:- if not telemetry_id:- return- with self._lock:- state = self._states.get(telemetry_id)- if state is None:- return- setattr(- state, queue_wait_attr, getattr(state, queue_wait_attr) + max(queue_wait_ms, 0.0)- )- setattr(state, execute_attr, getattr(state, execute_attr) + max(execute_ms, 0.0))-- def get_embedding_context_count(self, telemetry_id: str) -> int:- """Return contexts successfully indexed for one request."""- if not telemetry_id:- return 0- with self._lock:- state = self._states.get(telemetry_id)- return state.embedding_context_count if state is not None else 0-- def get_queue_timing(self, telemetry_id: str) -> Dict[str, Dict[str, float]]:- if not telemetry_id:- return {- "semantic": {"queue_wait_ms": 0.0, "execute_ms": 0.0},- "embedding": {"queue_wait_ms": 0.0, "execute_ms": 0.0},- }- with self._lock:- state = self._states.get(telemetry_id)- if state is None:- return {- "semantic": {"queue_wait_ms": 0.0, "execute_ms": 0.0},- "embedding": {"queue_wait_ms": 0.0, "execute_ms": 0.0},- }- return {- "semantic": {- "queue_wait_ms": state.semantic_queue_wait_ms,- "execute_ms": state.semantic_execute_ms,- },- "embedding": {- "queue_wait_ms": state.embedding_queue_wait_ms,- "execute_ms": state.embedding_execute_ms,- },- }-- def mark_semantic_done(- self,- telemetry_id: str,- root_id: str,- processed_delta: int = 1,- ) -> None:- if not telemetry_id:- return- with self._lock:- state = self._states.get(telemetry_id)- if state is None:- return- state.pending_semantic_roots.discard(root_id)- state.semantic_processed += max(processed_delta, 0)- self._release_retained_root(telemetry_id, root_id, state)-- def record_semantic_requeue(self, telemetry_id: str, delta: int = 1) -> None:- if not telemetry_id:- return- with self._lock:- state = self._states.get(telemetry_id)- if state is None:- return- state.semantic_requeue_count += max(delta, 0)-- def mark_semantic_failed(self, telemetry_id: str, root_id: str, message: str) -> None:- if not telemetry_id:- return- with self._lock:- state = self._states.get(telemetry_id)- if state is None:- return- state.pending_semantic_roots.discard(root_id)- state.semantic_error_count += 1- if message:- state.semantic_errors.append(message)- self._release_retained_root(telemetry_id, root_id, state)-- def mark_embedding_done(- self,- telemetry_id: str,- root_id: str,- processed_delta: int = 1,- *,- vector_written: bool = False,- ) -> None:- if not telemetry_id:- return- with self._lock:- state = self._states.get(telemetry_id)- if state is None:- return- was_pending = root_id in state.pending_embedding_roots- state.pending_embedding_roots.discard(root_id)- state.embedding_processed += max(processed_delta, 0)- if vector_written and was_pending:- state.embedding_context_count += 1-- def mark_embedding_failed(self, telemetry_id: str, root_id: str, message: str) -> None:- if not telemetry_id:- return- with self._lock:- state = self._states.get(telemetry_id)- if state is None:- return- state.pending_embedding_roots.discard(root_id)- state.embedding_error_count += 1- if message:- state.embedding_errors.append(message)-- def is_complete(self, telemetry_id: str) -> bool:- if not telemetry_id:- return True- with self._lock:- state = self._states.get(telemetry_id)- if state is None:- return True- return not state.pending_semantic_roots and not state.pending_embedding_roots-- def is_complete_except(self, telemetry_id: str, semantic_root_id: str) -> bool:- """Return whether all work except one coordinating semantic root has settled."""- if not telemetry_id:- return True- with self._lock:- state = self._states.get(telemetry_id)- if state is None:- return True- pending_semantic = state.pending_semantic_roots - {semantic_root_id}- return not pending_semantic and not state.pending_embedding_roots-- async def wait_for_request(- self,- telemetry_id: str,- timeout: Optional[float] = None,- ) -> None:- if not telemetry_id:- return- start = time.time()- while True:- if self.is_complete(telemetry_id):- return- if timeout is not None and (time.time() - start) > timeout:- raise TimeoutError(f"Request processing not complete after {timeout}s")- await asyncio.sleep(0.05)-- async def wait_for_embeddings(- self,- telemetry_id: str,- *,- stop_waiting: Optional[Callable[[], bool]] = None,- ) -> None:- """Drain embeddings while the producing semantic root remains pending."""- if not telemetry_id:- return- while True:- with self._lock:- state = self._states.get(telemetry_id)- if state is None or not state.pending_embedding_roots:- return- if stop_waiting is not None and stop_waiting():- # Shutdown has drained the embedding consumer's active writes.- # Leave this semantic delivery unacked for recovery rather than- # waiting forever for embeddings still in the persistent queue.- raise asyncio.CancelledError("Embedding worker stopped with queued work")- await asyncio.sleep(0.05)-- def build_queue_status(self, telemetry_id: str) -> Dict[str, Dict[str, object]]:- with self._lock:- state = self._states.get(telemetry_id) or _RequestWaitState()- return {- "Semantic": {- "processed": state.semantic_processed,- "requeue_count": state.semantic_requeue_count,- "error_count": state.semantic_error_count,- "errors": [{"message": msg} for msg in state.semantic_errors],- },- "Embedding": {- "processed": state.embedding_processed,- "requeue_count": state.embedding_requeue_count,- "error_count": state.embedding_error_count,- "errors": [{"message": msg} for msg in state.embedding_errors],- },- }-- def cleanup(self, telemetry_id: str) -> None:- if not telemetry_id:- return- with self._lock:- state = self._states.get(telemetry_id)- if state is not None and state.retained_roots:- state.cleanup_requested = True- else:- self._states.pop(telemetry_id, None)---def get_request_wait_tracker() -> RequestWaitTracker:- return RequestWaitTracker.get_instance()---__all__ = ["RequestWaitTracker", "get_request_wait_tracker"]diff --git a/openviking/telemetry/resource_summary.py b/openviking/telemetry/resource_summary.pyindex 4928d0c4b..ae21f24b0 100644--- a/openviking/telemetry/resource_summary.py+++ b/openviking/telemetry/resource_summary.py@@ -7,7 +7,6 @@ from __future__ import annotationsfrom typing import Any, Dictfrom .operation import OperationTelemetry-from .request_wait_tracker import get_request_wait_trackerdef _consume_semantic_request_stats(telemetry_id: str):@@ -52,6 +51,7 @@ def record_resource_queue_metrics(telemetry: OperationTelemetry,telemetry_id: str,root_uri: str | None,+ queue_timing: Dict[str, Dict[str, float]] | None = None,) -> None:"""Apply queue and semantic-tree metrics to a resource operation collector."""if not telemetry.enabled:@@ -59,18 +59,20 @@ def record_resource_queue_metrics(semantic = _queue_metrics(_consume_semantic_request_stats(telemetry_id))embedding = _queue_metrics(_consume_embedding_request_stats(telemetry_id))- timing = get_request_wait_tracker().get_queue_timing(telemetry_id)+ timing = queue_timing or {}+ semantic_timing = timing.get("semantic", {"queue_wait_ms": 0.0, "execute_ms": 0.0})+ embedding_timing = timing.get("embedding", {"queue_wait_ms": 0.0, "execute_ms": 0.0})telemetry.set("queue.semantic.processed", semantic["processed"])telemetry.set("queue.semantic.requeue_count", semantic["requeue_count"])telemetry.set("queue.semantic.error_count", semantic["error_count"])- telemetry.set("queue.semantic.queue_wait.duration_ms", timing["semantic"]["queue_wait_ms"])- telemetry.set("queue.semantic.execute.duration_ms", timing["semantic"]["execute_ms"])+ telemetry.set("queue.semantic.queue_wait.duration_ms", semantic_timing["queue_wait_ms"])+ telemetry.set("queue.semantic.execute.duration_ms", semantic_timing["execute_ms"])telemetry.set("queue.embedding.processed", embedding["processed"])telemetry.set("queue.embedding.requeue_count", embedding["requeue_count"])telemetry.set("queue.embedding.error_count", embedding["error_count"])- telemetry.set("queue.embedding.queue_wait.duration_ms", timing["embedding"]["queue_wait_ms"])- telemetry.set("queue.embedding.execute.duration_ms", timing["embedding"]["execute_ms"])+ telemetry.set("queue.embedding.queue_wait.duration_ms", embedding_timing["queue_wait_ms"])+ telemetry.set("queue.embedding.execute.duration_ms", embedding_timing["execute_ms"])tree_stats = _consume_semantic_tree_stats(telemetry_id, root_uri)if tree_stats is not None:diff --git a/openviking/utils/embedding_utils.py b/openviking/utils/embedding_utils.pyindex cb89b045b..0e062d10d 100644--- a/openviking/utils/embedding_utils.py+++ b/openviking/utils/embedding_utils.py@@ -32,7 +32,6 @@ from openviking.storage.queuefs import get_queue_managerfrom openviking.storage.queuefs.embedding_msg_converter import EmbeddingMsgConverterfrom openviking.storage.resource_rnfv import NON_PORTABLE_VECTOR_RECORD_FIELDSfrom openviking.storage.viking_fs import LS_ALL_NODES, get_viking_fs-from openviking.telemetry.request_wait_tracker import get_request_wait_trackerfrom openviking.utils.embedding_input import truncate_embedding_inputfrom openviking.utils.image_search import image_bytes_to_model_data_urifrom openviking.utils.ingest_options import IngestOptions@@ -135,29 +134,12 @@ async def _enqueue_embedding_message(embedding_queue,embedding_msg,*,- failure_message: str,+ failure_message: str = "Failed to enqueue embedding message",) -> bool:- """Persist one embedding message and settle request tracking on enqueue failure."""- wait_tracker = get_request_wait_tracker()- wait_tracker.register_embedding_root(embedding_msg.telemetry_id, embedding_msg.id)-- try:- enqueue_id = await embedding_queue.enqueue(embedding_msg)- except BaseException as exc:- wait_tracker.mark_embedding_failed(- embedding_msg.telemetry_id,- embedding_msg.id,- f"{failure_message}: {exc}",- )- raise-+ """Persist one embedding message."""+ enqueue_id = await embedding_queue.enqueue(embedding_msg)if not enqueue_id:- wait_tracker.mark_embedding_failed(- embedding_msg.telemetry_id,- embedding_msg.id,- failure_message,- )- return False+ raise RuntimeError(failure_message)return True@@ -492,7 +474,6 @@ async def vectorize_directory_meta(enqueued = await _enqueue_embedding_message(embedding_queue,msg_abstract,- failure_message=f"Failed to enqueue directory L0 vector for {uri}",)if enqueued:enqueued_levels.add(int(ContextLevel.ABSTRACT.value))@@ -554,7 +535,6 @@ async def vectorize_directory_meta(enqueued = await _enqueue_embedding_message(embedding_queue,msg_overview,- failure_message=f"Failed to enqueue directory L1 vector for {uri}",)if enqueued:enqueued_levels.add(int(ContextLevel.OVERVIEW.value))@@ -764,7 +744,6 @@ async def vectorize_file(enqueued = await _enqueue_embedding_message(embedding_queue,embedding_msg,- failure_message=f"Failed to enqueue file vector for {file_path}",)if not enqueued:return Falsediff --git a/openviking/utils/resource_processor.py b/openviking/utils/resource_processor.pyindex 2b3512d4b..d93d89ebe 100644--- a/openviking/utils/resource_processor.py+++ b/openviking/utils/resource_processor.py@@ -1429,7 +1429,6 @@ class ResourceProcessor:await _enqueue_embedding_message(embedding_queue,message,- failure_message="Failed to enqueue planned vector deletes",)completed_actions += len(delete_ids)directory_actions: dict[str, dict[int, Any]] = {}@@ -1480,7 +1479,6 @@ class ResourceProcessor:if await _enqueue_embedding_message(embedding_queue,message,- failure_message=f"Failed to enqueue scalar update for {action.uri}",):completed_actions += 1for uri, levels in directory_actions.items():diff --git a/openviking/utils/skill_processor.py b/openviking/utils/skill_processor.pyindex d7bd4eb03..2b7b31f2e 100644--- a/openviking/utils/skill_processor.py+++ b/openviking/utils/skill_processor.py@@ -31,7 +31,6 @@ from openviking.service.task_tracker_concurrency import run_to_completionfrom openviking.storage.viking_fs import VikingFSfrom openviking.storage.vikingdb_manager import VikingDBManagerfrom openviking.telemetry import get_current_telemetry, register_telemetry-from openviking.telemetry.request_wait_tracker import get_request_wait_trackerfrom openviking.utils.path_safety import safe_join_viking_urifrom openviking.utils.zip_safe import safe_extract_zipfrom openviking_cli.exceptions import InvalidArgumentError@@ -491,9 +490,7 @@ class SkillProcessor:"content": content,}if self._vlm_resolver is None:- raise RuntimeError(- "SkillProcessor requires a VLM resolver for account-owned work"- )+ raise RuntimeError("SkillProcessor requires a VLM resolver for account-owned work")extraction_kwargs["vlm"] = await self._vlm_resolver.get_vlm(ctx.account_id)extraction_result = await extract_skill_privacy_values(**extraction_kwargs)if not extraction_result.values:@@ -657,8 +654,6 @@ class SkillProcessor:telemetry = get_current_telemetry()register_telemetry(telemetry)- tracker = get_request_wait_tracker()- tracker.register_request(telemetry.telemetry_id)msg = SemanticMsg(uri=uri,context_type="skill",@@ -673,8 +668,6 @@ class SkillProcessor:propagate_to_parent=False,source={"path": source_path},)- tracker.register_semantic_root(msg.telemetry_id, msg.id)- tracker.retain_request(msg.telemetry_id, msg.id)handed_off = Falsetry:queue_manager = get_queue_manager()@@ -683,8 +676,7 @@ class SkillProcessor:await viking_fs._async_agfs.pathlock_handoff(lease)handed_off = Trueawait queue_manager.get_queue(queue_manager.SEMANTIC, allow_create=True).enqueue(msg)- except BaseException as exc:- tracker.mark_semantic_failed(msg.telemetry_id, msg.id, str(exc))+ except BaseException:if handed_off:# Enqueue was rejected/failed. Return ownership to the caller's# finally block; adopting creates a new owned lease reference.diff --git a/openviking/utils/summarizer.py b/openviking/utils/summarizer.pyindex 809b35f23..ddea97a24 100644--- a/openviking/utils/summarizer.py+++ b/openviking/utils/summarizer.py@@ -14,7 +14,6 @@ from openviking.storage.queuefs import SemanticMsg, get_queue_managerfrom openviking.storage.queuefs.semantic_msg import build_semantic_coalesce_keyfrom openviking.storage.viking_fs import LS_ALL_NODES, get_viking_fsfrom openviking.telemetry import get_current_telemetry-from openviking.telemetry.request_wait_tracker import get_request_wait_trackerfrom openviking.utils.ingest_options import IngestOptionsfrom openviking_cli.utils import get_loggerfrom openviking_cli.utils.uri import VikingURI@@ -217,25 +216,9 @@ class Summarizer:if change.ingest_options != IngestOptions()},)- if telemetry_id:- get_request_wait_tracker().register_semantic_root(telemetry_id, msg.id)- try:- enqueue_id = await semantic_queue.enqueue(msg)- except Exception as exc:- if telemetry_id:- get_request_wait_tracker().mark_semantic_failed(- telemetry_id,- msg.id,- str(exc),- )- raise+ enqueue_id = await semantic_queue.enqueue(msg)if enqueue_id == "deduplicated":- if telemetry_id:- get_request_wait_tracker().mark_semantic_done(- telemetry_id,- msg.id,- processed_delta=0,- )+ continueelse:enqueued_count += 1return {@@ -350,23 +333,8 @@ class Summarizer:file_abstracts=file_abstracts,plan=semantic_plan,)- if msg.telemetry_id:- get_request_wait_tracker().register_semantic_root(msg.telemetry_id, msg.id)- try:- enqueue_id = await semantic_queue.enqueue(msg)- except Exception as e:- if msg.telemetry_id:- get_request_wait_tracker().mark_semantic_failed(- msg.telemetry_id, msg.id, str(e)- )- raise+ enqueue_id = await semantic_queue.enqueue(msg)if enqueue_id == "deduplicated":- if msg.telemetry_id:- get_request_wait_tracker().mark_semantic_done(- msg.telemetry_id,- msg.id,- processed_delta=0,- )logger.info("Semantic generation already queued for: %s", target_uri)continueenqueued_count += 1diff --git a/tests/agfs/test_viking_fs_git.py b/tests/agfs/test_viking_fs_git.pyindex 412f4d777..94ed4253b 100644--- a/tests/agfs/test_viking_fs_git.py+++ b/tests/agfs/test_viking_fs_git.py@@ -608,6 +608,10 @@ class _SpyExecutor:self.calls.append(("reindex_file", uri))return {"ok": True}+ async def execute_in_current_task(self, *, uri, mode, ctx):+ self.calls.append(("reindex_file", uri))+ return {"ok": True}+async def reindex_directory_marker(self, *, dir_uri, level, ctx):self.calls.append(("reindex_marker", dir_uri, int(level)))@@ -631,6 +635,28 @@ def _patch_reindex_executor(monkeypatch, vfs, spy):monkeypatch.setattr(reindex_mod, "ReindexExecutor", factory)+class _MemTaskStore:+ def __init__(self, *, fail_creates: int = 0):+ self.fail_creates = fail_creates++ async def create(self, task):+ if self.fail_creates:+ self.fail_creates -= 1+ raise OSError("task store temporarily unavailable")++ async def update(self, task):+ return None++ async def get(self, task_id, *, account_id=None, user_id=None):+ return None++ async def list(self, account_id, *, user_id=None):+ return []++ async def delete(self, task_id, *, account_id, user_id=None):+ return None++@pytest.mark.asyncioasync def test_restore_schedules_reindex_for_derived_only_change(vfs, monkeypatch):"""When a restore only changes a directory `.abstract.md` (source file@@ -826,32 +852,27 @@ async def test_restore_relations_json_has_no_vector_side_effect(vfs, monkeypatch@pytest.mark.asyncioasync def test_restore_returns_pollable_task_id(vfs, monkeypatch):"""An applied restore with vector side-effects returns a ``task_id`` that- can be polled via the TaskTracker and reaches ``completed``.+ can be polled without leaving an orphaned admin reindex task."""- spy = _SpyExecutor()-- _patch_reindex_executor(monkeypatch, vfs, spy)-+ from openviking.service.reindex_executor import ReindexExecutorfrom openviking.service.task_tracker import (TaskTracker,+ get_task_tracker,set_task_tracker,)- class _MemTaskStore:- async def create(self, task):- return None+ reindexed = []- async def update(self, task):- return None+ async def run_reindex(_executor, **kwargs):+ reindexed.append(kwargs["uri"])+ return {+ "status": "completed",+ "uri": kwargs["uri"],+ "object_type": kwargs["object_type"],+ "mode": kwargs["mode"],+ }- async def get(self, task_id, *, account_id=None, user_id=None):- return None-- async def list(self, account_id, *, user_id=None):- return []-- async def delete(self, task_id, *, account_id, user_id=None):- return None+ monkeypatch.setattr(ReindexExecutor, "_run", run_reindex)set_task_tracker(TaskTracker(store=_MemTaskStore()))try:@@ -870,8 +891,6 @@ async def test_restore_returns_pollable_task_id(vfs, monkeypatch):task_id = result.get("task_id")assert task_id- from openviking.service.task_tracker import get_task_tracker-tracker = get_task_tracker()task = await tracker.wait(task_id,@@ -881,7 +900,81 @@ async def test_restore_returns_pollable_task_id(vfs, monkeypatch):)assert task.task_type == "snapshot_restore_reindex"assert task.status.value == "completed"- assert ("reindex_file", "viking://resources/proj/x.md") in spy.calls+ assert reindexed == ["viking://resources/proj/x.md"]++ tasks = await tracker.list_tasks(+ account_id=ctx.account_id,+ user_id=ctx.user.user_id,+ )+ assert [item.task_type for item in tasks] == ["snapshot_restore_reindex"]++ follow_up = await ReindexExecutor(+ vlm_resolver=vfs._vlm_resolver,+ vector_config_resolver=vfs._vector_config_resolver,+ ).execute(+ uri="viking://resources/proj/x.md",+ mode="vectors_only",+ wait=True,+ ctx=ctx,+ )+ assert follow_up["status"] == "completed"+ finally:+ set_task_tracker(None)+++async def test_restore_task_registration_failure_falls_back_to_file_reindex(+ vfs,+ monkeypatch,+):+ from openviking.service.reindex_executor import ReindexExecutor+ from openviking.service.task_tracker import TaskStatus, TaskTracker, set_task_tracker++ ctx = _make_ctx(account="acct_task_fallback")+ restored_contents = []+ rebuilt = asyncio.Event()++ async def run_reindex(_executor, **kwargs):+ restored_contents.append(await vfs.read(kwargs["uri"], ctx=kwargs["ctx"]))+ rebuilt.set()+ return {+ "status": "completed",+ "uri": kwargs["uri"],+ "object_type": kwargs["object_type"],+ "mode": kwargs["mode"],+ }++ monkeypatch.setattr(ReindexExecutor, "_run", run_reindex)+ tracker = TaskTracker(store=_MemTaskStore(fail_creates=1))+ set_task_tracker(tracker)+ try:+ uri = "viking://resources/proj/a.md"+ await vfs.write_file(uri, b"first version", ctx=ctx)+ first = await vfs.commit(message="v1", ctx=ctx)+ await vfs.write_file(uri, b"second version", ctx=ctx)+ await vfs.commit(message="v2", ctx=ctx)++ result = await vfs.restore(+ project_dir="viking://resources/proj",+ source_commit=first["commit_oid"],+ ctx=ctx,+ )++ assert result["result"] == "applied"+ assert "task_id" not in result+ await asyncio.wait_for(rebuilt.wait(), 1)+ assert restored_contents == [b"first version"]++ tasks = await tracker.list_tasks(account_id=ctx.account_id, user_id=ctx.user.user_id)+ assert len(tasks) == 1+ assert tasks[0].task_type == "admin_reindex"+ task = await tracker.wait(+ tasks[0].task_id,+ account_id=ctx.account_id,+ user_id=ctx.user.user_id,+ timeout=1,+ )+ assert task.status is TaskStatus.COMPLETEDfinally:set_task_tracker(None)diff --git a/tests/parse/test_feishu_parser_api.py b/tests/parse/test_feishu_parser_api.pyindex e6029179b..29edc074f 100644--- a/tests/parse/test_feishu_parser_api.py+++ b/tests/parse/test_feishu_parser_api.py@@ -22,7 +22,6 @@ from openviking.storage.queuefs.add_resource_msg import AddResourceMsg, AddResoufrom openviking.storage.queuefs.add_resource_processor import AddResourceProcessorfrom openviking.storage.queuefs.queue_manager import QueueManagerfrom openviking.telemetry import get_current_telemetry, resolve_telemetry-from openviking.telemetry.request_wait_tracker import get_request_wait_trackerfrom openviking.utils.media_processor import UnifiedResourceProcessorfrom openviking_cli.exceptions import InvalidArgumentErrorfrom openviking_cli.session.user_id import UserIdentifier@@ -190,7 +189,9 @@ async def test_feishu_parser_api_bypasses_accessor():async def test_feishu_parser_api_uses_app_credentials_for_tenant_token(monkeypatch):oauth_client = SimpleNamespace(get_tenant_access_token=AsyncMock(return_value="t-test"))from_config = Mock(return_value=oauth_client)- monkeypatch.setattr("openviking.resource.feishu_watch_auth.FeishuOAuthClient.from_config", from_config)+ monkeypatch.setattr(+ "openviking.resource.feishu_watch_auth.FeishuOAuthClient.from_config", from_config+ )api = _understanding_api_for_parse()feishu_config = FeishuConfig(app_id="app", app_secret="secret")@@ -1083,7 +1084,25 @@ async def test_add_resource_processor_persists_final_uri_and_cleans_staged_sourccomplete=AsyncMock(),fail=AsyncMock(),get_task_auth=AsyncMock(return_value={}),- wait_for_descendants=AsyncMock(),+ wait_for_task_work=AsyncMock(),+ build_queue_status=Mock(+ return_value={+ "Semantic": {+ "processed": 0,+ "requeue_count": 0,+ "error_count": 0,+ "errors": [],+ },+ "Embedding": {+ "processed": 9,+ "requeue_count": 0,+ "error_count": 0,+ "errors": [],+ },+ }+ ),+ get_context_count=Mock(return_value=9),+ get_queue_timing=Mock(return_value={}),)service = SimpleNamespace(@@ -1131,12 +1150,6 @@ async def test_add_resource_processor_persists_final_uri_and_cleans_staged_sourc},)- wait_tracker = get_request_wait_tracker()- wait_tracker.register_request(telemetry_id)- for index in range(9):- embedding_id = f"embedding-{index}"- wait_tracker.register_embedding_root(telemetry_id, embedding_id)- wait_tracker.mark_embedding_done(telemetry_id, embedding_id, vector_written=True)data = msg.to_dict()data[TASK_WORK_ID_FIELD] = "work-1"await processor._process(msg, data)@@ -1202,9 +1215,11 @@ async def test_add_resource_processor_persists_final_uri_and_cleans_staged_sourc@pytest.mark.asyncio@pytest.mark.parametrize("telemetry_id", [None, "telemetry-unregistered-resource"])[email protected]("context_count", [0, 1])async def test_add_resource_processor_collects_stats_without_registered_telemetry(monkeypatch,telemetry_id: str | None,+ context_count: int,):final_uri = "viking://resources/unregistered"observed_telemetry_ids = []@@ -1215,7 +1230,10 @@ async def test_add_resource_processor_collects_stats_without_registered_telemetrcomplete=AsyncMock(),fail=AsyncMock(),get_task_auth=AsyncMock(return_value={}),- wait_for_descendants=AsyncMock(),+ wait_for_task_work=AsyncMock(),+ build_queue_status=Mock(return_value={}),+ get_context_count=Mock(return_value=context_count),+ get_queue_timing=Mock(return_value={}),)monkeypatch.setattr("openviking.storage.queuefs.add_resource_processor.get_task_tracker",@@ -1233,13 +1251,6 @@ async def test_add_resource_processor_collects_stats_without_registered_telemetrtelemetry.record_token_usage("embedding", 21)telemetry.add_token_usage(13, 5)- wait_tracker = get_request_wait_tracker()- wait_tracker.register_embedding_root(effective_telemetry_id, "embedding-1")- wait_tracker.mark_embedding_done(- effective_telemetry_id,- "embedding-1",- vector_written=True,- )return {"status": "success", "root_uri": final_uri}processor = AddResourceProcessor(@@ -1266,7 +1277,8 @@ async def test_add_resource_processor_collects_stats_without_registered_telemetrtask_tracker.fail.assert_not_awaited()final_result = task_tracker.complete.await_args_list[-1].args[1]- assert final_result["context_count"] == 1+ assert final_result["root_uri"] == final_uri+ assert final_result["context_count"] == context_countassert final_result["usage"]["tokens"]["embedding"]["total"] == 21assert final_result["usage"]["tokens"]["llm"] == {"input": 13,@@ -1298,7 +1310,9 @@ async def test_add_resource_processor_replay_skips_lock_adopt_when_result_existsupdate_stage=AsyncMock(),complete=AsyncMock(),fail=AsyncMock(),- wait_for_descendants=AsyncMock(),+ wait_for_task_work=AsyncMock(),+ build_queue_status=Mock(return_value={}),+ get_queue_timing=Mock(return_value={}),)monkeypatch.setattr("openviking.storage.queuefs.add_resource_processor.get_task_tracker",@@ -1331,59 +1345,12 @@ async def test_add_resource_processor_replay_skips_lock_adopt_when_result_existsasync_agfs.pathlock_adopt.assert_not_awaited()service.execute_add_resource_job.assert_not_awaited()task_tracker.start.assert_not_awaited()- task_tracker.wait_for_descendants.assert_awaited_once_with("task-1", "work-1")task_tracker.complete.assert_awaited_once()completed_result = task_tracker.complete.await_args.args[1]assert completed_result["root_uri"] == final_uriassert completed_result["context_count"] == 4-async def test_add_resource_processor_reports_zero_vectors(monkeypatch):- final_uri = "viking://resources/no-vectors"- service = SimpleNamespace(- execute_add_resource_job=AsyncMock(- return_value={"status": "success", "root_uri": final_uri}- ),- _link_resource_reason_memory=AsyncMock(),- )- task_tracker = SimpleNamespace(- create=AsyncMock(return_value=SimpleNamespace(status=TaskStatus.PENDING)),- start=AsyncMock(),- update_stage=AsyncMock(),- complete=AsyncMock(),- fail=AsyncMock(),- get_task_auth=AsyncMock(return_value={}),- wait_for_descendants=AsyncMock(),- )- monkeypatch.setattr(- "openviking.storage.queuefs.add_resource_processor.get_task_tracker",- Mock(return_value=task_tracker),- )- processor = AddResourceProcessor(- service,- QueueManager.ADD_RESOURCE,- SimpleNamespace(_async_agfs=SimpleNamespace(pathlock_release=AsyncMock())),- )- msg = AddResourceMsg(- task_id="task-1",- path="/tmp/demo.md",- root_uri=final_uri,- account_id="account-1",- user_id="user-1",- role="user",- )- data = msg.to_dict()- data[TASK_WORK_ID_FIELD] = "work-1"-- await processor._process(msg, data)-- task_tracker.fail.assert_not_awaited()- final_result = task_tracker.complete.await_args.args[1]- assert final_result["root_uri"] == final_uri- assert final_result["context_count"] == 0--def test_feishu_direct_submission_requires_configured_auth(monkeypatch):api = _understanding_api_for_parse()source = "https://example.larkoffice.com/docx/doxcnToken"diff --git a/tests/server/conftest.py b/tests/server/conftest.pyindex 18a22268d..2871e184a 100644--- a/tests/server/conftest.py+++ b/tests/server/conftest.py@@ -96,7 +96,9 @@ def _install_session_commit_queue_fallback(service: OpenVikingService, monkeypatmonkeypatch.setattr(tracker,"has_work",- lambda task_id: task_id in queued_commit_tasks or original_has_work(task_id),+ lambda task_id, **kwargs: (+ task_id in queued_commit_tasks or original_has_work(task_id, **kwargs)+ ),)async def enqueue_with_session_commit_fallback(queue_name, data):diff --git a/tests/server/test_admin_rebuild_api.py b/tests/server/test_admin_rebuild_api.pyindex 57bece0d1..852c1fee9 100644--- a/tests/server/test_admin_rebuild_api.py+++ b/tests/server/test_admin_rebuild_api.py@@ -2572,51 +2572,10 @@ async def test_reindex_global_namespace_partitions_user_and_resources(monkeypatc@pytest.mark.asyncio-async def test_reindex_upsert_ignores_empty_replace_tags(monkeypatch):- from types import SimpleNamespace-- from openviking.service.reindex_executor import ReindexExecutor- from openviking.utils.ingest_options import IngestOptions-- queued = []-- class FakeVikingDB:- async def enqueue_embedding_msg(self, msg):- queued.append(msg)- return True-- monkeypatch.setattr(- "openviking.service.reindex_executor.get_service",- lambda: SimpleNamespace(vikingdb_manager=FakeVikingDB()),- )- monkeypatch.setattr(- "openviking.service.reindex_executor.get_request_wait_tracker",- lambda: SimpleNamespace(register_embedding_root=lambda *args: None),- )-- ctx = RequestContext(- user=UserIdentifier(account_id="test", user_id="alice"),- role=Role.ROOT,- )- await ReindexExecutor()._upsert_context(- uri="viking://resources/demo.md",- parent_uri="viking://resources",- abstract="demo",- vector_text="demo",- is_leaf=True,- context_type="resource",- level=ContextLevel.DETAIL,- ctx=ctx,- ingest_options=IngestOptions.from_search_tags([], mode="replace"),- )-- assert "search_tags" not in queued[0].context_data- assert "_upsert_options" not in queued[0].context_data---async def test_reindex_upsert_applies_clear_tags_to_embedding_message(monkeypatch):[email protected]("tag_mode", ["replace", "clear"])+async def test_reindex_upsert_empty_tags_preserves_replace_and_clear_semantics(+ monkeypatch, tag_mode+):from types import SimpleNamespacefrom openviking.service.reindex_executor import ReindexExecutor@@ -2633,10 +2592,6 @@ async def test_reindex_upsert_applies_clear_tags_to_embedding_message(monkeypatc"openviking.service.reindex_executor.get_service",lambda: SimpleNamespace(vikingdb_manager=FakeVikingDB()),)- monkeypatch.setattr(- "openviking.service.reindex_executor.get_request_wait_tracker",- lambda: SimpleNamespace(register_embedding_root=lambda *args: None),- )ctx = RequestContext(user=UserIdentifier(account_id="test", user_id="alice"),@@ -2651,11 +2606,17 @@ async def test_reindex_upsert_applies_clear_tags_to_embedding_message(monkeypatccontext_type="resource",level=ContextLevel.DETAIL,ctx=ctx,- ingest_options=IngestOptions.from_search_tags(None, mode="clear"),+ ingest_options=IngestOptions.from_search_tags(+ [] if tag_mode == "replace" else None, mode=tag_mode+ ),)- assert queued[0].context_data["search_tags"] == []- assert queued[0].context_data["_upsert_options"] == {"search_tag_mode": "replace"}+ if tag_mode == "replace":+ assert "search_tags" not in queued[0].context_data+ assert "_upsert_options" not in queued[0].context_data+ else:+ assert queued[0].context_data["search_tags"] == []+ assert queued[0].context_data["_upsert_options"] == {"search_tag_mode": "replace"}@pytest.mark.asynciodiff --git a/tests/server/test_api_snapshot.py b/tests/server/test_api_snapshot.pyindex 8468d6c7b..54645d737 100644--- a/tests/server/test_api_snapshot.py+++ b/tests/server/test_api_snapshot.py@@ -277,6 +277,10 @@ async def test_restore_acl_and_reindex_identity(client_with_resource, service, mreindex_contexts.append(ctx)return {"ok": True}+ async def execute_in_current_task(self, *, uri, mode, ctx):+ reindex_contexts.append(ctx)+ return {"ok": True}+async def reindex_directory_marker(self, *, dir_uri, level, ctx):reindex_contexts.append(ctx)@@ -649,6 +653,10 @@ async def test_restore_apply_triggers_reindex_hook(client_with_resource_and_blobcalls.append(uri)return {"ok": True}+ async def execute_in_current_task(self, *, uri, mode, ctx):+ calls.append(uri)+ return {"ok": True}+def factory(*, vlm_resolver, vector_config_resolver):assert vlm_resolver is service.viking_fs._vlm_resolverassert vector_config_resolver is service.vector_config_resolver@@ -696,6 +704,9 @@ async def test_restore_delete_removes_orphaned_vectors(client_with_resource_and_async def execute(self, *, uri, mode, wait, ctx):return {"ok": True}+ async def execute_in_current_task(self, *, uri, mode, ctx):+ return {"ok": True}+async def reindex_directory_marker(self, *, dir_uri, level, ctx):return Nonediff --git a/tests/server/test_content_batch_write.py b/tests/server/test_content_batch_write.pyindex d365f40ca..8f71a95f0 100644--- a/tests/server/test_content_batch_write.py+++ b/tests/server/test_content_batch_write.py@@ -7,6 +7,9 @@ import pytestimport openviking.storage.content_write as content_write_modulefrom openviking.server.identity import RequestContext, Role+from openviking.service.task_store import PersistentTaskStore+from openviking.service.task_tracker import TaskTracker+from openviking.service.task_work_index import TaskWorkIndex, bind_task_contextfrom openviking.session.memory.dataclass import MemoryFilefrom openviking.session.memory.utils import MemoryFileUtilsfrom openviking.storage.content_write import ContentWriteCoordinator@@ -15,6 +18,7 @@ from openviking.storage.context_update_plan import (ContextUpdatePlan,FileRefreshIntent,)+from openviking.storage.queuefs.queue_middleware import EnqueueReceiptfrom openviking.storage.queuefs.semantic_ops.freshness_policy import FreshnessActionfrom openviking.telemetry.context import bind_telemetryfrom openviking.telemetry.operation import OperationTelemetry@@ -25,6 +29,7 @@ from openviking_cli.exceptions import (OpenVikingError,)from openviking_cli.session.user_id import UserIdentifier+from tests.test_task_tracker import _FakeAgfsclass _PathLockClient:@@ -119,19 +124,22 @@ def _coordinator_queue_for_unit_tests(monkeypatch, request):yield Nonereturn- queue = SimpleNamespace(enqueue=AsyncMock(return_value="queued"))+ queue = SimpleNamespace(enqueue=AsyncMock(return_value=EnqueueReceipt("queued", "work-1")))queue_manager = SimpleNamespace(SEMANTIC="semantic",get_queue=lambda *args, **kwargs: queue,)monkeypatch.setattr(content_write_module, "get_queue_manager", lambda: queue_manager)- yield queue+ tracker = TaskTracker(PersistentTaskStore(_FakeAgfs()))+ tracker.attach_work_index(TaskWorkIndex())+ monkeypatch.setattr("openviking.service.task_tracker._instance", tracker)++ async def run_tracked_operation(_task_type, *, operation, **_kwargs):+ with bind_task_context("test-task", "default", "default", enqueued_work=[]):+ return await operation(), "test-task"- tracker = content_write_module.get_request_wait_tracker()- for call in queue.enqueue.await_args_list:- msg = call.args[0]- if msg.wait_for_request_completion:- tracker.mark_semantic_done(msg.telemetry_id, msg.id, processed_delta=0)+ monkeypatch.setattr(content_write_module, "run_tracked_operation", run_tracked_operation)+ yield queue@pytest.mark.asyncio@@ -677,7 +685,7 @@ async def test_batch_hands_off_file_locks_after_one_aggregated_refresh(monkeypatlocks = vfs._async_agfscoordinator = ContentWriteCoordinator(vfs)calls = []- queue = SimpleNamespace(enqueue=AsyncMock(return_value="queued"))+ queue = SimpleNamespace(enqueue=AsyncMock(return_value=EnqueueReceipt("queued", "work-1")))queue_manager = SimpleNamespace(SEMANTIC="semantic",get_queue=lambda *args, **kwargs: queue,@@ -709,11 +717,6 @@ async def test_batch_hands_off_file_locks_after_one_aggregated_refresh(monkeypatassert locks.handoffs == 1coordinator_msg = queue.enqueue.await_args.args[0]assert coordinator_msg.wait_for_request_completion is True- content_write_module.get_request_wait_tracker().mark_semantic_done(- telemetry.telemetry_id,- coordinator_msg.id,- processed_delta=0,- )@pytest.mark.asynciodiff --git a/tests/server/test_content_write_service.py b/tests/server/test_content_write_service.pyindex 76672ac86..4f11591a5 100644--- a/tests/server/test_content_write_service.py+++ b/tests/server/test_content_write_service.py@@ -8,6 +8,7 @@ from types import SimpleNamespaceimport pytestfrom openviking.server.identity import RequestContext, Role+from openviking.service.task_work_index import bind_task_contextfrom openviking.session.memory.dataclass import MemoryFilefrom openviking.session.memory.utils import MemoryFileUtilsfrom openviking.session.memory.utils.content_visibility import visible_content@@ -22,6 +23,21 @@ from openviking_cli.exceptions import (from openviking_cli.session.user_id import UserIdentifier[email protected](autouse=True)+def run_direct_coordinator_operations_inline(request, monkeypatch):+ if "service" in request.fixturenames:+ return++ async def run_tracked_operation(_task_type, *, operation, **_kwargs):+ with bind_task_context("test-task", "default", "test-user", enqueued_work=[]):+ return await operation(), "test-task"++ monkeypatch.setattr(+ "openviking.storage.content_write.run_tracked_operation",+ run_tracked_operation,+ )++@pytest.mark.asyncioasync def test_write_updates_memory_file_and_parent_overview(service):ctx = RequestContext(user=service.user, role=Role.USER)@@ -982,7 +998,11 @@ async def test_create_mode_valid_extensions_pass(monkeypatch):del uri, mode, ctx, lock_handle, lease_ref, is_new_filereturn content+ async def _fake_wait_for_queues(**_kwargs):+ return None+monkeypatch.setattr(coordinator, "_write_in_place", _fake_write_in_place)+ monkeypatch.setattr(coordinator, "_wait_for_request", _fake_wait_for_queues)result = await coordinator.write(uri=file_uri, content="content", mode="create", ctx=ctx, wait=Truediff --git a/tests/server/test_filesystem_router.py b/tests/server/test_filesystem_router.pyindex 9e9f8d461..1e7ae0940 100644--- a/tests/server/test_filesystem_router.py+++ b/tests/server/test_filesystem_router.py@@ -22,7 +22,10 @@ async def test_rm_preserves_memory_cleanup(monkeypatch):cleanup = {"status": "success", "memory_uris": ["viking://user/alice/memories/entities/a.md"]}async def fake_rm(uri, ctx=None, recursive=False, wait=False):- return {"estimated_deleted_count": 1, "memory_cleanup": cleanup}+ return {+ "estimated_deleted_count": 1,+ "memory_cleanup": cleanup,+ }monkeypatch.setattr(filesystem,diff --git a/tests/server/test_request_wait_tracking.py b/tests/server/test_request_wait_tracking.pydeleted file mode 100644index 0a87444a2..000000000--- a/tests/server/test_request_wait_tracking.py+++ /dev/null@@ -1,58 +0,0 @@-# Copyright (c) 2026 Beijing Volcano Engine Technology Co., Ltd.-# SPDX-License-Identifier: AGPL-3.0--"""Request-scoped waits must not depend on unrelated queue work."""--import asyncio-from types import SimpleNamespace--from openviking.server.identity import RequestContext, Role-from openviking.service.resource_service import ResourceService-from openviking.service.task_store import PersistentTaskStore-from openviking.service.task_tracker import TaskTracker-from openviking.telemetry.context import bind_telemetry-from openviking.telemetry.operation import OperationTelemetry-from openviking.telemetry.request_wait_tracker import RequestWaitTracker-from openviking_cli.session.user_id import UserIdentifier-from tests.test_task_tracker import _FakeAgfs---async def test_add_skill_wait_uses_request_tracker_when_telemetry_disabled(monkeypatch):- tracker = RequestWaitTracker()- telemetry = OperationTelemetry(operation="resources.add_skill", enabled=False)- tracker.register_request("unrelated")- tracker.register_embedding_root("unrelated", "pending-embedding")- ctx = RequestContext(user=UserIdentifier.the_default_user(), role=Role.ROOT)- root_uri = "viking://user/default/skills/demo"-- async def process_skill(**kwargs):- tracker.register_embedding_root(telemetry.telemetry_id, "skill-embedding")- tracker.mark_embedding_done(telemetry.telemetry_id, "skill-embedding")- return {"status": "success", "uri": root_uri, "name": "demo"}-- service = ResourceService(- resource_processor=object(),- skill_processor=SimpleNamespace(process_skill=process_skill),- viking_fs=object(),- )- monkeypatch.setattr(- "openviking.service.resource_service.get_request_wait_tracker", lambda: tracker- )- task_tracker = TaskTracker(store=PersistentTaskStore(_FakeAgfs()))- monkeypatch.setattr("openviking.service.task_tracker.get_task_tracker", lambda: task_tracker)-- with bind_telemetry(telemetry):- result = await asyncio.wait_for(- service.add_skill(- data={"name": "demo", "content": "# Demo"},- ctx=ctx,- wait=True,- target_uri="viking://user/default/skills",- ),- 1,- )-- assert result["root_uri"] == root_uri- assert result["queue_status"]["Embedding"]["processed"] == 1- assert not tracker.has_request(telemetry.telemetry_id)- assert not tracker.is_complete("unrelated")diff --git a/tests/service/test_fs_service.py b/tests/service/test_fs_service.pyindex 878246fc6..733682ff1 100644--- a/tests/service/test_fs_service.py+++ b/tests/service/test_fs_service.py@@ -12,6 +12,7 @@ import pytestfrom openviking.pyagfs.exceptions import AGFSAlreadyExistsErrorfrom openviking.server.identity import RequestContext, Rolefrom openviking.service.fs_service import FSService, ListingPage+from openviking.service.task_work_index import bind_task_contextfrom openviking.storage.abstract_overview import body_for_previewfrom openviking.storage.errors import LockAcquisitionErrorfrom openviking.storage.viking_fs import VikingFS@@ -19,6 +20,18 @@ from openviking_cli.exceptions import InvalidArgumentErrorfrom openviking_cli.session.user_id import UserIdentifier[email protected](autouse=True)+def run_fs_operations_inline(monkeypatch):+ async def run_tracked_operation(_task_type, *, operation, **_kwargs):+ with bind_task_context("test-task", "default", "test-user", enqueued_work=[]):+ return await operation(), "test-task"++ monkeypatch.setattr(+ "openviking.service.fs_service.run_tracked_operation",+ run_tracked_operation,+ )++class _FakeVikingFS:def __init__(self, *, rm_error=None, events=None, parent_exists=True):self.rm_calls = []@@ -248,41 +261,6 @@ class _FakeWatchScheduler:self.watch_manager = watch_manager-class _FakeWaitTracker:- def __init__(self):- self.registered_requests = []- self.registered_roots = []- self.wait_calls = []- self.cleaned = []-- def register_request(self, telemetry_id):- self.registered_requests.append(telemetry_id)-- def register_semantic_root(self, telemetry_id, root_id):- self.registered_roots.append(- {- "telemetry_id": telemetry_id,- "root_id": root_id,- "request_was_registered": telemetry_id in self.registered_requests,- }- )-- async def wait_for_request(self, telemetry_id):- self.wait_calls.append(telemetry_id)-- def build_queue_status(self, telemetry_id):- return {- "Semantic": {"processed": 1, "error_count": 0, "errors": []},- "Embedding": {"processed": 0, "error_count": 0, "errors": []},- }-- def mark_semantic_failed(self, telemetry_id, root_id, message):- pass-- def cleanup(self, telemetry_id):- self.cleaned.append(telemetry_id)--class _FakeQueueManager:SEMANTIC = "semantic"@@ -913,6 +891,34 @@ async def test_tree_tag_filter_keeps_zero_node_limit_unbounded(request_context):assert viking_fs.tree.await_args.kwargs["node_limit"] is None+async def test_resource_rm_enqueues_parent_delete_refresh_and_waits(request_context, monkeypatch):+ viking_fs = _FakeVikingFS()+ service = FSService(viking_fs=viking_fs)+ queue_manager = _FakeQueueManager()+ monkeypatch.setattr("openviking.service.fs_service.get_queue_manager", lambda: queue_manager)+ service._wait_for_refresh = AsyncMock(return_value={"Semantic": {"pending_count": 0}})++ uri = "viking://resources/images/2026/06/10/不二周助_jpeg"+ result = await service.rm(+ uri,+ ctx=request_context,+ recursive=True,+ wait=True,+ )++ assert viking_fs.rm_calls == [{"uri": uri, "recursive": True, "ctx": request_context}]+ assert len(queue_manager.messages) == 1+ msg = queue_manager.messages[0]+ assert msg.uri == "viking://resources/images/2026/06/10"+ assert msg.context_type == "resource"+ assert msg.recursive is False+ assert msg.changes == {"deleted": [uri]}+ assert result["semantic_root_uri"] == "viking://resources/images/2026/06/10"+ assert result["semantic_status"] == "complete"+ assert result["queue_status"] == {"Semantic": {"pending_count": 0}}++@pytest.mark.asyncioasync def test_resource_rm_reports_failed_semantic_status_when_wait_queue_has_errors(request_context, monkeypatch@@ -920,15 +926,15 @@ async def test_resource_rm_reports_failed_semantic_status_when_wait_queue_has_erviking_fs = _FakeVikingFS()service = FSService(viking_fs=viking_fs)service._enqueue_delete_refresh = AsyncMock()- tracker = _FakeWaitTracker()- tracker.build_queue_status = lambda _id: {- "Semantic": {- "processed": 1,- "error_count": 1,- "errors": [{"message": "refresh failed"}],+ service._wait_for_refresh = AsyncMock(+ return_value={+ "Semantic": {+ "processed": 1,+ "error_count": 1,+ "errors": [{"message": "refresh failed"}],+ }}- }- monkeypatch.setattr("openviking.service.fs_service.get_request_wait_tracker", lambda: tracker)+ )result = await service.rm("viking://resources/images/2026/06/10/不二周助_jpeg",@@ -1437,47 +1443,6 @@ async def test_transfer_refresh_message_records_deleted_source_entry(assert queue_manager.messages[0].changes == {"deleted": ["viking://resources/source/moved.md"]}-async def test_resource_rm_wait_registers_request_before_semantic_root(- request_context,- monkeypatch,-):- viking_fs = _FakeVikingFS()- service = FSService(viking_fs=viking_fs)- tracker = _FakeWaitTracker()- queue_manager = _FakeQueueManager()-- monkeypatch.setattr(- "openviking.service.fs_service.get_current_telemetry",- lambda: SimpleNamespace(telemetry_id="tm-fs-rm"),- )- monkeypatch.setattr(- "openviking.service.fs_service.get_request_wait_tracker",- lambda: tracker,- )- monkeypatch.setattr(- "openviking.service.fs_service.get_queue_manager",- lambda: queue_manager,- )-- result = await service.rm(- "viking://resources/images/2026/06/10/不二周助_jpeg",- ctx=request_context,- recursive=True,- wait=True,- )-- assert tracker.registered_requests == ["tm-fs-rm"]- assert tracker.registered_roots- assert tracker.registered_roots[0]["request_was_registered"] is True- assert queue_manager.messages[0].recursive is False- assert tracker.wait_calls == ["tm-fs-rm"]- assert tracker.cleaned == ["tm-fs-rm"]- assert result["semantic_root_uri"] == "viking://resources/images/2026/06/10"- assert result["semantic_status"] == "complete"- assert result["queue_status"]["Semantic"]["processed"] == 1--@pytest.mark.asyncioasync def test_resource_rm_does_not_cleanup_memory_if_resource_delete_fails(request_context):delete_error = RuntimeError("delete failed")diff --git a/tests/service/test_resource_service_watch.py b/tests/service/test_resource_service_watch.pyindex 74fbd90c0..f35a9813e 100644--- a/tests/service/test_resource_service_watch.py+++ b/tests/service/test_resource_service_watch.py@@ -390,9 +390,7 @@ class TestWatchTaskCreation:)def test_watch_persists_clear_without_tags(self, resource_service: ResourceService):- assert resource_service._watch_processor_kwargs({}, None, "clear") == {- "tag_mode": "clear"- }+ assert resource_service._watch_processor_kwargs({}, None, "clear") == {"tag_mode": "clear"}def test_add_resource_message_round_trip_preserves_clear_without_tags(self):message = AddResourceMsg(@@ -679,9 +677,7 @@ class TestAddResourceArgs:return resolver(SimpleNamespace(account=SimpleNamespace(feishu=None),- cluster=SimpleNamespace(- feishu=FeishuConfig(domain="https://open.feishu.cn")- ),+ cluster=SimpleNamespace(feishu=FeishuConfig(domain="https://open.feishu.cn")),))@@ -699,9 +695,7 @@ class TestAddResourceArgs:seen["config_domain"] = feishu_config.domainreturn SimpleNamespace(source_name=None, source_format="file")- resource_service._runtime_config_manager = SimpleNamespace(- resolve_account=resolve_account- )+ resource_service._runtime_config_manager = SimpleNamespace(resolve_account=resolve_account)monkeypatch.setattr("openviking.parse.accessors.feishu_accessor.FeishuAccessor.preflight_source",preflight,@@ -755,9 +749,7 @@ class TestAddResourceArgs:))- resource_service._runtime_config_manager = SimpleNamespace(- resolve_account=resolve_account- )+ resource_service._runtime_config_manager = SimpleNamespace(resolve_account=resolve_account)disable_task_tracker(monkeypatch)to_uri = "viking://resources/feishu-account-watch"resource_service._plan_source_job_target = AsyncMock(@@ -1033,7 +1025,11 @@ async def test_add_resource_processor_records_paused_watch_result(complete=AsyncMock(),fail=AsyncMock(),get_task_auth=AsyncMock(return_value={}),- wait_for_descendants=AsyncMock(),+ wait_for_task_work=AsyncMock(),+ build_queue_status=Mock(return_value={}),+ get_context_count=Mock(return_value=0),+ get_queue_timing=Mock(return_value={}),+ get=AsyncMock(return_value=SimpleNamespace(status=TaskStatus.CANCELLING)),)monkeypatch.setattr("openviking.storage.queuefs.add_resource_processor.get_task_tracker",diff --git a/tests/service/test_skill_processing_cancellation.py b/tests/service/test_skill_processing_cancellation.pyindex abcb3d430..eba7e505f 100644--- a/tests/service/test_skill_processing_cancellation.py+++ b/tests/service/test_skill_processing_cancellation.py@@ -11,8 +11,11 @@ from openviking.server.identity import RequestContext, Rolefrom openviking.service.resource_service import ResourceServicefrom openviking.service.task_store import PersistentTaskStorefrom openviking.service.task_tracker import TaskStatus, TaskTracker-from openviking.service.task_work_index import QueueTaskMetadata, TaskWorkIndex-from openviking.telemetry import OperationTelemetry, bind_telemetry+from openviking.service.task_work_index import QueueTaskMetadata, TaskWorkIndex, get_task_context+from openviking.telemetry import (+ OperationTelemetry,+ bind_telemetry,+)from openviking_cli.session.user_id import UserIdentifierfrom tests.test_task_tracker import _FakeAgfs@@ -100,3 +103,59 @@ async def test_cancel_during_skill_task_creation_settles_persisted_record(monkeyfinally:resume.set()await asyncio.gather(adding, return_exceptions=True)+++async def test_skill_success_remains_completed_when_queue_work_finishes_first(monkeypatch):+ tracker = TaskTracker(store=PersistentTaskStore(_FakeAgfs()))+ index = TaskWorkIndex()+ tracker.attach_work_index(index)+ monkeypatch.setattr("openviking.service.task_tracker.get_task_tracker", lambda: tracker)+ ctx = RequestContext(user=UserIdentifier.the_default_user(), role=Role.ROOT)+ owner = {"account_id": ctx.account_id, "user_id": ctx.user.user_id}++ async def process_skill(**kwargs):+ work = QueueTaskMetadata(get_task_context().task_id, "vector", **owner)+ assert index.register("Embedding", work)+ index.record_result(work, "completed")+ await index.prepare_ack("Embedding", work)+ return {"uri": "viking://agent/skills/demo"}++ service = ResourceService.__new__(ResourceService)+ service._resource_processor = object()+ service._viking_fs = object()+ service._skill_processor = SimpleNamespace(process_skill=process_skill)+ service._background_tasks = set()++ with bind_telemetry(OperationTelemetry("add-skill")):+ result = await service.add_skill("body", ctx, wait=True, target_uri="viking://agent/skills")++ task = await tracker.get(result["task_id"], **owner)+ assert task.status is TaskStatus.COMPLETED+ assert task.error is None+ assert task.result["root_uri"] == "viking://agent/skills/demo"+++async def test_skill_failure_preserves_business_error_without_queue_work(monkeypatch):+ tracker = TaskTracker(store=PersistentTaskStore(_FakeAgfs()))+ monkeypatch.setattr("openviking.service.task_tracker.get_task_tracker", lambda: tracker)+ ctx = RequestContext(user=UserIdentifier.the_default_user(), role=Role.ROOT)+ owner = {"account_id": ctx.account_id, "user_id": ctx.user.user_id}++ async def process_skill(**kwargs):+ raise RuntimeError("skill processing failed")++ service = ResourceService.__new__(ResourceService)+ service._resource_processor = object()+ service._viking_fs = object()+ service._skill_processor = SimpleNamespace(process_skill=process_skill)++ with bind_telemetry(OperationTelemetry("add-skill")):+ with pytest.raises(RuntimeError, match="skill processing failed"):+ await service.add_skill("body", ctx, wait=True, target_uri="viking://agent/skills")++ tasks = await tracker.list_tasks(**owner)+ assert len(tasks) == 1+ assert tasks[0].status is TaskStatus.FAILED+ assert tasks[0].error == "skill processing failed"diff --git a/tests/service/test_task_processing_time.py b/tests/service/test_task_processing_time.pyindex d3a161194..b7d138884 100644--- a/tests/service/test_task_processing_time.py+++ b/tests/service/test_task_processing_time.py@@ -16,7 +16,6 @@ from openviking.storage.queuefs.semantic_executor import (SemanticTreeExecutor,SemanticTreeScheduler,)-from openviking.telemetry.request_wait_tracker import RequestWaitTrackerdef test_work_timestamps_union_worker_occupancy_and_exclude_queue_time(monkeypatch):@@ -121,8 +120,7 @@ async def test_saturated_dag_scheduler_counts_claimed_handler_time(monkeypatch):@pytest.mark.asyncio[email protected]("method", ["wait_for_request", "wait_for_embeddings"])-async def test_downstream_wait_remains_worker_occupied_time(monkeypatch, method):+async def test_downstream_work_wait_remains_worker_occupied_time(monkeypatch):now = [0.0]monkeypatch.setattr("openviking.service.task_work_index.time",@@ -130,26 +128,25 @@ async def test_downstream_wait_remains_worker_occupied_time(monkeypatch, method))index = TaskWorkIndex()index.init_processing("task")- tracker = RequestWaitTracker()- request = "processing-time-" + method- tracker.register_request(request)- tracker.register_embedding_root(request, "embedding")+ metadata = QueueTaskMetadata("task", "embedding")+ assert index.register("Embedding", metadata)worker = asyncio.current_task()assert worker is not Noneindex.register_active("task", worker)try:now[0] = 3- waiter = asyncio.create_task(getattr(tracker, method)(request))+ waiter = asyncio.create_task(+ index.wait_for_work("task", ["embedding"], poll_interval=0.001)+ )await asyncio.sleep(0.01)now[0] = 20assert index.processing_seconds("task") == 20- tracker.mark_embedding_done(request, "embedding")+ index.record_result(metadata, "completed")await waiternow[0] = 22assert index.processing_seconds("task") == 22finally:index.unregister_active("task", worker)- tracker.cleanup(request)@pytest.mark.asynciodiff --git a/tests/service/test_task_queue_middleware.py b/tests/service/test_task_queue_middleware.pyindex 3d79bc89e..f6c54db28 100644--- a/tests/service/test_task_queue_middleware.py+++ b/tests/service/test_task_queue_middleware.py@@ -107,14 +107,38 @@ async def test_enqueue_registers_before_write_and_ack_finalizes_before_delete(trassert not index.has_work("task-1")[email protected]("failure", [OSError("write failed"), asyncio.CancelledError()])-async def test_enqueue_failure_discards_registered_work(tracked_queue, failure):+async def test_enqueue_failure_records_failed_work(tracked_queue):queue, index, transport, finalize, _ = tracked_queue- transport.write.side_effect = failure+ transport.write.side_effect = OSError("write failed")+ enqueued_work = []+ with bind_task_context(+ "task-1",+ "account",+ "user",+ enqueued_work=enqueued_work,+ ):+ with pytest.raises(OSError, match="write failed"):+ await queue.enqueue({})+ assert not index.has_work("task-1")+ payload = json.loads(transport.write.await_args.args[1])+ assert enqueued_work == [("Test", payload["_task_work_id"])]+ assert index.queue_status("task-1")["Test"] == {+ "processed": 0,+ "requeue_count": 0,+ "error_count": 1,+ "errors": [{"message": "write failed"}],+ }+ finalize.assert_awaited_once()+++async def test_cancelled_enqueue_discards_registered_work(tracked_queue):+ queue, index, transport, finalize, _ = tracked_queue+ transport.write.side_effect = asyncio.CancelledError()with bind_task_context("task-1", "account", "user"):- with pytest.raises(type(failure)):+ with pytest.raises(asyncio.CancelledError):await queue.enqueue({})assert not index.has_work("task-1")+ assert index.queue_status("task-1") == {}finalize.assert_awaited_once()@@ -252,7 +276,8 @@ async def test_process_result_tracks_children_and_errors(tracked_queue):class Handler(DequeueHandlerBase):async def on_dequeue(self, data):- contexts.append(get_task_context())+ with bind_task_context("task-1", "account", "user"):+ contexts.append(get_task_context())correlations.append(log_correlation(telemetry_id="tm-1", message_id="semantic-1"))await queue.enqueue({"child": True})return ProcessResult.failed("failed work")@@ -260,6 +285,7 @@ async def test_process_result_tracks_children_and_errors(tracked_queue):queue.set_dequeue_handler(Handler())assert (await queue.process_dequeued(message)).outcome is ProcessOutcome.FAILEDassert contexts[0].task_id == "task-1"+ assert contexts[0].work_id == extract_task_metadata(message).work_idassert correlations == ["task_id=task-1 telemetry_id=tm-1 message_id=semantic-1"]assert get_task_context() is Noneassert index.failure("task-1") == "failed work"@@ -347,6 +373,7 @@ async def test_cancel_before_handler_uses_legacy_discard_callback(tracked_queue)async def test_active_cancellation_distinguishes_user_from_shutdown(tracked_queue, user_cancel):queue, index, transport, _, cancelled = tracked_queuemessage = await enqueue_task(queue, transport)+ metadata = extract_task_metadata(message)started = asyncio.Event()class Handler(DequeueHandlerBase):@@ -363,6 +390,7 @@ async def test_active_cancellation_distinguishes_user_from_shutdown(tracked_queuindex.cancel_active("task-1")result = await asyncio.wait_for(running, timeout=1)assert result.outcome is ProcessOutcome.CANCELLED+ await index.wait_for_work("task-1", [metadata.work_id], timeout=0.1)await queue.ack("message-1", message)assert not index.has_work("task-1")assert (await queue.get_status()).processed == 1diff --git a/tests/service/test_task_work_contract.py b/tests/service/test_task_work_contract.pynew file mode 100644index 000000000..5bfc8dd9f--- /dev/null+++ b/tests/service/test_task_work_contract.py@@ -0,0 +1,163 @@+# Copyright (c) 2026 Beijing Volcano Engine Technology Co., Ltd.+# SPDX-License-Identifier: AGPL-3.0++import asyncio++import pytest++from openviking.service.task_store import PersistentTaskStore+from openviking.service.task_tracker import TaskStatus, TaskTracker+from openviking.service.task_work_index import QueueTaskMetadata, TaskWorkIndex+from tests.test_task_tracker import _FakeAgfs+++async def test_wait_for_work_observes_only_explicit_ids():+ index = TaskWorkIndex()+ first = QueueTaskMetadata("task", "first")+ unrelated = QueueTaskMetadata("task", "unrelated")+ assert index.register("Semantic", first)+ assert index.register("Embedding", unrelated)++ index.record_result(first, "completed")+ await index.wait_for_work("task", ["first"], timeout=0)++ assert index.has_work("task")++[email protected]("recovered", [False, True])+async def test_task_work_wait_includes_recovered_and_later_enqueued_work(recovered):+ index = TaskWorkIndex()+ root = QueueTaskMetadata("task", "root")+ semantic = QueueTaskMetadata("task", "semantic")+ embedding = QueueTaskMetadata("task", "embedding")+ if recovered:+ index.rebuild(+ {+ "Root": [{"task_id": "task", "_task_work_id": "root"}],+ "Semantic": [{"task_id": "task", "_task_work_id": "semantic"}],+ }+ )+ else:+ assert index.register("Root", root)+ assert index.register("Semantic", semantic)+ waiting = asyncio.create_task(+ index.wait_for_task_work("task", exclude_work_id="root", poll_interval=0.001)+ )+ try:+ await asyncio.sleep(0)+ assert not waiting.done()+ assert index.register("Embedding", embedding)+ index.record_result(semantic, "completed")+ await index.prepare_ack("Semantic", semantic)+ with pytest.raises(TimeoutError):+ await index.wait_for_task_work("task", exclude_work_id="root", timeout=0)+ index.record_result(embedding, "completed")+ await index.prepare_ack("Embedding", embedding)+ await asyncio.wait_for(waiting, timeout=1)+ assert index.has_work("task")+ finally:+ waiting.cancel()+ await asyncio.gather(waiting, return_exceptions=True)+++async def test_requeue_keeps_logical_work_until_replacement_ack():+ index = TaskWorkIndex()+ work = QueueTaskMetadata("task", "work")+ assert index.register("Semantic", work)+ assert index.register("Semantic", work)++ index.record_result(work, "requeued")+ await index.prepare_ack("Semantic", work)+ assert index.has_work("task")++ index.record_result(work, "completed")+ await index.wait_for_work("task", ["work"], timeout=0.1)+ assert index.queue_status("task")["Semantic"] == {+ "processed": 1,+ "requeue_count": 1,+ "error_count": 0,+ "errors": [],+ }++ await index.prepare_ack("Semantic", work)+ assert not index.has_work("task")++[email protected]("errors", [None, ["first", "second"]])+def test_terminal_work_result_is_monotonic_and_idempotent(errors):+ index = TaskWorkIndex()+ work = QueueTaskMetadata("task", "work")+ assert index.register("Embedding", work)+ messages = errors or ["first"]+ error = "\n".join(messages)++ index.record_result(work, "failed", error=error, errors=errors)+ index.record_result(work, "completed")+ index.record_result(work, "failed", error="later failure")++ assert index.failure("task") == error+ assert index.queue_status("task")["Embedding"] == {+ "processed": 0,+ "requeue_count": 0,+ "error_count": len(messages),+ "errors": [{"message": message} for message in messages],+ }+++async def test_enqueue_failure_finalizes_task_with_same_error():+ tracker = TaskTracker(PersistentTaskStore(_FakeAgfs()))+ index = TaskWorkIndex()+ tracker.attach_work_index(index)+ task = await tracker.create("content_write", account_id="account", user_id="user")+ await tracker.start(task.task_id, account_id="account", user_id="user")+ work = QueueTaskMetadata(task.task_id, "embedding", "account", "user")+ assert index.register("Embedding", work)++ await index.fail_enqueue("Embedding", work, "embedding enqueue returned false")++ failed = await tracker.get(task.task_id, account_id="account", user_id="user")+ assert failed.status is TaskStatus.FAILED+ assert failed.error == "embedding enqueue returned false"+++async def test_deleting_task_releases_retained_work_status():+ tracker = TaskTracker(PersistentTaskStore(_FakeAgfs()))+ index = TaskWorkIndex()+ tracker.attach_work_index(index)+ owner = {"account_id": "account", "user_id": "user"}+ task = await tracker.create("content_write", **owner)+ await tracker.start(task.task_id, **owner)+ work = QueueTaskMetadata(task.task_id, "semantic", "account", "user")+ assert index.register("Semantic", work)++ await tracker.complete(+ task.task_id,+ {"done": True, "queue_status": tracker.build_queue_status(task.task_id)},+ **owner,+ )+ index.record_result(work, "completed")+ await index.prepare_ack("Semantic", work)+ assert tracker.build_queue_status(task.task_id)["Semantic"]["processed"] == 1+ completed = await tracker.get(task.task_id, **owner)+ assert completed.result["queue_status"]["Semantic"]["processed"] == 1++ assert await tracker.delete_user_tasks(**owner) == 1+ assert tracker.build_queue_status(task.task_id) == {+ "Semantic": {+ "processed": 0,+ "requeue_count": 0,+ "error_count": 0,+ "errors": [],+ },+ "Embedding": {+ "processed": 0,+ "requeue_count": 0,+ "error_count": 0,+ "errors": [],+ },+ }diff --git a/tests/session/train/test_train_components.py b/tests/session/train/test_train_components.pyindex 831983cd5..f12858b70 100644--- a/tests/session/train/test_train_components.py+++ b/tests/session/train/test_train_components.py@@ -1279,12 +1279,8 @@ async def test_skill_policy_creation_preserves_lock_ownership(tmp_path, monkeypaawait fs._async_agfs.pathlock_release(lease)finally:from openviking.telemetry import unregister_telemetry- from openviking.telemetry.request_wait_tracker import get_request_wait_trackerif queue.enqueue.await_args:msg = queue.enqueue.await_args.args[0]- tracker = get_request_wait_tracker()- tracker.mark_semantic_done(msg.telemetry_id, msg.id)- tracker.cleanup(msg.telemetry_id)unregister_telemetry(msg.telemetry_id)agfs.close()diff --git a/tests/storage/test_collection_schemas.py b/tests/storage/test_collection_schemas.pyindex c0f149c01..da75841dd 100644--- a/tests/storage/test_collection_schemas.py+++ b/tests/storage/test_collection_schemas.py@@ -1324,10 +1324,17 @@ async def test_embedding_handler_honors_existing_record_id_override(monkeypatch)@pytest.mark.asyncio-async def test_embedding_handler_generates_id_when_no_existing_override(monkeypatch):+async def test_embedding_handler_generates_id_and_counts_written_context(monkeypatch):+ from openviking.service.task_tracker import TaskTracker+ from openviking.service.task_work_index import QueueTaskMetadata, TaskWorkIndexfrom openviking.storage.vector_ids import vector_record_idcaptured = {}+ tracker = TaskTracker(AsyncMock())+ index = TaskWorkIndex()+ tracker.attach_work_index(index)+ index.register("Embedding", QueueTaskMetadata("task-1", "work-1"))+ monkeypatch.setattr("openviking.storage.collection_schemas.get_task_tracker", lambda: tracker)class _CapturingVikingDB:is_closing = False@@ -1343,48 +1350,16 @@ async def test_embedding_handler_generates_id_when_no_existing_override(monkeypalambda: _DummyConfig(embedder),)payload = _build_queue_payload_for_account("acct-new-id")-- await TextEmbeddingHandler(_CapturingVikingDB()).on_dequeue(payload)-- assert captured["data"]["id"] == vector_record_id("acct-new-id", "viking://resources/sample", 2)---async def test_embedding_handler_settles_request_wait_by_message_id(monkeypatch):- class _CapturingVikingDB:- is_closing = False- mode = "local"- uses_content_field = False-- async def upsert(self, _data, *, ctx, options=UpsertOptions()):- assert options.partial_update is False- return "rec-1"-- embedder = _DummyEmbedder()- monkeypatch.setattr(- "openviking_cli.utils.config.get_openviking_config",- lambda: _DummyConfig(embedder),- )-- completed = []- monkeypatch.setattr(- "openviking.storage.collection_schemas.get_request_wait_tracker",- lambda: SimpleNamespace(- mark_embedding_done=lambda telemetry_id, root_id, **kwargs: completed.append(- (telemetry_id, root_id, kwargs)- )- ),- )-- handler = TextEmbeddingHandler(_CapturingVikingDB())- payload = _build_queue_payload()queue_data = json.loads(payload["data"])- queue_data["telemetry_id"] = "request-1"+ queue_data["task_id"] = "task-1"+ queue_data["_task_work_id"] = "work-1"payload["data"] = json.dumps(queue_data)- await handler.on_dequeue(payload)+ await TextEmbeddingHandler(_CapturingVikingDB()).on_dequeue(payload)- assert completed == [("request-1", queue_data["id"], {"vector_written": True})]+ assert captured["data"]["id"] == vector_record_id("acct-new-id", "viking://resources/sample", 2)+ assert tracker.get_context_count("task-1") == 1+ assert tracker.get_context_count("unrelated-task") == 0def test_context_collection_uses_acl_mode_and_excludes_parent_uri():diff --git a/tests/storage/test_content_write_processing_mode.py b/tests/storage/test_content_write_processing_mode.pyindex 1ed5b94f5..ef8e9a589 100644--- a/tests/storage/test_content_write_processing_mode.py+++ b/tests/storage/test_content_write_processing_mode.py@@ -8,6 +8,9 @@ import pytestfrom openviking.core.context import ContextLevelfrom openviking.server.identity import RequestContext, Role+from openviking.service.task_store import PersistentTaskStore+from openviking.service.task_tracker import TaskTracker+from openviking.service.task_work_index import TaskWorkIndex, bind_task_contextfrom openviking.session.memory.utils.memory_file_utils import MemoryFileUtilsfrom openviking.storage import content_write as content_write_modulefrom openviking.storage.abstract_overview import (@@ -15,12 +18,31 @@ from openviking.storage.abstract_overview import ()from openviking.storage.content_write import ContentWriteCoordinatorfrom openviking.storage.context_update_plan import ContextUpdatePlan, FileRefreshIntent+from openviking.storage.queuefs.queue_middleware import EnqueueReceiptfrom openviking.storage.queuefs.semantic_ops.freshness_policy import FreshnessActionfrom openviking.telemetry.context import bind_telemetryfrom openviking.telemetry.operation import OperationTelemetryfrom openviking.utils.ingest_options import IngestOptionsfrom openviking_cli.exceptions import InvalidArgumentError, NotFoundErrorfrom openviking_cli.session.user_id import UserIdentifier+from tests.test_task_tracker import _FakeAgfs++[email protected](autouse=True)+def run_content_write_operations_inline(monkeypatch):+ tracker = TaskTracker(PersistentTaskStore(_FakeAgfs()))+ tracker.attach_work_index(TaskWorkIndex())+ monkeypatch.setattr("openviking.service.task_tracker._instance", tracker)++ async def run_tracked_operation(_task_type, *, operation, **_kwargs):+ with bind_task_context("test-task", "account-1", "user-1", enqueued_work=[]):+ return await operation(), "test-task"++ monkeypatch.setattr(+ content_write_module,+ "run_tracked_operation",+ run_tracked_operation,+ )class _FakePathLock:@@ -283,12 +305,15 @@ async def test_direct_write_hands_exact_lock_to_request_coordinator(monkeypatch,)),)- queue = SimpleNamespace(enqueue=AsyncMock(return_value="queued"))+ queue = SimpleNamespace(enqueue=AsyncMock(return_value=EnqueueReceipt("queued", "work-1")))manager = SimpleNamespace(SEMANTIC="semantic", get_queue=lambda *args, **kwargs: queue)monkeypatch.setattr(content_write_module, "get_queue_manager", lambda: manager)telemetry = OperationTelemetry(operation="content.write", enabled=False)- with bind_telemetry(telemetry):+ with (+ bind_telemetry(telemetry),+ bind_task_context("test-task", "account-1", "user-1", enqueued_work=[]),+ ):result = await coordinator._write_direct_with_refresh(uri=uri,root_uri="viking://resources",@@ -312,12 +337,6 @@ async def test_direct_write_hands_exact_lock_to_request_coordinator(monkeypatch,assert result["semantic_status"] == "queued"assert result["vector_status"] == "queued"- content_write_module.get_request_wait_tracker().mark_semantic_done(- telemetry.telemetry_id,- coordinator_msg.id,- processed_delta=0,- )-@pytest.mark.asyncioasync def test_direct_write_commits_final_bytes_when_rnfv_content_is_unchanged(monkeypatch, ctx):diff --git a/tests/storage/test_context_update_plan.py b/tests/storage/test_context_update_plan.pyindex 5b195762c..dd7627a8b 100644--- a/tests/storage/test_context_update_plan.py+++ b/tests/storage/test_context_update_plan.py@@ -3410,6 +3410,7 @@ async def test_directory_output_unchanged_still_applies_planned_scalar_fields(moSemanticTreeSnapshot,)from openviking.storage.queuefs.semantic_executor import SemanticTreeExecutor+ from openviking.storage.queuefs.semantic_processor import SemanticProcessorfrom openviking_cli.session.user_id import UserIdentifierroot = "viking://resources/repo"@@ -3421,10 +3422,16 @@ async def test_directory_output_unchanged_still_applies_planned_scalar_fields(molambda: SimpleNamespace(semantic=SimpleNamespace(overview_sample_limit=32)),)- class Processor:+ embedding_queue = SimpleNamespace(enqueue=AsyncMock(return_value="embedding-id"))+ queue_manager = SimpleNamespace(+ EMBEDDING="Embedding",+ get_queue=lambda *args, **kwargs: embedding_queue,+ )+ monkeypatch.setattr("openviking.storage.queuefs.get_queue_manager", lambda: queue_manager)++ class Processor(SemanticProcessor):_generate_overview = AsyncMock(return_value="overview")_vectorize_directory = AsyncMock(return_value=set())- _update_vector_fields = AsyncMock(return_value=True)@staticmethoddef _normalize_overview_generation(overview):@@ -3481,22 +3488,12 @@ async def test_directory_output_unchanged_still_applies_planned_scalar_fields(moawait executor.run(root)processor._vectorize_directory.assert_not_awaited()- processor._update_vector_fields.assert_awaited_once_with(- record_id="root-l0",- uri=root,- level=0,- field_patch=FieldPatch(- {"search_tags": ["scope=new"]},- {"search_tags": "append"},- {- "abstract": "abstract",- "uri": root,- "level": 0,- "account_id": "acc",- },- ),- ctx=executor._ctx,- )+ embedding_queue.enqueue.assert_awaited_once()+ message = embedding_queue.enqueue.await_args.args[0]+ assert message.action.value == "update_fields"+ assert message.record_ids == ["root-l0"]+ assert message.update_fields == {"search_tags": ["scope=new"]}+ assert message.field_modes == {"search_tags": "append"}@pytest.mark.asyncio@@ -3665,7 +3662,8 @@ async def test_file_repair_passes_partial_update_to_embedding(monkeypatch):@pytest.mark.asyncio-async def test_direct_only_plan_skips_semantic_queue(monkeypatch):[email protected]("action_name", ["delete", "update_fields"])+async def test_direct_only_plan_skips_semantic_queue(monkeypatch, action_name):from openviking.server.identity import RequestContext, Rolefrom openviking.storage.context_update_plan import DirectIndexActionfrom openviking.utils.resource_processor import ResourceProcessor@@ -3678,15 +3676,22 @@ async def test_direct_only_plan_skips_semantic_queue(monkeypatch):)monkeypatch.setattr("openviking.utils.resource_processor.get_viking_fs", lambda: viking_fs)processor = ResourceProcessor(SimpleNamespace(get_embedder=lambda: None))- processor._enqueue_index_actions = AsyncMock()+ embedding_queue = SimpleNamespace(enqueue=AsyncMock(return_value="embedding-id"))+ queue_manager = SimpleNamespace(+ EMBEDDING="Embedding",+ get_queue=lambda *args, **kwargs: embedding_queue,+ )+ monkeypatch.setattr("openviking.storage.queuefs.get_queue_manager", lambda: queue_manager)summarizer = SimpleNamespace(summarize=AsyncMock())processor._get_summarizer = lambda: summarizeraction = DirectIndexAction(- "update_fields",+ action_name,root + "/a.py",2,"a-l2",- field_patch=FieldPatch({"search_tags": ["scope=new"]}),+ field_patch=(+ FieldPatch({"search_tags": ["scope=new"]}) if action_name == "update_fields" else None+ ),)await processor.finish_prepared_resource(@@ -3707,7 +3712,7 @@ async def test_direct_only_plan_skips_semantic_queue(monkeypatch):"uri": action.uri,"level": action.level,"record_id": action.record_id,- "field_patch": action.field_patch.to_dict(),+ "field_patch": action.field_patch.to_dict() if action.field_patch else None,"md5": action.md5,}],@@ -3719,7 +3724,12 @@ async def test_direct_only_plan_skips_semantic_queue(monkeypatch):processing_mode="semantic_and_vectors",)- processor._enqueue_index_actions.assert_awaited_once()+ embedding_queue.enqueue.assert_awaited_once()+ message = embedding_queue.enqueue.await_args.args[0]+ assert message.action.value == action_name+ assert message.record_ids == ["a-l2"]+ if action_name == "update_fields":+ assert message.update_fields == {"search_tags": ["scope=new"]}summarizer.summarize.assert_not_awaited()viking_fs._async_agfs.pathlock_release.assert_awaited_once_with(lock)diff --git a/tests/storage/test_embedding_msg.py b/tests/storage/test_embedding_msg.pyindex 3fb7edf01..0880409aa 100644--- a/tests/storage/test_embedding_msg.py+++ b/tests/storage/test_embedding_msg.py@@ -1,8 +1,6 @@# Copyright (c) 2026 Beijing Volcano Engine Technology Co., Ltd.# SPDX-License-Identifier: AGPL-3.0-from uuid import uuid4-import pytestfrom openviking.storage.index_action import FieldPatch, IndexAction@@ -13,39 +11,24 @@ from openviking.storage.queuefs.embedding_msg import (NoopPayload,UpdateFieldsPayload,)-from openviking.telemetry.request_wait_tracker import RequestWaitTracker---def test_embedding_msg_roundtrip_preserves_id_for_request_wait_tracker():- telemetry_id = f"tm_{uuid4().hex}"- tracker = RequestWaitTracker.get_instance()- tracker.register_request(telemetry_id)-- try:- msg = EmbeddingMsg(- "hello",- {"uri": "viking://user/default/skills/demo", "account_id": "default"},- telemetry_id=telemetry_id,- )- tracker.register_embedding_root(telemetry_id, msg.id)-- restored = EmbeddingMsg.from_dict(msg.to_dict())- assert restored.id == msg.id- tracker.mark_embedding_done(telemetry_id, restored.id)- assert tracker.is_complete(telemetry_id)- finally:- tracker.cleanup(telemetry_id)--def test_embedding_msg_roundtrip_preserves_queue_enqueue_time():+def test_embedding_msg_roundtrip_preserves_task_identity_and_queue_time():msg = EmbeddingMsg("hello",- {"uri": "viking://resources/demo", "account_id": "default"},+ {"uri": "viking://user/default/skills/demo", "account_id": "default"},+ telemetry_id="telemetry-1",+ task_id="task-1",+ _task_work_id="work-1",queue_enqueued_at=123.456,)- assert EmbeddingMsg.from_dict(msg.to_dict()).queue_enqueued_at == 123.456+ restored = EmbeddingMsg.from_dict(msg.to_dict())++ assert restored.id == msg.id+ assert restored.task_id == "task-1"+ assert restored._task_work_id == "work-1"+ assert restored.queue_enqueued_at == 123.456def test_embedding_msg_roundtrip_preserves_failure_requeue_count():@@ -316,14 +299,10 @@ def test_legacy_upsert_discards_historically_ignored_fields():assert restored.action is IndexAction.UPSERTassert restored.record_ids == []assert restored.field_patch is None- assert set(restored.to_dict()) == {- "id",- "telemetry_id",- "queue_enqueued_at",- "failure_requeue_count",- "action",- "payload",- }+ serialized = restored.to_dict()+ assert "record_ids" not in serialized+ assert "update_fields" not in serialized+ assert "initial_fields" not in serializeddef test_legacy_update_fields_message_normalizes_to_field_patch_payload():diff --git a/tests/storage/test_memory_semantic_stall.py b/tests/storage/test_memory_semantic_stall.pyindex 1e682013b..98572519f 100644--- a/tests/storage/test_memory_semantic_stall.py+++ b/tests/storage/test_memory_semantic_stall.py@@ -17,7 +17,6 @@ import pytestfrom openviking.storage.queuefs.process_result import ProcessOutcomefrom openviking.storage.queuefs.semantic_msg import SemanticMsgfrom openviking.storage.queuefs.semantic_processor import SemanticProcessor-from openviking.telemetry.request_wait_tracker import get_request_wait_trackerfrom openviking.utils.circuit_breaker import CircuitBreakerOpen@@ -53,7 +52,17 @@ def _build_data(msg: SemanticMsg) -> dict:@pytest.mark.parametrize("context_type", ["resource", "memory", "skill"])async def test_retry_cancellation_keeps_skill_wait_isolated(monkeypatch, context_type):+ from openviking.service.task_store import PersistentTaskStore+ from openviking.service.task_tracker import TaskTracker+ from tests.test_task_tracker import _FakeAgfs++ monkeypatch.setattr(+ "openviking.service.task_tracker._instance", TaskTracker(PersistentTaskStore(_FakeAgfs()))+ )processor = _processor()+ processor._circuit_breakers["acc1"] = SimpleNamespace(+ check=MagicMock(side_effect=CircuitBreakerOpen), retry_after=0+ )entered, release = asyncio.Event(), asyncio.Event()written = []@@ -88,7 +97,6 @@ async def test_retry_cancellation_keeps_skill_wait_isolated(monkeypatch, contextif not worker.done():worker.cancel()await asyncio.gather(worker, return_exceptions=True)- get_request_wait_tracker().cleanup(msg.telemetry_id)@pytest.mark.asynciodiff --git a/tests/storage/test_process_result.py b/tests/storage/test_process_result.pyindex 5d21e1c16..995e186f0 100644--- a/tests/storage/test_process_result.py+++ b/tests/storage/test_process_result.py@@ -32,3 +32,9 @@ def test_result_preserves_value_and_empty_error():payload = {"value": 1}assert ProcessResult.success(payload).value is payloadassert ProcessResult.failed("").error == ""++[email protected]("errors", [(), ("valid", 1)])+def test_result_rejects_invalid_structured_errors(errors):+ with pytest.raises(ValueError):+ ProcessResult.failed("failure", errors=errors)diff --git a/tests/storage/test_queue_manager.py b/tests/storage/test_queue_manager.pyindex b0aa487af..b691c8fe5 100644--- a/tests/storage/test_queue_manager.py+++ b/tests/storage/test_queue_manager.py@@ -14,7 +14,11 @@ from unittest.mock import AsyncMockimport pytestfrom openviking.pyagfs import AsyncAGFSClient-from openviking.service.task_work_index import bind_task_context+from openviking.service.task_work_index import (+ QueueTaskMetadata,+ bind_task_context,+ record_enqueued_work,+)from openviking.storage.queuefs.named_queue import DequeueHandlerBase, NamedQueuefrom openviking.storage.queuefs.process_result import ProcessResultfrom openviking.storage.queuefs.queue_manager import QueueManager@@ -86,17 +90,18 @@ async def test_skill_shutdown_releases_lock_after_embedding_worker_exits(from openviking.storage.queuefs.semantic_executor import SemanticTreeStatsfrom openviking.storage.queuefs.semantic_msg import SemanticMsgfrom openviking.storage.queuefs.semantic_processor import SemanticProcessor- from openviking.telemetry import OperationTelemetry- from openviking.telemetry.request_wait_tracker import get_request_wait_tracker- tracker = get_request_wait_tracker()- telemetry_id = OperationTelemetry("skill-shutdown").telemetry_id- tracker.register_request(telemetry_id)msg = SemanticMsg(- uri="viking://agent/skills/demo", context_type="skill", telemetry_id=telemetry_id+ uri="viking://agent/skills/demo",+ context_type="skill",+ task_id="skill-task",+ _task_work_id="semantic",)started, queued, release = threading.Event(), threading.Event(), threading.Event()events = []+ work_index = None+ active_work = QueueTaskMetadata("skill-task", "active")+ queued_work = QueueTaskMetadata("skill-task", "queued")class Lease:lock = {"lease_ref": "demo"}@@ -111,8 +116,11 @@ async def test_skill_shutdown_releases_lock_after_embedding_worker_exits(passasync def run(self, uri):- tracker.register_embedding_root(telemetry_id, "active")- tracker.register_embedding_root(telemetry_id, "queued")+ assert work_index is not None+ assert work_index.register("Embedding", active_work)+ assert work_index.register("Embedding", queued_work)+ record_enqueued_work("Embedding", active_work.work_id)+ record_enqueued_work("Embedding", queued_work.work_id)queued.set()def get_stats(self):@@ -124,7 +132,8 @@ async def test_skill_shutdown_releases_lock_after_embedding_worker_exits(while not release.is_set():await asyncio.sleep(0.01)events.append("write-finished")- tracker.mark_embedding_done(telemetry_id, "active")+ assert work_index is not None+ work_index.record_result(active_work, "completed")await run_to_completion(finish)return ProcessResult.success()@@ -146,6 +155,9 @@ async def test_skill_shutdown_releases_lock_after_embedding_worker_exits(return {"id": self.name, "data": msg.to_json()}async def process_dequeued(self, data):+ if self.name == QueueManager.SEMANTIC:+ with bind_task_context("skill-task", "", "", enqueued_work=[]):+ return await self.handler(data)return await self.handler(data)async def ack(self, *args):@@ -171,6 +183,11 @@ async def test_skill_shutdown_releases_lock_after_embedding_worker_exits(manager = QueueManager(object(), max_concurrent_semantic=concurrency, max_concurrent_embedding=concurrency)+ work_index = manager._task_work_index+ monkeypatch.setattr(+ "openviking.storage.queuefs.semantic_work.get_task_tracker",+ lambda: SimpleNamespace(wait_for_work=work_index.wait_for_work),+ )manager._poll_interval = 0.001manager.set_vlm_resolver(SimpleNamespace(get_vlm=AsyncMock(return_value=SimpleNamespace())))manager.setup_standard_queues(object(), start=False)@@ -197,14 +214,13 @@ async def test_skill_shutdown_releases_lock_after_embedding_worker_exits(assert all(not thread.is_alive() for thread in threads)assert events.index("write-finished") < events.index("lock-released")assert f"ack-{manager.SEMANTIC}" not in events- assert not tracker.is_complete(telemetry_id) # Queued vectors await restart.+ assert work_index.has_work("skill-task") # Queued vectors await restart.finally:release.set()if stopping is not None:await stoppingelse:await asyncio.to_thread(manager.stop)- tracker.cleanup(telemetry_id)async def test_status_waits_for_processing_messages_from_other_workers(monkeypatch) -> None:diff --git a/tests/storage/test_reindex_msg.py b/tests/storage/test_reindex_msg.pyindex 578982af6..5bb6883d3 100644--- a/tests/storage/test_reindex_msg.py+++ b/tests/storage/test_reindex_msg.py@@ -1,3 +1,4 @@+import asynciofrom types import SimpleNamespacefrom unittest.mock import AsyncMock, Mock@@ -62,15 +63,16 @@ def test_reindex_msg_rejects_missing_descriptor_fields():@pytest.mark.asyncioasync def test_reindex_processor_terminalizes_root_after_descendants(monkeypatch):+ from openviking.service.task_store import PersistentTaskStore+ from openviking.service.task_tracker import TaskTracker+ from openviking.service.task_work_index import QueueTaskMetadata, TaskWorkIndexfrom openviking.storage.queuefs.reindex_processor import ReindexProcessor+ from tests.test_task_tracker import _FakeAgfs- tracker = SimpleNamespace(- get=AsyncMock(return_value=SimpleNamespace(status=TaskStatus.RUNNING)),- start=AsyncMock(),- complete=AsyncMock(),- fail=AsyncMock(),- wait_for_descendants=AsyncMock(),- )+ owner = {"account_id": "account-1", "user_id": "user-1"}+ tracker = TaskTracker(PersistentTaskStore(_FakeAgfs()))+ index = TaskWorkIndex()+ tracker.attach_work_index(index)monkeypatch.setattr("openviking.storage.queuefs.reindex_processor.get_task_tracker", Mock(return_value=tracker))@@ -86,25 +88,49 @@ async def test_reindex_processor_terminalizes_root_after_descendants(monkeypatchviking_fs = SimpleNamespace(_async_agfs=SimpleNamespace(pathlock_release=AsyncMock()))processor = ReindexProcessor(viking_fs)processor._adopt_or_reacquire = AsyncMock(return_value={"lease": "root"})+ task = await tracker.create("admin_reindex", resource_id="viking://resources/demo", **owner)+ root_work = QueueTaskMetadata(task.task_id, "root-work", **owner)+ child_work = QueueTaskMetadata(task.task_id, "child-work", **owner)+ assert index.register("Reindex", root_work)+ assert index.register("Embedding", child_work)+ entered_descendant_wait = asyncio.Event()+ original_wait_for_task_work = tracker.wait_for_task_work++ async def wait_for_task_work(*args, **kwargs):+ entered_descendant_wait.set()+ await original_wait_for_task_work(*args, **kwargs)++ monkeypatch.setattr(tracker, "wait_for_task_work", wait_for_task_work)message = ReindexMsg(- task_id="task-1",+ task_id=task.task_id,uri="viking://resources/demo",object_type="resource",mode="vectors_only",- account_id="account-1",- user_id="user-1",+ **owner,role="admin",)- outcome = await processor.on_dequeue(- {"id": "queue-1", "data": {**message.to_dict(), "_task_work_id": "root-work"}}+ processing = asyncio.create_task(+ processor.on_dequeue(+ {"id": "queue-1", "data": {**message.to_dict(), "_task_work_id": "root-work"}}+ ))+ await asyncio.wait_for(entered_descendant_wait.wait(), 1)+ assert not processing.done()+ assert (await tracker.get(task.task_id, **owner)).status is TaskStatus.RUNNING+ index.record_result(child_work, "completed")+ await index.prepare_ack("Embedding", child_work)+ outcome = await asyncio.wait_for(processing, 1)assert outcome.outcome.value == "success"- tracker.wait_for_descendants.assert_awaited_once_with("task-1", "root-work")- tracker.complete.assert_awaited_once_with(- "task-1", result, account_id="account-1", user_id="user-1"- )+ assert (await tracker.get(task.task_id, **owner)).status is TaskStatus.RUNNING++ index.record_result(root_work, "completed")+ await index.prepare_ack("Reindex", root_work)+ completed = await tracker.wait(task.task_id, **owner, timeout=0)+ assert completed.status is TaskStatus.COMPLETED+ assert completed.result["rebuilt_records"] == 3+ assert completed.result["queue_status"]["Embedding"]["processed"] == 1@pytest.mark.asyncio@@ -122,7 +148,7 @@ async def test_reindex_processor_restores_root_telemetry_for_executor(monkeypatcstart=AsyncMock(),complete=AsyncMock(),fail=AsyncMock(),- wait_for_descendants=AsyncMock(),+ wait_for_task_work=AsyncMock(),)monkeypatch.setattr("openviking.storage.queuefs.reindex_processor.get_task_tracker", Mock(return_value=tracker)@@ -177,7 +203,7 @@ async def test_reindex_processor_replay_of_completed_task_skips_force_rebuild(mostart=AsyncMock(),complete=AsyncMock(),fail=AsyncMock(),- wait_for_descendants=AsyncMock(),+ wait_for_task_work=AsyncMock(),)monkeypatch.setattr("openviking.storage.queuefs.reindex_processor.get_task_tracker", Mock(return_value=tracker)@@ -225,7 +251,7 @@ async def test_reindex_processor_requeues_lock_handoff_failure_before_ack(monkeystart=AsyncMock(),complete=AsyncMock(),fail=AsyncMock(),- wait_for_descendants=AsyncMock(),+ wait_for_task_work=AsyncMock(),)queue_manager = SimpleNamespace(REINDEX="Reindex", enqueue=AsyncMock(return_value="retry-1"))monkeypatch.setattr(@@ -250,7 +276,12 @@ async def test_reindex_processor_requeues_lock_handoff_failure_before_ack(monkeyassert outcome.outcome.value == "requeued"queue_manager.enqueue.assert_awaited_once_with(- "Reindex", {**message.to_dict(), "lock_handoff_retry": 1}+ "Reindex",+ {+ **message.to_dict(),+ "_task_work_id": "root-work",+ "lock_handoff_retry": 1,+ },)tracker.fail.assert_not_awaited()@@ -264,7 +295,7 @@ async def test_reindex_processor_fails_after_lock_handoff_retry_limit(monkeypatcstart=AsyncMock(),complete=AsyncMock(),fail=AsyncMock(),- wait_for_descendants=AsyncMock(),+ wait_for_task_work=AsyncMock(),)monkeypatch.setattr("openviking.storage.queuefs.reindex_processor.get_task_tracker", Mock(return_value=tracker)@@ -324,7 +355,7 @@ async def test_reindex_processor_does_not_release_lease_owned_by_failed_executorstart=AsyncMock(),complete=AsyncMock(),fail=AsyncMock(),- wait_for_descendants=AsyncMock(),+ wait_for_task_work=AsyncMock(),)monkeypatch.setattr("openviking.storage.queuefs.reindex_processor.get_task_tracker", Mock(return_value=tracker)diff --git a/tests/storage/test_semantic_processor_lock_ownership.py b/tests/storage/test_semantic_processor_lock_ownership.pyindex 0e94ba804..534042adf 100644--- a/tests/storage/test_semantic_processor_lock_ownership.py+++ b/tests/storage/test_semantic_processor_lock_ownership.py@@ -3,17 +3,19 @@from __future__ import annotations+import asyncio+import json+from contextlib import nullcontextfrom types import SimpleNamespace+from typing import castfrom unittest.mock import AsyncMockimport pytestfrom openviking.storage.queuefs.process_result import ProcessOutcomefrom openviking.storage.queuefs.semantic_executor import SemanticTreeStats-from openviking.storage.queuefs.semantic_lock import SemanticLockScopefrom openviking.storage.queuefs.semantic_msg import SemanticMsgfrom openviking.storage.queuefs.semantic_processor import SemanticProcessor-from openviking.storage.queuefs.semantic_work import SemanticMessageWorkdef _processor():@@ -71,87 +73,6 @@ async def test_memory_semantic_directory_does_not_release_borrowed_lock(monkeypaassert pathlock.release_calls == []-async def test_durable_plan_waits_for_embeddings_before_releasing_handoff_lock(monkeypatch):- from openviking.storage.context_update_plan import (- SemanticPlan,- SemanticTreeEntry,- SemanticTreeSnapshot,- )-- events = []- pathlock = _FakePathLock()- lease = {"id": "durable-plan-lock"}- plan = SemanticPlan(- "viking://resources/demo",- "resource",- SemanticTreeSnapshot((SemanticTreeEntry("", "directory", "unchanged", "aggregate"),)),- )- msg = SemanticMsg(- uri=plan.root_uri,- context_type="resource",- telemetry_id="durable-plan",- lock_handoff={"owner_id": "producer"},- plan=plan,- )-- class Tracker:- async def wait_for_embeddings(self, telemetry_id, **kwargs):- assert telemetry_id == msg.telemetry_id- events.append("embeddings-settled")-- monkeypatch.setattr(- "openviking.storage.queuefs.semantic_work.get_request_wait_tracker", lambda: Tracker()- )- monkeypatch.setattr(- "openviking.storage.queuefs.semantic_lock.get_viking_fs",- lambda: _FakeVikingFS(pathlock),- )-- work = SemanticMessageWork(SimpleNamespace(), msg, caller_lock=None)- work.scope = SemanticLockScope(lease, _owned=True)-- await work.finish_processing(True)-- assert events == ["embeddings-settled"]- assert pathlock.release_calls == ["durable-plan-lock"]---async def test_content_write_waits_for_embeddings_before_releasing_handoff_lock(monkeypatch):- events = []- pathlock = _FakePathLock()- lease = {"id": "content-write-lock"}- msg = SemanticMsg(- uri="viking://resources/demo",- context_type="resource",- telemetry_id="content-write",- lock_handoff={"owner_id": "producer"},- generation_trigger="content_write",- )-- class Tracker:- async def wait_for_embeddings(self, telemetry_id, **kwargs):- assert telemetry_id == msg.telemetry_id- events.append("embeddings-settled")-- monkeypatch.setattr(- "openviking.storage.queuefs.semantic_work.get_request_wait_tracker", lambda: Tracker()- )- monkeypatch.setattr(- "openviking.storage.queuefs.semantic_lock.get_viking_fs",- lambda: _FakeVikingFS(pathlock),- )-- work = SemanticMessageWork(SimpleNamespace(), msg, caller_lock=None)- work.scope = SemanticLockScope(lease, _owned=True)-- await work.finish_processing(True)-- assert events == ["embeddings-settled"]- assert pathlock.release_calls == ["content-write-lock"]--def test_semantic_tree_stats_aggregate_multiple_plans_for_one_request():telemetry_id = "reindex-request-stats"SemanticProcessor._cache_tree_stats(@@ -177,6 +98,13 @@ def test_semantic_tree_stats_aggregate_multiple_plans_for_one_request():@pytest.mark.parametrize("context_type", ["resource", "memory", "skill"])async def test_missing_root_is_acked_before_lock_scope_is_resolved(monkeypatch, context_type):"""Resolving the lock for a deleted root would recreate it to hold lock metadata."""+ from openviking.service.task_store import PersistentTaskStore+ from openviking.service.task_tracker import TaskTracker+ from tests.test_task_tracker import _FakeAgfs++ monkeypatch.setattr(+ "openviking.service.task_tracker._instance", TaskTracker(PersistentTaskStore(_FakeAgfs()))+ )processor = _processor()fs = _FakeVikingFS()@@ -218,3 +146,179 @@ async def test_missing_root_is_acked_before_lock_scope_is_resolved(monkeypatch,assert fs._async_agfs.release_calls == (["queued-skill-lock"] if context_type == "skill" else [])+++ ("context_type", "tracked", "fail_generation"),+ [+ ("resource", True, False),+ ("skill", True, False),+ ("skill", False, False),+ ("skill", False, True),+ ],+)+async def test_shared_scheduler_drains_each_messages_embeddings_before_unlock(+ monkeypatch, context_type, tracked, fail_generation+):+ from openviking.pyagfs import AGFSSyncClientProtocol+ from openviking.server.identity import RequestContext, Role+ from openviking.service.task_queue_middleware import TaskWorkQueueMiddleware+ from openviking.service.task_store import PersistentTaskStore+ from openviking.service.task_tracker import TaskStatus, TaskTracker+ from openviking.service.task_work_index import (+ TaskWorkIndex,+ bind_task_context,+ extract_task_metadata,+ )+ from openviking.storage.context_update_plan import (+ ContentState,+ IndexAction,+ IndexSlot,+ SemanticAction,+ SemanticPlan,+ SemanticTreeEntry,+ SemanticTreeSnapshot,+ )+ from openviking.storage.queuefs.named_queue import NamedQueue+ from openviking.storage.queuefs.semantic_executor import SemanticTreeExecutor+ from openviking_cli.session.user_id import UserIdentifier+ from tests.storage.test_semantic_executor_skip_files import _FakeVikingFS as TreeFS+ from tests.test_task_tracker import _FakeAgfs, _owner_kwargs++ root = (+ "viking://resources/demo"+ if context_type == "resource"+ else "viking://agent/skills/demo/reference"+ )+ warm_root = "viking://resources/warm"+ released = []++ class FS(TreeFS):+ async def exists(self, uri, ctx=None):+ return True++ async def pathlock_adopt(self, lease):+ return lease++ async def pathlock_release(self, lease):+ if "id" in lease:+ released.append(lease["id"])++ fs = FS({root: [], warm_root: []})+ for module in ("semantic_processor", "semantic_executor", "semantic_lock"):+ monkeypatch.setattr(f"openviking.storage.queuefs.{module}.get_viking_fs", lambda: fs)+ agfs = _FakeAgfs()+ tracker = TaskTracker(PersistentTaskStore(agfs))+ index = TaskWorkIndex()+ tracker.attach_work_index(index)+ monkeypatch.setattr("openviking.service.task_tracker._instance", tracker)+ entered_embedding_drain = asyncio.Event()+ original_wait_for_work = tracker.wait_for_work++ async def wait_for_work(*args, **kwargs):+ entered_embedding_drain.set()+ await original_wait_for_work(*args, **kwargs)++ monkeypatch.setattr(tracker, "wait_for_work", wait_for_work)+ owner = _owner_kwargs("acc", "alice")+ source = await tracker.create("content_write", **owner)+ queue = NamedQueue(+ cast(AGFSSyncClientProtocol, agfs),+ "/queue",+ "Embedding",+ middlewares=[TaskWorkQueueMiddleware(index)],+ )+ enqueued = asyncio.Event()++ async def vectorize(uri, **kwargs):+ await queue.enqueue({"context_data": {"uri": uri, **owner}})+ enqueued.set()+ if fail_generation:+ raise RuntimeError("second embedding enqueue failed")+ return {0, 1}++ processor = _processor()+ processor._generate_overview = AsyncMock(return_value="overview")+ processor._vectorize_directory = vectorize+ ctx = RequestContext(UserIdentifier("acc", "alice"), Role(Role.USER))+ # Create the shared workers under a different Task, as happens across requests.+ with bind_task_context(source.task_id, **owner, enqueued_work=[]):+ await SemanticTreeExecutor(processor, "resource", 1, ctx, skip_vectorization=True).run(+ warm_root+ )+ await tracker.complete(source.task_id, {"ok": True}, **owner)++ plan = None+ if context_type == "resource":+ plan = SemanticPlan(+ root,+ context_type,+ SemanticTreeSnapshot(+ (+ SemanticTreeEntry(+ "",+ "directory",+ ContentState.UNCHANGED,+ SemanticAction.AGGREGATE,+ index_slots=(IndexSlot(0, "root-l0", action=IndexAction.UPSERT),),+ ),+ )+ ),+ )+ msg = SemanticMsg(+ uri=root,+ context_type=context_type,+ account_id="acc",+ user_id="alice",+ recursive=False,+ propagate_to_parent=False,+ generation_trigger="parent_refresh",+ lock_handoff={"id": "message-lock"},+ plan=plan,+ )+ task = await tracker.create("content_write", **owner) if tracked else None+ task_context = (+ bind_task_context(task.task_id, **owner, enqueued_work=[], work_id="semantic")+ if task+ else nullcontext()+ )+ with task_context:+ worker = asyncio.create_task(processor.on_dequeue({"data": msg.to_json()}))+ try:+ await asyncio.wait_for(enqueued.wait(), 1)+ await asyncio.wait_for(entered_embedding_drain.wait(), 1)+ assert not worker.done()+ assert released == []+ payload = json.loads(agfs.files["/queue/Embedding/enqueue"])+ metadata = extract_task_metadata(payload)+ assert metadata is not None+ assert metadata.task_id != source.task_id+ if task:+ assert metadata.task_id == task.task_id+ index.record_result(metadata, "completed")+ await index.prepare_ack("Embedding", metadata)+ result = await asyncio.wait_for(worker, 1)+ assert result.outcome is (+ ProcessOutcome.FAILED if fail_generation else ProcessOutcome.SUCCESS+ )+ assert released == ["message-lock"]+ if not tracked:+ completed = await tracker.wait(metadata.task_id, **owner, timeout=0)+ assert completed.status is (+ TaskStatus.FAILED if fail_generation else TaskStatus.COMPLETED+ )+ if fail_generation:+ assert completed.error is not None+ assert "second embedding enqueue failed" in completed.error+ source_result = await tracker.get(source.task_id, **owner)+ assert source_result is not None+ assert source_result.status is TaskStatus.COMPLETED+ finally:+ payload = agfs.files.get("/queue/Embedding/enqueue")+ if payload:+ metadata = extract_task_metadata(json.loads(payload))+ if metadata:+ index.record_result(metadata, "completed")+ worker.cancel()+ await asyncio.gather(worker, return_exceptions=True)diff --git a/tests/storage/test_semantic_processor_permanent_storage_error.py b/tests/storage/test_semantic_processor_permanent_storage_error.pyindex af31296ad..859af3855 100644--- a/tests/storage/test_semantic_processor_permanent_storage_error.py+++ b/tests/storage/test_semantic_processor_permanent_storage_error.py@@ -5,7 +5,7 @@import asyncioimport jsonfrom types import SimpleNamespace-from unittest.mock import AsyncMock, Mock, call+from unittest.mock import AsyncMock, Mockfrom uuid import uuid4import pytest@@ -21,7 +21,6 @@ from openviking.storage.queuefs.semantic_executor import SemanticTreeStatsfrom openviking.storage.queuefs.semantic_msg import SemanticMsgfrom openviking.storage.queuefs.semantic_processor import SemanticProcessorfrom openviking.storage.viking_fs import SyncDiff-from openviking.telemetry.request_wait_tracker import get_request_wait_trackerfrom openviking.utils.circuit_breaker import CircuitBreaker, CircuitBreakerOpen@@ -49,21 +48,15 @@ def semantic_env(monkeypatch):resolver = SimpleNamespace(get_vlm=AsyncMock(return_value=SimpleNamespace()))processor = SemanticProcessor(vlm_resolver=resolver)processor._sync_topdown_recursive = AsyncMock(return_value=SyncDiff())- tracker = get_request_wait_tracker()- telemetry_ids = []def message(**kwargs):- msg = SemanticMsg(+ return SemanticMsg(uri="viking://resources/notes.txt",context_type="resource",telemetry_id=str(uuid4()),propagate_to_parent=False,**kwargs,)- telemetry_ids.append(msg.telemetry_id)- tracker.register_request(msg.telemetry_id)- tracker.register_semantic_root(msg.telemetry_id, msg.id)- return msgyield SimpleNamespace(processor=processor,@@ -71,11 +64,8 @@ def semantic_env(monkeypatch):lease=lease,dag=dag,retry_queue=retry_queue,- tracker=tracker,message=message,)- for telemetry_id in telemetry_ids:- tracker.cleanup(telemetry_id)def _storage_error(wrapped):@@ -87,16 +77,6 @@ def _storage_error(wrapped):return error-async def _assert_request_failed(env, msg, error):- await asyncio.wait_for(env.tracker.wait_for_request(msg.telemetry_id), timeout=1)- assert env.tracker.build_queue_status(msg.telemetry_id)["Semantic"] == {- "processed": 0,- "requeue_count": 0,- "error_count": 1,- "errors": [{"message": str(error)}],- }--@pytest.mark.asyncio@pytest.mark.parametrize("wrapped", [False, True])@pytest.mark.parametrize("boundary", ["sync", "dag"])@@ -122,7 +102,6 @@ async def test_storage_error_finishes_request_and_respects_lock_ownership(assert result.error == str(error)operation.assert_awaited_once()env.retry_queue.enqueue.assert_not_awaited()- await _assert_request_failed(env, msg, error)env.processor._account_breaker(msg.account_id).check()if owned:env.pathlock.pathlock_adopt.assert_awaited_once_with(msg.lock_handoff)@@ -171,10 +150,6 @@ async def test_queue_acks_failed_semantic_work(semantic_env, monkeypatch, wrappeawait asyncio.wait_for(queue.dequeue(), timeout=1)- assert backend.write.await_args_list == [- call("/queue/Semantic/enqueue", queued_payload.encode()),- call("/queue/Semantic/ack", b"queue-message"),- ]status = await queue.get_status()assert status.is_completeassert status.in_progress == status.processed == status.requeue_count == 0@@ -189,7 +164,6 @@ async def test_queue_acks_failed_semantic_work(semantic_env, monkeypatch, wrappefinalize.assert_not_awaited()env.retry_queue.enqueue.assert_not_awaited()env.pathlock.pathlock_release.assert_awaited_once_with(env.lease)- await _assert_request_failed(env, msg, error)@pytest.mark.asyncio@@ -202,17 +176,14 @@ async def test_malformed_resources_do_not_pause_following_valid_work(semantic_enresult = await asyncio.wait_for(env.processor.on_dequeue(msg.to_dict()), timeout=1)assert result.outcome is ProcessOutcome.FAILEDassert result.error == str(error)- await _assert_request_failed(env, msg, error)env.dag.run.side_effect = Nonevalid = env.message()result = await asyncio.wait_for(env.processor.on_dequeue(valid.to_dict()), timeout=1)assert result.outcome is ProcessOutcome.SUCCESS- await asyncio.wait_for(env.tracker.wait_for_request(valid.telemetry_id), timeout=1)env.retry_queue.enqueue.assert_not_awaited()assert env.dag.run.await_count == 5- assert env.tracker.build_queue_status(valid.telemetry_id)["Semantic"]["processed"] == 1env.processor._account_breaker(valid.account_id).check()@@ -233,8 +204,6 @@ async def test_lock_error_requeues_without_consuming_failure_budget(semantic_envrequeued = env.retry_queue.enqueue.call_args.args[0]assert requeued.id == msg.idassert requeued.failure_requeue_count == 0- assert not env.tracker.is_complete(msg.telemetry_id)- assert env.tracker.build_queue_status(msg.telemetry_id)["Semantic"]["requeue_count"] == 1breaker.check()sleep.assert_not_awaited()@@ -251,8 +220,6 @@ async def test_semantic_model_error_fails_without_requeue(semantic_env):assert result.outcome is ProcessOutcome.FAILEDassert result.error == str(error)env.retry_queue.enqueue.assert_not_awaited()- await _assert_request_failed(env, msg, error)- env.processor._account_breaker(msg.account_id).check()@pytest.mark.asyncio@@ -267,10 +234,7 @@ async def test_invalid_request_does_not_open_breaker(semantic_env):assert result.outcome is ProcessOutcome.FAILEDassert result.error == str(error)env.retry_queue.enqueue.assert_not_awaited()- await _assert_request_failed(env, msg, error)- breaker = env.processor._account_breaker(msg.account_id)- breaker.check()- assert breaker._failure_count == 0+ env.processor._account_breaker(msg.account_id).check()@pytest.mark.asyncio@@ -290,14 +254,12 @@ async def test_open_breaker_requeues_then_recovers(semantic_env, monkeypatch):env.dag.run.assert_not_awaited()env.retry_queue.enqueue.assert_awaited_once()sleep.assert_awaited_once_with(30)- assert not env.tracker.is_complete(msg.telemetry_id)with pytest.raises(CircuitBreakerOpen):breaker.check()now[0] += 301result = await asyncio.wait_for(env.processor.on_dequeue(msg.to_dict()), timeout=1)assert result.outcome is ProcessOutcome.SUCCESS- await asyncio.wait_for(env.tracker.wait_for_request(msg.telemetry_id), timeout=1)env.dag.run.assert_awaited_once_with(msg.uri)assert breaker.retry_after == 0diff --git a/tests/storage/test_semantic_processor_target_preexisting.py b/tests/storage/test_semantic_processor_target_preexisting.pyindex e3dc12e4b..278a67a82 100644--- a/tests/storage/test_semantic_processor_target_preexisting.py+++ b/tests/storage/test_semantic_processor_target_preexisting.py@@ -6,7 +6,6 @@ import pytestfrom openviking.storage.queuefs.semantic_msg import SemanticMsgfrom openviking.storage.queuefs.semantic_processor import SemanticProcessorfrom openviking.storage.viking_fs import SyncDiff-from openviking.telemetry.request_wait_tracker import RequestWaitTrackerdef _processor(**kwargs):@@ -259,107 +258,6 @@ async def test_embedding_lock_wait_skips_semantic_execution(monkeypatch):scope.close.assert_awaited_once()-async def test_batch_lock_coordinator_requeues_while_sibling_work_is_pending(monkeypatch):- monkeypatch.setattr(- "openviking.storage.queuefs.semantic_processor.get_viking_fs",- lambda: _FakeVikingFS(),- )- monkeypatch.setattr(- "openviking.storage.queuefs.semantic_processor.is_semantic_msg_stale",- lambda msg: False,- )- monkeypatch.setattr(- "openviking.storage.queuefs.semantic_processor.asyncio.sleep",- AsyncMock(),- )- tracker = RequestWaitTracker()- telemetry_id = "tm_batch_coordinator_requeue"- tracker.register_request(telemetry_id)- tracker.register_semantic_root(telemetry_id, "coordinator")- tracker.register_semantic_root(telemetry_id, "semantic-1")- monkeypatch.setattr(- "openviking.storage.queuefs.semantic_processor.get_request_wait_tracker",- lambda: tracker,- )-- work = SimpleNamespace(- scope=SimpleNamespace(lock={"id": "batch-lock"}),- start=lambda: None,- resolve_telemetry=lambda: None,- acquire_lock=AsyncMock(return_value=True),- reenqueue=AsyncMock(),- close=AsyncMock(),- )- processor = SemanticProcessor(vlm_resolver=None)- monkeypatch.setattr(processor, "_message_work", lambda msg, lock=None: work)- msg = SemanticMsg(- uri="viking://resources/wiki",- context_type="resource",- telemetry_id=telemetry_id,- lock_handoff={"owner_id": "producer"},- wait_for_request_completion=True,- )- msg.id = "coordinator"-- result = await processor.on_dequeue(msg.to_dict())-- assert result.outcome.value == "requeued"- work.reenqueue.assert_awaited_once()- work.close.assert_awaited_once()- assert tracker.build_queue_status(telemetry_id)["Semantic"]["requeue_count"] == 0-- tracker.mark_semantic_done(telemetry_id, "semantic-1")- tracker.mark_semantic_done(telemetry_id, "coordinator")- tracker.cleanup(telemetry_id)---async def test_batch_lock_coordinator_releases_after_sibling_work_settles(monkeypatch):- monkeypatch.setattr(- "openviking.storage.queuefs.semantic_processor.get_viking_fs",- lambda: _FakeVikingFS(),- )- monkeypatch.setattr(- "openviking.storage.queuefs.semantic_processor.SemanticTreeExecutor",- _FakeTreeExecutor,- )- scope = SimpleNamespace(lock={"id": "batch-lock"}, close=AsyncMock())- monkeypatch.setattr(- "openviking.storage.queuefs.semantic_processor.SemanticLockScope.resolve",- AsyncMock(return_value=scope),- )- tracker = RequestWaitTracker()- telemetry_id = "tm_batch_coordinator_release"- tracker.register_request(telemetry_id)- tracker.register_semantic_root(telemetry_id, "coordinator")- monkeypatch.setattr(- "openviking.storage.queuefs.semantic_processor.get_request_wait_tracker",- lambda: tracker,- )-- _FakeTreeExecutor.calls = []- _FakeTreeExecutor.runs = []- processor = SemanticProcessor(vlm_resolver=None)- msg = SemanticMsg(- uri="viking://resources/wiki",- context_type="resource",- telemetry_id=telemetry_id,- lock_handoff={"owner_id": "producer"},- wait_for_request_completion=True,- )- msg.id = "coordinator"-- result = await processor.on_dequeue(msg.to_dict())-- assert result.outcome.value == "success"- assert _FakeTreeExecutor.calls == []- scope.close.assert_awaited_once()- assert tracker.is_complete(telemetry_id) is True- assert tracker.build_queue_status(telemetry_id)["Semantic"]["processed"] == 0- tracker.cleanup(telemetry_id)--@pytest.mark.asyncio@pytest.mark.parametrize("recursive", [False, True])async def test_memory_reindex_uses_semantic_executor(monkeypatch, recursive):diff --git a/tests/storage/test_skill_package_cancellation.py b/tests/storage/test_skill_package_cancellation.pyindex 7b2c3171a..a23ddcdaf 100644--- a/tests/storage/test_skill_package_cancellation.py+++ b/tests/storage/test_skill_package_cancellation.py@@ -6,7 +6,6 @@ import jsonimport threadingfrom types import SimpleNamespacefrom unittest.mock import AsyncMock-from uuid import uuid4import pytest@@ -18,6 +17,7 @@ from openviking.service.task_work_index import (bind_task_context,get_task_context,prepare_task_payload,+ record_enqueued_work,)from openviking.storage.collection_schemas import TextEmbeddingHandlerfrom openviking.storage.queuefs.embedding_msg import EmbeddingMsg@@ -26,7 +26,6 @@ from openviking.storage.queuefs.process_result import ProcessOutcomefrom openviking.storage.queuefs.semantic_executor import SemanticTreeExecutor, SemanticTreeStatsfrom openviking.storage.queuefs.semantic_msg import SemanticMsgfrom openviking.storage.queuefs.semantic_processor import SemanticProcessor-from openviking.telemetry.request_wait_tracker import get_request_wait_trackerfrom openviking_cli.session.user_id import UserIdentifierfrom tests.storage.test_collection_schemas import _DummyConfig, _DummyEmbedderfrom tests.storage.test_semantic_executor_skip_files import _FakeVikingFS@@ -182,9 +181,6 @@ async def test_active_skill_embedding_cancel_settles_only_after_write_exit(),SimpleNamespace(bind=lambda _account_id: Embedder()),)- telemetry_id = str(uuid4())- tracker = get_request_wait_tracker()- tracker.register_request(telemetry_id)msg = EmbeddingMsg("content",{@@ -192,15 +188,17 @@ async def test_active_skill_embedding_cancel_settles_only_after_write_exit("context_type": "skill","account_id": "acc",},- telemetry_id=telemetry_id,)- tracker.register_embedding_root(telemetry_id, msg.id)index = TaskWorkIndex()cancelling = set()index.set_callbacks(finalize_before_ack=AsyncMock(),is_cancellation_requested=lambda task_id: task_id in cancelling,)+ monkeypatch.setattr(+ "openviking.storage.collection_schemas.get_task_tracker",+ lambda: SimpleNamespace(record_context_count=index.record_context_count),+ )with bind_task_context("skill-task", "acc", "alice"):payload, metadata = prepare_task_payload(msg.to_dict())queue = NamedQueue(@@ -220,7 +218,6 @@ async def test_active_skill_embedding_cancel_settles_only_after_write_exit(await asyncio.sleep(0.03)if during_write:assert not worker.done()- assert not tracker.is_complete(telemetry_id)assert index.has_work("skill-task")assert not writeselse:@@ -232,23 +229,26 @@ async def test_active_skill_embedding_cancel_settles_only_after_write_exit(assert result.outcome is ProcessOutcome.CANCELLEDawait index.prepare_ack(queue.name, envelope)assert not index.has_work("skill-task")- assert tracker.is_complete(telemetry_id)- tracker.cleanup(telemetry_id)assert len(writes) == int(during_write)@pytest.mark.asyncioasync def test_semantic_cancel_drains_embeddings_before_releasing_package(monkeypatch):- tracker = get_request_wait_tracker()- telemetry_id = str(uuid4())- tracker.register_request(telemetry_id)+ index = TaskWorkIndex()+ child = None+ index.set_callbacks(+ finalize_before_ack=AsyncMock(),+ is_cancellation_requested=lambda _task_id: False,+ )+ monkeypatch.setattr(+ "openviking.storage.queuefs.semantic_work.get_task_tracker",+ lambda: SimpleNamespace(wait_for_work=index.wait_for_work),+ )msg = SemanticMsg(uri="viking://agent/skills/demo",context_type="skill",- telemetry_id=telemetry_id,propagate_to_parent=False,)- tracker.register_semantic_root(telemetry_id, msg.id)started = asyncio.Event()events = []@@ -257,7 +257,12 @@ async def test_semantic_cancel_drains_embeddings_before_releasing_package(monkeyself.stale = Falseasync def run(self, uri):- tracker.register_embedding_root(telemetry_id, "embedding")+ nonlocal child+ from openviking.service.task_work_index import QueueTaskMetadata++ child = QueueTaskMetadata("skill-task", "embedding")+ assert index.register("Embedding", child)+ record_enqueued_work("Embedding", child.work_id)started.set()def get_stats(self):@@ -280,9 +285,10 @@ async def test_semantic_cancel_drains_embeddings_before_releasing_package(monkeylambda: SimpleNamespace(exists=AsyncMock(return_value=True)),)resolver = SimpleNamespace(get_vlm=AsyncMock())- worker = asyncio.create_task(- SemanticProcessor(vlm_resolver=resolver).on_dequeue({"data": msg.to_json()})- )+ with bind_task_context("skill-task", "acc", "alice", enqueued_work=[]):+ worker = asyncio.create_task(+ SemanticProcessor(vlm_resolver=resolver).on_dequeue({"data": msg.to_json()})+ )await asyncio.wait_for(started.wait(), 1)worker.cancel()await asyncio.sleep(0)@@ -290,11 +296,9 @@ async def test_semantic_cancel_drains_embeddings_before_releasing_package(monkeyawait asyncio.sleep(0.03)assert not worker.done()assert events == []- assert not tracker.is_complete(telemetry_id)events.append("embedding-settled")- tracker.mark_embedding_done(telemetry_id, "embedding")+ assert child is not None+ index.record_result(child, "completed")with pytest.raises(asyncio.CancelledError):await asyncio.wait_for(worker, 1)assert events == ["embedding-settled", "released"]- assert tracker.is_complete(telemetry_id)- tracker.cleanup(telemetry_id)diff --git a/tests/storage/test_skill_package_semantics.py b/tests/storage/test_skill_package_semantics.pydeleted file mode 100644index 7acf0d729..000000000--- a/tests/storage/test_skill_package_semantics.py+++ /dev/null@@ -1,79 +0,0 @@-import asyncio-from types import SimpleNamespace-from unittest.mock import AsyncMock--import pytest--from openviking.storage.queuefs.semantic_executor import SemanticTreeStats-from openviking.storage.queuefs.semantic_msg import SemanticMsg-from openviking.storage.queuefs.semantic_processor import SemanticProcessor-from openviking.telemetry import get_current_telemetry, register_telemetry-from openviking.telemetry.request_wait_tracker import get_request_wait_tracker---async def test_skill_worker_keeps_package_locked_until_embeddings_finish(monkeypatch):- root = "viking://agent/skills/demo"- emitted = asyncio.Event()- locked = True- events = []- tracker = get_request_wait_tracker()- telemetry = get_current_telemetry()- register_telemetry(telemetry)- tracker.register_request(telemetry.telemetry_id)- msg = SemanticMsg(- uri=root,- context_type="skill",- telemetry_id=telemetry.telemetry_id,- generation_trigger="skill_ingest",- propagate_to_parent=False,- )- tracker.register_semantic_root(msg.telemetry_id, msg.id)-- class Lease:- lock = {"lease_ref": "package"}-- async def close(self):- nonlocal locked- locked = False- events.append("released")-- class Executor:- def __init__(self, **kwargs):- self.stale = False-- async def run(self, uri):- tracker.register_embedding_root(msg.telemetry_id, "file-embedding")- emitted.set()-- def get_stats(self):- return SemanticTreeStats()-- monkeypatch.setattr(- "openviking.storage.queuefs.semantic_processor.SemanticTreeExecutor", Executor- )- monkeypatch.setattr(- "openviking.storage.queuefs.semantic_processor.SemanticLockScope.resolve",- AsyncMock(return_value=Lease()),- )- monkeypatch.setattr(- "openviking.storage.queuefs.semantic_processor.get_viking_fs",- lambda: SimpleNamespace(exists=AsyncMock(return_value=True)),- )- resolver = SimpleNamespace(get_vlm=AsyncMock(return_value=SimpleNamespace()))- worker = asyncio.create_task(- SemanticProcessor(vlm_resolver=resolver).on_dequeue({"data": msg.to_json()})- )- await asyncio.wait_for(emitted.wait(), 1)- assert locked and not worker.done() and not tracker.is_complete(msg.telemetry_id)- # The HTTP timeout cleanup must not release the package while embedding runs.- tracker.cleanup(msg.telemetry_id)- await asyncio.sleep(0.06)- assert locked and not worker.done() and not tracker.is_complete(msg.telemetry_id)- # Deletion cannot pass the package lease while a file vector can still write.- events.append("embedding-written")- tracker.mark_embedding_done(msg.telemetry_id, "file-embedding", vector_written=True)- await asyncio.wait_for(worker, 1)- assert not locked and tracker.is_complete(msg.telemetry_id)- assert events == ["embedding-written", "released"]- tracker.cleanup(msg.telemetry_id)diff --git a/tests/telemetry/test_request_wait_tracker.py b/tests/telemetry/test_request_wait_tracker.pydeleted file mode 100644index e72a13b44..000000000--- a/tests/telemetry/test_request_wait_tracker.py+++ /dev/null@@ -1,140 +0,0 @@-# Copyright (c) 2026 Beijing Volcano Engine Technology Co., Ltd.-# SPDX-License-Identifier: AGPL-3.0--import pytest--from openviking.telemetry.request_wait_tracker import RequestWaitTracker---def test_request_wait_tracker_cleanup_prevents_state_recreation():- tracker = RequestWaitTracker()- telemetry_id = "tm_cleanup"-- tracker.register_request(telemetry_id)- tracker.register_semantic_root(telemetry_id, "semantic-1")- tracker.cleanup(telemetry_id)-- tracker.mark_semantic_done(telemetry_id, "semantic-1")- tracker.mark_embedding_done(telemetry_id, "embedding-1")-- assert tracker.build_queue_status(telemetry_id) == {- "Semantic": {"processed": 0, "requeue_count": 0, "error_count": 0, "errors": []},- "Embedding": {"processed": 0, "requeue_count": 0, "error_count": 0, "errors": []},- }---def test_request_wait_tracker_cleanup_prevents_root_recreation():- tracker = RequestWaitTracker()- telemetry_id = "tm_late_root"-- tracker.register_request(telemetry_id)- tracker.cleanup(telemetry_id)-- tracker.register_semantic_root(telemetry_id, "semantic-1")- tracker.register_embedding_root(telemetry_id, "embedding-1")-- assert tracker.is_complete(telemetry_id) is True- assert tracker.build_queue_status(telemetry_id) == {- "Semantic": {"processed": 0, "requeue_count": 0, "error_count": 0, "errors": []},- "Embedding": {"processed": 0, "requeue_count": 0, "error_count": 0, "errors": []},- }---def test_request_wait_tracker_records_requeues():- tracker = RequestWaitTracker()- telemetry_id = "tm_requeue"-- tracker.register_request(telemetry_id)- tracker.record_semantic_requeue(telemetry_id)- tracker.record_embedding_requeue(telemetry_id, delta=2)-- assert tracker.build_queue_status(telemetry_id) == {- "Semantic": {"processed": 0, "requeue_count": 1, "error_count": 0, "errors": []},- "Embedding": {"processed": 0, "requeue_count": 2, "error_count": 0, "errors": []},- }---def test_request_wait_tracker_completes_except_coordinator_after_siblings_settle():- tracker = RequestWaitTracker()- telemetry_id = "tm_batch_coordinator"-- tracker.register_request(telemetry_id)- tracker.register_semantic_root(telemetry_id, "coordinator")- tracker.register_semantic_root(telemetry_id, "semantic-1")- tracker.register_embedding_root(telemetry_id, "embedding-1")-- assert tracker.is_complete_except(telemetry_id, "coordinator") is False-- tracker.mark_semantic_done(telemetry_id, "semantic-1")- assert tracker.is_complete_except(telemetry_id, "coordinator") is False-- tracker.mark_embedding_done(telemetry_id, "embedding-1")- assert tracker.is_complete_except(telemetry_id, "coordinator") is True- assert tracker.is_complete(telemetry_id) is False-- tracker.mark_semantic_done(telemetry_id, "coordinator")- tracker.cleanup(telemetry_id)---def test_request_wait_tracker_retains_coordinator_after_request_cleanup():- tracker = RequestWaitTracker()- telemetry_id = "tm_batch_cleanup"-- tracker.register_request(telemetry_id)- tracker.register_semantic_root(telemetry_id, "coordinator")- tracker.retain_request(telemetry_id, "coordinator")- tracker.cleanup(telemetry_id)-- assert tracker.has_request(telemetry_id) is True-- tracker.mark_semantic_done(telemetry_id, "coordinator")-- assert tracker.has_request(telemetry_id) is False---def test_request_wait_tracker_counts_contexts_indexed_by_request():- tracker = RequestWaitTracker()- telemetry_id = "tm_vectors"-- tracker.register_request(telemetry_id)- for root_id in ("embedding-1", "embedding-2", "embedding-3"):- tracker.register_embedding_root(telemetry_id, root_id)- tracker.mark_embedding_done(telemetry_id, root_id, vector_written=True)- tracker.mark_embedding_done(telemetry_id, "embedding-3", vector_written=True)-- assert tracker.get_embedding_context_count(telemetry_id) == 3-- tracker.cleanup(telemetry_id)-- assert tracker.get_embedding_context_count(telemetry_id) == 0---def test_request_wait_tracker_accumulates_queue_wait_and_execution_durations():- tracker = RequestWaitTracker()- telemetry_id = "tm_queue_timing"-- tracker.register_request(telemetry_id)- tracker.record_semantic_timing(telemetry_id, queue_wait_ms=12.5, execute_ms=30.0)- tracker.record_semantic_timing(telemetry_id, queue_wait_ms=7.5, execute_ms=10.0)- tracker.record_embedding_timing(telemetry_id, queue_wait_ms=5.0, execute_ms=8.0)-- assert tracker.get_queue_timing(telemetry_id) == {- "semantic": {"queue_wait_ms": 20.0, "execute_ms": 40.0},- "embedding": {"queue_wait_ms": 5.0, "execute_ms": 8.0},- }---async def test_wait_for_request_timeout_preserves_pending_work():- tracker = RequestWaitTracker()- telemetry_id = "tm_wait_timeout"-- tracker.register_request(telemetry_id)- tracker.register_embedding_root(telemetry_id, "embedding-1")-- with pytest.raises(TimeoutError):- await tracker.wait_for_request(telemetry_id, timeout=0.01)-- assert not tracker.is_complete(telemetry_id)- tracker.mark_embedding_done(telemetry_id, "embedding-1")- await tracker.wait_for_request(telemetry_id)- assert tracker.build_queue_status(telemetry_id)["Embedding"]["processed"] == 1diff --git a/tests/telemetry/test_resource_summary.py b/tests/telemetry/test_resource_summary.pydeleted file mode 100644index 076c644fb..000000000--- a/tests/telemetry/test_resource_summary.py+++ /dev/null@@ -1,69 +0,0 @@-from openviking.telemetry.operation import OperationTelemetry---def test_record_resource_queue_metrics_collects_queue_and_tree_stats(monkeypatch):- from openviking.telemetry.resource_summary import record_resource_queue_metrics-- telemetry = OperationTelemetry(operation="resources.add_resource", enabled=True)- telemetry_id = telemetry.telemetry_id-- class _SemanticStats:- processed = 7- requeue_count = 0- error_count = 2-- class _EmbeddingStats:- processed = 11- requeue_count = 0- error_count = 1-- class _Timing:- @staticmethod- def get_queue_timing(_tid):- return {- "semantic": {"queue_wait_ms": 12.5, "execute_ms": 30.0},- "embedding": {"queue_wait_ms": 8.0, "execute_ms": 20.0},- }-- class _SemanticTreeStats:- total_nodes = 9- done_nodes = 8- pending_nodes = 1- in_progress_nodes = 0-- monkeypatch.setattr(- "openviking.telemetry.resource_summary._consume_semantic_request_stats",- lambda _tid: _SemanticStats(),- )- monkeypatch.setattr(- "openviking.telemetry.resource_summary._consume_embedding_request_stats",- lambda _tid: _EmbeddingStats(),- )- monkeypatch.setattr(- "openviking.telemetry.resource_summary._consume_semantic_tree_stats",- lambda _tid, _uri: _SemanticTreeStats(),- )- monkeypatch.setattr(- "openviking.telemetry.resource_summary.get_request_wait_tracker",- lambda: _Timing(),- )-- record_resource_queue_metrics(- telemetry=telemetry,- telemetry_id=telemetry_id,- root_uri="viking://resources/demo",- )-- summary = telemetry.finish().summary- assert summary["queue"]["semantic"]["processed"] == 7- assert summary["queue"]["semantic"]["error_count"] == 2- assert summary["queue"]["embedding"]["processed"] == 11- assert summary["queue"]["embedding"]["error_count"] == 1- assert summary["queue"]["semantic"]["queue_wait"]["duration_ms"] == 12.5- assert summary["queue"]["semantic"]["execute"]["duration_ms"] == 30.0- assert summary["queue"]["embedding"]["queue_wait"]["duration_ms"] == 8.0- assert summary["queue"]["embedding"]["execute"]["duration_ms"] == 20.0- assert summary["semantic_nodes"]["total"] == 9- assert summary["semantic_nodes"]["done"] == 8- assert summary["semantic_nodes"]["pending"] == 1- assert "running" not in summary["semantic_nodes"]diff --git a/tests/test_task_tracker.py b/tests/test_task_tracker.pyindex f7ecfa33c..b322bfad3 100644--- a/tests/test_task_tracker.py+++ b/tests/test_task_tracker.py@@ -5,6 +5,7 @@import asyncioimport json+import threadingimport timefrom copy import deepcopy@@ -19,8 +20,15 @@ from openviking.service.task_tracker import (TaskTracker,_sanitize_error,get_task_tracker,+ run_tracked_operation,set_task_tracker,)+from openviking.service.task_work_index import (+ QueueTaskMetadata,+ TaskWorkIndex,+ get_task_context,+)+from openviking_cli.exceptions import DeadlineExceededErrorfrom openviking_cli.session.user_id import UserIdentifierpytestmark = pytest.mark.asyncio@@ -620,6 +628,294 @@ async def test_get_task_tracker_requires_service_initialization():get_task_tracker()+async def test_wait_timeout_preserves_recorded_operation_result(tracker):+ index = TaskWorkIndex()+ tracker.attach_work_index(index)+ set_task_tracker(tracker)+ state = {}++ async def operation():+ context = get_task_context()+ state["task_id"] = context.task_id+ work = QueueTaskMetadata(+ context.task_id,+ "child-work",+ context.account_id,+ context.user_id,+ )+ assert index.register("Semantic", work)+ return {"status": "success", "root_uri": "resource-result"}++ with pytest.raises(DeadlineExceededError) as caught:+ await run_tracked_operation(+ "content_write",+ account_id="acme",+ user_id="alice",+ operation=operation,+ wait=True,+ timeout=0,+ )++ task = await tracker.get(state["task_id"], **_owner_kwargs())+ assert "task_id" not in caught.value.details+ assert task.status is TaskStatus.RUNNING+ assert task.result == {+ "status": "success",+ "root_uri": "resource-result",+ "task_id": state["task_id"],+ }+++async def test_accept_failed_result_returns_structured_partial_result(tracker):+ index = TaskWorkIndex()+ tracker.attach_work_index(index)+ set_task_tracker(tracker)+ workers = []++ async def operation():+ context = get_task_context()+ work = QueueTaskMetadata(+ context.task_id,+ "embedding-work",+ context.account_id,+ context.user_id,+ )+ assert index.register("Embedding", work)++ async def fail_work():+ await asyncio.sleep(0)+ index.record_result(work, "failed", error="embedding failed")+ await index.prepare_ack("Embedding", work)++ workers.append(asyncio.create_task(fail_work()))+ return {+ "status": "completed",+ "failed_records": 1,+ "warnings": ["Embedding queue failed during reindex: embedding failed"],+ }++ result, task_id = await run_tracked_operation(+ "admin_reindex",+ account_id="acme",+ user_id="alice",+ operation=operation,+ wait=True,+ accept_failed_result=lambda value: (+ isinstance(value, dict) and value.get("failed_records", 0) > 0+ ),+ )++ assert result["failed_records"] == 1+ task = await tracker.wait(task_id, **_owner_kwargs(), timeout=1)+ assert task.status is TaskStatus.FAILED+ assert task.result["failed_records"] == 1+ await asyncio.gather(*workers)+++ ("raised", "expected_message"),+ [+ (TimeoutError("operation timed out"), "operation timed out"),+ (DeadlineExceededError("queue processing", 0.1), "Queue processing timed out after 0.1s"),+ ],+)+async def test_operation_timeout_marks_task_failed(tracker, raised, expected_message):+ set_task_tracker(tracker)+ state = {}++ async def operation():+ state["task_id"] = get_task_context().task_id+ raise raised++ with pytest.raises(type(raised)):+ await run_tracked_operation(+ "content_write",+ account_id="acme",+ user_id="alice",+ operation=operation,+ wait=True,+ timeout=0.1,+ )++ task = await tracker.get(state["task_id"], **_owner_kwargs())+ assert task.status is TaskStatus.FAILED+ assert task.error == expected_message+++async def test_wait_false_still_finishes_top_level_operation(tracker):+ set_task_tracker(tracker)+ started = asyncio.Event()+ release = asyncio.Event()++ async def operation():+ started.set()+ await release.wait()+ return {"status": "success", "value": 1}++ running = asyncio.create_task(+ run_tracked_operation(+ "content_write",+ account_id="acme",+ user_id="alice",+ operation=operation,+ wait=False,+ )+ )+ await asyncio.wait_for(started.wait(), timeout=1)+ assert not running.done()++ release.set()+ (result, task_id) = await asyncio.wait_for(running, timeout=1)+ assert result == {"status": "success", "value": 1, "task_id": task_id}+ completed = await tracker.get(task_id, **_owner_kwargs())+ assert completed.status is TaskStatus.COMPLETED+ assert completed.result == {+ "status": "success",+ "value": 1,+ "task_id": task_id,+ }++[email protected]("cancel_status", ["pending", "running"])+async def test_cancel_during_operation_startup_settles_persisted_task(cancel_status):+ """Cancellation must not orphan a task whose store write has already started."""+ started = asyncio.Event()+ release = threading.Event()+ loop = asyncio.get_running_loop()++ class BlockingAgfs(_FakeAgfs):+ def write(self, path, data, *args, **kwargs):+ if json.loads(data)["status"] == cancel_status:+ loop.call_soon_threadsafe(started.set)+ assert release.wait(3)+ return super().write(path, data, *args, **kwargs)++ agfs = BlockingAgfs()+ tracker = TaskTracker(PersistentTaskStore(agfs))+ set_task_tracker(tracker)+ operation_called = False++ async def operation():+ nonlocal operation_called+ operation_called = True+ return {"ok": True}++ running = asyncio.create_task(+ run_tracked_operation("content_write", **_owner_kwargs(), operation=operation, wait=False)+ )+ try:+ await asyncio.wait_for(started.wait(), 1)+ running.cancel()+ await asyncio.sleep(0)+ running.cancel()+ finally:+ release.set()+ with pytest.raises(asyncio.CancelledError):+ await asyncio.wait_for(running, 1)++ assert not operation_called+ persisted = await TaskTracker(PersistentTaskStore(agfs)).list_tasks(**_owner_kwargs())+ assert len(persisted) == 1+ assert persisted[0].status is TaskStatus.FAILED+ assert persisted[0].error == "CancelledError"+ assert (+ await tracker.wait(persisted[0].task_id, **_owner_kwargs(), timeout=0)+ ).status is TaskStatus.FAILED+++async def test_operation_start_failure_settles_created_task():+ class FailingAgfs(_FakeAgfs):+ def write(self, path, data, *args, **kwargs):+ if json.loads(data)["status"] == "running":+ raise OSError("start write failed")+ return super().write(path, data, *args, **kwargs)++ tracker = TaskTracker(PersistentTaskStore(FailingAgfs()))+ set_task_tracker(tracker)++ async def operation():+ pytest.fail("operation must not run after start fails")++ with pytest.raises(OSError, match="start write failed"):+ await run_tracked_operation(+ "content_write", **_owner_kwargs(), operation=operation, wait=False+ )++ tasks = await tracker.list_tasks(**_owner_kwargs())+ assert len(tasks) == 1+ assert tasks[0].status is TaskStatus.FAILED+ assert tasks[0].error == "start write failed"++[email protected]("timeout", [0, 0.01])+async def test_wait_timeout_excludes_operation_time_for_completed_task(tracker, timeout):+ set_task_tracker(tracker)++ async def operation():+ await asyncio.sleep(0.02)+ return {"status": "success"}++ result, task_id = await run_tracked_operation(+ "content_write",+ account_id="acme",+ user_id="alice",+ operation=operation,+ wait=True,+ timeout=timeout,+ )++ assert result == {"status": "success"}+ completed = await tracker.wait(task_id, **_owner_kwargs(), timeout=0)+ assert completed.status is TaskStatus.COMPLETED+ assert completed.result["status"] == "success"+ assert completed.result["task_id"] == task_id+++async def test_multiple_queue_waits_share_budget_and_leave_task_pollable(tracker):+ index = TaskWorkIndex()+ tracker.attach_work_index(index)+ set_task_tracker(tracker)+ state = {}++ async def operation():+ task_id = get_task_context().task_id+ state["task_id"] = task_id+ first = QueueTaskMetadata(task_id, "first", "acme", "alice")+ assert index.register("Semantic", first)+ # Exhaust the budget on a caught wait, then make the first Work complete.+ with pytest.raises(TimeoutError):+ await tracker.wait_for_work(task_id, [first.work_id], timeout=0.01)+ index.record_result(first, "completed")+ await index.prepare_ack("Semantic", first)+ second = QueueTaskMetadata(task_id, "second", "acme", "alice")+ assert index.register("Embedding", second)+ state["second"] = second+ with pytest.raises(TimeoutError):+ await tracker.wait_for_task_work(task_id, timeout=60)+ return {"status": "success"}++ with pytest.raises(DeadlineExceededError) as caught:+ await asyncio.wait_for(+ run_tracked_operation(+ "content_write",+ **_owner_kwargs(),+ operation=operation,+ wait=True,+ timeout=0.01,+ ),+ timeout=1,+ )+ assert "task_id" not in caught.value.details+ task = await tracker.get(state["task_id"], **_owner_kwargs())+ assert task.status is TaskStatus.RUNNING+ assert task.error is None+ index.record_result(state["second"], "completed")+ await index.prepare_ack("Embedding", state["second"])+ assert (+ await tracker.wait(task.task_id, **_owner_kwargs(), timeout=0)+ ).status is TaskStatus.COMPLETED++async def test_persistent_store_cross_tracker_visibility():agfs = _FakeAgfs()store = PersistentTaskStore(agfs)@@ -896,27 +1192,11 @@ async def test_terminal_event_waits_for_owned_work(tracker, outcome):)snapshot = await tracker.get(task.task_id, **owner)assert snapshot.status == TaskStatus.RUNNING- waiting = asyncio.create_task(tracker.wait_for_descendants(task.task_id, "parent-work"))-- # Observe the public history while work remains, not a helper call sequence.- async def observed_wait():- while True:- record = await tracker.get(task.task_id, **owner)- if record.execution_events["items"][-1]["kind"] == "waiting_for_descendants":- return- await asyncio.sleep(0)-- try:- await asyncio.wait_for(observed_wait(), timeout=2)- finally:- await work_index.prepare_ack("Semantic", child)- await waiting+ await work_index.prepare_ack("Semantic", child)snapshot = await tracker.get(task.task_id, **owner)last = snapshot.execution_events["items"][-1]assert last["kind"] == "status_changed"assert last["status"] == ("completed" if outcome == "complete" else "failed")- await tracker.wait_for_descendants(task.task_id, "parent-work")- assert (await tracker.get(task.task_id, **owner)).execution_events == snapshot.execution_eventsasync def test_legacy_tasks_do_not_get_invented_history():diff --git a/tests/unit/test_summarizer_resources_root_split.py b/tests/unit/test_summarizer_resources_root_split.pyindex cab5eed99..905525dc9 100644--- a/tests/unit/test_summarizer_resources_root_split.py+++ b/tests/unit/test_summarizer_resources_root_split.py@@ -35,18 +35,6 @@ class _DummyQueueManager:return self._queue-class _DummyWaitTracker:- def __init__(self):- self.registered = []- self.failed = []-- def register_semantic_root(self, telemetry_id, msg_id):- self.registered.append((telemetry_id, msg_id))-- def mark_semantic_failed(self, telemetry_id, msg_id, error):- self.failed.append((telemetry_id, msg_id, error))--class _DummyVikingFS:def __init__(self, entries_by_uri):self.entries_by_uri = entries_by_uri@@ -63,7 +51,7 @@ async def test_resources_root_is_split_into_children():{"viking://temp/import_root": [{"name": "existing_a", "isDir": True},- {"name": "new_c", "isDir": True},+ {"name": "file.txt", "isDir": False},]})@@ -75,14 +63,11 @@ async def test_resources_root_is_split_into_children():"openviking.utils.summarizer.get_current_telemetry",return_value=SimpleNamespace(telemetry_id="tid"),),- patch(- "openviking.utils.summarizer.get_request_wait_tracker", return_value=_DummyWaitTracker()- ),patch("openviking.utils.summarizer.get_viking_fs", return_value=vfs),):summarizer = Summarizer(vlm_processor=None)res = await summarizer.summarize(- resource_uris=["viking://resources"],+ resource_uris=["viking://resources/"],ctx=ctx,temp_uris=["viking://temp/import_root"],)@@ -91,45 +76,14 @@ async def test_resources_root_is_split_into_children():assert res["enqueued_count"] == 2assert [m.target_uri for m in queue.msgs] == ["viking://resources/existing_a",- "viking://resources/new_c",+ "viking://resources/file.txt",]assert [m.uri for m in queue.msgs] == ["viking://temp/import_root/existing_a",- "viking://temp/import_root/new_c",+ "viking://temp/import_root/file.txt",]-async def test_resources_root_single_file_child():- queue = _DummyQueue()- qm = _DummyQueueManager(queue)- vfs = _DummyVikingFS({"viking://temp/import_root": [{"name": "file.txt", "isDir": False}]})- ctx = RequestContext(user=UserIdentifier.the_default_user(), role=Role.ROOT)-- with (- patch("openviking.utils.summarizer.get_queue_manager", return_value=qm),- patch(- "openviking.utils.summarizer.get_current_telemetry",- return_value=SimpleNamespace(telemetry_id="tid"),- ),- patch(- "openviking.utils.summarizer.get_request_wait_tracker", return_value=_DummyWaitTracker()- ),- patch("openviking.utils.summarizer.get_viking_fs", return_value=vfs),- ):- summarizer = Summarizer(vlm_processor=None)- res = await summarizer.summarize(- resource_uris=["viking://resources/"],- ctx=ctx,- temp_uris=["viking://temp/import_root"],- )-- assert res["status"] == "success"- assert res["enqueued_count"] == 1- assert queue.msgs[0].target_uri == "viking://resources/file.txt"- assert queue.msgs[0].uri == "viking://temp/import_root/file.txt"--@pytest.mark.asyncioasync def test_explicit_subpath_not_split():queue = _DummyQueue()@@ -143,9 +97,6 @@ async def test_explicit_subpath_not_split():"openviking.utils.summarizer.get_current_telemetry",return_value=SimpleNamespace(telemetry_id="tid"),),- patch(- "openviking.utils.summarizer.get_request_wait_tracker", return_value=_DummyWaitTracker()- ),patch("openviking.utils.summarizer.get_viking_fs", return_value=vfs),):summarizer = Summarizer(vlm_processor=None)@@ -181,9 +132,6 @@ async def test_resources_root_empty_import_is_error():"openviking.utils.summarizer.get_current_telemetry",return_value=SimpleNamespace(telemetry_id="tid"),),- patch(- "openviking.utils.summarizer.get_request_wait_tracker", return_value=_DummyWaitTracker()- ),patch("openviking.utils.summarizer.get_viking_fs", return_value=vfs),):summarizer = Summarizer(vlm_processor=None)@@ -201,7 +149,6 @@ async def test_resources_root_empty_import_is_error():async def test_flat_file_refresh_enqueues_incremental_parent_summary():queue = _DummyQueue()qm = _DummyQueueManager(queue)- wait_tracker = _DummyWaitTracker()ctx = RequestContext(user=UserIdentifier.the_default_user(), role=Role.ROOT)with (@@ -210,10 +157,6 @@ async def test_flat_file_refresh_enqueues_incremental_parent_summary():"openviking.utils.summarizer.get_current_telemetry",return_value=SimpleNamespace(telemetry_id="tid"),),- patch(- "openviking.utils.summarizer.get_request_wait_tracker",- return_value=wait_tracker,- ),):result = await Summarizer(vlm_processor=None).refresh_file_parent(file_uri="viking://resources/神雕.md",@@ -234,14 +177,12 @@ async def test_flat_file_refresh_enqueues_incremental_parent_summary():assert msg.skip_vectorization is Falseassert msg.role == Role.ROOTassert msg.telemetry_id == "tid"- assert wait_tracker.registered == [("tid", msg.id)]@pytest.mark.asyncioasync def test_grouped_file_refresh_coalesces_only_direct_parents():queue = _DummyQueue()qm = _DummyQueueManager(queue)- wait_tracker = _DummyWaitTracker()ctx = RequestContext(user=UserIdentifier.the_default_user(), role=Role.ROOT)with (@@ -250,10 +191,6 @@ async def test_grouped_file_refresh_coalesces_only_direct_parents():"openviking.utils.summarizer.get_current_telemetry",return_value=SimpleNamespace(telemetry_id="tid"),),- patch(- "openviking.utils.summarizer.get_request_wait_tracker",- return_value=wait_tracker,- ),):result = await Summarizer(vlm_processor=None).refresh_file_parents(changes=[@@ -298,9 +235,8 @@ async def test_grouped_file_refresh_coalesces_only_direct_parents():@pytest.mark.asyncio-async def test_flat_file_refresh_marks_wait_failed_when_enqueue_fails():+async def test_flat_file_refresh_propagates_enqueue_failure():qm = _DummyQueueManager(_FailingQueue())- wait_tracker = _DummyWaitTracker()ctx = RequestContext(user=UserIdentifier.the_default_user(), role=Role.ROOT)with (@@ -309,18 +245,9 @@ async def test_flat_file_refresh_marks_wait_failed_when_enqueue_fails():"openviking.utils.summarizer.get_current_telemetry",return_value=SimpleNamespace(telemetry_id="tid"),),- patch(- "openviking.utils.summarizer.get_request_wait_tracker",- return_value=wait_tracker,- ),):with pytest.raises(RuntimeError, match="queue unavailable"):await Summarizer(vlm_processor=None).refresh_file_parent(file_uri="viking://resources/神雕.md",ctx=ctx,)-- assert len(wait_tracker.registered) == 1- assert wait_tracker.failed == [- (*wait_tracker.registered[0], "queue unavailable"),- ]diff --git a/tests/unit/test_vectorize_file_strategy.py b/tests/unit/test_vectorize_file_strategy.pyindex 059439063..1f92b93b8 100644--- a/tests/unit/test_vectorize_file_strategy.py+++ b/tests/unit/test_vectorize_file_strategy.py@@ -356,82 +356,6 @@ async def test_vectorize_image_uses_supplied_bytes_without_second_read(monkeypatassert fs.read_file_bytes_calls == 0-async def test_vectorize_file_registers_request_wait_with_embedding_msg_id(monkeypatch):- queue = DummyQueueWithId()- registered = []- monkeypatch.setattr(embedding_utils, "get_queue_manager", lambda: DummyQueueManager(queue))- monkeypatch.setattr(embedding_utils, "get_viking_fs", lambda: DummyFS("hello"))- monkeypatch.setattr(- embedding_utils,- "get_openviking_config",- lambda: types.SimpleNamespace(- embedding=types.SimpleNamespace(text_source="content_only", max_input_tokens=1000)- ),- )- monkeypatch.setattr(- embedding_utils,- "get_request_wait_tracker",- lambda: types.SimpleNamespace(- register_embedding_root=lambda telemetry_id, root_id: registered.append(- (telemetry_id, root_id)- )- ),- )-- await embedding_utils.vectorize_file(- file_path="viking://user/default/resources/test.md",- summary_dict={"name": "test.md", "summary": ""},- parent_uri="viking://user/default/resources",- ctx=DummyReq(),- )-- assert len(queue.items) == 1- assert registered == [(queue.items[0].telemetry_id, queue.items[0].id)]- assert registered[0][1] != "queue-message-id"---async def test_vectorize_file_marks_registered_wait_root_failed_when_enqueue_raises(monkeypatch):- queue = FailingQueue()- registered = []- failed = []- monkeypatch.setattr(embedding_utils, "get_queue_manager", lambda: DummyQueueManager(queue))- monkeypatch.setattr(embedding_utils, "get_viking_fs", lambda: DummyFS("hello"))- monkeypatch.setattr(- embedding_utils,- "get_openviking_config",- lambda: types.SimpleNamespace(- embedding=types.SimpleNamespace(text_source="content_only", max_input_tokens=1000)- ),- )- monkeypatch.setattr(- embedding_utils,- "get_request_wait_tracker",- lambda: types.SimpleNamespace(- register_embedding_root=lambda telemetry_id, root_id: registered.append(- (telemetry_id, root_id)- ),- mark_embedding_failed=lambda telemetry_id, root_id, message: failed.append(- (telemetry_id, root_id, message)- ),- ),- )-- with pytest.raises(RuntimeError, match="queue unavailable"):- await embedding_utils.vectorize_file(- file_path="viking://user/default/resources/test.md",- summary_dict={"name": "test.md", "summary": ""},- parent_uri="viking://user/default/resources",- ctx=DummyReq(),- )-- assert len(queue.items) == 1- assert registered == [(queue.items[0].telemetry_id, queue.items[0].id)]- assert failed- assert failed[0][0:2] == registered[0]--@pytest.mark.asyncioasync def test_vectorize_file_propagates_enqueue_failure(monkeypatch):monkeypatch.setattr(