Self-evolving Context Database for AI Agents. Unify Agent Memory, Knowledge RAG and Skills.
Python agent-contextagent-memoryagent-pluginsagentic-ragcontext-databasedsh-pluginknowledge-basememorymemory-managementself-evolvingvector-database
refactor(tasks): converge task/work store contract (#6136)
runyunzhou013e988bb3d8
diff --git a/openviking/server/routers/tasks.py b/openviking/server/routers/tasks.pyindex b40c82989..e941f0f3d 100644--- a/openviking/server/routers/tasks.py+++ b/openviking/server/routers/tasks.py@@ -104,10 +104,6 @@ async def list_tasks(if pagination == "cursor":from openviking.service.task_pagination import decode_cursor, encode_cursor, page_scope- # ROOT retains the legacy system + cached visibility and its own- # persisted tasks, including submissions restored after restart.- account = SYSTEM_TASK_ACCOUNT_ID if _ctx.role == Role.ROOT else _ctx.account_id- user = SYSTEM_TASK_USER_ID if _ctx.role == Role.ROOT else _ctx.user.user_idfilters = {"task_type": task_type,"status": status,@@ -118,17 +114,32 @@ async def list_tasks(scope = page_scope(account=_ctx.account_id, user=_ctx.user.user_id, role=str(_ctx.role), **filters)- tasks = await tracker.list_page(- account_id=account,- user_id=user,- limit=limit + 1,- before=decode_cursor(cursor, scope),- include_cached=_ctx.role == Role.ROOT,- additional_owner=(_ctx.account_id, _ctx.user.user_id)+ list_call = (+ tracker.list_root_tasks(+ account_id=_ctx.account_id,+ user_id=_ctx.user.user_id,+ limit=limit + 1,+ before=decode_cursor(cursor, scope),+ task_type=task_type,+ status=status,+ resource_id=resource_id,+ include_internal=include_internal,+ q=q,+ )if _ctx.role == Role.ROOT- else None,- **filters,+ else tracker.list_tasks(+ account_id=_ctx.account_id,+ user_id=_ctx.user.user_id,+ limit=limit + 1,+ before=decode_cursor(cursor, scope),+ task_type=task_type,+ status=status,+ resource_id=resource_id,+ include_internal=include_internal,+ q=q,+ ))+ tasks = await list_callmore = len(tasks) > limititems = [t.to_dict() for t in tasks[:limit]]return Response(diff --git a/openviking/service/deletion.py b/openviking/service/deletion.pyindex 6a86b73d2..708a8255c 100644--- a/openviking/service/deletion.py+++ b/openviking/service/deletion.py@@ -319,8 +319,13 @@ class DeletionService:if scheduler is not None:await run_to_completion(lambda: scheduler.delete_tasks(account_id, user_id))await run_to_completion(lambda: self._cancel_tasks(account_id, user_id))- if user_id is not None:- await run_to_completion(lambda: tracker.delete_user_tasks(account_id, user_id))+ await run_to_completion(+ lambda: tracker.delete_tasks(+ account_id,+ user_id,+ allow_active_types={"user_delete"} if user_id is None else (),+ )+ )vectors = self._service.viking_fs.vector_storeif vectors is not None:@@ -383,7 +388,6 @@ class DeletionService:await agfs.stat(path)except Exception as exc:if is_not_found_error(exc):- get_task_tracker().forget_account_tasks(account_id)returnraiseraise RuntimeError("Account directory still exists after deletion")diff --git a/openviking/service/external_task_service.py b/openviking/service/external_task_service.pyindex c913ee861..c075ec561 100644--- a/openviking/service/external_task_service.py+++ b/openviking/service/external_task_service.py@@ -343,7 +343,9 @@ class ExternalTaskService:await self._execute_task(task, provider, payload, account_id, user_id)return Trueexcept asyncio.CancelledError:- if not tracker.is_cancellation_requested(task_id):+ if not await tracker.is_cancellation_requested(+ task_id, account_id=account_id, user_id=user_id+ ):raiseawait run_to_completion(lambda: self.cancel_recovered(task_id, account_id, user_id))raisediff --git a/openviking/service/fs_service.py b/openviking/service/fs_service.pyindex 89c453a79..c435476f6 100644--- a/openviking/service/fs_service.py+++ b/openviking/service/fs_service.py@@ -746,7 +746,7 @@ class FSService:)except (TimeoutError, asyncio.TimeoutError) as exc:raise DeadlineExceededError("queue processing", None) from exc- return tracker.build_queue_status(+ return await tracker.build_queue_status(context.task_id, work_ids if context.work_id is not None else None)diff --git a/openviking/service/memory_compile.py b/openviking/service/memory_compile.pyindex 25ae7e491..3c527ed64 100644--- a/openviking/service/memory_compile.py+++ b/openviking/service/memory_compile.py@@ -192,7 +192,11 @@ class MemoryCompileRunner:ctx: RequestContext,) -> None:tracker = get_task_tracker()- tracker.register_running_task(task_id)+ running_work_id = await tracker.register_running_task(+ task_id,+ account_id=ctx.account_id,+ user_id=ctx.user.user_id,+ )root_token = Noneif bind_root_observability_context is not None:root_attrs = create_root_span_attributes(@@ -213,7 +217,12 @@ class MemoryCompileRunner:user_id=ctx.user.user_id,stage="consolidating",)- with bind_task_context(task_id, ctx.account_id, ctx.user.user_id):+ with bind_task_context(+ task_id,+ ctx.account_id,+ ctx.user.user_id,+ work_id=running_work_id,+ ):result = await self._consolidate(target=target,memory_type=memory_type,diff --git a/openviking/service/reindex_executor.py b/openviking/service/reindex_executor.pyindex 8c7c9b574..671ac36ac 100644--- a/openviking/service/reindex_executor.py+++ b/openviking/service/reindex_executor.py@@ -685,7 +685,7 @@ class ReindexExecutor:)self._apply_embedding_wait_status(counters,- get_task_tracker().build_queue_status(task_context.task_id),+ await get_task_tracker().build_queue_status(task_context.task_id),)finally:active_lease = run.lease if run is not None else leasediff --git a/openviking/service/resource_service.py b/openviking/service/resource_service.pyindex b8f08bc71..df8a3c1c1 100644--- a/openviking/service/resource_service.py+++ b/openviking/service/resource_service.py@@ -2463,9 +2463,18 @@ class ResourceService:await task_tracker.start(task.task_id, account_id=ctx.account_id, user_id=ctx.user.user_id)- task_tracker.register_running_task(task.task_id)+ running_work_id = await task_tracker.register_running_task(+ task.task_id,+ account_id=ctx.account_id,+ user_id=ctx.user.user_id,+ )try:- with bind_task_context(task.task_id, ctx.account_id, ctx.user.user_id):+ with bind_task_context(+ task.task_id,+ ctx.account_id,+ ctx.user.user_id,+ work_id=running_work_id,+ ):if isinstance(data, SkillProcessingPreparation):result = await self._skill_processor.process_prepared_skill(data,@@ -2543,7 +2552,11 @@ class ResourceService:"queue.wait.duration_ms",round((time.perf_counter() - wait_start) * 1000, 3),)- status = task_tracker.build_queue_status(task.task_id)+ status = await task_tracker.build_queue_status(+ task.task_id,+ account_id=ctx.account_id,+ user_id=ctx.user.user_id,+ )result["queue_status"] = statusself._raise_queue_status_errors(status)# Cancellation settles queue entries without counting errors.@@ -2582,7 +2595,11 @@ class ResourceService:return # Synchronous preparation failed before the task was created.# This includes queued work that must be discarded and active handlers# that must finish their cancellation/write cleanup, not just task status.- while tracker.has_work(task_id):+ while await tracker.has_work(+ task_id,+ account_id=ctx.account_id,+ user_id=ctx.user.user_id,+ ):await asyncio.sleep(0.05)async def build_index(diff --git a/openviking/service/task_domain.py b/openviking/service/task_domain.pynew file mode 100644index 000000000..6aa5e3d0f--- /dev/null+++ b/openviking/service/task_domain.py@@ -0,0 +1,256 @@+# Copyright (c) 2026 Beijing Volcano Engine Technology Co., Ltd.+# SPDX-License-Identifier: AGPL-3.0+"""Persistence-neutral Task and Work records."""++from __future__ import annotations++import math+import re+import time+from copy import deepcopy+from dataclasses import dataclass, field+from datetime import datetime, timezone+from enum import Enum+from typing import Any, Dict, Iterable, Optional+++class TaskStatus(str, Enum):+ PENDING = "pending"+ RUNNING = "running"+ CANCELLING = "cancelling"+ COMPLETED = "completed"+ FAILED = "failed"+ CANCELLED = "cancelled"+++class WorkState(str, Enum):+ PENDING = "pending"+ RUNNING = "running"+ DONE = "done"+ FAILED = "failed"+++class WorkRecoveryMode(str, Enum):+ """How a Store rebuilds Work state after process startup."""++ QUEUE_SNAPSHOT = "queue_snapshot"+ DURABLE_STORE = "durable_store"+++TERMINAL_TASK_STATUSES = (+ TaskStatus.COMPLETED,+ TaskStatus.FAILED,+ TaskStatus.CANCELLED,+)+ACTIVE_TASK_STATUSES = (+ TaskStatus.PENDING,+ TaskStatus.RUNNING,+ TaskStatus.CANCELLING,+)+TERMINAL_WORK_STATES = (WorkState.DONE, WorkState.FAILED)+MAX_QUEUE_ERROR_DETAILS = 100+++class TaskWorkRejected(Exception):+ """Task lifecycle state prevented Work registration."""+++@dataclass(frozen=True)+class TaskRetentionPolicy:+ completed_seconds: float+ failed_seconds: float++ def is_expired(self, status: str, updated_at: float, now: float) -> bool:+ if status in ("completed", "cancelled"):+ return now - updated_at > self.completed_seconds+ return status == "failed" and now - updated_at > self.failed_seconds+++@dataclass(frozen=True)+class DeletedTask:+ task_id: str+ account_id: str+ user_id: str+++@dataclass(frozen=True)+class DeleteExpiredResult:+ deleted: tuple[DeletedTask, ...] = ()+ scanned: int = 0+ scan_complete: bool = False++ @property+ def deleted_count(self) -> int:+ return len(self.deleted)+++@dataclass+class QueueWorkSummary:+ processed: int = 0+ requeue_count: int = 0+ error_count: int = 0+ errors: list[str] = field(default_factory=list)+ queue_wait_ms: float = 0.0+ execute_ms: float = 0.0+++@dataclass+class WorkSummary:+ queues: Dict[str, QueueWorkSummary] = field(default_factory=dict)+ context_count: int = 0+++SENSITIVE_TASK_KEYS = frozenset({"api_key", "user_key"})+_SENSITIVE_PATTERNS = re.compile(+ r"(sk-|cr_|ghp_|ntn_|xox[baprs]-|Bearer\s+)[a-zA-Z0-9._-]+",+ re.IGNORECASE,+)+_MAX_ERROR_LEN = 500+++def sanitize_error(error: str) -> str:+ sanitized = _SENSITIVE_PATTERNS.sub("[REDACTED]", error)+ if len(sanitized) > _MAX_ERROR_LEN:+ sanitized = sanitized[:_MAX_ERROR_LEN] + "...[truncated]"+ return sanitized+++def sanitize_task_result(result: Any) -> Any:+ if isinstance(result, dict):+ return {+ key: sanitize_task_result(value)+ for key, value in result.items()+ if key not in SENSITIVE_TASK_KEYS+ }+ if isinstance(result, list):+ return [sanitize_task_result(item) for item in result]+ return result+++@dataclass+class TaskRecord:+ task_id: str+ task_type: str+ status: TaskStatus = TaskStatus.PENDING+ created_at: float = field(default_factory=time.time)+ updated_at: float = field(default_factory=time.time)+ resource_id: Optional[str] = None+ account_id: Optional[str] = None+ user_id: Optional[str] = None+ meta: Dict[str, Any] = field(default_factory=dict)+ stage: Optional[str] = None+ result: Optional[Dict[str, Any]] = None+ error: Optional[str] = None+ auth: Dict[str, Any] = field(default_factory=dict, repr=False)+ execution_events: Optional[Any] = None+ processing_seconds: Optional[float] = None+ _extra_fields: Dict[str, Any] = field(+ default_factory=dict, init=False, repr=False, compare=False+ )++ def to_dict(self, *, include_events: bool = False) -> Dict[str, Any]:+ public_meta = deepcopy(self.meta)+ public_meta.pop("submission_hash", None)+ public_meta.pop("submission_token", None)+ return {+ **({"execution_events": deepcopy(self.execution_events)} if include_events else {}),+ "task_id": self.task_id,+ "task_type": self.task_type,+ "status": self.status.value,+ "created_at": self.created_at,+ "updated_at": self.updated_at,+ "processing_seconds": self.processing_seconds,+ "resource_id": self.resource_id,+ "meta": sanitize_task_result(public_meta),+ "stage": self.stage,+ "result": sanitize_task_result(deepcopy(self.result)),+ "error": self.error,+ "created_at_iso": datetime.fromtimestamp(self.created_at, tz=timezone.utc).isoformat(),+ "updated_at_iso": datetime.fromtimestamp(self.updated_at, tz=timezone.utc).isoformat(),+ }++ def is_terminal(self) -> bool:+ return self.status in TERMINAL_TASK_STATUSES++ def is_active(self) -> bool:+ return self.status in ACTIVE_TASK_STATUSES++ @staticmethod+ def next_updated_at(previous: float) -> float:+ return max(time.time(), math.nextafter(previous, math.inf))+++@dataclass+class WorkRecord:+ work_id: str+ task_id: str+ queue_name: str+ state: WorkState = WorkState.PENDING+ requeue_count: int = 0+ error: Optional[str] = None+ errors: list[str] = field(default_factory=list)+ context_count: int = 0+ enqueued_at: float = field(default_factory=time.time)+ started_at: Optional[float] = None+ finished_at: Optional[float] = None+ updated_at: float = field(default_factory=time.time)++ def is_open(self) -> bool:+ return self.state not in TERMINAL_WORK_STATES+++def processing_seconds(+ works: Iterable[WorkRecord], *, now: Optional[float] = None+) -> Optional[float]:+ """Return the wall-clock union of claimed Work intervals."""+ current = time.time() if now is None else now+ intervals = sorted(+ (+ work.started_at,+ max(+ work.started_at,+ work.finished_at if work.finished_at is not None else current,+ ),+ )+ for work in works+ if work.started_at is not None+ )+ if not intervals:+ return None+ elapsed = 0.0+ left, right = intervals[0]+ for start, end in intervals[1:]:+ if start <= right:+ right = max(right, end)+ else:+ elapsed += right - left+ left, right = start, end+ return elapsed + right - left+++def summarize_works(works: Iterable[WorkRecord]) -> WorkSummary:+ """Aggregate persisted Work facts without imposing an API response shape."""+ summary = WorkSummary()+ for work in works:+ bucket = summary.queues.setdefault(work.queue_name, QueueWorkSummary())+ bucket.requeue_count += work.requeue_count+ summary.context_count += work.context_count+ if work.state == WorkState.DONE:+ bucket.processed += 1+ elif work.state == WorkState.FAILED:+ errors = work.errors or ([work.error] if work.error else [])+ bucket.error_count += len(errors) or 1+ bucket.errors.extend(errors)+ if work.started_at is None:+ continue+ bucket.queue_wait_ms += max(0.0, (work.started_at - work.enqueued_at) * 1000.0)+ if work.finished_at is not None:+ bucket.execute_ms += max(0.0, (work.finished_at - work.started_at) * 1000.0)+ return summary+++def first_work_error(works: Iterable[WorkRecord]) -> Optional[str]:+ return next(+ (work.error for work in works if work.state == WorkState.FAILED and work.error),+ None,+ )diff --git a/openviking/service/task_queue_middleware.py b/openviking/service/task_queue_middleware.pyindex c542cae9a..3f545bd4c 100644--- a/openviking/service/task_queue_middleware.py+++ b/openviking/service/task_queue_middleware.py@@ -1,11 +1,14 @@# Copyright (c) 2026 Beijing Volcano Engine Technology Co., Ltd.# SPDX-License-Identifier: AGPL-3.0-"""Adapt the existing in-memory TaskWorkIndex to queue lifecycle middleware."""+"""Bridge QueueFS lifecycle events to Store-authoritative Task Work."""import asyncio+from typing import TYPE_CHECKING+from openviking.service.task_domain import TERMINAL_TASK_STATUSES+from openviking.service.task_store import WorkRegistration+from openviking.service.task_tracker_concurrency import KeyedAsyncLockPool, run_to_completionfrom openviking.service.task_work_index import (- TaskWorkIndex,TaskWorkRejected,bind_task_context,extract_task_metadata,@@ -14,8 +17,6 @@ from openviking.service.task_work_index import ()from openviking.storage.queuefs.process_result import ProcessOutcome, ProcessResultfrom openviking.storage.queuefs.queue_middleware import (- AckContext,- AckNext,ClearContext,ClearNext,EnqueueContext,@@ -25,123 +26,260 @@ from openviking.storage.queuefs.queue_middleware import (QueueMiddleware,)+if TYPE_CHECKING:+ from openviking.service.task_tracker import TaskTracker+ from openviking.service.task_work_index import QueueTaskMetadata+class TaskWorkQueueMiddleware(QueueMiddleware):- """Keep QueueFS as the durable source of truth; no new task state or store."""+ """Map tracked queue deliveries to semantic TaskTracker operations."""++ def __init__(self) -> None:+ self._enqueue_locks = KeyedAsyncLockPool()- def __init__(self, index: TaskWorkIndex) -> None:- self._index = index+ @property+ def _tracker(self) -> "TaskTracker":+ from openviking.service.task_tracker import get_task_tracker++ return get_task_tracker()async def enqueue(self, ctx: EnqueueContext, call_next: EnqueueNext) -> str:if not isinstance(ctx.payload, dict):return await call_next(ctx)ctx.payload, metadata = prepare_task_payload(ctx.payload)ctx.work_id = metadata.work_id if metadata is not None else None- if not self._index.register(ctx.queue, metadata):+ if metadata is None:+ return await call_next(ctx)+ if not metadata.account_id or not metadata.user_id:+ raise TaskWorkRejected(f"Task {metadata.task_id} work is missing owner metadata")++ key = (metadata.account_id, metadata.user_id, metadata.task_id, metadata.work_id)+ async with self._enqueue_locks.acquire(key):+ return await self._enqueue_registered(ctx, call_next, metadata)++ async def _enqueue_registered(+ self,+ ctx: EnqueueContext,+ call_next: EnqueueNext,+ metadata: "QueueTaskMetadata",+ ) -> str:+ registration = WorkRegistration.REJECTED++ async def register() -> None:+ nonlocal registration+ registration = await self._tracker.register_work(+ metadata.task_id,+ metadata.work_id,+ ctx.queue,+ account_id=metadata.account_id,+ user_id=metadata.user_id,+ )++ await run_to_completion(register)+ if registration is WorkRegistration.REJECTED:raise TaskWorkRejected(- f"Task {metadata.task_id} is cancelling; rejected work for {ctx.queue}"+ f"Task {metadata.task_id} is unavailable; rejected work for {ctx.queue}")try:- result = await call_next(ctx)+ message_id = await call_next(ctx)except asyncio.CancelledError:- if metadata is not None:- if ctx.committed:- record_enqueued_work(ctx.queue, metadata.work_id)- else:- await self._index.discard(ctx.queue, metadata)- raise- except BaseException as exc:- if metadata is not None:- if ctx.committed:- record_enqueued_work(ctx.queue, metadata.work_id)- else:- await self._index.fail_enqueue(- ctx.queue,- metadata,- str(exc),+ if ctx.committed:+ record_enqueued_work(ctx.queue, metadata.work_id)+ elif registration is WorkRegistration.CREATED:+ await run_to_completion(+ lambda: self._tracker.discard_work(+ metadata.task_id,+ metadata.work_id,+ account_id=metadata.account_id,+ user_id=metadata.user_id,)- record_enqueued_work(ctx.queue, metadata.work_id)+ )raise- if metadata is not None:+ except BaseException as exc:if ctx.committed:record_enqueued_work(ctx.queue, metadata.work_id)else:- await self._index.discard(ctx.queue, metadata)- return result+ failure_message = str(exc) or type(exc).__name__+ await run_to_completion(+ lambda: self._tracker.mark_work_failed(+ metadata.task_id,+ metadata.work_id,+ failure_message,+ account_id=metadata.account_id,+ user_id=metadata.user_id,+ )+ )+ record_enqueued_work(ctx.queue, metadata.work_id)+ raise++ if ctx.committed:+ record_enqueued_work(ctx.queue, metadata.work_id)+ elif registration is WorkRegistration.CREATED:+ await run_to_completion(+ lambda: self._tracker.discard_work(+ metadata.task_id,+ metadata.work_id,+ account_id=metadata.account_id,+ user_id=metadata.user_id,+ )+ )+ return message_idasync def process(self, ctx: ProcessContext, call_next: ProcessNext) -> ProcessResult:metadata = extract_task_metadata(ctx.message)if metadata is None:return await call_next(ctx)+ if not metadata.account_id or not metadata.user_id:+ raise TaskWorkRejected(f"Task {metadata.task_id} work is missing owner metadata")- active_task = asyncio.current_task()- enqueued_work = []- with bind_task_context(+ duplicate = await self._reconcile_delivery(metadata, ctx.queue)+ if duplicate is not None:+ return duplicate++ if await self._tracker.is_cancelling(metadata.task_id,- metadata.account_id,- metadata.user_id,- enqueued_work=enqueued_work,- work_id=metadata.work_id,+ account_id=metadata.account_id,+ user_id=metadata.user_id,):- if self._index.cancellation_requested(metadata.task_id):- result = await ctx.cancel()- self._record_result(metadata, result)- return result- if active_task is not None:- self._index.register_active(- metadata.task_id,- active_task,- work_id=metadata.work_id,- )- try:- result = await call_next(ctx)- except asyncio.CancelledError:- if not self._index.cancellation_requested(metadata.task_id):- raise- # Handler unwinding finished; settle user cancellation before ACK.- result = ProcessResult.cancelled()- except Exception as exc:- self._index.record_failure(metadata.task_id, str(exc))- raise- finally:- if active_task is not None:- self._index.unregister_active(metadata.task_id, active_task)- self._record_result(metadata, result)+ result = await ctx.cancel()+ await self._persist_result(metadata, result)return result- def _record_result(self, metadata, result: ProcessResult) -> None:- outcome = {- ProcessOutcome.SUCCESS: "completed",- ProcessOutcome.FAILED: "failed",- ProcessOutcome.REQUEUED: "requeued",- ProcessOutcome.CANCELLED: "cancelled",- }[result.outcome]- self._index.record_result(- metadata,- outcome,- error=result.error,- errors=result.errors,+ await self._tracker.start_work(+ metadata.task_id,+ metadata.work_id,+ account_id=metadata.account_id,+ user_id=metadata.user_id,)-- async def ack(self, ctx: AckContext, call_next: AckNext) -> None:- prepared = None+ await self._tracker.register_running_task(+ metadata.task_id,+ work_id=metadata.work_id,+ account_id=metadata.account_id,+ user_id=metadata.user_id,+ )+ enqueued_work: list[tuple[str, str]] = []try:- if ctx.message is not None:- prepared = await self._index.prepare_ack(ctx.queue, ctx.message)- await call_next(ctx)+ with bind_task_context(+ metadata.task_id,+ metadata.account_id,+ metadata.user_id,+ enqueued_work=enqueued_work,+ work_id=metadata.work_id,+ ):+ try:+ result = await call_next(ctx)+ except asyncio.CancelledError:+ if not await self._tracker.is_cancelling(+ metadata.task_id,+ account_id=metadata.account_id,+ user_id=metadata.user_id,+ ):+ raise+ result = ProcessResult.cancelled()+ except Exception as exc:+ await self._tracker.record_work_error(+ metadata.task_id,+ metadata.work_id,+ str(exc) or type(exc).__name__,+ account_id=metadata.account_id,+ user_id=metadata.user_id,+ )+ raise+ await self._persist_result(metadata, result)+ return resultfinally:- if prepared is not None and not ctx.committed:- self._index.rollback_ack(ctx.queue, prepared)+ await self._tracker.unregister_running_task(metadata.task_id)++ async def _reconcile_delivery(+ self,+ metadata: "QueueTaskMetadata",+ queue_name: str,+ ) -> ProcessResult | None:+ from openviking.service.task_domain import WorkState++ state = await self._tracker.get_work_state(+ metadata.task_id,+ metadata.work_id,+ account_id=metadata.account_id,+ user_id=metadata.user_id,+ )+ if state is None:+ status = await self._tracker.get_task_status(+ metadata.task_id,+ account_id=metadata.account_id,+ user_id=metadata.user_id,+ )+ if status in TERMINAL_TASK_STATUSES:+ return ProcessResult.success()+ await self._tracker.reconcile_delivery(+ metadata.task_id,+ metadata.work_id,+ queue_name,+ account_id=metadata.account_id,+ user_id=metadata.user_id,+ )+ return None+ if state in (WorkState.DONE, WorkState.FAILED):+ await self._tracker.settle_work(+ metadata.task_id,+ metadata.work_id,+ account_id=metadata.account_id,+ user_id=metadata.user_id,+ )+ return ProcessResult.success()+ return None++ async def _persist_result(+ self,+ metadata: "QueueTaskMetadata",+ result: ProcessResult,+ ) -> None:+ if result.outcome is ProcessOutcome.FAILED:+ await self._tracker.mark_work_failed(+ metadata.task_id,+ metadata.work_id,+ result.error or "Queue handler failed",+ errors=result.errors,+ account_id=metadata.account_id,+ user_id=metadata.user_id,+ )+ elif result.outcome is ProcessOutcome.REQUEUED:+ await self._tracker.mark_work_requeued(+ metadata.task_id,+ metadata.work_id,+ delta=1,+ account_id=metadata.account_id,+ user_id=metadata.user_id,+ )+ else:+ vector_written = bool(+ isinstance(result.value, dict) and result.value.get("vector_written")+ )+ await self._tracker.mark_work_done(+ metadata.task_id,+ metadata.work_id,+ vector_written=vector_written,+ account_id=metadata.account_id,+ user_id=metadata.user_id,+ )async def clear(self, ctx: ClearContext, call_next: ClearNext) -> None:- prepared = []- try:- for message in ctx.messages:- metadata = await self._index.prepare_ack(ctx.queue, message)- if metadata is not None:- prepared.append(metadata)- await call_next(ctx)- finally:- if not ctx.committed:- for metadata in prepared:- self._index.rollback_ack(ctx.queue, metadata)+ tracked = [+ metadata+ for message in ctx.messages+ if (metadata := extract_task_metadata(message)) is not None+ and metadata.account_id+ and metadata.user_id+ ]+ await call_next(ctx)+ if not ctx.committed:+ return+ for metadata in tracked:+ await run_to_completion(+ lambda metadata=metadata: self._tracker.settle_work(+ metadata.task_id,+ metadata.work_id,+ account_id=metadata.account_id,+ user_id=metadata.user_id,+ )+ )diff --git a/openviking/service/task_store.py b/openviking/service/task_store.pyindex bcb8b4e20..ce9ccdfcb 100644--- a/openviking/service/task_store.py+++ b/openviking/service/task_store.py@@ -1,249 +1,1753 @@# Copyright (c) 2026 Beijing Volcano Engine Technology Co., Ltd.# SPDX-License-Identifier: AGPL-3.0-"""Internal storage backends for TaskTracker."""+"""Task/Work storage contract and the local file-backed implementation."""from __future__ import annotations+import asyncioimport heapq+import inspectimport json-from collections.abc import Awaitable, Callable+import threading+import time+from collections import defaultdict+from collections.abc import AsyncIterator, Callable, Collectionfrom copy import deepcopy+from dataclasses import dataclass, field, fields+from enum import Enumfrom typing import Any, Dict, List, Optional, Protocolfrom openviking.pyagfs import AsyncAGFSClientfrom openviking.pyagfs.exceptions import AGFSAlreadyExistsError, AGFSNotFoundError-from openviking.service.task_tracker_concurrency import StoreIOLimiter, run_to_completion+from openviking.service.task_domain import (+ ACTIVE_TASK_STATUSES,+ MAX_QUEUE_ERROR_DETAILS,+ TERMINAL_TASK_STATUSES,+ DeletedTask,+ DeleteExpiredResult,+ QueueWorkSummary,+ TaskRecord,+ TaskRetentionPolicy,+ TaskStatus,+ WorkRecord,+ WorkRecoveryMode,+ WorkState,+ WorkSummary,+ first_work_error,+ processing_seconds,+ sanitize_error,+ summarize_works,+)+from openviking.service.task_events import append_task_event+from openviking.service.task_tracker_concurrency import (+ KeyedAsyncLockPool,+ StoreIOLimiter,+ run_to_completion,+)SYSTEM_TASK_ACCOUNT_ID = "_system"SYSTEM_TASK_USER_ID = "root"+TaskStatsSnapshot = Dict[str, Dict[str, int]]-class TaskStore(Protocol):- async def create(self, task: Any) -> None: ...+class WorkRegistration(str, Enum):+ CREATED = "created"+ EXISTING = "existing"+ REJECTED = "rejected"- async def update(self, task: Any) -> None: ...+ def __bool__(self) -> bool:+ return self is not WorkRegistration.REJECTED- async def get(++class StoreOperationOutcome(str, Enum):+ APPLIED = "applied"+ NOOP = "noop"+ REJECTED = "rejected"+++class StoreOperationCode(str, Enum):+ OK = "ok"+ NOT_FOUND = "not_found"+ ACTIVE_TASK = "active_task"+ ALREADY_TERMINAL = "already_terminal"+ ALREADY_CANCELLED = "already_cancelled"+ INVALID_STATE = "invalid_state"+ NOT_CANCELLABLE = "not_cancellable"+ OPEN_WORK = "open_work"+ WORK_NOT_OPEN = "work_not_open"+ DUPLICATE = "duplicate"+++@dataclass(frozen=True)+class StoreOperationResult:+ outcome: StoreOperationOutcome+ code: StoreOperationCode = StoreOperationCode.OK+ message: Optional[str] = None++ @property+ def accepted(self) -> bool:+ return self.outcome is not StoreOperationOutcome.REJECTED++ @property+ def is_applied(self) -> bool:+ return self.outcome is StoreOperationOutcome.APPLIED++ @classmethod+ def applied(cls, code: StoreOperationCode = StoreOperationCode.OK) -> "StoreOperationResult":+ return cls(StoreOperationOutcome.APPLIED, code)++ @classmethod+ def noop(+ cls,+ code: StoreOperationCode = StoreOperationCode.OK,+ message: Optional[str] = None,+ ) -> "StoreOperationResult":+ return cls(StoreOperationOutcome.NOOP, code, message)++ @classmethod+ def rejected(+ cls,+ code: StoreOperationCode,+ message: Optional[str] = None,+ ) -> "StoreOperationResult":+ return cls(StoreOperationOutcome.REJECTED, code, message)+++@dataclass(frozen=True)+class TaskDeletionBlocker:+ task_id: str+ account_id: str+ user_id: str+ task_type: str+ code: StoreOperationCode+ message: str+++@dataclass(frozen=True)+class DeleteTasksResult:+ deleted: tuple[DeletedTask, ...] = ()+ result: StoreOperationResult = field(default_factory=StoreOperationResult.applied)+ blockers: tuple[TaskDeletionBlocker, ...] = ()++ @property+ def accepted(self) -> bool:+ return self.result.accepted+++class TaskAlreadyActive(RuntimeError):+ def __init__(self, task_type: str, resource_id: Optional[str]) -> None:+ self.task_type = task_type+ self.resource_id = resource_id+ super().__init__(f"An active {task_type} task already exists for {resource_id}")+++class TaskWorkStore(Protocol):+ """Atomic semantic operations required by TaskTracker.++ Implementations own transaction and revision details. Callers never replace+ a complete Task/Work aggregate or perform compare-and-set retries.+ """++ @property+ def work_recovery_mode(self) -> WorkRecoveryMode: ...++ async def create(self, task: TaskRecord, *, reject_if_active: bool = False) -> TaskRecord: ...++ async def get_task(self,task_id: str,*,account_id: Optional[str] = None,user_id: Optional[str] = None,- ) -> Optional[Dict[str, Any]]: ...+ ) -> Optional[TaskRecord]: ...async def list(- self, account_id: str, *, user_id: Optional[str] = None- ) -> List[Dict[str, Any]]: ...+ self,+ account_id: Optional[str] = None,+ *,+ user_id: Optional[str] = None,+ task_type: Optional[str] = None,+ status: Optional[str] = None,+ resource_id: Optional[str] = None,+ limit: Optional[int] = None,+ before: tuple[float, str] | None = None,+ include_internal: bool = True,+ q: Optional[str] = None,+ ) -> List[TaskRecord]: ...- async def list_page(+ async def delete_tasks(self,+ task_id: Optional[str] = None,+ *,account_id: str,+ user_id: Optional[str] = None,+ allow_active_types: Collection[str] = (),+ ) -> DeleteTasksResult: ...++ async def delete_expired(+ self, policy: TaskRetentionPolicy, *, now: float, limit: int+ ) -> DeleteExpiredResult: ...++ def snapshot_task_stats(self) -> TaskStatsSnapshot: ...++ async def start_task(+ self,+ task_id: str,+ *,+ stage: Optional[str],+ account_id: Optional[str],+ user_id: Optional[str],+ ) -> StoreOperationResult: ...++ async def update_stage(+ self,+ task_id: str,+ stage: str,+ *,+ meta: Optional[Dict[str, Any]],+ account_id: Optional[str],+ user_id: Optional[str],+ ) -> StoreOperationResult: ...++ async def update_auth(+ self,+ task_id: str,+ values: Dict[str, Any],*,+ account_id: str,user_id: str,- limit: int,- before: tuple[float, str] | None = None,- prune_expired: Callable[[Dict[str, Any]], Awaitable[bool]] | None = None,- io_limiter: StoreIOLimiter | None = None,- **filters: Any,- ) -> List[Dict[str, Any]]: ...+ ) -> StoreOperationResult: ...- async def delete(- self, task_id: str, *, account_id: str, user_id: Optional[str] = None- ) -> None: ...+ async def merge_meta(+ self,+ task_id: str,+ values: Dict[str, Any],+ *,+ account_id: str,+ user_id: str,+ ) -> StoreOperationResult: ...+ async def record_event(+ self,+ task_id: str,+ kind: str,+ *,+ operation: Optional[str],+ account_id: str,+ user_id: str,+ ) -> StoreOperationResult: ...++ async def record_outcome(+ self,+ task_id: str,+ *,+ result: Optional[Dict[str, Any]],+ error: Optional[str],+ resource_id: Optional[str],+ cancelled: bool,+ account_id: Optional[str],+ user_id: Optional[str],+ ) -> StoreOperationResult: ...++ async def request_cancellation(+ self,+ task_id: str,+ *,+ cancellable_types: Collection[str],+ allow_terminal: bool,+ account_id: str,+ user_id: str,+ ) -> StoreOperationResult: ...++ async def mark_enqueue_complete(+ self,+ task_id: str,+ work_id: str,+ *,+ account_id: str,+ user_id: str,+ ) -> StoreOperationResult: ...++ async def is_enqueue_complete(+ self,+ task_id: str,+ work_id: str,+ *,+ account_id: str,+ user_id: str,+ ) -> bool: ...++ async def add_work(+ self, work: WorkRecord, *, account_id: str, user_id: str+ ) -> StoreOperationResult: ...++ async def reconcile_delivery(+ self, work: WorkRecord, *, account_id: str, user_id: str+ ) -> StoreOperationResult: ...++ async def discard_work(+ self, task_id: str, work_id: str, *, account_id: str, user_id: str+ ) -> StoreOperationResult: ...++ async def get_work(+ self, task_id: str, work_id: str, *, account_id: str, user_id: str+ ) -> Optional[WorkRecord]: ...++ async def get_works(+ self,+ task_id: str,+ work_ids: Collection[str],+ *,+ account_id: Optional[str],+ user_id: Optional[str],+ ) -> Dict[str, WorkRecord]: ...++ async def start_work(+ self, task_id: str, work_id: str, *, account_id: str, user_id: str+ ) -> StoreOperationResult: ...++ async def finish_work(+ self,+ task_id: str,+ work_id: str,+ state: WorkState,+ *,+ error: Optional[str],+ errors: Optional[Collection[str]],+ context_count_delta: int,+ account_id: str,+ user_id: str,+ ) -> StoreOperationResult: ...++ async def requeue_work(+ self,+ task_id: str,+ work_id: str,+ *,+ delta: int,+ account_id: str,+ user_id: str,+ ) -> StoreOperationResult: ...++ async def record_work_error(+ self,+ task_id: str,+ work_id: str,+ error: str,+ *,+ account_id: str,+ user_id: str,+ ) -> StoreOperationResult: ...-class PersistentTaskStore:- """Persist task records into AGFS under account-scoped system task directories."""+ async def settle_work(+ self, task_id: str, work_id: str, *, account_id: str, user_id: str+ ) -> StoreOperationResult: ...++ async def has_open_work(+ self,+ task_id: str,+ *,+ exclude_work_id: Optional[str],+ account_id: Optional[str],+ user_id: Optional[str],+ ) -> bool: ...++ async def summarize_works(+ self,+ task_id: str,+ *,+ work_ids: Optional[Collection[str]],+ account_id: Optional[str],+ user_id: Optional[str],+ ) -> WorkSummary: ...++ async def filter_cancellation_requested(+ self,+ task_ids: Collection[str],+ *,+ account_id: str,+ user_id: str,+ ) -> set[str]: ...++ async def get_task_status(+ self,+ task_id: str,+ *,+ account_id: str,+ user_id: str,+ ) -> Optional[TaskStatus]: ...+++# Compatibility name for integrations that only annotate the constructor.+TaskStore = TaskWorkStore+++@dataclass+class _WorkSet:+ works: Dict[str, WorkRecord]+ open_count: int = 0+++@dataclass(frozen=True)+class _WorkMutationResult:+ changed: bool+ work: Optional[WorkRecord]+ result: StoreOperationResult = StoreOperationResult.applied()+++class PersistentTaskStore(TaskWorkStore):+ """File Task store with process-local Work state reconstructed from QueueFS.++ Only Task records are durable. QueueFS startup recovery can reconstruct+ unacknowledged Work, but terminal Work history, timings, and errors may be+ incomplete after restart.+ """ROOT_PREFIX = "/local"SYSTEM_DIRNAME = "_system"TASKS_DIRNAME = "tasks"- def __init__(self, agfs: Any) -> None:+ def __init__(self, agfs: Any, *, max_concurrent_io: int = 8) -> None:self._agfs = agfs if isinstance(agfs, AsyncAGFSClient) else AsyncAGFSClient(agfs)self._ensured_task_dirs: set[tuple[str, str]] = set()+ self._lock = threading.Lock()+ self._task_cache: Dict[tuple[str, str, str], TaskRecord] = {}+ self._work_sets: Dict[tuple[str, str, str], _WorkSet] = {}+ self._business_locks = KeyedAsyncLockPool()+ self._record_locks = KeyedAsyncLockPool()+ self._cleanup_lock = KeyedAsyncLockPool()+ self._cleanup_scan: Optional[AsyncIterator[Optional[Dict[str, Any]]]] = None+ self._io = StoreIOLimiter(max_concurrent_io)- async def create(self, task: Any) -> None:- await self._write_task(task)+ @property+ def work_recovery_mode(self) -> WorkRecoveryMode:+ return WorkRecoveryMode.QUEUE_SNAPSHOT- async def update(self, task: Any) -> None:- await self._write_task(task)+ async def create(self, task: TaskRecord, *, reject_if_active: bool = False) -> TaskRecord:+ if not task.account_id or not task.user_id:+ raise ValueError("PersistentTaskStore requires account_id and user_id")+ business_key = (task.account_id, task.user_id, task.task_type, task.resource_id)+ async with self._business_locks.acquire(business_key):+ if reject_if_active:+ candidates = await self.list(task.account_id, user_id=task.user_id)+ if any(+ item.task_type == task.task_type+ and item.resource_id == task.resource_id+ and item.status in ACTIVE_TASK_STATUSES+ for item in candidates+ ):+ raise TaskAlreadyActive(task.task_type, task.resource_id)+ owner = (task.account_id, task.user_id, task.task_id)+ async with self._record_locks.acquire(owner):+ existing = await self._load_task(task.task_id, task.account_id, task.user_id)+ if existing is not None:+ return self._copy_task(existing, self._work_sets.get(owner))+ stored = deepcopy(task)+ self._append_events(stored, None)+ await self._write_task(stored)+ with self._lock:+ self._task_cache[owner] = stored+ self._work_sets[owner] = _WorkSet({})+ return deepcopy(stored)- async def get(+ async def get_task(self,task_id: str,*,account_id: Optional[str] = None,user_id: Optional[str] = None,- ) -> Optional[Dict[str, Any]]:- if not account_id or not user_id:+ ) -> Optional[TaskRecord]:+ task = await self._load_task(task_id, account_id, user_id)+ if task is None:return None- path = self._task_path(account_id, user_id, task_id)- try:- raw = await self._agfs.read(path)- except (AGFSNotFoundError, FileNotFoundError):+ with self._lock:+ owner = self._find_cached_owner(task_id, account_id, user_id)+ if owner is None:+ return None+ return self._copy_task(self._task_cache[owner], self._work_sets.get(owner))++ async def start_task(+ self,+ task_id: str,+ *,+ stage: Optional[str],+ account_id: Optional[str],+ user_id: Optional[str],+ ) -> StoreOperationResult:+ def mutate(task: TaskRecord, _work_set: _WorkSet) -> StoreOperationResult:+ if task.status not in (TaskStatus.PENDING, TaskStatus.RUNNING):+ return _inactive_task_result(task, "Task cannot be started")+ task.status = TaskStatus.RUNNING+ if stage is not None:+ task.stage = stage+ return StoreOperationResult.applied()++ return await self._mutate_task(task_id, account_id, user_id, mutate)++ async def update_stage(+ self,+ task_id: str,+ stage: str,+ *,+ meta: Optional[Dict[str, Any]],+ account_id: Optional[str],+ user_id: Optional[str],+ ) -> StoreOperationResult:+ def mutate(task: TaskRecord, _work_set: _WorkSet) -> StoreOperationResult:+ if task.status not in ACTIVE_TASK_STATUSES:+ return _inactive_task_result(task, "Task stage cannot be updated")+ task.stage = stage+ if meta:+ _deep_merge(task.meta, deepcopy(meta))+ return StoreOperationResult.applied()++ return await self._mutate_task(task_id, account_id, user_id, mutate)++ async def update_auth(+ self,+ task_id: str,+ values: Dict[str, Any],+ *,+ account_id: str,+ user_id: str,+ ) -> StoreOperationResult:+ def mutate(task: TaskRecord, _work_set: _WorkSet) -> StoreOperationResult:+ if task.status not in ACTIVE_TASK_STATUSES:+ return _inactive_task_result(task, "Task auth cannot be updated")+ _deep_merge(task.auth, deepcopy(values))+ return StoreOperationResult.applied()++ return await self._mutate_task(task_id, account_id, user_id, mutate)++ async def merge_meta(+ self,+ task_id: str,+ values: Dict[str, Any],+ *,+ account_id: str,+ user_id: str,+ ) -> StoreOperationResult:+ def mutate(task: TaskRecord, _work_set: _WorkSet) -> StoreOperationResult:+ if task.status not in (TaskStatus.PENDING, TaskStatus.RUNNING):+ return _inactive_task_result(task, "Task meta cannot be updated")+ _deep_merge(task.meta, deepcopy(values))+ return StoreOperationResult.applied()++ return await self._mutate_task(task_id, account_id, user_id, mutate)++ async def record_event(+ self,+ task_id: str,+ kind: str,+ *,+ operation: Optional[str],+ account_id: str,+ user_id: str,+ ) -> StoreOperationResult:+ def mutate(task: TaskRecord, _work_set: _WorkSet) -> StoreOperationResult:+ if task.status not in ACTIVE_TASK_STATUSES:+ return StoreOperationResult.noop(+ StoreOperationCode.ALREADY_TERMINAL,+ f"Task is already {task.status.value}",+ )+ task.execution_events = append_task_event(+ task.execution_events,+ kind=kind,+ status=task.status.value,+ stage=sanitize_error(task.stage) if task.stage else None,+ operation=operation,+ )+ return StoreOperationResult.applied()++ return await self._mutate_task(task_id, account_id, user_id, mutate)++ async def record_outcome(+ self,+ task_id: str,+ *,+ result: Optional[Dict[str, Any]],+ error: Optional[str],+ resource_id: Optional[str],+ cancelled: bool,+ account_id: Optional[str],+ user_id: Optional[str],+ ) -> StoreOperationResult:+ def mutate(task: TaskRecord, work_set: _WorkSet) -> StoreOperationResult:+ if task.status not in ACTIVE_TASK_STATUSES:+ return StoreOperationResult.noop(+ StoreOperationCode.ALREADY_TERMINAL,+ f"Task is already {task.status.value}",+ )+ if result is not None:+ task.result = deepcopy(result)+ if resource_id is not None:+ task.resource_id = resource_id+ if error and task.error is None:+ task.error = sanitize_error(error)+ if cancelled:+ task.status = TaskStatus.CANCELLING+ enqueue_markers = task.auth.get("enqueue_complete_work_ids")+ task.auth = (+ {"enqueue_complete_work_ids": deepcopy(enqueue_markers)} if enqueue_markers else {}+ )+ self._try_finalize(task, work_set)+ return StoreOperationResult.applied()++ return await self._mutate_task(task_id, account_id, user_id, mutate)++ async def request_cancellation(+ self,+ task_id: str,+ *,+ cancellable_types: Collection[str],+ allow_terminal: bool,+ account_id: str,+ user_id: str,+ ) -> StoreOperationResult:+ def mutate(task: TaskRecord, work_set: _WorkSet) -> StoreOperationResult:+ if task.status in (TaskStatus.CANCELLING, TaskStatus.CANCELLED):+ return StoreOperationResult.noop(+ StoreOperationCode.ALREADY_CANCELLED,+ f"Task is already {task.status.value}",+ )+ if task.task_type not in cancellable_types:+ return StoreOperationResult.rejected(+ StoreOperationCode.NOT_CANCELLABLE,+ f"Task type '{task.task_type}' does not support cancellation",+ )+ if task.status in TERMINAL_TASK_STATUSES and not allow_terminal:+ return StoreOperationResult.rejected(+ StoreOperationCode.ALREADY_TERMINAL,+ f"Task is already {task.status.value}",+ )+ task.status = TaskStatus.CANCELLING+ self._try_finalize(task, work_set)+ return StoreOperationResult.applied()++ return await self._mutate_task(task_id, account_id, user_id, mutate)++ async def mark_enqueue_complete(+ self,+ task_id: str,+ work_id: str,+ *,+ account_id: str,+ user_id: str,+ ) -> StoreOperationResult:+ def mutate(task: TaskRecord, _work_set: _WorkSet) -> StoreOperationResult:+ if task.status not in ACTIVE_TASK_STATUSES:+ return StoreOperationResult.noop(+ StoreOperationCode.ALREADY_TERMINAL,+ f"Task is already {task.status.value}",+ )+ completed = set(task.auth.get("enqueue_complete_work_ids") or ())+ if work_id in completed:+ return StoreOperationResult.noop(StoreOperationCode.DUPLICATE)+ completed.add(work_id)+ task.auth["enqueue_complete_work_ids"] = sorted(completed)+ return StoreOperationResult.applied()++ return await self._mutate_task(task_id, account_id, user_id, mutate)++ async def is_enqueue_complete(+ self,+ task_id: str,+ work_id: str,+ *,+ account_id: str,+ user_id: str,+ ) -> bool:+ task = await self.get_task(task_id, account_id=account_id, user_id=user_id)+ return bool(task and work_id in set(task.auth.get("enqueue_complete_work_ids") or ()))++ async def add_work(+ self, work: WorkRecord, *, account_id: str, user_id: str+ ) -> StoreOperationResult:+ def mutate(task: TaskRecord, existing: Optional[WorkRecord]) -> _WorkMutationResult:+ if task.is_terminal():+ return _WorkMutationResult(+ False,+ existing,+ StoreOperationResult.rejected(+ StoreOperationCode.ALREADY_TERMINAL,+ f"Task {work.task_id} is {task.status.value}; rejected work",+ ),+ )+ if task.status == TaskStatus.CANCELLING:+ return _WorkMutationResult(+ False,+ existing,+ StoreOperationResult.rejected(+ StoreOperationCode.INVALID_STATE,+ f"Task {work.task_id} is cancelling; rejected work",+ ),+ )+ if existing is not None:+ return _WorkMutationResult(+ False,+ existing,+ StoreOperationResult.noop(StoreOperationCode.DUPLICATE),+ )+ return _WorkMutationResult(True, deepcopy(work))++ return await self._mutate_work(+ work.task_id,+ work.work_id,+ account_id,+ user_id,+ mutate,+ )++ async def reconcile_delivery(+ self, work: WorkRecord, *, account_id: str, user_id: str+ ) -> StoreOperationResult:+ """Ensure a delivered QueueFS message has local Work state."""++ def mutate(task: TaskRecord, existing: Optional[WorkRecord]) -> _WorkMutationResult:+ if task.is_terminal():+ return _WorkMutationResult(+ False,+ existing,+ StoreOperationResult.noop(+ StoreOperationCode.ALREADY_TERMINAL,+ f"Task is already {task.status.value}",+ ),+ )+ if existing is None:+ return _WorkMutationResult(True, deepcopy(work))+ if existing.is_open() and existing.queue_name != work.queue_name:+ existing.queue_name = work.queue_name+ return _WorkMutationResult(True, existing)+ return _WorkMutationResult(False, existing, StoreOperationResult.noop())++ return await self._mutate_work(+ work.task_id,+ work.work_id,+ account_id,+ user_id,+ mutate,+ )++ async def discard_work(+ self, task_id: str, work_id: str, *, account_id: str, user_id: str+ ) -> StoreOperationResult:+ def mutate(_task: TaskRecord, existing: Optional[WorkRecord]) -> _WorkMutationResult:+ if existing is None:+ return _WorkMutationResult(+ False,+ None,+ StoreOperationResult.noop(StoreOperationCode.NOT_FOUND),+ )+ return _WorkMutationResult(True, None)++ return await self._mutate_work(+ task_id,+ work_id,+ account_id,+ user_id,+ mutate,+ try_finalize=True,+ )++ async def get_work(+ self, task_id: str, work_id: str, *, account_id: str, user_id: str+ ) -> Optional[WorkRecord]:+ task = await self._load_task(task_id, account_id, user_id)+ if task is None:return None- return json.loads(_decode_bytes(raw))+ with self._lock:+ work_set = self._work_sets.get((account_id, user_id, task_id))+ work = work_set.works.get(work_id) if work_set else None+ return deepcopy(work) if work is not None else None- async def list(self, account_id: str, *, user_id: Optional[str] = None) -> List[Dict[str, Any]]:- if user_id is None:- try:- owners = await self._agfs.ls(self._task_root_dir(account_id))- except (AGFSNotFoundError, FileNotFoundError):- return []- tasks: List[Dict[str, Any]] = []- for owner in owners:- if owner.get("isDir") and owner.get("name") not in (".", ".."):- tasks.extend(await self.list(account_id, user_id=owner["name"]))- return tasks- directory = self._task_dir(account_id, user_id)- try:- items = await self._agfs.ls(directory)- except (AGFSNotFoundError, FileNotFoundError):- return []- tasks: List[Dict[str, Any]] = []- for item in items:- path = item.get("path") or f"{directory}/{item.get('name', '')}"- if not path.endswith(".json"):- continue- try:- raw = await self._agfs.read(path)- except (AGFSNotFoundError, FileNotFoundError):- continue- tasks.append(json.loads(_decode_bytes(raw)))- return tasks+ async def get_works(+ self,+ task_id: str,+ work_ids: Collection[str],+ *,+ account_id: Optional[str],+ user_id: Optional[str],+ ) -> Dict[str, WorkRecord]:+ task = await self._load_task(task_id, account_id, user_id)+ if task is None or not task.account_id or not task.user_id:+ return {}+ wanted = set(work_ids)+ with self._lock:+ work_set = self._work_sets.get((task.account_id, task.user_id, task_id))+ return {+ work_id: deepcopy(work)+ for work_id, work in (work_set.works.items() if work_set else ())+ if work_id in wanted+ }++ async def start_work(+ self, task_id: str, work_id: str, *, account_id: str, user_id: str+ ) -> StoreOperationResult:+ def mutate(_task: TaskRecord, work: Optional[WorkRecord]) -> _WorkMutationResult:+ if work is None or not work.is_open():+ return _WorkMutationResult(+ False,+ work,+ StoreOperationResult.rejected(+ StoreOperationCode.WORK_NOT_OPEN,+ f"Work {work_id} is unavailable",+ ),+ )+ now = time.time()+ work.state = WorkState.RUNNING+ work.started_at = work.started_at or now+ work.finished_at = None+ work.updated_at = now+ return _WorkMutationResult(True, work)++ return await self._mutate_work(task_id, work_id, account_id, user_id, mutate)++ async def finish_work(+ self,+ task_id: str,+ work_id: str,+ state: WorkState,+ *,+ error: Optional[str],+ errors: Optional[Collection[str]],+ context_count_delta: int,+ account_id: str,+ user_id: str,+ ) -> StoreOperationResult:+ if state not in (WorkState.DONE, WorkState.FAILED):+ raise ValueError(f"Non-terminal Work state: {state}")++ def mutate(_task: TaskRecord, work: Optional[WorkRecord]) -> _WorkMutationResult:+ if work is None or not work.is_open():+ return _WorkMutationResult(+ False,+ work,+ StoreOperationResult.noop(+ StoreOperationCode.WORK_NOT_OPEN,+ f"Work {work_id} is unavailable",+ ),+ )+ now = time.time()+ work.state = state+ work.finished_at = now+ work.updated_at = now+ if error:+ work.error = sanitize_error(error)+ if state == WorkState.FAILED:+ details = [sanitize_error(str(value)) for value in errors or () if str(value)]+ work.errors = details or ([work.error] if work.error else [])+ if context_count_delta > 0:+ work.context_count += context_count_delta+ return _WorkMutationResult(True, work)- async def list_page(+ return await self._mutate_work(+ task_id,+ work_id,+ account_id,+ user_id,+ mutate,+ try_finalize=True,+ )++ async def requeue_work(self,+ task_id: str,+ work_id: str,+ *,+ delta: int,account_id: str,+ user_id: str,+ ) -> StoreOperationResult:+ def mutate(_task: TaskRecord, work: Optional[WorkRecord]) -> _WorkMutationResult:+ if work is None or not work.is_open():+ return _WorkMutationResult(+ False,+ work,+ StoreOperationResult.noop(+ StoreOperationCode.WORK_NOT_OPEN,+ f"Work {work_id} is unavailable",+ ),+ )+ work.state = WorkState.PENDING+ work.requeue_count += max(delta, 0)+ work.started_at = None+ work.finished_at = None+ work.updated_at = time.time()+ return _WorkMutationResult(True, work)++ return await self._mutate_work(task_id, work_id, account_id, user_id, mutate)++ async def record_work_error(+ self,+ task_id: str,+ work_id: str,+ error: str,*,+ account_id: str,user_id: str,- limit: int,+ ) -> StoreOperationResult:+ def mutate(_task: TaskRecord, work: Optional[WorkRecord]) -> _WorkMutationResult:+ if work is None or work.error:+ return _WorkMutationResult(+ False,+ work,+ StoreOperationResult.noop(+ StoreOperationCode.WORK_NOT_OPEN if work is None else StoreOperationCode.OK+ ),+ )+ work.error = sanitize_error(error)+ work.updated_at = time.time()+ return _WorkMutationResult(True, work)++ return await self._mutate_work(task_id, work_id, account_id, user_id, mutate)++ async def settle_work(+ self, task_id: str, work_id: str, *, account_id: str, user_id: str+ ) -> StoreOperationResult:+ def mutate(_task: TaskRecord, work: Optional[WorkRecord]) -> _WorkMutationResult:+ if work is None:+ return _WorkMutationResult(+ False,+ None,+ StoreOperationResult.noop(StoreOperationCode.NOT_FOUND),+ )+ changed = False+ if work.is_open():+ work.state = WorkState.FAILED if work.error else WorkState.DONE+ work.finished_at = time.time()+ work.updated_at = work.finished_at+ changed = True+ return _WorkMutationResult(+ changed,+ work,+ StoreOperationResult.applied() if changed else StoreOperationResult.noop(),+ )++ return await self._mutate_work(+ task_id,+ work_id,+ account_id,+ user_id,+ mutate,+ try_finalize=True,+ finalize_when_unchanged=True,+ )++ async def has_open_work(+ self,+ task_id: str,+ *,+ exclude_work_id: Optional[str] = None,+ account_id: Optional[str] = None,+ user_id: Optional[str] = None,+ ) -> bool:+ task = await self._load_task(task_id, account_id, user_id)+ if task is None or not task.account_id or not task.user_id:+ return False+ with self._lock:+ work_set = self._work_sets.get((task.account_id, task.user_id, task_id))+ if work_set is None or work_set.open_count == 0:+ return False+ if exclude_work_id is None:+ return True+ excluded = work_set.works.get(exclude_work_id)+ return work_set.open_count > int(excluded is not None and excluded.is_open())++ async def summarize_works(+ self,+ task_id: str,+ *,+ work_ids: Optional[Collection[str]] = None,+ account_id: Optional[str] = None,+ user_id: Optional[str] = None,+ ) -> WorkSummary:+ works = await self._selected_works(task_id, work_ids, account_id, user_id)+ return summarize_works(works)++ async def filter_cancellation_requested(+ self,+ task_ids: Collection[str],+ *,+ account_id: str,+ user_id: str,+ ) -> set[str]:+ requested = set()+ for task_id in set(task_ids):+ status = await self.get_task_status(task_id, account_id=account_id, user_id=user_id)+ if status in (+ TaskStatus.CANCELLING,+ TaskStatus.CANCELLED,+ ):+ requested.add(task_id)+ return requested++ async def get_task_status(+ self,+ task_id: str,+ *,+ account_id: str,+ user_id: str,+ ) -> Optional[TaskStatus]:+ task = await self._load_task(task_id, account_id, user_id)+ if task is None:+ return None+ with self._lock:+ live = self._task_cache.get((account_id, user_id, task_id))+ return live.status if live is not None else None++ async def list(+ self,+ account_id: Optional[str] = None,+ *,+ user_id: Optional[str] = None,+ task_type: Optional[str] = None,+ status: Optional[str] = None,+ resource_id: Optional[str] = None,+ limit: Optional[int] = None,before: tuple[float, str] | None = None,- prune_expired: Callable[[Dict[str, Any]], Awaitable[bool]] | None = None,- io_limiter: StoreIOLimiter | None = None,- **filters: Any,- ) -> List[Dict[str, Any]]:- """Scan durable records, retaining only the requested page in memory.-- AGFS has no indexed query API. This bounds record memory, but still- performs O(n) reads; it deliberately does not load all tasks into the tracker.+ include_internal: bool = True,+ q: Optional[str] = None,+ ) -> List[TaskRecord]:+ """List matching Tasks.++ Owner-scoped reads include durable records. Without an account, the+ file backend can only return records already loaded into its cache."""from openviking.service.task_pagination import matches- async def read(operation: str, factory: Callable[[], Awaitable[Any]]) -> Any:- if io_limiter is None:- return await factory()- return await io_limiter.run(operation, lambda: run_to_completion(factory))+ if limit is not None and limit <= 0:+ return []+ if account_id and user_id and limit is not None:+ return await self._list_owner_page(+ account_id,+ user_id=user_id,+ limit=limit,+ before=before,+ task_type=task_type,+ status=status,+ resource_id=resource_id,+ include_internal=include_internal,+ q=q,+ )+ if account_id:+ users = [user_id] if user_id else await self._owner_users(account_id)+ for owner_user in users:+ for payload in await self._list_owner_payloads(account_id, owner_user):+ try:+ self._put_if_absent(+ _task_from_payload(payload, account_id=account_id, user_id=owner_user)+ )+ except (ValueError, TypeError, KeyError):+ continue+ with self._lock:+ tasks = [+ self._copy_task(task, self._work_sets.get(owner))+ for owner, task in self._task_cache.items()+ for account, user, _ in (owner,)+ if (account_id is None or account == account_id)+ and (user_id is None or user == user_id)+ ]+ tasks = [+ task+ for task in tasks+ if (before is None or (task.created_at, task.task_id) < before)+ and matches(+ task.to_dict(),+ task_type=task_type,+ status=status,+ resource_id=resource_id,+ include_internal=include_internal,+ q=q,+ )+ ]+ tasks.sort(key=lambda task: (task.created_at, task.task_id), reverse=True)+ return tasks if limit is None else tasks[:limit]++ async def _list_owner_page(+ self,+ account_id: str,+ *,+ user_id: str,+ limit: int,+ before: tuple[float, str] | None = None,+ task_type: Optional[str],+ status: Optional[str],+ resource_id: Optional[str],+ include_internal: bool,+ q: Optional[str],+ ) -> List[TaskRecord]:+ from openviking.service.task_pagination import matches- directory = self._task_dir(account_id, user_id)try:- entries = await read("list_page_ls", lambda: self._agfs.ls(directory))+ entries = await self._ls(self._task_dir(account_id, user_id))except (AGFSNotFoundError, FileNotFoundError):return []- heap: list = []- for entry in entries:- path = entry.get("path") or f"{directory}/{entry.get('name', '')}"+ heap: list[tuple[float, str, TaskRecord]] = []+ for item in entries:+ path = (+ item.get("path") or f"{self._task_dir(account_id, user_id)}/{item.get('name', '')}"+ )if not path.endswith(".json"):continuetry:- task = json.loads(- _decode_bytes(- await read("list_page_read", lambda path=path: self._agfs.read(path))- )- )- except (AGFSNotFoundError, FileNotFoundError):- continue- if task.get("account_id") != account_id or task.get("user_id") != user_id:+ payload = json.loads(_decode_bytes(await self._read(path)))+ record = _task_from_payload(payload, account_id=account_id, user_id=user_id)+ except (AGFSNotFoundError, FileNotFoundError, ValueError, TypeError, KeyError):continue- if prune_expired is not None and await prune_expired(task):- continue- key = (float(task["created_at"]), str(task["task_id"]))+ with self._lock:+ owner = (account_id, user_id, record.task_id)+ cached = self._task_cache.get(owner)+ if cached is not None:+ record = self._copy_task(cached, self._work_sets.get(owner))+ key = (record.created_at, record.task_id)if before is not None and key >= before:continue- if not matches(task, **filters):+ if not matches(+ record.to_dict(),+ task_type=task_type,+ status=status,+ resource_id=resource_id,+ include_internal=include_internal,+ q=q,+ ):continue- item = (*key, task)+ candidate = (*key, record)if len(heap) < limit:- heapq.heappush(heap, item)+ heapq.heappush(heap, candidate)elif key > heap[0][:2]:- heapq.heapreplace(heap, item)+ heapq.heapreplace(heap, candidate)return [item[2] for item in sorted(heap, reverse=True)]- async def delete(self, task_id: str, *, account_id: str, user_id: Optional[str] = None) -> None:- """Delete a task record, succeeding if it has already been removed."""- if not user_id:- return+ async def delete_tasks(+ self,+ task_id: Optional[str] = None,+ *,+ account_id: str,+ user_id: Optional[str] = None,+ allow_active_types: Collection[str] = (),+ ) -> DeleteTasksResult:+ """Delete one Task or all Tasks in an owner scope."""+ if task_id is not None:+ if not user_id:+ raise ValueError("Deleting one Task requires user_id")+ owner = (account_id, user_id, task_id)+ async with self._record_locks.acquire(owner):+ task = await self._load_task(task_id, account_id, user_id)+ if task is not None:+ blocker = self._deletion_blocker(+ task, self._work_sets.get(owner), allow_active_types+ )+ if blocker is not None:+ return self._delete_tasks_rejected((blocker,))+ deleted = await self._delete_task_unlocked(owner, task is not None)+ return DeleteTasksResult(+ (DeletedTask(task_id, account_id, user_id),) if deleted else (),+ StoreOperationResult.applied()+ if deleted+ else StoreOperationResult.noop(StoreOperationCode.NOT_FOUND),+ )++ return await self._delete_owner_tasks(+ account_id,+ user_id=user_id,+ allow_active_types=allow_active_types,+ )++ async def _delete_owner_tasks(+ self,+ account_id: str,+ *,+ user_id: Optional[str],+ allow_active_types: Collection[str],+ ) -> DeleteTasksResult:+ tasks = await self.list(account_id, user_id=user_id, limit=None)+ blockers = tuple(+ blocker+ for task in tasks+ if task.user_id+ for blocker in (+ self._deletion_blocker(+ task,+ self._work_sets.get((task.account_id, task.user_id, task.task_id)),+ allow_active_types,+ ),+ )+ if blocker is not None+ )+ if blockers:+ return self._delete_tasks_rejected(blockers)+ deleted: list[DeletedTask] = []+ for task in tasks:+ if task.user_id:+ result = await self.delete_tasks(+ task.task_id,+ account_id=account_id,+ user_id=task.user_id,+ allow_active_types=allow_active_types,+ )+ if not result.accepted:+ return result+ deleted.extend(result.deleted)+ self._ensured_task_dirs = {+ owner+ for owner in self._ensured_task_dirs+ if owner[0] != account_id or (user_id is not None and owner[1] != user_id)+ }+ return DeleteTasksResult(+ tuple(sorted(deleted, key=lambda item: (item.account_id, item.user_id, item.task_id)))+ )++ @staticmethod+ def _deletion_blocker(+ task: TaskRecord,+ work_set: Optional[_WorkSet],+ allow_active_types: Collection[str],+ ) -> Optional[TaskDeletionBlocker]:+ allowed = set(allow_active_types)+ if task.status in ACTIVE_TASK_STATUSES and task.task_type not in allowed:+ return TaskDeletionBlocker(+ task.task_id,+ task.account_id or "",+ task.user_id or "",+ task.task_type,+ StoreOperationCode.ACTIVE_TASK,+ f"Cannot delete active task record: {task.task_id}({task.task_type})",+ )+ if task.task_type in allowed:+ return None+ if work_set is not None and work_set.open_count:+ return TaskDeletionBlocker(+ task.task_id,+ task.account_id or "",+ task.user_id or "",+ task.task_type,+ StoreOperationCode.OPEN_WORK,+ f"Cannot delete active task record: {task.task_id}",+ )+ return None++ @staticmethod+ def _delete_tasks_rejected(blockers: Collection[TaskDeletionBlocker]) -> DeleteTasksResult:+ ordered = tuple(blockers)+ active = [item for item in ordered if item.code is StoreOperationCode.ACTIVE_TASK]+ if active:+ message = "Cannot delete active task records: " + ", ".join(+ f"{item.task_id}({item.task_type})" for item in active+ )+ code = StoreOperationCode.ACTIVE_TASK+ elif len(ordered) == 1:+ message = ordered[0].message+ code = ordered[0].code+ else:+ message = "Cannot delete active task records: " + ", ".join(+ item.task_id for item in ordered+ )+ code = StoreOperationCode.OPEN_WORK+ return DeleteTasksResult(+ (),+ StoreOperationResult.rejected(code, message),+ ordered,+ )++ async def _delete_task_unlocked(+ self,+ owner: tuple[str, str, str],+ task_exists: bool,+ ) -> bool:+ account_id, user_id, task_id = owner+ try:+ await self._rm(self._task_path(account_id, user_id, task_id))+ except (AGFSNotFoundError, FileNotFoundError):+ if not task_exists:+ return False+ with self._lock:+ self._task_cache.pop(owner, None)+ self._work_sets.pop(owner, None)+ return task_exists++ async def delete_expired(+ self, policy: TaskRetentionPolicy, *, now: float, limit: int+ ) -> DeleteExpiredResult:+ if limit <= 0:+ return DeleteExpiredResult()+ deleted: list[DeletedTask] = []+ scanned = 0+ scan_complete = False+ async with self._cleanup_lock.acquire("scan"):+ if self._cleanup_scan is None:+ self._cleanup_scan = self._iter_task_payloads()+ for _ in range(limit):+ try:+ payload = await anext(self._cleanup_scan)+ except StopAsyncIteration:+ self._cleanup_scan = None+ scan_complete = True+ break+ scanned += 1+ if payload is None:+ continue+ try:+ expired = policy.is_expired(+ str(payload.get("status", "")),+ float(payload.get("updated_at", 0.0)),+ now,+ )+ except (TypeError, ValueError):+ expired = False+ if not expired:+ continue+ owner = (+ str(payload.get("account_id") or ""),+ str(payload.get("user_id") or ""),+ str(payload.get("task_id") or ""),+ )+ if not all(owner):+ continue+ account_id, user_id, task_id = owner+ async with self._record_locks.acquire(owner):+ task = await self._load_task(task_id, account_id, user_id)+ work_set = self._work_sets.get(owner)+ if task is None or (work_set is not None and work_set.open_count):+ continue+ if not policy.is_expired(task.status.value, task.updated_at, now):+ continue+ try:+ await self._delete_task_unlocked(owner, task_exists=True)+ except BaseException:+ self._cleanup_scan = None+ raise+ deleted.append(DeletedTask(task_id, account_id, user_id))+ return DeleteExpiredResult(tuple(deleted), scanned, scan_complete)++ def snapshot_task_stats(self) -> TaskStatsSnapshot:+ grouped: Dict[str, Dict[str, int]] = defaultdict(lambda: defaultdict(int))+ with self._lock:+ for task in self._task_cache.values():+ grouped[task.task_type][task.status.value] += 1+ return {task_type: dict(statuses) for task_type, statuses in grouped.items()}++ async def _selected_works(+ self,+ task_id: str,+ work_ids: Optional[Collection[str]],+ account_id: Optional[str],+ user_id: Optional[str],+ ) -> List[WorkRecord]:+ task = await self._load_task(task_id, account_id, user_id)+ if task is None or not task.account_id or not task.user_id:+ return []+ selected = set(work_ids) if work_ids is not None else None+ with self._lock:+ work_set = self._work_sets.get((task.account_id, task.user_id, task_id))+ return [+ deepcopy(work)+ for work in (work_set.works.values() if work_set else ())+ if selected is None or work.work_id in selected+ ]++ async def _mutate_task(+ self,+ task_id: str,+ account_id: Optional[str],+ user_id: Optional[str],+ mutate: Callable[[TaskRecord, _WorkSet], StoreOperationResult],+ ) -> StoreOperationResult:+ task = await self._load_task(task_id, account_id, user_id)+ if task is None or not task.account_id or not task.user_id:+ return StoreOperationResult.rejected(StoreOperationCode.NOT_FOUND)+ owner = (task.account_id, task.user_id, task_id)+ async with self._record_locks.acquire(owner):+ await self._load_task(task_id, owner[0], owner[1])+ with self._lock:+ live_task = self._task_cache.get(owner)+ if live_task is None:+ return StoreOperationResult.rejected(StoreOperationCode.NOT_FOUND)+ work_set = self._work_sets.setdefault(owner, _WorkSet({}))+ candidate = deepcopy(live_task)+ previous = deepcopy(live_task)+ mutation = mutate(candidate, work_set)+ if inspect.isawaitable(mutation):+ raise TypeError("File store mutations must be synchronous")+ if not isinstance(mutation, StoreOperationResult):+ raise TypeError("Task mutations must return StoreOperationResult")+ if not mutation.is_applied:+ return mutation+ candidate.updated_at = TaskRecord.next_updated_at(candidate.updated_at)+ self._append_events(candidate, previous)+ snapshot = deepcopy(candidate)+ await self._write_task_and_publish(snapshot, owner, candidate)+ return mutation++ async def _mutate_work(+ self,+ task_id: str,+ work_id: str,+ account_id: Optional[str],+ user_id: Optional[str],+ mutate: Callable[[TaskRecord, Optional[WorkRecord]], _WorkMutationResult],+ *,+ try_finalize: bool = False,+ finalize_when_unchanged: bool = False,+ ) -> StoreOperationResult:+ task = await self._load_task(task_id, account_id, user_id)+ if task is None or not task.account_id or not task.user_id:+ return StoreOperationResult.rejected(StoreOperationCode.NOT_FOUND)+ owner = (task.account_id, task.user_id, task_id)+ async with self._record_locks.acquire(owner):+ await self._load_task(task_id, owner[0], owner[1])+ with self._lock:+ live_task = self._task_cache.get(owner)+ if live_task is None:+ return StoreOperationResult.rejected(StoreOperationCode.NOT_FOUND)+ work_set = self._work_sets.setdefault(owner, _WorkSet({}))+ existing = work_set.works.get(work_id)+ mutation = mutate(live_task, deepcopy(existing) if existing is not None else None)+ if inspect.isawaitable(mutation):+ raise TypeError("File store mutations must be synchronous")+ if not isinstance(mutation, _WorkMutationResult):+ raise TypeError("Work mutations must return _WorkMutationResult")+ old_open = existing is not None and existing.is_open()+ new_open = mutation.work is not None and mutation.work.is_open()+ open_count = work_set.open_count + int(new_open) - int(old_open)++ candidate_task: Optional[TaskRecord] = None+ candidate_work_set: Optional[_WorkSet] = None+ finalized = False+ if (+ try_finalize+ and open_count == 0+ and (mutation.changed or finalize_when_unchanged)+ ):+ candidate_task = deepcopy(live_task)+ previous = deepcopy(live_task)+ works = dict(work_set.works)+ if mutation.changed:+ if mutation.work is None:+ works.pop(work_id, None)+ else:+ works[work_id] = mutation.work+ candidate_work_set = _WorkSet(works, open_count)+ finalized = self._try_finalize(candidate_task, candidate_work_set)++ if finalized:+ assert candidate_task is not None+ assert candidate_work_set is not None+ candidate_task.updated_at = TaskRecord.next_updated_at(+ candidate_task.updated_at+ )+ self._append_events(candidate_task, previous)+ snapshot = deepcopy(candidate_task)+ else:+ if mutation.changed:+ if mutation.work is None:+ work_set.works.pop(work_id, None)+ else:+ work_set.works[work_id] = mutation.work+ work_set.open_count = open_count+ return mutation.result+ assert candidate_task is not None+ assert candidate_work_set is not None+ await self._write_task_and_publish(snapshot, owner, candidate_task, candidate_work_set)+ return StoreOperationResult.applied()++ async def _write_task_and_publish(+ self,+ snapshot: TaskRecord,+ owner: tuple[str, str, str],+ candidate_task: TaskRecord,+ candidate_work_set: Optional[_WorkSet] = None,+ ) -> None:+ write_task = asyncio.create_task(self._write_task(snapshot))+ cancellation: asyncio.CancelledError | None = Nonetry:- await self._agfs.rm(- self._task_path(account_id, user_id, task_id),- force=True,- auto_pathlock=False,+ while not write_task.done():+ try:+ await asyncio.shield(write_task)+ except asyncio.CancelledError as exc:+ if write_task.cancelled():+ raise+ cancellation = exc+ write_task.result()+ with self._lock:+ self._task_cache[owner] = candidate_task+ if candidate_work_set is not None:+ self._work_sets[owner] = candidate_work_set+ if cancellation is not None:+ raise cancellation+ finally:+ del write_task, cancellation++ def _try_finalize(self, task: TaskRecord, work_set: _WorkSet) -> bool:+ if task.status not in ACTIVE_TASK_STATUSES or work_set.open_count:+ return False+ if task.status == TaskStatus.CANCELLING:+ task.status = TaskStatus.CANCELLED+ else:+ work_error = first_work_error(work_set.works.values())+ if work_error and task.error is None:+ task.error = sanitize_error(work_error)+ if task.error is not None:+ task.status = TaskStatus.FAILED+ elif task.result is not None:+ task.status = TaskStatus.COMPLETED+ else:+ task.error = "Task operation interrupted before its result was persisted"+ task.status = TaskStatus.FAILED+ if isinstance(task.result, dict):+ # QueueFS reconstructs only unacknowledged Work after restart. For this+ # file backend, terminal Work history and its errors may therefore be+ # absent from the final projection until Work itself becomes durable.+ summary = summarize_works(work_set.works.values())+ projected = {+ queue_name: _queue_result(summary.queues.get(queue_name))+ for queue_name in ("Semantic", "Embedding")+ }+ has_projected_work = any(+ work.queue_name in projected for work in work_set.works.values()+ )+ if has_projected_work:+ task.result["queue_status"] = projected+ for queue_name, field_name in (+ ("Semantic", "semantic_status"),+ ("Embedding", "vector_status"),+ ):+ if task.result.get(field_name) == "queued":+ task.result[field_name] = (+ "failed"+ if projected.get(queue_name, {}).get("error_count", 0)+ else "complete"+ )+ task.stage = task.status.value+ task.auth = {}+ task.processing_seconds = processing_seconds(work_set.works.values())+ return True++ def _copy_task(self, task: TaskRecord, work_set: Optional[_WorkSet]) -> TaskRecord:+ copied = deepcopy(task)+ if copied.status in ACTIVE_TASK_STATUSES and work_set is not None:+ copied.processing_seconds = processing_seconds(work_set.works.values())+ return copied++ async def _load_task(+ self, task_id: str, account_id: Optional[str], user_id: Optional[str]+ ) -> Optional[TaskRecord]:+ with self._lock:+ owner = self._find_cached_owner(task_id, account_id, user_id)+ if owner is not None:+ return self._task_cache[owner]+ if not account_id or not user_id:+ return None+ try:+ payload = json.loads(+ _decode_bytes(await self._read(self._task_path(account_id, user_id, task_id))))except (AGFSNotFoundError, FileNotFoundError):+ return None+ task = _task_from_payload(payload, account_id=account_id, user_id=user_id)+ self._put_if_absent(task)+ with self._lock:+ return self._task_cache.get((account_id, user_id, task_id))++ def _find_cached_owner(+ self, task_id: str, account_id: Optional[str], user_id: Optional[str]+ ) -> Optional[tuple[str, str, str]]:+ if account_id and user_id:+ owner = (account_id, user_id, task_id)+ return owner if owner in self._task_cache else None+ return max(+ (+ owner+ for owner in self._task_cache+ for account, user, cached_id in (owner,)+ if cached_id == task_id+ and (account_id is None or account == account_id)+ and (user_id is None or user == user_id)+ ),+ key=lambda owner: self._task_cache[owner].updated_at,+ default=None,+ )++ def _put_if_absent(self, task: TaskRecord) -> None:+ if not task.account_id or not task.user_id:return+ owner = (task.account_id, task.user_id, task.task_id)+ with self._lock:+ self._task_cache.setdefault(owner, task)+ self._work_sets.setdefault(owner, _WorkSet({}))- async def _write_task(self, task: Any) -> None:- account_id = getattr(task, "account_id", None)- user_id = getattr(task, "user_id", None)- if not account_id or not user_id:+ @staticmethod+ def _append_events(task: TaskRecord, previous: Optional[TaskRecord]) -> None:+ from datetime import datetime, timezone++ recorded_at = datetime.now(timezone.utc).isoformat()+ if previous is None:+ kinds = ["created"]+ else:+ kinds = []+ if task.error is not None and previous.error is None:+ kinds.append("error_recorded")+ if task.status == TaskStatus.CANCELLED and previous.status != TaskStatus.CANCELLING:+ task.execution_events = append_task_event(+ task.execution_events,+ kind="status_changed",+ status=TaskStatus.CANCELLING.value,+ stage=sanitize_error(previous.stage) if previous.stage else None,+ recorded_at=recorded_at,+ )+ if task.status != previous.status:+ kinds.append("status_changed")+ if task.stage != previous.stage and task.status not in TERMINAL_TASK_STATUSES:+ kinds.append("stage_changed")++ stage = previous.stage if previous and task.status in TERMINAL_TASK_STATUSES else task.stage+ for kind in kinds:+ task.execution_events = append_task_event(+ task.execution_events,+ kind=kind,+ status=task.status.value,+ stage=sanitize_error(stage) if stage else None,+ error=sanitize_error(task.error)+ if kind == "error_recorded" and task.error+ else None,+ recorded_at=recorded_at,+ )++ async def _read(self, path: str) -> Any:+ return await self._io.run("read", lambda: run_to_completion(lambda: self._agfs.read(path)))++ async def _ls(self, path: str) -> Any:+ return await self._io.run("ls", lambda: run_to_completion(lambda: self._agfs.ls(path)))++ async def _rm(self, path: str) -> None:+ await self._io.run(+ "rm",+ lambda: run_to_completion(lambda: self._agfs.rm(path, force=True, auto_pathlock=False)),+ )++ async def _write_task(self, task: TaskRecord) -> None:+ if not task.account_id or not task.user_id:raise ValueError("PersistentTaskStore requires account_id and user_id")- await self._ensure_task_dir(account_id, user_id)- path = self._task_path(account_id, user_id, task.task_id)- payload = json.dumps(_task_to_payload(task), ensure_ascii=False).encode("utf-8")+ await self._ensure_task_dir(task.account_id, task.user_id)+ payload = json.dumps(_task_to_payload(task), ensure_ascii=False).encode()+ path = self._task_path(task.account_id, task.user_id, task.task_id)try:- await self._write_task_payload(path, payload)+ await self._io.run(+ "write",+ lambda: run_to_completion(+ lambda: self._agfs.write(path, payload, auto_pathlock=False)+ ),+ )except (AGFSNotFoundError, FileNotFoundError):- self._ensured_task_dirs.discard((account_id, user_id))- await self._ensure_task_dir(account_id, user_id)- await self._write_task_payload(path, payload)+ self._ensured_task_dirs.discard((task.account_id, task.user_id))+ await self._ensure_task_dir(task.account_id, task.user_id)+ await self._io.run(+ "write",+ lambda: run_to_completion(+ lambda: self._agfs.write(path, payload, auto_pathlock=False)+ ),+ )async def _ensure_task_dir(self, account_id: str, user_id: str) -> None:- cache_key = (account_id, user_id)- if cache_key in self._ensured_task_dirs:+ key = (account_id, user_id)+ if key in self._ensured_task_dirs:return-- seen_paths: set[str] = set()for path in self._task_dir_chain(account_id, user_id):- if path in seen_paths:+ try:+ await self._io.run(+ "mkdir",+ lambda path=path: run_to_completion(lambda: self._agfs.mkdir(path)),+ )+ except AGFSAlreadyExistsError:+ pass+ except Exception as exc:+ if "already exists" not in str(exc).lower():+ raise+ self._ensured_task_dirs.add(key)++ async def _owner_users(self, account_id: str) -> List[str]:+ try:+ return [+ str(item["name"])+ for item in await self._ls(self._task_root_dir(account_id))+ if item.get("name")+ ]+ except (AGFSNotFoundError, FileNotFoundError):+ return []++ async def _list_owner_payloads(self, account_id: str, user_id: str) -> List[Dict[str, Any]]:+ try:+ entries = await self._ls(self._task_dir(account_id, user_id))+ except (AGFSNotFoundError, FileNotFoundError):+ return []+ payloads = []+ for item in entries:+ path = (+ item.get("path") or f"{self._task_dir(account_id, user_id)}/{item.get('name', '')}"+ )+ if not path.endswith(".json"):continue- seen_paths.add(path)- await self._mkdir_if_missing(path)- self._ensured_task_dirs.add(cache_key)+ try:+ value = json.loads(_decode_bytes(await self._read(path)))+ if isinstance(value, dict):+ payloads.append(value)+ except (AGFSNotFoundError, FileNotFoundError, ValueError, TypeError):+ continue+ return payloads++ async def _iter_task_payloads(self) -> AsyncIterator[Optional[Dict[str, Any]]]:+ seen = set()+ with self._lock:+ cached = list(self._task_cache.items())+ for key, task in cached:+ seen.add(key)+ yield _task_to_payload(task)+ try:+ accounts = await self._ls(self.ROOT_PREFIX)+ except (AGFSNotFoundError, FileNotFoundError):+ return+ for account in accounts:+ raw_account_id = account.get("name")+ if not raw_account_id:+ continue+ account_id = str(raw_account_id)+ for user_id in await self._owner_users(account_id):+ try:+ entries = await self._ls(self._task_dir(account_id, user_id))+ except (AGFSNotFoundError, FileNotFoundError):+ continue+ for item in entries:+ path = item.get("path") or (+ f"{self._task_dir(account_id, user_id)}/{item.get('name', '')}"+ )+ if not path.endswith(".json"):+ continue+ task_id = path.rsplit("/", 1)[-1][:-5]+ if (account_id, user_id, task_id) in seen:+ continue+ try:+ payload = json.loads(_decode_bytes(await self._read(path)))+ except (+ AGFSNotFoundError,+ FileNotFoundError,+ ValueError,+ TypeError,+ ):+ yield None+ continue+ yield payload if isinstance(payload, dict) else Nonedef _task_dir_chain(self, account_id: str, user_id: str) -> tuple[str, ...]:return (- self._account_dir(account_id),+ f"{self.ROOT_PREFIX}/{account_id}",self._system_dir(account_id),self._task_root_dir(account_id),self._task_dir(account_id, user_id),)- async def _write_task_payload(self, path: str, payload: bytes) -> None:- # TaskTracker is the owner of task mutations and serializes updates per- # task. PersistentTaskStore does not implement store-level revision/CAS,- # so AGFS pathlock cannot make multiple independent writers correct; it- # only adds one storage lock around every task lifecycle write. Avoid- # that extra PathLock overhead on this internal task file.- await self._agfs.write(path, payload, auto_pathlock=False)-- async def _mkdir_if_missing(self, path: str) -> None:- try:- await self._agfs.mkdir(path)- except AGFSAlreadyExistsError:- return- except Exception as exc:- if "already exists" in str(exc).lower():- return- raise-- def _account_dir(self, account_id: str) -> str:- return f"{self.ROOT_PREFIX}/{account_id}"-def _system_dir(self, account_id: str) -> str:if account_id == SYSTEM_TASK_ACCOUNT_ID:- return self._account_dir(account_id)- return f"{self._account_dir(account_id)}/{self.SYSTEM_DIRNAME}"+ return f"{self.ROOT_PREFIX}/{account_id}"+ return f"{self.ROOT_PREFIX}/{account_id}/{self.SYSTEM_DIRNAME}"def _task_root_dir(self, account_id: str) -> str:return f"{self._system_dir(account_id)}/{self.TASKS_DIRNAME}"@@ -255,13 +1759,42 @@ class PersistentTaskStore:return f"{self._task_dir(account_id, user_id)}/{task_id}.json"-def _task_to_payload(task: Any) -> Dict[str, Any]:- status = getattr(task, "status", None)+def _deep_merge(target: Dict[str, Any], values: Dict[str, Any]) -> None:+ for key, value in values.items():+ if isinstance(value, dict) and isinstance(target.get(key), dict):+ _deep_merge(target[key], value)+ else:+ target[key] = value+++def _inactive_task_result(task: TaskRecord, message: str) -> StoreOperationResult:+ if task.status in TERMINAL_TASK_STATUSES:+ return StoreOperationResult.rejected(+ StoreOperationCode.ALREADY_TERMINAL,+ f"{message}: task is already {task.status.value}",+ )+ return StoreOperationResult.rejected(+ StoreOperationCode.INVALID_STATE,+ f"{message}: task is {task.status.value}",+ )+++def _queue_result(summary: Optional[QueueWorkSummary]) -> Dict[str, Any]:+ summary = summary or QueueWorkSummary()return {- **deepcopy(getattr(task, "_extra_fields", {})),+ "processed": summary.processed,+ "requeue_count": summary.requeue_count,+ "error_count": summary.error_count,+ "errors": [{"message": error} for error in summary.errors[:MAX_QUEUE_ERROR_DETAILS]],+ }+++def _task_to_payload(task: TaskRecord) -> Dict[str, Any]:+ return {+ **deepcopy(task._extra_fields),"task_id": task.task_id,"task_type": task.task_type,- "status": status.value if hasattr(status, "value") else status,+ "status": task.status.value,"created_at": task.created_at,"updated_at": task.updated_at,"processing_seconds": task.processing_seconds,@@ -277,7 +1810,27 @@ def _task_to_payload(task: Any) -> Dict[str, Any]:}+def _task_from_payload(+ payload: Dict[str, Any],+ *,+ account_id: Optional[str] = None,+ user_id: Optional[str] = None,+) -> TaskRecord:+ known = {+ item.name for item in fields(TaskRecord) if item.init and not item.name.startswith("_")+ }+ data = {key: deepcopy(value) for key, value in payload.items() if key in known}+ data["status"] = TaskStatus(data["status"])+ if account_id is not None:+ data["account_id"] = account_id+ if user_id is not None:+ data["user_id"] = user_id+ task = TaskRecord(**data)+ task._extra_fields = {+ key: deepcopy(value) for key, value in payload.items() if key not in known+ }+ return task++def _decode_bytes(raw: Any) -> str:- if isinstance(raw, bytes):- return raw.decode("utf-8")- return str(raw)+ return raw.decode() if isinstance(raw, bytes) else str(raw)diff --git a/openviking/service/task_tracker.py b/openviking/service/task_tracker.pyindex 201e34366..dab4495d2 100644--- a/openviking/service/task_tracker.py+++ b/openviking/service/task_tracker.py@@ -1,50 +1,94 @@# Copyright (c) 2026 Beijing Volcano Engine Technology Co., Ltd.# SPDX-License-Identifier: AGPL-3.0-"""-Async Task Tracker for OpenViking.+"""Coordinate Task lifecycle, queue Work, waiting, and cancellation."""-Provides a lightweight registry for tracking background operations-(e.g. session commit with wait=false). Callers receive a task_id that can be-polled via the /tasks API to check completion status, results, or errors.--Design decisions:- - Thread-safe (QueueManager workers run in separate threads).- - TTL-based cleanup applies to the process-local cache.- - Error messages are sanitized to avoid leaking sensitive data.-"""+from __future__ import annotationsimport asyncio-import mathimport reimport threadingimport timefrom contextlib import contextmanagerfrom contextvars import ContextVarfrom copy import deepcopy-from dataclasses import dataclass, field, fields-from datetime import datetime, timezone-from enum import Enum-from typing import Any, Awaitable, Callable, Dict, Iterator, List, Optional, TypeVar+from dataclasses import dataclass+from typing import (+ Any,+ Awaitable,+ Callable,+ Collection,+ Dict,+ Generator,+ List,+ Optional,+ TypeVar,+ cast,+)from uuid import uuid4-from openviking.service.task_events import (- PROCESS_EVENT_KINDS,- TaskEventHistory,- append_task_event,+from openviking.service.task_domain import (+ ACTIVE_TASK_STATUSES,+ MAX_QUEUE_ERROR_DETAILS,+ SENSITIVE_TASK_KEYS,+ TERMINAL_TASK_STATUSES,+ TERMINAL_WORK_STATES,+ DeleteExpiredResult,+ QueueWorkSummary,+ TaskRecord,+ TaskRetentionPolicy,+ TaskStatus,+ TaskWorkRejected,+ WorkRecord,+ WorkRecoveryMode,+ WorkState,+ WorkSummary,+ sanitize_error,+ sanitize_task_result,)-from openviking.service.task_store import TaskStore-from openviking.service.task_tracker_concurrency import (- KeyedAsyncLockPool,- StoreIOLimiter,- run_to_completion,+from openviking.service.task_events import PROCESS_EVENT_KINDS+from openviking.service.task_store import (+ SYSTEM_TASK_ACCOUNT_ID,+ SYSTEM_TASK_USER_ID,+ StoreOperationCode,+ StoreOperationResult,+ TaskAlreadyActive,+ TaskWorkStore,+ WorkRegistration,)-from openviking.service.task_work_index import QueueTaskMetadata, TaskWorkIndex+from openviking.service.task_tracker_concurrency import run_to_completion+from openviking.service.task_work_index import TaskWorkIndexfrom openviking_cli.exceptions import DeadlineExceededError, OpenVikingErrorfrom openviking_cli.utils.logger import get_loggerlogger = get_logger(__name__)_OperationResult = TypeVar("_OperationResult")+__all__ = [+ "SENSITIVE_TASK_KEYS",+ "TaskAlreadyActive",+ "TaskRecord",+ "TaskStatus",+ "TaskTracker",+ "get_task_tracker",+ "run_tracked_operation",+ "set_task_tracker",+]++_CANCELLABLE_TASK_TYPES = {+ "add_resource",+ "add_skill",+ "compile",+ "session_commit",+ "admin_reindex",+ "snapshot_restore_reindex",+}+_CANCEL_POLL_INTERVAL = 30+_TASK_WAIT_POLL_INTERVAL = 0.05+_PUBLIC_QUEUE_NAMES = ("Semantic", "Embedding")++_sanitize_error = sanitize_error+_sanitize_task_result = sanitize_task_result+@dataclassclass _WaitBudget:@@ -56,7 +100,9 @@ _wait_budget: ContextVar[Optional[_WaitBudget]] = ContextVar("task_wait_budget",@contextmanager-def _consume_wait_budget(task_id: str, timeout: Optional[float]) -> Iterator[Optional[float]]:+def _consume_wait_budget(+ task_id: str, timeout: Optional[float]+) -> Generator[Optional[float], None, None]:"""Charge only time spent waiting, shared across an inline operation."""budget = _wait_budget.get()if budget is None or budget.task_id != task_id:@@ -70,97 +116,14 @@ def _consume_wait_budget(task_id: str, timeout: Optional[float]) -> Iterator[Optbudget.remaining = max(0.0, budget.remaining - (time.monotonic() - started))-class _CommittedMutationCancelled(asyncio.CancelledError):- """Caller cancellation observed after store and cache commit completed."""---class TaskStatus(str, Enum):- """Lifecycle states of an async task."""-- PENDING = "pending"- RUNNING = "running"- CANCELLING = "cancelling"- COMPLETED = "completed"- FAILED = "failed"- CANCELLED = "cancelled"---_TERMINAL_STATUSES = (TaskStatus.COMPLETED, TaskStatus.FAILED, TaskStatus.CANCELLED)-_ACTIVE_STATUSES = (TaskStatus.PENDING, TaskStatus.RUNNING, TaskStatus.CANCELLING)-_PUBLIC_QUEUE_NAMES = ("Semantic", "Embedding")--_CANCELLABLE_TASK_TYPES = {- "add_resource",- "add_skill",- "compile",- "session_commit",- "admin_reindex",- "snapshot_restore_reindex",-}---@dataclass-class TaskRecord:- """Immutable snapshot of an async task."""-- task_id: str- task_type: str # e.g. "session_commit"- status: TaskStatus = TaskStatus.PENDING- created_at: float = field(default_factory=time.time)- updated_at: float = field(default_factory=time.time)- resource_id: Optional[str] = None # e.g. session_id- account_id: Optional[str] = None- user_id: Optional[str] = None- meta: Dict[str, Any] = field(default_factory=dict)- stage: Optional[str] = None- result: Optional[Dict[str, Any]] = None- error: Optional[str] = None- auth: Dict[str, Any] = field(default_factory=dict, repr=False)- execution_events: Optional[TaskEventHistory] = None- processing_seconds: Optional[float] = None- _extra_fields: Dict[str, Any] = field(- default_factory=dict, init=False, repr=False, compare=False- )-- def to_dict(self, *, include_events: bool = False) -> Dict[str, Any]:- """Serialize for JSON response."""- public_meta = deepcopy(self.meta)- public_meta.pop("submission_hash", None)- public_meta.pop("submission_token", None)- return {- **({"execution_events": deepcopy(self.execution_events)} if include_events else {}),- "task_id": self.task_id,- "task_type": self.task_type,- "status": self.status.value,- "created_at": self.created_at,- "updated_at": self.updated_at,- "processing_seconds": self.processing_seconds,- "resource_id": self.resource_id,- "meta": _sanitize_task_result(public_meta),- "stage": self.stage,- "result": _sanitize_task_result(deepcopy(self.result)),- "error": self.error,- "created_at_iso": datetime.fromtimestamp(self.created_at, tz=timezone.utc).isoformat(),- "updated_at_iso": datetime.fromtimestamp(self.updated_at, tz=timezone.utc).isoformat(),- }---# ── Singleton ──-_instance: Optional["TaskTracker"] = None_init_lock = threading.Lock()def get_task_tracker() -> "TaskTracker":- """Get the global TaskTracker singleton installed by service storage initialization."""+ """Return the tracker installed during service storage initialization."""with _init_lock:if _instance is None:- logger.error(- "TaskTracker accessed before service storage initialization; refusing to create "- "a separate AGFS client. Ensure OpenVikingService installs the shared tracker "- "with set_task_tracker() before task APIs are used.",- stack_info=True,- )raise RuntimeError("TaskTracker not initialized. OpenVikingService must install the shared ""tracker with set_task_tracker() during storage initialization."@@ -168,8 +131,7 @@ def get_task_tracker() -> "TaskTracker":return _instance-def set_task_tracker(tracker: "TaskTracker") -> None:- """Replace the global TaskTracker singleton."""+def set_task_tracker(tracker: Optional["TaskTracker"]) -> None:global _instancewith _init_lock:_instance = tracker@@ -186,11 +148,14 @@ async def run_tracked_operation(resource_id: Optional[str] = None,existing_task: Optional[TaskRecord] = None,accept_failed_result: Callable[[_OperationResult], bool] | None = None,+ return_terminal_result: bool = False,) -> tuple[_OperationResult, str]:"""Run a top-level operation under a Task, reusing an existing Task context.``accept_failed_result`` is reserved for operations whose result explicitlyreports record-level failures while the Task retains its failed status.+ ``return_terminal_result`` returns the Task's final projected result after+ waiting instead of the operation's pre-Work result."""from openviking.service.task_work_index import bind_task_context, get_task_context@@ -199,7 +164,7 @@ async def run_tracked_operation(return await operation(), current.task_idtracker = get_task_tracker()- created_task: Optional[TaskRecord] = existing_task+ created_task = existing_taskasync def create_task() -> None:nonlocal created_task@@ -211,8 +176,6 @@ async def run_tracked_operation()try:- # Keep the committed record reachable even if the caller is cancelled- # while the store write is still running.if created_task is None:await run_to_completion(create_task)assert created_task is not None@@ -222,27 +185,30 @@ async def run_tracked_operation()except BaseException as exc:if created_task is not None:- failed_task_id = created_task.task_idfailure_message = str(exc) or type(exc).__name__await run_to_completion(lambda: tracker.fail(- failed_task_id,+ created_task.task_id,failure_message,account_id=account_id,user_id=user_id,))raise- enqueued_work: list[tuple[str, str]] = []async def execute_operation() -> _OperationResult:- tracker.register_running_task(task.task_id)+ running_work_id = await tracker.register_running_task(+ task.task_id,+ account_id=account_id,+ user_id=user_id,+ )try:with bind_task_context(task.task_id,account_id,user_id,- enqueued_work=enqueued_work,+ enqueued_work=[],+ work_id=running_work_id,):result = await operation()task_result = deepcopy(result) if isinstance(result, dict) else {"result": result}@@ -257,8 +223,6 @@ async def run_tracked_operation(if isinstance(result, dict) and not wait:result.setdefault("task_id", task.task_id)return result- except _CommittedMutationCancelled:- raiseexcept BaseException as exc:failure_message = str(exc) or type(exc).__name__await run_to_completion(@@ -286,6 +250,10 @@ async def run_tracked_operation(user_id=user_id,timeout=timeout,)+ if return_terminal_result and terminal.result is not None:+ result = cast(_OperationResult, deepcopy(terminal.result))+ if isinstance(result, dict):+ result.pop("task_id", None)if terminal.status == TaskStatus.FAILED and not (accept_failed_result is not None and accept_failed_result(result)):@@ -307,86 +275,38 @@ async def run_tracked_operation(_wait_budget.reset(token)-# ── Sanitization ──--_SENSITIVE_PATTERNS = re.compile(- r"(sk-|cr_|ghp_|ntn_|xox[baprs]-|Bearer\s+)[a-zA-Z0-9._-]+",- re.IGNORECASE,-)--_MAX_ERROR_LEN = 500-SENSITIVE_TASK_KEYS = frozenset({"api_key", "user_key"})---def _sanitize_error(error: str) -> str:- """Remove potential secrets from error messages."""- sanitized = _SENSITIVE_PATTERNS.sub("[REDACTED]", error)- if len(sanitized) > _MAX_ERROR_LEN:- sanitized = sanitized[:_MAX_ERROR_LEN] + "...[truncated]"- return sanitized---def _sanitize_task_result(result: Any) -> Any:- """Remove sensitive fields from task results before exposing snapshots."""- if isinstance(result, dict):- return {- key: _sanitize_task_result(value)- for key, value in result.items()- if key not in SENSITIVE_TASK_KEYS- }- if isinstance(result, list):- return [_sanitize_task_result(item) for item in result]- return result---# ── TaskTracker ──--class TaskTracker:- """Async task tracker with persistent storage and a process-local cache.+ """Thin coordinator over the semantic :class:`TaskWorkStore` contract."""- Mutations are serialized per task across caller threads and event loops.- The thread lock only protects short accesses to immutable cache snapshots.- """-- MAX_TASKS = 10_000- TTL_COMPLETED = 86_400 # 24 hours- TTL_FAILED = 604_800 # 7 days- CLEANUP_INTERVAL = 300 # 5 minutes+ TTL_COMPLETED = 86_400+ TTL_FAILED = 604_800+ CLEANUP_INTERVAL = 300+ CLEANUP_BATCH = 256- def __init__(self, store: TaskStore, *, max_concurrent_store_io: int = 8) -> None:+ def __init__(self, store: TaskWorkStore, *, max_concurrent_store_io: int = 8) -> None:+ del max_concurrent_store_ioself._store = store- self._tasks: Dict[str, TaskRecord] = {}- self._lock = threading.Lock()- # The keyed registries provide process-local ordering. A deployment with- # multiple TaskTracker writers must add store-level revision/CAS first.- self._task_locks = KeyedAsyncLockPool[str]()- self._business_locks = KeyedAsyncLockPool[tuple[str, str, str, str]]()- self._store_io = StoreIOLimiter(max_concurrent_store_io)- self._cleanup_task: Optional[asyncio.Task] = None+ self._cleanup_task: Optional[asyncio.Task[Any]] = None+ self._cancel_poll_task: Optional[asyncio.Task[Any]] = Noneself._work_index = TaskWorkIndex()- self._install_work_index_callbacks()- logger.info(- "[TaskTracker] Initialized (store=%s, max_tasks=%d)",- self._store.__class__.__name__,- self.MAX_TASKS,+ self._running: ContextVar[tuple[str, str, str, bool] | None] = ContextVar(+ "task_tracker_running_work", default=None)- # ── Lifecycle ──-- def _install_work_index_callbacks(self) -> None:- self._work_index.set_callbacks(- finalize_before_ack=self._finalize_before_ack,- is_cancellation_requested=self.is_cancellation_requested,- )+ @staticmethod+ def _validate_owner(account_id: str, user_id: str) -> None:+ if not account_id or not user_id:+ raise ValueError("Task ownership requires non-empty account_id and user_id")def attach_work_index(self, work_index: TaskWorkIndex) -> None:- """Use the QueueManager-owned index as the task lifecycle authority."""+ """Replace the process-local active-handle index."""self._work_index = work_index- self._install_work_index_callbacks()++ @property+ def work_recovery_mode(self) -> WorkRecoveryMode:+ return self._store.work_recovery_modeasync def restore_work_tasks(self, owners: Dict[str, tuple[str, str]]) -> List[TaskRecord]:- """Restore task records referenced by rebuilt QueueFS work."""restored = []for task_id, (account_id, user_id) in owners.items():task = await self.get(task_id, account_id=account_id, user_id=user_id)@@ -394,143 +314,65 @@ class TaskTracker:restored.append(task)return restored- async def _finalize_before_ack(self, metadata: QueueTaskMetadata) -> None:- """Persist the terminal state before QueueFS removes the last recovery message."""- await self._finalize_task(- metadata.task_id,- account_id=metadata.account_id or None,- user_id=metadata.user_id or None,- )-- def is_cancellation_requested(self, task_id: str) -> bool:- """Fast thread-safe status check used by queue workers."""- with self._lock:- task = self._tasks.get(task_id)- return task is not None and task.status in (- TaskStatus.CANCELLING,- TaskStatus.CANCELLED,- )-def start_cleanup_loop(self) -> None:- """Start the background TTL cleanup coroutine.-- Safe to call multiple times; subsequent calls are no-ops.- Must be called from within a running event loop.- """- if self._cleanup_task is not None and not self._cleanup_task.done():- return- self._cleanup_task = asyncio.create_task(self._cleanup_loop())- logger.debug("[TaskTracker] Cleanup loop started")+ if self._cleanup_task is None or self._cleanup_task.done():+ self._cleanup_task = asyncio.create_task(self._cleanup_loop())+ if self._cancel_poll_task is None or self._cancel_poll_task.done():+ self._cancel_poll_task = asyncio.create_task(self._cancel_poll_loop())def stop_cleanup_loop(self) -> None:- """Cancel the background cleanup task. Safe to call if not started."""- if self._cleanup_task is not None and not self._cleanup_task.done():- self._cleanup_task.cancel()- logger.debug("[TaskTracker] Cleanup loop stopped")+ for task in (self._cleanup_task, self._cancel_poll_task):+ if task is not None and not task.done():+ task.cancel()async def _cleanup_loop(self) -> None:while True:try:await asyncio.sleep(self.CLEANUP_INTERVAL)- await self._evict_expired()+ await self.delete_expired()except asyncio.CancelledError:breakexcept Exception:- logger.exception("[TaskTracker] Cleanup error")-- async def _evict_expired(self) -> None:- """Remove expired tasks and enforce MAX_TASKS."""- now = time.time()- with self._lock:- expired_ids = [- task_id for task_id, task in self._tasks.items() if self._is_expired(task, now)- ]-- evicted_count = 0- for task_id in expired_ids:- evicted_count += await self._delete_expired_task(task_id, now)-- capacity_evicted: list[tuple[str, bool]] = []- with self._lock:- if len(self._tasks) > self.MAX_TASKS:- sorted_tasks = sorted(self._tasks.items(), key=lambda x: x[1].created_at)- excess = len(self._tasks) - self.MAX_TASKS- for tid, task in sorted_tasks[:excess]:- self._tasks.pop(tid, None)- capacity_evicted.append((tid, task.status in _TERMINAL_STATUSES))-- for tid, terminal in capacity_evicted:- if terminal:- self._work_index.forget_task(tid)- else:- self._work_index.forget_processing(tid)-- if evicted_count:- logger.debug("[TaskTracker] Evicted %d expired tasks", evicted_count)-- def _is_expired(self, task: TaskRecord, now: float) -> bool:- age = now - task.updated_at- if task.status in (TaskStatus.COMPLETED, TaskStatus.CANCELLED):- return age > self.TTL_COMPLETED- return task.status == TaskStatus.FAILED and age > self.TTL_FAILED-- async def _delete_expired_task(self, task_id: str, now: float) -> bool:- async with self._task_locks.acquire(task_id):- task = self._cached_task(task_id)- if task is None or not self._is_expired(task, now):- return False- account_id = task.account_id- user_id = task.user_id- if not account_id or not user_id:- logger.warning(- "[TaskTracker] Cannot delete expired ownerless task %s",- task_id,- )- return False+ logger.exception("[TaskTracker] cleanup error")++ async def _cancel_poll_loop(self) -> None:+ while True:try:- await self._store_io.run(- "delete",- lambda: run_to_completion(- lambda: self._store.delete(+ await asyncio.sleep(_CANCEL_POLL_INTERVAL)+ for (+ account_id,+ user_id,+ ), task_ids in self._work_index.active_tasks_by_owner().items():+ cancelling = await self._store.filter_cancellation_requested(+ task_ids,+ account_id=account_id,+ user_id=user_id,+ )+ for task_id in cancelling:+ self._work_index.cancel_active(task_id,account_id=account_id,user_id=user_id,)- ),- )+ except asyncio.CancelledError:+ breakexcept Exception:- logger.warning(- "[TaskTracker] Failed to delete expired task %s",- task_id,- exc_info=True,- )- return False-- with self._lock:- self._tasks.pop(task_id, None)- self._work_index.forget_task(task_id)- return True-- @staticmethod- def _matches_owner(- task: TaskRecord,- account_id: Optional[str] = None,- user_id: Optional[str] = None,- ) -> bool:- """Return True when a task belongs to the requested owner filter."""- if account_id is not None and task.account_id != account_id:- return False- if user_id is not None and task.user_id != user_id:- return False- return True-- @staticmethod- def _validate_owner(account_id: str, user_id: str) -> None:- """Reject ownerless task creation for user-originated background work."""- if not account_id or not user_id:- raise ValueError("Task ownership requires non-empty account_id and user_id")+ logger.exception("[TaskTracker] cancel-poll error")- # ── CRUD ──+ async def delete_expired(self) -> DeleteExpiredResult:+ result = await self._store.delete_expired(+ TaskRetentionPolicy(self.TTL_COMPLETED, self.TTL_FAILED),+ now=time.time(),+ limit=self.CLEANUP_BATCH,+ )+ for deleted in result.deleted:+ logger.info(+ "[TaskTracker] deleted expired task task_id=%s account_id=%s user_id=%s",+ deleted.task_id,+ deleted.account_id,+ deleted.user_id,+ )+ return resultasync def create(self,@@ -542,9 +384,13 @@ class TaskTracker:task_id: Optional[str] = None,meta: Optional[Dict[str, Any]] = None,auth: Optional[Dict[str, Any]] = None,+ reject_if_active: bool = False,) -> TaskRecord:- """Register a new pending task. Returns a snapshot copy."""self._validate_owner(account_id, user_id)+ if task_id is not None:+ existing = await self.get(task_id, account_id=account_id, user_id=user_id)+ if existing is not None:+ return existingtask = TaskRecord(task_id=task_id or str(uuid4()),task_type=task_type,@@ -554,31 +400,10 @@ class TaskTracker:meta=dict(meta or {}),auth=dict(auth or {}),)- async with self._task_locks.acquire(task.task_id):- if task_id is not None:- existing = self._cached_task(task.task_id)- if existing is not None:- if not self._matches_owner(existing, task.account_id, task.user_id):- raise ValueError(- f"Task ID already belongs to another owner: {task.task_id}"- )- return self._copy(existing)- existing = await self._load_from_store(- task.task_id,- task.account_id or "",- task.user_id,- )- if existing is not None:- self._publish_task(existing)- return self._copy(existing)- await self._persist_and_publish("create", task)- logger.debug(- "[TaskTracker] Created task %s type=%s resource=%s",- task.task_id,- task.task_type,- task.resource_id,+ stored = await run_to_completion(+ lambda: self._store.create(task, reject_if_active=reject_if_active))- return self._copy(task)+ return self._public(stored or task)async def create_if_no_running(self,@@ -588,42 +413,16 @@ class TaskTracker:account_id: str,user_id: str,) -> Optional[TaskRecord]:- """Atomically check for running tasks and create a new one if none exist.-- Returns TaskRecord on success, None if a running task already exists.- This eliminates the race condition between has_running() and create().- """- self._validate_owner(account_id, user_id)- business_key = (account_id, user_id, task_type, resource_id)- async with self._business_locks.acquire(business_key):- self._merge_loaded_tasks(await self._load_all_from_store(account_id, user_id))-- tasks = self._cache_snapshot()- has_active = any(- t.task_type == task_type- and t.resource_id == resource_id- and self._matches_owner(t, account_id, user_id)- and t.status in _ACTIVE_STATUSES- for t in tasks- )- if has_active:- return None- task = TaskRecord(- task_id=str(uuid4()),- task_type=task_type,- resource_id=resource_id,+ try:+ return await self.create(+ task_type,+ resource_id,account_id=account_id,user_id=user_id,+ reject_if_active=True,)- async with self._task_locks.acquire(task.task_id):- await self._persist_and_publish("create", task)- logger.debug(- "[TaskTracker] Created task %s type=%s resource=%s",- task.task_id,- task_type,- resource_id,- )- return self._copy(task)+ except TaskAlreadyActive:+ return Noneasync def start(self,@@ -632,16 +431,7 @@ class TaskTracker:user_id: Optional[str] = None,stage: Optional[str] = None,) -> None:- """Transition task to RUNNING."""- async with self._task_locks.acquire(task_id):- task = await self._load_for_update(task_id, account_id, user_id)- if task and task.status in (TaskStatus.PENDING, TaskStatus.RUNNING):- updated = deepcopy(task)- updated.status = TaskStatus.RUNNING- if stage is not None:- updated.stage = stage- updated.updated_at = self._next_updated_at(task)- await self._persist_and_publish("update", updated, previous=task)+ await self._store.start_task(task_id, stage=stage, account_id=account_id, user_id=user_id)async def update_stage(self,@@ -652,16 +442,9 @@ class TaskTracker:*,meta: Optional[Dict[str, Any]] = None,) -> None:- """Update task progress without changing its lifecycle status."""- async with self._task_locks.acquire(task_id):- task = await self._load_for_update(task_id, account_id, user_id)- if task and task.status in _ACTIVE_STATUSES:- updated = deepcopy(task)- updated.stage = stage- if meta:- updated.meta.update(deepcopy(meta))- updated.updated_at = self._next_updated_at(task)- await self._persist_and_publish("update", updated, previous=task)+ await self._store.update_stage(+ task_id, stage, meta=meta, account_id=account_id, user_id=user_id+ )async def update_task_auth(self,@@ -671,15 +454,8 @@ class TaskTracker:account_id: str,user_id: str,) -> None:- """Persist private state required to resume an active task."""self._validate_owner(account_id, user_id)- async with self._task_locks.acquire(task_id):- task = await self._load_for_update(task_id, account_id, user_id)- if task and task.status in _ACTIVE_STATUSES:- updated = deepcopy(task)- updated.auth.update(deepcopy(values))- updated.updated_at = self._next_updated_at(task)- await self._persist_and_publish("update", updated, previous=task)+ await self._store.update_auth(task_id, values, account_id=account_id, user_id=user_id)async def record_feishu_response(self,@@ -689,16 +465,13 @@ class TaskTracker:account_id: str,user_id: str,) -> None:- """Persist a child submission before polling so a source retry can resume it."""-- async with self._task_locks.acquire(task_id):- task = await self._load_for_update(task_id, account_id, user_id)- if task is None or task.status not in (TaskStatus.PENDING, TaskStatus.RUNNING):- raise ValueError("Feishu source task is no longer active")- updated = deepcopy(task)- updated.meta.setdefault("feishu_responses", {})[entry] = response_id- updated.updated_at = self._next_updated_at(task)- await self._persist_and_publish("update", updated, previous=task)+ self._validate_owner(account_id, user_id)+ await self._store.merge_meta(+ task_id,+ {"feishu_responses": {entry: response_id}},+ account_id=account_id,+ user_id=user_id,+ )async def complete(self,@@ -709,13 +482,14 @@ class TaskTracker:*,resource_id: Optional[str] = None,) -> None:- """Record successful completion and finalize after owned work settles."""await self._record_outcome(task_id,- account_id,- user_id,result=result,+ error=None,resource_id=resource_id,+ cancelled=False,+ account_id=account_id,+ user_id=user_id,)async def fail(@@ -727,13 +501,14 @@ class TaskTracker:*,result: Optional[Dict[str, Any]] = None,) -> None:- """Record failure and optional structured metadata, then finalize."""await self._record_outcome(task_id,- account_id,- user_id,result=result,- error=error,+ error=sanitize_error(error),+ resource_id=None,+ cancelled=False,+ account_id=account_id,+ user_id=user_id,)async def mark_cancelled(@@ -742,12 +517,14 @@ class TaskTracker:account_id: Optional[str] = None,user_id: Optional[str] = None,) -> None:- """Record an externally observed cancellation without requesting local cancellation."""await self._record_outcome(task_id,- account_id,- user_id,- terminal_status=TaskStatus.CANCELLED,+ result=None,+ error=None,+ resource_id=None,+ cancelled=True,+ account_id=account_id,+ user_id=user_id,)async def record_cancelled(@@ -756,189 +533,185 @@ class TaskTracker:account_id: Optional[str] = None,user_id: Optional[str] = None,) -> None:- """Record cancellation reported by the task owner without cancelling its worker."""- async with self._task_locks.acquire(task_id):- task = await self._load_for_update(task_id, account_id, user_id)- if task and task.status in _ACTIVE_STATUSES:- updated = deepcopy(task)- updated.status = TaskStatus.CANCELLING- updated.updated_at = self._next_updated_at(task)- await self._persist_and_publish("update", updated, previous=task)- await self._finalize_task(task_id, account_id, user_id)+ await self.mark_cancelled(task_id, account_id, user_id)async def _record_outcome(self,task_id: str,+ *,+ result: Optional[Dict[str, Any]],+ error: Optional[str],+ resource_id: Optional[str],+ cancelled: bool,account_id: Optional[str],user_id: Optional[str],- *,- result: Optional[Dict[str, Any]] = None,- error: Optional[str] = None,- resource_id: Optional[str] = None,- terminal_status: Optional[TaskStatus] = None,) -> None:- cancellation: asyncio.CancelledError | None = None- outcome_persisted = False- async with self._task_locks.acquire(task_id):- task = await self._load_for_update(task_id, account_id, user_id)- if task and task.status in (TaskStatus.PENDING, TaskStatus.RUNNING):- updated = deepcopy(task)- if result is not None:- updated.result = deepcopy(result)- if resource_id is not None:- updated.resource_id = resource_id- if error is not None and updated.error is None:- updated.error = _sanitize_error(error)- work_error = self._work_index.failure(task_id)- if work_error and updated.error is None:- updated.error = _sanitize_error(work_error)- if terminal_status is not None:- updated.status = terminal_status- updated.stage = terminal_status.value- updated.updated_at = self._next_updated_at(task)- enqueue_markers = updated.auth.get("enqueue_complete_work_ids")- updated.auth = (- {"enqueue_complete_work_ids": deepcopy(enqueue_markers)}- if enqueue_markers- else {}- )- try:- await self._persist_and_publish("update", updated, previous=task)- except _CommittedMutationCancelled as exc:- cancellation = exc- outcome_persisted = True- else:- outcome_persisted = True- if outcome_persisted:- try:- await run_to_completion(- lambda: self._finalize_task(- task_id,- account_id=account_id,- user_id=user_id,- )- )- except asyncio.CancelledError as exc:- cancellation = cancellation or exc- else:- await self._finalize_task(task_id, account_id=account_id, user_id=user_id)- if cancellation is not None:- raise cancellation+ await run_to_completion(+ lambda: self._store.record_outcome(+ task_id,+ result=result,+ error=error,+ resource_id=resource_id,+ cancelled=cancelled,+ account_id=account_id,+ user_id=user_id,+ )+ )async def cancel(self,task_id: str,- account_id: Optional[str] = None,- user_id: Optional[str] = None,+ *,+ account_id: str,+ user_id: str,+ ) -> Optional[TaskRecord]:+ self._validate_owner(account_id, user_id)+ found = await self._request_cancellation(+ task_id,+ account_id=account_id,+ user_id=user_id,+ allow_terminal=False,+ )+ if found.code is StoreOperationCode.NOT_FOUND:+ return None+ if not found.accepted:+ raise ValueError(found.message or found.code.value)+ self._work_index.cancel_active(task_id, account_id=account_id, user_id=user_id)+ return await self.get(task_id, account_id=account_id, user_id=user_id)++ async def cancel_skill_update_for_rollback(+ self, task_id: str, *, account_id: str, user_id: str) -> Optional[TaskRecord]:- """Request cooperative cancellation and return the current task snapshot."""- return await self._cancel_task(task_id, account_id, user_id)+ task = await self.get(task_id, account_id=account_id, user_id=user_id)+ if task is not None and task.task_type != "add_skill":+ raise ValueError("Only Skill processing can be cancelled for package rollback")+ found = await self._request_cancellation(+ task_id,+ account_id=account_id,+ user_id=user_id,+ allow_terminal=True,+ )+ if found.code is StoreOperationCode.NOT_FOUND:+ return None+ if not found.accepted:+ raise ValueError(found.message or found.code.value)+ self._work_index.cancel_active(task_id, account_id=account_id, user_id=user_id)+ return await self.get(task_id, account_id=account_id, user_id=user_id)- async def _cancel_task(+ async def _request_cancellation(self,task_id: str,- account_id: Optional[str],- user_id: Optional[str],*,- rollback_skill_update: bool = False,- ) -> Optional[TaskRecord]:- cancellation: asyncio.CancelledError | None = None- cancellation_persisted = False- async with self._task_locks.acquire(task_id):- task = await self._load_for_update(task_id, account_id, user_id)- if task is None:- return None- if rollback_skill_update and task.task_type != "add_skill":- raise ValueError("Only Skill processing can be cancelled for package rollback")- if task.status == TaskStatus.CANCELLED:- return self._copy(task)- if task.status != TaskStatus.CANCELLING:- if task.task_type not in _CANCELLABLE_TASK_TYPES:- raise ValueError(f"Task type '{task.task_type}' does not support cancellation")- if task.status in _TERMINAL_STATUSES and not rollback_skill_update:- raise ValueError(f"Task is already {task.status.value}")-- updated = deepcopy(task)- updated.status = TaskStatus.CANCELLING- updated.updated_at = self._next_updated_at(task)- try:- await self._persist_and_publish("update", updated, previous=task)- except _CommittedMutationCancelled as exc:- cancellation = exc- cancellation_persisted = True- else:- cancellation_persisted = True-- async def finish_cancellation() -> None:- self._work_index.cancel_active(task_id)- await self._finalize_task(+ account_id: str,+ user_id: str,+ allow_terminal: bool,+ ) -> StoreOperationResult:+ return await run_to_completion(+ lambda: self._store.request_cancellation(task_id,+ cancellable_types=_CANCELLABLE_TASK_TYPES,+ allow_terminal=allow_terminal,account_id=account_id,user_id=user_id,)+ )- if cancellation_persisted:+ async def register_running_task(+ self,+ task_id: str,+ *,+ work_id: Optional[str] = None,+ account_id: str,+ user_id: str,+ ) -> str:+ self._validate_owner(account_id, user_id)+ active_task = asyncio.current_task()+ status = await self._store.get_task_status(task_id, account_id=account_id, user_id=user_id)+ if active_task is None or status is None or status in TERMINAL_TASK_STATUSES:+ raise TaskWorkRejected(f"Task {task_id} is unavailable")+ inline = work_id is None+ running_work_id = work_id or f"inline:{uuid4()}"+ if inline:+ registration = await self.register_work(+ task_id,+ running_work_id,+ "Inline",+ account_id=account_id,+ user_id=user_id,+ )+ if registration is WorkRegistration.REJECTED:+ raise TaskWorkRejected(f"Task {task_id} rejects inline work")try:- await run_to_completion(finish_cancellation)- except asyncio.CancelledError as exc:- cancellation = cancellation or exc- else:- await finish_cancellation()- task = self._cached_task(task_id)- if task is None or not self._matches_owner(task, account_id, user_id):- return None- if cancellation is not None:- raise cancellation- return self._copy(task)-- async def cancel_skill_update_for_rollback(- self, task_id: str, *, account_id: str, user_id: str- ) -> Optional[TaskRecord]:- """Prevent even a late queue replay from writing into a restored Skill.+ await self.start_work(+ task_id,+ running_work_id,+ account_id=account_id,+ user_id=user_id,+ )+ except BaseException:+ if registration is WorkRegistration.CREATED:+ await run_to_completion(+ lambda: self.discard_work(+ task_id,+ running_work_id,+ account_id=account_id,+ user_id=user_id,+ )+ )+ raise+ self._work_index.register_active(+ task_id,+ active_task,+ account_id=account_id,+ user_id=user_id,+ )+ self._running.set((running_work_id, account_id, user_id, inline))+ if await self.is_cancellation_requested(task_id, account_id=account_id, user_id=user_id):+ active_task.cancel()+ return running_work_id- A last ACK can fail after the task appears finished. Persist cancellation- for this rolled-back update even if it has reached a terminal state.- Normal public cancellation keeps its existing terminal-state rules.- """- return await self._cancel_task(task_id, account_id, user_id, rollback_skill_update=True)+ async def unregister_running_task(self, task_id: str) -> None:+ active_task = asyncio.current_task()+ state = self._running.get()+ self._running.set(None)+ if active_task is not None and state is not None:+ _, account_id, user_id, _ = state+ self._work_index.unregister_active(+ task_id,+ active_task,+ account_id=account_id,+ user_id=user_id,+ )+ if state is not None and state[3]:+ work_id, account_id, user_id, _inline = state+ await run_to_completion(+ lambda: self.mark_work_done(+ task_id,+ work_id,+ account_id=account_id,+ user_id=user_id,+ )+ )- async def _finalize_task(+ async def get(self,task_id: str,account_id: Optional[str] = None,user_id: Optional[str] = None,- ) -> None:- """Apply the task's terminal outcome after all owned work is settled."""- if self._work_index.has_work(task_id):- return- async with self._task_locks.acquire(task_id):- task = await self._load_for_update(task_id, account_id, user_id)- if task and task.status in _ACTIVE_STATUSES:- if self._work_index.has_work(task_id):- return- updated = deepcopy(task)- if updated.status != TaskStatus.CANCELLING:- work_error = self._work_index.failure(task_id)- if work_error and updated.error is None:- updated.error = _sanitize_error(work_error)- if updated.status == TaskStatus.CANCELLING:- updated.status = TaskStatus.CANCELLED- elif updated.error is not None:- updated.status = TaskStatus.FAILED- elif updated.result is not None:- updated.status = TaskStatus.COMPLETED- else:- updated.error = "Task operation interrupted before its result was persisted"- updated.status = TaskStatus.FAILED- work_status = self._work_index.queue_status(task_id)- if updated.result is not None and work_status:- updated.result["queue_status"] = self._public_queue_status(work_status)- updated.stage = updated.status.value- updated.updated_at = self._next_updated_at(task)- updated.auth = {}- await self._persist_and_publish("update", updated, previous=task)- self._work_index.clear_failure(task_id)- logger.info("[TaskTracker] Task %s %s", task_id, updated.status.value)+ ) -> Optional[TaskRecord]:+ task = await self._store.get_task(task_id, account_id=account_id, user_id=user_id)+ return self._public(task) if task is not None else None++ async def get_task_status(+ self, task_id: str, *, account_id: str, user_id: str+ ) -> Optional[TaskStatus]:+ self._validate_owner(account_id, user_id)+ return await self._store.get_task_status(task_id, account_id=account_id, user_id=user_id)++ async def get_task_auth(self, task_id: str, *, account_id: str, user_id: str) -> Dict[str, Any]:+ self._validate_owner(account_id, user_id)+ task = await self._store.get_task(task_id, account_id=account_id, user_id=user_id)+ return deepcopy(task.auth) if task is not None else {}async def wait(self,@@ -946,21 +719,15 @@ class TaskTracker:account_id: Optional[str] = None,user_id: Optional[str] = None,timeout: Optional[float] = None,- poll_interval: float = 0.05,+ poll_interval: float = _TASK_WAIT_POLL_INTERVAL,) -> TaskRecord:- """Wait for one task's terminal state without changing its lifecycle."""-with _consume_wait_budget(task_id, timeout) as remaining:deadline = None if remaining is None else time.monotonic() + remainingwhile True:- lookup = self.get(task_id, account_id=account_id, user_id=user_id)- if remaining is not None and self._cached_task(task_id) is None:- task = await asyncio.wait_for(lookup, max(0.0, remaining))- else:- task = await lookup+ task = await self.get(task_id, account_id=account_id, user_id=user_id)if task is None:raise KeyError(f"Task not found: {task_id}")- if task.status in _TERMINAL_STATUSES:+ if task.status in TERMINAL_TASK_STATUSES:return taskremaining = None if deadline is None else deadline - time.monotonic()if remaining is not None and remaining <= 0:@@ -969,107 +736,376 @@ class TaskTracker:poll_interval if remaining is None else min(poll_interval, remaining))- async def get_task_auth(+ async def list_tasks(self,- task_id: str,+ task_type: Optional[str] = None,+ status: Optional[str] = None,+ resource_id: Optional[str] = None,+ limit: Optional[int] = 50,+ account_id: Optional[str] = None,+ user_id: Optional[str] = None,+ include_internal: bool = True,*,- account_id: str,- user_id: str,- ) -> Dict[str, Any]:- """Load task-owned authentication excluded from public snapshots."""- self._validate_owner(account_id, user_id)+ before: tuple[float, str] | None = None,+ q: Optional[str] = None,+ ) -> List[TaskRecord]:+ from openviking.service.task_pagination import matches- task = self._cached_task(task_id)- if task is None:- task = await self._load_from_store(task_id, account_id, user_id)- if task is not None:- self._publish_task(task)- if task is None or not self._matches_owner(task, account_id, user_id):- return {}- return deepcopy(task.auth)-- def forget_account_tasks(self, account_id: str) -> None:- """Invalidate snapshots after executions settle and account storage is removed."""- with self._lock:- task_ids = [- task_id for task_id, task in self._tasks.items() if task.account_id == account_id- ]- for task_id in task_ids:- del self._tasks[task_id]- for task_id in task_ids:- self._work_index.forget_task(task_id)-- async def delete_user_tasks(self, account_id: str, user_id: str) -> int:- """Delete terminal task records for one user from storage and cache."""- self._validate_owner(account_id, user_id)- self._merge_loaded_tasks(await self._load_all_from_store(account_id, user_id))- tasks = [- task- for task in self._cache_snapshot()- if self._matches_owner(task, account_id, user_id)- ]- active = [task for task in tasks if task.status in _ACTIVE_STATUSES]- if active:- raise RuntimeError(- "Cannot delete active task records: "- + ", ".join(f"{task.task_id}({task.task_type})" for task in active)+ records = await self._store.list(+ account_id,+ user_id=user_id,+ task_type=task_type,+ status=status,+ resource_id=resource_id,+ limit=limit,+ before=before,+ include_internal=include_internal,+ q=q,+ )+ visible = {+ task.task_id: task+ for task in records+ if (before is None or (task.created_at, task.task_id) < before)+ and matches(+ task.to_dict(),+ task_type=task_type,+ status=status,+ resource_id=resource_id,+ include_internal=include_internal,+ q=q,)+ }+ ordered = sorted(+ visible.values(),+ key=lambda task: (task.created_at, task.task_id),+ reverse=True,+ )+ selected = ordered if limit is None else ordered[:limit]+ return [self._public(task) for task in selected]- deleted = 0- for task in tasks:- async with self._task_locks.acquire(task.task_id):- current = self._cached_task(task.task_id)- if current is None or not self._matches_owner(current, account_id, user_id):- continue- if current.status in _ACTIVE_STATUSES or self._work_index.has_work(task.task_id):- raise RuntimeError(- f"Cannot delete active task record: {task.task_id}({task.task_type})"- )- await self._store_io.run(- "delete",- lambda task_id=task.task_id: run_to_completion(- lambda: self._store.delete(- task_id,- account_id=account_id,- user_id=user_id,- )- ),- )- with self._lock:- self._tasks.pop(task.task_id, None)- self._work_index.forget_task(task.task_id)- deleted += 1- return deleted+ async def list_root_tasks(+ self,+ *,+ account_id: str,+ user_id: str,+ task_type: Optional[str] = None,+ status: Optional[str] = None,+ resource_id: Optional[str] = None,+ limit: int = 50,+ before: tuple[float, str] | None = None,+ include_internal: bool = True,+ q: Optional[str] = None,+ ) -> List[TaskRecord]:+ """Build the ROOT compatibility view over the FileStore limitations.++ The logical meaning is "all Tasks". The file-backed implementation+ cannot enumerate every account without an owner, so include the+ durable system/current-owner scopes and its cache-only global view.+ """++ source_limit = limit + 1+ records = await asyncio.gather(+ self.list_tasks(+ task_type=task_type,+ status=status,+ resource_id=resource_id,+ limit=source_limit,+ account_id=SYSTEM_TASK_ACCOUNT_ID,+ user_id=SYSTEM_TASK_USER_ID,+ before=before,+ include_internal=include_internal,+ q=q,+ ),+ self.list_tasks(+ task_type=task_type,+ status=status,+ resource_id=resource_id,+ limit=source_limit,+ account_id=account_id,+ user_id=user_id,+ before=before,+ include_internal=include_internal,+ q=q,+ ),+ self.list_tasks(+ task_type=task_type,+ status=status,+ resource_id=resource_id,+ limit=None,+ before=before,+ include_internal=include_internal,+ q=q,+ ),+ )+ visible = {task.task_id: task for source in records for task in source}+ ordered = sorted(+ visible.values(),+ key=lambda task: (task.created_at, task.task_id),+ reverse=True,+ )+ return ordered[:limit]++ async def has_running(+ self,+ task_type: str,+ resource_id: str,+ account_id: Optional[str] = None,+ user_id: Optional[str] = None,+ ) -> bool:+ tasks = await self._store.list(+ account_id,+ user_id=user_id,+ task_type=task_type,+ resource_id=resource_id,+ )+ return any(task.status in ACTIVE_TASK_STATUSES for task in tasks)++ async def register_work(+ self,+ task_id: str,+ work_id: str,+ queue_name: str,+ *,+ account_id: str,+ user_id: str,+ ) -> WorkRegistration:+ work = WorkRecord(work_id=work_id, task_id=task_id, queue_name=queue_name)++ async def register() -> WorkRegistration:+ try:+ result = await self._store.add_work(work, account_id=account_id, user_id=user_id)+ except TaskWorkRejected:+ return WorkRegistration.REJECTED+ if result.is_applied:+ return WorkRegistration.CREATED+ if result.accepted and result.code is StoreOperationCode.DUPLICATE:+ return WorkRegistration.EXISTING+ return WorkRegistration.REJECTED++ return await register()++ async def discard_work(+ self, task_id: str, work_id: str, *, account_id: str, user_id: str+ ) -> None:+ await run_to_completion(+ lambda: self._store.discard_work(+ task_id, work_id, account_id=account_id, user_id=user_id+ )+ )++ async def reconcile_delivery(+ self,+ task_id: str,+ work_id: str,+ queue_name: str,+ *,+ account_id: str,+ user_id: str,+ ) -> None:+ work = WorkRecord(work_id=work_id, task_id=task_id, queue_name=queue_name)+ await self._store.reconcile_delivery(work, account_id=account_id, user_id=user_id)++ async def get_work_state(+ self, task_id: str, work_id: str, *, account_id: str, user_id: str+ ) -> Optional[WorkState]:+ work = await self._store.get_work(task_id, work_id, account_id=account_id, user_id=user_id)+ return work.state if work is not None else None++ async def start_work(+ self, task_id: str, work_id: str, *, account_id: str, user_id: str+ ) -> None:+ await self._store.start_work(task_id, work_id, account_id=account_id, user_id=user_id)++ async def mark_work_done(+ self,+ task_id: str,+ work_id: str,+ *,+ context_count_delta: int = 0,+ vector_written: bool = False,+ account_id: str,+ user_id: str,+ ) -> None:+ await self._finish_work(+ task_id,+ work_id,+ WorkState.DONE,+ error=None,+ errors=None,+ context_count_delta=max(context_count_delta, int(vector_written)),+ account_id=account_id,+ user_id=user_id,+ )++ async def mark_work_failed(+ self,+ task_id: str,+ work_id: str,+ error: Optional[str] = None,+ *,+ errors: Optional[Collection[str]] = None,+ account_id: str,+ user_id: str,+ ) -> None:+ await self._finish_work(+ task_id,+ work_id,+ WorkState.FAILED,+ error=sanitize_error(error) if error else None,+ errors=errors,+ context_count_delta=0,+ account_id=account_id,+ user_id=user_id,+ )++ async def mark_work_requeued(+ self,+ task_id: str,+ work_id: str,+ *,+ delta: int = 1,+ account_id: str,+ user_id: str,+ ) -> None:+ await self._store.requeue_work(+ task_id,+ work_id,+ delta=delta,+ account_id=account_id,+ user_id=user_id,+ )++ async def _finish_work(+ self,+ task_id: str,+ work_id: str,+ state: WorkState,+ *,+ error: Optional[str],+ errors: Optional[Collection[str]],+ context_count_delta: int,+ account_id: str,+ user_id: str,+ ) -> None:+ await run_to_completion(+ lambda: self._store.finish_work(+ task_id,+ work_id,+ state,+ error=error,+ errors=errors,+ context_count_delta=context_count_delta,+ account_id=account_id,+ user_id=user_id,+ )+ )++ async def record_work_error(+ self,+ task_id: str,+ work_id: str,+ error: str,+ *,+ account_id: str,+ user_id: str,+ ) -> None:+ await self._store.record_work_error(+ task_id,+ work_id,+ sanitize_error(error),+ account_id=account_id,+ user_id=user_id,+ )++ async def settle_work(+ self, task_id: str, work_id: str, *, account_id: str, user_id: str+ ) -> None:+ await run_to_completion(+ lambda: self._store.settle_work(+ task_id, work_id, account_id=account_id, user_id=user_id+ )+ )++ async def is_cancelling(+ self,+ task_id: str,+ *,+ account_id: str,+ user_id: str,+ ) -> bool:+ return await self.is_cancellation_requested(task_id, account_id=account_id, user_id=user_id)++ async def is_cancellation_requested(+ self,+ task_id: str,+ *,+ account_id: str,+ user_id: str,+ ) -> bool:+ status = await self.get_task_status(task_id, account_id=account_id, user_id=user_id)+ return status in (TaskStatus.CANCELLING, TaskStatus.CANCELLED)async def wait_for_work(self,task_id: str,- work_ids: List[str],+ work_ids: Collection[str],*,+ account_id: Optional[str] = None,+ user_id: Optional[str] = None,timeout: Optional[float] = None,- stop_waiting: Optional[Any] = None,+ stop_waiting: Optional[Callable[[], bool]] = None,+ poll_interval: float = _TASK_WAIT_POLL_INTERVAL,) -> None:- """Wait only for the supplied logical Work IDs to become terminal."""+ wanted = set(work_ids)+ if not wanted:+ returnwith _consume_wait_budget(task_id, timeout) as remaining:- await self._work_index.wait_for_work(- task_id,- work_ids,- timeout=remaining,- stop_waiting=stop_waiting,- )+ deadline = None if remaining is None else time.monotonic() + remaining+ while True:+ works = await self._store.get_works(+ task_id,+ wanted,+ account_id=account_id,+ user_id=user_id,+ )+ missing = wanted.difference(works)+ if missing:+ raise KeyError(f"Work not found for Task {task_id}: {sorted(missing)}")+ if all(work.state in TERMINAL_WORK_STATES for work in works.values()):+ return+ if stop_waiting is not None and stop_waiting():+ raise asyncio.CancelledError("Worker stopped with queued Work")+ remaining = None if deadline is None else deadline - time.monotonic()+ if remaining is not None and remaining <= 0:+ raise TimeoutError(f"Timed out waiting for Work in Task {task_id}")+ await asyncio.sleep(+ poll_interval if remaining is None else min(poll_interval, remaining)+ )async def wait_for_task_work(self,task_id: str,*,exclude_work_id: Optional[str] = None,+ account_id: Optional[str] = None,+ user_id: Optional[str] = None,timeout: Optional[float] = None,+ poll_interval: float = _TASK_WAIT_POLL_INTERVAL,) -> None:- """Let a root coordinator drain its Task's other queue deliveries."""with _consume_wait_budget(task_id, timeout) as remaining:- await self._work_index.wait_for_task_work(- task_id, exclude_work_id=exclude_work_id, timeout=remaining- )+ deadline = None if remaining is None else time.monotonic() + remaining+ while await self._store.has_open_work(+ task_id,+ exclude_work_id=exclude_work_id,+ account_id=account_id,+ user_id=user_id,+ ):+ remaining = None if deadline is None else deadline - time.monotonic()+ if remaining is not None and remaining <= 0:+ raise TimeoutError(f"Timed out waiting for Work in Task {task_id}")+ await asyncio.sleep(+ poll_interval if remaining is None else min(poll_interval, remaining)+ )async def mark_enqueue_complete(self,@@ -1079,17 +1115,12 @@ class TaskTracker:account_id: str,user_id: str,) -> None:- """Persist that a coordinator's complete child Work set is durable."""- async with self._task_locks.acquire(task_id):- task = await self._load_for_update(task_id, account_id, user_id)- if task is None or task.status not in _ACTIVE_STATUSES:- return- updated = deepcopy(task)- completed = set(updated.auth.get("enqueue_complete_work_ids") or ())- completed.add(work_id)- updated.auth["enqueue_complete_work_ids"] = sorted(completed)- updated.updated_at = self._next_updated_at(task)- await self._persist_and_publish("update", updated, previous=task)+ await self._store.mark_enqueue_complete(+ task_id,+ work_id,+ account_id=account_id,+ user_id=user_id,+ )async def is_enqueue_complete(self,@@ -1099,64 +1130,103 @@ class TaskTracker:account_id: str,user_id: str,) -> bool:- """Check the durable producer marker used by a lock coordinator."""- auth = await self.get_task_auth(+ return await self._store.is_enqueue_complete(task_id,+ work_id,account_id=account_id,user_id=user_id,)- return work_id in set(auth.get("enqueue_complete_work_ids") or ())- def build_queue_status(+ async def build_queue_status(self,task_id: str,- work_ids: Optional[List[str]] = None,+ work_ids: Optional[Collection[str]] = None,+ *,+ account_id: Optional[str] = None,+ user_id: Optional[str] = None,) -> Dict[str, Dict[str, object]]:- """Project Task-scoped queue outcomes from logical Work."""- return self._public_queue_status(self._work_index.queue_status(task_id, work_ids))+ summary = await self.get_work_summary(+ task_id,+ work_ids=work_ids,+ account_id=account_id,+ user_id=user_id,+ )+ return self.format_queue_status(summary)@staticmethod- def _public_queue_status(- status: Dict[str, Dict[str, object]],+ def format_queue_status(+ summary: WorkSummary,) -> Dict[str, Dict[str, object]]:result: Dict[str, Dict[str, object]] = {}for queue_name in _PUBLIC_QUEUE_NAMES:- queue_status = deepcopy(- status.get(- queue_name,- {- "processed": 0,- "requeue_count": 0,- "error_count": 0,- "errors": [],- },- )- )- errors = queue_status.get("errors")- if isinstance(errors, list):- queue_status["errors"] = [- {- **error,- "message": _sanitize_error(str(error.get("message") or "")),- }- if isinstance(error, dict)- else {"message": _sanitize_error(str(error))}- for error in errors[:100]- ]- result[queue_name] = queue_status+ queue = summary.queues.get(queue_name, QueueWorkSummary())+ result[queue_name] = {+ "processed": queue.processed,+ "requeue_count": queue.requeue_count,+ "error_count": queue.error_count,+ "errors": [+ {"message": sanitize_error(str(error))}+ for error in queue.errors[:MAX_QUEUE_ERROR_DETAILS]+ ],+ }return result- def get_queue_timing(self, task_id: str) -> Dict[str, Dict[str, float]]:- """Return Task-scoped queue wait and execution durations."""- return self._work_index.queue_timing(task_id)+ async def get_work_summary(+ self,+ task_id: str,+ work_ids: Optional[Collection[str]] = None,+ *,+ account_id: Optional[str] = None,+ user_id: Optional[str] = None,+ ) -> WorkSummary:+ return await self._store.summarize_works(+ task_id,+ work_ids=work_ids,+ account_id=account_id,+ user_id=user_id,+ )++ @staticmethod+ def format_queue_timing(+ summary: WorkSummary,+ ) -> Dict[str, Dict[str, float]]:+ return {+ queue_name.lower(): {+ "queue_wait_ms": queue.queue_wait_ms,+ "execute_ms": queue.execute_ms,+ }+ for queue_name, queue in summary.queues.items()+ }- def get_context_count(self, task_id: str) -> int:- """Return successful context writes recorded by Task Work."""- return self._work_index.context_count(task_id)+ async def get_queue_timing(+ self,+ task_id: str,+ *,+ account_id: Optional[str] = None,+ user_id: Optional[str] = None,+ ) -> Dict[str, Dict[str, float]]:+ summary = await self.get_work_summary(+ task_id,+ work_ids=None,+ account_id=account_id,+ user_id=user_id,+ )+ return self.format_queue_timing(summary)- def record_context_count(self, task_id: str, work_id: str, delta: int = 1) -> None:- """Record successful context writes against one logical Work."""- self._work_index.record_context_count(task_id, work_id, delta)+ async def get_context_count(+ self,+ task_id: str,+ *,+ account_id: Optional[str] = None,+ user_id: Optional[str] = None,+ ) -> int:+ summary = await self.get_work_summary(+ task_id,+ work_ids=None,+ account_id=account_id,+ user_id=user_id,+ )+ return summary.context_countasync def record_event(self,@@ -1167,356 +1237,78 @@ class TaskTracker:user_id: str,operation: Optional[str] = None,) -> None:- """Record a registered process event without changing task status or stage.-- Add process event kinds here and their Studio translations when instrumenting- new execution points. Callers report facts, not inferred lifecycle transitions.- """self._validate_owner(account_id, user_id)if kind not in PROCESS_EVENT_KINDS:raise ValueError(f"Unknown task process event: {kind}")if operation is not None and not re.fullmatch(r"[\w.:-]{1,128}", operation):raise ValueError("operation must be a bounded operation identifier")+ await self._store.record_event(+ task_id,+ kind,+ operation=operation,+ account_id=account_id,+ user_id=user_id,+ )- async with self._task_locks.acquire(task_id):- task = await self._load_for_update(task_id, account_id, user_id)- if task is None or task.status not in _ACTIVE_STATUSES:- return- updated = deepcopy(task)- updated.execution_events = append_task_event(- updated.execution_events,- kind=kind,- status=task.status.value,- stage=_sanitize_error(task.stage) if task.stage else None,- operation=operation,- )- updated.updated_at = self._next_updated_at(task)- await self._persist_and_publish("update", updated, previous=task)-- def has_work(self, task_id: str, *, exclude_work_id: Optional[str] = None) -> bool:- """Return whether a task still owns durable or active queue work."""- return self._work_index.has_work(task_id, exclude_work_id=exclude_work_id)-- def has_active_producer(self, task_id: str) -> bool:- """Return whether another active handler can persist an enqueue marker."""- return self._work_index.has_other_active(task_id, asyncio.current_task())-- def register_running_task(self, task_id: str) -> None:- """Register the current asyncio task so cancellation can interrupt it."""- active_task = asyncio.current_task()- if active_task is not None:- self._work_index.register_active(task_id, active_task)-- async def unregister_running_task(self, task_id: str) -> None:- """Release direct background work and persist its terminal outcome."""- active_task = asyncio.current_task()- if active_task is not None:- self._work_index.unregister_active(task_id, active_task)- await self._finalize_task(task_id)-- async def get(+ async def has_work(self,task_id: str,- account_id: Optional[str] = None,- user_id: Optional[str] = None,- ) -> Optional[TaskRecord]:- """Look up a single task. Returns a snapshot copy (None if not found)."""- task = self._cached_task(task_id)- if task is not None:- if not self._matches_owner(task, account_id, user_id):- return None- return self._copy(task)- if account_id is None:- return None- task = await self._load_from_store(task_id, account_id, user_id)- if task is not None:- self._merge_loaded_tasks([task])- task = self._cached_task(task_id)- if task is None or not self._matches_owner(task, account_id, user_id):- return None- return self._copy(task)-- async def _prune_persisted_expired(self, payload: Dict[str, Any]) -> bool:- """Prune cold records under the same lock and I/O budget as mutations."""- candidate = self._record_from_payload(payload)- now = time.time()- if not self._is_expired(candidate, now):- return False- async with self._task_locks.acquire(candidate.task_id):- # The scan may race with a lifecycle write. Re-read under the same- # lock used by mutations before removing any durable state.- current = await self._store_io.run(- "get",- lambda: self._store.get(- candidate.task_id, account_id=candidate.account_id, user_id=candidate.user_id- ),- )- if current is None:- return True- task = self._record_from_payload(current)- if not self._is_expired(task, now) or self._work_index.has_work(task.task_id):- payload.clear()- payload.update(current)- return False- await self._store_io.run(- "delete",- lambda: run_to_completion(- lambda: self._store.delete(- task.task_id, account_id=task.account_id, user_id=task.user_id- )- ),- )- with self._lock:- self._tasks.pop(task.task_id, None)- self._work_index.forget_task(task.task_id)- return True-- async def list_page(- self,*,- account_id: str,- user_id: str,- limit: int,- before: tuple[float, str] | None = None,- include_cached: bool = False,- additional_owner: tuple[str, str] | None = None,- **filters: Any,- ) -> list[TaskRecord]:- from openviking.service.task_pagination import matches-- owners = {(account_id, user_id)}- if additional_owner:- owners.add(additional_owner)- records = []- for account, user in owners:- page = await self._store.list_page(- account,- user_id=user,- limit=limit,- before=before,- prune_expired=self._prune_persisted_expired,- io_limiter=self._store_io,- **filters,- )- records.extend(self._copy(self._record_from_payload(record)) for record in page)- if include_cached:- records.extend(self._copy(t) for t in self._cache_snapshot())- visible = {- t.task_id: t- for t in records- if (before is None or (t.created_at, t.task_id) < before)- and matches(t.to_dict(), **filters)- }- return sorted(visible.values(), key=lambda t: (t.created_at, t.task_id), reverse=True)[- :limit- ]-- async def list_tasks(- self,- task_type: Optional[str] = None,- status: Optional[str] = None,- resource_id: Optional[str] = None,- limit: Optional[int] = 50,- account_id: Optional[str] = None,- user_id: Optional[str] = None,- include_internal: bool = True,- ) -> List[TaskRecord]:- """List tasks with optional filters. Most-recent first. Returns snapshot copies."""- if account_id is not None:- self._merge_loaded_tasks(await self._load_all_from_store(account_id, user_id))- source = self._cache_snapshot()- tasks = [t for t in source if self._matches_owner(t, account_id, user_id)]- if not include_internal:- tasks = [t for t in tasks if t.meta.get("internal") is not True]- if task_type:- tasks = [t for t in tasks if t.task_type == task_type]- if status:- tasks = [t for t in tasks if t.status.value == status]- if resource_id:- tasks = [t for t in tasks if t.resource_id == resource_id]- tasks.sort(key=lambda t: t.created_at, reverse=True)- return [self._copy(t) for t in tasks[:limit]]-- async def has_running(- self,- task_type: str,- resource_id: str,+ exclude_work_id: Optional[str] = None,account_id: Optional[str] = None,user_id: Optional[str] = None,) -> bool:- """Check if there is already a running task for the given type+resource."""- if account_id is not None:- self._merge_loaded_tasks(await self._load_all_from_store(account_id, user_id))- tasks = self._cache_snapshot()- return any(- t.task_type == task_type- and t.resource_id == resource_id- and self._matches_owner(t, account_id, user_id)- and t.status in _ACTIVE_STATUSES- for t in tasks+ return await self._store.has_open_work(+ task_id,+ exclude_work_id=exclude_work_id,+ account_id=account_id,+ user_id=user_id,)- async def _load_for_update(- self,- task_id: str,- account_id: Optional[str],- user_id: Optional[str],- ) -> Optional[TaskRecord]:- task = self._cached_task(task_id)- if task is not None:- return task if self._matches_owner(task, account_id, user_id) else None- if account_id is None or user_id is None:- return None- return await self._load_from_store(task_id, account_id, user_id)-- @staticmethod- def _record_from_payload(payload: Dict[str, Any]) -> TaskRecord:- known_fields = {item.name for item in fields(TaskRecord) if item.init}- data = {key: deepcopy(value) for key, value in payload.items() if key in known_fields}- data["status"] = TaskStatus(data["status"])- record = TaskRecord(**data)- # Keep fields from other writers for the next persistence update.- record._extra_fields = {- key: deepcopy(value) for key, value in payload.items() if key not in known_fields- }- return record+ def has_active_producer(self, task_id: str) -> bool:+ state = self._running.get()+ return self._work_index.has_other_active(+ task_id,+ asyncio.current_task(),+ account_id=state[1] if state is not None else None,+ user_id=state[2] if state is not None else None,+ )- async def _load_from_store(+ async def delete_tasks(self,- task_id: str,account_id: str,- user_id: Optional[str],- ) -> Optional[TaskRecord]:- payload = await self._store_io.run(- "get",- lambda: self._store.get(task_id, account_id=account_id, user_id=user_id),+ user_id: Optional[str] = None,+ *,+ allow_active_types: Collection[str] = (),+ ) -> int:+ if not account_id:+ raise ValueError("Task deletion requires a non-empty account_id")+ allowed = set(allow_active_types)+ delete_result = await self._store.delete_tasks(+ account_id=account_id,+ user_id=user_id,+ allow_active_types=allowed,)- if payload is None:- return None- return self._record_from_payload(payload)-- async def _load_all_from_store(- self, account_id: str, user_id: Optional[str]- ) -> List[TaskRecord]:- return [- self._record_from_payload(payload)- for payload in await self._store_io.run(- "list",- lambda: self._store.list(account_id, user_id=user_id),+ if not delete_result.accepted:+ raise RuntimeError(delete_result.result.message or delete_result.result.code.value)+ for task in delete_result.deleted:+ self._work_index.forget_task(+ task.task_id,+ account_id=task.account_id,+ user_id=task.user_id,)- ]+ return len(delete_result.deleted)- async def _persist_and_publish(- self, operation: str, task: TaskRecord, *, previous: Optional[TaskRecord] = None- ) -> None:- # All mutation callers supply the snapshot read under this task's lock.- # Build history before the same physical write as the state transition.- if operation == "update" and previous is None:- raise ValueError("Task updates require the previous snapshot")- # Persist the final Work interval union; restored active tasks lack full timing coverage.- if previous is None or previous.status in _ACTIVE_STATUSES:- task.processing_seconds = self._work_index.processing_seconds(task.task_id)- kinds = []- if previous is None:- kinds.append("created")- else:- if task.error is not None and previous.error is None:- kinds.append("error_recorded")- if task.status != previous.status:- kinds.append("status_changed")- if task.stage != previous.stage and task.status not in _TERMINAL_STATUSES:- kinds.append("stage_changed")- stage = previous.stage if previous and task.status in _TERMINAL_STATUSES else task.stage- recorded_at = datetime.now(timezone.utc).isoformat()- for kind in kinds:- task.execution_events = append_task_event(- task.execution_events,- kind=kind,- status=task.status.value,- stage=_sanitize_error(stage) if stage else None,- error=_sanitize_error(task.error)- if kind == "error_recorded" and task.error is not None- else None,- recorded_at=recorded_at,- )- committed = False-- async def write_and_publish() -> None:- nonlocal committed- if operation == "create":- await self._store.create(task)- else:- await self._store.update(task)- self._publish_task(task)- if operation == "create":- self._work_index.init_processing(task.task_id)- elif task.status in _TERMINAL_STATUSES:- self._work_index.forget_processing(task.task_id)- committed = True-- # Semaphore/lock waits remain cancellable. Once the store operation is- # admitted, settle the physical write and publish its cache snapshot- # before propagating caller cancellation.- try:- await self._store_io.run(- operation,- lambda: run_to_completion(write_and_publish),- )- except asyncio.CancelledError as exc:- if committed:- raise _CommittedMutationCancelled() from exc- raise+ def count(self) -> int:+ return sum(sum(statuses.values()) for statuses in self.snapshot_counts_by_type().values())- def _cached_task(self, task_id: str) -> Optional[TaskRecord]:- with self._lock:- task = self._tasks.get(task_id)- return deepcopy(task) if task is not None else None-- def _cache_snapshot(self) -> List[TaskRecord]:- # Published records are replaced, never mutated. Internal readers may- # share them; public callers receive defensive copies via _copy().- with self._lock:- return list(self._tasks.values())-- def _publish_task(self, task: TaskRecord) -> None:- published = deepcopy(task)- with self._lock:- self._tasks[task.task_id] = published-- def _merge_loaded_tasks(self, loaded_tasks: List[TaskRecord]) -> None:- candidates = [deepcopy(task) for task in loaded_tasks]- with self._lock:- for loaded in candidates:- cached = self._tasks.get(loaded.task_id)- if cached is None or loaded.updated_at > cached.updated_at:- self._tasks[loaded.task_id] = loaded+ def snapshot_counts_by_type(self) -> Dict[str, Dict[str, int]]:+ return self._store.snapshot_task_stats()@staticmethod- def _next_updated_at(task: TaskRecord) -> float:- return max(time.time(), math.nextafter(task.updated_at, math.inf))-- def _copy(self, task: TaskRecord) -> TaskRecord:- """Return a defensive copy of a TaskRecord."""+ def _public(task: TaskRecord) -> TaskRecord:copied = deepcopy(task)- if task.status in _ACTIVE_STATUSES:- copied.processing_seconds = self._work_index.processing_seconds(task.task_id)- copied.meta = _sanitize_task_result(copied.meta)- copied.result = _sanitize_task_result(copied.result)+ copied.meta = sanitize_task_result(copied.meta)+ copied.result = sanitize_task_result(copied.result)copied.auth = {}return copied-- def count(self) -> int:- """Return total task count."""- with self._lock:- return len(self._tasks)-- def snapshot_counts_by_type(self) -> Dict[str, Dict[str, int]]:- """Return a snapshot of task counts grouped by task_type and status."""- from collections import defaultdict-- grouped: Dict[str, Dict[str, int]] = defaultdict(lambda: defaultdict(int))- with self._lock:- tasks = list(self._tasks.values())- for t in tasks:- grouped[t.task_type][t.status.value] += 1- return {k: dict(v) for k, v in grouped.items()}diff --git a/openviking/service/task_work_index.py b/openviking/service/task_work_index.pyindex da65f42fb..d51242bb2 100644--- a/openviking/service/task_work_index.py+++ b/openviking/service/task_work_index.py@@ -1,32 +1,23 @@# Copyright (c) 2026 Beijing Volcano Engine Technology Co., Ltd.# SPDX-License-Identifier: AGPL-3.0-"""Runtime index of logical queue Work owned by tracked Tasks.--QueueFS remains the durable delivery source of truth. This index separates-logical Work state from delivery counts so a replacement delivery can retain-the same Work ID without allowing the old delivery's ACK to finish the Task.-"""+"""Process-local execution context and cancellable handles for Task Work."""from __future__ import annotationsimport asyncioimport jsonimport threading-import timefrom contextlib import contextmanagerfrom contextvars import ContextVarfrom dataclasses import dataclass-from typing import Any, Awaitable, Callable, Dict, Generator, Iterable, Mapping, Optional+from typing import Any, Dict, Generator, Mapping, Optionalfrom uuid import uuid4+from openviking.service.task_domain import TaskWorkRejected as TaskWorkRejected+TASK_WORK_ID_FIELD = "_task_work_id"TASK_ACCOUNT_ID_FIELD = "_task_account_id"TASK_USER_ID_FIELD = "_task_user_id"-_TERMINAL_WORK_STATUSES = frozenset({"completed", "failed", "cancelled"})---class TaskWorkRejected(Exception):- """Task cancellation prevented descendant queue work from being created."""@dataclass(frozen=True)@@ -46,35 +37,6 @@ class QueueTaskMetadata:user_id: str = ""-@dataclass-class WorkTiming:- """Timestamps for one logical Work across all of its deliveries."""-- enqueued_at: float- started_at: Optional[float] = None- finished_at: Optional[float] = None---@dataclass-class WorkRecord:- """Mutable process-local state for one logical Work."""-- task_id: str- work_id: str- queue_name: str- status: str = "pending"- delivery_count: int = 0- requeue_count: int = 0- error: Optional[str] = None- errors: Optional[list[str]] = None- context_count: int = 0- timing: Optional[WorkTiming] = None-- @property- def terminal(self) -> bool:- return self.status in _TERMINAL_WORK_STATUSES--_current_task_context: ContextVar[Optional[TaskExecutionContext]] = ContextVar("openviking_task_execution_context",default=None,@@ -266,494 +228,97 @@ def bind_task_context_from_message(message: Any) -> Generator[None, None, None]:class TaskWorkIndex:- """Thread-safe, rebuildable index of logical Work and active handlers."""+ """Process-local asyncio handles used to accelerate cancellation."""def __init__(self) -> None:self._lock = threading.Lock()- self._work: Dict[str, Dict[str, WorkRecord]] = {}- self._delivery_counts: Dict[str, int] = {}- self._active: Dict[str, set[asyncio.Task[Any]]] = {}- self._failures: Dict[str, str] = {}- self._work_timings: Dict[str, Dict[str, WorkTiming]] = {}- self._active_work_ids: Dict[tuple[str, asyncio.Task[Any]], str] = {}- self._finalize_before_ack: Optional[Callable[[QueueTaskMetadata], Awaitable[None]]] = None- self._is_cancellation_requested: Optional[Callable[[str], bool]] = None-- def set_callbacks(- self,- *,- finalize_before_ack: Callable[[QueueTaskMetadata], Awaitable[None]],- is_cancellation_requested: Callable[[str], bool],- ) -> None:- self._finalize_before_ack = finalize_before_ack- self._is_cancellation_requested = is_cancellation_requested-- def rebuild(- self,- snapshots: Mapping[str, Iterable[Any]],- ) -> Dict[str, tuple[str, str]]:- """Rebuild unacknowledged deliveries and return Task owners to restore."""- work: Dict[str, Dict[str, WorkRecord]] = {}- delivery_counts: Dict[str, int] = {}- owners: Dict[str, tuple[str, str]] = {}- now = time.time()- for queue_name, messages in snapshots.items():- for message in messages:- metadata = extract_task_metadata(message)- if metadata is None:- continue- records = work.setdefault(metadata.task_id, {})- record = records.setdefault(- metadata.work_id,- WorkRecord(- task_id=metadata.task_id,- work_id=metadata.work_id,- queue_name=queue_name,- timing=WorkTiming(enqueued_at=now),- ),- )- record.delivery_count += 1- delivery_counts[metadata.task_id] = delivery_counts.get(metadata.task_id, 0) + 1- if metadata.account_id and metadata.user_id:- owners[metadata.task_id] = (metadata.account_id, metadata.user_id)- with self._lock:- self._work = work- self._delivery_counts = delivery_counts- self._failures = {}- return owners-- def _add_delivery_locked(self, task_id: str) -> None:- self._delivery_counts[task_id] = self._delivery_counts.get(task_id, 0) + 1-- def _remove_delivery_locked(self, task_id: str) -> None:- remaining = self._delivery_counts.get(task_id, 0) - 1- if remaining > 0:- self._delivery_counts[task_id] = remaining- else:- self._delivery_counts.pop(task_id, None)-- def register(self, queue_name: str, metadata: Optional[QueueTaskMetadata]) -> bool:- """Reject cancelled Work or provisionally register one delivery."""- if metadata is None:- return True- with self._lock:- if self.cancellation_requested(metadata.task_id):- return False- records = self._work.setdefault(metadata.task_id, {})- record = records.get(metadata.work_id)- now = time.time()- if record is None:- record = WorkRecord(- task_id=metadata.task_id,- work_id=metadata.work_id,- queue_name=queue_name,- timing=WorkTiming(enqueued_at=now),- )- records[metadata.work_id] = record- elif record.queue_name != queue_name:- raise ValueError(- f"Work {metadata.work_id} already belongs to queue {record.queue_name}"- )- record.delivery_count += 1- self._add_delivery_locked(metadata.task_id)- timings = self._work_timings.get(metadata.task_id)- if timings is not None:- record.timing = timings.setdefault(metadata.work_id, record.timing)- return True-- def _remove_delivery(- self,- queue_name: str,- metadata: QueueTaskMetadata,- ) -> tuple[bool, bool]:- """Remove one delivery and report whether its Task became idle."""- with self._lock:- record = self._work.get(metadata.task_id, {}).get(metadata.work_id)- removed = bool(- record is not None and record.queue_name == queue_name and record.delivery_count > 0- )- if removed:- record.delivery_count -= 1- self._remove_delivery_locked(metadata.task_id)- became_idle = removed and not self._has_work_locked(metadata.task_id)- return removed, became_idle-- def _has_delivery_locked(self, task_id: str, exclude_work_id: Optional[str] = None) -> bool:- if exclude_work_id is not None:- record = self._work.get(task_id, {}).get(exclude_work_id)- excluded = record.delivery_count if record is not None else 0- return self._delivery_counts.get(task_id, 0) > excluded- return self._delivery_counts.get(task_id, 0) > 0-- def _has_work_locked(self, task_id: str, exclude_work_id: Optional[str] = None) -> bool:- if self._has_delivery_locked(task_id, exclude_work_id):- return True- if exclude_work_id is not None:- return False- return bool(self._active.get(task_id))-- async def discard(self, queue_name: str, message: Any) -> None:- """Discard a provisionally registered delivery that did not commit."""- metadata = (- message if isinstance(message, QueueTaskMetadata) else extract_task_metadata(message)- )- if metadata is None:- return- removed, became_idle = self._remove_delivery(queue_name, metadata)- if removed:- with self._lock:- record = self._work.get(metadata.task_id, {}).get(metadata.work_id)- if record is not None and record.delivery_count == 0 and not record.terminal:- self._work[metadata.task_id].pop(metadata.work_id, None)- timings = self._work_timings.get(metadata.task_id)- if timings is not None:- timings.pop(metadata.work_id, None)- callback = self._finalize_before_ack- if became_idle and callback is not None:- await callback(metadata)-- async def fail_enqueue(- self,- queue_name: str,- metadata: QueueTaskMetadata,- error: str,- ) -> None:- """Keep a failed logical Work visible after its delivery failed to persist."""- self.record_result(metadata, "failed", error=error)- removed, became_idle = self._remove_delivery(queue_name, metadata)- if not removed:- return- callback = self._finalize_before_ack- if became_idle and callback is not None:- await callback(metadata)-- async def prepare_ack(- self,- queue_name: str,- message: Any,- ) -> Optional[QueueTaskMetadata]:- """Provisionally remove one delivery and finalize before the last ACK."""- metadata = (- message if isinstance(message, QueueTaskMetadata) else extract_task_metadata(message)- )- if metadata is None:- return None-- removed, became_idle = self._remove_delivery(queue_name, metadata)- if not removed:- return None- if not became_idle:- return metadata-- callback = self._finalize_before_ack- if callback is None:- return metadata- try:- await callback(metadata)- except BaseException:- self.rollback_ack(queue_name, metadata)- raise- return metadata-- def rollback_ack(self, queue_name: str, metadata: QueueTaskMetadata) -> None:- """Restore a provisionally removed delivery when finalization or ACK fails."""- with self._lock:- record = self._work.get(metadata.task_id, {}).get(metadata.work_id)- if record is None:- record = WorkRecord(- task_id=metadata.task_id,- work_id=metadata.work_id,- queue_name=queue_name,- timing=WorkTiming(enqueued_at=time.time()),- )- self._work.setdefault(metadata.task_id, {})[metadata.work_id] = record- record.delivery_count += 1- self._add_delivery_locked(metadata.task_id)-- def has_work(self, task_id: str, exclude_work_id: Optional[str] = None) -> bool:- with self._lock:- return self._has_work_locked(task_id, exclude_work_id)+ self._active: Dict[tuple[str, str, str], set[asyncio.Task[Any]]] = {}def has_other_active(self,task_id: str,current_task: Optional[asyncio.Task[Any]],+ *,+ account_id: Optional[str] = None,+ user_id: Optional[str] = None,) -> bool:"""Return whether another in-process handler can finish producing Work."""with self._lock:- return any(task is not current_task for task in self._active.get(task_id, ()))-- def cancellation_requested(self, task_id: str) -> bool:- callback = self._is_cancellation_requested- return bool(callback is not None and callback(task_id))+ return any(+ task is not current_task+ for (account, user, active_task_id), tasks in self._active.items()+ if active_task_id == task_id+ and (account_id is None or account == account_id)+ and (user_id is None or user == user_id)+ for task in tasks+ )- def record_result(+ def forget_task(self,- metadata: QueueTaskMetadata,- outcome: str,+ task_id: str,*,- error: Optional[str] = None,- errors: Optional[Iterable[str]] = None,+ account_id: Optional[str] = None,+ user_id: Optional[str] = None,) -> None:- """Apply one delivery result to its logical Work monotonically."""with self._lock:- record = self._work.get(metadata.task_id, {}).get(metadata.work_id)- if record is None or record.terminal:- return- if outcome == "requeued":- record.requeue_count += 1- record.status = "pending"- return- if outcome not in _TERMINAL_WORK_STATUSES:- raise ValueError(f"Unsupported Work outcome: {outcome}")- record.status = outcome- record.error = error if outcome == "failed" and error else None- if outcome == "failed":- details = [str(value) for value in errors or () if str(value)]- record.errors = details or ([error] if error else [])- if record.timing is not None:- record.timing.finished_at = max(- record.timing.started_at or record.timing.enqueued_at,- time.time(),- )- if outcome == "failed" and error:- self._failures.setdefault(metadata.task_id, error)-- def record_context_count(self, task_id: str, work_id: str, delta: int = 1) -> None:- """Add a result count to one Work without changing its lifecycle."""- if delta <= 0:- return- with self._lock:- record = self._work.get(task_id, {}).get(work_id)- if record is not None:- record.context_count += delta-- def record_failure(self, task_id: str, error: str) -> None:- """Record the first failed Work owned by a Task."""- with self._lock:- self._failures.setdefault(task_id, error)-- def failure(self, task_id: str) -> Optional[str]:- with self._lock:- return self._failures.get(task_id)+ for key in [+ key+ for key in self._active+ if key[2] == task_id+ and (account_id is None or key[0] == account_id)+ and (user_id is None or key[1] == user_id)+ ]:+ self._active.pop(key, None)- def clear_failure(self, task_id: str) -> None:- with self._lock:- self._failures.pop(task_id, None)-- async def wait_for_work(+ def register_active(self,task_id: str,- work_ids: Iterable[str],+ active_task: asyncio.Task[Any],*,- timeout: Optional[float] = None,- poll_interval: float = 0.05,- stop_waiting: Optional[Callable[[], bool]] = None,+ account_id: str,+ user_id: str,) -> None:- """Wait until exactly the supplied logical Work IDs are terminal."""- requested = frozenset(str(work_id) for work_id in work_ids)- if not requested:- return-- deadline = None if timeout is None else time.monotonic() + timeout- while True:- with self._lock:- records = self._work.get(task_id, {})- missing = requested.difference(records)- done = not missing and all(records[work_id].terminal for work_id in requested)- if done:- return- if missing:- raise KeyError(f"Unknown Work for Task {task_id}: {sorted(missing)}")- if stop_waiting is not None and stop_waiting():- raise asyncio.CancelledError("Worker stopped with queued Work")- remaining = None if deadline is None else deadline - time.monotonic()- if remaining is not None and remaining <= 0:- raise TimeoutError(f"Timed out waiting for Work in Task {task_id}")- await asyncio.sleep(- poll_interval if remaining is None else min(poll_interval, remaining)- )+ with self._lock:+ self._active.setdefault((account_id, user_id, task_id), set()).add(active_task)- async def wait_for_task_work(+ def unregister_active(self,task_id: str,+ active_task: asyncio.Task[Any],*,- exclude_work_id: Optional[str] = None,- timeout: Optional[float] = None,- poll_interval: float = 0.05,+ account_id: str,+ user_id: str,) -> None:- """Drain Task deliveries, including newly enqueued and recovered Work.-- Only the Task coordinator may use this barrier, excluding its own Work.- Inline coordinators have no delivery to exclude. Other workers must wait- for explicit Work IDs to avoid waiting on each other.- """- deadline = None if timeout is None else time.monotonic() + timeout- while True:- with self._lock:- pending = self._has_delivery_locked(task_id, exclude_work_id)- if not pending:- return- remaining = None if deadline is None else deadline - time.monotonic()- if remaining is not None and remaining <= 0:- raise TimeoutError(f"Timed out waiting for Work in Task {task_id}")- await asyncio.sleep(- poll_interval if remaining is None else min(poll_interval, remaining)- )-- def queue_status(- self,- task_id: str,- work_ids: Optional[Iterable[str]] = None,- ) -> Dict[str, Dict[str, object]]:- """Project queue counters from logical Work records."""- selected = None if work_ids is None else frozenset(str(value) for value in work_ids)- with self._lock:- records = [- record- for work_id, record in self._work.get(task_id, {}).items()- if selected is None or work_id in selected- ]- result: Dict[str, Dict[str, object]] = {}- for record in records:- status = result.setdefault(- record.queue_name,- {- "processed": 0,- "requeue_count": 0,- "error_count": 0,- "errors": [],- },- )- if record.status == "completed":- status["processed"] += 1- status["requeue_count"] += record.requeue_count- if record.status == "failed":- errors = record.errors or ([record.error] if record.error else [])- status["error_count"] += len(errors) or 1- status["errors"].extend({"message": message} for message in errors)- return result-- def queue_timing(self, task_id: str) -> Dict[str, Dict[str, float]]:- """Project queue wait and execution timing in milliseconds."""with self._lock:- result: Dict[str, Dict[str, float]] = {}- for record in self._work.get(task_id, {}).values():- timing = record.timing- metrics = result.setdefault(- record.queue_name.lower(),- {"queue_wait_ms": 0.0, "execute_ms": 0.0},- )- if timing is None or timing.started_at is None:- continue- metrics["queue_wait_ms"] += max(- 0.0, (timing.started_at - timing.enqueued_at) * 1000.0- )- if timing.finished_at is not None:- metrics["execute_ms"] += max(- 0.0, (timing.finished_at - timing.started_at) * 1000.0- )- return result-- def context_count(self, task_id: str) -> int:- with self._lock:- return sum(record.context_count for record in self._work.get(task_id, {}).values())-- def init_processing(self, task_id: str) -> None:- with self._lock:- timings = self._work_timings.setdefault(task_id, {})- for work_id, record in self._work.get(task_id, {}).items():- if record.timing is not None:- timings.setdefault(work_id, record.timing)-- def processing_seconds(self, task_id: str) -> Optional[float]:- with self._lock:- timings = self._work_timings.get(task_id)- if timings is None:- return None- now = time.time()- intervals: list[tuple[float, float]] = []- for timing in timings.values():- if timing.started_at is None:- continue- end = timing.finished_at if timing.finished_at is not None else now- intervals.append((timing.started_at, max(timing.started_at, end)))- intervals.sort()- if not intervals:- return None-- elapsed = 0.0- current_start, current_end = intervals[0]- for start, end in intervals[1:]:- if start <= current_end:- current_end = max(current_end, end)- else:- elapsed += current_end - current_start- current_start, current_end = start, end- return elapsed + current_end - current_start-- def forget_processing(self, task_id: str) -> None:- with self._lock:- self._work_timings.pop(task_id, None)+ key = (account_id, user_id, task_id)+ entries = self._active.get(key)+ if entries is not None:+ entries.discard(active_task)+ if not entries:+ self._active.pop(key, None)- def forget_task(self, task_id: str) -> None:- """Release all process-local state after the owning Task is removed."""+ def active_tasks_by_owner(self) -> Dict[tuple[str, str], set[str]]:with self._lock:- self._work.pop(task_id, None)- self._delivery_counts.pop(task_id, None)- self._active.pop(task_id, None)- self._failures.pop(task_id, None)- self._work_timings.pop(task_id, None)- active_keys = [key for key in self._active_work_ids if key[0] == task_id]- for key in active_keys:- self._active_work_ids.pop(key, None)+ grouped: Dict[tuple[str, str], set[str]] = {}+ for account_id, user_id, task_id in self._active:+ grouped.setdefault((account_id, user_id), set()).add(task_id)+ return grouped- def register_active(+ def cancel_active(self,task_id: str,- active_task: asyncio.Task[Any],*,- work_id: Optional[str] = None,+ account_id: Optional[str] = None,+ user_id: Optional[str] = None,) -> None:with self._lock:- cancelled = self.cancellation_requested(task_id)- if not cancelled:- self._active.setdefault(task_id, set()).add(active_task)- now = time.time()- timing_id = work_id or f"inline:{id(active_task)}"- record = self._work.get(task_id, {}).get(timing_id)- if record is not None:- timing = record.timing- if timing is not None and timing.started_at is None:- timing.started_at = now- if not record.terminal:- record.status = "running"- timings = self._work_timings.get(task_id)- if timings is not None:- timing = timings.setdefault(timing_id, WorkTiming(enqueued_at=now))- if timing.started_at is None:- timing.started_at = now- timing.finished_at = None- self._active_work_ids[(task_id, active_task)] = timing_id- if cancelled:- active_task.get_loop().call_soon(active_task.cancel)-- def unregister_active(self, task_id: str, active_task: asyncio.Task[Any]) -> None:- with self._lock:- timing_id = self._active_work_ids.pop((task_id, active_task), None)- timings = self._work_timings.get(task_id)- if timing_id is not None and timings is not None:- timing = timings.get(timing_id)- if timing is not None and timing.started_at is not None:- timing.finished_at = max(timing.started_at, time.time())- entries = self._active.get(task_id)- if entries is not None:- entries.discard(active_task)- if not entries:- self._active.pop(task_id, None)-- def cancel_active(self, task_id: str) -> None:- with self._lock:- active = list(self._active.get(task_id, ()))+ active = [+ task+ for (account, user, active_task_id), tasks in self._active.items()+ if active_task_id == task_id+ and (account_id is None or account == account_id)+ and (user_id is None or user == user_id)+ for task in tasks+ ]for task in active:task.get_loop().call_soon_threadsafe(task.cancel)diff --git a/openviking/session/session.py b/openviking/session/session.pyindex 4dbede3e2..e98ba5ceb 100644--- a/openviking/session/session.py+++ b/openviking/session/session.py@@ -1197,7 +1197,7 @@ class Session:from openviking.service.task_tracker import get_task_trackertracker = get_task_tracker()- if not task_id or not tracker.has_work(str(task_id)):+ if not task_id or not await tracker.has_work(str(task_id)):error = "Phase 1 has no QueueFS work to resume"await self._write_failed_marker(archive_uri,@@ -2516,7 +2516,11 @@ class Session:telemetry.set_error("session.commit.phase2", "CANCELLED", "session commit cancelled")snapshot = telemetry.finish("cancelled")_publish_telemetry_summary_best_effort(snapshot)- if tracker.is_cancellation_requested(task_id):+ if await tracker.is_cancellation_requested(+ task_id,+ account_id=self.ctx.account_id,+ user_id=self.ctx.user.user_id,+ ):await self._write_failed_marker(archive_uri,stage="cancelled",@@ -2836,7 +2840,7 @@ class Session:from openviking.service.task_tracker import get_task_trackertracker = get_task_tracker()- if tracker.has_work(str(task_id)):+ if await tracker.has_work(str(task_id)):return Falseerror = "Session commit queue work is missing"diff --git a/openviking/storage/collection_schemas.py b/openviking/storage/collection_schemas.pyindex d10fea436..dfffc8e81 100644--- a/openviking/storage/collection_schemas.py+++ b/openviking/storage/collection_schemas.py@@ -21,7 +21,6 @@ from openviking.core.context import ContextType, ResourceContentTypefrom openviking.models.embedder.base import embed_compatfrom openviking.models.model_error import ModelErrorKind, model_error_kindfrom openviking.server.identity import RequestContext, Role-from openviking.service.task_tracker import get_task_trackerfrom openviking.service.task_tracker_concurrency import run_to_completionfrom openviking.storage.acl import ACL_GRANT_FIELDS, ACL_MODE_FIELD, AclModefrom openviking.storage.errors import (@@ -701,19 +700,16 @@ class TextEmbeddingHandler(DequeueHandlerBase):if self._vikingdb.is_closing:logger.debug("Skip embedding dequeue during shutdown")self._merge_request_stats(embedding_msg.telemetry_id, processed=1)- self._record_request_success(embedding_msg)return ProcessResult.success()if embedding_msg.action is IndexAction.NONE:self._merge_request_stats(embedding_msg.telemetry_id, processed=1)- self._record_request_success(embedding_msg)logger.debug("Skipping no-op embedding message: %s", embedding_msg.id)return ProcessResult.success()if embedding_msg.action not in {IndexAction.UPSERT, IndexAction.MERGE}:result = await self._apply_non_embedding_operation(embedding_msg, ctx)self._merge_request_stats(embedding_msg.telemetry_id, processed=1)- self._record_request_success(embedding_msg)return ProcessResult.success(result)if not isinstance(embedding_msg.message, (str, list)):@@ -721,7 +717,6 @@ class TextEmbeddingHandler(DequeueHandlerBase):f"Skipping unsupported message type: {type(embedding_msg.message)}")self._merge_request_stats(embedding_msg.telemetry_id, processed=1)- self._record_request_success(embedding_msg)return ProcessResult.success(data)# Circuit breaker: if API is known-broken, re-enqueue and wait@@ -942,7 +937,6 @@ class TextEmbeddingHandler(DequeueHandlerBase):if self._vikingdb.is_closing:logger.debug(f"Skip embedding write during shutdown: {db_err}")self._merge_request_stats(embedding_msg.telemetry_id, processed=1)- self._record_request_success(embedding_msg)return ProcessResult.success()error_msg = self._embedding_error_msg(embedding_msg,@@ -960,7 +954,6 @@ class TextEmbeddingHandler(DequeueHandlerBase):if self._vikingdb.is_closing:logger.debug(f"Skip embedding write during shutdown: {db_err}")self._merge_request_stats(embedding_msg.telemetry_id, processed=1)- self._record_request_success(embedding_msg)return ProcessResult.success()error_msg = self._embedding_error_msg(embedding_msg,@@ -973,20 +966,9 @@ class TextEmbeddingHandler(DequeueHandlerBase):return ProcessResult.failed(error_msg)self._merge_request_stats(embedding_msg.telemetry_id, processed=1)- self._record_request_success(- embedding_msg,- vector_written=bool(record_id),- )- return ProcessResult.success(inserted_data)+ return ProcessResult.success({"vector_written": bool(record_id)})except asyncio.CancelledError:- if (- embedding_msg is not None- and embedding_msg.context_data.get("context_type") == ContextType.SKILL.value- ):- # Active cancellation does not call on_cancelled in NamedQueue.- # Settle only after any already-started vector write has exited.- self._record_request_success(embedding_msg)raiseexcept Exception as e:error_msg = self._embedding_error_msg(@@ -1021,18 +1003,15 @@ class TextEmbeddingHandler(DequeueHandlerBase):async def on_cancelled(self, data: Optional[Dict[str, Any]]) -> ProcessResult:"""Settle request-scoped waiting when a queued embedding is cancelled."""- embedding_msg: Optional[EmbeddingMsg] = Nonetry:if data:payload = data.get("data", data)if isinstance(payload, str):payload = json.loads(payload)- embedding_msg = EmbeddingMsg.from_dict(payload)+ EmbeddingMsg.from_dict(payload)except (KeyError, TypeError, ValueError) as exc:return ProcessResult.failed(str(exc))- if embedding_msg is not None:- self._record_request_success(embedding_msg)return ProcessResult.cancelled()async def _reenqueue_embedding_msg(self, msg: EmbeddingMsg) -> None:@@ -1040,15 +1019,3 @@ class TextEmbeddingHandler(DequeueHandlerBase):await run_to_completion(lambda: self._vikingdb.enqueue_embedding_msg(msg))else:await self._vikingdb.enqueue_embedding_msg(msg)-- @staticmethod- def _record_request_success(- embedding_msg: EmbeddingMsg,- *,- vector_written: bool = False,- ) -> None:- if vector_written and embedding_msg.task_id and embedding_msg._task_work_id:- get_task_tracker().record_context_count(- embedding_msg.task_id,- embedding_msg._task_work_id,- )diff --git a/openviking/storage/content_write.py b/openviking/storage/content_write.pyindex f1e853c25..9cdacb092 100644--- a/openviking/storage/content_write.py+++ b/openviking/storage/content_write.py@@ -171,6 +171,13 @@ def _queue_has_errors(queue_status: Optional[Dict[str, Any]], name: str) -> boolreturn bool(status.get("errors"))+def _result_has_queue_errors(result: Any) -> bool:+ if not isinstance(result, dict):+ return False+ queue_status = result.get("queue_status")+ return any(_queue_has_errors(queue_status, name) for name in ("Semantic", "Embedding"))++@dataclass(frozen=True)class _BatchRefreshOutcome:queue_status: Optional[Dict[str, Any]]@@ -257,7 +264,13 @@ class ContentWriteCoordinator:tag_mode: str = "replace",acl: AclSpec | Dict[str, Any] | None = None,) -> Dict[str, Any]:+ return_terminal_result = wait and get_task_context() is None+async def execute() -> Dict[str, Any]:+ context = get_task_context()+ wait_for_queues = not (+ return_terminal_result and context is not None and context.work_id is not None+ )return await self._write(uri=uri,content=content,@@ -268,6 +281,7 @@ class ContentWriteCoordinator:tags=tags,tag_mode=tag_mode,acl=acl,+ wait_for_queues=wait_for_queues,)result, _task_id = await run_tracked_operation(@@ -277,6 +291,7 @@ class ContentWriteCoordinator:operation=execute,wait=wait,resource_id=uri,+ return_terminal_result=return_terminal_result,)return result@@ -292,6 +307,7 @@ class ContentWriteCoordinator:tags: list[str] | None = None,tag_mode: str = "replace",acl: AclSpec | Dict[str, Any] | None = None,+ wait_for_queues: bool = True,) -> Dict[str, Any]:self._validate_mode(mode)processing_mode = normalize_processing_mode(processing_mode)@@ -337,6 +353,7 @@ class ContentWriteCoordinator:ctx=ctx,telemetry_id=telemetry_id,ingest_options=ingest_options,+ wait_for_queues=wait_for_queues,)return await self._write_direct_with_refresh(@@ -351,6 +368,7 @@ class ContentWriteCoordinator:telemetry_id=telemetry_id,processing_mode=processing_mode,ingest_options=ingest_options,+ wait_for_queues=wait_for_queues,)async def batch_write(@@ -363,7 +381,13 @@ class ContentWriteCoordinator:skip_conflicts: bool = False,processing_mode: ProcessingMode = DEFAULT_PROCESSING_MODE,) -> Dict[str, Any]:+ return_terminal_result = wait and get_task_context() is None+async def execute() -> Dict[str, Any]:+ context = get_task_context()+ wait_for_queues = not (+ return_terminal_result and context is not None and context.work_id is not None+ )return await self._batch_write(root_uri=root_uri,operations=operations,@@ -371,6 +395,7 @@ class ContentWriteCoordinator:wait=wait,skip_conflicts=skip_conflicts,processing_mode=processing_mode,+ wait_for_queues=wait_for_queues,)result, _task_id = await run_tracked_operation(@@ -380,7 +405,11 @@ class ContentWriteCoordinator:operation=execute,wait=wait,resource_id=root_uri,+ accept_failed_result=_result_has_queue_errors if return_terminal_result else None,+ return_terminal_result=return_terminal_result,)+ if return_terminal_result:+ self._raise_refresh_errors(result.get("queue_status", {}))return resultasync def _batch_write(@@ -392,6 +421,7 @@ class ContentWriteCoordinator:wait: bool = True,skip_conflicts: bool = False,processing_mode: ProcessingMode = DEFAULT_PROCESSING_MODE,+ wait_for_queues: bool = True,) -> Dict[str, Any]:"""Write a bundle under one directory, then refresh it as a batch.@@ -697,12 +727,12 @@ class ContentWriteCoordinator:lock_settled = Truequeue_status = (await self._wait_for_request(telemetry_id=telemetry_id)- if wait+ if wait and wait_for_queueselse refresh_outcome.queue_statusif refresh_outcome is not Noneelse None)- if wait and queue_status is not None and write_error is None:+ if wait and wait_for_queues and queue_status is not None and write_error is None:self._raise_refresh_errors(queue_status)if refresh_outcome is not None and queue_status is not None:refresh_outcome = replace(refresh_outcome, queue_status=queue_status)@@ -734,7 +764,7 @@ class ContentWriteCoordinator:if skip_conflicts:result["conflicts"] = conflictsif refresh_outcome is not None:- semantic_status, vector_status = refresh_outcome.statuses(wait=wait)+ semantic_status, vector_status = refresh_outcome.statuses(wait=wait and wait_for_queues)result["semantic_status"] = semantic_statusresult["vector_status"] = vector_statusreturn result@@ -1560,6 +1590,7 @@ class ContentWriteCoordinator:telemetry_id: str,processing_mode: ProcessingMode = DEFAULT_PROCESSING_MODE,ingest_options: IngestOptions | None = None,+ wait_for_queues: bool = True,) -> Dict[str, Any]:# ``.abstract.md`` / ``.overview.md`` are generated sidecars whose body is# re-embedded in place (no LLM regeneration and no RNFV plan). Keep that@@ -1576,6 +1607,7 @@ class ContentWriteCoordinator:ctx=ctx,telemetry_id=telemetry_id,ingest_options=ingest_options,+ wait_for_queues=wait_for_queues,)lock_path = self._viking_fs._uri_to_path(uri, ctx=ctx)@@ -1672,10 +1704,14 @@ class ContentWriteCoordinator:await self._viking_fs._async_agfs.pathlock_release(lease)lock_settled = True- queue_status = await self._wait_for_request(telemetry_id=telemetry_id) if wait else None+ queue_status = (+ await self._wait_for_request(telemetry_id=telemetry_id)+ if wait and wait_for_queues+ else None+ )written_bytes = len(final_bytes)semantic_status, vector_status = self._plan_statuses(- work, wait=wait, queue_status=queue_status+ work, wait=wait and wait_for_queues, queue_status=queue_status)return self._build_write_result(uri=uri,@@ -1780,6 +1816,7 @@ class ContentWriteCoordinator:ctx: RequestContext,telemetry_id: str,ingest_options: IngestOptions | None = None,+ wait_for_queues: bool = True,) -> Dict[str, Any]:"""Re-embed a manually edited L0/L1 sidecar body without LLM regeneration."""lock_path = self._viking_fs._uri_to_path(uri, ctx=ctx)@@ -1805,10 +1842,14 @@ class ContentWriteCoordinator:)await self._viking_fs._async_agfs.pathlock_release(lease)lock_released = True- queue_status = await self._wait_for_request(telemetry_id=telemetry_id) if wait else None+ queue_status = (+ await self._wait_for_request(telemetry_id=telemetry_id)+ if wait and wait_for_queues+ else None+ )vector_status = self._queue_work_status(requested=vector_enqueued,- wait=wait,+ wait=wait and wait_for_queues,queue_status=queue_status,queue_name="Embedding",)@@ -2121,7 +2162,7 @@ class ContentWriteCoordinator:await tracker.wait_for_work(context.task_id, work_ids, timeout=None)except (TimeoutError, asyncio.TimeoutError) as exc:raise DeadlineExceededError("queue processing", None) from exc- return tracker.build_queue_status(+ return await tracker.build_queue_status(context.task_id, work_ids if context.work_id is not None else None)@@ -2137,6 +2178,7 @@ class ContentWriteCoordinator:ctx: RequestContext,telemetry_id: str,ingest_options: IngestOptions | None = None,+ wait_for_queues: bool = True,) -> Dict[str, Any]:lock_path = self._viking_fs._uri_to_path(uri, ctx=ctx)try:@@ -2181,13 +2223,13 @@ class ContentWriteCoordinator:ingest_options=ingest_options,)queue_status = None- if embedding_requested and wait:+ if embedding_requested and wait and wait_for_queues:queue_status = await self._wait_for_request(telemetry_id=telemetry_id,)vector_status = self._queue_work_status(requested=embedding_requested,- wait=wait,+ wait=wait and wait_for_queues,queue_status=queue_status,queue_name="Embedding",)diff --git a/openviking/storage/queuefs/add_resource_processor.py b/openviking/storage/queuefs/add_resource_processor.pyindex e0dd26266..6085ad1b9 100644--- a/openviking/storage/queuefs/add_resource_processor.py+++ b/openviking/storage/queuefs/add_resource_processor.py@@ -302,14 +302,19 @@ class AddResourceProcessor(DequeueHandlerBase):msg.task_id,exclude_work_id=metadata.work_id if metadata is not None else None,)- result["queue_status"] = tracker.build_queue_status(msg.task_id)+ work_summary = await tracker.get_work_summary(+ msg.task_id,+ account_id=ctx.account_id,+ user_id=ctx.user.user_id,+ )+ result["queue_status"] = tracker.format_queue_status(work_summary)if replay_result is None:- result["context_count"] = tracker.get_context_count(msg.task_id)+ result["context_count"] = work_summary.context_countrecord_resource_queue_metrics(telemetry=telemetry,telemetry_id=telemetry_id,root_uri=result.get("root_uri"),- queue_timing=tracker.get_queue_timing(msg.task_id),+ queue_timing=tracker.format_queue_timing(work_summary),)telemetry.set("resource.total.duration_ms", telemetry.elapsed_ms())diff --git a/openviking/storage/queuefs/external_task_processor.py b/openviking/storage/queuefs/external_task_processor.pyindex 47a8ea14b..6f296094a 100644--- a/openviking/storage/queuefs/external_task_processor.py+++ b/openviking/storage/queuefs/external_task_processor.py@@ -8,7 +8,11 @@ import jsonfrom typing import Any, Dict, Optionalfrom openviking.service.external_task_service import ExternalTaskService-from openviking.service.task_work_index import TaskWorkRejected+from openviking.service.task_work_index import (+ TASK_WORK_ID_FIELD,+ TaskWorkRejected,+ extract_task_metadata,+)from openviking.storage.queuefs import QueueManager, get_queue_managerfrom openviking.storage.queuefs.named_queue import DequeueHandlerBasefrom openviking.storage.queuefs.process_result import ProcessResult@@ -45,11 +49,15 @@ class ExternalTaskProcessor(DequeueHandlerBase):processed = await self._service.execute(task_id, account_id, user_id)if not processed:# Register a new delivery before the old one is ACKed. Keeping the- # task ID preserves cancellation and task-owned work across rotation.+ # Work ID preserves one logical Work across delivery rotation.+ replacement = {"task_id": task_id, "account_id": account_id, "user_id": user_id}+ metadata = extract_task_metadata(data)+ if metadata is not None:+ replacement[TASK_WORK_ID_FIELD] = metadata.work_idtry:await get_queue_manager().enqueue(QueueManager.EXTERNAL_TASK,- {"task_id": task_id, "account_id": account_id, "user_id": user_id},+ replacement,)except TaskWorkRejected:# Cancellation only needs the current delivery to be ACKed.diff --git a/openviking/storage/queuefs/queue_manager.py b/openviking/storage/queuefs/queue_manager.pyindex 1907850d1..6a29f9b6a 100644--- a/openviking/storage/queuefs/queue_manager.py+++ b/openviking/storage/queuefs/queue_manager.py@@ -11,7 +11,6 @@ import threadingimport tracebackfrom typing import TYPE_CHECKING, Any, Dict, Optional, Sequence, Set, Union-from openviking.service.task_work_index import TaskWorkIndexfrom openviking.utils.async_client_cache import LoopScopedAsyncClientCachefrom openviking_cli.utils.logger import get_logger@@ -136,13 +135,12 @@ class QueueManager:self._queue_stop_events: Dict[str, threading.Event] = {}self._embedding_worker_stopped = threading.Event()self._poll_interval = 0.2- self._task_work_index = TaskWorkIndex()self._vlm_resolver: Optional["VLMResolver"] = None# Import at composition time to avoid a service <-> queue package cycle.from openviking.service.task_queue_middleware import TaskWorkQueueMiddlewareself._middlewares: tuple[QueueMiddleware, ...] = (- TaskWorkQueueMiddleware(self._task_work_index),+ TaskWorkQueueMiddleware(),*middlewares,)@@ -167,10 +165,28 @@ class QueueManager:logger.info(f"[QueueManager] mount_point={self.mount_point} Started")async def prepare_task_tracking(self, tracker: Any) -> list[Any]:- """Rebuild task work from QueueFS before any consumer starts."""+ """Bootstrap Work only for stores that reconstruct it from QueueFS."""+ from openviking.service.task_domain import WorkRecoveryMode+ from openviking.service.task_work_index import extract_task_metadata++ if tracker.work_recovery_mode is WorkRecoveryMode.DURABLE_STORE:+ return []+snapshots = {name: await queue.snapshot() for name, queue in self._queues.items()}- owners = self._task_work_index.rebuild(snapshots)- tracker.attach_work_index(self._task_work_index)+ owners: Dict[str, tuple[str, str]] = {}+ for queue_name, messages in snapshots.items():+ for message in messages:+ metadata = extract_task_metadata(message)+ if metadata is None or not metadata.account_id or not metadata.user_id:+ continue+ owners[metadata.task_id] = (metadata.account_id, metadata.user_id)+ await tracker.reconcile_delivery(+ metadata.task_id,+ metadata.work_id,+ queue_name,+ account_id=metadata.account_id,+ user_id=metadata.user_id,+ )return await tracker.restore_work_tasks(owners)def setup_standard_queues(diff --git a/openviking/storage/queuefs/semantic_processor.py b/openviking/storage/queuefs/semantic_processor.pyindex 222494fdb..3ed865652 100644--- a/openviking/storage/queuefs/semantic_processor.py+++ b/openviking/storage/queuefs/semantic_processor.py@@ -598,9 +598,11 @@ class SemanticProcessor(DequeueHandlerBase):account_id=task_context.account_id,user_id=task_context.user_id,)- siblings_pending = tracker.has_work(+ siblings_pending = await tracker.has_work(task_context.task_id,exclude_work_id=task_context.work_id,+ account_id=task_context.account_id,+ user_id=task_context.user_id,)else:enqueue_complete = Truediff --git a/openviking/storage/viking_fs/_snapshot.py b/openviking/storage/viking_fs/_snapshot.pyindex b88c352ce..9a7aa5506 100644--- a/openviking/storage/viking_fs/_snapshot.py+++ b/openviking/storage/viking_fs/_snapshot.py@@ -858,7 +858,11 @@ class _SnapshotMixin:from openviking.service.task_work_index import bind_task_contexttracker = get_task_tracker()- tracker.register_running_task(task_id)+ running_work_id = await tracker.register_running_task(+ task_id,+ account_id=ctx.account_id,+ user_id=ctx.user.user_id,+ )try:await tracker.start(task_id,@@ -871,7 +875,12 @@ class _SnapshotMixin:vector_config_resolver=self._vector_config_resolver,)reindex_ctx = self._restore_reindex_context(ctx)- with bind_task_context(task_id, ctx.account_id, ctx.user.user_id):+ with bind_task_context(+ task_id,+ ctx.account_id,+ ctx.user.user_id,+ work_id=running_work_id,+ ):await asyncio.gather(*[self._run_vector_rebuild(diff --git a/tests/agfs/test_viking_fs_git.py b/tests/agfs/test_viking_fs_git.pyindex 94ed4253b..b6eba66ff 100644--- a/tests/agfs/test_viking_fs_git.py+++ b/tests/agfs/test_viking_fs_git.py@@ -21,9 +21,12 @@ from openviking.pyagfs.exceptions import (AGFSNotSupportedError,)from openviking.server.identity import RequestContext, Role+from openviking.service.task_store import PersistentTaskStore+from openviking.service.task_work_index import get_task_contextfrom openviking.storage.viking_fs import VikingFSfrom openviking_cli.exceptions import NotFoundErrorfrom openviking_cli.session.user_id import UserIdentifier+from tests.test_task_tracker import _FakeAgfsragfs_python = pytest.importorskip("ragfs_python")@@ -635,26 +638,16 @@ def _patch_reindex_executor(monkeypatch, vfs, spy):monkeypatch.setattr(reindex_mod, "ReindexExecutor", factory)-class _MemTaskStore:+class _MemTaskStore(PersistentTaskStore):def __init__(self, *, fail_creates: int = 0):+ super().__init__(_FakeAgfs())self.fail_creates = fail_creates- async def create(self, task):+ async def create(self, task, *, reject_if_active=False):if self.fail_creates:self.fail_creates -= 1raise OSError("task store temporarily unavailable")-- async def update(self, task):- return None-- async def get(self, task_id, *, account_id=None, user_id=None):- return None-- async def list(self, account_id, *, user_id=None):- return []-- async def delete(self, task_id, *, account_id, user_id=None):- return None+ return await super().create(task, reject_if_active=reject_if_active)@pytest.mark.asyncio@@ -864,6 +857,14 @@ async def test_restore_returns_pollable_task_id(vfs, monkeypatch):reindexed = []async def run_reindex(_executor, **kwargs):+ context = get_task_context()+ await get_task_tracker().wait_for_task_work(+ context.task_id,+ exclude_work_id=context.work_id,+ timeout=0,+ account_id=context.account_id,+ user_id=context.user_id,+ )reindexed.append(kwargs["uri"])return {"status": "completed",diff --git a/tests/server/test_admin_api.py b/tests/server/test_admin_api.pyindex d540a0403..24bc46195 100644--- a/tests/server/test_admin_api.py+++ b/tests/server/test_admin_api.py@@ -40,6 +40,10 @@ from openviking.service.task_store import (SYSTEM_TASK_USER_ID,)from openviking.service.task_tracker import get_task_tracker+from openviking.service.task_work_index import (+ TaskWorkRejected,+ bind_task_context,+)from openviking.session.memory.account_templates import (EDITABLE_MEMORY_TEMPLATE_FIELDS,account_memory_template_path,@@ -319,8 +323,6 @@ async def test_delete_account_files_disables_automatic_pathlock(monkeypatch):agfs.stat = AsyncMock(side_effect=AGFSNotFoundError(path))service = Mock()service.viking_fs._async_agfs = agfs- tracker = Mock()- monkeypatch.setattr("openviking.service.deletion.get_task_tracker", lambda: tracker)deletion = DeletionService(service=service,manager=Mock(),@@ -330,7 +332,6 @@ async def test_delete_account_files_disables_automatic_pathlock(monkeypatch):await deletion._delete_account_files("account-to-delete")agfs.rm.assert_awaited_once_with(path, recursive=True, auto_pathlock=False)- tracker.forget_account_tasks.assert_called_once_with("account-to-delete")@pytest_asyncio.fixture@@ -1036,10 +1037,12 @@ async def test_account_memory_templates_range_expressions_reach_file_body(fs = lightweight_admin_app.state.fake_service.viking_fssnapshot = await resolve_account_memory_registry(fs, account_id, MemoryTypeRegistry())schema = snapshot.get("events")- context = ExtractContext([- Message(id="m1", role="user", parts=[TextPart("Selected source message")]),- Message(id="m2", role="user", parts=[TextPart("Unrelated source message")]),- ])+ context = ExtractContext(+ [+ Message(id="m1", role="user", parts=[TextPart("Selected source message")]),+ Message(id="m2", role="user", parts=[TextPart("Unrelated source message")]),+ ]+ )rendered = MemoryFileUtils.write(MemoryFile(memory_type="events", extra_fields={"ranges": "0", "summary": "summary"}),content_template=schema.content_template,@@ -1362,29 +1365,33 @@ async def test_account_memory_templates_locked_fields_cannot_be_modified(for key, value in defaults.items()if key not in editable]- invalid.extend([- {"unknown": "ignored?"},- {"Description": "wrong key"},- {"_updated_at": "forged"},- {"_account_content_template": False},- {"deployment_content_template": "forged"},- {"enabled": 1},- {"fields": [{"name": "renamed_field", "description": "changed"}]},- {"description": None},- {"description": " \n"},- {"fields": [None]},- ])+ invalid.extend(+ [+ {"unknown": "ignored?"},+ {"Description": "wrong key"},+ {"_updated_at": "forged"},+ {"_account_content_template": False},+ {"deployment_content_template": "forged"},+ {"enabled": 1},+ {"fields": [{"name": "renamed_field", "description": "changed"}]},+ {"description": None},+ {"description": " \n"},+ {"fields": [None]},+ ]+ )for field in defaults["fields"]:invalid.extend({"fields": [{"name": field["name"], key: None if value is not None else "changed"}]}for key, value in field.items()if key not in {"name", "description"})- invalid.extend([- {"fields": [{"name": field["name"], "unknown": "ignored?"}]},- {"fields": [{"name": field["name"], "description": ""}]},- {"fields": [{"name": field["name"]}, {"name": field["name"]}]},- ])+ invalid.extend(+ [+ {"fields": [{"name": field["name"], "unknown": "ignored?"}]},+ {"fields": [{"name": field["name"], "description": ""}]},+ {"fields": [{"name": field["name"]}, {"name": field["name"]}]},+ ]+ )if memory_type == "events":invalid.extend({"fields": [{"name": name, "description": "changed"}]} for name in ("goal", "ranges")@@ -2378,18 +2385,29 @@ async def test_delete_account(account_id=SYSTEM_TASK_ACCOUNT_ID,user_id=SYSTEM_TASK_USER_ID,)- await queue.enqueue({- "task_id": user_cleanup.task_id,- "account_id": acct,- "user_id": "alice",- "target": {"account_id": acct, "user_id": "bob"},- })- await queue.enqueue({- "task_id": late_task_id,- "account_id": SYSTEM_TASK_ACCOUNT_ID,- "user_id": SYSTEM_TASK_USER_ID,- "target": {"account_id": acct, "user_id": "bob"},- })+ with bind_task_context(user_cleanup.task_id, acct, "alice"):+ with pytest.raises(TaskWorkRejected, match="unavailable"):+ await queue.enqueue(+ {+ "task_id": user_cleanup.task_id,+ "account_id": acct,+ "user_id": "alice",+ "target": {"account_id": acct, "user_id": "bob"},+ }+ )+ with bind_task_context(+ late_task_id,+ SYSTEM_TASK_ACCOUNT_ID,+ SYSTEM_TASK_USER_ID,+ ):+ await queue.enqueue(+ {+ "task_id": late_task_id,+ "account_id": SYSTEM_TASK_ACCOUNT_ID,+ "user_id": SYSTEM_TASK_USER_ID,+ "target": {"account_id": acct, "user_id": "bob"},+ }+ )skipped = await _wait_for_task(admin_client, late_task_id)assert skipped["status"] == "completed"assert skipped["result"] == {"deleted": not recreated, "stale": True}@@ -3521,10 +3539,12 @@ async def test_server_restart_requires_root_and_valid_revision(server_overrides = {"host": cli_host}lightweight_admin_app.state.server_config_overrides = server_overridespath.write_text(- json.dumps({- "server": server,- "vlm": {"provider": "openai", "model": "gpt-4o", "api_key": "test-key"},- })+ json.dumps(+ {+ "server": server,+ "vlm": {"provider": "openai", "model": "gpt-4o", "api_key": "test-key"},+ }+ ))monkeypatch.setattr(OpenVikingConfigSingleton, "_config_file", path)monkeypatch.setattr(OpenVikingConfigSingleton, "_config_file_content", path.read_bytes())diff --git a/tests/server/test_bot_proxy_auth.py b/tests/server/test_bot_proxy_auth.pyindex d2d4f7228..b5e1aacd9 100644--- a/tests/server/test_bot_proxy_auth.py+++ b/tests/server/test_bot_proxy_auth.py@@ -482,9 +482,9 @@ async def test_compile_api_client_session_protocol_retry_and_cancellation(}private_payload = {"args": private_args} if private_args else {}assert task.to_dict()["meta"]["request"].get("args", {}) == public_args- stored = await store.get(task.task_id, **owner)+ stored = await store.get_task(task.task_id, **owner)assert stored is not None- assert stored["meta"]["request"].get("args", {}) == public_args+ assert stored.meta["request"].get("args", {}) == public_argstracker = TaskTracker(store)restored = await tracker.get(task.task_id, **owner)@@ -594,6 +594,7 @@ async def test_compile_api_client_session_protocol_retry_and_cancellation(queue._async_agfs = backendstore = PersistentTaskStore(MockLocalAGFS(root_path=tmp_path))tracker = TaskTracker(store)+ monkeypatch.setattr("openviking.service.task_tracker._instance", tracker)monkeypatch.setattr(external_task_service_module, "get_task_tracker", lambda: tracker)monkeypatch.setattr(external_task_service_module, "get_queue_manager", lambda: manager)monkeypatch.setattr(@@ -670,13 +671,21 @@ async def test_compile_api_client_session_protocol_retry_and_cancellation(include_events=False,_ctx=ctx,)- assert queried.result == (await state(first)).to_dict()+ queried_task = dict(queried.result)+ current_task = (await state(first)).to_dict()+ assert queried_task.pop("processing_seconds") >= 0+ assert current_task.pop("processing_seconds") >= 0+ assert queried_task == current_taskassert queried.result["meta"]["request"].get("args", {}) == public_argsawait queue.dequeue() # Same target rotates, including across users.assert (await state(waiting)).status == TaskStatus.PENDINGassert waiting.task_id not in submitted- assert tracker.has_work(waiting.task_id) # Requeue survives ACK of the old delivery.+ assert await tracker.has_work(+ waiting.task_id,+ account_id=waiting.account_id,+ user_id=waiting.user_id,+ ) # Requeue survives ACK of the old delivery.await queue.dequeue()assert (await state(independent)).status == TaskStatus.COMPLETEDawait tracker.cancel(cancelled.task_id, account_id="acct", user_id="alice")@@ -695,6 +704,7 @@ async def test_compile_api_client_session_protocol_retry_and_cancellation(with pytest.raises(asyncio.CancelledError):await workertracker = TaskTracker(store)+ monkeypatch.setattr("openviking.service.task_tracker._instance", tracker)tasks = ExternalTaskService()tasks.register(CompileService(service._config, tasks, SimpleNamespace()))queue.set_dequeue_handler(ExternalTaskProcessor(tasks))diff --git a/tests/server/test_compile_studio.py b/tests/server/test_compile_studio.pyindex 3ef5f044c..c7423c0ec 100644--- a/tests/server/test_compile_studio.py+++ b/tests/server/test_compile_studio.py@@ -1,6 +1,7 @@"""Studio task pagination and submission retry contracts."""import asyncio+import jsonimport timefrom types import SimpleNamespacefrom unittest.mock import AsyncMock@@ -19,20 +20,20 @@ from tests.test_task_tracker import _FakeAgfs@pytest.mark.asyncioasync def test_page_reads_past_200_and_isolates_owner():- store = PersistentTaskStore(_FakeAgfs())+ agfs = _FakeAgfs()+ store = PersistentTaskStore(agfs)tracker = TaskTracker(store)owner = {"account_id": "acme", "user_id": "alice"}for index in range(205):- record = await tracker.create(+ await tracker.create("compile",task_id=f"cmp_{index:03}",meta={"request": {"to": "viking://resources/wiki"}},**owner,)- record.created_at = 100.0- await store.update(record)await tracker.create("compile", task_id="foreign", account_id="acme", user_id="bob")# Empty cache: pagination must read durable records without repopulating all tasks.+ store = PersistentTaskStore(agfs)tracker = TaskTracker(store)filters = {"task_type": "compile",@@ -44,7 +45,7 @@ async def test_page_reads_past_200_and_isolates_owner():seen = []before = Nonewhile True:- page = await tracker.list_page(**owner, limit=30, before=before, **filters)+ page = await tracker.list_tasks(**owner, limit=30, before=before, **filters)if not page:breakseen.extend(task.task_id for task in page)@@ -52,7 +53,6 @@ async def test_page_reads_past_200_and_isolates_owner():assert len(seen) == len(set(seen)) == 205assert seen[0] == "cmp_204"assert "foreign" not in seen- assert not tracker._tasksdef test_cursor_is_bound_to_identity_and_filters():@@ -69,7 +69,8 @@ def test_cursor_is_bound_to_identity_and_filters():async def test_concurrent_submission_retry_and_restart(monkeypatch):import openviking.service.external_task_service as module- store = PersistentTaskStore(_FakeAgfs())+ agfs = _FakeAgfs()+ store = PersistentTaskStore(agfs)tracker = TaskTracker(store)monkeypatch.setattr(module, "get_task_tracker", lambda: tracker)monkeypatch.setattr(module, "get_queue_manager", lambda: SimpleNamespace(enqueue=AsyncMock()))@@ -109,7 +110,8 @@ async def test_http_contract_preserves_legacy_list_and_retries(monkeypatch):from openviking_cli.exceptions import OpenVikingErrorfrom openviking_cli.utils.config.open_viking_config import CompileApiConfig- store = PersistentTaskStore(_FakeAgfs())+ agfs = _FakeAgfs()+ store = PersistentTaskStore(agfs)tracker = TaskTracker(store)monkeypatch.setattr(external, "get_task_tracker", lambda: tracker)monkeypatch.setattr(task_router, "get_task_tracker", lambda: tracker)@@ -189,6 +191,7 @@ async def test_http_contract_preserves_legacy_list_and_retries(monkeypatch):assert retry_after_cancel.json()["result"]["task_id"] == task_idassert "submission_hash" not in retry_after_cancel.json()["result"]["meta"]ctx = RequestContext(user=UserIdentifier("acme", "alice"), role=Role.ROOT)+ store = PersistentTaskStore(agfs)tracker = TaskTracker(store)root_page = await client.get("/api/v1/tasks", params={"pagination": "cursor", "task_type": "compile"}@@ -224,7 +227,8 @@ async def test_http_contract_preserves_legacy_list_and_retries(monkeypatch):@pytest.mark.asyncioasync def test_cursor_prunes_cold_expired_records_without_loading_cache():- store = PersistentTaskStore(_FakeAgfs())+ agfs = _FakeAgfs()+ store = PersistentTaskStore(agfs)tracker = TaskTracker(store)owner = {"account_id": "acme", "user_id": "alice"}for task_id, status, days in [@@ -235,12 +239,16 @@ async def test_cursor_prunes_cold_expired_records_without_loading_cache():("active", TaskStatus.RUNNING, 8),]:task = await tracker.create("compile", task_id=task_id, **owner)- task.status = status- task.updated_at = time.time() - days * 86400- await store.update(task)+ path = f"/local/acme/_system/tasks/alice/{task.task_id}.json"+ payload = json.loads(agfs.files[path])+ payload["status"] = status.value+ payload["updated_at"] = time.time() - days * 86400+ agfs.files[path] = json.dumps(payload).encode()+ store = PersistentTaskStore(agfs)tracker = TaskTracker(store, max_concurrent_store_io=1)+ await tracker.delete_expired()page = await asyncio.wait_for(- tracker.list_page(+ tracker.list_tasks(**owner,limit=30,task_type="compile",@@ -252,49 +260,41 @@ async def test_cursor_prunes_cold_expired_records_without_loading_cache():timeout=3,)assert {task.task_id for task in page} == {"recent-failed", "active"}- assert tracker.count() == 0- assert {task["task_id"] for task in await store.list("acme", user_id="alice")} == {+ assert {task.task_id for task in await store.list("acme", user_id="alice")} == {"recent-failed","active",}@pytest.mark.asyncio-async def test_cursor_cleanup_rechecks_task_updated_after_scan(monkeypatch):- store = PersistentTaskStore(_FakeAgfs())+async def test_cursor_cleanup_rechecks_task_updated_after_scan():+ agfs = _FakeAgfs()+ store = PersistentTaskStore(agfs)tracker = TaskTracker(store, max_concurrent_store_io=1)owner = {"account_id": "acme", "user_id": "alice"}task = await tracker.create("compile", task_id="updated-after-scan", **owner)- task.status = TaskStatus.COMPLETED- task.updated_at = time.time() - 2 * 86400- await store.update(task)+ path = f"/local/acme/_system/tasks/alice/{task.task_id}.json"+ payload = json.loads(agfs.files[path])+ payload["status"] = TaskStatus.COMPLETED.value+ payload["updated_at"] = time.time() - 2 * 86400+ agfs.files[path] = json.dumps(payload).encode()+ store = PersistentTaskStore(agfs)tracker = TaskTracker(store, max_concurrent_store_io=1)- prune = tracker._prune_persisted_expired- scanned = asyncio.Event()-- async def signal_before_prune(payload):- scanned.set()- return await prune(payload)-- monkeypatch.setattr(tracker, "_prune_persisted_expired", signal_before_prune)- async with tracker._task_locks.acquire(task.task_id):- reading = asyncio.create_task(- tracker.list_page(- **owner,- limit=30,- task_type="compile",- status=None,- resource_id=None,- include_internal=False,- q=None,- )- )- await asyncio.wait_for(scanned.wait(), timeout=3)- task.updated_at = time.time()- # A scan waiting for this task lock must not hold the sole I/O slot.- await asyncio.wait_for(- tracker._store_io.run("update", lambda: store.update(task)), timeout=3- )- page = await asyncio.wait_for(reading, timeout=3)+ record_lock = ("acme", "alice", task.task_id)+ async with store._record_locks.acquire(record_lock):+ cleanup = asyncio.create_task(tracker.delete_expired())+ await asyncio.sleep(0)+ payload["updated_at"] = time.time()+ agfs.files[path] = json.dumps(payload).encode()+ await asyncio.wait_for(cleanup, timeout=3)+ page = await tracker.list_tasks(+ **owner,+ limit=30,+ task_type="compile",+ status=None,+ resource_id=None,+ include_internal=False,+ q=None,+ )assert [record.task_id for record in page] == [task.task_id]- assert await store.get(task.task_id, **owner) is not None+ assert await store.get_task(task.task_id, **owner) is not Nonediff --git a/tests/server/test_content_batch_write.py b/tests/server/test_content_batch_write.pyindex 8f71a95f0..cc0f7fa0a 100644--- a/tests/server/test_content_batch_write.py+++ b/tests/server/test_content_batch_write.py@@ -1,3 +1,4 @@+import asyncioimport base64import hashlibfrom types import SimpleNamespace@@ -8,8 +9,12 @@ import pytestimport openviking.storage.content_write as content_write_modulefrom openviking.server.identity import RequestContext, Rolefrom openviking.service.task_store import PersistentTaskStore-from openviking.service.task_tracker import TaskTracker-from openviking.service.task_work_index import TaskWorkIndex, bind_task_context+from openviking.service.task_tracker import TaskTracker, get_task_tracker, run_tracked_operation+from openviking.service.task_work_index import (+ bind_task_context,+ get_task_context,+ record_enqueued_work,+)from openviking.session.memory.dataclass import MemoryFilefrom openviking.session.memory.utils import MemoryFileUtilsfrom openviking.storage.content_write import ContentWriteCoordinator@@ -131,7 +136,6 @@ def _coordinator_queue_for_unit_tests(monkeypatch, request):)monkeypatch.setattr(content_write_module, "get_queue_manager", lambda: queue_manager)tracker = TaskTracker(PersistentTaskStore(_FakeAgfs()))- tracker.attach_work_index(TaskWorkIndex())monkeypatch.setattr("openviking.service.task_tracker._instance", tracker)async def run_tracked_operation(_task_type, *, operation, **_kwargs):@@ -685,7 +689,25 @@ async def test_batch_hands_off_file_locks_after_one_aggregated_refresh(monkeypatlocks = vfs._async_agfscoordinator = ContentWriteCoordinator(vfs)calls = []- queue = SimpleNamespace(enqueue=AsyncMock(return_value=EnqueueReceipt("queued", "work-1")))+ workers = []+ tracker = get_task_tracker()++ async def enqueue(msg):+ context = get_task_context()+ owner = {"account_id": context.account_id, "user_id": context.user_id}+ await tracker.register_work(context.task_id, "coordinator", "Semantic", **owner)+ record_enqueued_work("Semantic", "coordinator")++ async def finish_after_producer():+ await tracker.wait_for_task_work(+ context.task_id, exclude_work_id="coordinator", poll_interval=0.001, **owner+ )+ await tracker.mark_work_done(context.task_id, "coordinator", **owner)++ workers.append(asyncio.create_task(finish_after_producer()))+ return EnqueueReceipt("queued", "coordinator")++ queue = SimpleNamespace(enqueue=AsyncMock(side_effect=enqueue))queue_manager = SimpleNamespace(SEMANTIC="semantic",get_queue=lambda *args, **kwargs: queue,@@ -699,19 +721,29 @@ async def test_batch_hands_off_file_locks_after_one_aggregated_refresh(monkeypatmonkeypatch.setattr(coordinator, "_refresh_batch", refresh)monkeypatch.setattr(content_write_module, "get_queue_manager", lambda: queue_manager)+ monkeypatch.setattr(content_write_module, "run_tracked_operation", run_tracked_operation)ctx = RequestContext(user=UserIdentifier.the_default_user(), role=Role.USER)- with bind_telemetry(telemetry):- result = await coordinator.batch_write(- root_uri=root,- operations=[- {"uri": b, "content": "B", "mode": "upsert"},- {"uri": a, "content": "A", "mode": "upsert"},- ],- ctx=ctx,- wait=False,- )+ try:+ with bind_telemetry(telemetry):+ result = await asyncio.wait_for(+ coordinator.batch_write(+ root_uri=root,+ operations=[+ {"uri": b, "content": "B", "mode": "upsert"},+ {"uri": a, "content": "A", "mode": "upsert"},+ ],+ ctx=ctx,+ wait=True,+ ),+ timeout=1,+ )+ finally:+ for worker in workers:+ worker.cancel()+ await asyncio.gather(*workers, return_exceptions=True)assert vfs.writes == [a, b]assert result["created"] == [a, b]+ assert result["queue_status"]["Semantic"]["processed"] == 1assert calls == [{a: "added", b: "added"}]assert locks.releases == 0assert locks.handoffs == 1diff --git a/tests/service/test_resource_service_watch.py b/tests/service/test_resource_service_watch.pyindex f35a9813e..0d2813782 100644--- a/tests/service/test_resource_service_watch.py+++ b/tests/service/test_resource_service_watch.py@@ -16,6 +16,7 @@ from openviking.resource.watch_manager import WatchManagerfrom openviking.server.identity import RequestContext, Rolefrom openviking.service import resource_service as resource_service_modulefrom openviking.service.resource_service import ResourceService+from openviking.service.task_domain import WorkSummaryfrom openviking.service.task_tracker import TaskStatusfrom openviking.service.task_work_index import TASK_WORK_ID_FIELDfrom openviking.storage.content_write import ContentWriteCoordinator@@ -1026,9 +1027,9 @@ async def test_add_resource_processor_records_paused_watch_result(fail=AsyncMock(),get_task_auth=AsyncMock(return_value={}),wait_for_task_work=AsyncMock(),- build_queue_status=Mock(return_value={}),- get_context_count=Mock(return_value=0),- get_queue_timing=Mock(return_value={}),+ get_work_summary=AsyncMock(return_value=WorkSummary()),+ format_queue_status=Mock(return_value={}),+ format_queue_timing=Mock(return_value={}),get=AsyncMock(return_value=SimpleNamespace(status=TaskStatus.CANCELLING)),)monkeypatch.setattr(diff --git a/tests/service/test_skill_processing_cancellation.py b/tests/service/test_skill_processing_cancellation.pyindex eba7e505f..0abca48f1 100644--- a/tests/service/test_skill_processing_cancellation.py+++ b/tests/service/test_skill_processing_cancellation.py@@ -9,9 +9,9 @@ import pytestfrom openviking.server.identity import RequestContext, Rolefrom openviking.service.resource_service import ResourceService-from openviking.service.task_store import PersistentTaskStore+from openviking.service.task_store import PersistentTaskStore, WorkRegistrationfrom openviking.service.task_tracker import TaskStatus, TaskTracker-from openviking.service.task_work_index import QueueTaskMetadata, TaskWorkIndex, get_task_context+from openviking.service.task_work_index import get_task_contextfrom openviking.telemetry import (OperationTelemetry,bind_telemetry,@@ -38,7 +38,7 @@ async def test_skill_rollback_persists_cancellation_for_finished_work(monkeypatcrestarted = TaskTracker(store=store)restored = await restarted.get(task.task_id, **owner)assert restored.status == TaskStatus.CANCELLED- assert restarted.is_cancellation_requested(task.task_id)+ assert await restarted.is_cancellation_requested(task.task_id, **owner)if outcome == "fail":assert restored.error == "failed"@@ -46,21 +46,23 @@ async def test_skill_rollback_persists_cancellation_for_finished_work(monkeypatc@pytest.mark.asyncioasync def test_skill_rollback_prevents_late_ack_failure_from_replaying_new_index(monkeypatch):tracker = TaskTracker(store=PersistentTaskStore(_FakeAgfs()))- work = TaskWorkIndex()- tracker.attach_work_index(work)monkeypatch.setattr("openviking.service.task_tracker.get_task_tracker", lambda: tracker)ctx = RequestContext(user=UserIdentifier.the_default_user(), role=Role.ROOT)owner = {"account_id": ctx.account_id, "user_id": ctx.user.user_id}task = await tracker.create("add_skill", **owner)- child = QueueTaskMetadata(task.task_id, "last-vector", ctx.account_id, ctx.user.user_id)- work.register("Embedding", child)+ assert (+ await tracker.register_work(task.task_id, "last-vector", "Embedding", **owner)+ is WorkRegistration.CREATED+ )await tracker.complete(task.task_id, {}, **owner)- await work.prepare_ack("Embedding", child)+ await tracker.mark_work_done(task.task_id, "last-vector", **owner)assert (await tracker.get(task.task_id, **owner)).status == TaskStatus.COMPLETEDawait ResourceService.__new__(ResourceService).cancel_skill_processing(task.task_id, ctx)- work.rollback_ack("Embedding", child)- assert work.cancellation_requested(task.task_id)- assert not work.register("Embedding", child)+ assert await tracker.is_cancellation_requested(task.task_id, **owner)+ assert (+ await tracker.register_work(task.task_id, "last-vector", "Embedding", **owner)+ is WorkRegistration.REJECTED+ )@pytest.mark.asyncio@@ -69,10 +71,10 @@ async def test_cancel_during_skill_task_creation_settles_persisted_record(monkeystarted, resume = asyncio.Event(), asyncio.Event()original_create = store.create- async def slow_create(task):+ async def slow_create(task, **kwargs):started.set()await resume.wait()- await original_create(task)+ await original_create(task, **kwargs)monkeypatch.setattr(store, "create", slow_create)tracker = TaskTracker(store=store)@@ -97,8 +99,8 @@ async def test_cancel_during_skill_task_creation_settles_persisted_record(monkeyawait asyncio.wait_for(adding, 1)record = await tracker.get("create-test", **owner)assert record.status == TaskStatus.FAILED- assert (await store.get("create-test", **owner))["status"] == "failed"- assert not tracker.has_work(record.task_id)+ assert (await store.get_task("create-test", **owner)).status is TaskStatus.FAILED+ assert not await tracker.has_work(record.task_id, **owner)service._skill_processor.process_skill.assert_not_awaited()finally:resume.set()@@ -108,17 +110,17 @@ async def test_cancel_during_skill_task_creation_settles_persisted_record(monkey@pytest.mark.asyncioasync def test_skill_success_remains_completed_when_queue_work_finishes_first(monkeypatch):tracker = TaskTracker(store=PersistentTaskStore(_FakeAgfs()))- index = TaskWorkIndex()- tracker.attach_work_index(index)monkeypatch.setattr("openviking.service.task_tracker.get_task_tracker", lambda: tracker)ctx = RequestContext(user=UserIdentifier.the_default_user(), role=Role.ROOT)owner = {"account_id": ctx.account_id, "user_id": ctx.user.user_id}async def process_skill(**kwargs):- work = QueueTaskMetadata(get_task_context().task_id, "vector", **owner)- assert index.register("Embedding", work)- index.record_result(work, "completed")- await index.prepare_ack("Embedding", work)+ task_id = get_task_context().task_id+ assert (+ await tracker.register_work(task_id, "vector", "Embedding", **owner)+ is WorkRegistration.CREATED+ )+ await tracker.mark_work_done(task_id, "vector", **owner)return {"uri": "viking://agent/skills/demo"}service = ResourceService.__new__(ResourceService)diff --git a/tests/service/test_task_processing_time.py b/tests/service/test_task_processing_time.pyindex b7d138884..b3991c60d 100644--- a/tests/service/test_task_processing_time.py+++ b/tests/service/test_task_processing_time.py@@ -7,58 +7,63 @@ from types import SimpleNamespaceimport pytest-from openviking.service.task_work_index import (- QueueTaskMetadata,- TaskWorkIndex,- bind_task_context,-)+from openviking.service.task_domain import WorkRecord, processing_seconds+from openviking.service.task_store import PersistentTaskStore+from openviking.service.task_tracker import TaskTracker+from openviking.service.task_work_index import bind_task_contextfrom openviking.storage.queuefs.semantic_executor import (SemanticTreeExecutor,SemanticTreeScheduler,)---def test_work_timestamps_union_worker_occupancy_and_exclude_queue_time(monkeypatch):- now = [0.0]- monkeypatch.setattr("openviking.service.task_work_index.time.time", lambda: now[0])- index = TaskWorkIndex()- index.init_processing("task")- first = QueueTaskMetadata("task", "first")- second = QueueTaskMetadata("task", "second")- assert index.register("Semantic", first)- assert index.processing_seconds("task") is None-- now[0] = 10 # Initial queue wait.- first_worker = object()- index.register_active("task", first_worker, work_id=first.work_id)- now[0] = 12- assert index.register("Embedding", second)- second_worker = object()- index.register_active("task", second_worker, work_id=second.work_id)- now[0] = 15- index.unregister_active("task", first_worker)- now[0] = 18- index.unregister_active("task", second_worker)- assert index.processing_seconds("task") == 8- now[0] = 40 # No worker: waiting for another queue.- assert index.processing_seconds("task") == 8- third_worker = object()- index.register_active("task", third_worker)- now[0] = 43- assert index.processing_seconds("task") == 11- index.unregister_active("task", third_worker)- now[0] = 100- assert index.processing_seconds("task") == 11+from tests.test_task_tracker import _FakeAgfs++OWNER = {"account_id": "account", "user_id": "user"}+++def test_work_timestamps_union_worker_occupancy_and_exclude_queue_time():+ works = [+ WorkRecord(+ work_id="first",+ task_id="task",+ queue_name="Semantic",+ started_at=10,+ finished_at=15,+ ),+ WorkRecord(+ work_id="second",+ task_id="task",+ queue_name="Embedding",+ started_at=12,+ finished_at=18,+ ),+ WorkRecord(+ work_id="third",+ task_id="task",+ queue_name="Inline",+ started_at=40,+ finished_at=43,+ ),+ ]++ assert processing_seconds(works, now=100) == 11+++async def _create_work(+ tracker: TaskTracker,+ task_id: str,+ work_id: str = "worker",+) -> None:+ task = await tracker.create("add_resource", task_id=task_id, **OWNER)+ await tracker.start(task.task_id, **OWNER)+ await tracker.register_work(task_id, work_id, "Semantic", **OWNER)+ await tracker.start_work(task_id, work_id, **OWNER)@pytest.mark.asyncioasync def test_saturated_dag_scheduler_counts_claimed_handler_time(monkeypatch):now = [0.0]- monkeypatch.setattr(- "openviking.service.task_work_index.time",- SimpleNamespace(time=lambda: now[0]),- )- index = TaskWorkIndex()+ monkeypatch.setattr("openviking.service.task_store.time.time", lambda: now[0])+ tracker = TaskTracker(PersistentTaskStore(_FakeAgfs()))scheduler = SemanticTreeScheduler(max_workers=1)monkeypatch.setattr("openviking.storage.queuefs.semantic_executor.get_semantic_tree_scheduler",@@ -70,7 +75,7 @@ async def test_saturated_dag_scheduler_counts_claimed_handler_time(monkeypatch):queued = asyncio.Event()executors = {}for name in ["a", "b"]:- index.init_processing(name)+ await _create_work(tracker, name)with bind_task_context(name, "account", "user"):executor = SemanticTreeExecutor(None, "resource", 1, None)executors[name] = executor@@ -90,30 +95,25 @@ async def test_saturated_dag_scheduler_counts_claimed_handler_time(monkeypatch):monkeypatch.setattr(executors["b"], "_schedule_dir", queue_b)async def coordinate(name):- worker = asyncio.current_task()- assert worker is not None- index.register_active(name, worker)try:await executors[name].run("viking://resources/" + name)finally:- index.unregister_active(name, worker)+ await tracker.mark_work_done(name, "worker", **OWNER)first = asyncio.create_task(coordinate("a"))await entered["a"].wait()second = asyncio.create_task(coordinate("b"))await queued.wait()now[0] = 20- assert index.processing_seconds("a") == 20- # B's QueueFS handler already owns a worker while its internal DAG node- # waits for the shared scheduler.- assert index.processing_seconds("b") == 20+ assert (await tracker.get("a", **OWNER)).processing_seconds == 20+ assert (await tracker.get("b", **OWNER)).processing_seconds == 20release["a"].set()await entered["b"].wait()now[0] = 25- assert index.processing_seconds("b") == 25+ assert (await tracker.get("b", **OWNER)).processing_seconds == 25release["b"].set()await asyncio.gather(first, second)- assert index.processing_seconds("b") == 25+ assert (await tracker.get("b", **OWNER)).processing_seconds == 25for worker in list(scheduler._workers):worker.cancel()await asyncio.gather(*scheduler._workers, return_exceptions=True)@@ -122,31 +122,23 @@ async def test_saturated_dag_scheduler_counts_claimed_handler_time(monkeypatch):@pytest.mark.asyncioasync def test_downstream_work_wait_remains_worker_occupied_time(monkeypatch):now = [0.0]- monkeypatch.setattr(- "openviking.service.task_work_index.time",- SimpleNamespace(time=lambda: now[0]),+ monkeypatch.setattr("openviking.service.task_store.time.time", lambda: now[0])+ tracker = TaskTracker(PersistentTaskStore(_FakeAgfs()))+ await _create_work(tracker, "task", "coordinator")+ await tracker.register_work("task", "embedding", "Embedding", **OWNER)++ now[0] = 3+ waiter = asyncio.create_task(+ tracker.wait_for_work("task", ["embedding"], poll_interval=0.001, **OWNER))- index = TaskWorkIndex()- index.init_processing("task")- metadata = QueueTaskMetadata("task", "embedding")- assert index.register("Embedding", metadata)- worker = asyncio.current_task()- assert worker is not None- index.register_active("task", worker)- try:- now[0] = 3- waiter = asyncio.create_task(- index.wait_for_work("task", ["embedding"], poll_interval=0.001)- )- await asyncio.sleep(0.01)- now[0] = 20- assert index.processing_seconds("task") == 20- index.record_result(metadata, "completed")- await waiter- now[0] = 22- assert index.processing_seconds("task") == 22- finally:- index.unregister_active("task", worker)+ await asyncio.sleep(0.01)+ now[0] = 20+ assert (await tracker.get("task", **OWNER)).processing_seconds == 20+ await tracker.mark_work_done("task", "embedding", **OWNER)+ await waiter+ now[0] = 22+ assert (await tracker.get("task", **OWNER)).processing_seconds == 22+ await tracker.mark_work_done("task", "coordinator", **OWNER)@pytest.mark.asyncio@@ -157,19 +149,16 @@ async def test_semantic_retry_cooldown_counts_full_worker_occupied_time(from openviking.storage.queuefs.semantic_processor import SemanticProcessornow = [0.0]- monkeypatch.setattr(- "openviking.service.task_work_index.time",- SimpleNamespace(time=lambda: now[0]),- )- index = TaskWorkIndex()- index.init_processing("task")+ monkeypatch.setattr("openviking.service.task_store.time.time", lambda: now[0])+ tracker = TaskTracker(PersistentTaskStore(_FakeAgfs()))+ await _create_work(tracker, "task")message = SimpleNamespace(uri="viking://resources/retry-test", account_id="account")enqueued = []async def sleep(delay):assert delay == 30now[0] += delay- assert index.processing_seconds("task") == 33+ assert (await tracker.get("task", **OWNER)).processing_seconds == 33async def enqueue(msg):assert msg is message@@ -190,15 +179,17 @@ async def test_semantic_retry_cooldown_counts_full_worker_occupied_time(processor = SimpleNamespace(_account_breaker=lambda _account_id: SimpleNamespace(retry_after=30))- worker = asyncio.current_task()- index.register_active("task", worker)- try:- now[0] = 3- await SemanticProcessor._reenqueue_semantic_msg(- processor, message, enqueue=custom if custom_enqueue else None- )- finally:- index.unregister_active("task", worker)-+ now[0] = 3+ await SemanticProcessor._reenqueue_semantic_msg(+ processor, message, enqueue=custom if custom_enqueue else None+ )assert enqueued == [message]- assert index.processing_seconds("task") == 35+ assert (await tracker.get("task", **OWNER)).processing_seconds == 35++ await tracker.mark_work_requeued("task", "worker", **OWNER)+ assert (await tracker.get("task", **OWNER)).processing_seconds is None+ now[0] = 50+ await tracker.start_work("task", "worker", **OWNER)+ now[0] = 55+ await tracker.mark_work_done("task", "worker", **OWNER)+ assert (await tracker.get("task", **OWNER)).processing_seconds == 5diff --git a/tests/service/test_task_queue_middleware.py b/tests/service/test_task_queue_middleware.pyindex aadc54583..3fdd400c4 100644--- a/tests/service/test_task_queue_middleware.py+++ b/tests/service/test_task_queue_middleware.py@@ -1,18 +1,19 @@# Copyright (c) 2026 Beijing Volcano Engine Technology Co., Ltd.# SPDX-License-Identifier: AGPL-3.0-"""Legacy task-index behavior through the middleware boundary."""+"""Task Store behavior through the QueueFS middleware boundary."""import asyncioimport json-import threadingfrom unittest.mock import AsyncMockimport pytestfrom openviking.pyagfs import AsyncAGFSClient+from openviking.service.task_domain import TaskStatus, WorkStatefrom openviking.service.task_queue_middleware import TaskWorkQueueMiddleware+from openviking.service.task_store import PersistentTaskStore+from openviking.service.task_tracker import TaskTracker, set_task_trackerfrom openviking.service.task_work_index import (- TaskWorkIndex,TaskWorkRejected,bind_task_context,extract_task_metadata,@@ -20,42 +21,51 @@ from openviking.service.task_work_index import ()from openviking.storage.queuefs.named_queue import DequeueHandlerBase, NamedQueuefrom openviking.storage.queuefs.process_result import ProcessOutcome, ProcessResult-from openviking.storage.queuefs.queue_middleware import QueueMiddleware-from openviking.storage.queuefs.session_commit_processor import SessionCommitProcessorfrom openviking.telemetry import OperationTelemetry, bind_telemetryfrom openviking.utils.log_correlation import log_correlation+from tests.test_task_tracker import _FakeAgfs++OWNER = {"account_id": "account", "user_id": "user"}@pytest.fixture-def tracked_queue(monkeypatch):+async def tracked_queue(monkeypatch):transport = AsyncMock(spec=AsyncAGFSClient)transport.write.return_value = "message-1"- # Tests override transport.read.return_value to feed /messages snapshots;- # status reads must still return the backend JSON shape.transport.read.return_value = b"0"async def read(path):if path.endswith("/status"):return b'{"pending":0,"processing":0}'- value = transport.read.return_value- return value+ return transport.read.return_valuetransport.read.side_effect = readmonkeypatch.setattr("openviking.storage.queuefs.named_queue.AsyncAGFSClient", lambda _client: transport)- index = TaskWorkIndex()- finalize = AsyncMock()- cancelled = set()- index.set_callbacks(- finalize_before_ack=finalize,- is_cancellation_requested=lambda task_id: task_id in cancelled,+ tracker = TaskTracker(PersistentTaskStore(_FakeAgfs()))+ set_task_tracker(tracker)+ queue = NamedQueue(+ object(),+ "/queue",+ "Embedding",+ middlewares=[TaskWorkQueueMiddleware()],)- queue = NamedQueue(object(), "/queue", "Test", middlewares=[TaskWorkQueueMiddleware(index)])- return queue, index, transport, finalize, cancelled+ try:+ yield queue, tracker, transport+ finally:+ set_task_tracker(None)++async def create_task(tracker: TaskTracker, task_id: str = "task-1") -> None:+ task = await tracker.get(task_id, **OWNER)+ if task is None:+ task = await tracker.create("add_resource", task_id=task_id, **OWNER)+ await tracker.start(task.task_id, **OWNER)-async def enqueue_task(queue, transport, task_id="task-1"):++async def enqueue_task(queue, tracker, transport, task_id="task-1"):+ await create_task(tracker, task_id)with bind_task_context(task_id, "account", "user"):await queue.enqueue({"value": 1})return {"id": "message-1", "data": transport.write.await_args.args[1].decode()}@@ -64,10 +74,7 @@ async def enqueue_task(queue, transport, task_id="task-1"):def test_log_correlation_uses_bound_task_and_telemetry() -> None:telemetry = OperationTelemetry(operation="add_resource_job", enabled=True)- with (- bind_task_context("task-123", "account", "user"),- bind_telemetry(telemetry),- ):+ with bind_task_context("task-123", "account", "user"), bind_telemetry(telemetry):assert log_correlation(message_id="semantic-456") == (f"task_id=task-123 telemetry_id={telemetry.telemetry_id} message_id=semantic-456")@@ -80,35 +87,11 @@ def test_log_correlation_accepts_explicit_queue_ids_without_context() -> None:)-async def test_enqueue_registers_before_write_and_ack_finalizes_before_delete(tracked_queue):- queue, index, transport, finalize, _ = tracked_queue-- async def write(path, data):- assert path.endswith("/enqueue")- assert index.has_work("task-1")- return "message-1"-- transport.write.side_effect = write- message = await enqueue_task(queue, transport)- metadata = extract_task_metadata(message)- assert metadata.task_id == "task-1"- assert metadata.account_id == "account"- assert metadata.user_id == "user"- assert get_task_context() is None-- async def ack_write(path, data):- assert path.endswith("/ack")- assert not index.has_work("task-1")- finalize.assert_awaited_once_with(metadata)- return ""-- transport.write.side_effect = ack_write- await queue.ack("message-1", message)- assert not index.has_work("task-1")--async def test_enqueue_keeps_content_owner_separate_from_task_owner(tracked_queue):- queue, _, transport, _, _ = tracked_queue+ queue, tracker, transport = tracked_queue+ task_owner = {"account_id": "account", "user_id": "task-owner"}+ task = await tracker.create("add_resource", task_id="task-1", **task_owner)+ await tracker.start(task.task_id, **task_owner)payload = {"task_id": "task-1","account_id": "account",@@ -134,270 +117,148 @@ async def test_enqueue_keeps_content_owner_separate_from_task_owner(tracked_queuassert persisted["_task_account_id"] == "account"assert persisted["_task_user_id"] == "task-owner"assert extract_task_metadata(persisted).user_id == "task-owner"+ metadata = extract_task_metadata(persisted)+ assert (+ await tracker.get_work_state(+ metadata.task_id,+ metadata.work_id,+ **task_owner,+ )+ is WorkState.PENDING+ )async def test_enqueue_does_not_stamp_task_owner_without_authoritative_context(tracked_queue):- queue, _, transport, _, _ = tracked_queue+ queue, _, transport = tracked_queue+ transport.write.reset_mock()with bind_task_context("task-1", "", ""):- await queue.enqueue(- {- "task_id": "task-1",- "account_id": "account",- "user_id": "content-owner",- }- )+ with pytest.raises(TaskWorkRejected, match="missing owner metadata"):+ await queue.enqueue(+ {+ "task_id": "task-1",+ "account_id": "account",+ "user_id": "content-owner",+ }+ )- persisted = json.loads(transport.write.await_args.args[1])- assert persisted["account_id"] == "account"- assert persisted["user_id"] == "content-owner"- assert "_task_account_id" not in persisted- assert "_task_user_id" not in persisted+ transport.write.assert_not_awaited()async def test_enqueue_failure_records_failed_work(tracked_queue):- queue, index, transport, finalize, _ = tracked_queue+ queue, tracker, transport = tracked_queue+ await create_task(tracker)transport.write.side_effect = OSError("write failed")enqueued_work = []- with bind_task_context(- "task-1",- "account",- "user",- enqueued_work=enqueued_work,++ with (+ bind_task_context("task-1", "account", "user", enqueued_work=enqueued_work),+ pytest.raises(OSError, match="write failed"),):- with pytest.raises(OSError, match="write failed"):- await queue.enqueue({})- assert not index.has_work("task-1")+ await queue.enqueue({})+payload = json.loads(transport.write.await_args.args[1])- assert enqueued_work == [("Test", payload["_task_work_id"])]- assert index.queue_status("task-1")["Test"] == {- "processed": 0,- "requeue_count": 0,- "error_count": 1,- "errors": [{"message": "write failed"}],- }- finalize.assert_awaited_once()+ assert enqueued_work == [("Embedding", payload["_task_work_id"])]+ status = await tracker.build_queue_status("task-1", **OWNER)+ assert status["Embedding"]["error_count"] == 1+ task = await tracker.get("task-1", **OWNER)+ assert task is not None and task.status is TaskStatus.FAILED+ assert task.error == "write failed"-async def test_cancelled_enqueue_discards_registered_work(tracked_queue):- queue, index, transport, finalize, _ = tracked_queue+async def test_cancelled_enqueue_discards_uncommitted_work(tracked_queue):+ queue, tracker, transport = tracked_queue+ await create_task(tracker)transport.write.side_effect = asyncio.CancelledError()- with bind_task_context("task-1", "account", "user"):- with pytest.raises(asyncio.CancelledError):- await queue.enqueue({})- assert not index.has_work("task-1")- assert index.queue_status("task-1") == {}- finalize.assert_awaited_once()--async def test_cancelled_task_rejects_enqueue_without_write(tracked_queue):- queue, index, transport, _, cancelled = tracked_queue- cancelled.add("task-1")with bind_task_context("task-1", "account", "user"):- with pytest.raises(TaskWorkRejected, match="cancelling"):- await queue.enqueue({})- transport.write.assert_not_awaited()- assert not index.has_work("task-1")--[email protected]("operation", ["ack", "clear"])[email protected]("failure", [OSError("write failed"), asyncio.CancelledError()])-async def test_failed_ack_or_clear_restores_work(tracked_queue, operation, failure):- queue, index, transport, finalize, _ = tracked_queue- message = await enqueue_task(queue, transport)- transport.read.return_value = json.dumps([message]).encode()- transport.write.reset_mock()- transport.write.side_effect = failure-- async def invoke():- if operation == "ack":- return await queue.ack("message-1", message)- return await queue.clear()-- if isinstance(failure, asyncio.CancelledError):with pytest.raises(asyncio.CancelledError):- await invoke()- elif operation == "clear":- assert await invoke() is False- else:- assert await invoke() is None- finalize.assert_awaited_once()- assert index.has_work("task-1")--[email protected]("operation", ["ack", "clear"])-async def test_finalization_failure_prevents_transport_delete(tracked_queue, operation):- queue, index, transport, finalize, _ = tracked_queue- message = await enqueue_task(queue, transport)- transport.read.return_value = json.dumps([message]).encode()- transport.write.reset_mock()- finalize.side_effect = OSError("task store unavailable")- if operation == "ack":- await queue.ack("message-1", message)- else:- assert await queue.clear() is False- transport.write.assert_not_awaited()- assert index.has_work("task-1")--[email protected]("operation", ["enqueue", "ack", "clear"])-async def test_inner_short_circuit_compensates_uncommitted_index_change(tracked_queue, operation):- queue, index, transport, _, _ = tracked_queue-- class ShortCircuit(QueueMiddleware):- async def invoke(self, ctx, call_next):- return "deduplicated"-- setattr(ShortCircuit, operation, ShortCircuit.invoke)- queue = NamedQueue(- object(), "/queue", "Test", middlewares=[TaskWorkQueueMiddleware(index), ShortCircuit()]- )- message = await enqueue_task(queue, transport) if operation != "enqueue" else None- transport.write.reset_mock()- if operation == "enqueue":- with bind_task_context("task-1", "account", "user"):- assert await queue.enqueue({}) == "deduplicated"- assert not index.has_work("task-1")- elif operation == "ack":- await queue.ack("message-1", message)- assert index.has_work("task-1")- else:- transport.read.return_value = json.dumps([message]).encode()- await queue.clear()- assert index.has_work("task-1")- transport.write.assert_not_awaited()--[email protected]("operation", ["enqueue", "ack", "clear"])-async def test_post_commit_error_does_not_undo_index_change(tracked_queue, operation):- queue, index, transport, _, _ = tracked_queue-- class FailAfter(QueueMiddleware):- async def invoke(self, ctx, call_next):- await call_next(ctx)- raise RuntimeError("after commit")+ await queue.enqueue({})- setattr(FailAfter, operation, FailAfter.invoke)- queue = NamedQueue(- object(), "/queue", "Test", middlewares=[TaskWorkQueueMiddleware(index), FailAfter()]- )- if operation == "enqueue":- with bind_task_context("task-1", "account", "user"):- with pytest.raises(RuntimeError, match="after commit"):- await queue.enqueue({})- assert index.has_work("task-1")- else:- message = await enqueue_task(queue, transport)- transport.read.return_value = json.dumps([message]).encode()- if operation == "ack":- await queue.ack("message-1", message)- else:- assert await queue.clear() is False- assert not index.has_work("task-1")---async def test_clear_rolls_back_all_prepared_messages(tracked_queue):- queue, index, transport, finalize, _ = tracked_queue- first = await enqueue_task(queue, transport, "first")- second = await enqueue_task(queue, transport, "second")- transport.read.return_value = json.dumps([first, second]).encode()-- async def finish(metadata):- if metadata.task_id == "second":- raise OSError("second task failed")-- finalize.side_effect = finish- transport.write.reset_mock()- assert await queue.clear() is False- transport.write.assert_not_awaited()- assert index.has_work("first") and index.has_work("second")- finalize.side_effect = None- assert await queue.clear()- assert not index.has_work("first") and not index.has_work("second")+ assert not await tracker.has_work("task-1", **OWNER)+ task = await tracker.get("task-1", **OWNER)+ assert task is not None and task.status is TaskStatus.FAILED+ assert task.error == "Task operation interrupted before its result was persisted"-async def test_process_result_tracks_children_and_errors(tracked_queue):- queue, index, transport, _, _ = tracked_queue- message = await enqueue_task(queue, transport)- contexts = []- correlations = []+async def test_successful_process_records_context_and_vector_count(tracked_queue):+ queue, tracker, transport = tracked_queue+ message = await enqueue_task(queue, tracker, transport)+ metadata = extract_task_metadata(message)+ seen = []class Handler(DequeueHandlerBase):async def on_dequeue(self, data):- with bind_task_context("task-1", "account", "user"):- contexts.append(get_task_context())- correlations.append(log_correlation(telemetry_id="tm-1", message_id="semantic-1"))- await queue.enqueue({"child": True})- return ProcessResult.failed("failed work")+ seen.append(get_task_context())+ return ProcessResult.success({"vector_written": True})+ await tracker.complete("task-1", {"done": True}, **OWNER)queue.set_dequeue_handler(Handler())- assert (await queue.process_dequeued(message)).outcome is ProcessOutcome.FAILED- assert contexts[0].task_id == "task-1"- assert contexts[0].work_id == extract_task_metadata(message).work_id- assert correlations == ["task_id=task-1 telemetry_id=tm-1 message_id=semantic-1"]+ result = await queue.process_dequeued(message)++ assert result.outcome is ProcessOutcome.SUCCESS+ assert seen[0].task_id == "task-1"+ assert seen[0].work_id == metadata.work_idassert get_task_context() is None- assert index.failure("task-1") == "failed work"- child_payload = next(- call.args[1]- for call in reversed(transport.write.await_args_list)- if call.args[0].endswith("/enqueue")- )- child = {"id": "child", "data": child_payload.decode()}- assert index.has_work("task-1")- await queue.ack("message-1", message)- assert index.has_work("task-1")- await queue.ack("child", child)- assert not index.has_work("task-1")- status = await queue.get_status()- assert status.error_count == 1- assert status.in_progress == 0+ assert await tracker.get_context_count("task-1", **OWNER) == 1+ task = await tracker.get("task-1", **OWNER)+ assert task is not None and task.status is TaskStatus.COMPLETED+ await queue.process_dequeued(message)+ assert len(seen) == 1+ assert await tracker.get_context_count("task-1", **OWNER) == 1+ assert (await tracker.build_queue_status("task-1", **OWNER))["Embedding"]["processed"] == 1-async def test_handler_exception_records_task_failure_and_propagates(tracked_queue):- queue, index, transport, _, _ = tracked_queue- message = await enqueue_task(queue, transport)+async def test_requeue_keeps_work_open_for_redelivery(tracked_queue):+ queue, tracker, transport = tracked_queue+ message = await enqueue_task(queue, tracker, transport)+ outcomes = [+ ProcessResult.requeued(),+ ProcessResult.failed("combined", errors=["first", "second"]),+ ]class Handler(DequeueHandlerBase):async def on_dequeue(self, data):- raise RuntimeError("handler failed")+ return outcomes.pop(0)queue.set_dequeue_handler(Handler())- with pytest.raises(RuntimeError, match="handler failed"):- await queue.process_dequeued(message)- assert index.failure("task-1") == "handler failed"- assert index.has_work("task-1")+ first = await queue.process_dequeued(message)+ metadata = extract_task_metadata(message)+ assert first.outcome is ProcessOutcome.REQUEUED+ assert await tracker.get_work_state("task-1", metadata.work_id, **OWNER) is WorkState.PENDING+ assert await tracker.has_work("task-1", **OWNER)++ await tracker.complete("task-1", {"done": True}, **OWNER)+ second = await queue.process_dequeued(message)+ assert second.outcome is ProcessOutcome.FAILED+ assert not await tracker.has_work("task-1", **OWNER)+ task = await tracker.get("task-1", **OWNER)+ assert task.status is TaskStatus.FAILED+ assert task.error == "combined"+ assert task.result["queue_status"]["Embedding"] == {+ "processed": 0,+ "requeue_count": 1,+ "error_count": 2,+ "errors": [{"message": "first"}, {"message": "second"}],+ }-async def test_concurrent_results_update_each_task(tracked_queue):- queue, index, transport, _, _ = tracked_queue- first = await enqueue_task(queue, transport, "first")- second = await enqueue_task(queue, transport, "second")- both_started = asyncio.Event()- started = 0+async def test_clear_settles_committed_work(tracked_queue):+ queue, tracker, transport = tracked_queue+ message = await enqueue_task(queue, tracker, transport)+ transport.read.return_value = json.dumps([message]).encode()+ transport.write.reset_mock()+ transport.write.return_value = ""- class Handler(DequeueHandlerBase):- async def on_dequeue(self, data):- nonlocal started- started += 1- if started == 2:- both_started.set()- await both_started.wait()- task_id = get_task_context().task_id- return ProcessResult.failed(f"{task_id} failed")+ assert await queue.clear() is True- queue.set_dequeue_handler(Handler())- await asyncio.wait_for(- asyncio.gather(queue.process_dequeued(first), queue.process_dequeued(second)),- timeout=1,- )- assert index.failure("first") == "first failed"- assert index.failure("second") == "second failed"+ transport.write.assert_awaited_once_with("/queue/Embedding/clear", b"")+ assert not await tracker.has_work("task-1", **OWNER)-async def test_cancel_before_handler_uses_legacy_discard_callback(tracked_queue):- queue, index, transport, _, cancelled = tracked_queue- message = await enqueue_task(queue, transport)- cancelled.add("task-1")+async def test_cancelled_task_uses_handler_cancellation_path(tracked_queue):+ queue, tracker, transport = tracked_queue+ message = await enqueue_task(queue, tracker, transport)calls = []class Handler(DequeueHandlerBase):@@ -406,186 +267,13 @@ async def test_cancel_before_handler_uses_legacy_discard_callback(tracked_queue)async def on_cancelled(self, data):calls.append(data)- return await super().on_cancelled(data)-- queue.set_dequeue_handler(Handler())- assert (await queue.process_dequeued(message)).outcome is ProcessOutcome.CANCELLED- assert calls == [message]- await queue.ack("message-1", message)- assert not index.has_work("task-1")- status = await queue.get_status()- assert status.processed == 1 and status.in_progress == 0--[email protected]("user_cancel", [True, False])-async def test_active_cancellation_distinguishes_user_from_shutdown(tracked_queue, user_cancel):- queue, index, transport, _, cancelled = tracked_queue- message = await enqueue_task(queue, transport)- metadata = extract_task_metadata(message)- started = asyncio.Event()-- class Handler(DequeueHandlerBase):- async def on_dequeue(self, data):- started.set()- await asyncio.Event().wait()-- queue.set_dequeue_handler(Handler())- running = asyncio.create_task(queue.process_dequeued(message))- await asyncio.wait_for(started.wait(), timeout=1)- try:- if user_cancel:- cancelled.add("task-1")- index.cancel_active("task-1")- result = await asyncio.wait_for(running, timeout=1)- assert result.outcome is ProcessOutcome.CANCELLED- await index.wait_for_work("task-1", [metadata.work_id], timeout=0.1)- await queue.ack("message-1", message)- assert not index.has_work("task-1")- assert (await queue.get_status()).processed == 1- else:- running.cancel()- with pytest.raises(asyncio.CancelledError):- await running- assert index.has_work("task-1")- finally:- if not running.done():- running.cancel()- await asyncio.gather(running, return_exceptions=True)---async def test_shared_middleware_keeps_concurrent_contexts_isolated(tracked_queue):- queue, index, transport, _, _ = tracked_queue- first = await enqueue_task(queue, transport, "first")- second = await enqueue_task(queue, transport, "second")- both_started = asyncio.Event()- seen = []-- class Handler(DequeueHandlerBase):- async def on_dequeue(self, data):- before = get_task_context().task_id- seen.append(before)- if len(seen) == 2:- both_started.set()- await both_started.wait()- assert get_task_context().task_id == before- return ProcessResult.success(data)-- queue.set_dequeue_handler(Handler())- await asyncio.wait_for(- asyncio.gather(queue.process_dequeued(first), queue.process_dequeued(second)), timeout=1- )- assert set(seen) == {"first", "second"}- assert get_task_context() is None- await queue.ack("first", first)- await queue.ack("second", second)- assert not index.has_work("first") and not index.has_work("second")---async def test_awaited_child_result_is_recorded_before_ack(tracked_queue):- queue, index, transport, finalize, _ = tracked_queue- message = await enqueue_task(queue, transport)-- class Handler(DequeueHandlerBase):- async def on_dequeue(self, data):- async def child():- await asyncio.sleep(0)- return ProcessResult.failed("child failure")-- return await asyncio.create_task(child())-- async def finish(metadata):- assert index.failure(metadata.task_id) == "child failure"-- finalize.side_effect = finish- queue.set_dequeue_handler(Handler())- await queue.process_dequeued(message)- finalize.assert_not_awaited()- await queue.ack("message-1", message)- finalize.assert_awaited_once()---async def test_cancel_cleanup_interruption_does_not_ack(tracked_queue):- queue, index, transport, _, cancelled = tracked_queue- message = await enqueue_task(queue, transport)- cancelled.add("task-1")- transport.write.reset_mock()-- class Handler(DequeueHandlerBase):- async def on_dequeue(self, data):- pytest.fail("cancelled work must not execute")-- async def on_cancelled(self, data):- raise asyncio.CancelledError+ return ProcessResult.cancelled()queue.set_dequeue_handler(Handler())- with pytest.raises(asyncio.CancelledError):- await queue.process_dequeued(message)- assert index.has_work("task-1")- transport.write.assert_not_awaited()---async def test_cross_loop_cancellation_waits_for_handler_cleanup(tracked_queue):- queue, index, transport, _, cancelled = tracked_queue- started = threading.Event()- cleanup_started = threading.Event()- cleanup_release = threading.Event()- request_loop = asyncio.get_running_loop()-- class Session:- async def exists(self):- return True-- async def load(self):- pass-- async def resume_queued_commit(self, msg):- assert asyncio.get_running_loop() is not request_loop- started.set()- try:- await asyncio.Event().wait()- finally:- cleanup_started.set()- await asyncio.to_thread(cleanup_release.wait)-- class Service:- def session(self, ctx, session_id, session_uri=None):- return Session()-- from openviking.storage.queuefs.session_commit_msg import SessionCommitMsg- from openviking_cli.session.user_id import UserIdentifier-- msg = SessionCommitMsg(- task_id="task-1",- session_id="session-1",- session_uri="viking://session/session-1",- archive_uri="viking://session/session-1/history/archive-1",- user=UserIdentifier("account", "user").to_dict(),- )- with bind_task_context("task-1", "account", "user"):- await queue.enqueue(msg.to_dict())- message = {"id": "m", "data": transport.write.await_args.args[1].decode()}- transport.write.reset_mock()- queue.set_dequeue_handler(SessionCommitProcessor(Service()))-- def consume():- return asyncio.run(queue.process_dequeued(message))+ await tracker.cancel("task-1", **OWNER)+ result = await queue.process_dequeued(message)- worker = asyncio.create_task(asyncio.to_thread(consume))- try:- assert await asyncio.to_thread(started.wait, 3)- cancelled.add("task-1")- index.cancel_active("task-1")- assert await asyncio.to_thread(cleanup_started.wait, 3)- assert not worker.done()- transport.write.assert_not_awaited()- finally:- cleanup_release.set()- if not cleanup_started.is_set():- cancelled.add("task-1")- index.cancel_active("task-1")- result = await asyncio.wait_for(worker, timeout=3)assert result.outcome is ProcessOutcome.CANCELLED- transport.write.assert_not_awaited()- await queue.ack("m", message)- assert not index.has_work("task-1")- transport.write.assert_awaited_once_with("/queue/Test/ack", b"m")+ assert calls == [message]+ task = await tracker.get("task-1", **OWNER)+ assert task is not None and task.status is TaskStatus.CANCELLEDdiff --git a/tests/service/test_task_work_contract.py b/tests/service/test_task_work_contract.pyindex 5bfc8dd9f..3dd66266e 100644--- a/tests/service/test_task_work_contract.py+++ b/tests/service/test_task_work_contract.py@@ -1,163 +1,163 @@# Copyright (c) 2026 Beijing Volcano Engine Technology Co., Ltd.# SPDX-License-Identifier: AGPL-3.0+"""Observable Task/Work lifecycle and persistence contracts."""import asyncio+from unittest.mock import Mock, create_autospecimport pytest-from openviking.service.task_store import PersistentTaskStore-from openviking.service.task_tracker import TaskStatus, TaskTracker-from openviking.service.task_work_index import QueueTaskMetadata, TaskWorkIndex+from openviking.service.task_domain import TaskStatus, WorkState+from openviking.service.task_store import PersistentTaskStore, WorkRegistration+from openviking.service.task_tracker import TaskTrackerfrom tests.test_task_tracker import _FakeAgfs+OWNER = {"account_id": "account", "user_id": "user"}+++async def tracker():+ tracker = TaskTracker(PersistentTaskStore(_FakeAgfs()))+ await tracker.create("content_write", task_id="task", **OWNER)+ await tracker.start("task", **OWNER)+ return tracker+@pytest.mark.asyncio-async def test_wait_for_work_observes_only_explicit_ids():- index = TaskWorkIndex()- first = QueueTaskMetadata("task", "first")- unrelated = QueueTaskMetadata("task", "unrelated")- assert index.register("Semantic", first)- assert index.register("Embedding", unrelated)+async def test_wait_for_work_observes_only_explicit_ids(tracker):+ for work_id in ("first", "unrelated"):+ await tracker.register_work("task", work_id, "Semantic", **OWNER)- index.record_result(first, "completed")- await index.wait_for_work("task", ["first"], timeout=0)+ await tracker.mark_work_done("task", "first", **OWNER)+ await tracker.wait_for_work("task", ["first"], timeout=0, **OWNER)- assert index.has_work("task")+ assert await tracker.has_work("task", **OWNER)@pytest.mark.asyncio[email protected]("recovered", [False, True])-async def test_task_work_wait_includes_recovered_and_later_enqueued_work(recovered):- index = TaskWorkIndex()- root = QueueTaskMetadata("task", "root")- semantic = QueueTaskMetadata("task", "semantic")- embedding = QueueTaskMetadata("task", "embedding")- if recovered:- index.rebuild(- {- "Root": [{"task_id": "task", "_task_work_id": "root"}],- "Semantic": [{"task_id": "task", "_task_work_id": "semantic"}],- }- )- else:- assert index.register("Root", root)- assert index.register("Semantic", semantic)+async def test_task_work_wait_includes_recovered_and_later_enqueued_work(tracker):+ await tracker.register_work("task", "root", "Inline", **OWNER)+ await tracker.reconcile_delivery("task", "semantic", "Semantic", **OWNER)waiting = asyncio.create_task(- index.wait_for_task_work("task", exclude_work_id="root", poll_interval=0.001)+ tracker.wait_for_task_work("task", exclude_work_id="root", poll_interval=0.001, **OWNER))try:await asyncio.sleep(0)assert not waiting.done()- assert index.register("Embedding", embedding)- index.record_result(semantic, "completed")- await index.prepare_ack("Semantic", semantic)+ await tracker.register_work("task", "embedding", "Embedding", **OWNER)+ await tracker.mark_work_done("task", "semantic", **OWNER)with pytest.raises(TimeoutError):- await index.wait_for_task_work("task", exclude_work_id="root", timeout=0)- index.record_result(embedding, "completed")- await index.prepare_ack("Embedding", embedding)+ await tracker.wait_for_task_work("task", exclude_work_id="root", timeout=0, **OWNER)+ await tracker.mark_work_done("task", "embedding", **OWNER)await asyncio.wait_for(waiting, timeout=1)- assert index.has_work("task")+ assert await tracker.has_work("task", **OWNER)finally:waiting.cancel()await asyncio.gather(waiting, return_exceptions=True)@pytest.mark.asyncio-async def test_requeue_keeps_logical_work_until_replacement_ack():- index = TaskWorkIndex()- work = QueueTaskMetadata("task", "work")- assert index.register("Semantic", work)- assert index.register("Semantic", work)-- index.record_result(work, "requeued")- await index.prepare_ack("Semantic", work)- assert index.has_work("task")-- index.record_result(work, "completed")- await index.wait_for_work("task", ["work"], timeout=0.1)- assert index.queue_status("task")["Semantic"] == {+async def test_requeue_keeps_logical_work_until_completion(tracker):+ assert await tracker.register_work("task", "work", "Semantic", **OWNER)+ assert (+ await tracker.register_work("task", "work", "Semantic", **OWNER)+ is WorkRegistration.EXISTING+ )+ await tracker.complete("task", {"done": True}, **OWNER)+ await tracker.start_work("task", "work", **OWNER)+ await tracker.mark_work_requeued("task", "work", **OWNER)+ assert (await tracker.get("task", **OWNER)).status is TaskStatus.RUNNING++ await tracker.start_work("task", "work", **OWNER)+ await tracker.mark_work_done("task", "work", **OWNER)+ completed = await tracker.wait("task", timeout=0, **OWNER)+ assert completed.status is TaskStatus.COMPLETED+ assert completed.result["queue_status"]["Semantic"] == {"processed": 1,"requeue_count": 1,"error_count": 0,"errors": [],}+ assert (+ await tracker.register_work("task", "late", "Semantic", **OWNER)+ is WorkRegistration.REJECTED+ )- await index.prepare_ack("Semantic", work)- assert not index.has_work("task")--[email protected]("errors", [None, ["first", "second"]])-def test_terminal_work_result_is_monotonic_and_idempotent(errors):- index = TaskWorkIndex()- work = QueueTaskMetadata("task", "work")- assert index.register("Embedding", work)- messages = errors or ["first"]- error = "\n".join(messages)-- index.record_result(work, "failed", error=error, errors=errors)- index.record_result(work, "completed")- index.record_result(work, "failed", error="later failure")- assert index.failure("task") == error- assert index.queue_status("task")["Embedding"] == {+async def test_terminal_work_result_is_monotonic_and_bounded():+ agfs = _FakeAgfs()+ tracker = TaskTracker(PersistentTaskStore(agfs))+ await tracker.create("content_write", task_id="task", **OWNER)+ await tracker.register_work("task", "work", "Embedding", **OWNER)+ await tracker.complete("task", {"done": True}, **OWNER)+ errors = [f"failure {index}" for index in range(150)]++ await tracker.mark_work_failed("task", "work", errors[0], errors=errors, **OWNER)+ await tracker.mark_work_done("task", "work", context_count_delta=3, **OWNER)+ await tracker.mark_work_failed("task", "work", "later failure", **OWNER)++ restored = await TaskTracker(PersistentTaskStore(agfs)).get("task", **OWNER)+ assert restored.status is TaskStatus.FAILED+ assert restored.error == errors[0]+ assert restored.result["queue_status"]["Embedding"] == {"processed": 0,"requeue_count": 0,- "error_count": len(messages),- "errors": [{"message": message} for message in messages],+ "error_count": 150,+ "errors": [{"message": error} for error in errors[:100]],}+ assert await tracker.get_context_count("task", **OWNER) == 0@pytest.mark.asyncio-async def test_enqueue_failure_finalizes_task_with_same_error():- tracker = TaskTracker(PersistentTaskStore(_FakeAgfs()))- index = TaskWorkIndex()- tracker.attach_work_index(index)- task = await tracker.create("content_write", account_id="account", user_id="user")- await tracker.start(task.task_id, account_id="account", user_id="user")- work = QueueTaskMetadata(task.task_id, "embedding", "account", "user")- assert index.register("Embedding", work)-- await index.fail_enqueue("Embedding", work, "embedding enqueue returned false")-- failed = await tracker.get(task.task_id, account_id="account", user_id="user")- assert failed.status is TaskStatus.FAILED- assert failed.error == "embedding enqueue returned false"+async def test_finalization_write_failure_keeps_work_retryable(monkeypatch):+ agfs = _FakeAgfs()+ tracker = TaskTracker(PersistentTaskStore(agfs))+ await tracker.create("content_write", task_id="task", **OWNER)+ await tracker.start("task", **OWNER)+ await tracker.register_work("task", "work", "Embedding", **OWNER)+ await tracker.complete("task", {"done": True}, **OWNER)+ write = agfs.write+ monkeypatch.setattr(agfs, "write", Mock(side_effect=OSError("write failed")))++ with pytest.raises(OSError, match="write failed"):+ await tracker.mark_work_done("task", "work", vector_written=True, **OWNER)++ assert (await tracker.get("task", **OWNER)).status is TaskStatus.RUNNING+ assert await tracker.get_work_state("task", "work", **OWNER) is WorkState.PENDING+ restored = await TaskTracker(PersistentTaskStore(agfs)).get("task", **OWNER)+ assert restored.status is TaskStatus.RUNNING++ monkeypatch.setattr(agfs, "write", write)+ await tracker.mark_work_done("task", "work", vector_written=True, **OWNER)+ assert (await tracker.wait("task", timeout=0, **OWNER)).status is TaskStatus.COMPLETED+ assert await tracker.get_context_count("task", **OWNER) == 1@pytest.mark.asyncio-async def test_deleting_task_releases_retained_work_status():- tracker = TaskTracker(PersistentTaskStore(_FakeAgfs()))- index = TaskWorkIndex()- tracker.attach_work_index(index)- owner = {"account_id": "account", "user_id": "user"}- task = await tracker.create("content_write", **owner)- await tracker.start(task.task_id, **owner)- work = QueueTaskMetadata(task.task_id, "semantic", "account", "user")- assert index.register("Semantic", work)-- await tracker.complete(- task.task_id,- {"done": True, "queue_status": tracker.build_queue_status(task.task_id)},- **owner,- )- index.record_result(work, "completed")- await index.prepare_ack("Semantic", work)- assert tracker.build_queue_status(task.task_id)["Semantic"]["processed"] == 1- completed = await tracker.get(task.task_id, **owner)- assert completed.result["queue_status"]["Semantic"]["processed"] == 1-- assert await tracker.delete_user_tasks(**owner) == 1- assert tracker.build_queue_status(task.task_id) == {- "Semantic": {- "processed": 0,- "requeue_count": 0,- "error_count": 0,- "errors": [],- },- "Embedding": {- "processed": 0,- "requeue_count": 0,- "error_count": 0,- "errors": [],- },- }+async def test_deleting_task_releases_retained_work_status(monkeypatch):+ agfs = _FakeAgfs()+ tracker = TaskTracker(PersistentTaskStore(agfs))+ foreign = {"account_id": "other", "user_id": "user"}+ for task_id in ("first", "second"):+ await tracker.create("content_write", task_id=task_id, **OWNER)+ await tracker.register_work(task_id, "work", "Semantic", **OWNER)+ await tracker.complete(task_id, {"done": True}, **OWNER)+ await tracker.create("content_write", task_id="foreign", **foreign)+ await tracker.complete("foreign", {"done": True}, **foreign)++ with pytest.raises(RuntimeError, match="Cannot delete active task"):+ await tracker.delete_tasks(**OWNER)+ for task_id in ("first", "second"):+ assert await tracker.get(task_id, **OWNER) is not None+ await tracker.mark_work_done(task_id, "work", **OWNER)++ read = create_autospec(agfs.read, side_effect=agfs.read)+ monkeypatch.setattr(agfs, "read", read)+ assert await tracker.delete_tasks(**OWNER) == 2+ # Cloud storage must read each owned task file only once for deletion.+ assert read.call_count == 2+ assert not await tracker.has_work("first", **OWNER)+ assert (await tracker.build_queue_status("first", **OWNER))["Semantic"]["processed"] == 0+ assert await tracker.get("second", **OWNER) is None+ assert await tracker.get("foreign", **foreign) is not Nonediff --git a/tests/session/conftest.py b/tests/session/conftest.pyindex 7e8eef5c7..88ae682a5 100644--- a/tests/session/conftest.py+++ b/tests/session/conftest.py@@ -32,10 +32,13 @@ async def client(tracker = get_task_tracker()original_has_work = tracker.has_work+ async def has_work(task_id, **kwargs):+ return task_id in queued_commit_tasks or await original_has_work(task_id, **kwargs)+monkeypatch.setattr(tracker,"has_work",- lambda task_id: task_id in queued_commit_tasks or original_has_work(task_id),+ has_work,)async def enqueue_with_session_commit_fallback(queue_name, data):diff --git a/tests/session/memory/test_consolidation_compile.py b/tests/session/memory/test_consolidation_compile.pyindex 085733b1c..13365cf1a 100644--- a/tests/session/memory/test_consolidation_compile.py+++ b/tests/session/memory/test_consolidation_compile.py@@ -16,6 +16,7 @@ from openviking.service.memory_compile import (_memory_type_from_target,_peer_id_from_memory_uri,)+from openviking.service.task_store import PersistentTaskStorefrom openviking.service.task_tracker import TaskStatus, TaskTrackerfrom openviking.service.task_work_index import get_task_contextfrom openviking.session.memory.consolidation_context_provider import (@@ -39,6 +40,7 @@ from openviking.session.memory.merge_op import FieldType, MergeOpfrom openviking.session.memory.utils.memory_file_utils import MemoryFileUtilsfrom openviking_cli.exceptions import InvalidArgumentError, NotFoundErrorfrom openviking_cli.session.user_id import UserIdentifier+from tests.test_task_tracker import _FakeAgfsdef _ctx(user_id: str = "u1") -> RequestContext:@@ -168,14 +170,7 @@ async def test_root_compile_creates_one_task_and_one_loop(monkeypatch, language_language_config.output_language_override = "en"vlm = SimpleNamespace(model="test", get_completion_async=AsyncMock(return_value="sdk.commit()"))language_config.vlm = SimpleNamespace(get_vlm_instance=lambda: vlm)- store = SimpleNamespace(- create=AsyncMock(),- update=AsyncMock(),- get=AsyncMock(return_value=None),- list=AsyncMock(return_value=[]),- delete=AsyncMock(),- )- tracker = TaskTracker(store)+ tracker = TaskTracker(PersistentTaskStore(_FakeAgfs()))monkeypatch.setattr("openviking.service.memory_compile.get_task_tracker", lambda: tracker)uris = [f"{target}/entities/person/alice.md",@@ -216,7 +211,6 @@ async def test_root_compile_creates_one_task_and_one_loop(monkeypatch, language_await asyncio.gather(*list(runner._running))final = await tracker.get(accepted.task_id, account_id="acc", user_id="u1")- store.create.assert_awaited_once()vlm.get_completion_async.assert_awaited_once()fs.glob.assert_awaited_once()assert fs.glob.await_args.kwargs["uri"] == target@@ -235,7 +229,11 @@ async def test_root_compile_creates_one_task_and_one_loop(monkeypatch, language_for name in ("entities", "preferences", "profile"):assert name in promptassert "create_events" not in prompt- assert not tracker.has_work(accepted.task_id)+ assert not await tracker.has_work(+ accepted.task_id,+ account_id="acc",+ user_id="u1",+ )@pytest.mark.asyncio@@ -736,14 +734,7 @@ async def test_compile_task_apply_outcomes(language_config.output_language_override = "en"language_config.vlm = SimpleNamespace(get_vlm_instance=lambda: SimpleNamespace(model="test"))ctx = _ctx()- store = SimpleNamespace(- create=AsyncMock(),- update=AsyncMock(),- get=AsyncMock(return_value=None),- list=AsyncMock(return_value=[]),- delete=AsyncMock(),- )- tracker = TaskTracker(store)+ tracker = TaskTracker(PersistentTaskStore(_FakeAgfs()))task = await tracker.create("compile", account_id=ctx.account_id, user_id=ctx.user.user_id)monkeypatch.setattr("openviking.service.memory_compile.get_task_tracker", lambda: tracker)monkeypatch.setattr(@@ -803,7 +794,11 @@ async def test_compile_task_apply_outcomes(assert final.error.startswith("MEMORY_CONSOLIDATION_FAILED:")else:assert final.error is None- assert not tracker.has_work(task.task_id)+ assert not await tracker.has_work(+ task.task_id,+ account_id=ctx.account_id,+ user_id=ctx.user.user_id,+ )if has_operations:apply.assert_awaited_once()viking_fs._async_agfs.pathlock_release.assert_awaited_once_with(lease)@@ -821,14 +816,7 @@ async def test_compile_task_apply_outcomes(@pytest.mark.asyncio@pytest.mark.parametrize("memory_type", [None, "entities"])async def test_cancel_running_compile_interrupts_before_memory_write(monkeypatch, memory_type):- store = SimpleNamespace(- create=AsyncMock(),- update=AsyncMock(),- get=AsyncMock(return_value=None),- list=AsyncMock(return_value=[]),- delete=AsyncMock(),- )- tracker = TaskTracker(store)+ tracker = TaskTracker(PersistentTaskStore(_FakeAgfs()))ctx = _ctx()task = await tracker.create("compile",@@ -869,7 +857,11 @@ async def test_cancel_running_compile_interrupts_before_memory_write(monkeypatchfinal = await tracker.get(task.task_id, account_id=ctx.account_id, user_id=ctx.user.user_id)assert final.status == TaskStatus.CANCELLED- assert not tracker.has_work(task.task_id)+ assert not await tracker.has_work(+ task.task_id,+ account_id=ctx.account_id,+ user_id=ctx.user.user_id,+ )memory_write.assert_not_awaited()diff --git a/tests/session/test_session_context.py b/tests/session/test_session_context.pyindex 866c57bc7..efc472543 100644--- a/tests/session/test_session_context.py+++ b/tests/session/test_session_context.py@@ -51,10 +51,14 @@ async def client(queued_commit_tasks: set[str] = set()tracker = get_task_tracker()original_has_work = tracker.has_work++ async def has_work(task_id, **kwargs):+ return task_id in queued_commit_tasks or await original_has_work(task_id, **kwargs)+monkeypatch.setattr(tracker,"has_work",- lambda task_id: task_id in queued_commit_tasks or original_has_work(task_id),+ has_work,)async def enqueue_with_session_commit_fallback(queue_name, data):diff --git a/tests/storage/test_collection_schemas.py b/tests/storage/test_collection_schemas.pyindex da75841dd..5cc66f653 100644--- a/tests/storage/test_collection_schemas.py+++ b/tests/storage/test_collection_schemas.py@@ -1259,11 +1259,7 @@ async def test_embedding_handler_preserves_parent_uri_for_backend_upsert_logic(massert result.outcome is ProcessOutcome.SUCCESSassert result.error is None- assert result.value == {- **queue_data["payload"]["context_data"],- "id": vector_record_id("default", "viking://resources/sample", 2),- "vector": [0.1, 0.2],- }+ assert result.value == {"vector_written": True}assert "data" in capturedassert captured["data"]["parent_uri"] == "viking://resources"@@ -1324,17 +1320,8 @@ async def test_embedding_handler_honors_existing_record_id_override(monkeypatch)@pytest.mark.asyncio-async def test_embedding_handler_generates_id_and_counts_written_context(monkeypatch):- from openviking.service.task_tracker import TaskTracker- from openviking.service.task_work_index import QueueTaskMetadata, TaskWorkIndex- from openviking.storage.vector_ids import vector_record_id-+async def test_embedding_handler_generates_id_and_reports_written_context(monkeypatch):captured = {}- tracker = TaskTracker(AsyncMock())- index = TaskWorkIndex()- tracker.attach_work_index(index)- index.register("Embedding", QueueTaskMetadata("task-1", "work-1"))- monkeypatch.setattr("openviking.storage.collection_schemas.get_task_tracker", lambda: tracker)class _CapturingVikingDB:is_closing = False@@ -1355,11 +1342,10 @@ async def test_embedding_handler_generates_id_and_counts_written_context(monkeypqueue_data["_task_work_id"] = "work-1"payload["data"] = json.dumps(queue_data)- await TextEmbeddingHandler(_CapturingVikingDB()).on_dequeue(payload)+ result = await TextEmbeddingHandler(_CapturingVikingDB()).on_dequeue(payload)assert captured["data"]["id"] == vector_record_id("acct-new-id", "viking://resources/sample", 2)- assert tracker.get_context_count("task-1") == 1- assert tracker.get_context_count("unrelated-task") == 0+ assert result.value == {"vector_written": True}def test_context_collection_uses_acl_mode_and_excludes_parent_uri():diff --git a/tests/storage/test_content_write_processing_mode.py b/tests/storage/test_content_write_processing_mode.pyindex ef8e9a589..fd2140968 100644--- a/tests/storage/test_content_write_processing_mode.py+++ b/tests/storage/test_content_write_processing_mode.py@@ -10,7 +10,7 @@ from openviking.core.context import ContextLevelfrom openviking.server.identity import RequestContext, Rolefrom openviking.service.task_store import PersistentTaskStorefrom openviking.service.task_tracker import TaskTracker-from openviking.service.task_work_index import TaskWorkIndex, bind_task_context+from openviking.service.task_work_index import bind_task_contextfrom openviking.session.memory.utils.memory_file_utils import MemoryFileUtilsfrom openviking.storage import content_write as content_write_modulefrom openviking.storage.abstract_overview import (@@ -31,7 +31,6 @@ from tests.test_task_tracker import _FakeAgfs@pytest.fixture(autouse=True)def run_content_write_operations_inline(monkeypatch):tracker = TaskTracker(PersistentTaskStore(_FakeAgfs()))- tracker.attach_work_index(TaskWorkIndex())monkeypatch.setattr("openviking.service.task_tracker._instance", tracker)async def run_tracked_operation(_task_type, *, operation, **_kwargs):diff --git a/tests/storage/test_embedding_msg.py b/tests/storage/test_embedding_msg.pyindex 0880409aa..b3eaf7d64 100644--- a/tests/storage/test_embedding_msg.py+++ b/tests/storage/test_embedding_msg.py@@ -303,6 +303,16 @@ def test_legacy_upsert_discards_historically_ignored_fields():assert "record_ids" not in serializedassert "update_fields" not in serializedassert "initial_fields" not in serialized+ assert set(restored.to_dict()) == {+ "id",+ "telemetry_id",+ "task_id",+ "_task_work_id",+ "queue_enqueued_at",+ "failure_requeue_count",+ "action",+ "payload",+ }def test_legacy_update_fields_message_normalizes_to_field_patch_payload():diff --git a/tests/storage/test_queue_manager.py b/tests/storage/test_queue_manager.pyindex 10f70eed9..210f0b096 100644--- a/tests/storage/test_queue_manager.py+++ b/tests/storage/test_queue_manager.py@@ -14,11 +14,8 @@ from unittest.mock import AsyncMockimport pytestfrom openviking.pyagfs import AsyncAGFSClient-from openviking.service.task_work_index import (- QueueTaskMetadata,- bind_task_context,- record_enqueued_work,-)+from openviking.service.task_domain import WorkRecoveryMode+from openviking.service.task_work_index import bind_task_context, record_enqueued_workfrom openviking.storage.queuefs.named_queue import DequeueHandlerBase, NamedQueuefrom openviking.storage.queuefs.process_result import ProcessResultfrom openviking.storage.queuefs.queue_manager import QueueManager@@ -86,10 +83,13 @@ def test_queue_worker_closes_loop_scoped_clients_before_closing_its_loop() -> Noasync def test_skill_shutdown_releases_lock_after_embedding_worker_exits(transport, monkeypatch, concurrency):+ from openviking.service.task_store import PersistentTaskStore+ from openviking.service.task_tracker import TaskTrackerfrom openviking.service.task_tracker_concurrency import run_to_completionfrom openviking.storage.queuefs.semantic_executor import SemanticTreeStatsfrom openviking.storage.queuefs.semantic_msg import SemanticMsgfrom openviking.storage.queuefs.semantic_processor import SemanticProcessor+ from tests.test_task_tracker import _FakeAgfsmsg = SemanticMsg(uri="viking://agent/skills/demo",@@ -99,9 +99,14 @@ async def test_skill_shutdown_releases_lock_after_embedding_worker_exits()started, queued, release = threading.Event(), threading.Event(), threading.Event()events = []- work_index = None- active_work = QueueTaskMetadata("skill-task", "active")- queued_work = QueueTaskMetadata("skill-task", "queued")+ tracker = TaskTracker(PersistentTaskStore(_FakeAgfs()))+ monkeypatch.setattr("openviking.service.task_tracker._instance", tracker)+ task = await tracker.create(+ "add_skill", task_id="skill-task", account_id="acc", user_id="alice"+ )+ await tracker.start(task.task_id, account_id="acc", user_id="alice")+ active_work = "active"+ queued_work = "queued"class Lease:lock = {"lease_ref": "demo"}@@ -116,11 +121,22 @@ async def test_skill_shutdown_releases_lock_after_embedding_worker_exits(passasync def run(self, uri):- assert work_index is not None- assert work_index.register("Embedding", active_work)- assert work_index.register("Embedding", queued_work)- record_enqueued_work("Embedding", active_work.work_id)- record_enqueued_work("Embedding", queued_work.work_id)+ await tracker.register_work(+ task.task_id,+ active_work,+ "Embedding",+ account_id="acc",+ user_id="alice",+ )+ await tracker.register_work(+ task.task_id,+ queued_work,+ "Embedding",+ account_id="acc",+ user_id="alice",+ )+ record_enqueued_work("Embedding", active_work)+ record_enqueued_work("Embedding", queued_work)queued.set()def get_stats(self):@@ -132,8 +148,12 @@ async def test_skill_shutdown_releases_lock_after_embedding_worker_exits(while not release.is_set():await asyncio.sleep(0.01)events.append("write-finished")- assert work_index is not None- work_index.record_result(active_work, "completed")+ await tracker.mark_work_done(+ task.task_id,+ active_work,+ account_id="acc",+ user_id="alice",+ )await run_to_completion(finish)return ProcessResult.success()@@ -156,7 +176,7 @@ async def test_skill_shutdown_releases_lock_after_embedding_worker_exits(async def process_dequeued(self, data):if self.name == QueueManager.SEMANTIC:- with bind_task_context("skill-task", "", "", enqueued_work=[]):+ with bind_task_context("skill-task", "acc", "alice", enqueued_work=[]):return await self.handler(data)return await self.handler(data)@@ -183,11 +203,6 @@ async def test_skill_shutdown_releases_lock_after_embedding_worker_exits(manager = QueueManager(object(), max_concurrent_semantic=concurrency, max_concurrent_embedding=concurrency)- work_index = manager._task_work_index- monkeypatch.setattr(- "openviking.storage.queuefs.semantic_work.get_task_tracker",- lambda: SimpleNamespace(wait_for_work=work_index.wait_for_work),- )manager._poll_interval = 0.001manager.set_vlm_resolver(SimpleNamespace(get_vlm=AsyncMock(return_value=SimpleNamespace())))manager.setup_standard_queues(object(), start=False)@@ -214,7 +229,7 @@ async def test_skill_shutdown_releases_lock_after_embedding_worker_exits(assert all(not thread.is_alive() for thread in threads)assert events.index("write-finished") < events.index("lock-released")assert f"ack-{manager.SEMANTIC}" not in events- assert work_index.has_work("skill-task") # Queued vectors await restart.+ assert await tracker.has_work(task.task_id, account_id="acc", user_id="alice")finally:release.set()if stopping is not None:@@ -267,13 +282,11 @@ async def test_constructor_middlewares_apply_to_all_queues(transport):manager = QueueManager(object(), middlewares=middlewares)middlewares.clear()existing = manager.get_queue("existing", allow_create=True)- with bind_task_context("task", "account", "user"):- await existing.enqueue({})- middlewares.append(Recorder())- future = manager.get_queue("future", allow_create=True)- await future.enqueue({})+ await existing.enqueue({})+ middlewares.append(Recorder())+ future = manager.get_queue("future", allow_create=True)+ await future.enqueue({})assert calls == ["existing", "future"]- assert manager._task_work_index.has_work("task")assert transport.write.await_count == 2@@ -334,30 +347,28 @@ async def test_concurrent_worker_uses_process_and_ack_middleware(transport, failtransport.write.assert_awaited_once_with("/queue/Test/ack", b"message-1")-async def test_bootstrap_restores_legacy_messages_into_middleware_index(transport):+async def test_bootstrap_restores_legacy_messages_into_store(transport):manager = QueueManager(object())- queue = manager.get_queue("Test", allow_create=True)+ manager.get_queue("Test", allow_create=True)message = {"id": "legacy-message","data": json.dumps({"task_id": "task", "account_id": "account", "user_id": "user"}),}transport.read.return_value = json.dumps([message]).encode()- attached = []+ restored_work = []class Tracker:- def attach_work_index(self, index):- attached.append(index)- assert index.has_work("task")+ work_recovery_mode = WorkRecoveryMode.QUEUE_SNAPSHOT++ async def reconcile_delivery(self, task_id, work_id, queue_name, *, account_id, user_id):+ restored_work.append((task_id, work_id, queue_name, account_id, user_id))async def restore_work_tasks(self, owners):assert owners == {"task": ("account", "user")}return ["restored"]assert await manager.prepare_task_tracking(Tracker()) == ["restored"]- assert attached == [manager._task_work_index]- await queue.ack("legacy-message", message)- assert not attached[0].has_work("task")- transport.write.assert_awaited_once_with("/queue/Test/ack", b"legacy-message")+ assert restored_work == [("task", "queuefs:legacy-message", "Test", "account", "user")]async def test_bootstrap_restores_task_owner_from_cross_user_child_work(transport):@@ -377,13 +388,17 @@ async def test_bootstrap_restores_task_owner_from_cross_user_child_work(transpor),}transport.read.return_value = json.dumps([message]).encode()+ restored_work = []class Tracker:- def attach_work_index(self, index):- assert index.has_work("task")+ work_recovery_mode = WorkRecoveryMode.QUEUE_SNAPSHOT++ async def reconcile_delivery(self, task_id, work_id, queue_name, *, account_id, user_id):+ restored_work.append((task_id, work_id, queue_name, account_id, user_id))async def restore_work_tasks(self, owners):assert owners == {"task": ("account", "task-owner")}return ["restored"]assert await manager.prepare_task_tracking(Tracker()) == ["restored"]+ assert restored_work == [("task", "work", "Test", "account", "task-owner")]diff --git a/tests/storage/test_reindex_msg.py b/tests/storage/test_reindex_msg.pyindex 5bb6883d3..b529efae2 100644--- a/tests/storage/test_reindex_msg.py+++ b/tests/storage/test_reindex_msg.py@@ -63,16 +63,13 @@ def test_reindex_msg_rejects_missing_descriptor_fields():@pytest.mark.asyncioasync def test_reindex_processor_terminalizes_root_after_descendants(monkeypatch):- from openviking.service.task_store import PersistentTaskStore+ from openviking.service.task_store import PersistentTaskStore, WorkRegistrationfrom openviking.service.task_tracker import TaskTracker- from openviking.service.task_work_index import QueueTaskMetadata, TaskWorkIndexfrom openviking.storage.queuefs.reindex_processor import ReindexProcessorfrom tests.test_task_tracker import _FakeAgfsowner = {"account_id": "account-1", "user_id": "user-1"}tracker = TaskTracker(PersistentTaskStore(_FakeAgfs()))- index = TaskWorkIndex()- tracker.attach_work_index(index)monkeypatch.setattr("openviking.storage.queuefs.reindex_processor.get_task_tracker", Mock(return_value=tracker))@@ -89,10 +86,14 @@ async def test_reindex_processor_terminalizes_root_after_descendants(monkeypatchprocessor = ReindexProcessor(viking_fs)processor._adopt_or_reacquire = AsyncMock(return_value={"lease": "root"})task = await tracker.create("admin_reindex", resource_id="viking://resources/demo", **owner)- root_work = QueueTaskMetadata(task.task_id, "root-work", **owner)- child_work = QueueTaskMetadata(task.task_id, "child-work", **owner)- assert index.register("Reindex", root_work)- assert index.register("Embedding", child_work)+ assert (+ await tracker.register_work(task.task_id, "root-work", "Reindex", **owner)+ is WorkRegistration.CREATED+ )+ assert (+ await tracker.register_work(task.task_id, "child-work", "Embedding", **owner)+ is WorkRegistration.CREATED+ )entered_descendant_wait = asyncio.Event()original_wait_for_task_work = tracker.wait_for_task_work@@ -119,14 +120,12 @@ async def test_reindex_processor_terminalizes_root_after_descendants(monkeypatchassert not processing.done()assert (await tracker.get(task.task_id, **owner)).status is TaskStatus.RUNNING- index.record_result(child_work, "completed")- await index.prepare_ack("Embedding", child_work)+ await tracker.mark_work_done(task.task_id, "child-work", **owner)outcome = await asyncio.wait_for(processing, 1)assert outcome.outcome.value == "success"assert (await tracker.get(task.task_id, **owner)).status is TaskStatus.RUNNING- index.record_result(root_work, "completed")- await index.prepare_ack("Reindex", root_work)+ await tracker.mark_work_done(task.task_id, "root-work", **owner)completed = await tracker.wait(task.task_id, **owner, timeout=0)assert completed.status is TaskStatus.COMPLETEDassert completed.result["rebuilt_records"] == 3diff --git a/tests/storage/test_semantic_executor_stats.py b/tests/storage/test_semantic_executor_stats.pyindex ee48eb8d1..f64edbe4b 100644--- a/tests/storage/test_semantic_executor_stats.py+++ b/tests/storage/test_semantic_executor_stats.py@@ -11,7 +11,6 @@ from openviking.core.context import ContextLevelfrom openviking.server.identity import RequestContext, Rolefrom openviking.service.task_queue_middleware import TaskWorkQueueMiddlewarefrom openviking.service.task_work_index import (- TaskWorkIndex,TaskWorkRejected,bind_task_context,get_task_context,@@ -468,20 +467,11 @@ async def test_semantic_executor_shares_node_scheduler_across_roots(monkeypatch)@pytest.mark.asyncioasync def test_task_work_rejection_does_not_stop_shared_semantic_worker():- work_index = TaskWorkIndex()-- async def finalize_before_ack(_metadata):- return None-- work_index.set_callbacks(- finalize_before_ack=finalize_before_ack,- is_cancellation_requested=lambda _task_id: True,- )embedding_queue = NamedQueue(None,"/queue","Embedding",- middlewares=[TaskWorkQueueMiddleware(work_index)],+ middlewares=[TaskWorkQueueMiddleware()],)embedding_queue._initialized = Trueunrelated_ran = asyncio.Event()diff --git a/tests/storage/test_semantic_processor_lock_ownership.py b/tests/storage/test_semantic_processor_lock_ownership.pyindex 534042adf..a3f5368e9 100644--- a/tests/storage/test_semantic_processor_lock_ownership.py+++ b/tests/storage/test_semantic_processor_lock_ownership.py@@ -167,7 +167,6 @@ async def test_shared_scheduler_drains_each_messages_embeddings_before_unlock(from openviking.service.task_store import PersistentTaskStorefrom openviking.service.task_tracker import TaskStatus, TaskTrackerfrom openviking.service.task_work_index import (- TaskWorkIndex,bind_task_context,extract_task_metadata,)@@ -210,8 +209,6 @@ async def test_shared_scheduler_drains_each_messages_embeddings_before_unlock(monkeypatch.setattr(f"openviking.storage.queuefs.{module}.get_viking_fs", lambda: fs)agfs = _FakeAgfs()tracker = TaskTracker(PersistentTaskStore(agfs))- index = TaskWorkIndex()- tracker.attach_work_index(index)monkeypatch.setattr("openviking.service.task_tracker._instance", tracker)entered_embedding_drain = asyncio.Event()original_wait_for_work = tracker.wait_for_work@@ -227,7 +224,7 @@ async def test_shared_scheduler_drains_each_messages_embeddings_before_unlock(cast(AGFSSyncClientProtocol, agfs),"/queue","Embedding",- middlewares=[TaskWorkQueueMiddleware(index)],+ middlewares=[TaskWorkQueueMiddleware()],)enqueued = asyncio.Event()@@ -296,8 +293,12 @@ async def test_shared_scheduler_drains_each_messages_embeddings_before_unlock(assert metadata.task_id != source.task_idif task:assert metadata.task_id == task.task_id- index.record_result(metadata, "completed")- await index.prepare_ack("Embedding", metadata)+ await tracker.mark_work_done(+ metadata.task_id,+ metadata.work_id,+ account_id=metadata.account_id,+ user_id=metadata.user_id,+ )result = await asyncio.wait_for(worker, 1)assert result.outcome is (ProcessOutcome.FAILED if fail_generation else ProcessOutcome.SUCCESS@@ -319,6 +320,11 @@ async def test_shared_scheduler_drains_each_messages_embeddings_before_unlock(if payload:metadata = extract_task_metadata(json.loads(payload))if metadata:- index.record_result(metadata, "completed")+ await tracker.mark_work_done(+ metadata.task_id,+ metadata.work_id,+ account_id=metadata.account_id,+ user_id=metadata.user_id,+ )worker.cancel()await asyncio.gather(worker, return_exceptions=True)diff --git a/tests/storage/test_semantic_processor_permanent_storage_error.py b/tests/storage/test_semantic_processor_permanent_storage_error.pyindex 859af3855..eaefba68f 100644--- a/tests/storage/test_semantic_processor_permanent_storage_error.py+++ b/tests/storage/test_semantic_processor_permanent_storage_error.py@@ -13,7 +13,9 @@ import pytestfrom openviking.models.model_error import ModelError, ModelErrorKindfrom openviking.pyagfs.exceptions import AGFSNotADirectoryErrorfrom openviking.service.task_queue_middleware import TaskWorkQueueMiddleware-from openviking.service.task_work_index import TaskWorkIndex+from openviking.service.task_store import PersistentTaskStore+from openviking.service.task_tracker import TaskStatus, TaskTracker+from openviking.service.task_work_index import bind_task_contextfrom openviking.storage.errors import LockAcquisitionErrorfrom openviking.storage.queuefs.named_queue import NamedQueuefrom openviking.storage.queuefs.process_result import ProcessOutcome@@ -22,6 +24,7 @@ from openviking.storage.queuefs.semantic_msg import SemanticMsgfrom openviking.storage.queuefs.semantic_processor import SemanticProcessorfrom openviking.storage.viking_fs import SyncDifffrom openviking.utils.circuit_breaker import CircuitBreaker, CircuitBreakerOpen+from tests.test_task_tracker import _FakeAgfs@pytest.fixture@@ -123,22 +126,33 @@ async def test_queue_acks_failed_semantic_work(semantic_env, monkeypatch, wrappemonkeypatch.setattr("openviking.storage.queuefs.named_queue.AsyncAGFSClient", lambda agfs: backend)- index = TaskWorkIndex()- finalize = AsyncMock()- index.set_callbacks(- finalize_before_ack=finalize, is_cancellation_requested=lambda task_id: False- )+ tracker = TaskTracker(PersistentTaskStore(_FakeAgfs()))+ monkeypatch.setattr("openviking.service.task_tracker._instance", tracker)queue = NamedQueue(object(),"/queue","Semantic",dequeue_handler=env.processor,- middlewares=[TaskWorkQueueMiddleware(index)],+ middlewares=[TaskWorkQueueMiddleware()],)payload = msg.to_dict()if task_owned:payload["task_id"] = "ingestion-task"- await queue.enqueue(payload)+ task = await tracker.create(+ "add_resource",+ task_id="ingestion-task",+ account_id=msg.account_id,+ user_id=msg.user_id,+ )+ await tracker.start(+ task.task_id,+ account_id=msg.account_id,+ user_id=msg.user_id,+ )+ with bind_task_context(task.task_id, msg.account_id, msg.user_id):+ await queue.enqueue(payload)+ else:+ await queue.enqueue(payload)queued_payload = backend.write.call_args.args[1].decode()backend.read = AsyncMock(side_effect=[@@ -146,7 +160,14 @@ async def test_queue_acks_failed_semantic_work(semantic_env, monkeypatch, wrappejson.dumps({"pending": 0, "processing": 0}).encode(),])- assert index.has_work("ingestion-task") is task_owned+ assert (+ await tracker.has_work(+ "ingestion-task",+ account_id=msg.account_id,+ user_id=msg.user_id,+ )+ is task_owned+ )await asyncio.wait_for(queue.dequeue(), timeout=1)@@ -155,13 +176,20 @@ async def test_queue_acks_failed_semantic_work(semantic_env, monkeypatch, wrappeassert status.in_progress == status.processed == status.requeue_count == 0assert status.error_count == 1assert [entry.message for entry in status.errors] == [str(error)]- assert not index.has_work("ingestion-task") # Includes active consumer registration.- assert index.failure("ingestion-task") == (str(error) if task_owned else None)+ assert not await tracker.has_work(+ "ingestion-task",+ account_id=msg.account_id,+ user_id=msg.user_id,+ )if task_owned:- finalize.assert_awaited_once()- assert finalize.call_args.args[0].task_id == "ingestion-task"- else:- finalize.assert_not_awaited()+ failed = await tracker.get(+ "ingestion-task",+ account_id=msg.account_id,+ user_id=msg.user_id,+ )+ assert failed is not None+ assert failed.status is TaskStatus.FAILED+ assert failed.error == str(error)env.retry_queue.enqueue.assert_not_awaited()env.pathlock.pathlock_release.assert_awaited_once_with(env.lease)diff --git a/tests/storage/test_skill_package_cancellation.py b/tests/storage/test_skill_package_cancellation.pyindex a23ddcdaf..a79e63ed3 100644--- a/tests/storage/test_skill_package_cancellation.py+++ b/tests/storage/test_skill_package_cancellation.py@@ -12,8 +12,9 @@ import pytestfrom openviking.models.embedder.base import EmbedResultfrom openviking.server.identity import RequestContext, Rolefrom openviking.service.task_queue_middleware import TaskWorkQueueMiddleware+from openviking.service.task_store import PersistentTaskStore+from openviking.service.task_tracker import TaskTrackerfrom openviking.service.task_work_index import (- TaskWorkIndex,bind_task_context,get_task_context,prepare_task_payload,@@ -29,6 +30,7 @@ from openviking.storage.queuefs.semantic_processor import SemanticProcessorfrom openviking_cli.session.user_id import UserIdentifierfrom tests.storage.test_collection_schemas import _DummyConfig, _DummyEmbedderfrom tests.storage.test_semantic_executor_skip_files import _FakeVikingFS+from tests.test_task_tracker import _FakeAgfsdef _ctx():@@ -189,16 +191,12 @@ async def test_active_skill_embedding_cancel_settles_only_after_write_exit("account_id": "acc",},)- index = TaskWorkIndex()- cancelling = set()- index.set_callbacks(- finalize_before_ack=AsyncMock(),- is_cancellation_requested=lambda task_id: task_id in cancelling,- )- monkeypatch.setattr(- "openviking.storage.collection_schemas.get_task_tracker",- lambda: SimpleNamespace(record_context_count=index.record_context_count),+ tracker = TaskTracker(PersistentTaskStore(_FakeAgfs()))+ monkeypatch.setattr("openviking.service.task_tracker._instance", tracker)+ task = await tracker.create(+ "add_skill", task_id="skill-task", account_id="acc", user_id="alice")+ await tracker.start(task.task_id, account_id="acc", user_id="alice")with bind_task_context("skill-task", "acc", "alice"):payload, metadata = prepare_task_payload(msg.to_dict())queue = NamedQueue(@@ -206,19 +204,27 @@ async def test_active_skill_embedding_cancel_settles_only_after_write_exit("/queue","Embedding",dequeue_handler=handler,- middlewares=[TaskWorkQueueMiddleware(index)],+ middlewares=[TaskWorkQueueMiddleware()],)envelope = {"data": json.dumps(payload)}- assert index.register(queue.name, metadata)+ assert metadata is not None+ await tracker.register_work(+ task.task_id,+ metadata.work_id,+ queue.name,+ account_id="acc",+ user_id="alice",+ )worker = asyncio.create_task(queue.process_dequeued(envelope))try:await asyncio.wait_for(started.wait(), 1)- cancelling.add("skill-task")- index.cancel_active("skill-task")+ await tracker.cancel(task.task_id, account_id="acc", user_id="alice")await asyncio.sleep(0.03)if during_write:assert not worker.done()- assert index.has_work("skill-task")+ assert await tracker.has_work(+ task.task_id, account_id="acc", user_id="alice"+ )assert not writeselse:await asyncio.wait_for(worker, 1)@@ -227,23 +233,21 @@ async def test_active_skill_embedding_cancel_settles_only_after_write_exit(release.set()result = await asyncio.wait_for(worker, 1)assert result.outcome is ProcessOutcome.CANCELLED- await index.prepare_ack(queue.name, envelope)- assert not index.has_work("skill-task")+ assert not await tracker.has_work(+ task.task_id, account_id="acc", user_id="alice"+ )assert len(writes) == int(during_write)@pytest.mark.asyncioasync def test_semantic_cancel_drains_embeddings_before_releasing_package(monkeypatch):- index = TaskWorkIndex()- child = None- index.set_callbacks(- finalize_before_ack=AsyncMock(),- is_cancellation_requested=lambda _task_id: False,- )- monkeypatch.setattr(- "openviking.storage.queuefs.semantic_work.get_task_tracker",- lambda: SimpleNamespace(wait_for_work=index.wait_for_work),+ tracker = TaskTracker(PersistentTaskStore(_FakeAgfs()))+ monkeypatch.setattr("openviking.service.task_tracker._instance", tracker)+ task = await tracker.create(+ "add_skill", task_id="skill-task", account_id="acc", user_id="alice")+ await tracker.start(task.task_id, account_id="acc", user_id="alice")+ child = Nonemsg = SemanticMsg(uri="viking://agent/skills/demo",context_type="skill",@@ -258,11 +262,15 @@ async def test_semantic_cancel_drains_embeddings_before_releasing_package(monkeyasync def run(self, uri):nonlocal child- from openviking.service.task_work_index import QueueTaskMetadata-- child = QueueTaskMetadata("skill-task", "embedding")- assert index.register("Embedding", child)- record_enqueued_work("Embedding", child.work_id)+ child = "embedding"+ await tracker.register_work(+ task.task_id,+ child,+ "Embedding",+ account_id="acc",+ user_id="alice",+ )+ record_enqueued_work("Embedding", child)started.set()def get_stats(self):@@ -298,7 +306,12 @@ async def test_semantic_cancel_drains_embeddings_before_releasing_package(monkeyassert events == []events.append("embedding-settled")assert child is not None- index.record_result(child, "completed")+ await tracker.mark_work_done(+ task.task_id,+ child,+ account_id="acc",+ user_id="alice",+ )with pytest.raises(asyncio.CancelledError):await asyncio.wait_for(worker, 1)assert events == ["embedding-settled", "released"]diff --git a/tests/test_task_tracker.py b/tests/test_task_tracker.pyindex b322bfad3..e06d8dbae 100644--- a/tests/test_task_tracker.py+++ b/tests/test_task_tracker.py@@ -8,13 +8,18 @@ import jsonimport threadingimport timefrom copy import deepcopy+from unittest.mock import create_autospecimport pytestfrom openviking.pyagfs.exceptions import AGFSAlreadyExistsError, AGFSNotFoundErrorfrom openviking.server.identity import RequestContext, Rolefrom openviking.service.session_service import SessionService-from openviking.service.task_store import PersistentTaskStore, _task_to_payload+from openviking.service.task_store import (+ PersistentTaskStore,+ _task_from_payload,+ _task_to_payload,+)from openviking.service.task_tracker import (TaskStatus,TaskTracker,@@ -23,11 +28,7 @@ from openviking.service.task_tracker import (run_tracked_operation,set_task_tracker,)-from openviking.service.task_work_index import (- QueueTaskMetadata,- TaskWorkIndex,- get_task_context,-)+from openviking.service.task_work_index import get_task_contextfrom openviking_cli.exceptions import DeadlineExceededErrorfrom openviking_cli.session.user_id import UserIdentifier@@ -549,62 +550,67 @@ async def test_sanitize_preserves_safe_error():@pytest.mark.parametrize("already_missing", [False, True])async def test_evict_expired_completed(already_missing):agfs = _FakeAgfs()- tracker = TaskTracker(store=PersistentTaskStore(agfs))- t = await tracker.create("session_commit", **_owner_kwargs())- await tracker.start(t.task_id)- await tracker.complete(t.task_id, {})- assert await tracker._store.get(t.task_id, **_owner_kwargs()) is not None+ writer = TaskTracker(store=PersistentTaskStore(agfs))+ task = await writer.create("session_commit", **_owner_kwargs())+ await writer.start(task.task_id)+ await writer.complete(task.task_id, {})+ path = f"/local/acme/_system/tasks/alice/{task.task_id}.json"if already_missing:agfs.files.clear()- # Simulate old timestamp (access internal state; get() returns defensive copies)- tracker._tasks[t.task_id].updated_at = time.time() - tracker.TTL_COMPLETED - 1- await tracker._evict_expired()- assert await tracker.get(t.task_id) is None- assert await tracker._store.get(t.task_id, **_owner_kwargs()) is None+ else:+ payload = json.loads(agfs.files[path])+ payload["updated_at"] = time.time() - writer.TTL_COMPLETED - 1+ agfs.files[path] = json.dumps(payload).encode()++ tracker = TaskTracker(store=PersistentTaskStore(agfs))+ await tracker.delete_expired()+ assert await tracker.get(task.task_id, **_owner_kwargs()) is None-async def test_evict_keeps_cached_task_when_persistent_delete_fails():+async def test_delete_expired_retries_after_persistent_delete_fails():agfs = _FakeAgfs()+ writer = TaskTracker(store=PersistentTaskStore(agfs))+ task = await writer.create("session_commit", **_owner_kwargs())+ await writer.start(task.task_id)+ await writer.complete(task.task_id, {})+ path = f"/local/acme/_system/tasks/alice/{task.task_id}.json"+ payload = json.loads(agfs.files[path])+ payload["updated_at"] = time.time() - writer.TTL_COMPLETED - 1+ agfs.files[path] = json.dumps(payload).encode()tracker = TaskTracker(store=PersistentTaskStore(agfs))- t = await tracker.create("session_commit", **_owner_kwargs())- await tracker.start(t.task_id)- await tracker.complete(t.task_id, {})- tracker._tasks[t.task_id].updated_at = time.time() - tracker.TTL_COMPLETED - 1agfs.fail_rm = True- await tracker._evict_expired()-- assert await tracker.get(t.task_id) is not None- assert await tracker._store.get(t.task_id, **_owner_kwargs()) is not None+ with pytest.raises(OSError, match="simulated delete failure"):+ await tracker.delete_expired()+ assert await tracker.get(task.task_id, **_owner_kwargs()) is not Noneagfs.fail_rm = False- await tracker._evict_expired()+ assert (await tracker.delete_expired()).deleted_count == 1- assert await tracker.get(t.task_id) is None- assert await tracker._store.get(t.task_id, **_owner_kwargs()) is None+ assert await tracker.get(task.task_id, **_owner_kwargs()) is Noneassert all(call["ctx"]["disable_auto_pathlock"] == "true" for call in agfs.rm_calls)-async def test_evict_keeps_recent_completed(tracker: TaskTracker):- t = await tracker.create("session_commit", **_owner_kwargs())- await tracker.start(t.task_id)- await tracker.complete(t.task_id, {})- await tracker._evict_expired()- assert await tracker.get(t.task_id) is not None-+async def test_evict_keeps_recent_completed_with_bounded_reads(monkeypatch):+ agfs = _FakeAgfs()+ writer = TaskTracker(PersistentTaskStore(agfs))+ for index in range(4):+ task = await writer.create("session_commit", task_id=f"task-{index}", **_owner_kwargs())+ await writer.complete(task.task_id, {}, **_owner_kwargs())+ tracker = TaskTracker(PersistentTaskStore(agfs))+ monkeypatch.setattr(tracker, "CLEANUP_BATCH", 2)+ for task_id in ("task-0", "task-1"):+ await tracker.get(task_id, **_owner_kwargs())+ read = create_autospec(agfs.read, side_effect=agfs.read)+ monkeypatch.setattr(agfs, "read", read)-async def test_evict_fifo_when_over_limit(tracker: TaskTracker):- tracker.MAX_TASKS = 5- tasks = []- for i in range(7):- tasks.append(await tracker.create("session_commit", resource_id=f"s{i}", **_owner_kwargs()))- await tracker._evict_expired()- assert tracker.count() == 5- # Oldest should be gone- assert await tracker.get(tasks[0].task_id) is None- assert await tracker.get(tasks[1].task_id) is None- # Newest should remain- assert await tracker.get(tasks[6].task_id) is not None+ first = await tracker.delete_expired()+ assert first.scanned == 2 and first.deleted_count == 0+ assert read.call_count == 0+ second = await tracker.delete_expired()+ assert second.scanned == 2 and second.deleted_count == 0+ assert read.call_count == 2+ assert len(agfs.files) == 4# ── Singleton ──@@ -629,21 +635,19 @@ async def test_get_task_tracker_requires_service_initialization():async def test_wait_timeout_preserves_recorded_operation_result(tracker):- index = TaskWorkIndex()- tracker.attach_work_index(index)set_task_tracker(tracker)state = {}async def operation():context = get_task_context()state["task_id"] = context.task_id- work = QueueTaskMetadata(+ await tracker.register_work(context.task_id,"child-work",- context.account_id,- context.user_id,+ "Semantic",+ account_id=context.account_id,+ user_id=context.user_id,)- assert index.register("Semantic", work)return {"status": "success", "root_uri": "resource-result"}with pytest.raises(DeadlineExceededError) as caught:@@ -667,25 +671,28 @@ async def test_wait_timeout_preserves_recorded_operation_result(tracker):async def test_accept_failed_result_returns_structured_partial_result(tracker):- index = TaskWorkIndex()- tracker.attach_work_index(index)set_task_tracker(tracker)workers = []async def operation():context = get_task_context()- work = QueueTaskMetadata(+ await tracker.register_work(context.task_id,"embedding-work",- context.account_id,- context.user_id,+ "Embedding",+ account_id=context.account_id,+ user_id=context.user_id,)- assert index.register("Embedding", work)async def fail_work():await asyncio.sleep(0)- index.record_result(work, "failed", error="embedding failed")- await index.prepare_ack("Embedding", work)+ await tracker.mark_work_failed(+ context.task_id,+ "embedding-work",+ "embedding failed",+ account_id=context.account_id,+ user_id=context.user_id,+ )workers.append(asyncio.create_task(fail_work()))return {@@ -850,10 +857,34 @@ async def test_operation_start_failure_settles_created_task():@pytest.mark.parametrize("timeout", [0, 0.01])async def test_wait_timeout_excludes_operation_time_for_completed_task(tracker, timeout):set_task_tracker(tracker)+ workers = []async def operation():await asyncio.sleep(0.02)- return {"status": "success"}+ context = get_task_context()+ await tracker.wait_for_task_work(+ context.task_id, exclude_work_id=context.work_id, timeout=0, **_owner_kwargs()+ )+ if timeout:+ await tracker.register_work(+ context.task_id,+ "embedding",+ "Embedding",+ account_id="acme",+ user_id="alice",+ )++ async def complete_work():+ await asyncio.sleep(0.001)+ await tracker.mark_work_done(+ context.task_id,+ "embedding",+ account_id="acme",+ user_id="alice",+ )++ workers.append(asyncio.create_task(complete_work()))+ return {"status": "success", "vector_status": "queued" if timeout else "skipped"}result, task_id = await run_tracked_operation("content_write",@@ -862,34 +893,34 @@ async def test_wait_timeout_excludes_operation_time_for_completed_task(tracker,operation=operation,wait=True,timeout=timeout,+ return_terminal_result=True,)- assert result == {"status": "success"}+ assert result["status"] == "success"+ assert "task_id" not in result+ assert result["vector_status"] == ("complete" if timeout else "skipped")+ if timeout:+ assert result["queue_status"]["Embedding"]["processed"] == 1completed = await tracker.wait(task_id, **_owner_kwargs(), timeout=0)assert completed.status is TaskStatus.COMPLETEDassert completed.result["status"] == "success"assert completed.result["task_id"] == task_id+ await asyncio.gather(*workers)async def test_multiple_queue_waits_share_budget_and_leave_task_pollable(tracker):- index = TaskWorkIndex()- tracker.attach_work_index(index)set_task_tracker(tracker)state = {}async def operation():task_id = get_task_context().task_idstate["task_id"] = task_id- first = QueueTaskMetadata(task_id, "first", "acme", "alice")- assert index.register("Semantic", first)+ await tracker.register_work(task_id, "first", "Semantic", **_owner_kwargs())# Exhaust the budget on a caught wait, then make the first Work complete.with pytest.raises(TimeoutError):- await tracker.wait_for_work(task_id, [first.work_id], timeout=0.01)- index.record_result(first, "completed")- await index.prepare_ack("Semantic", first)- second = QueueTaskMetadata(task_id, "second", "acme", "alice")- assert index.register("Embedding", second)- state["second"] = second+ await tracker.wait_for_work(task_id, ["first"], timeout=0.01)+ await tracker.mark_work_done(task_id, "first", **_owner_kwargs())+ await tracker.register_work(task_id, "second", "Embedding", **_owner_kwargs())with pytest.raises(TimeoutError):await tracker.wait_for_task_work(task_id, timeout=60)return {"status": "success"}@@ -909,14 +940,13 @@ async def test_multiple_queue_waits_share_budget_and_leave_task_pollable(trackertask = await tracker.get(state["task_id"], **_owner_kwargs())assert task.status is TaskStatus.RUNNINGassert task.error is None- index.record_result(state["second"], "completed")- await index.prepare_ack("Embedding", state["second"])+ await tracker.mark_work_done(task.task_id, "second", **_owner_kwargs())assert (await tracker.wait(task.task_id, **_owner_kwargs(), timeout=0)).status is TaskStatus.COMPLETED-async def test_persistent_store_cross_tracker_visibility():+async def test_persistent_store_cross_tracker_visibility_after_cancelled_write(monkeypatch):agfs = _FakeAgfs()store = PersistentTaskStore(agfs)tracker1 = TaskTracker(store=store)@@ -924,13 +954,39 @@ async def test_persistent_store_cross_tracker_visibility():task = await tracker1.create("session_commit", resource_id="sess-123", **_owner_kwargs())await tracker1.start(task.task_id, account_id="acme", user_id="alice")- await tracker1.complete(task.task_id, {"ok": True}, account_id="acme", user_id="alice")+ entered, release = threading.Event(), threading.Event()+ write = agfs.write++ def blocked_write(*args, **kwargs):+ entered.set()+ assert release.wait(timeout=3)+ return write(*args, **kwargs)++ monkeypatch.setattr(agfs, "write", blocked_write)+ completing = asyncio.create_task(+ tracker1.complete(task.task_id, {"ok": True}, **_owner_kwargs())+ )+ try:+ assert await asyncio.to_thread(entered.wait, 3)+ completing.cancel()+ await asyncio.sleep(0)+ assert not completing.done()+ assert (await tracker2.get(task.task_id, **_owner_kwargs())).status is TaskStatus.RUNNING+ release.set()+ with pytest.raises(asyncio.CancelledError):+ await completing+ finally:+ release.set()+ await asyncio.gather(completing, return_exceptions=True)loaded = await tracker2.get(task.task_id, account_id="acme", user_id="alice")assert loaded is not Noneassert loaded.status == TaskStatus.COMPLETEDassert loaded.result == {"ok": True}+ restored = await TaskTracker(PersistentTaskStore(agfs)).get(task.task_id, **_owner_kwargs())+ assert restored.status is TaskStatus.COMPLETED+ assert restored.result == loaded.result@pytest.mark.parametrize(@@ -956,7 +1012,7 @@ async def test_persistent_task_extensions_survive_updates(extra):agfs.files[path] = json.dumps(payload).encode()original = agfs.files[path]- reader = TaskTracker(store=store)+ reader = TaskTracker(store=PersistentTaskStore(agfs))loaded = await reader.get(task.task_id, **_owner_kwargs())assert loaded is not Noneassert loaded.status == TaskStatus.PENDING@@ -971,7 +1027,9 @@ async def test_persistent_task_extensions_survive_updates(extra):assert saved["status"] == "completed"assert saved["result"] == {"ok": True}assert {key: saved[key] for key in extra} == extra- reloaded = await TaskTracker(store=store).get(task.task_id, **_owner_kwargs())+ reloaded = await TaskTracker(store=PersistentTaskStore(agfs)).get(+ task.task_id, **_owner_kwargs()+ )assert reloaded is not Noneassert reloaded.status == TaskStatus.COMPLETEDassert {key: _task_to_payload(reloaded)[key] for key in extra} == extra@@ -986,7 +1044,7 @@ async def test_task_extension_payload_is_defensively_copied():"meta": {"values": [2]},}original = deepcopy(payload)- record = TaskTracker._record_from_payload(payload)+ record = _task_from_payload(payload)record._extra_fields["future_field"]["values"].append(3)record.meta["values"].append(4)assert payload == original@@ -1000,7 +1058,7 @@ async def test_task_extension_payload_is_defensively_copied():@pytest.mark.parametrize("status", ["unknown", None])async def test_task_extensions_do_not_hide_invalid_status(status):with pytest.raises(ValueError):- TaskTracker._record_from_payload(+ _task_from_payload({"task_id": "task-1", "task_type": "session_commit", "status": status, "extra": 1})@@ -1040,12 +1098,6 @@ async def test_persistent_store_writes_task_record_json():assert all(call["ctx"]["disable_auto_pathlock"] == "true" for call in agfs.write_calls)-async def test_persistent_store_keeps_tasktracker_tasks_dict():- tracker = TaskTracker(store=PersistentTaskStore(_FakeAgfs()))- task = await tracker.create("session_commit", **_owner_kwargs())- assert task.task_id in tracker._tasks--async def test_persistent_store_survives_tracker_reset():agfs = _FakeAgfs()tracker1 = TaskTracker(store=PersistentTaskStore(agfs))@@ -1178,21 +1230,16 @@ async def test_execution_events_survive_restart_and_preserve_reported_facts():@pytest.mark.parametrize("outcome", ["complete", "fail"])async def test_terminal_event_waits_for_owned_work(tracker, outcome):- from openviking.service.task_work_index import QueueTaskMetadata, TaskWorkIndex-owner = _owner_kwargs()task = await tracker.create("add_resource", **owner)await tracker.start(task.task_id, **owner)- work_index = TaskWorkIndex()- tracker.attach_work_index(work_index)- child = QueueTaskMetadata(task.task_id, "child-work", "acme", "alice")- work_index.register("Semantic", child)+ await tracker.register_work(task.task_id, "child-work", "Semantic", **owner)await getattr(tracker, outcome)(task.task_id, {"done": True} if outcome == "complete" else "error", **owner)snapshot = await tracker.get(task.task_id, **owner)assert snapshot.status == TaskStatus.RUNNING- await work_index.prepare_ack("Semantic", child)+ await tracker.mark_work_done(task.task_id, "child-work", **owner)snapshot = await tracker.get(task.task_id, **owner)last = snapshot.execution_events["items"][-1]assert last["kind"] == "status_changed"@@ -1207,7 +1254,7 @@ async def test_legacy_tasks_do_not_get_invented_history():payload = json.loads(agfs.files[path])payload.pop("execution_events")agfs.files[path] = json.dumps(payload).encode()- reader = TaskTracker(store=store)+ reader = TaskTracker(store=PersistentTaskStore(agfs))assert (await reader.get(task.task_id, **_owner_kwargs())).execution_events is Noneawait reader.start(task.task_id, **_owner_kwargs())history = (await reader.get(task.task_id, **_owner_kwargs())).execution_events@@ -1258,18 +1305,20 @@ async def test_process_events_respect_owner_and_terminal_boundaries(tracker):).execution_events == cancelled.execution_events-async def test_processing_time_persists_and_restored_active_tasks_are_unknown(tracker, monkeypatch):+async def test_processing_time_persists_and_restored_active_tasks_are_unknown(monkeypatch):now = [0.0]- monkeypatch.setattr("openviking.service.task_work_index.time.time", lambda: now[0])+ monkeypatch.setattr("openviking.service.task_store.time.time", lambda: now[0])+ agfs = _FakeAgfs()+ tracker = TaskTracker(PersistentTaskStore(agfs))task = await tracker.create("add_resource", **_owner_kwargs())await tracker.start(task.task_id, **_owner_kwargs())now[0] = 10- tracker.register_running_task(task.task_id)+ await tracker.register_running_task(task.task_id, **_owner_kwargs())now[0] = 13assert (await tracker.get(task.task_id)).processing_seconds == 3# A stage update neither resets nor stops the processing clock.await tracker.update_stage(task.task_id, "semantic", **_owner_kwargs())- restarted = TaskTracker(store=tracker._store)+ restarted = TaskTracker(store=PersistentTaskStore(agfs))assert (await restarted.get(task.task_id, **_owner_kwargs())).processing_seconds is None# Waiting, I/O, and backoff after a worker claims the task remain part of# worker-occupied wall time.@@ -1281,5 +1330,7 @@ async def test_processing_time_persists_and_restored_active_tasks_are_unknown(trfinished = await tracker.get(task.task_id)assert finished.status == TaskStatus.COMPLETEDassert finished.to_dict()["processing_seconds"] == 22- restored = await TaskTracker(store=tracker._store).get(task.task_id, **_owner_kwargs())+ restored = await TaskTracker(store=PersistentTaskStore(agfs)).get(+ task.task_id, **_owner_kwargs()+ )assert restored.processing_seconds == 22diff --git a/tests/test_task_tracker_concurrency.py b/tests/test_task_tracker_concurrency.pyindex 342c4566d..7292d3efd 100644--- a/tests/test_task_tracker_concurrency.py+++ b/tests/test_task_tracker_concurrency.py@@ -5,149 +5,38 @@ import asyncioimport gcimport threadingimport weakref-from copy import deepcopy-from typing import Anyimport pytest-class _ControllableTaskStore:- def __init__(self) -> None:- self.payloads: dict[str, dict[str, Any]] = {}- self.create_calls = 0- self.update_started: dict[str, asyncio.Event] = {}- self.update_release: dict[str, asyncio.Event] = {}- self.update_errors: dict[str, Exception] = {}- self.list_started: asyncio.Event | None = None- self.list_release: asyncio.Event | None = None- self.get_calls = 0- self.operation_loops: list[asyncio.AbstractEventLoop] = []-- @staticmethod- def _payload(task: Any) -> dict[str, Any]:- return {- "task_id": task.task_id,- "task_type": task.task_type,- "status": task.status.value,- "created_at": task.created_at,- "updated_at": task.updated_at,- "resource_id": task.resource_id,- "account_id": task.account_id,- "user_id": task.user_id,- "meta": deepcopy(task.meta),- "stage": task.stage,- "result": deepcopy(task.result),- "error": task.error,- "execution_events": deepcopy(task.execution_events),- }-- async def create(self, task: Any) -> None:- self.operation_loops.append(asyncio.get_running_loop())- self.create_calls += 1- self.payloads[task.task_id] = self._payload(task)-- async def update(self, task: Any) -> None:- self.operation_loops.append(asyncio.get_running_loop())- started = self.update_started.get(task.task_id)- if started is not None:- started.set()- release = self.update_release.get(task.task_id)- if release is not None:- await release.wait()- error = self.update_errors.get(task.task_id)- if error is not None:- raise error- self.payloads[task.task_id] = self._payload(task)-- async def get(- self,- task_id: str,- *,- account_id: str | None = None,- user_id: str | None = None,- ) -> dict[str, Any] | None:- self.operation_loops.append(asyncio.get_running_loop())- self.get_calls += 1- payload = self.payloads.get(task_id)- if payload is None:- return None- if account_id is not None and payload["account_id"] != account_id:- return None- if user_id is not None and payload["user_id"] != user_id:- return None- return deepcopy(payload)-- async def list(- self,- account_id: str,- *,- user_id: str | None = None,- ) -> list[dict[str, Any]]:- self.operation_loops.append(asyncio.get_running_loop())- snapshot = [- deepcopy(payload)- for payload in self.payloads.values()- if payload["account_id"] == account_id- and (user_id is None or payload["user_id"] == user_id)- ]- if self.list_started is not None:- self.list_started.set()- if self.list_release is not None:- await self.list_release.wait()- return snapshot-- async def delete(- self,- task_id: str,- *,- account_id: str,- user_id: str | None = None,- ) -> None:- self.payloads.pop(task_id, None)---class _BlockingCreateTaskStore(_ControllableTaskStore):- def __init__(self, expected_concurrency: int) -> None:- super().__init__()- self.expected_concurrency = expected_concurrency- self.create_release = asyncio.Event()- self.expected_entered = asyncio.Event()- self.create_inflight = 0- self.max_create_inflight = 0-- async def create(self, task: Any) -> None:- self.create_inflight += 1- self.max_create_inflight = max(self.max_create_inflight, self.create_inflight)- if self.create_inflight == self.expected_concurrency:- self.expected_entered.set()- try:- await self.create_release.wait()- await super().create(task)- finally:- self.create_inflight -= 1---class _ThreadBlockingUpdateTaskStore(_ControllableTaskStore):- def __init__(self) -> None:- super().__init__()- self.thread_update_started = threading.Event()- self.thread_update_release = threading.Event()- self._block_next_update = True+async def test_work_index_groups_and_cancels_active_tasks_by_owner():+ from openviking.service.task_work_index import TaskWorkIndex++ index = TaskWorkIndex()+ first = asyncio.create_task(asyncio.Event().wait())+ second = asyncio.create_task(asyncio.Event().wait())+ index.register_active(+ "shared-id", first, account_id="account-a", user_id="user"+ )+ index.register_active(+ "shared-id", second, account_id="account-b", user_id="user"+ )- async def update(self, task: Any) -> None:- if not self._block_next_update:- await super().update(task)- return- self._block_next_update = False- payload = self._payload(task)+ assert index.active_tasks_by_owner() == {+ ("account-a", "user"): {"shared-id"},+ ("account-b", "user"): {"shared-id"},+ }- def blocking_write() -> None:- self.thread_update_started.set()- self.thread_update_release.wait()- self.payloads[task.task_id] = payload+ index.cancel_active(+ "shared-id", account_id="account-a", user_id="user"+ )+ await asyncio.gather(first, return_exceptions=True)+ assert first.cancelled()+ assert not second.done()- self.operation_loops.append(asyncio.get_running_loop())- await asyncio.to_thread(blocking_write)+ second.cancel()+ await asyncio.gather(second, return_exceptions=True)@pytest.mark.asyncio@@ -164,32 +53,12 @@ async def test_owner_loop_dispatcher_runs_foreign_loop_work_on_owner_loop():work_loop = asyncio.get_running_loop()return "done"- def run_from_foreign_loop() -> str:- return asyncio.run(dispatcher.run(work))-- result = await asyncio.to_thread(run_from_foreign_loop)+ result = await asyncio.to_thread(lambda: asyncio.run(dispatcher.run(work)))assert result == "done"assert work_loop is owner_loop-async def test_owner_loop_dispatcher_uses_explicit_owner_loop():- from openviking.service.task_tracker_concurrency import OwnerLoopDispatcher-- owner_loop = asyncio.get_running_loop()- dispatcher = await asyncio.to_thread(OwnerLoopDispatcher, owner_loop)- work_loop: asyncio.AbstractEventLoop | None = None-- async def work() -> None:- nonlocal work_loop- work_loop = asyncio.get_running_loop()-- await asyncio.to_thread(lambda: asyncio.run(dispatcher.run(work)))-- assert work_loop is owner_loop--@pytest.mark.asyncioasync def test_owner_loop_dispatcher_propagates_foreign_cancellation():from openviking.service.task_tracker_concurrency import OwnerLoopDispatcher@@ -228,45 +97,36 @@ async def test_owner_loop_dispatcher_propagates_foreign_cancellation():await asyncio.wait_for(work_cancelled.wait(), timeout=1)-def test_owner_loop_dispatcher_rejects_calls_after_owner_loop_closes():+def test_owner_loop_dispatcher_rejects_closed_or_stopped_owner_loop():from openviking.service.task_tracker_concurrency import OwnerLoopDispatcher- dispatcher = OwnerLoopDispatcher()+ closed_dispatcher = OwnerLoopDispatcher()- async def bind() -> None:- dispatcher.bind_current_loop()+ async def bind_closed() -> None:+ closed_dispatcher.bind_current_loop()- asyncio.run(bind())+ asyncio.run(bind_closed())- async def invoke() -> None:+ async def invoke_closed() -> None:with pytest.raises(RuntimeError, match="owner event loop is closed"):- await dispatcher.run(lambda: asyncio.sleep(0))-- asyncio.run(invoke())-+ await closed_dispatcher.run(lambda: asyncio.sleep(0))-def test_owner_loop_dispatcher_rejects_stopped_owner_loop():- from openviking.service.task_tracker_concurrency import OwnerLoopDispatcher+ asyncio.run(invoke_closed())- dispatcher = OwnerLoopDispatcher()owner_loop = asyncio.new_event_loop()+ stopped_dispatcher = OwnerLoopDispatcher(owner_loop)owner_loop.run_until_complete(asyncio.sleep(0))- owner_loop.run_until_complete(_bind_dispatcher(dispatcher))- async def invoke() -> None:+ async def invoke_stopped() -> None:with pytest.raises(RuntimeError, match="owner event loop is not running"):- await dispatcher.run(lambda: asyncio.sleep(0))+ await stopped_dispatcher.run(lambda: asyncio.sleep(0))try:- asyncio.run(invoke())+ asyncio.run(invoke_stopped())finally:owner_loop.close()-async def _bind_dispatcher(dispatcher: Any) -> None:- dispatcher.bind_current_loop()--def test_owner_loop_dispatcher_concurrent_first_calls_share_winning_loop():from openviking.service.task_tracker_concurrency import OwnerLoopDispatcher@@ -298,97 +158,6 @@ def test_owner_loop_dispatcher_concurrent_first_calls_share_winning_loop():assert results[0] == results[1]-def test_owner_loop_dispatcher_fails_if_loop_stops_during_dispatch():- from openviking.service.task_tracker_concurrency import OwnerLoopDispatcher-- dispatcher = OwnerLoopDispatcher()- owner_loop = asyncio.new_event_loop()- owner_ready = threading.Event()- stop_requested = threading.Event()- let_stop_callback_return = threading.Event()- first_run_stopped = threading.Event()- cleanup_queued_callbacks = threading.Event()-- def owner_thread_main() -> None:- asyncio.set_event_loop(owner_loop)-- def bind() -> None:- dispatcher.bind_current_loop()- owner_ready.set()-- owner_loop.call_soon(bind)- owner_loop.run_forever()- first_run_stopped.set()- cleanup_queued_callbacks.wait()- owner_loop.call_soon(owner_loop.stop)- owner_loop.run_forever()- owner_loop.close()-- owner_thread = threading.Thread(target=owner_thread_main)- owner_thread.start()- assert owner_ready.wait(timeout=1)-- def stop_inside_owner_callback() -> None:- owner_loop.stop()- stop_requested.set()- let_stop_callback_return.wait()-- owner_loop.call_soon_threadsafe(stop_inside_owner_callback)- assert stop_requested.wait(timeout=1)-- async def invoke() -> None:- async def release_callback() -> None:- await asyncio.sleep(0.05)- let_stop_callback_return.set()-- release_task = asyncio.create_task(release_callback())- with pytest.raises(RuntimeError, match="owner event loop stopped"):- await asyncio.wait_for(- dispatcher.run(lambda: asyncio.sleep(0)),- timeout=1,- )- await release_task-- asyncio.run(invoke())- assert first_run_stopped.wait(timeout=1)- cleanup_queued_callbacks.set()- owner_thread.join(timeout=2)- assert not owner_thread.is_alive()---def test_owner_loop_dispatcher_prefers_completed_result_over_simultaneous_stop():- from openviking.service.task_tracker_concurrency import OwnerLoopDispatcher-- dispatcher = OwnerLoopDispatcher()- owner_loop = asyncio.new_event_loop()- owner_ready = threading.Event()-- def owner_thread_main() -> None:- asyncio.set_event_loop(owner_loop)-- def bind() -> None:- dispatcher.bind_current_loop()- owner_ready.set()-- owner_loop.call_soon(bind)- owner_loop.run_forever()- owner_loop.close()-- owner_thread = threading.Thread(target=owner_thread_main)- owner_thread.start()- assert owner_ready.wait(timeout=1)-- async def finish_and_stop() -> str:- owner_loop.stop()- return "committed"-- result = asyncio.run(dispatcher.run(finish_and_stop))-- owner_thread.join(timeout=2)- assert result == "committed"- assert not owner_thread.is_alive()--@pytest.mark.asyncioasync def test_store_io_limiter_bounds_concurrency():from openviking.service.task_tracker_concurrency import StoreIOLimiter@@ -428,7 +197,7 @@ def test_store_io_limiter_rejects_non_positive_limit():@pytest.mark.asyncio@pytest.mark.parametrize("cancel", [False, True])-async def test_run_to_completion_releases_failed_request_and_preserves_cancellation(cancel):+async def test_run_to_completion_releases_request_and_preserves_cancellation(cancel):from openviking.service.task_tracker_concurrency import run_to_completionstarted = asyncio.Event()@@ -442,7 +211,7 @@ async def test_run_to_completion_releases_failed_request_and_preserves_cancellatawait release.wait()raise RuntimeError("late store failure")- async def request():+ async def request() -> weakref.ReferenceType[RequestPayload]:payload = RequestPayload()try:await run_to_completion(fail_later)@@ -451,10 +220,9 @@ async def test_run_to_completion_releases_failed_request_and_preserves_cancellatif not cancel:assert str(exc) == "late store failure"else:- pytest.fail("The operation must propagate its failure or caller cancellation")+ pytest.fail("The operation must propagate failure or caller cancellation")return weakref.ref(payload)- # Finished requests must be released without waiting for cyclic GC.gc_was_enabled = gc.isenabled()gc.disable()try:@@ -503,15 +271,10 @@ async def test_keyed_lock_serializes_same_key_and_allows_different_keys():@pytest.mark.asyncio-async def test_keyed_lock_registry_is_cleaned_after_use_and_cancellation():+async def test_keyed_lock_registry_is_cleaned_after_waiter_cancellation():from openviking.service.task_tracker_concurrency import KeyedAsyncLockPoolpool = KeyedAsyncLockPool[str]()-- async with pool.acquire("completed"):- assert pool.entry_count == 1- assert pool.entry_count == 0-blocker_entered = asyncio.Event()release_blocker = asyncio.Event()@@ -521,13 +284,12 @@ async def test_keyed_lock_registry_is_cleaned_after_use_and_cancellation():await release_blocker.wait()async def waiter() -> None:- await blocker_entered.wait()async with pool.acquire("cancelled"):passblocker_task = asyncio.create_task(blocker())- waiter_task = asyncio.create_task(waiter())await blocker_entered.wait()+ waiter_task = asyncio.create_task(waiter())await asyncio.sleep(0)waiter_task.cancel()@@ -538,272 +300,14 @@ async def test_keyed_lock_registry_is_cleaned_after_use_and_cancellation():await blocker_taskassert pool.entry_count == 0- # Cancellation can arrive after release hands over the slot, but before- # the waiting coroutine resumes. The slot must still become available.- async with pool.acquire("cancelled"):- granted_waiter = asyncio.create_task(waiter())- await asyncio.sleep(0)- granted_waiter.cancel()- with pytest.raises(asyncio.CancelledError):- await granted_waiter- await asyncio.wait_for(waiter(), timeout=1)- assert pool.entry_count == 0-@pytest.mark.asyncio-async def test_task_tracker_different_task_updates_do_not_block_each_other():- from openviking.service.task_tracker import TaskStatus, TaskTracker-- store = _ControllableTaskStore()- tracker = TaskTracker(store=store)- first = await tracker.create("add_resource", account_id="acme", user_id="alice")- second = await tracker.create("add_resource", account_id="acme", user_id="alice")- store.update_started[first.task_id] = asyncio.Event()- store.update_release[first.task_id] = asyncio.Event()-- first_update = asyncio.create_task(tracker.start(first.task_id))- await store.update_started[first.task_id].wait()-- await asyncio.wait_for(tracker.start(second.task_id), timeout=1)- second_snapshot = await tracker.get(second.task_id)- assert second_snapshot is not None- assert second_snapshot.status == TaskStatus.RUNNING-- store.update_release[first.task_id].set()- await first_update---async def test_task_tracker_cache_hit_get_is_not_blocked_by_in_flight_update():- from openviking.service.task_tracker import TaskStatus, TaskTracker-- store = _ControllableTaskStore()- tracker = TaskTracker(store=store)- task = await tracker.create("add_resource", account_id="acme", user_id="alice")- store.update_started[task.task_id] = asyncio.Event()- store.update_release[task.task_id] = asyncio.Event()-- update = asyncio.create_task(tracker.start(task.task_id))- await store.update_started[task.task_id].wait()-- snapshot = await asyncio.wait_for(tracker.get(task.task_id), timeout=0.1)- assert snapshot is not None- assert snapshot.status == TaskStatus.PENDING-- store.update_release[task.task_id].set()- await update---async def test_task_tracker_failed_update_does_not_contaminate_cached_snapshot():- from openviking.service.task_tracker import TaskStatus, TaskTracker-- store = _ControllableTaskStore()- tracker = TaskTracker(store=store)- task = await tracker.create("add_resource", account_id="acme", user_id="alice")- store.update_errors[task.task_id] = RuntimeError("store unavailable")-- with pytest.raises(RuntimeError, match="store unavailable"):- await tracker.start(task.task_id)-- snapshot = await tracker.get(task.task_id)- assert snapshot is not None- assert snapshot.status == TaskStatus.PENDING- assert snapshot.execution_events == task.execution_events- assert store.payloads[task.task_id]["execution_events"] == task.execution_events---async def test_cancelled_thread_write_settles_before_later_same_task_mutation():- from openviking.service.task_tracker import TaskStatus, TaskTracker-- store = _ThreadBlockingUpdateTaskStore()- tracker = TaskTracker(store=store)- task = await tracker.create("add_resource", account_id="acme", user_id="alice")-- starting = asyncio.create_task(tracker.start(task.task_id))- await asyncio.to_thread(store.thread_update_started.wait)- starting.cancel()- completing = asyncio.create_task(tracker.complete(task.task_id, {"done": True}))- await asyncio.sleep(0)- assert not completing.done()-- store.thread_update_release.set()- with pytest.raises(asyncio.CancelledError):- await starting- await completing-- snapshot = await tracker.get(task.task_id)- assert snapshot is not None- assert snapshot.status == TaskStatus.COMPLETED- assert store.payloads[task.task_id]["status"] == TaskStatus.COMPLETED.value- assert store.payloads[task.task_id]["execution_events"] == snapshot.execution_events- assert [event["status"] for event in snapshot.execution_events["items"]] == [- "pending",- "running",- "completed",- ]- assert tracker._task_locks.entry_count == 0- assert tracker._store_io.inflight == 0--- ("outcome", "expected_status"),- [("complete", "completed"), ("fail", "failed")],-)-async def test_cancelled_outcome_finishes_terminal_transition(outcome, expected_status):+async def test_file_store_concurrent_fixed_id_create_is_idempotent():+ from openviking.service.task_store import PersistentTaskStorefrom openviking.service.task_tracker import TaskTracker+ from tests.test_task_tracker import _FakeAgfs- store = _ThreadBlockingUpdateTaskStore()- tracker = TaskTracker(store=store)- task = await tracker.create("add_resource", account_id="acme", user_id="alice")-- if outcome == "complete":- operation = asyncio.create_task(tracker.complete(task.task_id, {"done": True}))- else:- operation = asyncio.create_task(tracker.fail(task.task_id, "failed on purpose"))- await asyncio.to_thread(store.thread_update_started.wait)- operation.cancel()- store.thread_update_release.set()-- with pytest.raises(asyncio.CancelledError):- await operation-- snapshot = await tracker.get(task.task_id)- assert snapshot is not None- assert snapshot.status.value == expected_status- assert store.payloads[task.task_id]["status"] == expected_status- assert tracker._task_locks.entry_count == 0- assert tracker._store_io.inflight == 0---async def test_cancelled_cancel_request_still_cancels_active_work_and_finalizes():- from openviking.service.task_tracker import TaskStatus, TaskTracker-- store = _ThreadBlockingUpdateTaskStore()- tracker = TaskTracker(store=store)- task = await tracker.create("add_resource", account_id="acme", user_id="alice")- work_started = asyncio.Event()- work_cancelled = asyncio.Event()-- async def active_work() -> None:- tracker.register_running_task(task.task_id)- work_started.set()- try:- await asyncio.Future()- except asyncio.CancelledError:- work_cancelled.set()- raise- finally:- await tracker.unregister_running_task(task.task_id)-- worker = asyncio.create_task(active_work())- await work_started.wait()- cancelling = asyncio.create_task(tracker.cancel(task.task_id))- await asyncio.to_thread(store.thread_update_started.wait)- cancelling.cancel()- store.thread_update_release.set()-- with pytest.raises(asyncio.CancelledError):- await cancelling- with pytest.raises(asyncio.CancelledError):- await asyncio.wait_for(worker, timeout=1)- await asyncio.wait_for(work_cancelled.wait(), timeout=1)-- snapshot = await tracker.get(task.task_id)- assert snapshot is not None- assert snapshot.status == TaskStatus.CANCELLED- assert store.payloads[task.task_id]["status"] == TaskStatus.CANCELLED.value- assert tracker._task_locks.entry_count == 0- assert tracker._store_io.inflight == 0---async def test_cancel_request_cancelled_before_store_admission_does_not_cancel_work():- from openviking.service.task_tracker import TaskStatus, TaskTracker-- store = _ControllableTaskStore()- tracker = TaskTracker(store=store, max_concurrent_store_io=1)- blocker = await tracker.create("add_resource", account_id="acme", user_id="alice")- target = await tracker.create("add_resource", account_id="acme", user_id="alice")- await tracker.start(target.task_id)- store.update_started[blocker.task_id] = asyncio.Event()- store.update_release[blocker.task_id] = asyncio.Event()-- blocking_update = asyncio.create_task(tracker.start(blocker.task_id))- await store.update_started[blocker.task_id].wait()-- work_started = asyncio.Event()- work_cancelled = asyncio.Event()-- async def active_work() -> None:- tracker.register_running_task(target.task_id)- work_started.set()- try:- await asyncio.Future()- except asyncio.CancelledError:- work_cancelled.set()- raise-- worker = asyncio.create_task(active_work())- await work_started.wait()- cancelling = asyncio.create_task(tracker.cancel(target.task_id))- for _ in range(100):- if tracker._task_locks.entry_count == 2:- break- await asyncio.sleep(0.01)-- cancelling.cancel()- with pytest.raises(asyncio.CancelledError):- await asyncio.wait_for(cancelling, timeout=1)-- snapshot = await tracker.get(target.task_id)- assert snapshot is not None- assert snapshot.status == TaskStatus.RUNNING- assert not work_cancelled.is_set()- assert not worker.done()-- worker.cancel()- with pytest.raises(asyncio.CancelledError):- await worker- store.update_release[blocker.task_id].set()- await blocking_update---async def test_same_task_late_mutations_cannot_revert_terminal_status():- from openviking.service.task_tracker import TaskStatus, TaskTracker-- store = _ControllableTaskStore()- tracker = TaskTracker(store=store)- task = await tracker.create("add_resource", account_id="acme", user_id="alice")- await tracker.start(task.task_id)- store.update_started[task.task_id] = asyncio.Event()- store.update_release[task.task_id] = asyncio.Event()-- completing = asyncio.create_task(tracker.complete(task.task_id, {"done": True}))- await store.update_started[task.task_id].wait()- late_start = asyncio.create_task(tracker.start(task.task_id, stage="late-start"))- late_stage = asyncio.create_task(tracker.update_stage(task.task_id, "late-stage"))-- store.update_release[task.task_id].set()- await asyncio.gather(completing, late_start, late_stage)-- snapshot = await tracker.get(task.task_id)- assert snapshot is not None- assert snapshot.status == TaskStatus.COMPLETED- assert snapshot.stage == "completed"---async def test_concurrent_create_with_fixed_task_id_is_idempotent():- from openviking.service.task_tracker import TaskTracker-- store = _ControllableTaskStore()- tracker = TaskTracker(store=store)-+ tracker = TaskTracker(PersistentTaskStore(_FakeAgfs()))first, second = await asyncio.gather(tracker.create("add_resource",@@ -820,268 +324,16 @@ async def test_concurrent_create_with_fixed_task_id_is_idempotent():)assert first.task_id == second.task_id == "fixed"- assert store.create_calls == 1- with pytest.raises(ValueError, match="already belongs to another owner"):- await tracker.create(- "add_resource",- task_id="fixed",- account_id="acme",- user_id="bob",- )---async def test_task_tracker_cache_hit_does_not_read_store():- from openviking.service.task_tracker import TaskTracker-- store = _ControllableTaskStore()- tracker = TaskTracker(store=store)- task = await tracker.create("add_resource", account_id="acme", user_id="alice")-- snapshot = await tracker.get(task.task_id, account_id="acme", user_id="alice")-- assert snapshot is not None- assert snapshot.task_id == task.task_id- assert store.get_calls == 0---async def test_task_tracker_list_does_not_replace_newer_cache_with_stale_store_data():- from openviking.service.task_tracker import TaskStatus, TaskTracker-- store = _ControllableTaskStore()- tracker = TaskTracker(store=store)- task = await tracker.create("add_resource", account_id="acme", user_id="alice")- stale_payload = deepcopy(store.payloads[task.task_id])-- await tracker.start(task.task_id)- store.payloads[task.task_id] = stale_payload-- snapshots = await tracker.list_tasks(account_id="acme", user_id="alice")-- assert len(snapshots) == 1- assert snapshots[0].status == TaskStatus.RUNNING---async def test_blocked_store_list_does_not_block_cached_get():- from openviking.service.task_tracker import TaskTracker-- store = _ControllableTaskStore()- tracker = TaskTracker(store=store)- task = await tracker.create("add_resource", account_id="acme", user_id="alice")- store.list_started = asyncio.Event()- store.list_release = asyncio.Event()-- listing = asyncio.create_task(tracker.list_tasks(account_id="acme", user_id="alice"))- await store.list_started.wait()-- snapshot = await asyncio.wait_for(tracker.get(task.task_id), timeout=0.1)- assert snapshot is not None- assert snapshot.task_id == task.task_id-- store.list_release.set()- await listing---async def test_stale_in_flight_list_does_not_overwrite_completed_cache():- from openviking.service.task_tracker import TaskStatus, TaskTracker-- store = _ControllableTaskStore()- tracker = TaskTracker(store=store)- task = await tracker.create("add_resource", account_id="acme", user_id="alice")- await tracker.start(task.task_id)- store.list_started = asyncio.Event()- store.list_release = asyncio.Event()-- listing = asyncio.create_task(tracker.list_tasks(account_id="acme", user_id="alice"))- await store.list_started.wait()- await tracker.complete(task.task_id, {"root_uri": "viking://resources/done"})-- store.list_release.set()- await listing-- snapshot = await tracker.get(task.task_id)- assert snapshot is not None- assert snapshot.status == TaskStatus.COMPLETED---async def test_clock_rollback_cannot_make_stale_store_snapshot_look_newer(monkeypatch):- from openviking.service.task_tracker import TaskStatus, TaskTracker-- store = _ControllableTaskStore()- tracker = TaskTracker(store=store)- task = await tracker.create("add_resource", account_id="acme", user_id="alice")- await tracker.start(task.task_id)- stale_payload = deepcopy(store.payloads[task.task_id])- monkeypatch.setattr("openviking.service.task_tracker.time.time", lambda: 0.0)-- await tracker.complete(task.task_id, {"done": True})- store.payloads[task.task_id] = stale_payload- await tracker.list_tasks(account_id="acme", user_id="alice")-- snapshot = await tracker.get(task.task_id)- assert snapshot is not None- assert snapshot.status == TaskStatus.COMPLETED- assert snapshot.updated_at > stale_payload["updated_at"]---async def test_task_tracker_store_io_limit_allows_bounded_parallelism():- from openviking.service.task_tracker import TaskTracker-- store = _BlockingCreateTaskStore(expected_concurrency=3)- tracker = TaskTracker(store=store, max_concurrent_store_io=3)-- creates = [- asyncio.create_task(tracker.create("add_resource", account_id="acme", user_id="alice"))- for _ in range(10)- ]- await asyncio.wait_for(store.expected_entered.wait(), timeout=1)- await asyncio.sleep(0)-- assert store.max_create_inflight == 3- store.create_release.set()- await asyncio.gather(*creates)- assert tracker._store_io.max_observed_inflight == 3- assert tracker._store_io.inflight == 0---async def test_cancelled_create_waiting_for_store_slot_has_no_side_effect():- from openviking.service.task_tracker import TaskTracker-- store = _BlockingCreateTaskStore(expected_concurrency=1)- tracker = TaskTracker(store=store, max_concurrent_store_io=1)-- first = asyncio.create_task(tracker.create("add_resource", account_id="acme", user_id="alice"))- await asyncio.wait_for(store.expected_entered.wait(), timeout=1)-- waiting = asyncio.create_task(- tracker.create("add_resource", account_id="acme", user_id="alice")- )- for _ in range(100):- if tracker._task_locks.entry_count == 2:- break- await asyncio.sleep(0.01)- assert tracker._task_locks.entry_count == 2-- waiting.cancel()- await asyncio.sleep(0.05)- cancelled_before_release = waiting.done()- store.create_release.set()- with pytest.raises(asyncio.CancelledError):- await waiting-- created = await first-- assert cancelled_before_release- assert store.create_calls == 1- assert set(store.payloads) == {created.task_id}- assert tracker._task_locks.entry_count == 0- assert tracker._store_io.inflight == 0---async def test_cancelled_create_waiting_for_task_lock_finishes_before_lock_release():- from openviking.service.task_tracker import TaskTracker-- store = _BlockingCreateTaskStore(expected_concurrency=1)- tracker = TaskTracker(store=store)- create_args = {- "task_type": "add_resource",- "task_id": "fixed",- "account_id": "acme",- "user_id": "alice",- }-- first = asyncio.create_task(tracker.create(**create_args))- await asyncio.wait_for(store.expected_entered.wait(), timeout=1)- waiting = asyncio.create_task(tracker.create(**create_args))- await asyncio.sleep(0)-- waiting.cancel()- await asyncio.sleep(0.05)- cancelled_before_release = waiting.done()- store.create_release.set()- with pytest.raises(asyncio.CancelledError):- await waiting-- await first- assert cancelled_before_release- assert store.create_calls == 1---async def test_task_tracker_public_mutation_stays_on_calling_loop():- from openviking.service.task_tracker import TaskStatus, TaskTracker-- store = _ControllableTaskStore()- tracker = TaskTracker(store=store)- request_loop = asyncio.get_running_loop()- task = await tracker.create("add_resource", account_id="acme", user_id="alice")-- await asyncio.to_thread(lambda: asyncio.run(tracker.start(task.task_id)))-- with tracker._lock:- tracker._tasks.pop(task.task_id)- loaded = await asyncio.to_thread(- lambda: asyncio.run(tracker.get(task.task_id, account_id="acme", user_id="alice"))- )-- snapshot = await tracker.get(task.task_id)- assert loaded is not None- assert snapshot is not None- assert snapshot.status == TaskStatus.RUNNING- assert store.operation_loops[0] is request_loop- assert all(loop is not request_loop for loop in store.operation_loops[1:])---async def test_cancelling_foreign_mutation_cleans_task_lock_and_store_slot():- from openviking.service.task_tracker import TaskStatus, TaskTracker-- store = _ThreadBlockingUpdateTaskStore()- tracker = TaskTracker(store=store)- task = await tracker.create("add_resource", account_id="acme", user_id="alice")- cancel_foreign = threading.Event()-- def run_from_foreign_loop() -> None:- async def invoke() -> None:- mutation = asyncio.create_task(tracker.start(task.task_id))- await asyncio.to_thread(cancel_foreign.wait)- mutation.cancel()- with pytest.raises(asyncio.CancelledError):- await mutation-- asyncio.run(invoke())-- foreign_call = asyncio.create_task(asyncio.to_thread(run_from_foreign_loop))- assert await asyncio.to_thread(store.thread_update_started.wait, 3)- cancel_foreign.set()- await asyncio.sleep(0.05)- returned_before_store_settled = foreign_call.done()- store.thread_update_release.set()- await foreign_call-- assert not returned_before_store_settled- snapshot = await tracker.get(task.task_id)- assert snapshot is not None- assert snapshot.status == TaskStatus.RUNNING- assert tracker._task_locks.entry_count == 0- assert tracker._store_io.inflight == 0- assert store.payloads[task.task_id]["status"] == TaskStatus.RUNNING.value+ assert len(await tracker.list_tasks(account_id="acme", user_id="alice")) == 1@pytest.mark.asyncio-async def test_create_if_no_running_is_unique_per_business_key():+async def test_file_store_create_if_no_running_is_unique_per_business_key():+ from openviking.service.task_store import PersistentTaskStorefrom openviking.service.task_tracker import TaskTracker+ from tests.test_task_tracker import _FakeAgfs- store = _ControllableTaskStore()- tracker = TaskTracker(store=store)-+ tracker = TaskTracker(PersistentTaskStore(_FakeAgfs()))results = await asyncio.gather(*[tracker.create_if_no_running(@@ -1094,67 +346,4 @@ async def test_create_if_no_running_is_unique_per_business_key():])- created = [task for task in results if task is not None]- assert len(created) == 1- assert tracker._business_locks.entry_count == 0- assert tracker._task_locks.entry_count == 0---async def test_create_if_no_running_uses_independent_business_keys():- from openviking.service.task_tracker import TaskTracker-- store = _BlockingCreateTaskStore(expected_concurrency=2)- tracker = TaskTracker(store=store)-- first_call = asyncio.create_task(- tracker.create_if_no_running(- "add_resource",- "viking://resources/one",- account_id="acme",- user_id="alice",- )- )- second_call = asyncio.create_task(- tracker.create_if_no_running(- "add_resource",- "viking://resources/two",- account_id="acme",- user_id="alice",- )- )- await asyncio.wait_for(store.expected_entered.wait(), timeout=1)- assert store.max_create_inflight == 2- store.create_release.set()- first, second = await asyncio.gather(first_call, second_call)-- assert first is not None- assert second is not None- assert first.task_id != second.task_id---async def test_task_and_business_lock_registries_do_not_grow_with_completed_calls():- from openviking.service.task_tracker import TaskTracker-- store = _ControllableTaskStore()- tracker = TaskTracker(store=store)-- tasks = await asyncio.gather(- *[tracker.create("add_resource", account_id="acme", user_id="alice") for _ in range(100)]- )- await asyncio.gather(*(tracker.start(task.task_id) for task in tasks))- await asyncio.gather(- *[- tracker.create_if_no_running(- "reindex",- f"viking://resources/{index}",- account_id="acme",- user_id="alice",- )- for index in range(20)- ]- )-- assert tracker._task_locks.entry_count == 0- assert tracker._business_locks.entry_count == 0+ assert sum(task is not None for task in results) == 1diff --git a/tests/unit/session/test_session_commit_resume.py b/tests/unit/session/test_session_commit_resume.pyindex 90adac7c7..500ed3cd0 100644--- a/tests/unit/session/test_session_commit_resume.py+++ b/tests/unit/session/test_session_commit_resume.py@@ -11,10 +11,12 @@ from unittest.mock import AsyncMockimport pytestfrom openviking.message import Message, TextPart, ToolPart+from openviking.service.task_store import PersistentTaskStorefrom openviking.service.task_tracker import TaskStatus, TaskTracker, set_task_trackerfrom openviking.session.memory_policy import MemoryPolicyfrom openviking.session.session import Sessionfrom openviking.storage.queuefs.session_commit_msg import SessionCommitMsg+from tests.test_task_tracker import _FakeAgfs@pytest.mark.parametrize("enabled", [False, True])@@ -49,24 +51,9 @@ def test_legacy_queued_work_keeps_old_default_without_migrating_user_policy():assert message.memory_policy["memory_types"] == policy["memory_types"]-class _TaskStore:+class _TaskStore(PersistentTaskStore):def __init__(self):- self.tasks = {}-- async def create(self, task):- self.tasks[task.task_id] = task-- async def update(self, task):- self.tasks[task.task_id] = task-- async def get(self, task_id, *, account_id=None, user_id=None):- return None-- async def list(self, account_id, *, user_id=None):- return []-- async def delete(self, task_id, *, account_id, user_id=None):- self.tasks.pop(task_id, None)+ super().__init__(_FakeAgfs())class _MemoryVikingFS: