Self-evolving Context Database for AI Agents. Unify Agent Memory, Knowledge RAG and Skills.
Python agent-contextagent-memoryagent-pluginsagentic-ragcontext-databasedsh-pluginknowledge-basememorymemory-managementself-evolvingvector-database
fix(studio): distinguish task processing time from waiting time (#5207)
yufengee6cc6ce1bae
diff --git a/openviking/service/task_processing_time.py b/openviking/service/task_processing_time.pynew file mode 100644index 000000000..e52375ad8--- /dev/null+++ b/openviking/service/task_processing_time.py@@ -0,0 +1,57 @@+# Copyright (c) 2026 Beijing Volcano Engine Technology Co., Ltd.+# SPDX-License-Identifier: AGPL-3.0+"""Union of worker execution intervals; queue waits do not advance the clock.++Only tasks created in this process have complete coverage. After a restart we+leave active tasks unmeasured rather than inventing time across the downtime.+"""++import time+from contextlib import contextmanager+from contextvars import ContextVar+from dataclasses import dataclass, field+from typing import TYPE_CHECKING, Any, Iterator++if TYPE_CHECKING:+ from openviking.service.task_work_index import TaskWorkIndex++# Inherited by shielded child coroutines; DAG nodes install their own owner.+processing_owner: ContextVar[tuple["TaskWorkIndex", str, Any] | None] = ContextVar(+ "processing_owner", default=None+)+++@contextmanager+def pause_task_processing() -> Iterator[None]:+ owner = processing_owner.get()+ if owner is None:+ yield+ else:+ index, task_id, worker = owner+ with index.pause_processing(task_id, worker=worker):+ yield+++@dataclass+class ProcessingClock:+ elapsed: float = 0.0+ started: float | None = None+ workers: set[Any] = field(default_factory=set)+ observed: bool = False++ def enter(self, worker: Any) -> None:+ if not self.workers:+ self.started = time.monotonic()+ self.workers.add(worker)+ self.observed = True++ def leave(self, worker: Any) -> None:+ self.workers.discard(worker)+ if not self.workers and self.started is not None:+ self.elapsed += time.monotonic() - self.started+ self.started = None++ def seconds(self) -> float | None:+ if not self.observed:+ return None+ return self.elapsed + (time.monotonic() - self.started if self.started is not None else 0.0)diff --git a/openviking/service/task_store.py b/openviking/service/task_store.pyindex 7920d9139..bcb8b4e20 100644--- a/openviking/service/task_store.py+++ b/openviking/service/task_store.py@@ -264,6 +264,7 @@ def _task_to_payload(task: Any) -> Dict[str, Any]:"status": status.value if hasattr(status, "value") else status,"created_at": task.created_at,"updated_at": task.updated_at,+ "processing_seconds": task.processing_seconds,"resource_id": task.resource_id,"account_id": task.account_id,"user_id": task.user_id,diff --git a/openviking/service/task_tracker.py b/openviking/service/task_tracker.pyindex 901f1e14c..252a8a388 100644--- a/openviking/service/task_tracker.py+++ b/openviking/service/task_tracker.py@@ -30,6 +30,7 @@ from openviking.service.task_events import (TaskEventHistory,append_task_event,)+from openviking.service.task_processing_time import pause_task_processingfrom openviking.service.task_store import TaskStorefrom openviking.service.task_tracker_concurrency import (KeyedAsyncLockPool,@@ -88,6 +89,7 @@ class TaskRecord:error: Optional[str] = Noneauth: Dict[str, Any] = field(default_factory=dict, repr=False)execution_events: Optional[TaskEventHistory] = None+ processing_seconds: Optional[float] = None_extra_fields: Dict[str, Any] = field(default_factory=dict, init=False, repr=False, compare=False)@@ -104,6 +106,7 @@ class TaskRecord:"status": self.status.value,"created_at": self.created_at,"updated_at": self.updated_at,+ "processing_seconds": self.processing_seconds,"resource_id": self.resource_id,"meta": _sanitize_task_result(public_meta),"stage": self.stage,@@ -287,12 +290,17 @@ class TaskTracker:for task_id in expired_ids:evicted_count += await self._delete_expired_task(task_id, now)+ capacity_evicted = []with self._lock:if len(self._tasks) > self.MAX_TASKS:sorted_tasks = sorted(self._tasks.items(), key=lambda x: x[1].created_at)excess = len(self._tasks) - self.MAX_TASKSfor tid, _ in sorted_tasks[:excess]:self._tasks.pop(tid, None)+ capacity_evicted.append(tid)++ for tid in capacity_evicted:+ self._work_index.forget_processing(tid)if evicted_count:logger.debug("[TaskTracker] Evicted %d expired tasks", evicted_count)@@ -337,6 +345,7 @@ class TaskTracker:with self._lock:self._tasks.pop(task_id, None)+ self._work_index.forget_processing(task_id)return True@staticmethod@@ -811,6 +820,7 @@ class TaskTracker:del self._tasks[task_id]for task_id in task_ids:self._work_index.clear_failure(task_id)+ self._work_index.forget_processing(task_id)async def delete_user_tasks(self, account_id: str, user_id: str) -> int:"""Delete terminal task records for one user from storage and cache."""@@ -850,6 +860,7 @@ class TaskTracker:)with self._lock:self._tasks.pop(task.task_id, None)+ self._work_index.forget_processing(task.task_id)self._work_index.clear_failure(task.task_id)deleted += 1return deleted@@ -866,8 +877,9 @@ class TaskTracker:account_id=task.account_id,user_id=task.user_id,)- while self._work_index.has_work(task_id, exclude_work_id=current_work_id):- await asyncio.sleep(0.05)+ with pause_task_processing():+ while self._work_index.has_work(task_id, exclude_work_id=current_work_id):+ await asyncio.sleep(0.05)async def record_event(self,@@ -975,6 +987,7 @@ class TaskTracker:)with self._lock:self._tasks.pop(task.task_id, None)+ self._work_index.forget_processing(task.task_id)return Trueasync def list_page(@@ -1004,7 +1017,7 @@ class TaskTracker:io_limiter=self._store_io,**filters,)- records.extend(self._record_from_payload(record) for record in page)+ records.extend(self._copy(self._record_from_payload(record)) for record in page)if include_cached:records.extend(self._copy(t) for t in self._cache_snapshot())visible = {@@ -1119,6 +1132,9 @@ class TaskTracker:# Build history before the same physical write as the state transition.if operation == "update" and previous is None:raise ValueError("Task updates require the previous snapshot")+ # Persist the final measured union; restored active tasks have no complete clock.+ if previous is None or previous.status in _ACTIVE_STATUSES:+ task.processing_seconds = self._work_index.processing_seconds(task.task_id)kinds = []if previous is None:kinds.append("created")@@ -1151,6 +1167,10 @@ class TaskTracker:else:await self._store.update(task)self._publish_task(task)+ if operation == "create":+ self._work_index.init_processing(task.task_id)+ elif task.status in _TERMINAL_STATUSES:+ self._work_index.forget_processing(task.task_id)committed = True# Semaphore/lock waits remain cancellable. Once the store operation is@@ -1194,10 +1214,11 @@ class TaskTracker:def _next_updated_at(task: TaskRecord) -> float:return max(time.time(), math.nextafter(task.updated_at, math.inf))- @staticmethod- def _copy(task: TaskRecord) -> TaskRecord:+ def _copy(self, task: TaskRecord) -> TaskRecord:"""Return a defensive copy of a TaskRecord."""copied = deepcopy(task)+ if task.status in _ACTIVE_STATUSES:+ copied.processing_seconds = self._work_index.processing_seconds(task.task_id)copied.meta = _sanitize_task_result(copied.meta)copied.result = _sanitize_task_result(copied.result)copied.auth = {}diff --git a/openviking/service/task_work_index.py b/openviking/service/task_work_index.pyindex b9a520974..ae9ee9c90 100644--- a/openviking/service/task_work_index.py+++ b/openviking/service/task_work_index.py@@ -18,6 +18,8 @@ from dataclasses import dataclassfrom typing import Any, Awaitable, Callable, Dict, Iterable, Iterator, Mapping, Optionalfrom uuid import uuid4+from openviking.service.task_processing_time import ProcessingClock, processing_owner+TASK_WORK_ID_FIELD = "_task_work_id"@@ -158,6 +160,7 @@ class TaskWorkIndex:self._work: Dict[str, set[tuple[str, str]]] = {}self._active: Dict[str, set[asyncio.Task[Any]]] = {}self._failures: Dict[str, str] = {}+ self._processing: Dict[str, ProcessingClock] = {}self._finalize_before_ack: Optional[Callable[[QueueTaskMetadata], Awaitable[None]]] = Noneself._is_cancellation_requested: Optional[Callable[[str], bool]] = None@@ -293,6 +296,52 @@ class TaskWorkIndex:with self._lock:self._failures.pop(task_id, None)+ def init_processing(self, task_id: str) -> None:+ with self._lock:+ self._processing.setdefault(task_id, ProcessingClock())++ def processing_seconds(self, task_id: str) -> Optional[float]:+ with self._lock:+ clock = self._processing.get(task_id)+ return clock.seconds() if clock else None++ def forget_processing(self, task_id: str) -> None:+ with self._lock:+ self._processing.pop(task_id, None)++ @contextmanager+ def pause_processing(self, task_id: str, *, worker: Any = None) -> Iterator[None]:+ worker = worker if worker is not None else asyncio.current_task()+ with self._lock:+ clock = self._processing.get(task_id)+ was_processing = clock is not None and worker in clock.workers+ if was_processing:+ clock.leave(worker)+ try:+ yield+ finally:+ with self._lock:+ if was_processing and self._processing.get(task_id) is clock:+ clock.enter(worker)++ @contextmanager+ def measure_processing(self, task_id: str) -> Iterator[None]:+ # Timing ownership is separate from cancellation ownership: a shared DAG+ # worker must never become cancellable on behalf of one queued task.+ worker = object()+ token = processing_owner.set((self, task_id, worker))+ with self._lock:+ clock = self._processing.get(task_id)+ if clock is not None:+ clock.enter(worker)+ try:+ yield+ finally:+ with self._lock:+ if clock is not None:+ clock.leave(worker)+ processing_owner.reset(token)+def register_active(self,task_id: str,@@ -302,11 +351,20 @@ class TaskWorkIndex:cancelled = self.cancellation_requested(task_id)if not cancelled:self._active.setdefault(task_id, set()).add(active_task)+ clock = self._processing.get(task_id)+ if clock is not None:+ clock.enter(active_task)+ processing_owner.set((self, task_id, active_task))if cancelled:active_task.get_loop().call_soon(active_task.cancel)def unregister_active(self, task_id: str, active_task: asyncio.Task[Any]) -> None:+ if processing_owner.get() == (self, task_id, active_task):+ processing_owner.set(None)with self._lock:+ clock = self._processing.get(task_id)+ if clock is not None:+ clock.leave(active_task)entries = self._active.get(task_id)if entries is not None:entries.discard(active_task)diff --git a/openviking/storage/queuefs/semantic_dag.py b/openviking/storage/queuefs/semantic_dag.pyindex 9f9e74e28..a470caa83 100644--- a/openviking/storage/queuefs/semantic_dag.py+++ b/openviking/storage/queuefs/semantic_dag.py@@ -5,6 +5,7 @@import asyncioimport reimport threading+from contextlib import nullcontextfrom dataclasses import dataclass, fieldfrom typing import Any, ClassVar, Dict, List, Optional, Setfrom weakref import WeakKeyDictionary@@ -12,6 +13,7 @@ from weakref import WeakKeyDictionaryfrom openviking.core.namespace import classify_urifrom openviking.parse.parsers.media import get_media_typefrom openviking.server.identity import RequestContext+from openviking.service.task_processing_time import pause_task_processing, processing_ownerfrom openviking.service.task_tracker_concurrency import run_to_completionfrom openviking.service.task_work_index import (bind_task_context,@@ -207,6 +209,7 @@ class SemanticDagExecutor:self._aggregate_directory = aggregate_directoryself._copy_source_uri = copy_source_uriself._task_context = get_task_context()+ self._processing_index = Noneself._telemetry = get_current_telemetry()self._stale = Falseself._changed_paths = {@@ -254,31 +257,36 @@ class SemanticDagExecutor:self._root_done = asyncio.Event()self._scheduler = get_semantic_node_scheduler(self._node_concurrency)- try:- self._register_active()- self._schedule_dir(root_uri, parent_uri=None)- await self._root_done.wait()- if self._failure:- raise self._failure- except BaseException:- self._closed = True- if self._context_type == "skill":- for task in self._skill_node_tasks:- task.cancel()- await run_to_completion(- lambda: asyncio.gather(*self._skill_node_tasks, return_exceptions=True)- )- await self._active_work_idle.wait()- raise- finally:- self._closed = True+ owner = processing_owner.get()+ if owner is not None and self._task_context is not None:+ if owner[1] == self._task_context.task_id:+ self._processing_index = owner[0]+ with pause_task_processing():try:+ self._register_active()+ self._schedule_dir(root_uri, parent_uri=None)+ await self._root_done.wait()+ if self._failure:+ raise self._failure+ except BaseException:+ self._closed = Trueif self._context_type == "skill":- await run_to_completion(self._active_work_idle.wait)- else:- await self._active_work_idle.wait()+ for task in self._skill_node_tasks:+ task.cancel()+ await run_to_completion(+ lambda: asyncio.gather(*self._skill_node_tasks, return_exceptions=True)+ )+ await self._active_work_idle.wait()+ raisefinally:- self._unregister_active()+ self._closed = True+ try:+ if self._context_type == "skill":+ await run_to_completion(self._active_work_idle.wait)+ else:+ await self._active_work_idle.wait()+ finally:+ self._unregister_active()def _schedule_work(self, work: DagWork) -> None:if self._closed:@@ -384,8 +392,19 @@ class SemanticDagExecutor:if self._task_context is not Noneelse detach_task_context())- with bind_telemetry(self._telemetry), task_context:- await self._run_work_bound(work)+ timing = (+ self._processing_index.measure_processing(self._task_context.task_id)+ if self._processing_index is not None and self._task_context is not None+ else nullcontext()+ )+ # Shared workers inherit their creator's ContextVars. Never let an+ # unrelated node pause the creator's clock when it has no timing owner.+ token = processing_owner.set(None)+ try:+ with bind_telemetry(self._telemetry), task_context, timing:+ await self._run_work_bound(work)+ finally:+ processing_owner.reset(token)async def _await_write(self, operation):if self._context_type == "skill":diff --git a/openviking/storage/queuefs/semantic_processor.py b/openviking/storage/queuefs/semantic_processor.pyindex 7fb73d552..b8b1d6cf0 100644--- a/openviking/storage/queuefs/semantic_processor.py+++ b/openviking/storage/queuefs/semantic_processor.py@@ -35,6 +35,7 @@ from openviking.parse.parsers.media.utils import (from openviking.prompts import render_promptfrom openviking.pyagfs.exceptions import AGFSNotADirectoryErrorfrom openviking.server.identity import RequestContext, Role+from openviking.service.task_processing_time import pause_task_processingfrom openviking.service.task_tracker_concurrency import run_to_completionfrom openviking.service.task_work_index import detach_task_contextfrom openviking.storage.abstract_overview import (@@ -234,7 +235,8 @@ class SemanticProcessor(DequeueHandlerBase):# Throttle to prevent re-enqueue storm during OPEN windowwait = self._circuit_breaker.retry_afterif wait > 0:- await asyncio.sleep(wait)+ with pause_task_processing():+ await asyncio.sleep(wait)queue_manager = get_queue_manager()if queue_manager is not None:diff --git a/openviking/telemetry/request_wait_tracker.py b/openviking/telemetry/request_wait_tracker.pyindex fb1cd5ecc..1a27bd35e 100644--- a/openviking/telemetry/request_wait_tracker.py+++ b/openviking/telemetry/request_wait_tracker.py@@ -10,6 +10,8 @@ import timefrom dataclasses import dataclass, fieldfrom typing import Callable, Dict, List, Optional, Set+from openviking.service.task_processing_time import pause_task_processing+@dataclassclass _RequestWaitState:@@ -199,13 +201,14 @@ class RequestWaitTracker:) -> None:if not telemetry_id:return- start = time.time()- while True:- if self.is_complete(telemetry_id):- return- if timeout is not None and (time.time() - start) > timeout:- raise TimeoutError(f"Request processing not complete after {timeout}s")- await asyncio.sleep(poll_interval)+ with pause_task_processing():+ start = time.time()+ while True:+ if self.is_complete(telemetry_id):+ return+ if timeout is not None and (time.time() - start) > timeout:+ raise TimeoutError(f"Request processing not complete after {timeout}s")+ await asyncio.sleep(poll_interval)async def wait_for_embeddings(self,@@ -217,17 +220,18 @@ class RequestWaitTracker:"""Drain embeddings while the producing semantic root remains pending."""if not telemetry_id:return- while True:- with self._lock:- state = self._states.get(telemetry_id)- if state is None or not state.pending_embedding_roots:- return- if stop_waiting is not None and stop_waiting():- # Shutdown has drained the embedding consumer's active writes.- # Leave this semantic delivery unacked for recovery rather than- # waiting forever for embeddings still in the persistent queue.- raise asyncio.CancelledError("Embedding worker stopped with queued work")- await asyncio.sleep(poll_interval)+ with pause_task_processing():+ while True:+ with self._lock:+ state = self._states.get(telemetry_id)+ if state is None or not state.pending_embedding_roots:+ return+ if stop_waiting is not None and stop_waiting():+ # Shutdown has drained the embedding consumer's active writes.+ # Leave this semantic delivery unacked for recovery rather than+ # waiting forever for embeddings still in the persistent queue.+ raise asyncio.CancelledError("Embedding worker stopped with queued work")+ await asyncio.sleep(poll_interval)def build_queue_status(self, telemetry_id: str) -> Dict[str, Dict[str, object]]:with self._lock:diff --git a/tests/service/test_task_processing_time.py b/tests/service/test_task_processing_time.pynew file mode 100644index 000000000..31d32531d--- /dev/null+++ b/tests/service/test_task_processing_time.py@@ -0,0 +1,214 @@+# Copyright (c) 2026 Beijing Volcano Engine Technology Co., Ltd.+# SPDX-License-Identifier: AGPL-3.0+"""Processing time excludes idle gaps and unions overlapping workers."""++import asyncio+from types import SimpleNamespace++import pytest++from openviking.service.task_processing_time import ProcessingClock+from openviking.service.task_tracker_concurrency import run_to_completion+from openviking.service.task_work_index import TaskWorkIndex, bind_task_context+from openviking.storage.queuefs.semantic_dag import SemanticDagExecutor, SemanticNodeScheduler+from openviking.telemetry.request_wait_tracker import RequestWaitTracker+++def test_processing_clock_unions_workers_and_excludes_idle_time(monkeypatch):+ now = [0.0]+ monkeypatch.setattr("openviking.service.task_processing_time.time.monotonic", lambda: now[0])+ clock = ProcessingClock()+ assert clock.seconds() is None+ now[0] = 10 # Initial queue wait.+ clock.enter("parse")+ now[0] = 12+ clock.enter("embedding")+ now[0] = 15+ clock.leave("parse")+ now[0] = 18+ clock.leave("embedding")+ assert clock.seconds() == 8+ now[0] = 40 # No worker: waiting for another queue.+ assert clock.seconds() == 8+ clock.enter("semantic")+ now[0] = 43+ assert clock.seconds() == 11+ clock.leave("semantic")+ now[0] = 100+ assert clock.seconds() == 11+++async def test_saturated_dag_scheduler_counts_only_dispatched_nodes(monkeypatch):+ now = [0.0]+ monkeypatch.setattr(+ "openviking.service.task_processing_time.time",+ SimpleNamespace(monotonic=lambda: now[0]),+ )+ index = TaskWorkIndex()+ scheduler = SemanticNodeScheduler(max_workers=1)+ monkeypatch.setattr(+ "openviking.storage.queuefs.semantic_dag.get_semantic_node_scheduler",+ lambda _: scheduler,+ )+ monkeypatch.setattr("openviking.storage.queuefs.semantic_dag.get_viking_fs", lambda: None)+ entered = {name: asyncio.Event() for name in ["a", "b"]}+ release = {name: asyncio.Event() for name in ["a", "b"]}+ queued = asyncio.Event()+ executors = {}+ for name in ["a", "b"]:+ index.init_processing(name)+ with bind_task_context(name, "account", "user"):+ executor = SemanticDagExecutor(None, "resource", 1, None)+ executors[name] = executor++ async def node(_work, name=name, executor=executor):+ entered[name].set()+ await release[name].wait()+ executor._root_done.set()++ monkeypatch.setattr(executor, "_run_work_bound", node)+ schedule_b = executors["b"]._schedule_dir++ def queue_b(*args, **kwargs):+ schedule_b(*args, **kwargs)+ queued.set()++ monkeypatch.setattr(executors["b"], "_schedule_dir", queue_b)++ async def coordinate(name):+ with index.measure_processing(name):+ await executors[name].run("viking://resources/" + name)++ first = asyncio.create_task(coordinate("a"))+ await entered["a"].wait()+ second = asyncio.create_task(coordinate("b"))+ await queued.wait()+ now[0] = 20+ assert index.processing_seconds("a") == 20+ assert index.processing_seconds("b") == 0 # Still queued behind A.+ release["a"].set()+ await entered["b"].wait()+ now[0] = 25+ assert index.processing_seconds("b") == 5+ release["b"].set()+ await asyncio.gather(first, second)+ assert index.processing_seconds("b") == 5+ for worker in list(scheduler._workers):+ worker.cancel()+ await asyncio.gather(*scheduler._workers, return_exceptions=True)++[email protected]("method", ["wait_for_request", "wait_for_embeddings"])+async def test_downstream_wait_pauses_inherited_owner_in_shielded_child(monkeypatch, method):+ now = [0.0]+ monkeypatch.setattr(+ "openviking.service.task_processing_time.time",+ SimpleNamespace(monotonic=lambda: now[0]),+ )+ index = TaskWorkIndex()+ index.init_processing("task")+ tracker = RequestWaitTracker()+ request = "processing-time-" + method+ tracker.register_request(request)+ tracker.register_embedding_root(request, "embedding")+ with index.measure_processing("task"):+ now[0] = 3+ waiter = asyncio.create_task(+ run_to_completion(lambda: getattr(tracker, method)(request, poll_interval=0.001))+ )+ # Let run_to_completion and its shielded child enter the wait loop.+ await asyncio.sleep(0.01)+ now[0] = 20+ assert index.processing_seconds("task") == 3+ tracker.mark_embedding_done(request, "embedding")+ await waiter+ now[0] = 22+ assert index.processing_seconds("task") == 5+ tracker.cleanup(request)+++async def test_nested_wait_and_cancellation_restore_processing_owner(monkeypatch):+ from openviking.service.task_processing_time import pause_task_processing, processing_owner++ now = [0.0]+ monkeypatch.setattr(+ "openviking.service.task_processing_time.time",+ SimpleNamespace(monotonic=lambda: now[0]),+ )+ index = TaskWorkIndex()+ index.init_processing("task")+ entered = asyncio.Event()++ async def work():+ with index.measure_processing("task"):+ now[0] = 2+ try:+ with pause_task_processing():+ with pause_task_processing():+ entered.set()+ await asyncio.Event().wait()+ except asyncio.CancelledError:+ now[0] += 3 # Cancellation cleanup resumes processing.+ assert processing_owner.get() is None++ task = asyncio.create_task(work())+ await entered.wait()+ now[0] = 20+ assert index.processing_seconds("task") == 2+ task.cancel()+ await task+ assert index.processing_seconds("task") == 5++[email protected]("custom_enqueue", [False, True])+async def test_semantic_retry_cooldown_excludes_wait_but_counts_enqueue(monkeypatch, custom_enqueue):+ from openviking.storage.queuefs.semantic_processor import SemanticProcessor++ now = [0.0]+ monkeypatch.setattr(+ "openviking.service.task_processing_time.time",+ SimpleNamespace(monotonic=lambda: now[0]),+ )+ index = TaskWorkIndex()+ index.init_processing("task")+ message = SimpleNamespace(uri="viking://resources/retry-test")+ enqueued = []++ async def sleep(delay):+ assert delay == 30+ now[0] += delay+ assert index.processing_seconds("task") == 3++ async def enqueue(msg):+ assert msg is message+ now[0] += 2+ enqueued.append(msg)++ queue = SimpleNamespace(enqueue=enqueue)++ async def custom(queue_arg, msg):+ assert queue_arg is queue+ await enqueue(msg)++ monkeypatch.setattr(asyncio, "sleep", sleep)+ monkeypatch.setattr(+ "openviking.storage.queuefs.get_queue_manager",+ lambda: SimpleNamespace(SEMANTIC="semantic", get_queue=lambda _: queue),+ )+ processor = SimpleNamespace(_circuit_breaker=SimpleNamespace(retry_after=30))+ worker = asyncio.current_task()+ index.register_active("task", worker)+ try:+ now[0] = 3+ await SemanticProcessor._reenqueue_semantic_msg(+ processor, message, enqueue=custom if custom_enqueue else None+ )+ finally:+ index.unregister_active("task", worker)++ assert enqueued == [message]+ assert index.processing_seconds("task") == 5diff --git a/tests/test_task_tracker.py b/tests/test_task_tracker.pyindex 6128ec1cb..796c00458 100644--- a/tests/test_task_tracker.py+++ b/tests/test_task_tracker.py@@ -486,6 +486,7 @@ async def test_public_serialization_skips_private_payloads(tracker: TaskTracker)public = task.to_dict()assert set(public) == {+ "processing_seconds","task_id","task_type","status",@@ -975,3 +976,29 @@ async def test_process_events_respect_owner_and_terminal_boundaries(tracker):assert (await tracker.get(task.task_id, **_owner_kwargs())).execution_events == cancelled.execution_events+++async def test_processing_time_persists_and_restored_active_tasks_are_unknown(tracker, monkeypatch):+ now = [0.0]+ monkeypatch.setattr("openviking.service.task_processing_time.time.monotonic", lambda: now[0])+ task = await tracker.create("add_resource", **_owner_kwargs())+ await tracker.start(task.task_id, **_owner_kwargs())+ now[0] = 10+ tracker.register_running_task(task.task_id)+ now[0] = 13+ assert (await tracker.get(task.task_id)).processing_seconds == 3+ # A stage update neither resets nor stops the processing clock.+ await tracker.update_stage(task.task_id, "semantic", **_owner_kwargs())+ restarted = TaskTracker(store=tracker._store)+ assert (await restarted.get(task.task_id, **_owner_kwargs())).processing_seconds is None+ with tracker._work_index.pause_processing(task.task_id):+ now[0] = 30+ assert (await tracker.get(task.task_id)).processing_seconds == 3+ now[0] = 32+ await tracker.complete(task.task_id, result={"ok": True}, **_owner_kwargs())+ await tracker.unregister_running_task(task.task_id)+ finished = await tracker.get(task.task_id)+ assert finished.status == TaskStatus.COMPLETED+ assert finished.to_dict()["processing_seconds"] == 5+ restored = await TaskTracker(store=tracker._store).get(task.task_id, **_owner_kwargs())+ assert restored.processing_seconds == 5diff --git a/web-studio/src/i18n/locales/en/workspace.ts b/web-studio/src/i18n/locales/en/workspace.tsindex 68c16c5e2..a7428a3e6 100644--- a/web-studio/src/i18n/locales/en/workspace.ts+++ b/web-studio/src/i18n/locales/en/workspace.ts@@ -389,19 +389,27 @@ const workspace = {},tasksPage: {labels: {+ timing: 'Duration',+ totalDuration: 'Total Time',+ processingNotStarted: 'Not started',+ processingDurationHelp: 'Worker processing time, including model and I/O calls. Excludes queue and downstream waits; overlapping workers count once. Incomplete or legacy records are unavailable.',+ processingDuration: 'Processing Time',+ waitingDuration: 'Waiting Time',+ timingUnavailable: 'Not recorded',+missingResource: 'Missing resource ID for task',requeueFailed: 'Re-queue failed',requeueSubmitted: 'Re-queue request submitted successfully!',successRate: 'Success Rate',- avgDuration: 'Avg Duration',- avgProcessingTime: 'Avg Processing Time',+ avgDuration: 'Avg Total Duration',+ avgProcessingTime: 'Finished tasks only, including queue time',totalTasks: 'Total Tasks',activePending: 'Active/Pending',runningPending: 'Running / Pending Workloads',taskQueueStatus: 'Task Queue Status',processQueueStatus: 'Process Queue Status',queuePipeline: 'Queue Pipeline',- duration: 'Duration',+ duration: 'Total Duration (incl. queue)',taskSummary: 'Total {{total}} ({{failed}} failed)',taskCount: 'Tasks: {{count}}',completedTasks: 'Completed: {{count}}',diff --git a/web-studio/src/i18n/locales/zh-CN/workspace.ts b/web-studio/src/i18n/locales/zh-CN/workspace.tsindex 7c2bf0f7b..357bbb205 100644--- a/web-studio/src/i18n/locales/zh-CN/workspace.ts+++ b/web-studio/src/i18n/locales/zh-CN/workspace.ts@@ -378,19 +378,27 @@ const workspace = {},tasksPage: {labels: {+ timing: '耗时',+ totalDuration: '总耗时',+ processingNotStarted: '未开始',+ processingDurationHelp: '仅累计工序实际处理区间,含模型与 I/O 调用;排除排队和等待下游,并行重叠只计一次。旧任务或重启后记录不完整时显示未记录。',+ processingDuration: '处理耗时',+ waitingDuration: '等待耗时',+ timingUnavailable: '未记录',+missingResource: '任务缺少关联资源 ID,无法重新入队',requeueFailed: '重新入队失败',requeueSubmitted: '重新入队请求已发送,后端正在处理新任务!',successRate: '任务成功率',- avgDuration: '平均处理耗时',- avgProcessingTime: '全流程平均处理时长',+ avgDuration: '平均总耗时',+ avgProcessingTime: '仅统计已结束任务,含排队时间',totalTasks: '任务总数',activePending: '并发与排队',runningPending: '进行中 / 等待中任务',taskQueueStatus: '任务队列状态',processQueueStatus: '工序队列状态',queuePipeline: '工序队列流转',- duration: '耗时',+ duration: '总耗时(含排队)',taskSummary: '共 {{total}} 条任务({{failed}} 异常)',taskCount: '{{count}} 条',completedTasks: '已完成 {{count}} 条',diff --git a/web-studio/src/routes/tasks/-components/task-detail-sheet.tsx b/web-studio/src/routes/tasks/-components/task-detail-sheet.tsxindex 7321bf036..2ca5440fc 100644--- a/web-studio/src/routes/tasks/-components/task-detail-sheet.tsx+++ b/web-studio/src/routes/tasks/-components/task-detail-sheet.tsx@@ -24,7 +24,7 @@ import {SheetTitle,} from '#/components/ui/sheet'import { getOvResult, getTaskByTaskId } from '#/lib/ov-client'-import { formatTaskDuration, getTaskDate } from '#/routes/tasks/-lib/task-time'+import { formatTaskDuration, formatTaskProcessingDuration, formatTaskWaitingDuration, getTaskDate } from '#/routes/tasks/-lib/task-time'import {hasTaskResult,@@ -201,10 +201,22 @@ export function TaskDetailSheet({'updated',)}/>+ <DetailField+ icon={<TimerResetIcon />}+ label={t('labels.processingDuration')}+ value={formatTaskProcessingDuration(task) ?? t('labels.timingUnavailable')}+ mono+ />+ <DetailField+ icon={<TimerResetIcon />}+ label={t('labels.waitingDuration')}+ value={formatTaskWaitingDuration(task) ?? t('labels.timingUnavailable')}+ mono+ /><DetailFieldclassName="col-span-2"icon={<TimerResetIcon />}- label={t('pipeline.duration')}+ label={t('labels.duration')}value={formatTaskDuration(task)}mono/>diff --git a/web-studio/src/routes/tasks/-lib/task-time.test.ts b/web-studio/src/routes/tasks/-lib/task-time.test.tsindex aadd49c79..d352d600a 100644--- a/web-studio/src/routes/tasks/-lib/task-time.test.ts+++ b/web-studio/src/routes/tasks/-lib/task-time.test.ts@@ -1,6 +1,13 @@-import { describe, expect, it } from 'vitest'+import { afterEach, describe, expect, it, vi } from 'vitest'-import { getTaskDate } from './task-time'+import {+ formatTaskDuration,+ formatTaskProcessingDuration,+ formatTaskWaitingDuration,+ getAverageTaskDurationSeconds,+ getTaskDate,+ getTaskDurationSeconds,+} from './task-time'describe('getTaskDate', () => {it('prefers the ISO timestamp returned by the task API', () => {@@ -20,3 +27,114 @@ describe('getTaskDate', () => {expect(date?.getUTCFullYear()).toBe(2026)})})++describe('task total duration', () => {+ afterEach(() => vi.useRealTimers())++ it.each(['pending', 'running', 'cancelling'])(+ 'includes queue time for %s without resetting on stage updates',+ (status) => {+ vi.useFakeTimers()+ vi.setSystemTime(new Date('2026-09-20T04:10:00Z'))+ const task = {+ status,+ created_at_iso: '2026-09-20T04:00:00Z',+ updated_at_iso: '2026-09-20T04:09:00Z',+ }+ expect(formatTaskDuration(task)).toBe('10m')+ expect(+ formatTaskDuration({ ...task, updated_at_iso: '2026-09-20T04:10:00Z' }),+ ).toBe('10m')+ vi.advanceTimersByTime(2000)+ expect(formatTaskDuration(task)).toBe('10m 2s')+ },+ )++ it.each(['completed', 'failed', 'cancelled'])(+ 'freezes %s duration after completion',+ (status) => {+ vi.useFakeTimers()+ vi.setSystemTime(new Date('2026-09-20T05:00:00Z'))+ const task = {+ status,+ created_at_iso: '2026-09-20T04:00:00Z',+ updated_at_iso: '2026-09-20T04:10:00Z',+ }+ expect(formatTaskDuration(task)).toBe('10m')+ vi.advanceTimersByTime(60_000)+ expect(formatTaskDuration(task)).toBe('10m')+ },+ )++ it('keeps the same total when a running task completes', () => {+ const task = { created_at: 100, updated_at: 700 }+ expect(+ getTaskDurationSeconds({ ...task, status: 'running' }, 700_000),+ ).toBe(getTaskDurationSeconds({ ...task, status: 'completed' }))+ })++ it('does not invent durations from missing or invalid timestamps', () => {+ for (const task of [+ { status: 'completed', updated_at: 700 },+ { status: 'completed', created_at: 100 },+ { status: 'completed', created_at: 'invalid', updated_at: 700 },+ { status: 'completed', created_at: 700, updated_at: 100 },+ { status: 'unknown', created_at: 100, updated_at: 700 },+ ])+ expect(formatTaskDuration(task)).toBe('-')+ })++ it('averages valid terminal tasks only, with normalized timestamps', () => {+ expect(+ getAverageTaskDurationSeconds([+ { status: 'completed', created_at: 100, updated_at: 160 },+ { status: 'failed', created_at: '100', updated_at: '220' },+ {+ status: 'cancelled',+ created_at_iso: '2026-09-20T04:00:00Z',+ updated_at_iso: '2026-09-20T04:03:00Z',+ },+ { status: 'pending', created_at: 100, updated_at: 100 },+ { status: 'running', created_at: 100, updated_at: 101 },+ { status: 'cancelling', created_at: 100, updated_at: 102 },+ { status: 'completed', created_at: 100 },+ ]),+ ).toBe(120)+ expect(getAverageTaskDurationSeconds([])).toBeUndefined()+ expect(+ getAverageTaskDurationSeconds([{ status: 'pending', created_at: 100 }]),+ ).toBeUndefined()+ expect(+ getAverageTaskDurationSeconds([+ { status: 'completed', created_at: 100, updated_at: 100 },+ ]),+ ).toBe(0)+ })+})++describe('measured processing and waiting time', () => {+ it('separates worker time from queue and other waiting time', () => {+ const task = {+ status: 'completed',+ created_at: 100,+ updated_at: 700,+ processing_seconds: 61,+ }+ expect(formatTaskProcessingDuration(task)).toBe('1m 1s')+ expect(formatTaskWaitingDuration(task)).toBe('8m 59s')+ })++ it('does not substitute total time when measurement is unavailable', () => {+ for (const processing_seconds of [undefined, null, NaN, -1, Infinity]) {+ const task = {+ status: 'completed',+ created_at: 100,+ updated_at: 700,+ processing_seconds,+ }+ expect(formatTaskProcessingDuration(task)).toBeUndefined()+ expect(formatTaskWaitingDuration(task)).toBeUndefined()+ }+ expect(formatTaskProcessingDuration({ processing_seconds: 0 })).toBe('< 1s')+ })+})diff --git a/web-studio/src/routes/tasks/-lib/task-time.ts b/web-studio/src/routes/tasks/-lib/task-time.tsindex 2b3b96471..b751f42d8 100644--- a/web-studio/src/routes/tasks/-lib/task-time.ts+++ b/web-studio/src/routes/tasks/-lib/task-time.ts@@ -1,4 +1,7 @@+import { isActiveTaskStatus } from './task-record'+export type TaskTimestamp = {+ processing_seconds?: number | nullcreated_at?: number | stringcreated_at_iso?: stringstatus?: string@@ -30,37 +33,77 @@ export function getTaskDate(task: TaskTimestamp): Date | undefined {return Number.isNaN(date.getTime()) ? undefined : date}-export function formatTaskDuration(task: TaskTimestamp): string {- const status = task.status || 'unknown'+const terminalStatuses = new Set(['completed', 'failed', 'cancelled'])++/** Wall-clock time since submission, including queue waits between stages. */+export function getTaskDurationSeconds(+ task: TaskTimestamp,+ nowMs = Date.now(),+): number | undefined {+ // A missing creation timestamp cannot be replaced with the last update.+ const start = getTaskDate({+ created_at: task.created_at,+ created_at_iso: task.created_at_iso,+ })+ if (!start) return undefined- // Pending tasks have not started execution yet- if (status === 'pending') {- return '-'+ let endMs: number | undefined+ if (isActiveTaskStatus(task.status)) {+ endMs = nowMs+ } else if (terminalStatuses.has(task.status ?? '')) {+ endMs = getTaskDate({+ created_at: task.updated_at,+ created_at_iso: task.updated_at_iso,+ })?.getTime()}+ if (endMs === undefined || !Number.isFinite(endMs)) return undefined+ if (endMs < start.getTime()) return undefined+ return (endMs - start.getTime()) / 1000+}- const createdDate = getTaskDate(task)- if (!createdDate) return '-'+export function getAverageTaskDurationSeconds(+ tasks: TaskTimestamp[],+): number | undefined {+ const durations = tasks+ .filter((task) => terminalStatuses.has(task.status ?? ''))+ .map((task) => getTaskDurationSeconds(task))+ .filter((duration): duration is number => duration !== undefined)+ if (durations.length === 0) return undefined+ return (+ durations.reduce((sum, duration) => sum + duration, 0) / durations.length+ )+}- const createdMs = createdDate.getTime()- const updatedDate =- task.updated_at || task.updated_at_iso- ? getTaskDate({- created_at: task.updated_at,- created_at_iso: task.updated_at_iso,- })- : undefined+export function formatTaskDuration(task: TaskTimestamp): string {+ const seconds = getTaskDurationSeconds(task)+ return seconds === undefined ? '-' : formatDurationString(Math.floor(seconds))+}- const startMs = updatedDate ? updatedDate.getTime() : createdMs+export function getTaskProcessingSeconds(+ task: TaskTimestamp,+): number | undefined {+ const seconds = task.processing_seconds+ return typeof seconds === 'number' && Number.isFinite(seconds) && seconds >= 0+ ? seconds+ : undefined+}- if (status === 'running') {- const elapsedSec = Math.max(0, Math.floor((Date.now() - startMs) / 1000))- return formatDurationString(elapsedSec)- }+export function formatTaskProcessingDuration(+ task: TaskTimestamp,+): string | undefined {+ const seconds = getTaskProcessingSeconds(task)+ return seconds === undefined+ ? undefined+ : formatDurationString(Math.floor(seconds))+}- // Completed or Failed tasks- const endMs = updatedDate ? updatedDate.getTime() : createdMs- const durationSec = Math.max(0, Math.floor((endMs - createdMs) / 1000))- return formatDurationString(durationSec)+export function formatTaskWaitingDuration(+ task: TaskTimestamp,+): string | undefined {+ const processing = getTaskProcessingSeconds(task)+ const total = getTaskDurationSeconds(task)+ if (processing === undefined || total === undefined) return undefined+ return formatDurationString(Math.floor(Math.max(0, total - processing)))}function formatDurationString(diffSec: number): string {diff --git a/web-studio/src/routes/tasks/route.test.tsx b/web-studio/src/routes/tasks/route.test.tsxindex fe943462c..a0abcc1c5 100644--- a/web-studio/src/routes/tasks/route.test.tsx+++ b/web-studio/src/routes/tasks/route.test.tsx@@ -305,3 +305,22 @@ describe('session commit re-trigger feedback', () => {expect(toast.success).not.toHaveBeenCalled()})})+++it('groups processing and total time without inventing legacy timing', async () => {+ records = [+ { task_id: 'measured', status: 'completed', task_type: 'add_resource', created_at: 100, updated_at: 700, processing_seconds: 61 },+ { task_id: 'pending', status: 'pending', task_type: 'add_resource', created_at: 100, processing_seconds: 0 },+ { task_id: 'legacy', status: 'completed', task_type: 'add_resource', created_at: 100, updated_at: 700 },+ ]+ await renderPage()+ const measured = await screen.findByRole('row', { name: 'View details for task measured' })+ expect(within(measured).getByText('1m 1s')).toBeDefined()+ expect(within(measured).queryByText(/Waiting Time/)).toBeNull()+ expect(screen.getByRole('columnheader', { name: 'Duration' })).toBeDefined()+ expect(within(measured).getByText('1m 1s').closest('td')).toBe(within(measured).getByText('10m').closest('td'))+ expect(within(measured).getByText('10m')).toBeDefined()+ expect(within(screen.getByRole('row', { name: 'View details for task pending' })).getByText('Not started')).toBeDefined()+ const legacy = screen.getByRole('row', { name: 'View details for task legacy' })+ expect(within(legacy).getByText('Not recorded')).toBeDefined()+})diff --git a/web-studio/src/routes/tasks/route.tsx b/web-studio/src/routes/tasks/route.tsxindex d96d33bef..974f0712f 100644--- a/web-studio/src/routes/tasks/route.tsx+++ b/web-studio/src/routes/tasks/route.tsx@@ -51,7 +51,12 @@ import { QueueStatusCard } from '#/routes/monitoring/-components/queue-status-caimport { TaskDetailSheet } from '#/routes/tasks/-components/task-detail-sheet'import { normalizeTaskStatus } from '#/routes/tasks/-lib/task-record'import type { TaskRecord } from '#/routes/tasks/-lib/task-record'-import { formatTaskDuration, getTaskDate } from '#/routes/tasks/-lib/task-time'+import {+ formatTaskDuration,+ formatTaskProcessingDuration,+ getAverageTaskDurationSeconds,+ getTaskDate,+} from '#/routes/tasks/-lib/task-time'import { fetchTasks, MAX_TASKS } from './-lib/task-list'import { localizeSkippedCommit } from './-lib/localize-commit-result'import type { TaskStatusFilter, TaskTypeFilter } from './-lib/task-list'@@ -494,18 +499,7 @@ function TasksRoute() {const successRate = total > 0 ? (completed / total) * 100 : 100- const durations = allTasks- .map((item) => {- const start = Number(item.created_at || 0)- const end = Number(item.updated_at || start)- return start > 0 && end >= start ? end - start : null- })- .filter((d): d is number => d !== null && d >= 0)-- const avgDurationSec =- durations.length > 0- ? durations.reduce((a, b) => a + b, 0) / durations.length- : 0+ const avgDurationSec = getAverageTaskDurationSeconds(allTasks)const ALL_TASK_TYPES = ['add_resource',@@ -645,9 +639,11 @@ function TasksRoute() {</div><div className="flex items-baseline gap-1"><span className="font-mono text-xl font-bold tabular-nums text-foreground">- {kpiData.avgDurationSec < 1- ? `${(kpiData.avgDurationSec * 1000).toFixed(0)}ms`- : `${kpiData.avgDurationSec.toFixed(1)}s`}+ {kpiData.avgDurationSec === undefined+ ? '-'+ : kpiData.avgDurationSec < 1+ ? `${(kpiData.avgDurationSec * 1000).toFixed(0)}ms`+ : `${kpiData.avgDurationSec.toFixed(1)}s`}</span></div><p className="text-[11px] text-muted-foreground truncate">@@ -842,7 +838,7 @@ function TasksRoute() {<TableHead>{t('table.resource')}</TableHead><TableHead>{t('labels.queuePipeline')}</TableHead><TableHead>{t('table.status')}</TableHead>- <TableHead>{t('labels.duration')}</TableHead>+ <TableHead title={t('labels.processingDurationHelp')}>{t('labels.timing')}</TableHead><TableHead className="text-right">{t('table.createdAt')}</TableHead>@@ -909,8 +905,15 @@ function TasksRoute() {</TableCell><TableCell>{renderQueuePipeline(task)}</TableCell><TableCell>{renderStatus(task)}</TableCell>- <TableCell className="whitespace-nowrap font-mono text-xs text-muted-foreground">- {formatTaskDuration(task)}+ <TableCell className="whitespace-nowrap text-xs tabular-nums">+ <div>+ {t('labels.processingDuration')}: <span>{task.status === 'pending' && (task.processing_seconds == null || task.processing_seconds === 0)+ ? t('labels.processingNotStarted')+ : formatTaskProcessingDuration(task) ?? t('labels.timingUnavailable')}</span>+ </div>+ <div className="text-muted-foreground">+ {t('labels.totalDuration')}: <span>{formatTaskDuration(task)}</span>+ </div></TableCell><TableCell className="whitespace-nowrap text-right text-muted-foreground">{formatTime(task)}