Skip to content

Self-evolving Context Database for AI Agents. Unify Agent Memory, Knowledge RAG and Skills.

fix(studio): distinguish task processing time from waiting time (#5207)

yufengee6cc6ce1bae
diff --git a/openviking/service/task_processing_time.py b/openviking/service/task_processing_time.py
new file mode 100644
index 000000000..e52375ad8
--- /dev/null
+++ b/openviking/service/task_processing_time.py
@@ -0,0 +1,57 @@
+# Copyright (c) 2026 Beijing Volcano Engine Technology Co., Ltd.
+# SPDX-License-Identifier: AGPL-3.0
+"""Union of worker execution intervals; queue waits do not advance the clock.
+
+Only tasks created in this process have complete coverage. After a restart we
+leave active tasks unmeasured rather than inventing time across the downtime.
+"""
+
+import time
+from contextlib import contextmanager
+from contextvars import ContextVar
+from dataclasses import dataclass, field
+from typing import TYPE_CHECKING, Any, Iterator
+
+if TYPE_CHECKING:
+ from openviking.service.task_work_index import TaskWorkIndex
+
+# Inherited by shielded child coroutines; DAG nodes install their own owner.
+processing_owner: ContextVar[tuple["TaskWorkIndex", str, Any] | None] = ContextVar(
+ "processing_owner", default=None
+)
+
+
+@contextmanager
+def pause_task_processing() -> Iterator[None]:
+ owner = processing_owner.get()
+ if owner is None:
+ yield
+ else:
+ index, task_id, worker = owner
+ with index.pause_processing(task_id, worker=worker):
+ yield
+
+
+@dataclass
+class ProcessingClock:
+ elapsed: float = 0.0
+ started: float | None = None
+ workers: set[Any] = field(default_factory=set)
+ observed: bool = False
+
+ def enter(self, worker: Any) -> None:
+ if not self.workers:
+ self.started = time.monotonic()
+ self.workers.add(worker)
+ self.observed = True
+
+ def leave(self, worker: Any) -> None:
+ self.workers.discard(worker)
+ if not self.workers and self.started is not None:
+ self.elapsed += time.monotonic() - self.started
+ self.started = None
+
+ def seconds(self) -> float | None:
+ if not self.observed:
+ return None
+ return self.elapsed + (time.monotonic() - self.started if self.started is not None else 0.0)
diff --git a/openviking/service/task_store.py b/openviking/service/task_store.py
index 7920d9139..bcb8b4e20 100644
--- a/openviking/service/task_store.py
+++ b/openviking/service/task_store.py
@@ -264,6 +264,7 @@ def _task_to_payload(task: Any) -> Dict[str, Any]:
"status": status.value if hasattr(status, "value") else status,
"created_at": task.created_at,
"updated_at": task.updated_at,
+ "processing_seconds": task.processing_seconds,
"resource_id": task.resource_id,
"account_id": task.account_id,
"user_id": task.user_id,
diff --git a/openviking/service/task_tracker.py b/openviking/service/task_tracker.py
index 901f1e14c..252a8a388 100644
--- a/openviking/service/task_tracker.py
+++ b/openviking/service/task_tracker.py
@@ -30,6 +30,7 @@ from openviking.service.task_events import (
TaskEventHistory,
append_task_event,
)
+from openviking.service.task_processing_time import pause_task_processing
from openviking.service.task_store import TaskStore
from openviking.service.task_tracker_concurrency import (
KeyedAsyncLockPool,
@@ -88,6 +89,7 @@ class TaskRecord:
error: Optional[str] = None
auth: Dict[str, Any] = field(default_factory=dict, repr=False)
execution_events: Optional[TaskEventHistory] = None
+ processing_seconds: Optional[float] = None
_extra_fields: Dict[str, Any] = field(
default_factory=dict, init=False, repr=False, compare=False
)
@@ -104,6 +106,7 @@ class TaskRecord:
"status": self.status.value,
"created_at": self.created_at,
"updated_at": self.updated_at,
+ "processing_seconds": self.processing_seconds,
"resource_id": self.resource_id,
"meta": _sanitize_task_result(public_meta),
"stage": self.stage,
@@ -287,12 +290,17 @@ class TaskTracker:
for task_id in expired_ids:
evicted_count += await self._delete_expired_task(task_id, now)
+ capacity_evicted = []
with self._lock:
if len(self._tasks) > self.MAX_TASKS:
sorted_tasks = sorted(self._tasks.items(), key=lambda x: x[1].created_at)
excess = len(self._tasks) - self.MAX_TASKS
for tid, _ in sorted_tasks[:excess]:
self._tasks.pop(tid, None)
+ capacity_evicted.append(tid)
+
+ for tid in capacity_evicted:
+ self._work_index.forget_processing(tid)
if evicted_count:
logger.debug("[TaskTracker] Evicted %d expired tasks", evicted_count)
@@ -337,6 +345,7 @@ class TaskTracker:
with self._lock:
self._tasks.pop(task_id, None)
+ self._work_index.forget_processing(task_id)
return True
@staticmethod
@@ -811,6 +820,7 @@ class TaskTracker:
del self._tasks[task_id]
for task_id in task_ids:
self._work_index.clear_failure(task_id)
+ self._work_index.forget_processing(task_id)
async def delete_user_tasks(self, account_id: str, user_id: str) -> int:
"""Delete terminal task records for one user from storage and cache."""
@@ -850,6 +860,7 @@ class TaskTracker:
)
with self._lock:
self._tasks.pop(task.task_id, None)
+ self._work_index.forget_processing(task.task_id)
self._work_index.clear_failure(task.task_id)
deleted += 1
return deleted
@@ -866,8 +877,9 @@ class TaskTracker:
account_id=task.account_id,
user_id=task.user_id,
)
- while self._work_index.has_work(task_id, exclude_work_id=current_work_id):
- await asyncio.sleep(0.05)
+ with pause_task_processing():
+ while self._work_index.has_work(task_id, exclude_work_id=current_work_id):
+ await asyncio.sleep(0.05)
async def record_event(
self,
@@ -975,6 +987,7 @@ class TaskTracker:
)
with self._lock:
self._tasks.pop(task.task_id, None)
+ self._work_index.forget_processing(task.task_id)
return True
async def list_page(
@@ -1004,7 +1017,7 @@ class TaskTracker:
io_limiter=self._store_io,
**filters,
)
- records.extend(self._record_from_payload(record) for record in page)
+ records.extend(self._copy(self._record_from_payload(record)) for record in page)
if include_cached:
records.extend(self._copy(t) for t in self._cache_snapshot())
visible = {
@@ -1119,6 +1132,9 @@ class TaskTracker:
# Build history before the same physical write as the state transition.
if operation == "update" and previous is None:
raise ValueError("Task updates require the previous snapshot")
+ # Persist the final measured union; restored active tasks have no complete clock.
+ if previous is None or previous.status in _ACTIVE_STATUSES:
+ task.processing_seconds = self._work_index.processing_seconds(task.task_id)
kinds = []
if previous is None:
kinds.append("created")
@@ -1151,6 +1167,10 @@ class TaskTracker:
else:
await self._store.update(task)
self._publish_task(task)
+ if operation == "create":
+ self._work_index.init_processing(task.task_id)
+ elif task.status in _TERMINAL_STATUSES:
+ self._work_index.forget_processing(task.task_id)
committed = True
# Semaphore/lock waits remain cancellable. Once the store operation is
@@ -1194,10 +1214,11 @@ class TaskTracker:
def _next_updated_at(task: TaskRecord) -> float:
return max(time.time(), math.nextafter(task.updated_at, math.inf))
- @staticmethod
- def _copy(task: TaskRecord) -> TaskRecord:
+ def _copy(self, task: TaskRecord) -> TaskRecord:
"""Return a defensive copy of a TaskRecord."""
copied = deepcopy(task)
+ if task.status in _ACTIVE_STATUSES:
+ copied.processing_seconds = self._work_index.processing_seconds(task.task_id)
copied.meta = _sanitize_task_result(copied.meta)
copied.result = _sanitize_task_result(copied.result)
copied.auth = {}
diff --git a/openviking/service/task_work_index.py b/openviking/service/task_work_index.py
index b9a520974..ae9ee9c90 100644
--- a/openviking/service/task_work_index.py
+++ b/openviking/service/task_work_index.py
@@ -18,6 +18,8 @@ from dataclasses import dataclass
from typing import Any, Awaitable, Callable, Dict, Iterable, Iterator, Mapping, Optional
from uuid import uuid4
+from openviking.service.task_processing_time import ProcessingClock, processing_owner
+
TASK_WORK_ID_FIELD = "_task_work_id"
@@ -158,6 +160,7 @@ class TaskWorkIndex:
self._work: Dict[str, set[tuple[str, str]]] = {}
self._active: Dict[str, set[asyncio.Task[Any]]] = {}
self._failures: Dict[str, str] = {}
+ self._processing: Dict[str, ProcessingClock] = {}
self._finalize_before_ack: Optional[Callable[[QueueTaskMetadata], Awaitable[None]]] = None
self._is_cancellation_requested: Optional[Callable[[str], bool]] = None
@@ -293,6 +296,52 @@ class TaskWorkIndex:
with self._lock:
self._failures.pop(task_id, None)
+ def init_processing(self, task_id: str) -> None:
+ with self._lock:
+ self._processing.setdefault(task_id, ProcessingClock())
+
+ def processing_seconds(self, task_id: str) -> Optional[float]:
+ with self._lock:
+ clock = self._processing.get(task_id)
+ return clock.seconds() if clock else None
+
+ def forget_processing(self, task_id: str) -> None:
+ with self._lock:
+ self._processing.pop(task_id, None)
+
+ @contextmanager
+ def pause_processing(self, task_id: str, *, worker: Any = None) -> Iterator[None]:
+ worker = worker if worker is not None else asyncio.current_task()
+ with self._lock:
+ clock = self._processing.get(task_id)
+ was_processing = clock is not None and worker in clock.workers
+ if was_processing:
+ clock.leave(worker)
+ try:
+ yield
+ finally:
+ with self._lock:
+ if was_processing and self._processing.get(task_id) is clock:
+ clock.enter(worker)
+
+ @contextmanager
+ def measure_processing(self, task_id: str) -> Iterator[None]:
+ # Timing ownership is separate from cancellation ownership: a shared DAG
+ # worker must never become cancellable on behalf of one queued task.
+ worker = object()
+ token = processing_owner.set((self, task_id, worker))
+ with self._lock:
+ clock = self._processing.get(task_id)
+ if clock is not None:
+ clock.enter(worker)
+ try:
+ yield
+ finally:
+ with self._lock:
+ if clock is not None:
+ clock.leave(worker)
+ processing_owner.reset(token)
+
def register_active(
self,
task_id: str,
@@ -302,11 +351,20 @@ class TaskWorkIndex:
cancelled = self.cancellation_requested(task_id)
if not cancelled:
self._active.setdefault(task_id, set()).add(active_task)
+ clock = self._processing.get(task_id)
+ if clock is not None:
+ clock.enter(active_task)
+ processing_owner.set((self, task_id, active_task))
if cancelled:
active_task.get_loop().call_soon(active_task.cancel)
def unregister_active(self, task_id: str, active_task: asyncio.Task[Any]) -> None:
+ if processing_owner.get() == (self, task_id, active_task):
+ processing_owner.set(None)
with self._lock:
+ clock = self._processing.get(task_id)
+ if clock is not None:
+ clock.leave(active_task)
entries = self._active.get(task_id)
if entries is not None:
entries.discard(active_task)
diff --git a/openviking/storage/queuefs/semantic_dag.py b/openviking/storage/queuefs/semantic_dag.py
index 9f9e74e28..a470caa83 100644
--- a/openviking/storage/queuefs/semantic_dag.py
+++ b/openviking/storage/queuefs/semantic_dag.py
@@ -5,6 +5,7 @@
import asyncio
import re
import threading
+from contextlib import nullcontext
from dataclasses import dataclass, field
from typing import Any, ClassVar, Dict, List, Optional, Set
from weakref import WeakKeyDictionary
@@ -12,6 +13,7 @@ from weakref import WeakKeyDictionary
from openviking.core.namespace import classify_uri
from openviking.parse.parsers.media import get_media_type
from openviking.server.identity import RequestContext
+from openviking.service.task_processing_time import pause_task_processing, processing_owner
from openviking.service.task_tracker_concurrency import run_to_completion
from openviking.service.task_work_index import (
bind_task_context,
@@ -207,6 +209,7 @@ class SemanticDagExecutor:
self._aggregate_directory = aggregate_directory
self._copy_source_uri = copy_source_uri
self._task_context = get_task_context()
+ self._processing_index = None
self._telemetry = get_current_telemetry()
self._stale = False
self._changed_paths = {
@@ -254,31 +257,36 @@ class SemanticDagExecutor:
self._root_done = asyncio.Event()
self._scheduler = get_semantic_node_scheduler(self._node_concurrency)
- try:
- self._register_active()
- self._schedule_dir(root_uri, parent_uri=None)
- await self._root_done.wait()
- if self._failure:
- raise self._failure
- except BaseException:
- self._closed = True
- if self._context_type == "skill":
- for task in self._skill_node_tasks:
- task.cancel()
- await run_to_completion(
- lambda: asyncio.gather(*self._skill_node_tasks, return_exceptions=True)
- )
- await self._active_work_idle.wait()
- raise
- finally:
- self._closed = True
+ owner = processing_owner.get()
+ if owner is not None and self._task_context is not None:
+ if owner[1] == self._task_context.task_id:
+ self._processing_index = owner[0]
+ with pause_task_processing():
try:
+ self._register_active()
+ self._schedule_dir(root_uri, parent_uri=None)
+ await self._root_done.wait()
+ if self._failure:
+ raise self._failure
+ except BaseException:
+ self._closed = True
if self._context_type == "skill":
- await run_to_completion(self._active_work_idle.wait)
- else:
- await self._active_work_idle.wait()
+ for task in self._skill_node_tasks:
+ task.cancel()
+ await run_to_completion(
+ lambda: asyncio.gather(*self._skill_node_tasks, return_exceptions=True)
+ )
+ await self._active_work_idle.wait()
+ raise
finally:
- self._unregister_active()
+ self._closed = True
+ try:
+ if self._context_type == "skill":
+ await run_to_completion(self._active_work_idle.wait)
+ else:
+ await self._active_work_idle.wait()
+ finally:
+ self._unregister_active()
def _schedule_work(self, work: DagWork) -> None:
if self._closed:
@@ -384,8 +392,19 @@ class SemanticDagExecutor:
if self._task_context is not None
else detach_task_context()
)
- with bind_telemetry(self._telemetry), task_context:
- await self._run_work_bound(work)
+ timing = (
+ self._processing_index.measure_processing(self._task_context.task_id)
+ if self._processing_index is not None and self._task_context is not None
+ else nullcontext()
+ )
+ # Shared workers inherit their creator's ContextVars. Never let an
+ # unrelated node pause the creator's clock when it has no timing owner.
+ token = processing_owner.set(None)
+ try:
+ with bind_telemetry(self._telemetry), task_context, timing:
+ await self._run_work_bound(work)
+ finally:
+ processing_owner.reset(token)
async def _await_write(self, operation):
if self._context_type == "skill":
diff --git a/openviking/storage/queuefs/semantic_processor.py b/openviking/storage/queuefs/semantic_processor.py
index 7fb73d552..b8b1d6cf0 100644
--- a/openviking/storage/queuefs/semantic_processor.py
+++ b/openviking/storage/queuefs/semantic_processor.py
@@ -35,6 +35,7 @@ from openviking.parse.parsers.media.utils import (
from openviking.prompts import render_prompt
from openviking.pyagfs.exceptions import AGFSNotADirectoryError
from openviking.server.identity import RequestContext, Role
+from openviking.service.task_processing_time import pause_task_processing
from openviking.service.task_tracker_concurrency import run_to_completion
from openviking.service.task_work_index import detach_task_context
from openviking.storage.abstract_overview import (
@@ -234,7 +235,8 @@ class SemanticProcessor(DequeueHandlerBase):
# Throttle to prevent re-enqueue storm during OPEN window
wait = self._circuit_breaker.retry_after
if wait > 0:
- await asyncio.sleep(wait)
+ with pause_task_processing():
+ await asyncio.sleep(wait)
queue_manager = get_queue_manager()
if queue_manager is not None:
diff --git a/openviking/telemetry/request_wait_tracker.py b/openviking/telemetry/request_wait_tracker.py
index fb1cd5ecc..1a27bd35e 100644
--- a/openviking/telemetry/request_wait_tracker.py
+++ b/openviking/telemetry/request_wait_tracker.py
@@ -10,6 +10,8 @@ import time
from dataclasses import dataclass, field
from typing import Callable, Dict, List, Optional, Set
+from openviking.service.task_processing_time import pause_task_processing
+
@dataclass
class _RequestWaitState:
@@ -199,13 +201,14 @@ class RequestWaitTracker:
) -> None:
if not telemetry_id:
return
- start = time.time()
- while True:
- if self.is_complete(telemetry_id):
- return
- if timeout is not None and (time.time() - start) > timeout:
- raise TimeoutError(f"Request processing not complete after {timeout}s")
- await asyncio.sleep(poll_interval)
+ with pause_task_processing():
+ start = time.time()
+ while True:
+ if self.is_complete(telemetry_id):
+ return
+ if timeout is not None and (time.time() - start) > timeout:
+ raise TimeoutError(f"Request processing not complete after {timeout}s")
+ await asyncio.sleep(poll_interval)
async def wait_for_embeddings(
self,
@@ -217,17 +220,18 @@ class RequestWaitTracker:
"""Drain embeddings while the producing semantic root remains pending."""
if not telemetry_id:
return
- while True:
- with self._lock:
- state = self._states.get(telemetry_id)
- if state is None or not state.pending_embedding_roots:
- return
- if stop_waiting is not None and stop_waiting():
- # Shutdown has drained the embedding consumer's active writes.
- # Leave this semantic delivery unacked for recovery rather than
- # waiting forever for embeddings still in the persistent queue.
- raise asyncio.CancelledError("Embedding worker stopped with queued work")
- await asyncio.sleep(poll_interval)
+ with pause_task_processing():
+ while True:
+ with self._lock:
+ state = self._states.get(telemetry_id)
+ if state is None or not state.pending_embedding_roots:
+ return
+ if stop_waiting is not None and stop_waiting():
+ # Shutdown has drained the embedding consumer's active writes.
+ # Leave this semantic delivery unacked for recovery rather than
+ # waiting forever for embeddings still in the persistent queue.
+ raise asyncio.CancelledError("Embedding worker stopped with queued work")
+ await asyncio.sleep(poll_interval)
def build_queue_status(self, telemetry_id: str) -> Dict[str, Dict[str, object]]:
with self._lock:
diff --git a/tests/service/test_task_processing_time.py b/tests/service/test_task_processing_time.py
new file mode 100644
index 000000000..31d32531d
--- /dev/null
+++ b/tests/service/test_task_processing_time.py
@@ -0,0 +1,214 @@
+# Copyright (c) 2026 Beijing Volcano Engine Technology Co., Ltd.
+# SPDX-License-Identifier: AGPL-3.0
+"""Processing time excludes idle gaps and unions overlapping workers."""
+
+import asyncio
+from types import SimpleNamespace
+
+import pytest
+
+from openviking.service.task_processing_time import ProcessingClock
+from openviking.service.task_tracker_concurrency import run_to_completion
+from openviking.service.task_work_index import TaskWorkIndex, bind_task_context
+from openviking.storage.queuefs.semantic_dag import SemanticDagExecutor, SemanticNodeScheduler
+from openviking.telemetry.request_wait_tracker import RequestWaitTracker
+
+
+def test_processing_clock_unions_workers_and_excludes_idle_time(monkeypatch):
+ now = [0.0]
+ monkeypatch.setattr("openviking.service.task_processing_time.time.monotonic", lambda: now[0])
+ clock = ProcessingClock()
+ assert clock.seconds() is None
+ now[0] = 10 # Initial queue wait.
+ clock.enter("parse")
+ now[0] = 12
+ clock.enter("embedding")
+ now[0] = 15
+ clock.leave("parse")
+ now[0] = 18
+ clock.leave("embedding")
+ assert clock.seconds() == 8
+ now[0] = 40 # No worker: waiting for another queue.
+ assert clock.seconds() == 8
+ clock.enter("semantic")
+ now[0] = 43
+ assert clock.seconds() == 11
+ clock.leave("semantic")
+ now[0] = 100
+ assert clock.seconds() == 11
+
+
+async def test_saturated_dag_scheduler_counts_only_dispatched_nodes(monkeypatch):
+ now = [0.0]
+ monkeypatch.setattr(
+ "openviking.service.task_processing_time.time",
+ SimpleNamespace(monotonic=lambda: now[0]),
+ )
+ index = TaskWorkIndex()
+ scheduler = SemanticNodeScheduler(max_workers=1)
+ monkeypatch.setattr(
+ "openviking.storage.queuefs.semantic_dag.get_semantic_node_scheduler",
+ lambda _: scheduler,
+ )
+ monkeypatch.setattr("openviking.storage.queuefs.semantic_dag.get_viking_fs", lambda: None)
+ entered = {name: asyncio.Event() for name in ["a", "b"]}
+ release = {name: asyncio.Event() for name in ["a", "b"]}
+ queued = asyncio.Event()
+ executors = {}
+ for name in ["a", "b"]:
+ index.init_processing(name)
+ with bind_task_context(name, "account", "user"):
+ executor = SemanticDagExecutor(None, "resource", 1, None)
+ executors[name] = executor
+
+ async def node(_work, name=name, executor=executor):
+ entered[name].set()
+ await release[name].wait()
+ executor._root_done.set()
+
+ monkeypatch.setattr(executor, "_run_work_bound", node)
+ schedule_b = executors["b"]._schedule_dir
+
+ def queue_b(*args, **kwargs):
+ schedule_b(*args, **kwargs)
+ queued.set()
+
+ monkeypatch.setattr(executors["b"], "_schedule_dir", queue_b)
+
+ async def coordinate(name):
+ with index.measure_processing(name):
+ await executors[name].run("viking://resources/" + name)
+
+ first = asyncio.create_task(coordinate("a"))
+ await entered["a"].wait()
+ second = asyncio.create_task(coordinate("b"))
+ await queued.wait()
+ now[0] = 20
+ assert index.processing_seconds("a") == 20
+ assert index.processing_seconds("b") == 0 # Still queued behind A.
+ release["a"].set()
+ await entered["b"].wait()
+ now[0] = 25
+ assert index.processing_seconds("b") == 5
+ release["b"].set()
+ await asyncio.gather(first, second)
+ assert index.processing_seconds("b") == 5
+ for worker in list(scheduler._workers):
+ worker.cancel()
+ await asyncio.gather(*scheduler._workers, return_exceptions=True)
+
+
[email protected]("method", ["wait_for_request", "wait_for_embeddings"])
+async def test_downstream_wait_pauses_inherited_owner_in_shielded_child(monkeypatch, method):
+ now = [0.0]
+ monkeypatch.setattr(
+ "openviking.service.task_processing_time.time",
+ SimpleNamespace(monotonic=lambda: now[0]),
+ )
+ index = TaskWorkIndex()
+ index.init_processing("task")
+ tracker = RequestWaitTracker()
+ request = "processing-time-" + method
+ tracker.register_request(request)
+ tracker.register_embedding_root(request, "embedding")
+ with index.measure_processing("task"):
+ now[0] = 3
+ waiter = asyncio.create_task(
+ run_to_completion(lambda: getattr(tracker, method)(request, poll_interval=0.001))
+ )
+ # Let run_to_completion and its shielded child enter the wait loop.
+ await asyncio.sleep(0.01)
+ now[0] = 20
+ assert index.processing_seconds("task") == 3
+ tracker.mark_embedding_done(request, "embedding")
+ await waiter
+ now[0] = 22
+ assert index.processing_seconds("task") == 5
+ tracker.cleanup(request)
+
+
+async def test_nested_wait_and_cancellation_restore_processing_owner(monkeypatch):
+ from openviking.service.task_processing_time import pause_task_processing, processing_owner
+
+ now = [0.0]
+ monkeypatch.setattr(
+ "openviking.service.task_processing_time.time",
+ SimpleNamespace(monotonic=lambda: now[0]),
+ )
+ index = TaskWorkIndex()
+ index.init_processing("task")
+ entered = asyncio.Event()
+
+ async def work():
+ with index.measure_processing("task"):
+ now[0] = 2
+ try:
+ with pause_task_processing():
+ with pause_task_processing():
+ entered.set()
+ await asyncio.Event().wait()
+ except asyncio.CancelledError:
+ now[0] += 3 # Cancellation cleanup resumes processing.
+ assert processing_owner.get() is None
+
+ task = asyncio.create_task(work())
+ await entered.wait()
+ now[0] = 20
+ assert index.processing_seconds("task") == 2
+ task.cancel()
+ await task
+ assert index.processing_seconds("task") == 5
+
+
[email protected]("custom_enqueue", [False, True])
+async def test_semantic_retry_cooldown_excludes_wait_but_counts_enqueue(monkeypatch, custom_enqueue):
+ from openviking.storage.queuefs.semantic_processor import SemanticProcessor
+
+ now = [0.0]
+ monkeypatch.setattr(
+ "openviking.service.task_processing_time.time",
+ SimpleNamespace(monotonic=lambda: now[0]),
+ )
+ index = TaskWorkIndex()
+ index.init_processing("task")
+ message = SimpleNamespace(uri="viking://resources/retry-test")
+ enqueued = []
+
+ async def sleep(delay):
+ assert delay == 30
+ now[0] += delay
+ assert index.processing_seconds("task") == 3
+
+ async def enqueue(msg):
+ assert msg is message
+ now[0] += 2
+ enqueued.append(msg)
+
+ queue = SimpleNamespace(enqueue=enqueue)
+
+ async def custom(queue_arg, msg):
+ assert queue_arg is queue
+ await enqueue(msg)
+
+ monkeypatch.setattr(asyncio, "sleep", sleep)
+ monkeypatch.setattr(
+ "openviking.storage.queuefs.get_queue_manager",
+ lambda: SimpleNamespace(SEMANTIC="semantic", get_queue=lambda _: queue),
+ )
+ processor = SimpleNamespace(_circuit_breaker=SimpleNamespace(retry_after=30))
+ worker = asyncio.current_task()
+ index.register_active("task", worker)
+ try:
+ now[0] = 3
+ await SemanticProcessor._reenqueue_semantic_msg(
+ processor, message, enqueue=custom if custom_enqueue else None
+ )
+ finally:
+ index.unregister_active("task", worker)
+
+ assert enqueued == [message]
+ assert index.processing_seconds("task") == 5
diff --git a/tests/test_task_tracker.py b/tests/test_task_tracker.py
index 6128ec1cb..796c00458 100644
--- a/tests/test_task_tracker.py
+++ b/tests/test_task_tracker.py
@@ -486,6 +486,7 @@ async def test_public_serialization_skips_private_payloads(tracker: TaskTracker)
public = task.to_dict()
assert set(public) == {
+ "processing_seconds",
"task_id",
"task_type",
"status",
@@ -975,3 +976,29 @@ async def test_process_events_respect_owner_and_terminal_boundaries(tracker):
assert (
await tracker.get(task.task_id, **_owner_kwargs())
).execution_events == cancelled.execution_events
+
+
+async def test_processing_time_persists_and_restored_active_tasks_are_unknown(tracker, monkeypatch):
+ now = [0.0]
+ monkeypatch.setattr("openviking.service.task_processing_time.time.monotonic", lambda: now[0])
+ task = await tracker.create("add_resource", **_owner_kwargs())
+ await tracker.start(task.task_id, **_owner_kwargs())
+ now[0] = 10
+ tracker.register_running_task(task.task_id)
+ now[0] = 13
+ assert (await tracker.get(task.task_id)).processing_seconds == 3
+ # A stage update neither resets nor stops the processing clock.
+ await tracker.update_stage(task.task_id, "semantic", **_owner_kwargs())
+ restarted = TaskTracker(store=tracker._store)
+ assert (await restarted.get(task.task_id, **_owner_kwargs())).processing_seconds is None
+ with tracker._work_index.pause_processing(task.task_id):
+ now[0] = 30
+ assert (await tracker.get(task.task_id)).processing_seconds == 3
+ now[0] = 32
+ await tracker.complete(task.task_id, result={"ok": True}, **_owner_kwargs())
+ await tracker.unregister_running_task(task.task_id)
+ finished = await tracker.get(task.task_id)
+ assert finished.status == TaskStatus.COMPLETED
+ assert finished.to_dict()["processing_seconds"] == 5
+ restored = await TaskTracker(store=tracker._store).get(task.task_id, **_owner_kwargs())
+ assert restored.processing_seconds == 5
diff --git a/web-studio/src/i18n/locales/en/workspace.ts b/web-studio/src/i18n/locales/en/workspace.ts
index 68c16c5e2..a7428a3e6 100644
--- a/web-studio/src/i18n/locales/en/workspace.ts
+++ b/web-studio/src/i18n/locales/en/workspace.ts
@@ -389,19 +389,27 @@ const workspace = {
},
tasksPage: {
labels: {
+ timing: 'Duration',
+ totalDuration: 'Total Time',
+ processingNotStarted: 'Not started',
+ processingDurationHelp: 'Worker processing time, including model and I/O calls. Excludes queue and downstream waits; overlapping workers count once. Incomplete or legacy records are unavailable.',
+ processingDuration: 'Processing Time',
+ waitingDuration: 'Waiting Time',
+ timingUnavailable: 'Not recorded',
+
missingResource: 'Missing resource ID for task',
requeueFailed: 'Re-queue failed',
requeueSubmitted: 'Re-queue request submitted successfully!',
successRate: 'Success Rate',
- avgDuration: 'Avg Duration',
- avgProcessingTime: 'Avg Processing Time',
+ avgDuration: 'Avg Total Duration',
+ avgProcessingTime: 'Finished tasks only, including queue time',
totalTasks: 'Total Tasks',
activePending: 'Active/Pending',
runningPending: 'Running / Pending Workloads',
taskQueueStatus: 'Task Queue Status',
processQueueStatus: 'Process Queue Status',
queuePipeline: 'Queue Pipeline',
- duration: 'Duration',
+ duration: 'Total Duration (incl. queue)',
taskSummary: 'Total {{total}} ({{failed}} failed)',
taskCount: 'Tasks: {{count}}',
completedTasks: 'Completed: {{count}}',
diff --git a/web-studio/src/i18n/locales/zh-CN/workspace.ts b/web-studio/src/i18n/locales/zh-CN/workspace.ts
index 7c2bf0f7b..357bbb205 100644
--- a/web-studio/src/i18n/locales/zh-CN/workspace.ts
+++ b/web-studio/src/i18n/locales/zh-CN/workspace.ts
@@ -378,19 +378,27 @@ const workspace = {
},
tasksPage: {
labels: {
+ timing: '耗时',
+ totalDuration: '总耗时',
+ processingNotStarted: '未开始',
+ processingDurationHelp: '仅累计工序实际处理区间,含模型与 I/O 调用;排除排队和等待下游,并行重叠只计一次。旧任务或重启后记录不完整时显示未记录。',
+ processingDuration: '处理耗时',
+ waitingDuration: '等待耗时',
+ timingUnavailable: '未记录',
+
missingResource: '任务缺少关联资源 ID,无法重新入队',
requeueFailed: '重新入队失败',
requeueSubmitted: '重新入队请求已发送,后端正在处理新任务!',
successRate: '任务成功率',
- avgDuration: '平均处理耗时',
- avgProcessingTime: '全流程平均处理时长',
+ avgDuration: '平均总耗时',
+ avgProcessingTime: '仅统计已结束任务,含排队时间',
totalTasks: '任务总数',
activePending: '并发与排队',
runningPending: '进行中 / 等待中任务',
taskQueueStatus: '任务队列状态',
processQueueStatus: '工序队列状态',
queuePipeline: '工序队列流转',
- duration: '耗时',
+ duration: '总耗时(含排队)',
taskSummary: '共 {{total}} 条任务({{failed}} 异常)',
taskCount: '{{count}} 条',
completedTasks: '已完成 {{count}} 条',
diff --git a/web-studio/src/routes/tasks/-components/task-detail-sheet.tsx b/web-studio/src/routes/tasks/-components/task-detail-sheet.tsx
index 7321bf036..2ca5440fc 100644
--- a/web-studio/src/routes/tasks/-components/task-detail-sheet.tsx
+++ b/web-studio/src/routes/tasks/-components/task-detail-sheet.tsx
@@ -24,7 +24,7 @@ import {
SheetTitle,
} from '#/components/ui/sheet'
import { getOvResult, getTaskByTaskId } from '#/lib/ov-client'
-import { formatTaskDuration, getTaskDate } from '#/routes/tasks/-lib/task-time'
+import { formatTaskDuration, formatTaskProcessingDuration, formatTaskWaitingDuration, getTaskDate } from '#/routes/tasks/-lib/task-time'
import {
hasTaskResult,
@@ -201,10 +201,22 @@ export function TaskDetailSheet({
'updated',
)}
/>
+ <DetailField
+ icon={<TimerResetIcon />}
+ label={t('labels.processingDuration')}
+ value={formatTaskProcessingDuration(task) ?? t('labels.timingUnavailable')}
+ mono
+ />
+ <DetailField
+ icon={<TimerResetIcon />}
+ label={t('labels.waitingDuration')}
+ value={formatTaskWaitingDuration(task) ?? t('labels.timingUnavailable')}
+ mono
+ />
<DetailField
className="col-span-2"
icon={<TimerResetIcon />}
- label={t('pipeline.duration')}
+ label={t('labels.duration')}
value={formatTaskDuration(task)}
mono
/>
diff --git a/web-studio/src/routes/tasks/-lib/task-time.test.ts b/web-studio/src/routes/tasks/-lib/task-time.test.ts
index aadd49c79..d352d600a 100644
--- a/web-studio/src/routes/tasks/-lib/task-time.test.ts
+++ b/web-studio/src/routes/tasks/-lib/task-time.test.ts
@@ -1,6 +1,13 @@
-import { describe, expect, it } from 'vitest'
+import { afterEach, describe, expect, it, vi } from 'vitest'
-import { getTaskDate } from './task-time'
+import {
+ formatTaskDuration,
+ formatTaskProcessingDuration,
+ formatTaskWaitingDuration,
+ getAverageTaskDurationSeconds,
+ getTaskDate,
+ getTaskDurationSeconds,
+} from './task-time'
describe('getTaskDate', () => {
it('prefers the ISO timestamp returned by the task API', () => {
@@ -20,3 +27,114 @@ describe('getTaskDate', () => {
expect(date?.getUTCFullYear()).toBe(2026)
})
})
+
+describe('task total duration', () => {
+ afterEach(() => vi.useRealTimers())
+
+ it.each(['pending', 'running', 'cancelling'])(
+ 'includes queue time for %s without resetting on stage updates',
+ (status) => {
+ vi.useFakeTimers()
+ vi.setSystemTime(new Date('2026-09-20T04:10:00Z'))
+ const task = {
+ status,
+ created_at_iso: '2026-09-20T04:00:00Z',
+ updated_at_iso: '2026-09-20T04:09:00Z',
+ }
+ expect(formatTaskDuration(task)).toBe('10m')
+ expect(
+ formatTaskDuration({ ...task, updated_at_iso: '2026-09-20T04:10:00Z' }),
+ ).toBe('10m')
+ vi.advanceTimersByTime(2000)
+ expect(formatTaskDuration(task)).toBe('10m 2s')
+ },
+ )
+
+ it.each(['completed', 'failed', 'cancelled'])(
+ 'freezes %s duration after completion',
+ (status) => {
+ vi.useFakeTimers()
+ vi.setSystemTime(new Date('2026-09-20T05:00:00Z'))
+ const task = {
+ status,
+ created_at_iso: '2026-09-20T04:00:00Z',
+ updated_at_iso: '2026-09-20T04:10:00Z',
+ }
+ expect(formatTaskDuration(task)).toBe('10m')
+ vi.advanceTimersByTime(60_000)
+ expect(formatTaskDuration(task)).toBe('10m')
+ },
+ )
+
+ it('keeps the same total when a running task completes', () => {
+ const task = { created_at: 100, updated_at: 700 }
+ expect(
+ getTaskDurationSeconds({ ...task, status: 'running' }, 700_000),
+ ).toBe(getTaskDurationSeconds({ ...task, status: 'completed' }))
+ })
+
+ it('does not invent durations from missing or invalid timestamps', () => {
+ for (const task of [
+ { status: 'completed', updated_at: 700 },
+ { status: 'completed', created_at: 100 },
+ { status: 'completed', created_at: 'invalid', updated_at: 700 },
+ { status: 'completed', created_at: 700, updated_at: 100 },
+ { status: 'unknown', created_at: 100, updated_at: 700 },
+ ])
+ expect(formatTaskDuration(task)).toBe('-')
+ })
+
+ it('averages valid terminal tasks only, with normalized timestamps', () => {
+ expect(
+ getAverageTaskDurationSeconds([
+ { status: 'completed', created_at: 100, updated_at: 160 },
+ { status: 'failed', created_at: '100', updated_at: '220' },
+ {
+ status: 'cancelled',
+ created_at_iso: '2026-09-20T04:00:00Z',
+ updated_at_iso: '2026-09-20T04:03:00Z',
+ },
+ { status: 'pending', created_at: 100, updated_at: 100 },
+ { status: 'running', created_at: 100, updated_at: 101 },
+ { status: 'cancelling', created_at: 100, updated_at: 102 },
+ { status: 'completed', created_at: 100 },
+ ]),
+ ).toBe(120)
+ expect(getAverageTaskDurationSeconds([])).toBeUndefined()
+ expect(
+ getAverageTaskDurationSeconds([{ status: 'pending', created_at: 100 }]),
+ ).toBeUndefined()
+ expect(
+ getAverageTaskDurationSeconds([
+ { status: 'completed', created_at: 100, updated_at: 100 },
+ ]),
+ ).toBe(0)
+ })
+})
+
+describe('measured processing and waiting time', () => {
+ it('separates worker time from queue and other waiting time', () => {
+ const task = {
+ status: 'completed',
+ created_at: 100,
+ updated_at: 700,
+ processing_seconds: 61,
+ }
+ expect(formatTaskProcessingDuration(task)).toBe('1m 1s')
+ expect(formatTaskWaitingDuration(task)).toBe('8m 59s')
+ })
+
+ it('does not substitute total time when measurement is unavailable', () => {
+ for (const processing_seconds of [undefined, null, NaN, -1, Infinity]) {
+ const task = {
+ status: 'completed',
+ created_at: 100,
+ updated_at: 700,
+ processing_seconds,
+ }
+ expect(formatTaskProcessingDuration(task)).toBeUndefined()
+ expect(formatTaskWaitingDuration(task)).toBeUndefined()
+ }
+ expect(formatTaskProcessingDuration({ processing_seconds: 0 })).toBe('< 1s')
+ })
+})
diff --git a/web-studio/src/routes/tasks/-lib/task-time.ts b/web-studio/src/routes/tasks/-lib/task-time.ts
index 2b3b96471..b751f42d8 100644
--- a/web-studio/src/routes/tasks/-lib/task-time.ts
+++ b/web-studio/src/routes/tasks/-lib/task-time.ts
@@ -1,4 +1,7 @@
+import { isActiveTaskStatus } from './task-record'
+
export type TaskTimestamp = {
+ processing_seconds?: number | null
created_at?: number | string
created_at_iso?: string
status?: string
@@ -30,37 +33,77 @@ export function getTaskDate(task: TaskTimestamp): Date | undefined {
return Number.isNaN(date.getTime()) ? undefined : date
}
-export function formatTaskDuration(task: TaskTimestamp): string {
- const status = task.status || 'unknown'
+const terminalStatuses = new Set(['completed', 'failed', 'cancelled'])
+
+/** Wall-clock time since submission, including queue waits between stages. */
+export function getTaskDurationSeconds(
+ task: TaskTimestamp,
+ nowMs = Date.now(),
+): number | undefined {
+ // A missing creation timestamp cannot be replaced with the last update.
+ const start = getTaskDate({
+ created_at: task.created_at,
+ created_at_iso: task.created_at_iso,
+ })
+ if (!start) return undefined
- // Pending tasks have not started execution yet
- if (status === 'pending') {
- return '-'
+ let endMs: number | undefined
+ if (isActiveTaskStatus(task.status)) {
+ endMs = nowMs
+ } else if (terminalStatuses.has(task.status ?? '')) {
+ endMs = getTaskDate({
+ created_at: task.updated_at,
+ created_at_iso: task.updated_at_iso,
+ })?.getTime()
}
+ if (endMs === undefined || !Number.isFinite(endMs)) return undefined
+ if (endMs < start.getTime()) return undefined
+ return (endMs - start.getTime()) / 1000
+}
- const createdDate = getTaskDate(task)
- if (!createdDate) return '-'
+export function getAverageTaskDurationSeconds(
+ tasks: TaskTimestamp[],
+): number | undefined {
+ const durations = tasks
+ .filter((task) => terminalStatuses.has(task.status ?? ''))
+ .map((task) => getTaskDurationSeconds(task))
+ .filter((duration): duration is number => duration !== undefined)
+ if (durations.length === 0) return undefined
+ return (
+ durations.reduce((sum, duration) => sum + duration, 0) / durations.length
+ )
+}
- const createdMs = createdDate.getTime()
- const updatedDate =
- task.updated_at || task.updated_at_iso
- ? getTaskDate({
- created_at: task.updated_at,
- created_at_iso: task.updated_at_iso,
- })
- : undefined
+export function formatTaskDuration(task: TaskTimestamp): string {
+ const seconds = getTaskDurationSeconds(task)
+ return seconds === undefined ? '-' : formatDurationString(Math.floor(seconds))
+}
- const startMs = updatedDate ? updatedDate.getTime() : createdMs
+export function getTaskProcessingSeconds(
+ task: TaskTimestamp,
+): number | undefined {
+ const seconds = task.processing_seconds
+ return typeof seconds === 'number' && Number.isFinite(seconds) && seconds >= 0
+ ? seconds
+ : undefined
+}
- if (status === 'running') {
- const elapsedSec = Math.max(0, Math.floor((Date.now() - startMs) / 1000))
- return formatDurationString(elapsedSec)
- }
+export function formatTaskProcessingDuration(
+ task: TaskTimestamp,
+): string | undefined {
+ const seconds = getTaskProcessingSeconds(task)
+ return seconds === undefined
+ ? undefined
+ : formatDurationString(Math.floor(seconds))
+}
- // Completed or Failed tasks
- const endMs = updatedDate ? updatedDate.getTime() : createdMs
- const durationSec = Math.max(0, Math.floor((endMs - createdMs) / 1000))
- return formatDurationString(durationSec)
+export function formatTaskWaitingDuration(
+ task: TaskTimestamp,
+): string | undefined {
+ const processing = getTaskProcessingSeconds(task)
+ const total = getTaskDurationSeconds(task)
+ if (processing === undefined || total === undefined) return undefined
+ return formatDurationString(Math.floor(Math.max(0, total - processing)))
}
function formatDurationString(diffSec: number): string {
diff --git a/web-studio/src/routes/tasks/route.test.tsx b/web-studio/src/routes/tasks/route.test.tsx
index fe943462c..a0abcc1c5 100644
--- a/web-studio/src/routes/tasks/route.test.tsx
+++ b/web-studio/src/routes/tasks/route.test.tsx
@@ -305,3 +305,22 @@ describe('session commit re-trigger feedback', () => {
expect(toast.success).not.toHaveBeenCalled()
})
})
+
+
+it('groups processing and total time without inventing legacy timing', async () => {
+ records = [
+ { task_id: 'measured', status: 'completed', task_type: 'add_resource', created_at: 100, updated_at: 700, processing_seconds: 61 },
+ { task_id: 'pending', status: 'pending', task_type: 'add_resource', created_at: 100, processing_seconds: 0 },
+ { task_id: 'legacy', status: 'completed', task_type: 'add_resource', created_at: 100, updated_at: 700 },
+ ]
+ await renderPage()
+ const measured = await screen.findByRole('row', { name: 'View details for task measured' })
+ expect(within(measured).getByText('1m 1s')).toBeDefined()
+ expect(within(measured).queryByText(/Waiting Time/)).toBeNull()
+ expect(screen.getByRole('columnheader', { name: 'Duration' })).toBeDefined()
+ expect(within(measured).getByText('1m 1s').closest('td')).toBe(within(measured).getByText('10m').closest('td'))
+ expect(within(measured).getByText('10m')).toBeDefined()
+ expect(within(screen.getByRole('row', { name: 'View details for task pending' })).getByText('Not started')).toBeDefined()
+ const legacy = screen.getByRole('row', { name: 'View details for task legacy' })
+ expect(within(legacy).getByText('Not recorded')).toBeDefined()
+})
diff --git a/web-studio/src/routes/tasks/route.tsx b/web-studio/src/routes/tasks/route.tsx
index d96d33bef..974f0712f 100644
--- a/web-studio/src/routes/tasks/route.tsx
+++ b/web-studio/src/routes/tasks/route.tsx
@@ -51,7 +51,12 @@ import { QueueStatusCard } from '#/routes/monitoring/-components/queue-status-ca
import { TaskDetailSheet } from '#/routes/tasks/-components/task-detail-sheet'
import { normalizeTaskStatus } from '#/routes/tasks/-lib/task-record'
import type { TaskRecord } from '#/routes/tasks/-lib/task-record'
-import { formatTaskDuration, getTaskDate } from '#/routes/tasks/-lib/task-time'
+import {
+ formatTaskDuration,
+ formatTaskProcessingDuration,
+ getAverageTaskDurationSeconds,
+ getTaskDate,
+} from '#/routes/tasks/-lib/task-time'
import { fetchTasks, MAX_TASKS } from './-lib/task-list'
import { localizeSkippedCommit } from './-lib/localize-commit-result'
import type { TaskStatusFilter, TaskTypeFilter } from './-lib/task-list'
@@ -494,18 +499,7 @@ function TasksRoute() {
const successRate = total > 0 ? (completed / total) * 100 : 100
- const durations = allTasks
- .map((item) => {
- const start = Number(item.created_at || 0)
- const end = Number(item.updated_at || start)
- return start > 0 && end >= start ? end - start : null
- })
- .filter((d): d is number => d !== null && d >= 0)
-
- const avgDurationSec =
- durations.length > 0
- ? durations.reduce((a, b) => a + b, 0) / durations.length
- : 0
+ const avgDurationSec = getAverageTaskDurationSeconds(allTasks)
const ALL_TASK_TYPES = [
'add_resource',
@@ -645,9 +639,11 @@ function TasksRoute() {
</div>
<div className="flex items-baseline gap-1">
<span className="font-mono text-xl font-bold tabular-nums text-foreground">
- {kpiData.avgDurationSec < 1
- ? `${(kpiData.avgDurationSec * 1000).toFixed(0)}ms`
- : `${kpiData.avgDurationSec.toFixed(1)}s`}
+ {kpiData.avgDurationSec === undefined
+ ? '-'
+ : kpiData.avgDurationSec < 1
+ ? `${(kpiData.avgDurationSec * 1000).toFixed(0)}ms`
+ : `${kpiData.avgDurationSec.toFixed(1)}s`}
</span>
</div>
<p className="text-[11px] text-muted-foreground truncate">
@@ -842,7 +838,7 @@ function TasksRoute() {
<TableHead>{t('table.resource')}</TableHead>
<TableHead>{t('labels.queuePipeline')}</TableHead>
<TableHead>{t('table.status')}</TableHead>
- <TableHead>{t('labels.duration')}</TableHead>
+ <TableHead title={t('labels.processingDurationHelp')}>{t('labels.timing')}</TableHead>
<TableHead className="text-right">
{t('table.createdAt')}
</TableHead>
@@ -909,8 +905,15 @@ function TasksRoute() {
</TableCell>
<TableCell>{renderQueuePipeline(task)}</TableCell>
<TableCell>{renderStatus(task)}</TableCell>
- <TableCell className="whitespace-nowrap font-mono text-xs text-muted-foreground">
- {formatTaskDuration(task)}
+ <TableCell className="whitespace-nowrap text-xs tabular-nums">
+ <div>
+ {t('labels.processingDuration')}: <span>{task.status === 'pending' && (task.processing_seconds == null || task.processing_seconds === 0)
+ ? t('labels.processingNotStarted')
+ : formatTaskProcessingDuration(task) ?? t('labels.timingUnavailable')}</span>
+ </div>
+ <div className="text-muted-foreground">
+ {t('labels.totalDuration')}: <span>{formatTaskDuration(task)}</span>
+ </div>
</TableCell>
<TableCell className="whitespace-nowrap text-right text-muted-foreground">
{formatTime(task)}