Skip to content

Self-evolving Context Database for AI Agents. Unify Agent Memory, Knowledge RAG and Skills.

fix(tasks): derive processing time from work timestamps (#5991)

runyunzhoud41cd6db1c17
diff --git a/openviking/service/task_processing_time.py b/openviking/service/task_processing_time.py
deleted file mode 100644
index e52375ad8..000000000
--- a/openviking/service/task_processing_time.py
+++ /dev/null
@@ -1,57 +0,0 @@
-# Copyright (c) 2026 Beijing Volcano Engine Technology Co., Ltd.
-# SPDX-License-Identifier: AGPL-3.0
-"""Union of worker execution intervals; queue waits do not advance the clock.
-
-Only tasks created in this process have complete coverage. After a restart we
-leave active tasks unmeasured rather than inventing time across the downtime.
-"""
-
-import time
-from contextlib import contextmanager
-from contextvars import ContextVar
-from dataclasses import dataclass, field
-from typing import TYPE_CHECKING, Any, Iterator
-
-if TYPE_CHECKING:
- from openviking.service.task_work_index import TaskWorkIndex
-
-# Inherited by shielded child coroutines; DAG nodes install their own owner.
-processing_owner: ContextVar[tuple["TaskWorkIndex", str, Any] | None] = ContextVar(
- "processing_owner", default=None
-)
-
-
-@contextmanager
-def pause_task_processing() -> Iterator[None]:
- owner = processing_owner.get()
- if owner is None:
- yield
- else:
- index, task_id, worker = owner
- with index.pause_processing(task_id, worker=worker):
- yield
-
-
-@dataclass
-class ProcessingClock:
- elapsed: float = 0.0
- started: float | None = None
- workers: set[Any] = field(default_factory=set)
- observed: bool = False
-
- def enter(self, worker: Any) -> None:
- if not self.workers:
- self.started = time.monotonic()
- self.workers.add(worker)
- self.observed = True
-
- def leave(self, worker: Any) -> None:
- self.workers.discard(worker)
- if not self.workers and self.started is not None:
- self.elapsed += time.monotonic() - self.started
- self.started = None
-
- def seconds(self) -> float | None:
- if not self.observed:
- return None
- return self.elapsed + (time.monotonic() - self.started if self.started is not None else 0.0)
diff --git a/openviking/service/task_queue_middleware.py b/openviking/service/task_queue_middleware.py
index e7381cda6..64006c570 100644
--- a/openviking/service/task_queue_middleware.py
+++ b/openviking/service/task_queue_middleware.py
@@ -58,7 +58,11 @@ class TaskWorkQueueMiddleware(QueueMiddleware):
self._index.record_failure(metadata.task_id, result.error)
return result
if active_task is not None:
- self._index.register_active(metadata.task_id, active_task)
+ self._index.register_active(
+ metadata.task_id,
+ active_task,
+ work_id=metadata.work_id,
+ )
try:
result = await call_next(ctx)
except asyncio.CancelledError:
diff --git a/openviking/service/task_tracker.py b/openviking/service/task_tracker.py
index 252a8a388..90bb232e6 100644
--- a/openviking/service/task_tracker.py
+++ b/openviking/service/task_tracker.py
@@ -30,7 +30,6 @@ from openviking.service.task_events import (
TaskEventHistory,
append_task_event,
)
-from openviking.service.task_processing_time import pause_task_processing
from openviking.service.task_store import TaskStore
from openviking.service.task_tracker_concurrency import (
KeyedAsyncLockPool,
@@ -877,9 +876,8 @@ class TaskTracker:
account_id=task.account_id,
user_id=task.user_id,
)
- with pause_task_processing():
- while self._work_index.has_work(task_id, exclude_work_id=current_work_id):
- await asyncio.sleep(0.05)
+ while self._work_index.has_work(task_id, exclude_work_id=current_work_id):
+ await asyncio.sleep(0.05)
async def record_event(
self,
@@ -1132,7 +1130,7 @@ class TaskTracker:
# Build history before the same physical write as the state transition.
if operation == "update" and previous is None:
raise ValueError("Task updates require the previous snapshot")
- # Persist the final measured union; restored active tasks have no complete clock.
+ # Persist the final Work interval union; restored active tasks lack full timing coverage.
if previous is None or previous.status in _ACTIVE_STATUSES:
task.processing_seconds = self._work_index.processing_seconds(task.task_id)
kinds = []
diff --git a/openviking/service/task_work_index.py b/openviking/service/task_work_index.py
index ae9ee9c90..cb034afa0 100644
--- a/openviking/service/task_work_index.py
+++ b/openviking/service/task_work_index.py
@@ -12,14 +12,13 @@ from __future__ import annotations
import asyncio
import json
import threading
+import time
from contextlib import contextmanager
from contextvars import ContextVar
from dataclasses import dataclass
from typing import Any, Awaitable, Callable, Dict, Iterable, Iterator, Mapping, Optional
from uuid import uuid4
-from openviking.service.task_processing_time import ProcessingClock, processing_owner
-
TASK_WORK_ID_FIELD = "_task_work_id"
@@ -42,6 +41,15 @@ class QueueTaskMetadata:
user_id: str = ""
+@dataclass
+class WorkTiming:
+ """Local timestamps matching the future persisted Work timing contract."""
+
+ enqueued_at: float
+ started_at: Optional[float] = None
+ finished_at: Optional[float] = None
+
+
_current_task_context: ContextVar[Optional[TaskExecutionContext]] = ContextVar(
"openviking_task_execution_context",
default=None,
@@ -160,7 +168,8 @@ class TaskWorkIndex:
self._work: Dict[str, set[tuple[str, str]]] = {}
self._active: Dict[str, set[asyncio.Task[Any]]] = {}
self._failures: Dict[str, str] = {}
- self._processing: Dict[str, ProcessingClock] = {}
+ self._work_timings: Dict[str, Dict[str, WorkTiming]] = {}
+ self._active_work_ids: Dict[tuple[str, asyncio.Task[Any]], str] = {}
self._finalize_before_ack: Optional[Callable[[QueueTaskMetadata], Awaitable[None]]] = None
self._is_cancellation_requested: Optional[Callable[[str], bool]] = None
@@ -201,6 +210,9 @@ class TaskWorkIndex:
if self.cancellation_requested(metadata.task_id):
return False
self._work.setdefault(metadata.task_id, set()).add((queue_name, metadata.work_id))
+ timings = self._work_timings.get(metadata.task_id)
+ if timings is not None:
+ timings.setdefault(metadata.work_id, WorkTiming(enqueued_at=time.time()))
return True
def _remove_work(
@@ -232,7 +244,12 @@ class TaskWorkIndex:
)
if metadata is None:
return
- _removed, became_idle = self._remove_work(queue_name, metadata)
+ removed, became_idle = self._remove_work(queue_name, metadata)
+ if removed:
+ with self._lock:
+ timings = self._work_timings.get(metadata.task_id)
+ if timings is not None:
+ timings.pop(metadata.work_id, None)
callback = self._finalize_before_ack
if became_idle and callback is not None:
await callback(metadata)
@@ -298,73 +315,69 @@ class TaskWorkIndex:
def init_processing(self, task_id: str) -> None:
with self._lock:
- self._processing.setdefault(task_id, ProcessingClock())
+ self._work_timings.setdefault(task_id, {})
def processing_seconds(self, task_id: str) -> Optional[float]:
with self._lock:
- clock = self._processing.get(task_id)
- return clock.seconds() if clock else None
+ timings = self._work_timings.get(task_id)
+ if timings is None:
+ return None
+ now = time.time()
+ intervals: list[tuple[float, float]] = []
+ for timing in timings.values():
+ if timing.started_at is None:
+ continue
+ end = timing.finished_at if timing.finished_at is not None else now
+ intervals.append((timing.started_at, max(timing.started_at, end)))
+ intervals.sort()
+ if not intervals:
+ return None
- def forget_processing(self, task_id: str) -> None:
- with self._lock:
- self._processing.pop(task_id, None)
+ elapsed = 0.0
+ current_start, current_end = intervals[0]
+ for start, end in intervals[1:]:
+ if start <= current_end:
+ current_end = max(current_end, end)
+ else:
+ elapsed += current_end - current_start
+ current_start, current_end = start, end
+ return elapsed + current_end - current_start
- @contextmanager
- def pause_processing(self, task_id: str, *, worker: Any = None) -> Iterator[None]:
- worker = worker if worker is not None else asyncio.current_task()
- with self._lock:
- clock = self._processing.get(task_id)
- was_processing = clock is not None and worker in clock.workers
- if was_processing:
- clock.leave(worker)
- try:
- yield
- finally:
- with self._lock:
- if was_processing and self._processing.get(task_id) is clock:
- clock.enter(worker)
-
- @contextmanager
- def measure_processing(self, task_id: str) -> Iterator[None]:
- # Timing ownership is separate from cancellation ownership: a shared DAG
- # worker must never become cancellable on behalf of one queued task.
- worker = object()
- token = processing_owner.set((self, task_id, worker))
+ def forget_processing(self, task_id: str) -> None:
with self._lock:
- clock = self._processing.get(task_id)
- if clock is not None:
- clock.enter(worker)
- try:
- yield
- finally:
- with self._lock:
- if clock is not None:
- clock.leave(worker)
- processing_owner.reset(token)
+ self._work_timings.pop(task_id, None)
def register_active(
self,
task_id: str,
active_task: asyncio.Task[Any],
+ *,
+ work_id: Optional[str] = None,
) -> None:
with self._lock:
cancelled = self.cancellation_requested(task_id)
if not cancelled:
self._active.setdefault(task_id, set()).add(active_task)
- clock = self._processing.get(task_id)
- if clock is not None:
- clock.enter(active_task)
- processing_owner.set((self, task_id, active_task))
+ timings = self._work_timings.get(task_id)
+ if timings is not None:
+ now = time.time()
+ timing_id = work_id or f"inline:{id(active_task)}"
+ timing = timings.setdefault(timing_id, WorkTiming(enqueued_at=now))
+ if timing.started_at is None:
+ timing.started_at = now
+ timing.finished_at = None
+ self._active_work_ids[(task_id, active_task)] = timing_id
if cancelled:
active_task.get_loop().call_soon(active_task.cancel)
def unregister_active(self, task_id: str, active_task: asyncio.Task[Any]) -> None:
- if processing_owner.get() == (self, task_id, active_task):
- processing_owner.set(None)
with self._lock:
- clock = self._processing.get(task_id)
- if clock is not None:
- clock.leave(active_task)
+ timing_id = self._active_work_ids.pop((task_id, active_task), None)
+ timings = self._work_timings.get(task_id)
+ if timing_id is not None and timings is not None:
+ timing = timings.get(timing_id)
+ if timing is not None and timing.started_at is not None:
+ timing.finished_at = max(timing.started_at, time.time())
entries = self._active.get(task_id)
if entries is not None:
entries.discard(active_task)
diff --git a/openviking/storage/queuefs/semantic_executor.py b/openviking/storage/queuefs/semantic_executor.py
index f41029520..61580f403 100644
--- a/openviking/storage/queuefs/semantic_executor.py
+++ b/openviking/storage/queuefs/semantic_executor.py
@@ -5,7 +5,6 @@
import asyncio
import re
import threading
-from contextlib import nullcontext
from dataclasses import dataclass, field
from typing import TYPE_CHECKING, Any, ClassVar, Dict, List, Mapping, Optional, Set
from weakref import WeakKeyDictionary
@@ -13,7 +12,6 @@ from weakref import WeakKeyDictionary
from openviking.core.namespace import classify_uri
from openviking.parse.parsers.media import get_media_type
from openviking.server.identity import RequestContext
-from openviking.service.task_processing_time import pause_task_processing, processing_owner
from openviking.service.task_tracker_concurrency import run_to_completion
from openviking.service.task_work_index import (
bind_task_context,
@@ -240,7 +238,6 @@ class SemanticTreeExecutor:
self._aggregate_directory = aggregate_directory
self._copy_source_uri = copy_source_uri
self._task_context = get_task_context()
- self._processing_index = None
self._telemetry = get_current_telemetry()
self._telemetry_id = telemetry_id
self._stale = False
@@ -386,36 +383,31 @@ class SemanticTreeExecutor:
)
return
- owner = processing_owner.get()
- if owner is not None and self._task_context is not None:
- if owner[1] == self._task_context.task_id:
- self._processing_index = owner[0]
- with pause_task_processing():
+ try:
+ self._register_active()
+ self._schedule_dir(root_uri, parent_uri=None)
+ await self._root_done.wait()
+ if self._failure:
+ raise self._failure
+ except BaseException:
+ self._closed = True
+ if self._context_type == "skill":
+ for task in self._skill_node_tasks:
+ task.cancel()
+ await run_to_completion(
+ lambda: asyncio.gather(*self._skill_node_tasks, return_exceptions=True)
+ )
+ await self._active_work_idle.wait()
+ raise
+ finally:
+ self._closed = True
try:
- self._register_active()
- self._schedule_dir(root_uri, parent_uri=None)
- await self._root_done.wait()
- if self._failure:
- raise self._failure
- except BaseException:
- self._closed = True
if self._context_type == "skill":
- for task in self._skill_node_tasks:
- task.cancel()
- await run_to_completion(
- lambda: asyncio.gather(*self._skill_node_tasks, return_exceptions=True)
- )
- await self._active_work_idle.wait()
- raise
+ await run_to_completion(self._active_work_idle.wait)
+ else:
+ await self._active_work_idle.wait()
finally:
- self._closed = True
- try:
- if self._context_type == "skill":
- await run_to_completion(self._active_work_idle.wait)
- else:
- await self._active_work_idle.wait()
- finally:
- self._unregister_active()
+ self._unregister_active()
def _schedule_work(self, work: SemanticTreeWork) -> None:
if self._closed:
@@ -521,19 +513,8 @@ class SemanticTreeExecutor:
if self._task_context is not None
else detach_task_context()
)
- timing = (
- self._processing_index.measure_processing(self._task_context.task_id)
- if self._processing_index is not None and self._task_context is not None
- else nullcontext()
- )
- # Shared workers inherit their creator's ContextVars. Never let an
- # unrelated node pause the creator's clock when it has no timing owner.
- token = processing_owner.set(None)
- try:
- with bind_telemetry(self._telemetry), task_context, timing:
- await self._run_work_bound(work)
- finally:
- processing_owner.reset(token)
+ with bind_telemetry(self._telemetry), task_context:
+ await self._run_work_bound(work)
async def _await_write(self, operation):
if self._context_type == "skill":
diff --git a/openviking/storage/queuefs/semantic_processor.py b/openviking/storage/queuefs/semantic_processor.py
index 53eed513c..c248ddced 100644
--- a/openviking/storage/queuefs/semantic_processor.py
+++ b/openviking/storage/queuefs/semantic_processor.py
@@ -35,7 +35,6 @@ from openviking.parse.parsers.media.utils import (
from openviking.prompts import render_prompt
from openviking.pyagfs.exceptions import AGFSNotADirectoryError
from openviking.server.identity import RequestContext, Role
-from openviking.service.task_processing_time import pause_task_processing
from openviking.service.task_tracker_concurrency import run_to_completion
from openviking.service.task_work_index import detach_task_context
from openviking.storage.abstract_overview import (
@@ -282,8 +281,7 @@ class SemanticProcessor(DequeueHandlerBase):
# Throttle to prevent re-enqueue storm during OPEN window
wait = self._account_breaker(msg.account_id).retry_after
if wait > 0:
- with pause_task_processing():
- await asyncio.sleep(wait)
+ await asyncio.sleep(wait)
queue_manager = get_queue_manager()
if queue_manager is not None:
diff --git a/openviking/telemetry/request_wait_tracker.py b/openviking/telemetry/request_wait_tracker.py
index 85223d798..8553a543c 100644
--- a/openviking/telemetry/request_wait_tracker.py
+++ b/openviking/telemetry/request_wait_tracker.py
@@ -10,8 +10,6 @@ import time
from dataclasses import dataclass, field
from typing import Callable, Dict, List, Optional, Set
-from openviking.service.task_processing_time import pause_task_processing
-
@dataclass
class _RequestWaitState:
@@ -271,14 +269,13 @@ class RequestWaitTracker:
) -> None:
if not telemetry_id:
return
- with pause_task_processing():
- start = time.time()
- while True:
- if self.is_complete(telemetry_id):
- return
- if timeout is not None and (time.time() - start) > timeout:
- raise TimeoutError(f"Request processing not complete after {timeout}s")
- await asyncio.sleep(poll_interval)
+ start = time.time()
+ while True:
+ if self.is_complete(telemetry_id):
+ return
+ if timeout is not None and (time.time() - start) > timeout:
+ raise TimeoutError(f"Request processing not complete after {timeout}s")
+ await asyncio.sleep(poll_interval)
async def wait_for_embeddings(
self,
@@ -290,18 +287,17 @@ class RequestWaitTracker:
"""Drain embeddings while the producing semantic root remains pending."""
if not telemetry_id:
return
- with pause_task_processing():
- while True:
- with self._lock:
- state = self._states.get(telemetry_id)
- if state is None or not state.pending_embedding_roots:
- return
- if stop_waiting is not None and stop_waiting():
- # Shutdown has drained the embedding consumer's active writes.
- # Leave this semantic delivery unacked for recovery rather than
- # waiting forever for embeddings still in the persistent queue.
- raise asyncio.CancelledError("Embedding worker stopped with queued work")
- await asyncio.sleep(poll_interval)
+ while True:
+ with self._lock:
+ state = self._states.get(telemetry_id)
+ if state is None or not state.pending_embedding_roots:
+ return
+ if stop_waiting is not None and stop_waiting():
+ # Shutdown has drained the embedding consumer's active writes.
+ # Leave this semantic delivery unacked for recovery rather than
+ # waiting forever for embeddings still in the persistent queue.
+ raise asyncio.CancelledError("Embedding worker stopped with queued work")
+ await asyncio.sleep(poll_interval)
def build_queue_status(self, telemetry_id: str) -> Dict[str, Dict[str, object]]:
with self._lock:
diff --git a/tests/service/test_task_processing_time.py b/tests/service/test_task_processing_time.py
index 35956ee10..9ca2ca9f3 100644
--- a/tests/service/test_task_processing_time.py
+++ b/tests/service/test_task_processing_time.py
@@ -1,15 +1,17 @@
# Copyright (c) 2026 Beijing Volcano Engine Technology Co., Ltd.
# SPDX-License-Identifier: AGPL-3.0
-"""Processing time excludes idle gaps and unions overlapping workers."""
+"""Processing time measures the wall-clock union of worker-owned execution."""
import asyncio
from types import SimpleNamespace
import pytest
-from openviking.service.task_processing_time import ProcessingClock
-from openviking.service.task_tracker_concurrency import run_to_completion
-from openviking.service.task_work_index import TaskWorkIndex, bind_task_context
+from openviking.service.task_work_index import (
+ QueueTaskMetadata,
+ TaskWorkIndex,
+ bind_task_context,
+)
from openviking.storage.queuefs.semantic_executor import (
SemanticTreeExecutor,
SemanticTreeScheduler,
@@ -17,36 +19,45 @@ from openviking.storage.queuefs.semantic_executor import (
from openviking.telemetry.request_wait_tracker import RequestWaitTracker
-def test_processing_clock_unions_workers_and_excludes_idle_time(monkeypatch):
+def test_work_timestamps_union_worker_occupancy_and_exclude_queue_time(monkeypatch):
now = [0.0]
- monkeypatch.setattr("openviking.service.task_processing_time.time.monotonic", lambda: now[0])
- clock = ProcessingClock()
- assert clock.seconds() is None
+ monkeypatch.setattr("openviking.service.task_work_index.time.time", lambda: now[0])
+ index = TaskWorkIndex()
+ index.init_processing("task")
+ first = QueueTaskMetadata("task", "first")
+ second = QueueTaskMetadata("task", "second")
+ assert index.register("Semantic", first)
+ assert index.processing_seconds("task") is None
+
now[0] = 10 # Initial queue wait.
- clock.enter("parse")
+ first_worker = object()
+ index.register_active("task", first_worker, work_id=first.work_id)
now[0] = 12
- clock.enter("embedding")
+ assert index.register("Embedding", second)
+ second_worker = object()
+ index.register_active("task", second_worker, work_id=second.work_id)
now[0] = 15
- clock.leave("parse")
+ index.unregister_active("task", first_worker)
now[0] = 18
- clock.leave("embedding")
- assert clock.seconds() == 8
+ index.unregister_active("task", second_worker)
+ assert index.processing_seconds("task") == 8
now[0] = 40 # No worker: waiting for another queue.
- assert clock.seconds() == 8
- clock.enter("semantic")
+ assert index.processing_seconds("task") == 8
+ third_worker = object()
+ index.register_active("task", third_worker)
now[0] = 43
- assert clock.seconds() == 11
- clock.leave("semantic")
+ assert index.processing_seconds("task") == 11
+ index.unregister_active("task", third_worker)
now[0] = 100
- assert clock.seconds() == 11
+ assert index.processing_seconds("task") == 11
@pytest.mark.asyncio
-async def test_saturated_dag_scheduler_counts_only_dispatched_nodes(monkeypatch):
+async def test_saturated_dag_scheduler_counts_claimed_handler_time(monkeypatch):
now = [0.0]
monkeypatch.setattr(
- "openviking.service.task_processing_time.time",
- SimpleNamespace(monotonic=lambda: now[0]),
+ "openviking.service.task_work_index.time",
+ SimpleNamespace(time=lambda: now[0]),
)
index = TaskWorkIndex()
scheduler = SemanticTreeScheduler(max_workers=1)
@@ -80,8 +91,13 @@ async def test_saturated_dag_scheduler_counts_only_dispatched_nodes(monkeypatch)
monkeypatch.setattr(executors["b"], "_schedule_dir", queue_b)
async def coordinate(name):
- with index.measure_processing(name):
+ worker = asyncio.current_task()
+ assert worker is not None
+ index.register_active(name, worker)
+ try:
await executors[name].run("viking://resources/" + name)
+ finally:
+ index.unregister_active(name, worker)
first = asyncio.create_task(coordinate("a"))
await entered["a"].wait()
@@ -89,14 +105,16 @@ async def test_saturated_dag_scheduler_counts_only_dispatched_nodes(monkeypatch)
await queued.wait()
now[0] = 20
assert index.processing_seconds("a") == 20
- assert index.processing_seconds("b") == 0 # Still queued behind A.
+ # B's QueueFS handler already owns a worker while its internal DAG node
+ # waits for the shared scheduler.
+ assert index.processing_seconds("b") == 20
release["a"].set()
await entered["b"].wait()
now[0] = 25
- assert index.processing_seconds("b") == 5
+ assert index.processing_seconds("b") == 25
release["b"].set()
await asyncio.gather(first, second)
- assert index.processing_seconds("b") == 5
+ assert index.processing_seconds("b") == 25
for worker in list(scheduler._workers):
worker.cancel()
await asyncio.gather(*scheduler._workers, return_exceptions=True)
@@ -104,11 +122,11 @@ async def test_saturated_dag_scheduler_counts_only_dispatched_nodes(monkeypatch)
@pytest.mark.asyncio
@pytest.mark.parametrize("method", ["wait_for_request", "wait_for_embeddings"])
-async def test_downstream_wait_pauses_inherited_owner_in_shielded_child(monkeypatch, method):
+async def test_downstream_wait_remains_worker_occupied_time(monkeypatch, method):
now = [0.0]
monkeypatch.setattr(
- "openviking.service.task_processing_time.time",
- SimpleNamespace(monotonic=lambda: now[0]),
+ "openviking.service.task_work_index.time",
+ SimpleNamespace(time=lambda: now[0]),
)
index = TaskWorkIndex()
index.init_processing("task")
@@ -116,67 +134,35 @@ async def test_downstream_wait_pauses_inherited_owner_in_shielded_child(monkeypa
request = "processing-time-" + method
tracker.register_request(request)
tracker.register_embedding_root(request, "embedding")
- with index.measure_processing("task"):
+ worker = asyncio.current_task()
+ assert worker is not None
+ index.register_active("task", worker)
+ try:
now[0] = 3
- waiter = asyncio.create_task(
- run_to_completion(lambda: getattr(tracker, method)(request, poll_interval=0.001))
- )
- # Let run_to_completion and its shielded child enter the wait loop.
+ waiter = asyncio.create_task(getattr(tracker, method)(request, poll_interval=0.001))
await asyncio.sleep(0.01)
now[0] = 20
- assert index.processing_seconds("task") == 3
+ assert index.processing_seconds("task") == 20
tracker.mark_embedding_done(request, "embedding")
await waiter
now[0] = 22
- assert index.processing_seconds("task") == 5
+ assert index.processing_seconds("task") == 22
+ finally:
+ index.unregister_active("task", worker)
tracker.cleanup(request)
-async def test_nested_wait_and_cancellation_restore_processing_owner(monkeypatch):
- from openviking.service.task_processing_time import pause_task_processing, processing_owner
-
- now = [0.0]
- monkeypatch.setattr(
- "openviking.service.task_processing_time.time",
- SimpleNamespace(monotonic=lambda: now[0]),
- )
- index = TaskWorkIndex()
- index.init_processing("task")
- entered = asyncio.Event()
-
- async def work():
- with index.measure_processing("task"):
- now[0] = 2
- try:
- with pause_task_processing():
- with pause_task_processing():
- entered.set()
- await asyncio.Event().wait()
- except asyncio.CancelledError:
- now[0] += 3 # Cancellation cleanup resumes processing.
- assert processing_owner.get() is None
-
- task = asyncio.create_task(work())
- await entered.wait()
- now[0] = 20
- assert index.processing_seconds("task") == 2
- task.cancel()
- await task
- assert index.processing_seconds("task") == 5
-
-
@pytest.mark.asyncio
@pytest.mark.parametrize("custom_enqueue", [False, True])
-async def test_semantic_retry_cooldown_excludes_wait_but_counts_enqueue(
+async def test_semantic_retry_cooldown_counts_full_worker_occupied_time(
monkeypatch, custom_enqueue
):
from openviking.storage.queuefs.semantic_processor import SemanticProcessor
now = [0.0]
monkeypatch.setattr(
- "openviking.service.task_processing_time.time",
- SimpleNamespace(monotonic=lambda: now[0]),
+ "openviking.service.task_work_index.time",
+ SimpleNamespace(time=lambda: now[0]),
)
index = TaskWorkIndex()
index.init_processing("task")
@@ -186,7 +172,7 @@ async def test_semantic_retry_cooldown_excludes_wait_but_counts_enqueue(
async def sleep(delay):
assert delay == 30
now[0] += delay
- assert index.processing_seconds("task") == 3
+ assert index.processing_seconds("task") == 33
async def enqueue(msg):
assert msg is message
@@ -218,4 +204,4 @@ async def test_semantic_retry_cooldown_excludes_wait_but_counts_enqueue(
index.unregister_active("task", worker)
assert enqueued == [message]
- assert index.processing_seconds("task") == 5
+ assert index.processing_seconds("task") == 35
diff --git a/tests/test_task_tracker.py b/tests/test_task_tracker.py
index 796c00458..f7ecfa33c 100644
--- a/tests/test_task_tracker.py
+++ b/tests/test_task_tracker.py
@@ -980,7 +980,7 @@ async def test_process_events_respect_owner_and_terminal_boundaries(tracker):
async def test_processing_time_persists_and_restored_active_tasks_are_unknown(tracker, monkeypatch):
now = [0.0]
- monkeypatch.setattr("openviking.service.task_processing_time.time.monotonic", lambda: now[0])
+ monkeypatch.setattr("openviking.service.task_work_index.time.time", lambda: now[0])
task = await tracker.create("add_resource", **_owner_kwargs())
await tracker.start(task.task_id, **_owner_kwargs())
now[0] = 10
@@ -991,14 +991,15 @@ async def test_processing_time_persists_and_restored_active_tasks_are_unknown(tr
await tracker.update_stage(task.task_id, "semantic", **_owner_kwargs())
restarted = TaskTracker(store=tracker._store)
assert (await restarted.get(task.task_id, **_owner_kwargs())).processing_seconds is None
- with tracker._work_index.pause_processing(task.task_id):
- now[0] = 30
- assert (await tracker.get(task.task_id)).processing_seconds == 3
+ # Waiting, I/O, and backoff after a worker claims the task remain part of
+ # worker-occupied wall time.
+ now[0] = 30
+ assert (await tracker.get(task.task_id)).processing_seconds == 20
now[0] = 32
await tracker.complete(task.task_id, result={"ok": True}, **_owner_kwargs())
await tracker.unregister_running_task(task.task_id)
finished = await tracker.get(task.task_id)
assert finished.status == TaskStatus.COMPLETED
- assert finished.to_dict()["processing_seconds"] == 5
+ assert finished.to_dict()["processing_seconds"] == 22
restored = await TaskTracker(store=tracker._store).get(task.task_id, **_owner_kwargs())
- assert restored.processing_seconds == 5
+ assert restored.processing_seconds == 22
diff --git a/web-studio/src/i18n/locales/en/workspace.ts b/web-studio/src/i18n/locales/en/workspace.ts
index aa5430b39..972091f85 100644
--- a/web-studio/src/i18n/locales/en/workspace.ts
+++ b/web-studio/src/i18n/locales/en/workspace.ts
@@ -396,7 +396,7 @@ const workspace = {
totalDuration: 'Total Time',
processingNotStarted: 'Not started',
processingDurationHelp:
- 'Worker processing time, including model and I/O calls. Excludes queue and downstream waits; overlapping workers count once. Incomplete or legacy records are unavailable.',
+ 'Processing time is wall-clock time after at least one Work is claimed by a worker, including model calls, I/O, downstream waits, and backoff; overlapping workers count once. Waiting time is total time minus processing time, mainly queue and scheduling gaps.',
processingDuration: 'Processing Time',
waitingDuration: 'Waiting Time',
timingUnavailable: 'Not recorded',
diff --git a/web-studio/src/i18n/locales/zh-CN/workspace.ts b/web-studio/src/i18n/locales/zh-CN/workspace.ts
index e19f9d0fa..8416aec09 100644
--- a/web-studio/src/i18n/locales/zh-CN/workspace.ts
+++ b/web-studio/src/i18n/locales/zh-CN/workspace.ts
@@ -386,7 +386,7 @@ const workspace = {
totalDuration: '总耗时',
processingNotStarted: '未开始',
processingDurationHelp:
- '仅累计工序实际处理区间,含模型与 I/O 调用;排除排队和等待下游,并行重叠只计一次。旧任务或重启后记录不完整时显示未记录。',
+ '处理耗时为至少一个 Work 已被 Worker 领取后的墙钟时间,包含模型调用、I/O、下游等待和退避,并行重叠只计一次;等待耗时为总耗时减去处理耗时,主要包含排队和调度空档。',
processingDuration: '处理耗时',
waitingDuration: '等待耗时',
timingUnavailable: '未记录',
diff --git a/web-studio/src/routes/tasks/-lib/task-time.ts b/web-studio/src/routes/tasks/-lib/task-time.ts
index b751f42d8..7a8d93540 100644
--- a/web-studio/src/routes/tasks/-lib/task-time.ts
+++ b/web-studio/src/routes/tasks/-lib/task-time.ts
@@ -97,6 +97,7 @@ export function formatTaskProcessingDuration(
: formatDurationString(Math.floor(seconds))
}
+/** Task wall time not covered by a locally observed worker-owned interval. */
export function formatTaskWaitingDuration(
task: TaskTimestamp,
): string | undefined {