Skip to content

Self-evolving Context Database for AI Agents. Unify Agent Memory, Knowledge RAG and Skills.

refactor(tasks): unify task and queue work tracking (#6043)

runyunzhouec56b9443750
diff --git a/openviking/server/routers/filesystem.py b/openviking/server/routers/filesystem.py
index d3768cae5..064140023 100644
--- a/openviking/server/routers/filesystem.py
+++ b/openviking/server/routers/filesystem.py
@@ -332,6 +332,8 @@ async def rm(
response_result["semantic_status"] = result["semantic_status"]
if isinstance(result, dict) and "queue_status" in result:
response_result["queue_status"] = result["queue_status"]
+ if isinstance(result, dict) and "task_id" in result:
+ response_result["task_id"] = result["task_id"]
return Response(status="ok", result=response_result)
diff --git a/openviking/service/fs_service.py b/openviking/service/fs_service.py
index 5b1910751..d61a7e033 100644
--- a/openviking/service/fs_service.py
+++ b/openviking/service/fs_service.py
@@ -27,6 +27,8 @@ from openviking.privacy import (
from openviking.resource.uri_mutation_coordinator import UriMutationCoordinator
from openviking.resource.watch_storage import is_watch_task_control_uri
from openviking.server.identity import RequestContext
+from openviking.service.task_tracker import get_task_tracker, run_tracked_operation
+from openviking.service.task_work_index import get_enqueued_work_ids, get_task_context
from openviking.session.memory.memory_updater import MemoryUpdater
from openviking.session.memory.utils.content_visibility import visible_content
from openviking.storage.abstract_overview import (
@@ -45,12 +47,12 @@ from openviking.storage.vector_ids import is_vector_record_id
from openviking.storage.viking_fs import VikingFS
from openviking.storage.vikingdb_manager import VikingDBManagerProxy
from openviking.telemetry import get_current_telemetry
-from openviking.telemetry.request_wait_tracker import get_request_wait_tracker
from openviking.utils.embedding_utils import vectorize_directory_meta
from openviking.utils.ingest_options import IngestOptions
from openviking.utils.path_safety import validate_safe_viking_uri_path
from openviking.utils.tags import normalize_search_tags
from openviking_cli.exceptions import (
+ DeadlineExceededError,
InvalidArgumentError,
NotFoundError,
NotInitializedError,
@@ -454,6 +456,31 @@ class FSService:
ctx: RequestContext,
recursive: bool = False,
wait: bool = False,
+ ) -> Optional[Dict[str, Any]]:
+ async def execute() -> Optional[Dict[str, Any]]:
+ return await self._rm(
+ uri,
+ ctx,
+ recursive=recursive,
+ wait=wait,
+ )
+
+ result, _task_id = await run_tracked_operation(
+ "fs_rm",
+ account_id=ctx.account_id,
+ user_id=ctx.user.user_id,
+ operation=execute,
+ wait=wait,
+ resource_id=uri,
+ )
+ return result
+
+ async def _rm(
+ self,
+ uri: str,
+ ctx: RequestContext,
+ recursive: bool = False,
+ wait: bool = False,
) -> Optional[Dict[str, Any]]:
"""Remove resource."""
viking_fs = self._ensure_initialized()
@@ -470,13 +497,8 @@ class FSService:
refresh_parent_uri = None
queue_status = None
refresh_action: Optional[FreshnessAction] = None
- request_registered = False
- telemetry_id = get_current_telemetry().telemetry_id
try:
if refresh_parent_uri:
- if wait and telemetry_id:
- get_request_wait_tracker().register_request(telemetry_id)
- request_registered = True
refresh_action = await self._enqueue_delete_refresh(
root_uri=refresh_parent_uri,
deleted_uri=uri,
@@ -505,12 +527,9 @@ class FSService:
ctx=ctx,
)
if refresh_parent_uri and wait and refresh_action is not FreshnessAction.MARK_PENDING:
- tracker = get_request_wait_tracker()
- await tracker.wait_for_request(telemetry_id)
- queue_status = tracker.build_queue_status(telemetry_id)
+ queue_status = await self._wait_for_refresh()
finally:
- if request_registered:
- get_request_wait_tracker().cleanup(telemetry_id)
+ pass
if cleanup_result is not None and isinstance(result, dict):
result["memory_cleanup"] = cleanup_result
if refresh_parent_uri and isinstance(result, dict):
@@ -651,14 +670,7 @@ class FSService:
changes={"deleted": [deleted_uri]},
generation_trigger="content_delete",
)
- if telemetry_id:
- get_request_wait_tracker().register_semantic_root(telemetry_id, msg.id)
- try:
- await semantic_queue.enqueue(msg)
- except Exception as exc:
- if telemetry_id:
- get_request_wait_tracker().mark_semantic_failed(telemetry_id, msg.id, str(exc))
- raise
+ await semantic_queue.enqueue(msg)
return decision.action
async def _enqueue_copy_refresh(
@@ -707,16 +719,30 @@ class FSService:
generation_trigger="content_copy",
copy_source_uri=source_uri,
)
- if telemetry_id:
- get_request_wait_tracker().register_semantic_root(telemetry_id, msg.id)
- try:
- await semantic_queue.enqueue(msg)
- except Exception as exc:
- if telemetry_id:
- get_request_wait_tracker().mark_semantic_failed(telemetry_id, msg.id, str(exc))
- raise
+ await semantic_queue.enqueue(msg)
return "queued"
+ async def _wait_for_refresh(self) -> Dict[str, Any]:
+ context = get_task_context()
+ if context is None:
+ raise RuntimeError("TaskContext is required for request-scoped queue waiting")
+ work_ids = get_enqueued_work_ids()
+ tracker = get_task_tracker()
+ try:
+ if context.work_id is None:
+ await tracker.wait_for_task_work(context.task_id, timeout=None)
+ elif work_ids:
+ await tracker.wait_for_work(
+ context.task_id,
+ work_ids,
+ timeout=None,
+ )
+ except (TimeoutError, asyncio.TimeoutError) as exc:
+ raise DeadlineExceededError("queue processing", None) from exc
+ return tracker.build_queue_status(
+ context.task_id, work_ids if context.work_id is not None else None
+ )
+
async def cp(
self,
from_uri: str,
diff --git a/openviking/service/reindex_executor.py b/openviking/service/reindex_executor.py
index bd8131a67..f44ff7b5d 100644
--- a/openviking/service/reindex_executor.py
+++ b/openviking/service/reindex_executor.py
@@ -26,7 +26,8 @@ from openviking.core.namespace import (
)
from openviking.server.dependencies import get_service
from openviking.server.identity import RequestContext
-from openviking.service.task_tracker import get_task_tracker
+from openviking.service.task_tracker import get_task_tracker, run_tracked_operation
+from openviking.service.task_work_index import get_task_context
from openviking.session.memory.utils.memory_file_utils import MemoryFileUtils
from openviking.storage.abstract_overview import body_for_preview, embedding_text_for_body
from openviking.storage.queuefs.embedding_msg_converter import EmbeddingMsgConverter
@@ -34,7 +35,6 @@ from openviking.storage.queuefs.semantic_msg import SemanticMsg
from openviking.storage.queuefs.semantic_processor import SemanticProcessor
from openviking.storage.viking_fs import get_viking_fs
from openviking.telemetry import get_current_telemetry
-from openviking.telemetry.request_wait_tracker import get_request_wait_tracker
from openviking.utils.embedding_input import truncate_embedding_input
from openviking.utils.embedding_utils import (
_apply_ingest_options,
@@ -218,26 +218,41 @@ class ReindexExecutor:
tracker = get_task_tracker()
if wait:
- if await tracker.has_running(
+ task = await tracker.create_if_no_running(
REINDEX_TASK_TYPE,
uri,
account_id=ctx.account_id,
user_id=ctx.user.user_id,
- ):
+ )
+ if task is None:
raise OpenVikingError(
f"URI {uri} already has a reindex in progress",
code="CONFLICT",
details={"uri": uri},
)
- return await self._run(
- uri=uri,
- object_type=object_type,
- mode=mode,
- force=force,
- recursive=recursive,
- ingest_options=ingest_options,
- ctx=ctx,
+
+ async def execute_run() -> dict[str, Any]:
+ return await self._run(
+ uri=uri,
+ object_type=object_type,
+ mode=mode,
+ force=force,
+ recursive=recursive,
+ ingest_options=ingest_options,
+ ctx=ctx,
+ )
+
+ result, _task_id = await run_tracked_operation(
+ REINDEX_TASK_TYPE,
+ account_id=ctx.account_id,
+ user_id=ctx.user.user_id,
+ operation=execute_run,
+ wait=True,
+ resource_id=uri,
+ existing_task=task,
+ accept_failed_result=self._accept_partial_result,
)
+ return result
task = await tracker.create_if_no_running(
REINDEX_TASK_TYPE,
@@ -314,6 +329,35 @@ class ReindexExecutor:
"mode": mode,
}
+ async def execute_in_current_task(
+ self,
+ *,
+ uri: str,
+ mode: str,
+ ctx: RequestContext,
+ ) -> dict[str, Any]:
+ """Run reindex work under an already-bound task.
+
+ This is for composite operations such as snapshot restore. Public
+ reindex requests must use execute(), which owns admin task creation and
+ URI-level duplicate detection.
+ """
+ if get_task_context() is None:
+ raise RuntimeError("TaskContext is required for nested reindex execution")
+ object_type = self._infer_target_type(uri)
+ self._validate_mode(object_type, mode)
+ return await self._run(
+ uri=uri,
+ object_type=object_type,
+ mode=mode,
+ ctx=ctx,
+ )
+
+ @staticmethod
+ def _accept_partial_result(result: Any) -> bool:
+ """Keep record-level reindex failures in the structured result."""
+ return isinstance(result, dict) and int(result.get("failed_records", 0) or 0) > 0
+
@staticmethod
def _resolve_ingest_options(
*,
@@ -569,10 +613,6 @@ class ReindexExecutor:
started_at = time.perf_counter()
counters = _ReindexCounters()
telemetry_id = get_current_telemetry().telemetry_id
- wait_tracker = get_request_wait_tracker()
- if telemetry_id:
- wait_tracker.register_request(telemetry_id)
-
acquire_lock = service.viking_fs._async_agfs.pathlock_acquire_tree
stat = await service.viking_fs.stat(uri, ctx=ctx, skip_count=True)
root_is_dir = bool(stat.get("isDir", stat.get("is_dir")))
@@ -633,22 +673,23 @@ class ReindexExecutor:
details={"uri": uri},
)
- if telemetry_id:
- await wait_tracker.wait_for_request(telemetry_id)
+ task_context = get_task_context()
+ if task_context is not None:
+ await get_task_tracker().wait_for_task_work(
+ task_context.task_id, exclude_work_id=task_context.work_id
+ )
self._apply_semantic_tree_stats(
counters,
SemanticProcessor.consume_tree_stats(telemetry_id=telemetry_id),
)
self._apply_embedding_wait_status(
counters,
- wait_tracker.build_queue_status(telemetry_id),
+ get_task_tracker().build_queue_status(task_context.task_id),
)
finally:
active_lease = run.lease if run is not None else lease
if active_lease is not None and (run is None or not run.lease_handed_off):
await service.viking_fs._async_agfs.pathlock_release(active_lease)
- if telemetry_id:
- wait_tracker.cleanup(telemetry_id)
return {
"status": "completed",
@@ -842,23 +883,10 @@ class ReindexExecutor:
propagate_to_parent=recursive,
plan=semantic_plan,
)
- tracker = get_request_wait_tracker()
- if telemetry_id:
- tracker.register_semantic_root(telemetry_id, msg.id)
-
queue_manager = get_queue_manager()
semantic_queue = queue_manager.get_queue(queue_manager.SEMANTIC, allow_create=True)
if run.lease is None:
- try:
- enqueue_id = await semantic_queue.enqueue(msg)
- except Exception:
- if telemetry_id:
- tracker.mark_semantic_failed(
- telemetry_id, msg.id, "semantic plan enqueue failed"
- )
- raise
- if enqueue_id == "deduplicated" and telemetry_id:
- tracker.mark_semantic_done(telemetry_id, msg.id, processed_delta=0)
+ await semantic_queue.enqueue(msg)
return
viking_fs = get_service().viking_fs
@@ -895,8 +923,6 @@ class ReindexExecutor:
run.lease_handed_off = True
run.lease = None
run.lock = None
- if telemetry_id:
- tracker.mark_semantic_done(telemetry_id, msg.id, processed_delta=0)
return
if not acquired_plan_lease:
run.lease_handed_off = True
@@ -912,8 +938,6 @@ class ReindexExecutor:
run.lock = await agfs.pathlock_as_borrowed(run.lease)
elif acquired_plan_lease:
await agfs.pathlock_release(plan_lease)
- if telemetry_id:
- tracker.mark_semantic_failed(telemetry_id, msg.id, "semantic plan enqueue failed")
raise
async def _reindex_skill(
@@ -1579,17 +1603,11 @@ class ReindexExecutor:
code="FAILED_PRECONDITION",
details={"uri": uri},
)
- wait_tracker = get_request_wait_tracker()
- wait_tracker.register_embedding_root(msg.telemetry_id, msg.id)
+ enqueue_error = f"Failed to enqueue reindex vector for {uri}"
enqueued = await service.vikingdb_manager.enqueue_embedding_msg(msg)
if not enqueued:
- wait_tracker.mark_embedding_failed(
- msg.telemetry_id,
- msg.id,
- f"Failed to enqueue reindex vector for {uri}",
- )
raise OpenVikingError(
- f"Failed to enqueue reindex vector for {uri}",
+ enqueue_error,
code="PROCESSING_ERROR",
details={"uri": uri, "level": int(level)},
)
diff --git a/openviking/service/resource_service.py b/openviking/service/resource_service.py
index 308a0410f..7d72cf23e 100644
--- a/openviking/service/resource_service.py
+++ b/openviking/service/resource_service.py
@@ -67,7 +67,6 @@ from openviking.storage.queuefs.add_resource_msg import AddResourcePhase
from openviking.storage.viking_fs import LS_ALL_NODES, VikingFS
from openviking.storage.vikingdb_manager import VikingDBManager
from openviking.telemetry import get_current_telemetry, register_telemetry, unregister_telemetry
-from openviking.telemetry.request_wait_tracker import get_request_wait_tracker
from openviking.utils import is_git_repo_url, is_github_url, parse_code_hosting_url
from openviking.utils.git_auth import (
GitHttpAuthConfig,
@@ -84,6 +83,7 @@ from openviking.utils.resource_processor import ResourceProcessor
from openviking.utils.skill_processor import SkillProcessingPreparation, SkillProcessor
from openviking_cli.exceptions import (
ConflictError,
+ DeadlineExceededError,
FailedPreconditionError,
InternalError,
InvalidArgumentError,
@@ -497,7 +497,9 @@ class ResourceService:
self._runtime_config_manager,
ctx.account_id,
)
- app_credentials = self._load_feishu_credentials_for_watch(app_id, app_secret, feishu_config)
+ app_credentials = self._load_feishu_credentials_for_watch(
+ app_id, app_secret, feishu_config
+ )
watch_auth_state = create_feishu_auth_state(
token,
refresh_token.strip(),
@@ -657,6 +659,30 @@ class ResourceService:
await asyncio.gather(*tasks, return_exceptions=True)
self._background_tasks.clear()
+ def _unregister_telemetry_after_task(
+ self,
+ task_id: str,
+ telemetry_id: str,
+ *,
+ account_id: str,
+ user_id: str,
+ ) -> None:
+ async def wait_and_unregister() -> None:
+ from openviking.service.task_tracker import get_task_tracker
+
+ try:
+ await get_task_tracker().wait(
+ task_id,
+ account_id=account_id,
+ user_id=user_id,
+ )
+ finally:
+ unregister_telemetry(telemetry_id)
+
+ background = asyncio.create_task(wait_and_unregister())
+ self._background_tasks.add(background)
+ background.add_done_callback(self._background_tasks.discard)
+
async def _enqueue_add_resource_job(
self,
msg: Any,
@@ -2256,42 +2282,6 @@ class ResourceService:
logger.warning("[ResourceService] Failed to link resource reason memory: %s", exc)
result.setdefault("warnings", []).append(f"Memory linking failed: {exc}")
- async def _monitor_queue_processing(
- self,
- task_id: str,
- telemetry_id: str,
- account_id: str,
- user_id: str,
- ) -> None:
- from openviking.service.task_tracker import get_task_tracker
-
- task_tracker = get_task_tracker()
- request_wait_tracker = get_request_wait_tracker()
- await task_tracker.start(task_id, account_id=account_id, user_id=user_id)
- try:
- await request_wait_tracker.wait_for_request(telemetry_id)
- status = request_wait_tracker.build_queue_status(telemetry_id)
- errors = sum(int(group.get("error_count", 0) or 0) for group in status.values())
- if errors:
- await task_tracker.fail(
- task_id,
- f"queue processing failed: {status}",
- account_id=account_id,
- user_id=user_id,
- )
- else:
- await task_tracker.complete(
- task_id,
- {"queue_status": status},
- account_id=account_id,
- user_id=user_id,
- )
- except Exception as exc:
- await task_tracker.fail(task_id, str(exc), account_id=account_id, user_id=user_id)
- finally:
- request_wait_tracker.cleanup(telemetry_id)
- unregister_telemetry(telemetry_id)
-
@staticmethod
def _raise_queue_status_errors(status: Dict[str, Any]) -> None:
failed = {
@@ -2440,18 +2430,15 @@ class ResourceService:
ctx=ctx,
server_config=get_server_config(),
)
- telemetry_id = get_current_telemetry().telemetry_id
- request_wait_tracker = get_request_wait_tracker()
- monitor_started = False
- processing_started = False
from openviking.service.task_tracker import get_task_tracker
from openviking.service.task_tracker_concurrency import run_to_completion
from openviking.service.task_work_index import bind_task_context
task_tracker = get_task_tracker()
task = None
- if telemetry_id:
- request_wait_tracker.register_request(telemetry_id)
+ outcome_recorded = False
+ telemetry_id = get_current_telemetry().telemetry_id
+ telemetry_cleanup_scheduled = False
try:
# Establish ownership before any queue work can start. Descendant
@@ -2468,6 +2455,7 @@ class ResourceService:
# Keep the returned record before propagating cancellation, so a
# committed task can still be settled by the exception handler.
await run_to_completion(create_skill_task)
+ assert task is not None
await task_tracker.start(
task.task_id, account_id=ctx.account_id, user_id=ctx.user.user_id
)
@@ -2498,73 +2486,84 @@ class ResourceService:
source_metadata=source_metadata,
owner_lease_ref=owner_lease_ref,
)
- finally:
- await task_tracker.unregister_running_task(task.task_id)
- processing_started = True
- if not wait:
- monitor_started = True
- asyncio.create_task(
- self._monitor_queue_processing(
- task.task_id, telemetry_id, ctx.account_id, ctx.user.user_id
+ if isinstance(result, dict) and "root_uri" not in result and result.get("uri"):
+ result["root_uri"] = result["uri"]
+
+ result["task_id"] = task.task_id
+ if telemetry_id:
+ self._unregister_telemetry_after_task(
+ task.task_id,
+ telemetry_id,
+ account_id=ctx.account_id,
+ user_id=ctx.user.user_id,
)
+ telemetry_cleanup_scheduled = True
+ await task_tracker.complete(
+ task.task_id,
+ result,
+ account_id=ctx.account_id,
+ user_id=ctx.user.user_id,
)
- if isinstance(result, dict) and "root_uri" not in result and result.get("uri"):
- result["root_uri"] = result["uri"]
-
+ outcome_recorded = True
+ except BaseException as exc:
+ failure_message = str(exc) or "Skill processing cancelled"
+ await run_to_completion(
+ lambda: task_tracker.fail(
+ task.task_id,
+ failure_message,
+ account_id=ctx.account_id,
+ user_id=ctx.user.user_id,
+ )
+ )
+ outcome_recorded = True
+ raise
+ finally:
+ await task_tracker.unregister_running_task(task.task_id)
if wait:
wait_start = time.perf_counter()
- await request_wait_tracker.wait_for_request(telemetry_id)
- status = request_wait_tracker.build_queue_status(telemetry_id)
+ try:
+ completed = await task_tracker.wait(
+ task.task_id,
+ account_id=ctx.account_id,
+ user_id=ctx.user.user_id,
+ timeout=None,
+ )
+ except (TimeoutError, asyncio.TimeoutError) as exc:
+ get_current_telemetry().set_error(
+ "resource_service.wait_complete",
+ "DEADLINE_EXCEEDED",
+ str(exc),
+ )
+ raise DeadlineExceededError("queue processing", None) from exc
get_current_telemetry().set(
"queue.wait.duration_ms",
round((time.perf_counter() - wait_start) * 1000, 3),
)
+ status = task_tracker.build_queue_status(task.task_id)
result["queue_status"] = status
self._raise_queue_status_errors(status)
- await task_tracker.complete(
- task.task_id,
- {"queue_status": status},
- account_id=ctx.account_id,
- user_id=ctx.user.user_id,
- )
# Cancellation settles queue entries without counting errors.
- # complete() preserves cancellation, so check it before the
- # update caller commits the package and discards its backup.
- if task_tracker.is_cancellation_requested(task.task_id):
+ if completed.status.value == "cancelled":
raise OpenVikingError(
"Skill processing was cancelled",
code="PROCESSING_ERROR",
details={"task_id": task.task_id, "status": "cancelled"},
)
- else:
- result["task_id"] = task.task_id
-
return result
except BaseException as exc:
- if task is not None and not monitor_started:
- if processing_started and not request_wait_tracker.is_complete(telemetry_id):
- # A cancelled standalone request leaves processing in the background.
- # An update's caller separately cancels before restoring.
- monitor_started = True
- asyncio.create_task(
- self._monitor_queue_processing(
- task.task_id, telemetry_id, ctx.account_id, ctx.user.user_id
- )
- )
- else:
- failure_message = str(exc) or "Skill processing cancelled"
- await run_to_completion(
- lambda: task_tracker.fail(
- task.task_id,
- failure_message,
- account_id=ctx.account_id,
- user_id=ctx.user.user_id,
- )
+ if task is not None and not outcome_recorded:
+ failure_message = str(exc) or "Skill processing cancelled"
+ await run_to_completion(
+ lambda: task_tracker.fail(
+ task.task_id,
+ failure_message,
+ account_id=ctx.account_id,
+ user_id=ctx.user.user_id,
)
+ )
raise
finally:
- if not monitor_started:
- request_wait_tracker.cleanup(telemetry_id)
+ if not telemetry_cleanup_scheduled:
unregister_telemetry(telemetry_id)
async def cancel_skill_processing(self, task_id: str, ctx: RequestContext) -> None:
diff --git a/openviking/service/task_queue_middleware.py b/openviking/service/task_queue_middleware.py
index 64006c570..c542cae9a 100644
--- a/openviking/service/task_queue_middleware.py
+++ b/openviking/service/task_queue_middleware.py
@@ -10,6 +10,7 @@ from openviking.service.task_work_index import (
bind_task_context,
extract_task_metadata,
prepare_task_payload,
+ record_enqueued_work,
)
from openviking.storage.queuefs.process_result import ProcessOutcome, ProcessResult
from openviking.storage.queuefs.queue_middleware import (
@@ -35,15 +36,38 @@ class TaskWorkQueueMiddleware(QueueMiddleware):
if not isinstance(ctx.payload, dict):
return await call_next(ctx)
ctx.payload, metadata = prepare_task_payload(ctx.payload)
+ ctx.work_id = metadata.work_id if metadata is not None else None
if not self._index.register(ctx.queue, metadata):
raise TaskWorkRejected(
f"Task {metadata.task_id} is cancelling; rejected work for {ctx.queue}"
)
try:
- return await call_next(ctx)
- finally:
- if metadata is not None and not ctx.committed:
+ result = await call_next(ctx)
+ except asyncio.CancelledError:
+ if metadata is not None:
+ if ctx.committed:
+ record_enqueued_work(ctx.queue, metadata.work_id)
+ else:
+ await self._index.discard(ctx.queue, metadata)
+ raise
+ except BaseException as exc:
+ if metadata is not None:
+ if ctx.committed:
+ record_enqueued_work(ctx.queue, metadata.work_id)
+ else:
+ await self._index.fail_enqueue(
+ ctx.queue,
+ metadata,
+ str(exc),
+ )
+ record_enqueued_work(ctx.queue, metadata.work_id)
+ raise
+ if metadata is not None:
+ if ctx.committed:
+ record_enqueued_work(ctx.queue, metadata.work_id)
+ else:
await self._index.discard(ctx.queue, metadata)
+ return result
async def process(self, ctx: ProcessContext, call_next: ProcessNext) -> ProcessResult:
metadata = extract_task_metadata(ctx.message)
@@ -51,11 +75,17 @@ class TaskWorkQueueMiddleware(QueueMiddleware):
return await call_next(ctx)
active_task = asyncio.current_task()
- with bind_task_context(metadata.task_id, metadata.account_id, metadata.user_id):
+ enqueued_work = []
+ with bind_task_context(
+ metadata.task_id,
+ metadata.account_id,
+ metadata.user_id,
+ enqueued_work=enqueued_work,
+ work_id=metadata.work_id,
+ ):
if self._index.cancellation_requested(metadata.task_id):
result = await ctx.cancel()
- if result.outcome is ProcessOutcome.FAILED:
- self._index.record_failure(metadata.task_id, result.error)
+ self._record_result(metadata, result)
return result
if active_task is not None:
self._index.register_active(
@@ -66,20 +96,33 @@ class TaskWorkQueueMiddleware(QueueMiddleware):
try:
result = await call_next(ctx)
except asyncio.CancelledError:
- if self._index.cancellation_requested(metadata.task_id):
- # Handler unwinding must finish before this delivery is settled.
- return ProcessResult.cancelled()
- raise
+ if not self._index.cancellation_requested(metadata.task_id):
+ raise
+ # Handler unwinding finished; settle user cancellation before ACK.
+ result = ProcessResult.cancelled()
except Exception as exc:
self._index.record_failure(metadata.task_id, str(exc))
raise
finally:
if active_task is not None:
self._index.unregister_active(metadata.task_id, active_task)
- if result.outcome is ProcessOutcome.FAILED:
- self._index.record_failure(metadata.task_id, result.error)
+ self._record_result(metadata, result)
return result
+ def _record_result(self, metadata, result: ProcessResult) -> None:
+ outcome = {
+ ProcessOutcome.SUCCESS: "completed",
+ ProcessOutcome.FAILED: "failed",
+ ProcessOutcome.REQUEUED: "requeued",
+ ProcessOutcome.CANCELLED: "cancelled",
+ }[result.outcome]
+ self._index.record_result(
+ metadata,
+ outcome,
+ error=result.error,
+ errors=result.errors,
+ )
+
async def ack(self, ctx: AckContext, call_next: AckNext) -> None:
prepared = None
try:
diff --git a/openviking/service/task_tracker.py b/openviking/service/task_tracker.py
index 90bb232e6..201e34366 100644
--- a/openviking/service/task_tracker.py
+++ b/openviking/service/task_tracker.py
@@ -18,11 +18,13 @@ import math
import re
import threading
import time
+from contextlib import contextmanager
+from contextvars import ContextVar
from copy import deepcopy
from dataclasses import dataclass, field, fields
from datetime import datetime, timezone
from enum import Enum
-from typing import Any, Dict, List, Optional
+from typing import Any, Awaitable, Callable, Dict, Iterator, List, Optional, TypeVar
from uuid import uuid4
from openviking.service.task_events import (
@@ -37,9 +39,35 @@ from openviking.service.task_tracker_concurrency import (
run_to_completion,
)
from openviking.service.task_work_index import QueueTaskMetadata, TaskWorkIndex
+from openviking_cli.exceptions import DeadlineExceededError, OpenVikingError
from openviking_cli.utils.logger import get_logger
logger = get_logger(__name__)
+_OperationResult = TypeVar("_OperationResult")
+
+
+@dataclass
+class _WaitBudget:
+ task_id: str
+ remaining: float
+
+
+_wait_budget: ContextVar[Optional[_WaitBudget]] = ContextVar("task_wait_budget", default=None)
+
+
+@contextmanager
+def _consume_wait_budget(task_id: str, timeout: Optional[float]) -> Iterator[Optional[float]]:
+ """Charge only time spent waiting, shared across an inline operation."""
+ budget = _wait_budget.get()
+ if budget is None or budget.task_id != task_id:
+ yield timeout
+ return
+ allowed = budget.remaining if timeout is None else min(timeout, budget.remaining)
+ started = time.monotonic()
+ try:
+ yield max(0.0, allowed)
+ finally:
+ budget.remaining = max(0.0, budget.remaining - (time.monotonic() - started))
class _CommittedMutationCancelled(asyncio.CancelledError):
@@ -59,6 +87,7 @@ class TaskStatus(str, Enum):
_TERMINAL_STATUSES = (TaskStatus.COMPLETED, TaskStatus.FAILED, TaskStatus.CANCELLED)
_ACTIVE_STATUSES = (TaskStatus.PENDING, TaskStatus.RUNNING, TaskStatus.CANCELLING)
+_PUBLIC_QUEUE_NAMES = ("Semantic", "Embedding")
_CANCELLABLE_TASK_TYPES = {
"add_resource",
@@ -146,6 +175,138 @@ def set_task_tracker(tracker: "TaskTracker") -> None:
_instance = tracker
+async def run_tracked_operation(
+ task_type: str,
+ *,
+ account_id: str,
+ user_id: str,
+ operation: Callable[[], Awaitable[_OperationResult]],
+ wait: bool,
+ timeout: Optional[float] = None,
+ resource_id: Optional[str] = None,
+ existing_task: Optional[TaskRecord] = None,
+ accept_failed_result: Callable[[_OperationResult], bool] | None = None,
+) -> tuple[_OperationResult, str]:
+ """Run a top-level operation under a Task, reusing an existing Task context.
+
+ ``accept_failed_result`` is reserved for operations whose result explicitly
+ reports record-level failures while the Task retains its failed status.
+ """
+ from openviking.service.task_work_index import bind_task_context, get_task_context
+
+ current = get_task_context()
+ if current is not None:
+ return await operation(), current.task_id
+
+ tracker = get_task_tracker()
+ created_task: Optional[TaskRecord] = existing_task
+
+ async def create_task() -> None:
+ nonlocal created_task
+ created_task = await tracker.create(
+ task_type,
+ resource_id=resource_id,
+ account_id=account_id,
+ user_id=user_id,
+ )
+
+ try:
+ # Keep the committed record reachable even if the caller is cancelled
+ # while the store write is still running.
+ if created_task is None:
+ await run_to_completion(create_task)
+ assert created_task is not None
+ task = created_task
+ await run_to_completion(
+ lambda: tracker.start(task.task_id, account_id=account_id, user_id=user_id)
+ )
+ except BaseException as exc:
+ if created_task is not None:
+ failed_task_id = created_task.task_id
+ failure_message = str(exc) or type(exc).__name__
+ await run_to_completion(
+ lambda: tracker.fail(
+ failed_task_id,
+ failure_message,
+ account_id=account_id,
+ user_id=user_id,
+ )
+ )
+ raise
+ enqueued_work: list[tuple[str, str]] = []
+
+ async def execute_operation() -> _OperationResult:
+ tracker.register_running_task(task.task_id)
+ try:
+ with bind_task_context(
+ task.task_id,
+ account_id,
+ user_id,
+ enqueued_work=enqueued_work,
+ ):
+ result = await operation()
+ task_result = deepcopy(result) if isinstance(result, dict) else {"result": result}
+ task_result.setdefault("task_id", task.task_id)
+ await tracker.complete(
+ task.task_id,
+ task_result,
+ account_id=account_id,
+ user_id=user_id,
+ resource_id=resource_id,
+ )
+ if isinstance(result, dict) and not wait:
+ result.setdefault("task_id", task.task_id)
+ return result
+ except _CommittedMutationCancelled:
+ raise
+ except BaseException as exc:
+ failure_message = str(exc) or type(exc).__name__
+ await run_to_completion(
+ lambda: tracker.fail(
+ task.task_id,
+ failure_message,
+ account_id=account_id,
+ user_id=user_id,
+ )
+ )
+ raise
+ finally:
+ await tracker.unregister_running_task(task.task_id)
+
+ token = _wait_budget.set(
+ _WaitBudget(task.task_id, timeout) if wait and timeout is not None else None
+ )
+ try:
+ result = await execute_operation()
+ if wait:
+ try:
+ terminal = await tracker.wait(
+ task.task_id,
+ account_id=account_id,
+ user_id=user_id,
+ timeout=timeout,
+ )
+ if terminal.status == TaskStatus.FAILED and not (
+ accept_failed_result is not None and accept_failed_result(result)
+ ):
+ raise OpenVikingError(
+ terminal.error or "Task processing failed",
+ code="PROCESSING_ERROR",
+ details={"task_id": task.task_id},
+ )
+ if terminal.status == TaskStatus.CANCELLED:
+ raise OpenVikingError(
+ "Task processing was cancelled",
+ code="PROCESSING_ERROR",
+ details={"task_id": task.task_id, "status": "cancelled"},
+ )
+ except (TimeoutError, asyncio.TimeoutError) as exc:
+ raise DeadlineExceededError("queue processing", timeout) from exc
+ return result, task.task_id
+ finally:
+ _wait_budget.reset(token)
+
+
# ── Sanitization ──
_SENSITIVE_PATTERNS = re.compile(
@@ -289,17 +450,20 @@ class TaskTracker:
for task_id in expired_ids:
evicted_count += await self._delete_expired_task(task_id, now)
- capacity_evicted = []
+ capacity_evicted: list[tuple[str, bool]] = []
with self._lock:
if len(self._tasks) > self.MAX_TASKS:
sorted_tasks = sorted(self._tasks.items(), key=lambda x: x[1].created_at)
excess = len(self._tasks) - self.MAX_TASKS
- for tid, _ in sorted_tasks[:excess]:
+ for tid, task in sorted_tasks[:excess]:
self._tasks.pop(tid, None)
- capacity_evicted.append(tid)
+ capacity_evicted.append((tid, task.status in _TERMINAL_STATUSES))
- for tid in capacity_evicted:
- self._work_index.forget_processing(tid)
+ for tid, terminal in capacity_evicted:
+ if terminal:
+ self._work_index.forget_task(tid)
+ else:
+ self._work_index.forget_processing(tid)
if evicted_count:
logger.debug("[TaskTracker] Evicted %d expired tasks", evicted_count)
@@ -344,7 +508,7 @@ class TaskTracker:
with self._lock:
self._tasks.pop(task_id, None)
- self._work_index.forget_processing(task_id)
+ self._work_index.forget_task(task_id)
return True
@staticmethod
@@ -632,7 +796,12 @@ class TaskTracker:
updated.status = terminal_status
updated.stage = terminal_status.value
updated.updated_at = self._next_updated_at(task)
- updated.auth = {}
+ enqueue_markers = updated.auth.get("enqueue_complete_work_ids")
+ updated.auth = (
+ {"enqueue_complete_work_ids": deepcopy(enqueue_markers)}
+ if enqueue_markers
+ else {}
+ )
try:
await self._persist_and_publish("update", updated, previous=task)
except _CommittedMutationCancelled as exc:
@@ -759,7 +928,11 @@ class TaskTracker:
elif updated.result is not None:
updated.status = TaskStatus.COMPLETED
else:
- return
+ updated.error = "Task operation interrupted before its result was persisted"
+ updated.status = TaskStatus.FAILED
+ work_status = self._work_index.queue_status(task_id)
+ if updated.result is not None and work_status:
+ updated.result["queue_status"] = self._public_queue_status(work_status)
updated.stage = updated.status.value
updated.updated_at = self._next_updated_at(task)
updated.auth = {}
@@ -777,18 +950,24 @@ class TaskTracker:
) -> TaskRecord:
"""Wait for one task's terminal state without changing its lifecycle."""
- async def _poll() -> TaskRecord:
+ with _consume_wait_budget(task_id, timeout) as remaining:
+ deadline = None if remaining is None else time.monotonic() + remaining
while True:
- task = await self.get(task_id, account_id=account_id, user_id=user_id)
+ lookup = self.get(task_id, account_id=account_id, user_id=user_id)
+ if remaining is not None and self._cached_task(task_id) is None:
+ task = await asyncio.wait_for(lookup, max(0.0, remaining))
+ else:
+ task = await lookup
if task is None:
raise KeyError(f"Task not found: {task_id}")
if task.status in _TERMINAL_STATUSES:
return task
- await asyncio.sleep(poll_interval)
-
- if timeout is None:
- return await _poll()
- return await asyncio.wait_for(_poll(), timeout)
+ remaining = None if deadline is None else deadline - time.monotonic()
+ if remaining is not None and remaining <= 0:
+ raise TimeoutError(f"Timed out waiting for Task {task_id}")
+ await asyncio.sleep(
+ poll_interval if remaining is None else min(poll_interval, remaining)
+ )
async def get_task_auth(
self,
@@ -818,8 +997,7 @@ class TaskTracker:
for task_id in task_ids:
del self._tasks[task_id]
for task_id in task_ids:
- self._work_index.clear_failure(task_id)
- self._work_index.forget_processing(task_id)
+ self._work_index.forget_task(task_id)
async def delete_user_tasks(self, account_id: str, user_id: str) -> int:
"""Delete terminal task records for one user from storage and cache."""
@@ -859,25 +1037,126 @@ class TaskTracker:
)
with self._lock:
self._tasks.pop(task.task_id, None)
- self._work_index.forget_processing(task.task_id)
- self._work_index.clear_failure(task.task_id)
+ self._work_index.forget_task(task.task_id)
deleted += 1
return deleted
- async def wait_for_descendants(self, task_id: str, current_work_id: str) -> None:
- """Wait on the same durable work index used by completion and cancellation."""
- if self._work_index.has_work(task_id, exclude_work_id=current_work_id):
- task = self._cached_task(task_id)
- if task and task.account_id and task.user_id:
- await self.record_event(
- task_id,
- "waiting_for_descendants",
- operation=current_work_id,
- account_id=task.account_id,
- user_id=task.user_id,
+ async def wait_for_work(
+ self,
+ task_id: str,
+ work_ids: List[str],
+ *,
+ timeout: Optional[float] = None,
+ stop_waiting: Optional[Any] = None,
+ ) -> None:
+ """Wait only for the supplied logical Work IDs to become terminal."""
+ with _consume_wait_budget(task_id, timeout) as remaining:
+ await self._work_index.wait_for_work(
+ task_id,
+ work_ids,
+ timeout=remaining,
+ stop_waiting=stop_waiting,
+ )
+
+ async def wait_for_task_work(
+ self,
+ task_id: str,
+ *,
+ exclude_work_id: Optional[str] = None,
+ timeout: Optional[float] = None,
+ ) -> None:
+ """Let a root coordinator drain its Task's other queue deliveries."""
+ with _consume_wait_budget(task_id, timeout) as remaining:
+ await self._work_index.wait_for_task_work(
+ task_id, exclude_work_id=exclude_work_id, timeout=remaining
+ )
+
+ async def mark_enqueue_complete(
+ self,
+ task_id: str,
+ work_id: str,
+ *,
+ account_id: str,
+ user_id: str,
+ ) -> None:
+ """Persist that a coordinator's complete child Work set is durable."""
+ async with self._task_locks.acquire(task_id):
+ task = await self._load_for_update(task_id, account_id, user_id)
+ if task is None or task.status not in _ACTIVE_STATUSES:
+ return
+ updated = deepcopy(task)
+ completed = set(updated.auth.get("enqueue_complete_work_ids") or ())
+ completed.add(work_id)
+ updated.auth["enqueue_complete_work_ids"] = sorted(completed)
+ updated.updated_at = self._next_updated_at(task)
+ await self._persist_and_publish("update", updated, previous=task)
+
+ async def is_enqueue_complete(
+ self,
+ task_id: str,
+ work_id: str,
+ *,
+ account_id: str,
+ user_id: str,
+ ) -> bool:
+ """Check the durable producer marker used by a lock coordinator."""
+ auth = await self.get_task_auth(
+ task_id,
+ account_id=account_id,
+ user_id=user_id,
+ )
+ return work_id in set(auth.get("enqueue_complete_work_ids") or ())
+
+ def build_queue_status(
+ self,
+ task_id: str,
+ work_ids: Optional[List[str]] = None,
+ ) -> Dict[str, Dict[str, object]]:
+ """Project Task-scoped queue outcomes from logical Work."""
+ return self._public_queue_status(self._work_index.queue_status(task_id, work_ids))
+
+ @staticmethod
+ def _public_queue_status(
+ status: Dict[str, Dict[str, object]],
+ ) -> Dict[str, Dict[str, object]]:
+ result: Dict[str, Dict[str, object]] = {}
+ for queue_name in _PUBLIC_QUEUE_NAMES:
+ queue_status = deepcopy(
+ status.get(
+ queue_name,
+ {
+ "processed": 0,
+ "requeue_count": 0,
+ "error_count": 0,
+ "errors": [],
+ },
)
- while self._work_index.has_work(task_id, exclude_work_id=current_work_id):
- await asyncio.sleep(0.05)
+ )
+ errors = queue_status.get("errors")
+ if isinstance(errors, list):
+ queue_status["errors"] = [
+ {
+ **error,
+ "message": _sanitize_error(str(error.get("message") or "")),
+ }
+ if isinstance(error, dict)
+ else {"message": _sanitize_error(str(error))}
+ for error in errors[:100]
+ ]
+ result[queue_name] = queue_status
+ return result
+
+ def get_queue_timing(self, task_id: str) -> Dict[str, Dict[str, float]]:
+ """Return Task-scoped queue wait and execution durations."""
+ return self._work_index.queue_timing(task_id)
+
+ def get_context_count(self, task_id: str) -> int:
+ """Return successful context writes recorded by Task Work."""
+ return self._work_index.context_count(task_id)
+
+ def record_context_count(self, task_id: str, work_id: str, delta: int = 1) -> None:
+ """Record successful context writes against one logical Work."""
+ self._work_index.record_context_count(task_id, work_id, delta)
async def record_event(
self,
@@ -914,9 +1193,13 @@ class TaskTracker:
updated.updated_at = self._next_updated_at(task)
await self._persist_and_publish("update", updated, previous=task)
- def has_work(self, task_id: str) -> bool:
+ def has_work(self, task_id: str, *, exclude_work_id: Optional[str] = None) -> bool:
"""Return whether a task still owns durable or active queue work."""
- return self._work_index.has_work(task_id)
+ return self._work_index.has_work(task_id, exclude_work_id=exclude_work_id)
+
+ def has_active_producer(self, task_id: str) -> bool:
+ """Return whether another active handler can persist an enqueue marker."""
+ return self._work_index.has_other_active(task_id, asyncio.current_task())
def register_running_task(self, task_id: str) -> None:
"""Register the current asyncio task so cancellation can interrupt it."""
@@ -985,7 +1268,7 @@ class TaskTracker:
)
with self._lock:
self._tasks.pop(task.task_id, None)
- self._work_index.forget_processing(task.task_id)
+ self._work_index.forget_task(task.task_id)
return True
async def list_page(
diff --git a/openviking/service/task_work_index.py b/openviking/service/task_work_index.py
index cb034afa0..dfb6ff93d 100644
--- a/openviking/service/task_work_index.py
+++ b/openviking/service/task_work_index.py
@@ -1,10 +1,10 @@
# Copyright (c) 2026 Beijing Volcano Engine Technology Co., Ltd.
# SPDX-License-Identifier: AGPL-3.0
-"""Runtime index of durable queue work owned by a tracked task.
+"""Runtime index of logical queue Work owned by tracked Tasks.
-QueueFS remains the durable source of truth. This index is rebuilt from all
-unacknowledged queue messages during startup and then maintained by enqueue/ACK
-hooks. It deliberately does not persist a second counter beside QueueFS.
+QueueFS remains the durable delivery source of truth. This index separates
+logical Work state from delivery counts so a replacement delivery can retain
+the same Work ID without allowing the old delivery's ACK to finish the Task.
"""
from __future__ import annotations
@@ -16,10 +16,11 @@ import time
from contextlib import contextmanager
from contextvars import ContextVar
from dataclasses import dataclass
-from typing import Any, Awaitable, Callable, Dict, Iterable, Iterator, Mapping, Optional
+from typing import Any, Awaitable, Callable, Dict, Generator, Iterable, Mapping, Optional
from uuid import uuid4
TASK_WORK_ID_FIELD = "_task_work_id"
+_TERMINAL_WORK_STATUSES = frozenset({"completed", "failed", "cancelled"})
class TaskWorkRejected(Exception):
@@ -31,6 +32,8 @@ class TaskExecutionContext:
task_id: str
account_id: str
user_id: str
+ enqueued_work: Optional[list[tuple[str, str]]] = None
+ work_id: Optional[str] = None
@dataclass(frozen=True)
@@ -43,13 +46,33 @@ class QueueTaskMetadata:
@dataclass
class WorkTiming:
- """Local timestamps matching the future persisted Work timing contract."""
+ """Timestamps for one logical Work across all of its deliveries."""
enqueued_at: float
started_at: Optional[float] = None
finished_at: Optional[float] = None
+@dataclass
+class WorkRecord:
+ """Mutable process-local state for one logical Work."""
+
+ task_id: str
+ work_id: str
+ queue_name: str
+ status: str = "pending"
+ delivery_count: int = 0
+ requeue_count: int = 0
+ error: Optional[str] = None
+ errors: Optional[list[str]] = None
+ context_count: int = 0
+ timing: Optional[WorkTiming] = None
+
+ @property
+ def terminal(self) -> bool:
+ return self.status in _TERMINAL_WORK_STATUSES
+
+
_current_task_context: ContextVar[Optional[TaskExecutionContext]] = ContextVar(
"openviking_task_execution_context",
default=None,
@@ -61,13 +84,23 @@ def bind_task_context(
task_id: str,
account_id: str,
user_id: str,
-) -> Iterator[None]:
- """Propagate task ownership to queue messages produced by this coroutine."""
+ *,
+ enqueued_work: Optional[list[tuple[str, str]]] = None,
+ work_id: Optional[str] = None,
+) -> Generator[None, None, None]:
+ """Propagate Task ownership to queue messages produced by this coroutine."""
+ current = _current_task_context.get()
+ if enqueued_work is None and current is not None and current.task_id == str(task_id):
+ enqueued_work = current.enqueued_work
+ if work_id is None and current is not None and current.task_id == str(task_id):
+ work_id = current.work_id
token = _current_task_context.set(
TaskExecutionContext(
task_id=str(task_id),
account_id=str(account_id),
user_id=str(user_id),
+ enqueued_work=enqueued_work,
+ work_id=work_id,
)
)
try:
@@ -77,8 +110,8 @@ def bind_task_context(
@contextmanager
-def detach_task_context() -> Iterator[None]:
- """Keep independently scheduled work outside the current task lifecycle."""
+def detach_task_context() -> Generator[None, None, None]:
+ """Keep independently scheduled work outside the current Task lifecycle."""
token = _current_task_context.set(None)
try:
yield
@@ -90,6 +123,25 @@ def get_task_context() -> Optional[TaskExecutionContext]:
return _current_task_context.get()
+def record_enqueued_work(queue_name: str, work_id: str) -> None:
+ """Record a committed child enqueue for the currently executing handler."""
+ current = _current_task_context.get()
+ if current is not None and current.enqueued_work is not None:
+ current.enqueued_work.append((queue_name, work_id))
+
+
+def get_enqueued_work_ids(queue_name: Optional[str] = None) -> list[str]:
+ """Return committed child Work IDs produced by the current handler."""
+ current = _current_task_context.get()
+ if current is None or current.enqueued_work is None:
+ return []
+ return [
+ work_id
+ for work_queue, work_id in current.enqueued_work
+ if queue_name is None or work_queue == queue_name
+ ]
+
+
def _payload_dict(message: Any) -> Optional[Dict[str, Any]]:
if not isinstance(message, dict):
return None
@@ -115,13 +167,19 @@ def _owner_from_payload(payload: Mapping[str, Any]) -> tuple[str, str]:
if isinstance(context_data, dict):
account_id = account_id or context_data.get("account_id")
user_id = user_id or context_data.get("owner_user_id")
+ payload_data = payload.get("payload")
+ if isinstance(payload_data, dict):
+ context_data = payload_data.get("context_data")
+ if isinstance(context_data, dict):
+ account_id = account_id or context_data.get("account_id")
+ user_id = user_id or context_data.get("owner_user_id")
return str(account_id or ""), str(user_id or "")
def prepare_task_payload(
data: Dict[str, Any],
) -> tuple[Dict[str, Any], Optional[QueueTaskMetadata]]:
- """Copy a payload and attach durable task metadata when it belongs to a task."""
+ """Copy a payload and attach durable Task metadata when it belongs to a Task."""
payload = dict(data)
current = _current_task_context.get()
task_id = payload.get("task_id") or (current.task_id if current is not None else "")
@@ -144,15 +202,13 @@ def prepare_task_payload(
def extract_task_metadata(message: Any) -> Optional[QueueTaskMetadata]:
- """Read task metadata from either a QueueFS envelope or its inner payload."""
+ """Read Task metadata from either a QueueFS envelope or its inner payload."""
payload = _payload_dict(message)
if payload is None:
return None
task_id = payload.get("task_id")
work_id = payload.get(TASK_WORK_ID_FIELD)
if not work_id and isinstance(message, dict) and "data" in message and message.get("id"):
- # Messages produced before task work IDs were introduced can still be
- # rebuilt from their stable QueueFS envelope ID.
work_id = f"queuefs:{message['id']}"
if not task_id or not work_id:
return None
@@ -161,11 +217,12 @@ def extract_task_metadata(message: Any) -> Optional[QueueTaskMetadata]:
class TaskWorkIndex:
- """Thread-safe, rebuildable index of persistent and currently active task work."""
+ """Thread-safe, rebuildable index of logical Work and active handlers."""
def __init__(self) -> None:
self._lock = threading.Lock()
- self._work: Dict[str, set[tuple[str, str]]] = {}
+ self._work: Dict[str, Dict[str, WorkRecord]] = {}
+ self._delivery_counts: Dict[str, int] = {}
self._active: Dict[str, set[asyncio.Task[Any]]] = {}
self._failures: Dict[str, str] = {}
self._work_timings: Dict[str, Dict[str, WorkTiming]] = {}
@@ -186,67 +243,119 @@ class TaskWorkIndex:
self,
snapshots: Mapping[str, Iterable[Any]],
) -> Dict[str, tuple[str, str]]:
- """Rebuild work from QueueFS and return owners needed to restore task records."""
- work: Dict[str, set[tuple[str, str]]] = {}
+ """Rebuild unacknowledged deliveries and return Task owners to restore."""
+ work: Dict[str, Dict[str, WorkRecord]] = {}
+ delivery_counts: Dict[str, int] = {}
owners: Dict[str, tuple[str, str]] = {}
+ now = time.time()
for queue_name, messages in snapshots.items():
for message in messages:
metadata = extract_task_metadata(message)
if metadata is None:
continue
- work.setdefault(metadata.task_id, set()).add((queue_name, metadata.work_id))
+ records = work.setdefault(metadata.task_id, {})
+ record = records.setdefault(
+ metadata.work_id,
+ WorkRecord(
+ task_id=metadata.task_id,
+ work_id=metadata.work_id,
+ queue_name=queue_name,
+ timing=WorkTiming(enqueued_at=now),
+ ),
+ )
+ record.delivery_count += 1
+ delivery_counts[metadata.task_id] = delivery_counts.get(metadata.task_id, 0) + 1
if metadata.account_id and metadata.user_id:
owners[metadata.task_id] = (metadata.account_id, metadata.user_id)
with self._lock:
self._work = work
+ self._delivery_counts = delivery_counts
self._failures = {}
return owners
+ def _add_delivery_locked(self, task_id: str) -> None:
+ self._delivery_counts[task_id] = self._delivery_counts.get(task_id, 0) + 1
+
+ def _remove_delivery_locked(self, task_id: str) -> None:
+ remaining = self._delivery_counts.get(task_id, 0) - 1
+ if remaining > 0:
+ self._delivery_counts[task_id] = remaining
+ else:
+ self._delivery_counts.pop(task_id, None)
+
def register(self, queue_name: str, metadata: Optional[QueueTaskMetadata]) -> bool:
- """Atomically reject cancelled work or add it to the runtime index."""
+ """Reject cancelled Work or provisionally register one delivery."""
if metadata is None:
return True
with self._lock:
if self.cancellation_requested(metadata.task_id):
return False
- self._work.setdefault(metadata.task_id, set()).add((queue_name, metadata.work_id))
+ records = self._work.setdefault(metadata.task_id, {})
+ record = records.get(metadata.work_id)
+ now = time.time()
+ if record is None:
+ record = WorkRecord(
+ task_id=metadata.task_id,
+ work_id=metadata.work_id,
+ queue_name=queue_name,
+ timing=WorkTiming(enqueued_at=now),
+ )
+ records[metadata.work_id] = record
+ elif record.queue_name != queue_name:
+ raise ValueError(
+ f"Work {metadata.work_id} already belongs to queue {record.queue_name}"
+ )
+ record.delivery_count += 1
+ self._add_delivery_locked(metadata.task_id)
timings = self._work_timings.get(metadata.task_id)
if timings is not None:
- timings.setdefault(metadata.work_id, WorkTiming(enqueued_at=time.time()))
+ record.timing = timings.setdefault(metadata.work_id, record.timing)
return True
- def _remove_work(
+ def _remove_delivery(
self,
queue_name: str,
metadata: QueueTaskMetadata,
) -> tuple[bool, bool]:
- """Remove one work item and report whether it existed and made its task idle."""
- removed = False
+ """Remove one delivery and report whether its Task became idle."""
with self._lock:
- entries = self._work.get(metadata.task_id)
- if entries is not None:
- entry = (queue_name, metadata.work_id)
- removed = entry in entries
- entries.discard(entry)
- if not entries:
- self._work.pop(metadata.task_id, None)
- became_idle = (
- removed
- and not self._work.get(metadata.task_id)
- and not self._active.get(metadata.task_id)
+ record = self._work.get(metadata.task_id, {}).get(metadata.work_id)
+ removed = bool(
+ record is not None and record.queue_name == queue_name and record.delivery_count > 0
)
+ if removed:
+ record.delivery_count -= 1
+ self._remove_delivery_locked(metadata.task_id)
+ became_idle = removed and not self._has_work_locked(metadata.task_id)
return removed, became_idle
+ def _has_delivery_locked(self, task_id: str, exclude_work_id: Optional[str] = None) -> bool:
+ if exclude_work_id is not None:
+ record = self._work.get(task_id, {}).get(exclude_work_id)
+ excluded = record.delivery_count if record is not None else 0
+ return self._delivery_counts.get(task_id, 0) > excluded
+ return self._delivery_counts.get(task_id, 0) > 0
+
+ def _has_work_locked(self, task_id: str, exclude_work_id: Optional[str] = None) -> bool:
+ if self._has_delivery_locked(task_id, exclude_work_id):
+ return True
+ if exclude_work_id is not None:
+ return False
+ return bool(self._active.get(task_id))
+
async def discard(self, queue_name: str, message: Any) -> None:
- """Discard work that was never durably enqueued and finalize if it was last."""
+ """Discard a provisionally registered delivery that did not commit."""
metadata = (
message if isinstance(message, QueueTaskMetadata) else extract_task_metadata(message)
)
if metadata is None:
return
- removed, became_idle = self._remove_work(queue_name, metadata)
+ removed, became_idle = self._remove_delivery(queue_name, metadata)
if removed:
with self._lock:
+ record = self._work.get(metadata.task_id, {}).get(metadata.work_id)
+ if record is not None and record.delivery_count == 0 and not record.terminal:
+ self._work[metadata.task_id].pop(metadata.work_id, None)
timings = self._work_timings.get(metadata.task_id)
if timings is not None:
timings.pop(metadata.work_id, None)
@@ -254,19 +363,34 @@ class TaskWorkIndex:
if became_idle and callback is not None:
await callback(metadata)
+ async def fail_enqueue(
+ self,
+ queue_name: str,
+ metadata: QueueTaskMetadata,
+ error: str,
+ ) -> None:
+ """Keep a failed logical Work visible after its delivery failed to persist."""
+ self.record_result(metadata, "failed", error=error)
+ removed, became_idle = self._remove_delivery(queue_name, metadata)
+ if not removed:
+ return
+ callback = self._finalize_before_ack
+ if became_idle and callback is not None:
+ await callback(metadata)
+
async def prepare_ack(
self,
queue_name: str,
message: Any,
) -> Optional[QueueTaskMetadata]:
- """Remove work provisionally and finalize its task before the last durable ACK."""
+ """Provisionally remove one delivery and finalize before the last ACK."""
metadata = (
message if isinstance(message, QueueTaskMetadata) else extract_task_metadata(message)
)
if metadata is None:
return None
- removed, became_idle = self._remove_work(queue_name, metadata)
+ removed, became_idle = self._remove_delivery(queue_name, metadata)
if not removed:
return None
if not became_idle:
@@ -283,25 +407,80 @@ class TaskWorkIndex:
return metadata
def rollback_ack(self, queue_name: str, metadata: QueueTaskMetadata) -> None:
- """Restore provisionally removed work when finalization or ACK fails."""
+ """Restore a provisionally removed delivery when finalization or ACK fails."""
with self._lock:
- self._work.setdefault(metadata.task_id, set()).add((queue_name, metadata.work_id))
+ record = self._work.get(metadata.task_id, {}).get(metadata.work_id)
+ if record is None:
+ record = WorkRecord(
+ task_id=metadata.task_id,
+ work_id=metadata.work_id,
+ queue_name=queue_name,
+ timing=WorkTiming(enqueued_at=time.time()),
+ )
+ self._work.setdefault(metadata.task_id, {})[metadata.work_id] = record
+ record.delivery_count += 1
+ self._add_delivery_locked(metadata.task_id)
def has_work(self, task_id: str, exclude_work_id: Optional[str] = None) -> bool:
with self._lock:
- if exclude_work_id is not None:
- return any(
- work_id != exclude_work_id
- for _queue_name, work_id in self._work.get(task_id, ())
- )
- return bool(self._work.get(task_id) or self._active.get(task_id))
+ return self._has_work_locked(task_id, exclude_work_id)
+
+ def has_other_active(
+ self,
+ task_id: str,
+ current_task: Optional[asyncio.Task[Any]],
+ ) -> bool:
+ """Return whether another in-process handler can finish producing Work."""
+ with self._lock:
+ return any(task is not current_task for task in self._active.get(task_id, ()))
def cancellation_requested(self, task_id: str) -> bool:
callback = self._is_cancellation_requested
return bool(callback is not None and callback(task_id))
+ def record_result(
+ self,
+ metadata: QueueTaskMetadata,
+ outcome: str,
+ *,
+ error: Optional[str] = None,
+ errors: Optional[Iterable[str]] = None,
+ ) -> None:
+ """Apply one delivery result to its logical Work monotonically."""
+ with self._lock:
+ record = self._work.get(metadata.task_id, {}).get(metadata.work_id)
+ if record is None or record.terminal:
+ return
+ if outcome == "requeued":
+ record.requeue_count += 1
+ record.status = "pending"
+ return
+ if outcome not in _TERMINAL_WORK_STATUSES:
+ raise ValueError(f"Unsupported Work outcome: {outcome}")
+ record.status = outcome
+ record.error = error if outcome == "failed" and error else None
+ if outcome == "failed":
+ details = [str(value) for value in errors or () if str(value)]
+ record.errors = details or ([error] if error else [])
+ if record.timing is not None:
+ record.timing.finished_at = max(
+ record.timing.started_at or record.timing.enqueued_at,
+ time.time(),
+ )
+ if outcome == "failed" and error:
+ self._failures.setdefault(metadata.task_id, error)
+
+ def record_context_count(self, task_id: str, work_id: str, delta: int = 1) -> None:
+ """Add a result count to one Work without changing its lifecycle."""
+ if delta <= 0:
+ return
+ with self._lock:
+ record = self._work.get(task_id, {}).get(work_id)
+ if record is not None:
+ record.context_count += delta
+
def record_failure(self, task_id: str, error: str) -> None:
- """Record the first failed work item owned by a task."""
+ """Record the first failed Work owned by a Task."""
with self._lock:
self._failures.setdefault(task_id, error)
@@ -313,9 +492,130 @@ class TaskWorkIndex:
with self._lock:
self._failures.pop(task_id, None)
+ async def wait_for_work(
+ self,
+ task_id: str,
+ work_ids: Iterable[str],
+ *,
+ timeout: Optional[float] = None,
+ poll_interval: float = 0.05,
+ stop_waiting: Optional[Callable[[], bool]] = None,
+ ) -> None:
+ """Wait until exactly the supplied logical Work IDs are terminal."""
+ requested = frozenset(str(work_id) for work_id in work_ids)
+ if not requested:
+ return
+
+ deadline = None if timeout is None else time.monotonic() + timeout
+ while True:
+ with self._lock:
+ records = self._work.get(task_id, {})
+ missing = requested.difference(records)
+ done = not missing and all(records[work_id].terminal for work_id in requested)
+ if done:
+ return
+ if missing:
+ raise KeyError(f"Unknown Work for Task {task_id}: {sorted(missing)}")
+ if stop_waiting is not None and stop_waiting():
+ raise asyncio.CancelledError("Worker stopped with queued Work")
+ remaining = None if deadline is None else deadline - time.monotonic()
+ if remaining is not None and remaining <= 0:
+ raise TimeoutError(f"Timed out waiting for Work in Task {task_id}")
+ await asyncio.sleep(
+ poll_interval if remaining is None else min(poll_interval, remaining)
+ )
+
+ async def wait_for_task_work(
+ self,
+ task_id: str,
+ *,
+ exclude_work_id: Optional[str] = None,
+ timeout: Optional[float] = None,
+ poll_interval: float = 0.05,
+ ) -> None:
+ """Drain Task deliveries, including newly enqueued and recovered Work.
+
+ Only the Task coordinator may use this barrier, excluding its own Work.
+ Inline coordinators have no delivery to exclude. Other workers must wait
+ for explicit Work IDs to avoid waiting on each other.
+ """
+ deadline = None if timeout is None else time.monotonic() + timeout
+ while True:
+ with self._lock:
+ pending = self._has_delivery_locked(task_id, exclude_work_id)
+ if not pending:
+ return
+ remaining = None if deadline is None else deadline - time.monotonic()
+ if remaining is not None and remaining <= 0:
+ raise TimeoutError(f"Timed out waiting for Work in Task {task_id}")
+ await asyncio.sleep(
+ poll_interval if remaining is None else min(poll_interval, remaining)
+ )
+
+ def queue_status(
+ self,
+ task_id: str,
+ work_ids: Optional[Iterable[str]] = None,
+ ) -> Dict[str, Dict[str, object]]:
+ """Project queue counters from logical Work records."""
+ selected = None if work_ids is None else frozenset(str(value) for value in work_ids)
+ with self._lock:
+ records = [
+ record
+ for work_id, record in self._work.get(task_id, {}).items()
+ if selected is None or work_id in selected
+ ]
+ result: Dict[str, Dict[str, object]] = {}
+ for record in records:
+ status = result.setdefault(
+ record.queue_name,
+ {
+ "processed": 0,
+ "requeue_count": 0,
+ "error_count": 0,
+ "errors": [],
+ },
+ )
+ if record.status == "completed":
+ status["processed"] += 1
+ status["requeue_count"] += record.requeue_count
+ if record.status == "failed":
+ errors = record.errors or ([record.error] if record.error else [])
+ status["error_count"] += len(errors) or 1
+ status["errors"].extend({"message": message} for message in errors)
+ return result
+
+ def queue_timing(self, task_id: str) -> Dict[str, Dict[str, float]]:
+ """Project queue wait and execution timing in milliseconds."""
+ with self._lock:
+ result: Dict[str, Dict[str, float]] = {}
+ for record in self._work.get(task_id, {}).values():
+ timing = record.timing
+ metrics = result.setdefault(
+ record.queue_name.lower(),
+ {"queue_wait_ms": 0.0, "execute_ms": 0.0},
+ )
+ if timing is None or timing.started_at is None:
+ continue
+ metrics["queue_wait_ms"] += max(
+ 0.0, (timing.started_at - timing.enqueued_at) * 1000.0
+ )
+ if timing.finished_at is not None:
+ metrics["execute_ms"] += max(
+ 0.0, (timing.finished_at - timing.started_at) * 1000.0
+ )
+ return result
+
+ def context_count(self, task_id: str) -> int:
+ with self._lock:
+ return sum(record.context_count for record in self._work.get(task_id, {}).values())
+
def init_processing(self, task_id: str) -> None:
with self._lock:
- self._work_timings.setdefault(task_id, {})
+ timings = self._work_timings.setdefault(task_id, {})
+ for work_id, record in self._work.get(task_id, {}).items():
+ if record.timing is not None:
+ timings.setdefault(work_id, record.timing)
def processing_seconds(self, task_id: str) -> Optional[float]:
with self._lock:
@@ -347,6 +647,18 @@ class TaskWorkIndex:
with self._lock:
self._work_timings.pop(task_id, None)
+ def forget_task(self, task_id: str) -> None:
+ """Release all process-local state after the owning Task is removed."""
+ with self._lock:
+ self._work.pop(task_id, None)
+ self._delivery_counts.pop(task_id, None)
+ self._active.pop(task_id, None)
+ self._failures.pop(task_id, None)
+ self._work_timings.pop(task_id, None)
+ active_keys = [key for key in self._active_work_ids if key[0] == task_id]
+ for key in active_keys:
+ self._active_work_ids.pop(key, None)
+
def register_active(
self,
task_id: str,
@@ -358,10 +670,17 @@ class TaskWorkIndex:
cancelled = self.cancellation_requested(task_id)
if not cancelled:
self._active.setdefault(task_id, set()).add(active_task)
+ now = time.time()
+ timing_id = work_id or f"inline:{id(active_task)}"
+ record = self._work.get(task_id, {}).get(timing_id)
+ if record is not None:
+ timing = record.timing
+ if timing is not None and timing.started_at is None:
+ timing.started_at = now
+ if not record.terminal:
+ record.status = "running"
timings = self._work_timings.get(task_id)
if timings is not None:
- now = time.time()
- timing_id = work_id or f"inline:{id(active_task)}"
timing = timings.setdefault(timing_id, WorkTiming(enqueued_at=now))
if timing.started_at is None:
timing.started_at = now
diff --git a/openviking/session/memory/memory_updater.py b/openviking/session/memory/memory_updater.py
index 01bba2f7e..591f42486 100644
--- a/openviking/session/memory/memory_updater.py
+++ b/openviking/session/memory/memory_updater.py
@@ -45,7 +45,6 @@ from openviking.session.memory.utils.uri import render_template
from openviking.storage.abstract_overview import freshness_metadata, render_abstract_overview
from openviking.storage.viking_fs import get_viking_fs
from openviking.telemetry import tracer
-from openviking.telemetry.request_wait_tracker import get_request_wait_tracker
from openviking.telemetry.tracer import get_trace_id
from openviking.utils.ingest_options import IngestOptions
from openviking.utils.tags import merge_search_tags
@@ -835,6 +834,7 @@ class MemoryUpdater:
ctx,
uri_memory_type_map={uri: memory_type} if memory_type else {},
ingest_options=ingest_options,
+ raise_on_enqueue_failure=True,
)
return attempted > 0
except Exception:
@@ -1749,6 +1749,7 @@ class MemoryUpdater:
uri_memory_type_map: Dict[str, str] = None,
search_tags_by_uri: Dict[str, List[str]] = None,
ingest_options: Any = None,
+ raise_on_enqueue_failure: bool = False,
) -> int:
"""Vectorize written and edited memory files.
@@ -1768,7 +1769,6 @@ class MemoryUpdater:
uri_memory_type_map = uri_memory_type_map or {}
search_tags_by_uri = search_tags_by_uri or {}
viking_fs = self._get_viking_fs()
- request_wait_tracker = get_request_wait_tracker()
attempted_count = 0
# Collect all URIs to vectorize (skip .overview.md and .abstract.md - they are handled separately)
@@ -1865,31 +1865,20 @@ class MemoryUpdater:
embedding_msg.context_data["_upsert_options"][
"extracted_memory_type"
] = memory_type
- if embedding_msg.telemetry_id:
- request_wait_tracker.register_embedding_root(
- embedding_msg.telemetry_id, embedding_msg.id
- )
+ enqueued = await self._vikingdb.enqueue_embedding_msg(embedding_msg)
+ if not enqueued:
+ message = f"Failed to enqueue memory for vectorization: {uri}"
+ if raise_on_enqueue_failure:
+ raise RuntimeError(message)
+ tracer.error(message)
+ continue
attempted_count += 1
- try:
- enqueued = await self._vikingdb.enqueue_embedding_msg(embedding_msg)
- except Exception as e:
- if embedding_msg.telemetry_id:
- request_wait_tracker.mark_embedding_failed(
- embedding_msg.telemetry_id,
- embedding_msg.id,
- str(e),
- )
- raise
- if not enqueued and embedding_msg.telemetry_id:
- request_wait_tracker.mark_embedding_failed(
- embedding_msg.telemetry_id,
- embedding_msg.id,
- "embedding enqueue returned false",
- )
logger.debug(f"Enqueued memory for vectorization: {uri}")
except Exception as e:
tracer.error(f"Failed to vectorize memory {uri}: {e}")
+ if raise_on_enqueue_failure:
+ raise
return attempted_count
@staticmethod
diff --git a/openviking/session/session.py b/openviking/session/session.py
index c66cce97b..847f6999e 100644
--- a/openviking/session/session.py
+++ b/openviking/session/session.py
@@ -64,7 +64,6 @@ from openviking.session.working_memory import (
)
from openviking.storage.abstract_overview import render_abstract_overview
from openviking.telemetry import get_current_telemetry, tracer
-from openviking.telemetry.request_wait_tracker import get_request_wait_tracker
from openviking.utils.model_retry import is_retryable_api_error, retry_async
from openviking.utils.time_utils import get_current_timestamp
from openviking.utils.token_estimation import estimate_text_tokens
@@ -2048,11 +2047,11 @@ class Session:
) -> None:
"""Phase 2: Extract memories and enqueue semantic work in the background."""
from openviking.service.task_tracker import get_task_tracker
+ from openviking.service.task_work_index import get_task_context
from openviking.telemetry import OperationTelemetry, bind_telemetry
from openviking.telemetry.registry import register_telemetry, unregister_telemetry
tracker = get_task_tracker()
- request_wait_tracker = get_request_wait_tracker()
memories_extracted: Dict[str, int] = {}
usage_events_extracted = 0
@@ -2081,7 +2080,6 @@ class Session:
account_id=self.ctx.account_id,
user_id=self.ctx.user.user_id,
)
- request_wait_tracker.register_request(telemetry.telemetry_id)
register_telemetry(telemetry)
try:
with bind_telemetry(telemetry):
@@ -2426,11 +2424,13 @@ class Session:
)
try:
- await request_wait_tracker.wait_for_request(
- telemetry.telemetry_id,
+ task_context = get_task_context()
+ await tracker.wait_for_task_work(
+ task_id,
+ exclude_work_id=task_context.work_id if task_context else None,
timeout=_PHASE2_QUEUE_WAIT_TIMEOUT_SECONDS,
)
- except TimeoutError as exc:
+ except (TimeoutError, asyncio.TimeoutError) as exc:
telemetry.set_error(
"session.commit.phase2.wait_for_request",
"DEADLINE_EXCEEDED",
@@ -2443,7 +2443,6 @@ class Session:
_PHASE2_QUEUE_WAIT_TIMEOUT_SECONDS,
)
finally:
- request_wait_tracker.cleanup(telemetry.telemetry_id)
unregister_telemetry(telemetry.telemetry_id)
# Phase 2 complete — update meta with telemetry and commit info
diff --git a/openviking/storage/collection_schemas.py b/openviking/storage/collection_schemas.py
index 73b9b2d78..d10fea436 100644
--- a/openviking/storage/collection_schemas.py
+++ b/openviking/storage/collection_schemas.py
@@ -21,6 +21,7 @@ from openviking.core.context import ContextType, ResourceContentType
from openviking.models.embedder.base import embed_compat
from openviking.models.model_error import ModelErrorKind, model_error_kind
from openviking.server.identity import RequestContext, Role
+from openviking.service.task_tracker import get_task_tracker
from openviking.service.task_tracker_concurrency import run_to_completion
from openviking.storage.acl import ACL_GRANT_FIELDS, ACL_MODE_FIELD, AclMode
from openviking.storage.errors import (
@@ -43,7 +44,6 @@ from openviking.storage.viking_vector_index_backend import (
normalize_upsert_options,
)
from openviking.telemetry import bind_telemetry, resolve_telemetry
-from openviking.telemetry.request_wait_tracker import get_request_wait_tracker
from openviking.utils.circuit_breaker import CircuitBreakerOpen
from openviking.utils.log_correlation import log_correlation
from openviking.utils.tags import merge_search_tags
@@ -761,9 +761,6 @@ class TextEmbeddingHandler(DequeueHandlerBase):
await asyncio.sleep(wait)
await self._reenqueue_embedding_msg(embedding_msg)
self._merge_request_stats(embedding_msg.telemetry_id, requeue_count=1)
- get_request_wait_tracker().record_embedding_requeue(
- embedding_msg.telemetry_id
- )
return ProcessResult.requeued()
execute_status = "error"
request_failed_message = self._embedding_error_msg(
@@ -857,9 +854,7 @@ class TextEmbeddingHandler(DequeueHandlerBase):
# Write to vector database
try:
raw_upsert_options = inserted_data.pop("_upsert_options", {})
- extracted_memory_type = raw_upsert_options.pop(
- "extracted_memory_type", None
- )
+ extracted_memory_type = raw_upsert_options.pop("extracted_memory_type", None)
# Reuse the actual vector-store ID when a semantic plan
# rebuilds an existing same-level record. Only genuinely new
# records derive an ID locally from (account, uri, level).
@@ -1009,14 +1004,6 @@ class TextEmbeddingHandler(DequeueHandlerBase):
return ProcessResult.failed(error_msg)
finally:
if embedding_msg is not None and execute_started_at is not None:
- tracker = get_request_wait_tracker()
- record_timing = getattr(tracker, "record_embedding_timing", None)
- if callable(record_timing):
- record_timing(
- embedding_msg.telemetry_id,
- queue_wait_ms=queue_wait_ms,
- execute_ms=(time.perf_counter() - execute_started_at) * 1000.0,
- )
from openviking.metrics.datasources.resource import ResourceIngestionEventDataSource
ResourceIngestionEventDataSource.record_stage(
@@ -1031,8 +1018,6 @@ class TextEmbeddingHandler(DequeueHandlerBase):
duration_seconds=(time.perf_counter() - execute_started_at),
account_id=embedding_msg.context_data.get("account_id"),
)
- if embedding_msg is not None and request_failed_message is not None:
- self._record_request_failure(embedding_msg, request_failed_message)
async def on_cancelled(self, data: Optional[Dict[str, Any]]) -> ProcessResult:
"""Settle request-scoped waiting when a queued embedding is cancelled."""
@@ -1062,17 +1047,8 @@ class TextEmbeddingHandler(DequeueHandlerBase):
*,
vector_written: bool = False,
) -> None:
- tracker = get_request_wait_tracker()
- tracker.mark_embedding_done(
- embedding_msg.telemetry_id,
- embedding_msg.id,
- vector_written=vector_written,
- )
-
- @staticmethod
- def _record_request_failure(embedding_msg: EmbeddingMsg, message: str) -> None:
- get_request_wait_tracker().mark_embedding_failed(
- embedding_msg.telemetry_id,
- embedding_msg.id,
- message,
- )
+ if vector_written and embedding_msg.task_id and embedding_msg._task_work_id:
+ get_task_tracker().record_context_count(
+ embedding_msg.task_id,
+ embedding_msg._task_work_id,
+ )
diff --git a/openviking/storage/content_write.py b/openviking/storage/content_write.py
index 2c474fce8..9f3455c8b 100644
--- a/openviking/storage/content_write.py
+++ b/openviking/storage/content_write.py
@@ -4,6 +4,7 @@
from __future__ import annotations
+import asyncio
import base64
import binascii
import hashlib
@@ -26,6 +27,9 @@ from openviking.resource.processing_mode import (
)
from openviking.resource.watch_storage import is_watch_task_control_uri
from openviking.server.identity import RequestContext
+from openviking.service.task_tracker import get_task_tracker, run_tracked_operation
+from openviking.service.task_tracker_concurrency import run_to_completion
+from openviking.service.task_work_index import get_enqueued_work_ids, get_task_context
from openviking.session.memory.memory_updater import MemoryUpdater
from openviking.session.memory.utils.memory_file_utils import MemoryFileUtils
from openviking.session.memory.utils.resource_refs import (
@@ -75,7 +79,6 @@ from openviking.storage.resource_target import AgfsResourceTarget
from openviking.storage.vector_ids import vector_record_id
from openviking.storage.viking_fs import VikingFS
from openviking.telemetry import get_current_telemetry
-from openviking.telemetry.request_wait_tracker import get_request_wait_tracker
from openviking.utils.content_hash import content_md5
from openviking.utils.embedding_utils import vectorize_directory_meta
from openviking.utils.ingest_options import IngestOptions
@@ -84,6 +87,7 @@ from openviking.utils.summarizer import FileRefreshRequest, Summarizer
from openviking.utils.tags import normalize_search_tags
from openviking_cli.exceptions import (
ConflictError,
+ DeadlineExceededError,
InvalidArgumentError,
NotFoundError,
OpenVikingError,
@@ -208,6 +212,42 @@ class ContentWriteCoordinator:
tags: list[str] | None = None,
tag_mode: str = "replace",
acl: AclSpec | Dict[str, Any] | None = None,
+ ) -> Dict[str, Any]:
+ async def execute() -> Dict[str, Any]:
+ return await self._write(
+ uri=uri,
+ content=content,
+ ctx=ctx,
+ mode=mode,
+ wait=wait,
+ processing_mode=processing_mode,
+ tags=tags,
+ tag_mode=tag_mode,
+ acl=acl,
+ )
+
+ result, _task_id = await run_tracked_operation(
+ "content_write",
+ account_id=ctx.account_id,
+ user_id=ctx.user.user_id,
+ operation=execute,
+ wait=wait,
+ resource_id=uri,
+ )
+ return result
+
+ async def _write(
+ self,
+ *,
+ uri: str,
+ content: str,
+ ctx: RequestContext,
+ mode: str = "replace",
+ wait: bool = False,
+ processing_mode: ProcessingMode = DEFAULT_PROCESSING_MODE,
+ tags: list[str] | None = None,
+ tag_mode: str = "replace",
+ acl: AclSpec | Dict[str, Any] | None = None,
) -> Dict[str, Any]:
self._validate_mode(mode)
processing_mode = normalize_processing_mode(processing_mode)
@@ -277,6 +317,36 @@ class ContentWriteCoordinator:
wait: bool = True,
skip_conflicts: bool = False,
processing_mode: ProcessingMode = DEFAULT_PROCESSING_MODE,
+ ) -> Dict[str, Any]:
+ async def execute() -> Dict[str, Any]:
+ return await self._batch_write(
+ root_uri=root_uri,
+ operations=operations,
+ ctx=ctx,
+ wait=wait,
+ skip_conflicts=skip_conflicts,
+ processing_mode=processing_mode,
+ )
+
+ result, _task_id = await run_tracked_operation(
+ "content_batch_write",
+ account_id=ctx.account_id,
+ user_id=ctx.user.user_id,
+ operation=execute,
+ wait=wait,
+ resource_id=root_uri,
+ )
+ return result
+
+ async def _batch_write(
+ self,
+ *,
+ root_uri: str,
+ operations: list[dict[str, Any]],
+ ctx: RequestContext,
+ wait: bool = True,
+ skip_conflicts: bool = False,
+ processing_mode: ProcessingMode = DEFAULT_PROCESSING_MODE,
) -> Dict[str, Any]:
"""Write a bundle under one directory, then refresh it as a batch.
@@ -445,25 +515,20 @@ class ContentWriteCoordinator:
raise
telemetry_id = get_current_telemetry().telemetry_id
- request_registered = False
- enqueue_barrier_id: str | None = None
+ coordinator_work_id: str | None = None
handoff_pending = False
refresh_outcome: Optional[_BatchRefreshOutcome] = None
try:
if refresh_kinds or sidecar_directories or committed_resource_plans:
- if telemetry_id:
- tracker = get_request_wait_tracker()
- tracker.register_request(telemetry_id)
- request_registered = True
- lock_handoff = await self._viking_fs._async_agfs.pathlock_to_handoff(lease)
- _, enqueue_barrier_id = await self._enqueue_lock_coordinator(
- root_uri=normalized_root,
- context_type=context_type_for_uri(normalized_root),
- ctx=ctx,
- telemetry_id=telemetry_id,
- lock_handoff=lock_handoff,
- )
- handoff_pending = True
+ lock_handoff = await self._viking_fs._async_agfs.pathlock_to_handoff(lease)
+ _, coordinator_work_id = await self._enqueue_lock_coordinator(
+ root_uri=normalized_root,
+ context_type=context_type_for_uri(normalized_root),
+ ctx=ctx,
+ telemetry_id=telemetry_id,
+ lock_handoff=lock_handoff,
+ )
+ handoff_pending = True
try:
for directory_uri, options in sorted(sidecar_directories.items()):
await self._vectorize_semantic_directory(
@@ -566,13 +631,13 @@ class ContentWriteCoordinator:
},
) from exc
finally:
- if enqueue_barrier_id is not None:
- get_request_wait_tracker().mark_semantic_done(
- telemetry_id,
- enqueue_barrier_id,
- processed_delta=0,
+ if coordinator_work_id is not None:
+ await run_to_completion(
+ lambda: self._mark_coordinator_enqueue_complete(
+ coordinator_work_id, ctx
+ )
)
- enqueue_barrier_id = None
+ coordinator_work_id = None
if handoff_pending:
try:
@@ -586,7 +651,7 @@ class ContentWriteCoordinator:
lock_settled = True
queue_status = (
await self._wait_for_request(telemetry_id=telemetry_id)
- if wait and request_registered
+ if wait
else refresh_outcome.queue_status
if refresh_outcome is not None
else None
@@ -600,12 +665,6 @@ class ContentWriteCoordinator:
await self._viking_fs._async_agfs.pathlock_release(lease)
lock_settled = True
finally:
- if enqueue_barrier_id is not None:
- get_request_wait_tracker().mark_semantic_done(
- telemetry_id,
- enqueue_barrier_id,
- processed_delta=0,
- )
if not lock_settled:
if handoff_pending:
try:
@@ -616,8 +675,6 @@ class ContentWriteCoordinator:
else:
await self._viking_fs._async_agfs.pathlock_release(lease)
lock_settled = True
- if request_registered:
- get_request_wait_tracker().cleanup(telemetry_id)
if write_error is not None:
raise write_error
@@ -929,14 +986,7 @@ class ContentWriteCoordinator:
file_abstracts=file_abstracts,
file_ingest_options=file_ingest_options,
)
- if msg.telemetry_id:
- get_request_wait_tracker().register_semantic_root(msg.telemetry_id, msg.id)
- try:
- await semantic_queue.enqueue(msg)
- except Exception as exc:
- if msg.telemetry_id:
- get_request_wait_tracker().mark_semantic_failed(msg.telemetry_id, msg.id, str(exc))
- raise
+ await semantic_queue.enqueue(msg)
return action
async def _enqueue_lock_coordinator(
@@ -949,6 +999,9 @@ class ContentWriteCoordinator:
lock_handoff: dict[str, Any],
) -> tuple[SemanticMsg, str]:
"""Enqueue the sole lease owner before scheduling request work."""
+ context = get_task_context()
+ if context is None:
+ raise RuntimeError("TaskContext is required for lock-coordinated content writes")
queue_manager = get_queue_manager()
semantic_queue = queue_manager.get_queue(queue_manager.SEMANTIC, allow_create=True)
msg = SemanticMsg(
@@ -967,18 +1020,26 @@ class ContentWriteCoordinator:
propagate_to_parent=False,
wait_for_request_completion=True,
)
- enqueue_barrier_id = f"{msg.id}:enqueue"
- tracker = get_request_wait_tracker()
- tracker.register_semantic_root(telemetry_id, msg.id)
- tracker.retain_request(telemetry_id, msg.id)
- tracker.register_semantic_root(telemetry_id, enqueue_barrier_id)
- try:
- await semantic_queue.enqueue(msg)
- except BaseException as exc:
- tracker.mark_semantic_failed(telemetry_id, enqueue_barrier_id, str(exc))
- tracker.mark_semantic_failed(telemetry_id, msg.id, str(exc))
- raise
- return msg, enqueue_barrier_id
+ msg.coalesce_key = f"lock-coordinator|{msg.id}"
+ receipt = await semantic_queue.enqueue(msg)
+ if not receipt.work_id:
+ raise RuntimeError("Lock coordinator was enqueued without Task Work identity")
+ return msg, receipt.work_id
+
+ async def _mark_coordinator_enqueue_complete(
+ self,
+ work_id: str,
+ ctx: RequestContext,
+ ) -> None:
+ context = get_task_context()
+ if context is None:
+ raise RuntimeError("TaskContext is required for lock-coordinated content writes")
+ await get_task_tracker().mark_enqueue_complete(
+ context.task_id,
+ work_id,
+ account_id=ctx.account_id,
+ user_id=ctx.user.user_id,
+ )
@staticmethod
def _raise_refresh_errors(queue_status: Dict[str, Any]) -> None:
@@ -1482,8 +1543,7 @@ class ContentWriteCoordinator:
previous_content: Optional[str] = None
lock_settled = False
- request_registered = False
- enqueue_barrier_id: str | None = None
+ coordinator_work_id: str | None = None
handoff_pending = False
try:
target_state = await self._classify_locked_write_target(uri, ctx=ctx)
@@ -1521,12 +1581,9 @@ class ContentWriteCoordinator:
has_async_work = bool(
prepared.plan.direct_index_actions or prepared.plan.file_refresh is not None
)
- if has_async_work and telemetry_id:
- tracker = get_request_wait_tracker()
- tracker.register_request(telemetry_id)
- request_registered = True
+ if has_async_work:
lock_handoff = await self._viking_fs._async_agfs.pathlock_to_handoff(lease)
- _, enqueue_barrier_id = await self._enqueue_lock_coordinator(
+ _, coordinator_work_id = await self._enqueue_lock_coordinator(
root_uri=uri,
context_type=context_type,
ctx=ctx,
@@ -1534,7 +1591,6 @@ class ContentWriteCoordinator:
lock_handoff=lock_handoff,
)
handoff_pending = True
-
work = await commit_and_enqueue_plan(
prepared.plan,
ctx=ctx,
@@ -1554,13 +1610,11 @@ class ContentWriteCoordinator:
ingest_options.acl_update, ctx
)
- if enqueue_barrier_id is not None:
- get_request_wait_tracker().mark_semantic_done(
- telemetry_id,
- enqueue_barrier_id,
- processed_delta=0,
+ if coordinator_work_id is not None:
+ await run_to_completion(
+ lambda: self._mark_coordinator_enqueue_complete(coordinator_work_id, ctx)
)
- enqueue_barrier_id = None
+ coordinator_work_id = None
if handoff_pending:
try:
await self._viking_fs._async_agfs.pathlock_handoff(lease)
@@ -1572,11 +1626,7 @@ class ContentWriteCoordinator:
await self._viking_fs._async_agfs.pathlock_release(lease)
lock_settled = True
- queue_status = (
- await self._wait_for_request(telemetry_id=telemetry_id)
- if wait
- else None
- )
+ queue_status = await self._wait_for_request(telemetry_id=telemetry_id) if wait else None
written_bytes = len(final_bytes)
semantic_status, vector_status = self._plan_statuses(
work, wait=wait, queue_status=queue_status
@@ -1598,12 +1648,6 @@ class ContentWriteCoordinator:
# when N already matches V, which makes that retry safe while V lags F.
raise
finally:
- if enqueue_barrier_id is not None:
- get_request_wait_tracker().mark_semantic_done(
- telemetry_id,
- enqueue_barrier_id,
- processed_delta=0,
- )
if not lock_settled:
if handoff_pending:
try:
@@ -1614,8 +1658,6 @@ class ContentWriteCoordinator:
else:
await self._viking_fs._async_agfs.pathlock_release(lease)
lock_settled = True
- if request_registered:
- get_request_wait_tracker().cleanup(telemetry_id)
@staticmethod
def _ensure_explicit_write_content_action(
@@ -1704,7 +1746,6 @@ class ContentWriteCoordinator:
) from exc
lock_released = False
- request_registered = False
try:
# Preserve the public invariant that generated sidecars cannot be
# materialized, while keeping the state check inside the exact lock.
@@ -1713,19 +1754,12 @@ class ContentWriteCoordinator:
await self._write_in_place(
uri, content, mode=mode, ctx=ctx, lease_ref=lease, existing_raw=previous_content
)
- if wait and telemetry_id:
- get_request_wait_tracker().register_request(telemetry_id)
- request_registered = True
vector_enqueued = await self._vectorize_abstract_overview(
uri=uri, ctx=ctx, ingest_options=ingest_options
)
await self._viking_fs._async_agfs.pathlock_release(lease)
lock_released = True
- queue_status = (
- await self._wait_for_request(telemetry_id=telemetry_id)
- if wait
- else None
- )
+ queue_status = await self._wait_for_request(telemetry_id=telemetry_id) if wait else None
vector_status = self._queue_work_status(
requested=vector_enqueued,
wait=wait,
@@ -1746,9 +1780,6 @@ class ContentWriteCoordinator:
if not lock_released:
await self._viking_fs._async_agfs.pathlock_release(lease)
raise
- finally:
- if request_registered:
- get_request_wait_tracker().cleanup(telemetry_id)
async def _vectorize_abstract_overview(
self,
@@ -1998,14 +2029,55 @@ class ContentWriteCoordinator:
except AbstractOverviewFormatError as exc:
raise InvalidArgumentError(str(exc)) from exc
+ async def _enqueue_semantic_refresh(
+ self,
+ *,
+ root_uri: str,
+ changed_uri: str,
+ context_type: str,
+ ctx: RequestContext,
+ change_type: str = "modified",
+ target_uri: str = "",
+ recursive: bool = False,
+ force_refresh: bool = False,
+ ingest_options: IngestOptions | None = None,
+ file_md5: str | None = None,
+ file_abstract: str = "",
+ ) -> FreshnessAction:
+ return await self._enqueue_semantic_refresh_changes(
+ root_uri=root_uri,
+ context_type=context_type,
+ ctx=ctx,
+ changes={change_type: [changed_uri]},
+ target_uri=target_uri,
+ recursive=recursive,
+ force_refresh=force_refresh,
+ ingest_options=ingest_options,
+ file_md5s={changed_uri: file_md5} if file_md5 else None,
+ file_abstracts={changed_uri: file_abstract} if file_abstract else None,
+ )
+
async def _wait_for_request(
self,
*,
telemetry_id: str,
) -> Dict[str, Any]:
- tracker = get_request_wait_tracker()
- await tracker.wait_for_request(telemetry_id)
- return tracker.build_queue_status(telemetry_id)
+ del telemetry_id
+ context = get_task_context()
+ if context is None:
+ raise RuntimeError("TaskContext is required for request-scoped queue waiting")
+ work_ids = get_enqueued_work_ids()
+ tracker = get_task_tracker()
+ try:
+ if context.work_id is None:
+ await tracker.wait_for_task_work(context.task_id, timeout=None)
+ elif work_ids:
+ await tracker.wait_for_work(context.task_id, work_ids, timeout=None)
+ except (TimeoutError, asyncio.TimeoutError) as exc:
+ raise DeadlineExceededError("queue processing", None) from exc
+ return tracker.build_queue_status(
+ context.task_id, work_ids if context.work_id is not None else None
+ )
async def _write_memory_with_refresh(
self,
@@ -2030,7 +2102,6 @@ class ContentWriteCoordinator:
) from exc
released = False
- request_registered = False
try:
target_state = await self._classify_locked_write_target(uri, ctx=ctx)
is_new_file = target_state.is_new_file
@@ -2049,9 +2120,6 @@ class ContentWriteCoordinator:
written_bytes = len(final_bytes)
await self._viking_fs._async_agfs.pathlock_release(lease)
released = True
- if wait and telemetry_id and self._vikingdb_has_queue():
- get_request_wait_tracker().register_request(telemetry_id)
- request_registered = True
overview_refreshed = await MemoryUpdater.refresh_schema_overview(
viking_fs=self._viking_fs,
directory_uri=root_uri,
@@ -2063,11 +2131,14 @@ class ContentWriteCoordinator:
uri=uri,
memory_type=MemoryUpdater.memory_type_from_uri(root_uri),
ctx=ctx,
+ strict=True,
ingest_options=ingest_options,
)
queue_status = None
if embedding_requested and wait:
- queue_status = await self._wait_for_request(telemetry_id=telemetry_id)
+ queue_status = await self._wait_for_request(
+ telemetry_id=telemetry_id,
+ )
vector_status = self._queue_work_status(
requested=embedding_requested,
wait=wait,
@@ -2090,8 +2161,7 @@ class ContentWriteCoordinator:
await self._viking_fs._async_agfs.pathlock_release(lease)
raise
finally:
- if request_registered:
- get_request_wait_tracker().cleanup(telemetry_id)
+ pass
def _vikingdb_has_queue(self) -> bool:
if not self._vikingdb:
diff --git a/openviking/storage/queuefs/add_resource_msg.py b/openviking/storage/queuefs/add_resource_msg.py
index e9b16364b..43e1a53d6 100644
--- a/openviking/storage/queuefs/add_resource_msg.py
+++ b/openviking/storage/queuefs/add_resource_msg.py
@@ -22,6 +22,7 @@ class AddResourceMsg:
user_id: str
group_ids: list[str] = field(default_factory=list)
role: str
+ _task_work_id: str = ""
path: str = ""
source_path: str = ""
telemetry_id: Optional[str] = None
@@ -159,6 +160,7 @@ class AddResourceMsg:
return cls(
task_id=str(task_id),
+ _task_work_id=str(data.get("_task_work_id") or ""),
path=str(path or ""),
source_path=str(data.get("source_path") or path or ""),
root_uri=str(root_uri),
diff --git a/openviking/storage/queuefs/add_resource_processor.py b/openviking/storage/queuefs/add_resource_processor.py
index 214a151ef..f7b4ec865 100644
--- a/openviking/storage/queuefs/add_resource_processor.py
+++ b/openviking/storage/queuefs/add_resource_processor.py
@@ -11,7 +11,10 @@ from typing import Any, Dict, Optional
from openviking.observability.context import bind_execution_context
from openviking.server.identity import RequestContext, Role
from openviking.service.task_tracker import TaskStatus, get_task_tracker
-from openviking.service.task_work_index import bind_task_context, extract_task_metadata
+from openviking.service.task_work_index import (
+ bind_task_context,
+ extract_task_metadata,
+)
from openviking.storage.queuefs.add_resource_msg import AddResourceMsg
from openviking.storage.queuefs.named_queue import DequeueHandlerBase
from openviking.storage.queuefs.process_result import ProcessResult
@@ -22,7 +25,6 @@ from openviking.telemetry import (
resolve_telemetry,
unregister_telemetry,
)
-from openviking.telemetry.request_wait_tracker import get_request_wait_tracker
from openviking.telemetry.resource_summary import record_resource_queue_metrics
from openviking.utils.log_correlation import log_correlation
from openviking_cli.exceptions import OpenVikingError
@@ -177,7 +179,6 @@ class AddResourceProcessor(DequeueHandlerBase):
unregister_telemetry(telemetry_id)
return ProcessResult.success()
- metadata = extract_task_metadata(data)
replay_result = getattr(task, "result", None)
resource_lock = None
if replay_result is None:
@@ -212,8 +213,6 @@ class AddResourceProcessor(DequeueHandlerBase):
else:
telemetry_id = telemetry.telemetry_id
register_telemetry(telemetry)
- request_wait_tracker = get_request_wait_tracker()
- request_wait_tracker.register_request(telemetry_id)
current_stage = "queued"
async def _set_stage(stage: str) -> None:
@@ -298,18 +297,19 @@ class AddResourceProcessor(DequeueHandlerBase):
)
else:
result = deepcopy(replay_result)
- await tracker.wait_for_descendants(msg.task_id, metadata.work_id)
- result.setdefault(
- "queue_status", request_wait_tracker.build_queue_status(telemetry_id)
+ metadata = extract_task_metadata(data)
+ await tracker.wait_for_task_work(
+ msg.task_id,
+ exclude_work_id=metadata.work_id if metadata is not None else None,
)
+ result["queue_status"] = tracker.build_queue_status(msg.task_id)
if replay_result is None:
- result["context_count"] = request_wait_tracker.get_embedding_context_count(
- telemetry_id
- )
+ result["context_count"] = tracker.get_context_count(msg.task_id)
record_resource_queue_metrics(
telemetry=telemetry,
telemetry_id=telemetry_id,
root_uri=result.get("root_uri"),
+ queue_timing=tracker.get_queue_timing(msg.task_id),
)
telemetry.set("resource.total.duration_ms", telemetry.elapsed_ms())
@@ -406,7 +406,6 @@ class AddResourceProcessor(DequeueHandlerBase):
terminal = True
return ProcessResult.failed(str(exc))
finally:
- request_wait_tracker.cleanup(telemetry_id)
unregister_telemetry(telemetry_id)
with suppress(Exception):
if resource_lock is not None:
diff --git a/openviking/storage/queuefs/embedding_msg.py b/openviking/storage/queuefs/embedding_msg.py
index b5f7203ca..09c7ed6d4 100644
--- a/openviking/storage/queuefs/embedding_msg.py
+++ b/openviking/storage/queuefs/embedding_msg.py
@@ -100,6 +100,8 @@ class EmbeddingMsg:
payload: EmbeddingPayload
id: str = field(default_factory=lambda: str(uuid4()))
telemetry_id: str = ""
+ task_id: str = ""
+ _task_work_id: str = ""
queue_enqueued_at: float = 0.0
failure_requeue_count: int = 0
@@ -108,6 +110,8 @@ class EmbeddingMsg:
message: Optional[EmbeddingInput] = None,
context_data: Optional[Dict[str, Any]] = None,
telemetry_id: str = "",
+ task_id: str = "",
+ _task_work_id: str = "",
action: IndexAction | str = IndexAction.UPSERT,
record_ids: Optional[List[str]] = None,
update_fields: Optional[Dict[str, Any]] = None,
@@ -120,6 +124,8 @@ class EmbeddingMsg:
) -> None:
self.id = str(uuid4())
self.telemetry_id = telemetry_id
+ self.task_id = task_id
+ self._task_work_id = _task_work_id
self.queue_enqueued_at = max(float(queue_enqueued_at or 0.0), 0.0)
self.failure_requeue_count = max(int(failure_requeue_count or 0), 0)
self.action = IndexAction(action)
@@ -325,6 +331,8 @@ class EmbeddingMsg:
return {
"id": self.id,
"telemetry_id": self.telemetry_id,
+ "task_id": self.task_id,
+ "_task_work_id": self._task_work_id,
"queue_enqueued_at": self.queue_enqueued_at,
"failure_requeue_count": self.failure_requeue_count,
"action": self.action.value,
@@ -391,6 +399,8 @@ class EmbeddingMsg:
action=action,
payload=payload,
telemetry_id=str(data.get("telemetry_id") or ""),
+ task_id=str(data.get("task_id") or ""),
+ _task_work_id=str(data.get("_task_work_id") or ""),
queue_enqueued_at=data.get("queue_enqueued_at", 0.0),
failure_requeue_count=data.get("failure_requeue_count", 0),
)
@@ -399,6 +409,8 @@ class EmbeddingMsg:
message=data.get("message"),
context_data=dict(data.get("context_data") or {}),
telemetry_id=str(data.get("telemetry_id") or ""),
+ task_id=str(data.get("task_id") or ""),
+ _task_work_id=str(data.get("_task_work_id") or ""),
action=action,
record_ids=data.get("record_ids"),
update_fields=data.get("update_fields"),
diff --git a/openviking/storage/queuefs/named_queue.py b/openviking/storage/queuefs/named_queue.py
index 9b17120d1..c0cb7dd0e 100644
--- a/openviking/storage/queuefs/named_queue.py
+++ b/openviking/storage/queuefs/named_queue.py
@@ -18,6 +18,7 @@ from openviking.storage.queuefs.queue_middleware import (
AckContext,
ClearContext,
EnqueueContext,
+ EnqueueReceipt,
ProcessContext,
QueueMiddleware,
)
@@ -171,10 +172,12 @@ class NamedQueue:
pass
self._initialized = True
- async def enqueue(self, data: Union[str, Dict[str, Any]]) -> str:
+ async def enqueue(self, data: Union[str, Dict[str, Any]]) -> EnqueueReceipt:
"""Send message to queue (enqueue)."""
await self._ensure_initialized()
- return await self._run_middleware("enqueue", EnqueueContext(self.name, data), self._enqueue)
+ ctx = EnqueueContext(self.name, data)
+ message_id = await self._run_middleware("enqueue", ctx, self._enqueue)
+ return EnqueueReceipt(message_id, ctx.work_id)
async def _enqueue(self, ctx: EnqueueContext) -> str:
data = json.dumps(ctx.payload) if isinstance(ctx.payload, dict) else ctx.payload
diff --git a/openviking/storage/queuefs/process_result.py b/openviking/storage/queuefs/process_result.py
index 455f4ab6c..c9ce2d12d 100644
--- a/openviking/storage/queuefs/process_result.py
+++ b/openviking/storage/queuefs/process_result.py
@@ -4,7 +4,7 @@
from dataclasses import dataclass
from enum import Enum
-from typing import Any, Optional
+from typing import Any, Optional, Tuple
class ProcessOutcome(str, Enum):
@@ -25,6 +25,7 @@ class ProcessResult:
outcome: ProcessOutcome
value: Any = None
error: Optional[str] = None
+ errors: Optional[Tuple[str, ...]] = None
def __post_init__(self) -> None:
if not isinstance(self.outcome, ProcessOutcome):
@@ -32,16 +33,25 @@ class ProcessResult:
if self.outcome is ProcessOutcome.FAILED:
if not isinstance(self.error, str):
raise ValueError("failed results require an error string")
- elif self.error is not None:
- raise ValueError("only failed results may contain an error")
+ if self.errors is not None and (
+ not self.errors or any(not isinstance(value, str) for value in self.errors)
+ ):
+ raise ValueError("failed result errors must be a non-empty tuple of strings")
+ elif self.error is not None or self.errors is not None:
+ raise ValueError("only failed results may contain errors")
@classmethod
def success(cls, value: Any = None) -> "ProcessResult":
return cls(ProcessOutcome.SUCCESS, value=value)
@classmethod
- def failed(cls, error: str) -> "ProcessResult":
- return cls(ProcessOutcome.FAILED, error=error)
+ def failed(
+ cls,
+ error: str,
+ *,
+ errors: Optional[Tuple[str, ...]] = None,
+ ) -> "ProcessResult":
+ return cls(ProcessOutcome.FAILED, error=error, errors=errors)
@classmethod
def requeued(cls) -> "ProcessResult":
diff --git a/openviking/storage/queuefs/queue_middleware.py b/openviking/storage/queuefs/queue_middleware.py
index c105b0ddd..4ac3c7f2a 100644
--- a/openviking/storage/queuefs/queue_middleware.py
+++ b/openviking/storage/queuefs/queue_middleware.py
@@ -16,6 +16,22 @@ class EnqueueContext:
queue: str
payload: Payload
committed: bool = False
+ work_id: Optional[str] = None
+
+
+class EnqueueReceipt(str):
+ """String-compatible QueueFS message ID with optional logical Work identity."""
+
+ work_id: Optional[str]
+
+ def __new__(cls, message_id: str, work_id: Optional[str] = None) -> "EnqueueReceipt":
+ value = super().__new__(cls, message_id)
+ value.work_id = work_id
+ return value
+
+ @property
+ def message_id(self) -> str:
+ return str(self)
@dataclass
diff --git a/openviking/storage/queuefs/reindex_msg.py b/openviking/storage/queuefs/reindex_msg.py
index 0db0397db..420df78bc 100644
--- a/openviking/storage/queuefs/reindex_msg.py
+++ b/openviking/storage/queuefs/reindex_msg.py
@@ -15,6 +15,7 @@ class ReindexMsg:
account_id: str
user_id: str
role: str
+ _task_work_id: str = ""
force: bool = False
recursive: bool = True
tags: Optional[list[str]] = None
@@ -41,6 +42,7 @@ class ReindexMsg:
lock_handoff_retry = 0
return cls(
task_id=str(data["task_id"]),
+ _task_work_id=str(data.get("_task_work_id") or ""),
uri=str(data["uri"]),
object_type=str(data["object_type"]),
mode=str(data["mode"]),
diff --git a/openviking/storage/queuefs/reindex_processor.py b/openviking/storage/queuefs/reindex_processor.py
index a9c15ea03..229a1779b 100644
--- a/openviking/storage/queuefs/reindex_processor.py
+++ b/openviking/storage/queuefs/reindex_processor.py
@@ -81,7 +81,6 @@ class ReindexProcessor(DequeueHandlerBase):
msg = ReindexMsg.from_dict(payload)
ctx = self._ctx(msg)
tracker = get_task_tracker()
- metadata = extract_task_metadata(data)
task = await tracker.get(
msg.task_id,
account_id=ctx.account_id,
@@ -151,8 +150,11 @@ class ReindexProcessor(DequeueHandlerBase):
ctx=ctx,
existing_lease=executor_lease,
)
- if metadata is not None:
- await tracker.wait_for_descendants(msg.task_id, metadata.work_id)
+ metadata = extract_task_metadata(data)
+ await tracker.wait_for_task_work(
+ msg.task_id,
+ exclude_work_id=metadata.work_id if metadata is not None else None,
+ )
await tracker.complete(
msg.task_id, result, account_id=ctx.account_id, user_id=ctx.user.user_id
)
diff --git a/openviking/storage/queuefs/semantic_executor.py b/openviking/storage/queuefs/semantic_executor.py
index 2c3532143..8dd96d5e0 100644
--- a/openviking/storage/queuefs/semantic_executor.py
+++ b/openviking/storage/queuefs/semantic_executor.py
@@ -516,6 +516,8 @@ class SemanticTreeExecutor:
self._task_context.task_id,
self._task_context.account_id,
self._task_context.user_id,
+ enqueued_work=self._task_context.enqueued_work,
+ work_id=self._task_context.work_id,
)
if self._task_context is not None
else detach_task_context()
diff --git a/openviking/storage/queuefs/semantic_msg.py b/openviking/storage/queuefs/semantic_msg.py
index f4e41d297..4bbb151f6 100644
--- a/openviking/storage/queuefs/semantic_msg.py
+++ b/openviking/storage/queuefs/semantic_msg.py
@@ -60,6 +60,8 @@ class SemanticMsg:
# Additional flags
skip_vectorization: bool = False
telemetry_id: str = ""
+ task_id: str = ""
+ _task_work_id: str = ""
target_uri: str = ""
lock_handoff: Optional[Dict[str, Any]] = None
is_code_repo: bool = False
@@ -102,6 +104,8 @@ class SemanticMsg:
role: str = "root",
skip_vectorization: bool = False,
telemetry_id: str = "",
+ task_id: str = "",
+ _task_work_id: str = "",
target_uri: str = "",
lock_handoff: Optional[Dict[str, Any]] = None,
is_code_repo: bool = False,
@@ -141,6 +145,8 @@ class SemanticMsg:
self.role = role
self.skip_vectorization = skip_vectorization
self.telemetry_id = telemetry_id
+ self.task_id = task_id
+ self._task_work_id = _task_work_id
self.target_uri = target_uri
self.lock_handoff = lock_handoff
self.is_code_repo = is_code_repo
@@ -224,6 +230,8 @@ class SemanticMsg:
role=data.get("role", "root"),
skip_vectorization=data.get("skip_vectorization", False),
telemetry_id=data.get("telemetry_id", ""),
+ task_id=data.get("task_id", ""),
+ _task_work_id=data.get("_task_work_id", ""),
target_uri=data.get("target_uri", ""),
lock_handoff=data.get("lock_handoff"),
is_code_repo=data.get("is_code_repo", False),
diff --git a/openviking/storage/queuefs/semantic_processor.py b/openviking/storage/queuefs/semantic_processor.py
index 4163c3216..222494fdb 100644
--- a/openviking/storage/queuefs/semantic_processor.py
+++ b/openviking/storage/queuefs/semantic_processor.py
@@ -36,8 +36,9 @@ from openviking.parse.parsers.media.utils import (
from openviking.prompts import render_prompt
from openviking.pyagfs.exceptions import AGFSNotADirectoryError
from openviking.server.identity import RequestContext, Role
+from openviking.service.task_tracker import get_task_tracker, run_tracked_operation
from openviking.service.task_tracker_concurrency import run_to_completion
-from openviking.service.task_work_index import detach_task_context
+from openviking.service.task_work_index import detach_task_context, get_task_context
from openviking.storage.abstract_overview import (
AbstractOverviewWriteResult,
body_for_preview,
@@ -63,7 +64,6 @@ from openviking.telemetry import (
bind_telemetry,
bind_telemetry_stage,
)
-from openviking.telemetry.request_wait_tracker import get_request_wait_tracker
from openviking.telemetry.span_models import create_root_span_attributes
from openviking.utils.circuit_breaker import CircuitBreaker, CircuitBreakerOpen
from openviking.utils.ingest_options import IngestOptions
@@ -338,11 +338,9 @@ class SemanticProcessor(DequeueHandlerBase):
try:
await work.reenqueue_after_error()
self._merge_request_stats(msg.telemetry_id, requeue_count=1)
- get_request_wait_tracker().record_semantic_requeue(msg.telemetry_id)
except Exception as requeue_err:
logger.error(f"Failed to re-enqueue semantic message: {requeue_err}")
self._merge_request_stats(msg.telemetry_id, error_count=1)
- get_request_wait_tracker().mark_semantic_failed(msg.telemetry_id, msg.id, str(error))
await self._cleanup_local_artifact(msg)
return ProcessResult.failed(str(error))
return ProcessResult.requeued()
@@ -448,7 +446,55 @@ class SemanticProcessor(DequeueHandlerBase):
lock: Optional[Dict[str, Any]] = None,
) -> ProcessResult:
"""Process dequeued SemanticMsg, recursively process all subdirectories."""
- msg: Optional[SemanticMsg] = None
+ import json
+
+ if not data:
+ return ProcessResult.success()
+ try:
+ if "data" in data and isinstance(data["data"], str):
+ data = json.loads(data["data"])
+ msg = SemanticMsg.from_dict(data)
+ except Exception as exc:
+ logger.error("Invalid semantic message: %s", exc, exc_info=True)
+ return ProcessResult.failed(str(exc))
+
+ if msg.context_type != "skill" or get_task_context() is not None:
+ return await self._process_message(msg, lock)
+
+ # Parent refreshes and recovered taskless Skill messages need their own
+ # child-work collector so package ownership lasts through vector writes.
+ result: Optional[ProcessResult] = None
+
+ async def process() -> Dict[str, Any]:
+ nonlocal result
+ result = await self._process_message(msg, lock)
+ if result.error is not None:
+ context = get_task_context()
+ assert context is not None
+ await get_task_tracker().fail(
+ context.task_id,
+ result.error,
+ account_id=msg.account_id,
+ user_id=msg.user_id,
+ )
+ return {"uri": msg.uri, "outcome": result.outcome.value}
+
+ await run_tracked_operation(
+ "semantic_refresh",
+ account_id=msg.account_id,
+ user_id=msg.user_id,
+ resource_id=msg.uri,
+ operation=process,
+ wait=False,
+ )
+ assert result is not None
+ return result
+
+ async def _process_message(
+ self,
+ msg: SemanticMsg,
+ lock: Optional[Dict[str, Any]],
+ ) -> ProcessResult:
collector = None
work: Optional[SemanticMessageWork] = None
execute_started_at: float | None = None
@@ -456,16 +502,6 @@ class SemanticProcessor(DequeueHandlerBase):
execute_status = "ok"
breaker = None
try:
- import json
-
- if not data:
- return ProcessResult.success()
-
- if "data" in data and isinstance(data["data"], str):
- data = json.loads(data["data"])
-
- assert data is not None
- msg = SemanticMsg.from_dict(data)
breaker = self._account_breaker(msg.account_id)
work = self._message_work(msg, lock)
work.start()
@@ -474,8 +510,6 @@ class SemanticProcessor(DequeueHandlerBase):
queue_wait_ms = max((time.time() - msg.queue_enqueued_at) * 1000.0, 0.0)
if VikingURI(msg.uri).parent is None:
logger.warning("Skipping semantic generation for root URI: %s", msg.uri)
- if msg.telemetry_id and msg.id:
- get_request_wait_tracker().mark_semantic_done(msg.telemetry_id, msg.id)
await self._cleanup_local_artifact(msg)
return ProcessResult.success()
if is_semantic_msg_stale(msg):
@@ -504,8 +538,6 @@ class SemanticProcessor(DequeueHandlerBase):
msg.coalesce_version,
)
await work.skip()
- if msg.telemetry_id and msg.id:
- get_request_wait_tracker().mark_semantic_done(msg.telemetry_id, msg.id)
await self._cleanup_local_artifact(msg)
return ProcessResult.success()
# Circuit breaker: if API is known-broken, re-enqueue and wait
@@ -517,7 +549,6 @@ class SemanticProcessor(DequeueHandlerBase):
)
await work.reenqueue()
self._merge_request_stats(msg.telemetry_id, requeue_count=1)
- get_request_wait_tracker().record_semantic_requeue(msg.telemetry_id)
return ProcessResult.requeued()
collector = work.resolve_telemetry()
telemetry_ctx = bind_telemetry(collector) if collector is not None else nullcontext()
@@ -544,8 +575,6 @@ class SemanticProcessor(DequeueHandlerBase):
if not await get_viking_fs().exists(msg.uri, ctx=current_ctx):
logger.info("Skipping semantic message for missing root: uri=%s", msg.uri)
await work.skip()
- if msg.telemetry_id and msg.id:
- get_request_wait_tracker().mark_semantic_done(msg.telemetry_id, msg.id)
return ProcessResult.success()
if (
@@ -557,16 +586,32 @@ class SemanticProcessor(DequeueHandlerBase):
"SemanticProcessor requires a VLM resolver for account-owned work"
)
if not await work.acquire_lock(current_ctx):
- get_request_wait_tracker().mark_semantic_done(msg.telemetry_id, msg.id)
return ProcessResult.success()
semantic_lock = work.scope
assert semantic_lock is not None
- if (
- msg.wait_for_request_completion
- and not get_request_wait_tracker().is_complete_except(
- msg.telemetry_id, msg.id
+ task_context = get_task_context()
+ if msg.wait_for_request_completion and task_context is not None:
+ tracker = get_task_tracker()
+ enqueue_complete = await tracker.is_enqueue_complete(
+ task_context.task_id,
+ task_context.work_id or "",
+ account_id=task_context.account_id,
+ user_id=task_context.user_id,
+ )
+ siblings_pending = tracker.has_work(
+ task_context.task_id,
+ exclude_work_id=task_context.work_id,
)
+ else:
+ enqueue_complete = True
+ siblings_pending = False
+ if not enqueue_complete and not tracker.has_active_producer(
+ task_context.task_id
):
+ return ProcessResult.failed(
+ "Content write producer ended before child Work enqueue completed"
+ )
+ if not enqueue_complete or siblings_pending:
# Keep a single-worker semantic queue moving while this
# coordinator retains the producer's lease.
await asyncio.sleep(0.05)
@@ -755,11 +800,6 @@ class SemanticProcessor(DequeueHandlerBase):
if failure is not None:
return failure
is_request_coordinator = msg.wait_for_request_completion
- get_request_wait_tracker().mark_semantic_done(
- msg.telemetry_id,
- msg.id,
- processed_delta=0 if is_request_coordinator else 1,
- )
if not is_request_coordinator:
self._merge_request_stats(msg.telemetry_id, processed=1)
logger.info(f"Completed semantic generation for: {msg.uri}")
@@ -802,9 +842,6 @@ class SemanticProcessor(DequeueHandlerBase):
)
if msg is not None:
self._merge_request_stats(msg.telemetry_id, error_count=1)
- get_request_wait_tracker().mark_semantic_failed(
- msg.telemetry_id, msg.id, str(e)
- )
if msg is not None:
await self._cleanup_local_artifact(msg)
return ProcessResult.failed(str(e))
@@ -823,9 +860,6 @@ class SemanticProcessor(DequeueHandlerBase):
breaker.record_failure(e)
if msg is not None:
self._merge_request_stats(msg.telemetry_id, error_count=1)
- get_request_wait_tracker().mark_semantic_failed(
- msg.telemetry_id, msg.id, str(e)
- )
if msg is not None:
await self._cleanup_local_artifact(msg)
return ProcessResult.failed(str(e))
@@ -839,21 +873,10 @@ class SemanticProcessor(DequeueHandlerBase):
)
if msg is not None:
self._merge_request_stats(msg.telemetry_id, error_count=1)
- get_request_wait_tracker().mark_semantic_failed(
- msg.telemetry_id, msg.id, str(e)
- )
await self._cleanup_local_artifact(msg)
return ProcessResult.failed(str(e))
finally:
if msg is not None and execute_started_at is not None:
- tracker = get_request_wait_tracker()
- record_timing = getattr(tracker, "record_semantic_timing", None)
- if callable(record_timing):
- record_timing(
- msg.telemetry_id,
- queue_wait_ms=queue_wait_ms,
- execute_ms=(time.perf_counter() - execute_started_at) * 1000.0,
- )
from openviking.metrics.datasources.resource import ResourceIngestionEventDataSource
ResourceIngestionEventDataSource.record_stage(
@@ -1991,7 +2014,6 @@ class SemanticProcessor(DequeueHandlerBase):
return await _enqueue_embedding_message(
embedding_queue,
embedding_msg,
- failure_message=f"Failed to enqueue vector scalar update for {uri}",
)
async def _vectorize_single_file(
diff --git a/openviking/storage/queuefs/semantic_work.py b/openviking/storage/queuefs/semantic_work.py
index 116088eda..ab8973bc2 100644
--- a/openviking/storage/queuefs/semantic_work.py
+++ b/openviking/storage/queuefs/semantic_work.py
@@ -8,7 +8,9 @@ import asyncio
from typing import TYPE_CHECKING, Any, Awaitable, Callable, TypeVar
from openviking.server.identity import RequestContext
+from openviking.service.task_tracker import get_task_tracker
from openviking.service.task_tracker_concurrency import run_to_completion
+from openviking.service.task_work_index import get_enqueued_work_ids, get_task_context
from openviking.storage.queuefs.process_result import ProcessResult
from openviking.storage.queuefs.semantic_executor import SemanticTreeStats
from openviking.storage.queuefs.semantic_lock import SemanticLockScope
@@ -20,7 +22,6 @@ from openviking.telemetry import (
resolve_telemetry,
unregister_telemetry,
)
-from openviking.telemetry.request_wait_tracker import get_request_wait_tracker
if TYPE_CHECKING:
from openviking.storage.queuefs.semantic_processor import SemanticProcessor
@@ -62,9 +63,14 @@ class SemanticMessageWork:
async def finish_processing(self, succeeded: bool) -> None:
assert self.scope is not None
- if self.msg.lock_handoff is not None:
- await get_request_wait_tracker().wait_for_embeddings(
- self.msg.telemetry_id,
+ context = get_task_context()
+ if self.msg.wait_for_request_completion and context is not None:
+ await get_task_tracker().wait_for_task_work(
+ context.task_id,
+ exclude_work_id=context.work_id,
+ )
+ elif self.msg.lock_handoff is not None:
+ await self._wait_for_embedding_work(
stop_waiting=getattr(self.processor, "_embedding_worker_stopped", None),
)
if succeeded:
@@ -74,6 +80,20 @@ class SemanticMessageWork:
def failure_result(self, stats: SemanticTreeStats | None) -> ProcessResult | None:
return None
+ async def _wait_for_embedding_work(
+ self,
+ *,
+ stop_waiting: Callable[[], bool] | None = None,
+ ) -> None:
+ context = get_task_context()
+ work_ids = get_enqueued_work_ids("Embedding")
+ if context is not None and work_ids:
+ await get_task_tracker().wait_for_work(
+ context.task_id,
+ work_ids,
+ stop_waiting=stop_waiting,
+ )
+
async def reenqueue(self) -> None:
async def enqueue(queue, msg):
if self.scope is not None and self.scope.lock is not None and self.scope._owned:
@@ -93,11 +113,8 @@ class SemanticMessageWork:
async def cancel(self) -> None:
if self.scope is None:
await self.skip()
- get_request_wait_tracker().mark_semantic_done(self.msg.telemetry_id, self.msg.id)
async def cancel_queued(self) -> None:
- if self.msg.telemetry_id and self.msg.id:
- get_request_wait_tracker().mark_semantic_done(self.msg.telemetry_id, self.msg.id)
await self.processor._release_cancelled_semantic_lock(self.msg)
async def close(self) -> None:
@@ -120,7 +137,7 @@ class SkillSemanticMessageWork(SemanticMessageWork):
caller_lock: dict[str, Any] | None,
) -> None:
super().__init__(processor, msg, caller_lock)
- self._internal_tracking = False
+ self._owns_telemetry = False
self._lock_started = False
self._lock_closed = False
@@ -130,12 +147,6 @@ class SkillSemanticMessageWork(SemanticMessageWork):
msg.telemetry_id = OperationTelemetry(
operation="skill_index", enabled=False
).telemetry_id
- tracker = get_request_wait_tracker()
- self._internal_tracking = not tracker.has_request(msg.telemetry_id)
- tracker.retain_request(msg.telemetry_id, msg.id)
- tracker.register_semantic_root(msg.telemetry_id, msg.id)
- if self._internal_tracking:
- tracker.cleanup(msg.telemetry_id)
def resolve_telemetry(self) -> OperationTelemetry:
collector = super().resolve_telemetry()
@@ -143,6 +154,7 @@ class SkillSemanticMessageWork(SemanticMessageWork):
collector = OperationTelemetry(operation="skill_index", enabled=False)
collector.telemetry_id = self.msg.telemetry_id
register_telemetry(collector)
+ self._owns_telemetry = True
return collector
async def acquire_lock(self, ctx: RequestContext) -> bool:
@@ -158,8 +170,7 @@ class SkillSemanticMessageWork(SemanticMessageWork):
async def finish_processing(self, succeeded: bool) -> None:
async def finish():
if not self.msg.skip_vectorization:
- await get_request_wait_tracker().wait_for_embeddings(
- self.msg.telemetry_id,
+ await self._wait_for_embedding_work(
stop_waiting=self.processor._embedding_worker_stopped,
)
if succeeded:
@@ -173,13 +184,12 @@ class SkillSemanticMessageWork(SemanticMessageWork):
def failure_result(self, stats: SemanticTreeStats | None) -> ProcessResult | None:
if stats is None or not stats.failures:
return None
- for failure in stats.failures:
- get_request_wait_tracker().mark_semantic_failed(
- self.msg.telemetry_id, self.msg.id, failure
- )
# Report every file error, but settle this queue item only once.
self.processor._merge_request_stats(self.msg.telemetry_id, error_count=len(stats.failures))
- return ProcessResult.failed("\n".join(stats.failures))
+ return ProcessResult.failed(
+ "\n".join(stats.failures),
+ errors=tuple(stats.failures),
+ )
async def reenqueue(self) -> None:
async def enqueue(queue, msg):
@@ -201,7 +211,6 @@ class SkillSemanticMessageWork(SemanticMessageWork):
await run_to_completion(
lambda: self.processor._release_cancelled_semantic_lock(self.msg)
)
- get_request_wait_tracker().mark_semantic_done(self.msg.telemetry_id, self.msg.id)
raise
async def skip(self) -> None:
@@ -214,18 +223,13 @@ class SkillSemanticMessageWork(SemanticMessageWork):
if not self._lock_started:
# Cancellation during the circuit-breaker delay precedes adoption.
await self.skip()
- get_request_wait_tracker().mark_semantic_done(self.msg.telemetry_id, self.msg.id)
async def close(self) -> None:
if self.scope is not None and not self._lock_closed:
await run_to_completion(self.scope.close)
- if self._internal_tracking and get_request_wait_tracker().is_complete(
- self.msg.telemetry_id
- ):
+ if self._owns_telemetry:
unregister_telemetry(self.msg.telemetry_id)
async def cancel_queued(self) -> None:
# Skill rollback must observe release before this item is settled.
await self.processor._release_cancelled_semantic_lock(self.msg)
- if self.msg.telemetry_id and self.msg.id:
- get_request_wait_tracker().mark_semantic_done(self.msg.telemetry_id, self.msg.id)
diff --git a/openviking/storage/queuefs/session_commit_msg.py b/openviking/storage/queuefs/session_commit_msg.py
index d66dce1a6..a62b67072 100644
--- a/openviking/storage/queuefs/session_commit_msg.py
+++ b/openviking/storage/queuefs/session_commit_msg.py
@@ -13,6 +13,7 @@ class SessionCommitMsg:
session_uri: str
archive_uri: str
user: Dict[str, str]
+ _task_work_id: str = ""
memory_policy: Dict[str, Any] = field(default_factory=dict)
# When True, Phase 2's final meta merge also clears the auto-commit error
# fields and stamps last_auto_commit_at. Defaults keep old producers working.
diff --git a/openviking/storage/viking_fs/_snapshot.py b/openviking/storage/viking_fs/_snapshot.py
index 0b1fc7421..b88c352ce 100644
--- a/openviking/storage/viking_fs/_snapshot.py
+++ b/openviking/storage/viking_fs/_snapshot.py
@@ -5,7 +5,7 @@
import asyncio
import sys
from dataclasses import replace
-from typing import Any, Dict, List, Optional, Union
+from typing import Any, Dict, List, Literal, Optional, Union
from openviking.pyagfs.exceptions import (
AGFSInvalidOperationError,
@@ -829,7 +829,14 @@ class _SnapshotMixin:
)
for op, uri, level in tasks:
loop.create_task(
- self._run_vector_rebuild(executor, op, uri, level, ctx),
+ self._run_vector_rebuild(
+ executor,
+ op,
+ uri,
+ level,
+ ctx,
+ file_task_owner="standalone",
+ ),
name=f"vikingfs-git-{op}:{uri}:{int(level)}",
)
@@ -867,7 +874,14 @@ class _SnapshotMixin:
with bind_task_context(task_id, ctx.account_id, ctx.user.user_id):
await asyncio.gather(
*[
- self._run_vector_rebuild(executor, op, uri, level, reindex_ctx)
+ self._run_vector_rebuild(
+ executor,
+ op,
+ uri,
+ level,
+ reindex_ctx,
+ file_task_owner="current",
+ )
for (op, uri, level) in tasks
]
)
@@ -897,13 +911,27 @@ class _SnapshotMixin:
uri: str,
level: Any,
ctx: RequestContext,
+ *,
+ file_task_owner: Literal["current", "standalone"],
) -> None:
"""Wrapper coroutine: dispatch one vector task and swallow errors."""
try:
if op == "reindex_marker":
await executor.reindex_directory_marker(dir_uri=uri, level=level, ctx=ctx)
elif op == "reindex_file":
- await executor.execute(uri=uri, mode="vectors_only", wait=True, ctx=ctx)
+ if file_task_owner == "current":
+ await executor.execute_in_current_task(
+ uri=uri,
+ mode="vectors_only",
+ ctx=ctx,
+ )
+ else:
+ await executor.execute(
+ uri=uri,
+ mode="vectors_only",
+ wait=True,
+ ctx=ctx,
+ )
elif op == "delete":
await executor.delete_uri_level(uri=uri, level=level, ctx=ctx)
else: # pragma: no cover - defensive
diff --git a/openviking/telemetry/request_wait_tracker.py b/openviking/telemetry/request_wait_tracker.py
deleted file mode 100644
index 3e2d21363..000000000
--- a/openviking/telemetry/request_wait_tracker.py
+++ /dev/null
@@ -1,344 +0,0 @@
-# Copyright (c) 2026 Beijing Volcano Engine Technology Co., Ltd.
-# SPDX-License-Identifier: AGPL-3.0
-"""Request-scoped wait tracker for write APIs."""
-
-from __future__ import annotations
-
-import asyncio
-import threading
-import time
-from dataclasses import dataclass, field
-from typing import Callable, Dict, List, Optional, Set
-
-
-@dataclass
-class _RequestWaitState:
- pending_semantic_roots: Set[str] = field(default_factory=set)
- pending_embedding_roots: Set[str] = field(default_factory=set)
- semantic_processed: int = 0
- semantic_requeue_count: int = 0
- semantic_error_count: int = 0
- semantic_errors: List[str] = field(default_factory=list)
- embedding_processed: int = 0
- embedding_context_count: int = 0
- embedding_requeue_count: int = 0
- embedding_error_count: int = 0
- embedding_errors: List[str] = field(default_factory=list)
- semantic_queue_wait_ms: float = 0.0
- semantic_execute_ms: float = 0.0
- embedding_queue_wait_ms: float = 0.0
- embedding_execute_ms: float = 0.0
- created_at: float = field(default_factory=time.time)
- retained_roots: Set[str] = field(default_factory=set)
- cleanup_requested: bool = False
-
-
-class RequestWaitTracker:
- """Track request-scoped queue completion using telemetry_id."""
-
- _instance: Optional["RequestWaitTracker"] = None
-
- def __new__(cls) -> "RequestWaitTracker":
- if cls._instance is None:
- cls._instance = super().__new__(cls)
- return cls._instance
-
- def __init__(self) -> None:
- if hasattr(self, "_lock"):
- return
- self._lock = threading.Lock()
- self._states: Dict[str, _RequestWaitState] = {}
-
- @classmethod
- def get_instance(cls) -> "RequestWaitTracker":
- return cls()
-
- def _create_state(self, telemetry_id: str) -> Optional[_RequestWaitState]:
- if not telemetry_id:
- return None
- with self._lock:
- return self._states.setdefault(telemetry_id, _RequestWaitState())
-
- def register_request(self, telemetry_id: str) -> None:
- self._create_state(telemetry_id)
-
- def has_request(self, telemetry_id: str) -> bool:
- with self._lock:
- return telemetry_id in self._states
-
- def retain_request(self, telemetry_id: str, root_id: str) -> None:
- """Keep a Skill worker's accounting alive if its HTTP waiter times out."""
- if not telemetry_id or not root_id:
- return
- with self._lock:
- state = self._states.setdefault(telemetry_id, _RequestWaitState())
- state.retained_roots.add(root_id)
-
- def _release_retained_root(
- self, telemetry_id: str, root_id: str, state: _RequestWaitState
- ) -> None:
- state.retained_roots.discard(root_id)
- if state.cleanup_requested and not state.retained_roots:
- self._states.pop(telemetry_id, None)
-
- def register_semantic_root(self, telemetry_id: str, root_id: str) -> None:
- if not telemetry_id or not root_id:
- return
- with self._lock:
- state = self._states.get(telemetry_id)
- if state is None:
- return
- state.pending_semantic_roots.add(root_id)
-
- def register_embedding_root(self, telemetry_id: str, root_id: str) -> None:
- if not telemetry_id or not root_id:
- return
- with self._lock:
- state = self._states.get(telemetry_id)
- if state is None:
- return
- state.pending_embedding_roots.add(root_id)
-
- def record_embedding_requeue(self, telemetry_id: str, delta: int = 1) -> None:
- if not telemetry_id:
- return
- with self._lock:
- state = self._states.get(telemetry_id)
- if state is None:
- return
- state.embedding_requeue_count += max(delta, 0)
-
- def record_semantic_timing(
- self, telemetry_id: str, *, queue_wait_ms: float = 0.0, execute_ms: float = 0.0
- ) -> None:
- self._record_timing(
- telemetry_id,
- queue_wait_ms=queue_wait_ms,
- execute_ms=execute_ms,
- queue_wait_attr="semantic_queue_wait_ms",
- execute_attr="semantic_execute_ms",
- )
-
- def record_embedding_timing(
- self, telemetry_id: str, *, queue_wait_ms: float = 0.0, execute_ms: float = 0.0
- ) -> None:
- self._record_timing(
- telemetry_id,
- queue_wait_ms=queue_wait_ms,
- execute_ms=execute_ms,
- queue_wait_attr="embedding_queue_wait_ms",
- execute_attr="embedding_execute_ms",
- )
-
- def _record_timing(
- self,
- telemetry_id: str,
- *,
- queue_wait_ms: float,
- execute_ms: float,
- queue_wait_attr: str,
- execute_attr: str,
- ) -> None:
- if not telemetry_id:
- return
- with self._lock:
- state = self._states.get(telemetry_id)
- if state is None:
- return
- setattr(
- state, queue_wait_attr, getattr(state, queue_wait_attr) + max(queue_wait_ms, 0.0)
- )
- setattr(state, execute_attr, getattr(state, execute_attr) + max(execute_ms, 0.0))
-
- def get_embedding_context_count(self, telemetry_id: str) -> int:
- """Return contexts successfully indexed for one request."""
- if not telemetry_id:
- return 0
- with self._lock:
- state = self._states.get(telemetry_id)
- return state.embedding_context_count if state is not None else 0
-
- def get_queue_timing(self, telemetry_id: str) -> Dict[str, Dict[str, float]]:
- if not telemetry_id:
- return {
- "semantic": {"queue_wait_ms": 0.0, "execute_ms": 0.0},
- "embedding": {"queue_wait_ms": 0.0, "execute_ms": 0.0},
- }
- with self._lock:
- state = self._states.get(telemetry_id)
- if state is None:
- return {
- "semantic": {"queue_wait_ms": 0.0, "execute_ms": 0.0},
- "embedding": {"queue_wait_ms": 0.0, "execute_ms": 0.0},
- }
- return {
- "semantic": {
- "queue_wait_ms": state.semantic_queue_wait_ms,
- "execute_ms": state.semantic_execute_ms,
- },
- "embedding": {
- "queue_wait_ms": state.embedding_queue_wait_ms,
- "execute_ms": state.embedding_execute_ms,
- },
- }
-
- def mark_semantic_done(
- self,
- telemetry_id: str,
- root_id: str,
- processed_delta: int = 1,
- ) -> None:
- if not telemetry_id:
- return
- with self._lock:
- state = self._states.get(telemetry_id)
- if state is None:
- return
- state.pending_semantic_roots.discard(root_id)
- state.semantic_processed += max(processed_delta, 0)
- self._release_retained_root(telemetry_id, root_id, state)
-
- def record_semantic_requeue(self, telemetry_id: str, delta: int = 1) -> None:
- if not telemetry_id:
- return
- with self._lock:
- state = self._states.get(telemetry_id)
- if state is None:
- return
- state.semantic_requeue_count += max(delta, 0)
-
- def mark_semantic_failed(self, telemetry_id: str, root_id: str, message: str) -> None:
- if not telemetry_id:
- return
- with self._lock:
- state = self._states.get(telemetry_id)
- if state is None:
- return
- state.pending_semantic_roots.discard(root_id)
- state.semantic_error_count += 1
- if message:
- state.semantic_errors.append(message)
- self._release_retained_root(telemetry_id, root_id, state)
-
- def mark_embedding_done(
- self,
- telemetry_id: str,
- root_id: str,
- processed_delta: int = 1,
- *,
- vector_written: bool = False,
- ) -> None:
- if not telemetry_id:
- return
- with self._lock:
- state = self._states.get(telemetry_id)
- if state is None:
- return
- was_pending = root_id in state.pending_embedding_roots
- state.pending_embedding_roots.discard(root_id)
- state.embedding_processed += max(processed_delta, 0)
- if vector_written and was_pending:
- state.embedding_context_count += 1
-
- def mark_embedding_failed(self, telemetry_id: str, root_id: str, message: str) -> None:
- if not telemetry_id:
- return
- with self._lock:
- state = self._states.get(telemetry_id)
- if state is None:
- return
- state.pending_embedding_roots.discard(root_id)
- state.embedding_error_count += 1
- if message:
- state.embedding_errors.append(message)
-
- def is_complete(self, telemetry_id: str) -> bool:
- if not telemetry_id:
- return True
- with self._lock:
- state = self._states.get(telemetry_id)
- if state is None:
- return True
- return not state.pending_semantic_roots and not state.pending_embedding_roots
-
- def is_complete_except(self, telemetry_id: str, semantic_root_id: str) -> bool:
- """Return whether all work except one coordinating semantic root has settled."""
- if not telemetry_id:
- return True
- with self._lock:
- state = self._states.get(telemetry_id)
- if state is None:
- return True
- pending_semantic = state.pending_semantic_roots - {semantic_root_id}
- return not pending_semantic and not state.pending_embedding_roots
-
- async def wait_for_request(
- self,
- telemetry_id: str,
- timeout: Optional[float] = None,
- ) -> None:
- if not telemetry_id:
- return
- start = time.time()
- while True:
- if self.is_complete(telemetry_id):
- return
- if timeout is not None and (time.time() - start) > timeout:
- raise TimeoutError(f"Request processing not complete after {timeout}s")
- await asyncio.sleep(0.05)
-
- async def wait_for_embeddings(
- self,
- telemetry_id: str,
- *,
- stop_waiting: Optional[Callable[[], bool]] = None,
- ) -> None:
- """Drain embeddings while the producing semantic root remains pending."""
- if not telemetry_id:
- return
- while True:
- with self._lock:
- state = self._states.get(telemetry_id)
- if state is None or not state.pending_embedding_roots:
- return
- if stop_waiting is not None and stop_waiting():
- # Shutdown has drained the embedding consumer's active writes.
- # Leave this semantic delivery unacked for recovery rather than
- # waiting forever for embeddings still in the persistent queue.
- raise asyncio.CancelledError("Embedding worker stopped with queued work")
- await asyncio.sleep(0.05)
-
- def build_queue_status(self, telemetry_id: str) -> Dict[str, Dict[str, object]]:
- with self._lock:
- state = self._states.get(telemetry_id) or _RequestWaitState()
- return {
- "Semantic": {
- "processed": state.semantic_processed,
- "requeue_count": state.semantic_requeue_count,
- "error_count": state.semantic_error_count,
- "errors": [{"message": msg} for msg in state.semantic_errors],
- },
- "Embedding": {
- "processed": state.embedding_processed,
- "requeue_count": state.embedding_requeue_count,
- "error_count": state.embedding_error_count,
- "errors": [{"message": msg} for msg in state.embedding_errors],
- },
- }
-
- def cleanup(self, telemetry_id: str) -> None:
- if not telemetry_id:
- return
- with self._lock:
- state = self._states.get(telemetry_id)
- if state is not None and state.retained_roots:
- state.cleanup_requested = True
- else:
- self._states.pop(telemetry_id, None)
-
-
-def get_request_wait_tracker() -> RequestWaitTracker:
- return RequestWaitTracker.get_instance()
-
-
-__all__ = ["RequestWaitTracker", "get_request_wait_tracker"]
diff --git a/openviking/telemetry/resource_summary.py b/openviking/telemetry/resource_summary.py
index 4928d0c4b..ae21f24b0 100644
--- a/openviking/telemetry/resource_summary.py
+++ b/openviking/telemetry/resource_summary.py
@@ -7,7 +7,6 @@ from __future__ import annotations
from typing import Any, Dict
from .operation import OperationTelemetry
-from .request_wait_tracker import get_request_wait_tracker
def _consume_semantic_request_stats(telemetry_id: str):
@@ -52,6 +51,7 @@ def record_resource_queue_metrics(
telemetry: OperationTelemetry,
telemetry_id: str,
root_uri: str | None,
+ queue_timing: Dict[str, Dict[str, float]] | None = None,
) -> None:
"""Apply queue and semantic-tree metrics to a resource operation collector."""
if not telemetry.enabled:
@@ -59,18 +59,20 @@ def record_resource_queue_metrics(
semantic = _queue_metrics(_consume_semantic_request_stats(telemetry_id))
embedding = _queue_metrics(_consume_embedding_request_stats(telemetry_id))
- timing = get_request_wait_tracker().get_queue_timing(telemetry_id)
+ timing = queue_timing or {}
+ semantic_timing = timing.get("semantic", {"queue_wait_ms": 0.0, "execute_ms": 0.0})
+ embedding_timing = timing.get("embedding", {"queue_wait_ms": 0.0, "execute_ms": 0.0})
telemetry.set("queue.semantic.processed", semantic["processed"])
telemetry.set("queue.semantic.requeue_count", semantic["requeue_count"])
telemetry.set("queue.semantic.error_count", semantic["error_count"])
- telemetry.set("queue.semantic.queue_wait.duration_ms", timing["semantic"]["queue_wait_ms"])
- telemetry.set("queue.semantic.execute.duration_ms", timing["semantic"]["execute_ms"])
+ telemetry.set("queue.semantic.queue_wait.duration_ms", semantic_timing["queue_wait_ms"])
+ telemetry.set("queue.semantic.execute.duration_ms", semantic_timing["execute_ms"])
telemetry.set("queue.embedding.processed", embedding["processed"])
telemetry.set("queue.embedding.requeue_count", embedding["requeue_count"])
telemetry.set("queue.embedding.error_count", embedding["error_count"])
- telemetry.set("queue.embedding.queue_wait.duration_ms", timing["embedding"]["queue_wait_ms"])
- telemetry.set("queue.embedding.execute.duration_ms", timing["embedding"]["execute_ms"])
+ telemetry.set("queue.embedding.queue_wait.duration_ms", embedding_timing["queue_wait_ms"])
+ telemetry.set("queue.embedding.execute.duration_ms", embedding_timing["execute_ms"])
tree_stats = _consume_semantic_tree_stats(telemetry_id, root_uri)
if tree_stats is not None:
diff --git a/openviking/utils/embedding_utils.py b/openviking/utils/embedding_utils.py
index cb89b045b..0e062d10d 100644
--- a/openviking/utils/embedding_utils.py
+++ b/openviking/utils/embedding_utils.py
@@ -32,7 +32,6 @@ from openviking.storage.queuefs import get_queue_manager
from openviking.storage.queuefs.embedding_msg_converter import EmbeddingMsgConverter
from openviking.storage.resource_rnfv import NON_PORTABLE_VECTOR_RECORD_FIELDS
from openviking.storage.viking_fs import LS_ALL_NODES, get_viking_fs
-from openviking.telemetry.request_wait_tracker import get_request_wait_tracker
from openviking.utils.embedding_input import truncate_embedding_input
from openviking.utils.image_search import image_bytes_to_model_data_uri
from openviking.utils.ingest_options import IngestOptions
@@ -135,29 +134,12 @@ async def _enqueue_embedding_message(
embedding_queue,
embedding_msg,
*,
- failure_message: str,
+ failure_message: str = "Failed to enqueue embedding message",
) -> bool:
- """Persist one embedding message and settle request tracking on enqueue failure."""
- wait_tracker = get_request_wait_tracker()
- wait_tracker.register_embedding_root(embedding_msg.telemetry_id, embedding_msg.id)
-
- try:
- enqueue_id = await embedding_queue.enqueue(embedding_msg)
- except BaseException as exc:
- wait_tracker.mark_embedding_failed(
- embedding_msg.telemetry_id,
- embedding_msg.id,
- f"{failure_message}: {exc}",
- )
- raise
-
+ """Persist one embedding message."""
+ enqueue_id = await embedding_queue.enqueue(embedding_msg)
if not enqueue_id:
- wait_tracker.mark_embedding_failed(
- embedding_msg.telemetry_id,
- embedding_msg.id,
- failure_message,
- )
- return False
+ raise RuntimeError(failure_message)
return True
@@ -492,7 +474,6 @@ async def vectorize_directory_meta(
enqueued = await _enqueue_embedding_message(
embedding_queue,
msg_abstract,
- failure_message=f"Failed to enqueue directory L0 vector for {uri}",
)
if enqueued:
enqueued_levels.add(int(ContextLevel.ABSTRACT.value))
@@ -554,7 +535,6 @@ async def vectorize_directory_meta(
enqueued = await _enqueue_embedding_message(
embedding_queue,
msg_overview,
- failure_message=f"Failed to enqueue directory L1 vector for {uri}",
)
if enqueued:
enqueued_levels.add(int(ContextLevel.OVERVIEW.value))
@@ -764,7 +744,6 @@ async def vectorize_file(
enqueued = await _enqueue_embedding_message(
embedding_queue,
embedding_msg,
- failure_message=f"Failed to enqueue file vector for {file_path}",
)
if not enqueued:
return False
diff --git a/openviking/utils/resource_processor.py b/openviking/utils/resource_processor.py
index 2b3512d4b..d93d89ebe 100644
--- a/openviking/utils/resource_processor.py
+++ b/openviking/utils/resource_processor.py
@@ -1429,7 +1429,6 @@ class ResourceProcessor:
await _enqueue_embedding_message(
embedding_queue,
message,
- failure_message="Failed to enqueue planned vector deletes",
)
completed_actions += len(delete_ids)
directory_actions: dict[str, dict[int, Any]] = {}
@@ -1480,7 +1479,6 @@ class ResourceProcessor:
if await _enqueue_embedding_message(
embedding_queue,
message,
- failure_message=f"Failed to enqueue scalar update for {action.uri}",
):
completed_actions += 1
for uri, levels in directory_actions.items():
diff --git a/openviking/utils/skill_processor.py b/openviking/utils/skill_processor.py
index d7bd4eb03..2b7b31f2e 100644
--- a/openviking/utils/skill_processor.py
+++ b/openviking/utils/skill_processor.py
@@ -31,7 +31,6 @@ from openviking.service.task_tracker_concurrency import run_to_completion
from openviking.storage.viking_fs import VikingFS
from openviking.storage.vikingdb_manager import VikingDBManager
from openviking.telemetry import get_current_telemetry, register_telemetry
-from openviking.telemetry.request_wait_tracker import get_request_wait_tracker
from openviking.utils.path_safety import safe_join_viking_uri
from openviking.utils.zip_safe import safe_extract_zip
from openviking_cli.exceptions import InvalidArgumentError
@@ -491,9 +490,7 @@ class SkillProcessor:
"content": content,
}
if self._vlm_resolver is None:
- raise RuntimeError(
- "SkillProcessor requires a VLM resolver for account-owned work"
- )
+ raise RuntimeError("SkillProcessor requires a VLM resolver for account-owned work")
extraction_kwargs["vlm"] = await self._vlm_resolver.get_vlm(ctx.account_id)
extraction_result = await extract_skill_privacy_values(**extraction_kwargs)
if not extraction_result.values:
@@ -657,8 +654,6 @@ class SkillProcessor:
telemetry = get_current_telemetry()
register_telemetry(telemetry)
- tracker = get_request_wait_tracker()
- tracker.register_request(telemetry.telemetry_id)
msg = SemanticMsg(
uri=uri,
context_type="skill",
@@ -673,8 +668,6 @@ class SkillProcessor:
propagate_to_parent=False,
source={"path": source_path},
)
- tracker.register_semantic_root(msg.telemetry_id, msg.id)
- tracker.retain_request(msg.telemetry_id, msg.id)
handed_off = False
try:
queue_manager = get_queue_manager()
@@ -683,8 +676,7 @@ class SkillProcessor:
await viking_fs._async_agfs.pathlock_handoff(lease)
handed_off = True
await queue_manager.get_queue(queue_manager.SEMANTIC, allow_create=True).enqueue(msg)
- except BaseException as exc:
- tracker.mark_semantic_failed(msg.telemetry_id, msg.id, str(exc))
+ except BaseException:
if handed_off:
# Enqueue was rejected/failed. Return ownership to the caller's
# finally block; adopting creates a new owned lease reference.
diff --git a/openviking/utils/summarizer.py b/openviking/utils/summarizer.py
index 809b35f23..ddea97a24 100644
--- a/openviking/utils/summarizer.py
+++ b/openviking/utils/summarizer.py
@@ -14,7 +14,6 @@ from openviking.storage.queuefs import SemanticMsg, get_queue_manager
from openviking.storage.queuefs.semantic_msg import build_semantic_coalesce_key
from openviking.storage.viking_fs import LS_ALL_NODES, get_viking_fs
from openviking.telemetry import get_current_telemetry
-from openviking.telemetry.request_wait_tracker import get_request_wait_tracker
from openviking.utils.ingest_options import IngestOptions
from openviking_cli.utils import get_logger
from openviking_cli.utils.uri import VikingURI
@@ -217,25 +216,9 @@ class Summarizer:
if change.ingest_options != IngestOptions()
},
)
- if telemetry_id:
- get_request_wait_tracker().register_semantic_root(telemetry_id, msg.id)
- try:
- enqueue_id = await semantic_queue.enqueue(msg)
- except Exception as exc:
- if telemetry_id:
- get_request_wait_tracker().mark_semantic_failed(
- telemetry_id,
- msg.id,
- str(exc),
- )
- raise
+ enqueue_id = await semantic_queue.enqueue(msg)
if enqueue_id == "deduplicated":
- if telemetry_id:
- get_request_wait_tracker().mark_semantic_done(
- telemetry_id,
- msg.id,
- processed_delta=0,
- )
+ continue
else:
enqueued_count += 1
return {
@@ -350,23 +333,8 @@ class Summarizer:
file_abstracts=file_abstracts,
plan=semantic_plan,
)
- if msg.telemetry_id:
- get_request_wait_tracker().register_semantic_root(msg.telemetry_id, msg.id)
- try:
- enqueue_id = await semantic_queue.enqueue(msg)
- except Exception as e:
- if msg.telemetry_id:
- get_request_wait_tracker().mark_semantic_failed(
- msg.telemetry_id, msg.id, str(e)
- )
- raise
+ enqueue_id = await semantic_queue.enqueue(msg)
if enqueue_id == "deduplicated":
- if msg.telemetry_id:
- get_request_wait_tracker().mark_semantic_done(
- msg.telemetry_id,
- msg.id,
- processed_delta=0,
- )
logger.info("Semantic generation already queued for: %s", target_uri)
continue
enqueued_count += 1
diff --git a/tests/agfs/test_viking_fs_git.py b/tests/agfs/test_viking_fs_git.py
index 412f4d777..94ed4253b 100644
--- a/tests/agfs/test_viking_fs_git.py
+++ b/tests/agfs/test_viking_fs_git.py
@@ -608,6 +608,10 @@ class _SpyExecutor:
self.calls.append(("reindex_file", uri))
return {"ok": True}
+ async def execute_in_current_task(self, *, uri, mode, ctx):
+ self.calls.append(("reindex_file", uri))
+ return {"ok": True}
+
async def reindex_directory_marker(self, *, dir_uri, level, ctx):
self.calls.append(("reindex_marker", dir_uri, int(level)))
@@ -631,6 +635,28 @@ def _patch_reindex_executor(monkeypatch, vfs, spy):
monkeypatch.setattr(reindex_mod, "ReindexExecutor", factory)
+class _MemTaskStore:
+ def __init__(self, *, fail_creates: int = 0):
+ self.fail_creates = fail_creates
+
+ async def create(self, task):
+ if self.fail_creates:
+ self.fail_creates -= 1
+ raise OSError("task store temporarily unavailable")
+
+ async def update(self, task):
+ return None
+
+ async def get(self, task_id, *, account_id=None, user_id=None):
+ return None
+
+ async def list(self, account_id, *, user_id=None):
+ return []
+
+ async def delete(self, task_id, *, account_id, user_id=None):
+ return None
+
+
@pytest.mark.asyncio
async def test_restore_schedules_reindex_for_derived_only_change(vfs, monkeypatch):
"""When a restore only changes a directory `.abstract.md` (source file
@@ -826,32 +852,27 @@ async def test_restore_relations_json_has_no_vector_side_effect(vfs, monkeypatch
@pytest.mark.asyncio
async def test_restore_returns_pollable_task_id(vfs, monkeypatch):
"""An applied restore with vector side-effects returns a ``task_id`` that
- can be polled via the TaskTracker and reaches ``completed``.
+ can be polled without leaving an orphaned admin reindex task.
"""
- spy = _SpyExecutor()
-
- _patch_reindex_executor(monkeypatch, vfs, spy)
-
+ from openviking.service.reindex_executor import ReindexExecutor
from openviking.service.task_tracker import (
TaskTracker,
+ get_task_tracker,
set_task_tracker,
)
- class _MemTaskStore:
- async def create(self, task):
- return None
+ reindexed = []
- async def update(self, task):
- return None
+ async def run_reindex(_executor, **kwargs):
+ reindexed.append(kwargs["uri"])
+ return {
+ "status": "completed",
+ "uri": kwargs["uri"],
+ "object_type": kwargs["object_type"],
+ "mode": kwargs["mode"],
+ }
- async def get(self, task_id, *, account_id=None, user_id=None):
- return None
-
- async def list(self, account_id, *, user_id=None):
- return []
-
- async def delete(self, task_id, *, account_id, user_id=None):
- return None
+ monkeypatch.setattr(ReindexExecutor, "_run", run_reindex)
set_task_tracker(TaskTracker(store=_MemTaskStore()))
try:
@@ -870,8 +891,6 @@ async def test_restore_returns_pollable_task_id(vfs, monkeypatch):
task_id = result.get("task_id")
assert task_id
- from openviking.service.task_tracker import get_task_tracker
-
tracker = get_task_tracker()
task = await tracker.wait(
task_id,
@@ -881,7 +900,81 @@ async def test_restore_returns_pollable_task_id(vfs, monkeypatch):
)
assert task.task_type == "snapshot_restore_reindex"
assert task.status.value == "completed"
- assert ("reindex_file", "viking://resources/proj/x.md") in spy.calls
+ assert reindexed == ["viking://resources/proj/x.md"]
+
+ tasks = await tracker.list_tasks(
+ account_id=ctx.account_id,
+ user_id=ctx.user.user_id,
+ )
+ assert [item.task_type for item in tasks] == ["snapshot_restore_reindex"]
+
+ follow_up = await ReindexExecutor(
+ vlm_resolver=vfs._vlm_resolver,
+ vector_config_resolver=vfs._vector_config_resolver,
+ ).execute(
+ uri="viking://resources/proj/x.md",
+ mode="vectors_only",
+ wait=True,
+ ctx=ctx,
+ )
+ assert follow_up["status"] == "completed"
+ finally:
+ set_task_tracker(None)
+
+
+async def test_restore_task_registration_failure_falls_back_to_file_reindex(
+ vfs,
+ monkeypatch,
+):
+ from openviking.service.reindex_executor import ReindexExecutor
+ from openviking.service.task_tracker import TaskStatus, TaskTracker, set_task_tracker
+
+ ctx = _make_ctx(account="acct_task_fallback")
+ restored_contents = []
+ rebuilt = asyncio.Event()
+
+ async def run_reindex(_executor, **kwargs):
+ restored_contents.append(await vfs.read(kwargs["uri"], ctx=kwargs["ctx"]))
+ rebuilt.set()
+ return {
+ "status": "completed",
+ "uri": kwargs["uri"],
+ "object_type": kwargs["object_type"],
+ "mode": kwargs["mode"],
+ }
+
+ monkeypatch.setattr(ReindexExecutor, "_run", run_reindex)
+ tracker = TaskTracker(store=_MemTaskStore(fail_creates=1))
+ set_task_tracker(tracker)
+ try:
+ uri = "viking://resources/proj/a.md"
+ await vfs.write_file(uri, b"first version", ctx=ctx)
+ first = await vfs.commit(message="v1", ctx=ctx)
+ await vfs.write_file(uri, b"second version", ctx=ctx)
+ await vfs.commit(message="v2", ctx=ctx)
+
+ result = await vfs.restore(
+ project_dir="viking://resources/proj",
+ source_commit=first["commit_oid"],
+ ctx=ctx,
+ )
+
+ assert result["result"] == "applied"
+ assert "task_id" not in result
+ await asyncio.wait_for(rebuilt.wait(), 1)
+ assert restored_contents == [b"first version"]
+
+ tasks = await tracker.list_tasks(account_id=ctx.account_id, user_id=ctx.user.user_id)
+ assert len(tasks) == 1
+ assert tasks[0].task_type == "admin_reindex"
+ task = await tracker.wait(
+ tasks[0].task_id,
+ account_id=ctx.account_id,
+ user_id=ctx.user.user_id,
+ timeout=1,
+ )
+ assert task.status is TaskStatus.COMPLETED
finally:
set_task_tracker(None)
diff --git a/tests/parse/test_feishu_parser_api.py b/tests/parse/test_feishu_parser_api.py
index e6029179b..29edc074f 100644
--- a/tests/parse/test_feishu_parser_api.py
+++ b/tests/parse/test_feishu_parser_api.py
@@ -22,7 +22,6 @@ from openviking.storage.queuefs.add_resource_msg import AddResourceMsg, AddResou
from openviking.storage.queuefs.add_resource_processor import AddResourceProcessor
from openviking.storage.queuefs.queue_manager import QueueManager
from openviking.telemetry import get_current_telemetry, resolve_telemetry
-from openviking.telemetry.request_wait_tracker import get_request_wait_tracker
from openviking.utils.media_processor import UnifiedResourceProcessor
from openviking_cli.exceptions import InvalidArgumentError
from openviking_cli.session.user_id import UserIdentifier
@@ -190,7 +189,9 @@ async def test_feishu_parser_api_bypasses_accessor():
async def test_feishu_parser_api_uses_app_credentials_for_tenant_token(monkeypatch):
oauth_client = SimpleNamespace(get_tenant_access_token=AsyncMock(return_value="t-test"))
from_config = Mock(return_value=oauth_client)
- monkeypatch.setattr("openviking.resource.feishu_watch_auth.FeishuOAuthClient.from_config", from_config)
+ monkeypatch.setattr(
+ "openviking.resource.feishu_watch_auth.FeishuOAuthClient.from_config", from_config
+ )
api = _understanding_api_for_parse()
feishu_config = FeishuConfig(app_id="app", app_secret="secret")
@@ -1083,7 +1084,25 @@ async def test_add_resource_processor_persists_final_uri_and_cleans_staged_sourc
complete=AsyncMock(),
fail=AsyncMock(),
get_task_auth=AsyncMock(return_value={}),
- wait_for_descendants=AsyncMock(),
+ wait_for_task_work=AsyncMock(),
+ build_queue_status=Mock(
+ return_value={
+ "Semantic": {
+ "processed": 0,
+ "requeue_count": 0,
+ "error_count": 0,
+ "errors": [],
+ },
+ "Embedding": {
+ "processed": 9,
+ "requeue_count": 0,
+ "error_count": 0,
+ "errors": [],
+ },
+ }
+ ),
+ get_context_count=Mock(return_value=9),
+ get_queue_timing=Mock(return_value={}),
)
service = SimpleNamespace(
@@ -1131,12 +1150,6 @@ async def test_add_resource_processor_persists_final_uri_and_cleans_staged_sourc
},
)
- wait_tracker = get_request_wait_tracker()
- wait_tracker.register_request(telemetry_id)
- for index in range(9):
- embedding_id = f"embedding-{index}"
- wait_tracker.register_embedding_root(telemetry_id, embedding_id)
- wait_tracker.mark_embedding_done(telemetry_id, embedding_id, vector_written=True)
data = msg.to_dict()
data[TASK_WORK_ID_FIELD] = "work-1"
await processor._process(msg, data)
@@ -1202,9 +1215,11 @@ async def test_add_resource_processor_persists_final_uri_and_cleans_staged_sourc
@pytest.mark.asyncio
@pytest.mark.parametrize("telemetry_id", [None, "telemetry-unregistered-resource"])
[email protected]("context_count", [0, 1])
async def test_add_resource_processor_collects_stats_without_registered_telemetry(
monkeypatch,
telemetry_id: str | None,
+ context_count: int,
):
final_uri = "viking://resources/unregistered"
observed_telemetry_ids = []
@@ -1215,7 +1230,10 @@ async def test_add_resource_processor_collects_stats_without_registered_telemetr
complete=AsyncMock(),
fail=AsyncMock(),
get_task_auth=AsyncMock(return_value={}),
- wait_for_descendants=AsyncMock(),
+ wait_for_task_work=AsyncMock(),
+ build_queue_status=Mock(return_value={}),
+ get_context_count=Mock(return_value=context_count),
+ get_queue_timing=Mock(return_value={}),
)
monkeypatch.setattr(
"openviking.storage.queuefs.add_resource_processor.get_task_tracker",
@@ -1233,13 +1251,6 @@ async def test_add_resource_processor_collects_stats_without_registered_telemetr
telemetry.record_token_usage("embedding", 21)
telemetry.add_token_usage(13, 5)
- wait_tracker = get_request_wait_tracker()
- wait_tracker.register_embedding_root(effective_telemetry_id, "embedding-1")
- wait_tracker.mark_embedding_done(
- effective_telemetry_id,
- "embedding-1",
- vector_written=True,
- )
return {"status": "success", "root_uri": final_uri}
processor = AddResourceProcessor(
@@ -1266,7 +1277,8 @@ async def test_add_resource_processor_collects_stats_without_registered_telemetr
task_tracker.fail.assert_not_awaited()
final_result = task_tracker.complete.await_args_list[-1].args[1]
- assert final_result["context_count"] == 1
+ assert final_result["root_uri"] == final_uri
+ assert final_result["context_count"] == context_count
assert final_result["usage"]["tokens"]["embedding"]["total"] == 21
assert final_result["usage"]["tokens"]["llm"] == {
"input": 13,
@@ -1298,7 +1310,9 @@ async def test_add_resource_processor_replay_skips_lock_adopt_when_result_exists
update_stage=AsyncMock(),
complete=AsyncMock(),
fail=AsyncMock(),
- wait_for_descendants=AsyncMock(),
+ wait_for_task_work=AsyncMock(),
+ build_queue_status=Mock(return_value={}),
+ get_queue_timing=Mock(return_value={}),
)
monkeypatch.setattr(
"openviking.storage.queuefs.add_resource_processor.get_task_tracker",
@@ -1331,59 +1345,12 @@ async def test_add_resource_processor_replay_skips_lock_adopt_when_result_exists
async_agfs.pathlock_adopt.assert_not_awaited()
service.execute_add_resource_job.assert_not_awaited()
task_tracker.start.assert_not_awaited()
- task_tracker.wait_for_descendants.assert_awaited_once_with("task-1", "work-1")
task_tracker.complete.assert_awaited_once()
completed_result = task_tracker.complete.await_args.args[1]
assert completed_result["root_uri"] == final_uri
assert completed_result["context_count"] == 4
-async def test_add_resource_processor_reports_zero_vectors(monkeypatch):
- final_uri = "viking://resources/no-vectors"
- service = SimpleNamespace(
- execute_add_resource_job=AsyncMock(
- return_value={"status": "success", "root_uri": final_uri}
- ),
- _link_resource_reason_memory=AsyncMock(),
- )
- task_tracker = SimpleNamespace(
- create=AsyncMock(return_value=SimpleNamespace(status=TaskStatus.PENDING)),
- start=AsyncMock(),
- update_stage=AsyncMock(),
- complete=AsyncMock(),
- fail=AsyncMock(),
- get_task_auth=AsyncMock(return_value={}),
- wait_for_descendants=AsyncMock(),
- )
- monkeypatch.setattr(
- "openviking.storage.queuefs.add_resource_processor.get_task_tracker",
- Mock(return_value=task_tracker),
- )
- processor = AddResourceProcessor(
- service,
- QueueManager.ADD_RESOURCE,
- SimpleNamespace(_async_agfs=SimpleNamespace(pathlock_release=AsyncMock())),
- )
- msg = AddResourceMsg(
- task_id="task-1",
- path="/tmp/demo.md",
- root_uri=final_uri,
- account_id="account-1",
- user_id="user-1",
- role="user",
- )
- data = msg.to_dict()
- data[TASK_WORK_ID_FIELD] = "work-1"
-
- await processor._process(msg, data)
-
- task_tracker.fail.assert_not_awaited()
- final_result = task_tracker.complete.await_args.args[1]
- assert final_result["root_uri"] == final_uri
- assert final_result["context_count"] == 0
-
-
def test_feishu_direct_submission_requires_configured_auth(monkeypatch):
api = _understanding_api_for_parse()
source = "https://example.larkoffice.com/docx/doxcnToken"
diff --git a/tests/server/conftest.py b/tests/server/conftest.py
index 18a22268d..2871e184a 100644
--- a/tests/server/conftest.py
+++ b/tests/server/conftest.py
@@ -96,7 +96,9 @@ def _install_session_commit_queue_fallback(service: OpenVikingService, monkeypat
monkeypatch.setattr(
tracker,
"has_work",
- lambda task_id: task_id in queued_commit_tasks or original_has_work(task_id),
+ lambda task_id, **kwargs: (
+ task_id in queued_commit_tasks or original_has_work(task_id, **kwargs)
+ ),
)
async def enqueue_with_session_commit_fallback(queue_name, data):
diff --git a/tests/server/test_admin_rebuild_api.py b/tests/server/test_admin_rebuild_api.py
index 57bece0d1..852c1fee9 100644
--- a/tests/server/test_admin_rebuild_api.py
+++ b/tests/server/test_admin_rebuild_api.py
@@ -2572,51 +2572,10 @@ async def test_reindex_global_namespace_partitions_user_and_resources(monkeypatc
@pytest.mark.asyncio
-async def test_reindex_upsert_ignores_empty_replace_tags(monkeypatch):
- from types import SimpleNamespace
-
- from openviking.service.reindex_executor import ReindexExecutor
- from openviking.utils.ingest_options import IngestOptions
-
- queued = []
-
- class FakeVikingDB:
- async def enqueue_embedding_msg(self, msg):
- queued.append(msg)
- return True
-
- monkeypatch.setattr(
- "openviking.service.reindex_executor.get_service",
- lambda: SimpleNamespace(vikingdb_manager=FakeVikingDB()),
- )
- monkeypatch.setattr(
- "openviking.service.reindex_executor.get_request_wait_tracker",
- lambda: SimpleNamespace(register_embedding_root=lambda *args: None),
- )
-
- ctx = RequestContext(
- user=UserIdentifier(account_id="test", user_id="alice"),
- role=Role.ROOT,
- )
- await ReindexExecutor()._upsert_context(
- uri="viking://resources/demo.md",
- parent_uri="viking://resources",
- abstract="demo",
- vector_text="demo",
- is_leaf=True,
- context_type="resource",
- level=ContextLevel.DETAIL,
- ctx=ctx,
- ingest_options=IngestOptions.from_search_tags([], mode="replace"),
- )
-
- assert "search_tags" not in queued[0].context_data
- assert "_upsert_options" not in queued[0].context_data
-
-
-async def test_reindex_upsert_applies_clear_tags_to_embedding_message(monkeypatch):
[email protected]("tag_mode", ["replace", "clear"])
+async def test_reindex_upsert_empty_tags_preserves_replace_and_clear_semantics(
+ monkeypatch, tag_mode
+):
from types import SimpleNamespace
from openviking.service.reindex_executor import ReindexExecutor
@@ -2633,10 +2592,6 @@ async def test_reindex_upsert_applies_clear_tags_to_embedding_message(monkeypatc
"openviking.service.reindex_executor.get_service",
lambda: SimpleNamespace(vikingdb_manager=FakeVikingDB()),
)
- monkeypatch.setattr(
- "openviking.service.reindex_executor.get_request_wait_tracker",
- lambda: SimpleNamespace(register_embedding_root=lambda *args: None),
- )
ctx = RequestContext(
user=UserIdentifier(account_id="test", user_id="alice"),
@@ -2651,11 +2606,17 @@ async def test_reindex_upsert_applies_clear_tags_to_embedding_message(monkeypatc
context_type="resource",
level=ContextLevel.DETAIL,
ctx=ctx,
- ingest_options=IngestOptions.from_search_tags(None, mode="clear"),
+ ingest_options=IngestOptions.from_search_tags(
+ [] if tag_mode == "replace" else None, mode=tag_mode
+ ),
)
- assert queued[0].context_data["search_tags"] == []
- assert queued[0].context_data["_upsert_options"] == {"search_tag_mode": "replace"}
+ if tag_mode == "replace":
+ assert "search_tags" not in queued[0].context_data
+ assert "_upsert_options" not in queued[0].context_data
+ else:
+ assert queued[0].context_data["search_tags"] == []
+ assert queued[0].context_data["_upsert_options"] == {"search_tag_mode": "replace"}
@pytest.mark.asyncio
diff --git a/tests/server/test_api_snapshot.py b/tests/server/test_api_snapshot.py
index 8468d6c7b..54645d737 100644
--- a/tests/server/test_api_snapshot.py
+++ b/tests/server/test_api_snapshot.py
@@ -277,6 +277,10 @@ async def test_restore_acl_and_reindex_identity(client_with_resource, service, m
reindex_contexts.append(ctx)
return {"ok": True}
+ async def execute_in_current_task(self, *, uri, mode, ctx):
+ reindex_contexts.append(ctx)
+ return {"ok": True}
+
async def reindex_directory_marker(self, *, dir_uri, level, ctx):
reindex_contexts.append(ctx)
@@ -649,6 +653,10 @@ async def test_restore_apply_triggers_reindex_hook(client_with_resource_and_blob
calls.append(uri)
return {"ok": True}
+ async def execute_in_current_task(self, *, uri, mode, ctx):
+ calls.append(uri)
+ return {"ok": True}
+
def factory(*, vlm_resolver, vector_config_resolver):
assert vlm_resolver is service.viking_fs._vlm_resolver
assert vector_config_resolver is service.vector_config_resolver
@@ -696,6 +704,9 @@ async def test_restore_delete_removes_orphaned_vectors(client_with_resource_and_
async def execute(self, *, uri, mode, wait, ctx):
return {"ok": True}
+ async def execute_in_current_task(self, *, uri, mode, ctx):
+ return {"ok": True}
+
async def reindex_directory_marker(self, *, dir_uri, level, ctx):
return None
diff --git a/tests/server/test_content_batch_write.py b/tests/server/test_content_batch_write.py
index d365f40ca..8f71a95f0 100644
--- a/tests/server/test_content_batch_write.py
+++ b/tests/server/test_content_batch_write.py
@@ -7,6 +7,9 @@ import pytest
import openviking.storage.content_write as content_write_module
from openviking.server.identity import RequestContext, Role
+from openviking.service.task_store import PersistentTaskStore
+from openviking.service.task_tracker import TaskTracker
+from openviking.service.task_work_index import TaskWorkIndex, bind_task_context
from openviking.session.memory.dataclass import MemoryFile
from openviking.session.memory.utils import MemoryFileUtils
from openviking.storage.content_write import ContentWriteCoordinator
@@ -15,6 +18,7 @@ from openviking.storage.context_update_plan import (
ContextUpdatePlan,
FileRefreshIntent,
)
+from openviking.storage.queuefs.queue_middleware import EnqueueReceipt
from openviking.storage.queuefs.semantic_ops.freshness_policy import FreshnessAction
from openviking.telemetry.context import bind_telemetry
from openviking.telemetry.operation import OperationTelemetry
@@ -25,6 +29,7 @@ from openviking_cli.exceptions import (
OpenVikingError,
)
from openviking_cli.session.user_id import UserIdentifier
+from tests.test_task_tracker import _FakeAgfs
class _PathLockClient:
@@ -119,19 +124,22 @@ def _coordinator_queue_for_unit_tests(monkeypatch, request):
yield None
return
- queue = SimpleNamespace(enqueue=AsyncMock(return_value="queued"))
+ queue = SimpleNamespace(enqueue=AsyncMock(return_value=EnqueueReceipt("queued", "work-1")))
queue_manager = SimpleNamespace(
SEMANTIC="semantic",
get_queue=lambda *args, **kwargs: queue,
)
monkeypatch.setattr(content_write_module, "get_queue_manager", lambda: queue_manager)
- yield queue
+ tracker = TaskTracker(PersistentTaskStore(_FakeAgfs()))
+ tracker.attach_work_index(TaskWorkIndex())
+ monkeypatch.setattr("openviking.service.task_tracker._instance", tracker)
+
+ async def run_tracked_operation(_task_type, *, operation, **_kwargs):
+ with bind_task_context("test-task", "default", "default", enqueued_work=[]):
+ return await operation(), "test-task"
- tracker = content_write_module.get_request_wait_tracker()
- for call in queue.enqueue.await_args_list:
- msg = call.args[0]
- if msg.wait_for_request_completion:
- tracker.mark_semantic_done(msg.telemetry_id, msg.id, processed_delta=0)
+ monkeypatch.setattr(content_write_module, "run_tracked_operation", run_tracked_operation)
+ yield queue
@pytest.mark.asyncio
@@ -677,7 +685,7 @@ async def test_batch_hands_off_file_locks_after_one_aggregated_refresh(monkeypat
locks = vfs._async_agfs
coordinator = ContentWriteCoordinator(vfs)
calls = []
- queue = SimpleNamespace(enqueue=AsyncMock(return_value="queued"))
+ queue = SimpleNamespace(enqueue=AsyncMock(return_value=EnqueueReceipt("queued", "work-1")))
queue_manager = SimpleNamespace(
SEMANTIC="semantic",
get_queue=lambda *args, **kwargs: queue,
@@ -709,11 +717,6 @@ async def test_batch_hands_off_file_locks_after_one_aggregated_refresh(monkeypat
assert locks.handoffs == 1
coordinator_msg = queue.enqueue.await_args.args[0]
assert coordinator_msg.wait_for_request_completion is True
- content_write_module.get_request_wait_tracker().mark_semantic_done(
- telemetry.telemetry_id,
- coordinator_msg.id,
- processed_delta=0,
- )
@pytest.mark.asyncio
diff --git a/tests/server/test_content_write_service.py b/tests/server/test_content_write_service.py
index 76672ac86..4f11591a5 100644
--- a/tests/server/test_content_write_service.py
+++ b/tests/server/test_content_write_service.py
@@ -8,6 +8,7 @@ from types import SimpleNamespace
import pytest
from openviking.server.identity import RequestContext, Role
+from openviking.service.task_work_index import bind_task_context
from openviking.session.memory.dataclass import MemoryFile
from openviking.session.memory.utils import MemoryFileUtils
from openviking.session.memory.utils.content_visibility import visible_content
@@ -22,6 +23,21 @@ from openviking_cli.exceptions import (
from openviking_cli.session.user_id import UserIdentifier
[email protected](autouse=True)
+def run_direct_coordinator_operations_inline(request, monkeypatch):
+ if "service" in request.fixturenames:
+ return
+
+ async def run_tracked_operation(_task_type, *, operation, **_kwargs):
+ with bind_task_context("test-task", "default", "test-user", enqueued_work=[]):
+ return await operation(), "test-task"
+
+ monkeypatch.setattr(
+ "openviking.storage.content_write.run_tracked_operation",
+ run_tracked_operation,
+ )
+
+
@pytest.mark.asyncio
async def test_write_updates_memory_file_and_parent_overview(service):
ctx = RequestContext(user=service.user, role=Role.USER)
@@ -982,7 +998,11 @@ async def test_create_mode_valid_extensions_pass(monkeypatch):
del uri, mode, ctx, lock_handle, lease_ref, is_new_file
return content
+ async def _fake_wait_for_queues(**_kwargs):
+ return None
+
monkeypatch.setattr(coordinator, "_write_in_place", _fake_write_in_place)
+ monkeypatch.setattr(coordinator, "_wait_for_request", _fake_wait_for_queues)
result = await coordinator.write(
uri=file_uri, content="content", mode="create", ctx=ctx, wait=True
diff --git a/tests/server/test_filesystem_router.py b/tests/server/test_filesystem_router.py
index 9e9f8d461..1e7ae0940 100644
--- a/tests/server/test_filesystem_router.py
+++ b/tests/server/test_filesystem_router.py
@@ -22,7 +22,10 @@ async def test_rm_preserves_memory_cleanup(monkeypatch):
cleanup = {"status": "success", "memory_uris": ["viking://user/alice/memories/entities/a.md"]}
async def fake_rm(uri, ctx=None, recursive=False, wait=False):
- return {"estimated_deleted_count": 1, "memory_cleanup": cleanup}
+ return {
+ "estimated_deleted_count": 1,
+ "memory_cleanup": cleanup,
+ }
monkeypatch.setattr(
filesystem,
diff --git a/tests/server/test_request_wait_tracking.py b/tests/server/test_request_wait_tracking.py
deleted file mode 100644
index 0a87444a2..000000000
--- a/tests/server/test_request_wait_tracking.py
+++ /dev/null
@@ -1,58 +0,0 @@
-# Copyright (c) 2026 Beijing Volcano Engine Technology Co., Ltd.
-# SPDX-License-Identifier: AGPL-3.0
-
-"""Request-scoped waits must not depend on unrelated queue work."""
-
-import asyncio
-from types import SimpleNamespace
-
-from openviking.server.identity import RequestContext, Role
-from openviking.service.resource_service import ResourceService
-from openviking.service.task_store import PersistentTaskStore
-from openviking.service.task_tracker import TaskTracker
-from openviking.telemetry.context import bind_telemetry
-from openviking.telemetry.operation import OperationTelemetry
-from openviking.telemetry.request_wait_tracker import RequestWaitTracker
-from openviking_cli.session.user_id import UserIdentifier
-from tests.test_task_tracker import _FakeAgfs
-
-
-async def test_add_skill_wait_uses_request_tracker_when_telemetry_disabled(monkeypatch):
- tracker = RequestWaitTracker()
- telemetry = OperationTelemetry(operation="resources.add_skill", enabled=False)
- tracker.register_request("unrelated")
- tracker.register_embedding_root("unrelated", "pending-embedding")
- ctx = RequestContext(user=UserIdentifier.the_default_user(), role=Role.ROOT)
- root_uri = "viking://user/default/skills/demo"
-
- async def process_skill(**kwargs):
- tracker.register_embedding_root(telemetry.telemetry_id, "skill-embedding")
- tracker.mark_embedding_done(telemetry.telemetry_id, "skill-embedding")
- return {"status": "success", "uri": root_uri, "name": "demo"}
-
- service = ResourceService(
- resource_processor=object(),
- skill_processor=SimpleNamespace(process_skill=process_skill),
- viking_fs=object(),
- )
- monkeypatch.setattr(
- "openviking.service.resource_service.get_request_wait_tracker", lambda: tracker
- )
- task_tracker = TaskTracker(store=PersistentTaskStore(_FakeAgfs()))
- monkeypatch.setattr("openviking.service.task_tracker.get_task_tracker", lambda: task_tracker)
-
- with bind_telemetry(telemetry):
- result = await asyncio.wait_for(
- service.add_skill(
- data={"name": "demo", "content": "# Demo"},
- ctx=ctx,
- wait=True,
- target_uri="viking://user/default/skills",
- ),
- 1,
- )
-
- assert result["root_uri"] == root_uri
- assert result["queue_status"]["Embedding"]["processed"] == 1
- assert not tracker.has_request(telemetry.telemetry_id)
- assert not tracker.is_complete("unrelated")
diff --git a/tests/service/test_fs_service.py b/tests/service/test_fs_service.py
index 878246fc6..733682ff1 100644
--- a/tests/service/test_fs_service.py
+++ b/tests/service/test_fs_service.py
@@ -12,6 +12,7 @@ import pytest
from openviking.pyagfs.exceptions import AGFSAlreadyExistsError
from openviking.server.identity import RequestContext, Role
from openviking.service.fs_service import FSService, ListingPage
+from openviking.service.task_work_index import bind_task_context
from openviking.storage.abstract_overview import body_for_preview
from openviking.storage.errors import LockAcquisitionError
from openviking.storage.viking_fs import VikingFS
@@ -19,6 +20,18 @@ from openviking_cli.exceptions import InvalidArgumentError
from openviking_cli.session.user_id import UserIdentifier
[email protected](autouse=True)
+def run_fs_operations_inline(monkeypatch):
+ async def run_tracked_operation(_task_type, *, operation, **_kwargs):
+ with bind_task_context("test-task", "default", "test-user", enqueued_work=[]):
+ return await operation(), "test-task"
+
+ monkeypatch.setattr(
+ "openviking.service.fs_service.run_tracked_operation",
+ run_tracked_operation,
+ )
+
+
class _FakeVikingFS:
def __init__(self, *, rm_error=None, events=None, parent_exists=True):
self.rm_calls = []
@@ -248,41 +261,6 @@ class _FakeWatchScheduler:
self.watch_manager = watch_manager
-class _FakeWaitTracker:
- def __init__(self):
- self.registered_requests = []
- self.registered_roots = []
- self.wait_calls = []
- self.cleaned = []
-
- def register_request(self, telemetry_id):
- self.registered_requests.append(telemetry_id)
-
- def register_semantic_root(self, telemetry_id, root_id):
- self.registered_roots.append(
- {
- "telemetry_id": telemetry_id,
- "root_id": root_id,
- "request_was_registered": telemetry_id in self.registered_requests,
- }
- )
-
- async def wait_for_request(self, telemetry_id):
- self.wait_calls.append(telemetry_id)
-
- def build_queue_status(self, telemetry_id):
- return {
- "Semantic": {"processed": 1, "error_count": 0, "errors": []},
- "Embedding": {"processed": 0, "error_count": 0, "errors": []},
- }
-
- def mark_semantic_failed(self, telemetry_id, root_id, message):
- pass
-
- def cleanup(self, telemetry_id):
- self.cleaned.append(telemetry_id)
-
-
class _FakeQueueManager:
SEMANTIC = "semantic"
@@ -913,6 +891,34 @@ async def test_tree_tag_filter_keeps_zero_node_limit_unbounded(request_context):
assert viking_fs.tree.await_args.kwargs["node_limit"] is None
+async def test_resource_rm_enqueues_parent_delete_refresh_and_waits(request_context, monkeypatch):
+ viking_fs = _FakeVikingFS()
+ service = FSService(viking_fs=viking_fs)
+ queue_manager = _FakeQueueManager()
+ monkeypatch.setattr("openviking.service.fs_service.get_queue_manager", lambda: queue_manager)
+ service._wait_for_refresh = AsyncMock(return_value={"Semantic": {"pending_count": 0}})
+
+ uri = "viking://resources/images/2026/06/10/不二周助_jpeg"
+ result = await service.rm(
+ uri,
+ ctx=request_context,
+ recursive=True,
+ wait=True,
+ )
+
+ assert viking_fs.rm_calls == [{"uri": uri, "recursive": True, "ctx": request_context}]
+ assert len(queue_manager.messages) == 1
+ msg = queue_manager.messages[0]
+ assert msg.uri == "viking://resources/images/2026/06/10"
+ assert msg.context_type == "resource"
+ assert msg.recursive is False
+ assert msg.changes == {"deleted": [uri]}
+ assert result["semantic_root_uri"] == "viking://resources/images/2026/06/10"
+ assert result["semantic_status"] == "complete"
+ assert result["queue_status"] == {"Semantic": {"pending_count": 0}}
+
+
@pytest.mark.asyncio
async def test_resource_rm_reports_failed_semantic_status_when_wait_queue_has_errors(
request_context, monkeypatch
@@ -920,15 +926,15 @@ async def test_resource_rm_reports_failed_semantic_status_when_wait_queue_has_er
viking_fs = _FakeVikingFS()
service = FSService(viking_fs=viking_fs)
service._enqueue_delete_refresh = AsyncMock()
- tracker = _FakeWaitTracker()
- tracker.build_queue_status = lambda _id: {
- "Semantic": {
- "processed": 1,
- "error_count": 1,
- "errors": [{"message": "refresh failed"}],
+ service._wait_for_refresh = AsyncMock(
+ return_value={
+ "Semantic": {
+ "processed": 1,
+ "error_count": 1,
+ "errors": [{"message": "refresh failed"}],
+ }
}
- }
- monkeypatch.setattr("openviking.service.fs_service.get_request_wait_tracker", lambda: tracker)
+ )
result = await service.rm(
"viking://resources/images/2026/06/10/不二周助_jpeg",
@@ -1437,47 +1443,6 @@ async def test_transfer_refresh_message_records_deleted_source_entry(
assert queue_manager.messages[0].changes == {"deleted": ["viking://resources/source/moved.md"]}
-async def test_resource_rm_wait_registers_request_before_semantic_root(
- request_context,
- monkeypatch,
-):
- viking_fs = _FakeVikingFS()
- service = FSService(viking_fs=viking_fs)
- tracker = _FakeWaitTracker()
- queue_manager = _FakeQueueManager()
-
- monkeypatch.setattr(
- "openviking.service.fs_service.get_current_telemetry",
- lambda: SimpleNamespace(telemetry_id="tm-fs-rm"),
- )
- monkeypatch.setattr(
- "openviking.service.fs_service.get_request_wait_tracker",
- lambda: tracker,
- )
- monkeypatch.setattr(
- "openviking.service.fs_service.get_queue_manager",
- lambda: queue_manager,
- )
-
- result = await service.rm(
- "viking://resources/images/2026/06/10/不二周助_jpeg",
- ctx=request_context,
- recursive=True,
- wait=True,
- )
-
- assert tracker.registered_requests == ["tm-fs-rm"]
- assert tracker.registered_roots
- assert tracker.registered_roots[0]["request_was_registered"] is True
- assert queue_manager.messages[0].recursive is False
- assert tracker.wait_calls == ["tm-fs-rm"]
- assert tracker.cleaned == ["tm-fs-rm"]
- assert result["semantic_root_uri"] == "viking://resources/images/2026/06/10"
- assert result["semantic_status"] == "complete"
- assert result["queue_status"]["Semantic"]["processed"] == 1
-
-
@pytest.mark.asyncio
async def test_resource_rm_does_not_cleanup_memory_if_resource_delete_fails(request_context):
delete_error = RuntimeError("delete failed")
diff --git a/tests/service/test_resource_service_watch.py b/tests/service/test_resource_service_watch.py
index 74fbd90c0..f35a9813e 100644
--- a/tests/service/test_resource_service_watch.py
+++ b/tests/service/test_resource_service_watch.py
@@ -390,9 +390,7 @@ class TestWatchTaskCreation:
)
def test_watch_persists_clear_without_tags(self, resource_service: ResourceService):
- assert resource_service._watch_processor_kwargs({}, None, "clear") == {
- "tag_mode": "clear"
- }
+ assert resource_service._watch_processor_kwargs({}, None, "clear") == {"tag_mode": "clear"}
def test_add_resource_message_round_trip_preserves_clear_without_tags(self):
message = AddResourceMsg(
@@ -679,9 +677,7 @@ class TestAddResourceArgs:
return resolver(
SimpleNamespace(
account=SimpleNamespace(feishu=None),
- cluster=SimpleNamespace(
- feishu=FeishuConfig(domain="https://open.feishu.cn")
- ),
+ cluster=SimpleNamespace(feishu=FeishuConfig(domain="https://open.feishu.cn")),
)
)
@@ -699,9 +695,7 @@ class TestAddResourceArgs:
seen["config_domain"] = feishu_config.domain
return SimpleNamespace(source_name=None, source_format="file")
- resource_service._runtime_config_manager = SimpleNamespace(
- resolve_account=resolve_account
- )
+ resource_service._runtime_config_manager = SimpleNamespace(resolve_account=resolve_account)
monkeypatch.setattr(
"openviking.parse.accessors.feishu_accessor.FeishuAccessor.preflight_source",
preflight,
@@ -755,9 +749,7 @@ class TestAddResourceArgs:
)
)
- resource_service._runtime_config_manager = SimpleNamespace(
- resolve_account=resolve_account
- )
+ resource_service._runtime_config_manager = SimpleNamespace(resolve_account=resolve_account)
disable_task_tracker(monkeypatch)
to_uri = "viking://resources/feishu-account-watch"
resource_service._plan_source_job_target = AsyncMock(
@@ -1033,7 +1025,11 @@ async def test_add_resource_processor_records_paused_watch_result(
complete=AsyncMock(),
fail=AsyncMock(),
get_task_auth=AsyncMock(return_value={}),
- wait_for_descendants=AsyncMock(),
+ wait_for_task_work=AsyncMock(),
+ build_queue_status=Mock(return_value={}),
+ get_context_count=Mock(return_value=0),
+ get_queue_timing=Mock(return_value={}),
+ get=AsyncMock(return_value=SimpleNamespace(status=TaskStatus.CANCELLING)),
)
monkeypatch.setattr(
"openviking.storage.queuefs.add_resource_processor.get_task_tracker",
diff --git a/tests/service/test_skill_processing_cancellation.py b/tests/service/test_skill_processing_cancellation.py
index abcb3d430..eba7e505f 100644
--- a/tests/service/test_skill_processing_cancellation.py
+++ b/tests/service/test_skill_processing_cancellation.py
@@ -11,8 +11,11 @@ from openviking.server.identity import RequestContext, Role
from openviking.service.resource_service import ResourceService
from openviking.service.task_store import PersistentTaskStore
from openviking.service.task_tracker import TaskStatus, TaskTracker
-from openviking.service.task_work_index import QueueTaskMetadata, TaskWorkIndex
-from openviking.telemetry import OperationTelemetry, bind_telemetry
+from openviking.service.task_work_index import QueueTaskMetadata, TaskWorkIndex, get_task_context
+from openviking.telemetry import (
+ OperationTelemetry,
+ bind_telemetry,
+)
from openviking_cli.session.user_id import UserIdentifier
from tests.test_task_tracker import _FakeAgfs
@@ -100,3 +103,59 @@ async def test_cancel_during_skill_task_creation_settles_persisted_record(monkey
finally:
resume.set()
await asyncio.gather(adding, return_exceptions=True)
+
+
+async def test_skill_success_remains_completed_when_queue_work_finishes_first(monkeypatch):
+ tracker = TaskTracker(store=PersistentTaskStore(_FakeAgfs()))
+ index = TaskWorkIndex()
+ tracker.attach_work_index(index)
+ monkeypatch.setattr("openviking.service.task_tracker.get_task_tracker", lambda: tracker)
+ ctx = RequestContext(user=UserIdentifier.the_default_user(), role=Role.ROOT)
+ owner = {"account_id": ctx.account_id, "user_id": ctx.user.user_id}
+
+ async def process_skill(**kwargs):
+ work = QueueTaskMetadata(get_task_context().task_id, "vector", **owner)
+ assert index.register("Embedding", work)
+ index.record_result(work, "completed")
+ await index.prepare_ack("Embedding", work)
+ return {"uri": "viking://agent/skills/demo"}
+
+ service = ResourceService.__new__(ResourceService)
+ service._resource_processor = object()
+ service._viking_fs = object()
+ service._skill_processor = SimpleNamespace(process_skill=process_skill)
+ service._background_tasks = set()
+
+ with bind_telemetry(OperationTelemetry("add-skill")):
+ result = await service.add_skill("body", ctx, wait=True, target_uri="viking://agent/skills")
+
+ task = await tracker.get(result["task_id"], **owner)
+ assert task.status is TaskStatus.COMPLETED
+ assert task.error is None
+ assert task.result["root_uri"] == "viking://agent/skills/demo"
+
+
+async def test_skill_failure_preserves_business_error_without_queue_work(monkeypatch):
+ tracker = TaskTracker(store=PersistentTaskStore(_FakeAgfs()))
+ monkeypatch.setattr("openviking.service.task_tracker.get_task_tracker", lambda: tracker)
+ ctx = RequestContext(user=UserIdentifier.the_default_user(), role=Role.ROOT)
+ owner = {"account_id": ctx.account_id, "user_id": ctx.user.user_id}
+
+ async def process_skill(**kwargs):
+ raise RuntimeError("skill processing failed")
+
+ service = ResourceService.__new__(ResourceService)
+ service._resource_processor = object()
+ service._viking_fs = object()
+ service._skill_processor = SimpleNamespace(process_skill=process_skill)
+
+ with bind_telemetry(OperationTelemetry("add-skill")):
+ with pytest.raises(RuntimeError, match="skill processing failed"):
+ await service.add_skill("body", ctx, wait=True, target_uri="viking://agent/skills")
+
+ tasks = await tracker.list_tasks(**owner)
+ assert len(tasks) == 1
+ assert tasks[0].status is TaskStatus.FAILED
+ assert tasks[0].error == "skill processing failed"
diff --git a/tests/service/test_task_processing_time.py b/tests/service/test_task_processing_time.py
index d3a161194..b7d138884 100644
--- a/tests/service/test_task_processing_time.py
+++ b/tests/service/test_task_processing_time.py
@@ -16,7 +16,6 @@ from openviking.storage.queuefs.semantic_executor import (
SemanticTreeExecutor,
SemanticTreeScheduler,
)
-from openviking.telemetry.request_wait_tracker import RequestWaitTracker
def test_work_timestamps_union_worker_occupancy_and_exclude_queue_time(monkeypatch):
@@ -121,8 +120,7 @@ async def test_saturated_dag_scheduler_counts_claimed_handler_time(monkeypatch):
@pytest.mark.asyncio
[email protected]("method", ["wait_for_request", "wait_for_embeddings"])
-async def test_downstream_wait_remains_worker_occupied_time(monkeypatch, method):
+async def test_downstream_work_wait_remains_worker_occupied_time(monkeypatch):
now = [0.0]
monkeypatch.setattr(
"openviking.service.task_work_index.time",
@@ -130,26 +128,25 @@ async def test_downstream_wait_remains_worker_occupied_time(monkeypatch, method)
)
index = TaskWorkIndex()
index.init_processing("task")
- tracker = RequestWaitTracker()
- request = "processing-time-" + method
- tracker.register_request(request)
- tracker.register_embedding_root(request, "embedding")
+ metadata = QueueTaskMetadata("task", "embedding")
+ assert index.register("Embedding", metadata)
worker = asyncio.current_task()
assert worker is not None
index.register_active("task", worker)
try:
now[0] = 3
- waiter = asyncio.create_task(getattr(tracker, method)(request))
+ waiter = asyncio.create_task(
+ index.wait_for_work("task", ["embedding"], poll_interval=0.001)
+ )
await asyncio.sleep(0.01)
now[0] = 20
assert index.processing_seconds("task") == 20
- tracker.mark_embedding_done(request, "embedding")
+ index.record_result(metadata, "completed")
await waiter
now[0] = 22
assert index.processing_seconds("task") == 22
finally:
index.unregister_active("task", worker)
- tracker.cleanup(request)
@pytest.mark.asyncio
diff --git a/tests/service/test_task_queue_middleware.py b/tests/service/test_task_queue_middleware.py
index 3d79bc89e..f6c54db28 100644
--- a/tests/service/test_task_queue_middleware.py
+++ b/tests/service/test_task_queue_middleware.py
@@ -107,14 +107,38 @@ async def test_enqueue_registers_before_write_and_ack_finalizes_before_delete(tr
assert not index.has_work("task-1")
[email protected]("failure", [OSError("write failed"), asyncio.CancelledError()])
-async def test_enqueue_failure_discards_registered_work(tracked_queue, failure):
+async def test_enqueue_failure_records_failed_work(tracked_queue):
queue, index, transport, finalize, _ = tracked_queue
- transport.write.side_effect = failure
+ transport.write.side_effect = OSError("write failed")
+ enqueued_work = []
+ with bind_task_context(
+ "task-1",
+ "account",
+ "user",
+ enqueued_work=enqueued_work,
+ ):
+ with pytest.raises(OSError, match="write failed"):
+ await queue.enqueue({})
+ assert not index.has_work("task-1")
+ payload = json.loads(transport.write.await_args.args[1])
+ assert enqueued_work == [("Test", payload["_task_work_id"])]
+ assert index.queue_status("task-1")["Test"] == {
+ "processed": 0,
+ "requeue_count": 0,
+ "error_count": 1,
+ "errors": [{"message": "write failed"}],
+ }
+ finalize.assert_awaited_once()
+
+
+async def test_cancelled_enqueue_discards_registered_work(tracked_queue):
+ queue, index, transport, finalize, _ = tracked_queue
+ transport.write.side_effect = asyncio.CancelledError()
with bind_task_context("task-1", "account", "user"):
- with pytest.raises(type(failure)):
+ with pytest.raises(asyncio.CancelledError):
await queue.enqueue({})
assert not index.has_work("task-1")
+ assert index.queue_status("task-1") == {}
finalize.assert_awaited_once()
@@ -252,7 +276,8 @@ async def test_process_result_tracks_children_and_errors(tracked_queue):
class Handler(DequeueHandlerBase):
async def on_dequeue(self, data):
- contexts.append(get_task_context())
+ with bind_task_context("task-1", "account", "user"):
+ contexts.append(get_task_context())
correlations.append(log_correlation(telemetry_id="tm-1", message_id="semantic-1"))
await queue.enqueue({"child": True})
return ProcessResult.failed("failed work")
@@ -260,6 +285,7 @@ async def test_process_result_tracks_children_and_errors(tracked_queue):
queue.set_dequeue_handler(Handler())
assert (await queue.process_dequeued(message)).outcome is ProcessOutcome.FAILED
assert contexts[0].task_id == "task-1"
+ assert contexts[0].work_id == extract_task_metadata(message).work_id
assert correlations == ["task_id=task-1 telemetry_id=tm-1 message_id=semantic-1"]
assert get_task_context() is None
assert index.failure("task-1") == "failed work"
@@ -347,6 +373,7 @@ async def test_cancel_before_handler_uses_legacy_discard_callback(tracked_queue)
async def test_active_cancellation_distinguishes_user_from_shutdown(tracked_queue, user_cancel):
queue, index, transport, _, cancelled = tracked_queue
message = await enqueue_task(queue, transport)
+ metadata = extract_task_metadata(message)
started = asyncio.Event()
class Handler(DequeueHandlerBase):
@@ -363,6 +390,7 @@ async def test_active_cancellation_distinguishes_user_from_shutdown(tracked_queu
index.cancel_active("task-1")
result = await asyncio.wait_for(running, timeout=1)
assert result.outcome is ProcessOutcome.CANCELLED
+ await index.wait_for_work("task-1", [metadata.work_id], timeout=0.1)
await queue.ack("message-1", message)
assert not index.has_work("task-1")
assert (await queue.get_status()).processed == 1
diff --git a/tests/service/test_task_work_contract.py b/tests/service/test_task_work_contract.py
new file mode 100644
index 000000000..5bfc8dd9f
--- /dev/null
+++ b/tests/service/test_task_work_contract.py
@@ -0,0 +1,163 @@
+# Copyright (c) 2026 Beijing Volcano Engine Technology Co., Ltd.
+# SPDX-License-Identifier: AGPL-3.0
+
+import asyncio
+
+import pytest
+
+from openviking.service.task_store import PersistentTaskStore
+from openviking.service.task_tracker import TaskStatus, TaskTracker
+from openviking.service.task_work_index import QueueTaskMetadata, TaskWorkIndex
+from tests.test_task_tracker import _FakeAgfs
+
+
+async def test_wait_for_work_observes_only_explicit_ids():
+ index = TaskWorkIndex()
+ first = QueueTaskMetadata("task", "first")
+ unrelated = QueueTaskMetadata("task", "unrelated")
+ assert index.register("Semantic", first)
+ assert index.register("Embedding", unrelated)
+
+ index.record_result(first, "completed")
+ await index.wait_for_work("task", ["first"], timeout=0)
+
+ assert index.has_work("task")
+
+
[email protected]("recovered", [False, True])
+async def test_task_work_wait_includes_recovered_and_later_enqueued_work(recovered):
+ index = TaskWorkIndex()
+ root = QueueTaskMetadata("task", "root")
+ semantic = QueueTaskMetadata("task", "semantic")
+ embedding = QueueTaskMetadata("task", "embedding")
+ if recovered:
+ index.rebuild(
+ {
+ "Root": [{"task_id": "task", "_task_work_id": "root"}],
+ "Semantic": [{"task_id": "task", "_task_work_id": "semantic"}],
+ }
+ )
+ else:
+ assert index.register("Root", root)
+ assert index.register("Semantic", semantic)
+ waiting = asyncio.create_task(
+ index.wait_for_task_work("task", exclude_work_id="root", poll_interval=0.001)
+ )
+ try:
+ await asyncio.sleep(0)
+ assert not waiting.done()
+ assert index.register("Embedding", embedding)
+ index.record_result(semantic, "completed")
+ await index.prepare_ack("Semantic", semantic)
+ with pytest.raises(TimeoutError):
+ await index.wait_for_task_work("task", exclude_work_id="root", timeout=0)
+ index.record_result(embedding, "completed")
+ await index.prepare_ack("Embedding", embedding)
+ await asyncio.wait_for(waiting, timeout=1)
+ assert index.has_work("task")
+ finally:
+ waiting.cancel()
+ await asyncio.gather(waiting, return_exceptions=True)
+
+
+async def test_requeue_keeps_logical_work_until_replacement_ack():
+ index = TaskWorkIndex()
+ work = QueueTaskMetadata("task", "work")
+ assert index.register("Semantic", work)
+ assert index.register("Semantic", work)
+
+ index.record_result(work, "requeued")
+ await index.prepare_ack("Semantic", work)
+ assert index.has_work("task")
+
+ index.record_result(work, "completed")
+ await index.wait_for_work("task", ["work"], timeout=0.1)
+ assert index.queue_status("task")["Semantic"] == {
+ "processed": 1,
+ "requeue_count": 1,
+ "error_count": 0,
+ "errors": [],
+ }
+
+ await index.prepare_ack("Semantic", work)
+ assert not index.has_work("task")
+
+
[email protected]("errors", [None, ["first", "second"]])
+def test_terminal_work_result_is_monotonic_and_idempotent(errors):
+ index = TaskWorkIndex()
+ work = QueueTaskMetadata("task", "work")
+ assert index.register("Embedding", work)
+ messages = errors or ["first"]
+ error = "\n".join(messages)
+
+ index.record_result(work, "failed", error=error, errors=errors)
+ index.record_result(work, "completed")
+ index.record_result(work, "failed", error="later failure")
+
+ assert index.failure("task") == error
+ assert index.queue_status("task")["Embedding"] == {
+ "processed": 0,
+ "requeue_count": 0,
+ "error_count": len(messages),
+ "errors": [{"message": message} for message in messages],
+ }
+
+
+async def test_enqueue_failure_finalizes_task_with_same_error():
+ tracker = TaskTracker(PersistentTaskStore(_FakeAgfs()))
+ index = TaskWorkIndex()
+ tracker.attach_work_index(index)
+ task = await tracker.create("content_write", account_id="account", user_id="user")
+ await tracker.start(task.task_id, account_id="account", user_id="user")
+ work = QueueTaskMetadata(task.task_id, "embedding", "account", "user")
+ assert index.register("Embedding", work)
+
+ await index.fail_enqueue("Embedding", work, "embedding enqueue returned false")
+
+ failed = await tracker.get(task.task_id, account_id="account", user_id="user")
+ assert failed.status is TaskStatus.FAILED
+ assert failed.error == "embedding enqueue returned false"
+
+
+async def test_deleting_task_releases_retained_work_status():
+ tracker = TaskTracker(PersistentTaskStore(_FakeAgfs()))
+ index = TaskWorkIndex()
+ tracker.attach_work_index(index)
+ owner = {"account_id": "account", "user_id": "user"}
+ task = await tracker.create("content_write", **owner)
+ await tracker.start(task.task_id, **owner)
+ work = QueueTaskMetadata(task.task_id, "semantic", "account", "user")
+ assert index.register("Semantic", work)
+
+ await tracker.complete(
+ task.task_id,
+ {"done": True, "queue_status": tracker.build_queue_status(task.task_id)},
+ **owner,
+ )
+ index.record_result(work, "completed")
+ await index.prepare_ack("Semantic", work)
+ assert tracker.build_queue_status(task.task_id)["Semantic"]["processed"] == 1
+ completed = await tracker.get(task.task_id, **owner)
+ assert completed.result["queue_status"]["Semantic"]["processed"] == 1
+
+ assert await tracker.delete_user_tasks(**owner) == 1
+ assert tracker.build_queue_status(task.task_id) == {
+ "Semantic": {
+ "processed": 0,
+ "requeue_count": 0,
+ "error_count": 0,
+ "errors": [],
+ },
+ "Embedding": {
+ "processed": 0,
+ "requeue_count": 0,
+ "error_count": 0,
+ "errors": [],
+ },
+ }
diff --git a/tests/session/train/test_train_components.py b/tests/session/train/test_train_components.py
index 831983cd5..f12858b70 100644
--- a/tests/session/train/test_train_components.py
+++ b/tests/session/train/test_train_components.py
@@ -1279,12 +1279,8 @@ async def test_skill_policy_creation_preserves_lock_ownership(tmp_path, monkeypa
await fs._async_agfs.pathlock_release(lease)
finally:
from openviking.telemetry import unregister_telemetry
- from openviking.telemetry.request_wait_tracker import get_request_wait_tracker
if queue.enqueue.await_args:
msg = queue.enqueue.await_args.args[0]
- tracker = get_request_wait_tracker()
- tracker.mark_semantic_done(msg.telemetry_id, msg.id)
- tracker.cleanup(msg.telemetry_id)
unregister_telemetry(msg.telemetry_id)
agfs.close()
diff --git a/tests/storage/test_collection_schemas.py b/tests/storage/test_collection_schemas.py
index c0f149c01..da75841dd 100644
--- a/tests/storage/test_collection_schemas.py
+++ b/tests/storage/test_collection_schemas.py
@@ -1324,10 +1324,17 @@ async def test_embedding_handler_honors_existing_record_id_override(monkeypatch)
@pytest.mark.asyncio
-async def test_embedding_handler_generates_id_when_no_existing_override(monkeypatch):
+async def test_embedding_handler_generates_id_and_counts_written_context(monkeypatch):
+ from openviking.service.task_tracker import TaskTracker
+ from openviking.service.task_work_index import QueueTaskMetadata, TaskWorkIndex
from openviking.storage.vector_ids import vector_record_id
captured = {}
+ tracker = TaskTracker(AsyncMock())
+ index = TaskWorkIndex()
+ tracker.attach_work_index(index)
+ index.register("Embedding", QueueTaskMetadata("task-1", "work-1"))
+ monkeypatch.setattr("openviking.storage.collection_schemas.get_task_tracker", lambda: tracker)
class _CapturingVikingDB:
is_closing = False
@@ -1343,48 +1350,16 @@ async def test_embedding_handler_generates_id_when_no_existing_override(monkeypa
lambda: _DummyConfig(embedder),
)
payload = _build_queue_payload_for_account("acct-new-id")
-
- await TextEmbeddingHandler(_CapturingVikingDB()).on_dequeue(payload)
-
- assert captured["data"]["id"] == vector_record_id("acct-new-id", "viking://resources/sample", 2)
-
-
-async def test_embedding_handler_settles_request_wait_by_message_id(monkeypatch):
- class _CapturingVikingDB:
- is_closing = False
- mode = "local"
- uses_content_field = False
-
- async def upsert(self, _data, *, ctx, options=UpsertOptions()):
- assert options.partial_update is False
- return "rec-1"
-
- embedder = _DummyEmbedder()
- monkeypatch.setattr(
- "openviking_cli.utils.config.get_openviking_config",
- lambda: _DummyConfig(embedder),
- )
-
- completed = []
- monkeypatch.setattr(
- "openviking.storage.collection_schemas.get_request_wait_tracker",
- lambda: SimpleNamespace(
- mark_embedding_done=lambda telemetry_id, root_id, **kwargs: completed.append(
- (telemetry_id, root_id, kwargs)
- )
- ),
- )
-
- handler = TextEmbeddingHandler(_CapturingVikingDB())
- payload = _build_queue_payload()
queue_data = json.loads(payload["data"])
- queue_data["telemetry_id"] = "request-1"
+ queue_data["task_id"] = "task-1"
+ queue_data["_task_work_id"] = "work-1"
payload["data"] = json.dumps(queue_data)
- await handler.on_dequeue(payload)
+ await TextEmbeddingHandler(_CapturingVikingDB()).on_dequeue(payload)
- assert completed == [("request-1", queue_data["id"], {"vector_written": True})]
+ assert captured["data"]["id"] == vector_record_id("acct-new-id", "viking://resources/sample", 2)
+ assert tracker.get_context_count("task-1") == 1
+ assert tracker.get_context_count("unrelated-task") == 0
def test_context_collection_uses_acl_mode_and_excludes_parent_uri():
diff --git a/tests/storage/test_content_write_processing_mode.py b/tests/storage/test_content_write_processing_mode.py
index 1ed5b94f5..ef8e9a589 100644
--- a/tests/storage/test_content_write_processing_mode.py
+++ b/tests/storage/test_content_write_processing_mode.py
@@ -8,6 +8,9 @@ import pytest
from openviking.core.context import ContextLevel
from openviking.server.identity import RequestContext, Role
+from openviking.service.task_store import PersistentTaskStore
+from openviking.service.task_tracker import TaskTracker
+from openviking.service.task_work_index import TaskWorkIndex, bind_task_context
from openviking.session.memory.utils.memory_file_utils import MemoryFileUtils
from openviking.storage import content_write as content_write_module
from openviking.storage.abstract_overview import (
@@ -15,12 +18,31 @@ from openviking.storage.abstract_overview import (
)
from openviking.storage.content_write import ContentWriteCoordinator
from openviking.storage.context_update_plan import ContextUpdatePlan, FileRefreshIntent
+from openviking.storage.queuefs.queue_middleware import EnqueueReceipt
from openviking.storage.queuefs.semantic_ops.freshness_policy import FreshnessAction
from openviking.telemetry.context import bind_telemetry
from openviking.telemetry.operation import OperationTelemetry
from openviking.utils.ingest_options import IngestOptions
from openviking_cli.exceptions import InvalidArgumentError, NotFoundError
from openviking_cli.session.user_id import UserIdentifier
+from tests.test_task_tracker import _FakeAgfs
+
+
[email protected](autouse=True)
+def run_content_write_operations_inline(monkeypatch):
+ tracker = TaskTracker(PersistentTaskStore(_FakeAgfs()))
+ tracker.attach_work_index(TaskWorkIndex())
+ monkeypatch.setattr("openviking.service.task_tracker._instance", tracker)
+
+ async def run_tracked_operation(_task_type, *, operation, **_kwargs):
+ with bind_task_context("test-task", "account-1", "user-1", enqueued_work=[]):
+ return await operation(), "test-task"
+
+ monkeypatch.setattr(
+ content_write_module,
+ "run_tracked_operation",
+ run_tracked_operation,
+ )
class _FakePathLock:
@@ -283,12 +305,15 @@ async def test_direct_write_hands_exact_lock_to_request_coordinator(monkeypatch,
)
),
)
- queue = SimpleNamespace(enqueue=AsyncMock(return_value="queued"))
+ queue = SimpleNamespace(enqueue=AsyncMock(return_value=EnqueueReceipt("queued", "work-1")))
manager = SimpleNamespace(SEMANTIC="semantic", get_queue=lambda *args, **kwargs: queue)
monkeypatch.setattr(content_write_module, "get_queue_manager", lambda: manager)
telemetry = OperationTelemetry(operation="content.write", enabled=False)
- with bind_telemetry(telemetry):
+ with (
+ bind_telemetry(telemetry),
+ bind_task_context("test-task", "account-1", "user-1", enqueued_work=[]),
+ ):
result = await coordinator._write_direct_with_refresh(
uri=uri,
root_uri="viking://resources",
@@ -312,12 +337,6 @@ async def test_direct_write_hands_exact_lock_to_request_coordinator(monkeypatch,
assert result["semantic_status"] == "queued"
assert result["vector_status"] == "queued"
- content_write_module.get_request_wait_tracker().mark_semantic_done(
- telemetry.telemetry_id,
- coordinator_msg.id,
- processed_delta=0,
- )
-
@pytest.mark.asyncio
async def test_direct_write_commits_final_bytes_when_rnfv_content_is_unchanged(monkeypatch, ctx):
diff --git a/tests/storage/test_context_update_plan.py b/tests/storage/test_context_update_plan.py
index 5b195762c..dd7627a8b 100644
--- a/tests/storage/test_context_update_plan.py
+++ b/tests/storage/test_context_update_plan.py
@@ -3410,6 +3410,7 @@ async def test_directory_output_unchanged_still_applies_planned_scalar_fields(mo
SemanticTreeSnapshot,
)
from openviking.storage.queuefs.semantic_executor import SemanticTreeExecutor
+ from openviking.storage.queuefs.semantic_processor import SemanticProcessor
from openviking_cli.session.user_id import UserIdentifier
root = "viking://resources/repo"
@@ -3421,10 +3422,16 @@ async def test_directory_output_unchanged_still_applies_planned_scalar_fields(mo
lambda: SimpleNamespace(semantic=SimpleNamespace(overview_sample_limit=32)),
)
- class Processor:
+ embedding_queue = SimpleNamespace(enqueue=AsyncMock(return_value="embedding-id"))
+ queue_manager = SimpleNamespace(
+ EMBEDDING="Embedding",
+ get_queue=lambda *args, **kwargs: embedding_queue,
+ )
+ monkeypatch.setattr("openviking.storage.queuefs.get_queue_manager", lambda: queue_manager)
+
+ class Processor(SemanticProcessor):
_generate_overview = AsyncMock(return_value="overview")
_vectorize_directory = AsyncMock(return_value=set())
- _update_vector_fields = AsyncMock(return_value=True)
@staticmethod
def _normalize_overview_generation(overview):
@@ -3481,22 +3488,12 @@ async def test_directory_output_unchanged_still_applies_planned_scalar_fields(mo
await executor.run(root)
processor._vectorize_directory.assert_not_awaited()
- processor._update_vector_fields.assert_awaited_once_with(
- record_id="root-l0",
- uri=root,
- level=0,
- field_patch=FieldPatch(
- {"search_tags": ["scope=new"]},
- {"search_tags": "append"},
- {
- "abstract": "abstract",
- "uri": root,
- "level": 0,
- "account_id": "acc",
- },
- ),
- ctx=executor._ctx,
- )
+ embedding_queue.enqueue.assert_awaited_once()
+ message = embedding_queue.enqueue.await_args.args[0]
+ assert message.action.value == "update_fields"
+ assert message.record_ids == ["root-l0"]
+ assert message.update_fields == {"search_tags": ["scope=new"]}
+ assert message.field_modes == {"search_tags": "append"}
@pytest.mark.asyncio
@@ -3665,7 +3662,8 @@ async def test_file_repair_passes_partial_update_to_embedding(monkeypatch):
@pytest.mark.asyncio
-async def test_direct_only_plan_skips_semantic_queue(monkeypatch):
[email protected]("action_name", ["delete", "update_fields"])
+async def test_direct_only_plan_skips_semantic_queue(monkeypatch, action_name):
from openviking.server.identity import RequestContext, Role
from openviking.storage.context_update_plan import DirectIndexAction
from openviking.utils.resource_processor import ResourceProcessor
@@ -3678,15 +3676,22 @@ async def test_direct_only_plan_skips_semantic_queue(monkeypatch):
)
monkeypatch.setattr("openviking.utils.resource_processor.get_viking_fs", lambda: viking_fs)
processor = ResourceProcessor(SimpleNamespace(get_embedder=lambda: None))
- processor._enqueue_index_actions = AsyncMock()
+ embedding_queue = SimpleNamespace(enqueue=AsyncMock(return_value="embedding-id"))
+ queue_manager = SimpleNamespace(
+ EMBEDDING="Embedding",
+ get_queue=lambda *args, **kwargs: embedding_queue,
+ )
+ monkeypatch.setattr("openviking.storage.queuefs.get_queue_manager", lambda: queue_manager)
summarizer = SimpleNamespace(summarize=AsyncMock())
processor._get_summarizer = lambda: summarizer
action = DirectIndexAction(
- "update_fields",
+ action_name,
root + "/a.py",
2,
"a-l2",
- field_patch=FieldPatch({"search_tags": ["scope=new"]}),
+ field_patch=(
+ FieldPatch({"search_tags": ["scope=new"]}) if action_name == "update_fields" else None
+ ),
)
await processor.finish_prepared_resource(
@@ -3707,7 +3712,7 @@ async def test_direct_only_plan_skips_semantic_queue(monkeypatch):
"uri": action.uri,
"level": action.level,
"record_id": action.record_id,
- "field_patch": action.field_patch.to_dict(),
+ "field_patch": action.field_patch.to_dict() if action.field_patch else None,
"md5": action.md5,
}
],
@@ -3719,7 +3724,12 @@ async def test_direct_only_plan_skips_semantic_queue(monkeypatch):
processing_mode="semantic_and_vectors",
)
- processor._enqueue_index_actions.assert_awaited_once()
+ embedding_queue.enqueue.assert_awaited_once()
+ message = embedding_queue.enqueue.await_args.args[0]
+ assert message.action.value == action_name
+ assert message.record_ids == ["a-l2"]
+ if action_name == "update_fields":
+ assert message.update_fields == {"search_tags": ["scope=new"]}
summarizer.summarize.assert_not_awaited()
viking_fs._async_agfs.pathlock_release.assert_awaited_once_with(lock)
diff --git a/tests/storage/test_embedding_msg.py b/tests/storage/test_embedding_msg.py
index 3fb7edf01..0880409aa 100644
--- a/tests/storage/test_embedding_msg.py
+++ b/tests/storage/test_embedding_msg.py
@@ -1,8 +1,6 @@
# Copyright (c) 2026 Beijing Volcano Engine Technology Co., Ltd.
# SPDX-License-Identifier: AGPL-3.0
-from uuid import uuid4
-
import pytest
from openviking.storage.index_action import FieldPatch, IndexAction
@@ -13,39 +11,24 @@ from openviking.storage.queuefs.embedding_msg import (
NoopPayload,
UpdateFieldsPayload,
)
-from openviking.telemetry.request_wait_tracker import RequestWaitTracker
-
-
-def test_embedding_msg_roundtrip_preserves_id_for_request_wait_tracker():
- telemetry_id = f"tm_{uuid4().hex}"
- tracker = RequestWaitTracker.get_instance()
- tracker.register_request(telemetry_id)
-
- try:
- msg = EmbeddingMsg(
- "hello",
- {"uri": "viking://user/default/skills/demo", "account_id": "default"},
- telemetry_id=telemetry_id,
- )
- tracker.register_embedding_root(telemetry_id, msg.id)
-
- restored = EmbeddingMsg.from_dict(msg.to_dict())
- assert restored.id == msg.id
- tracker.mark_embedding_done(telemetry_id, restored.id)
- assert tracker.is_complete(telemetry_id)
- finally:
- tracker.cleanup(telemetry_id)
-
-def test_embedding_msg_roundtrip_preserves_queue_enqueue_time():
+def test_embedding_msg_roundtrip_preserves_task_identity_and_queue_time():
msg = EmbeddingMsg(
"hello",
- {"uri": "viking://resources/demo", "account_id": "default"},
+ {"uri": "viking://user/default/skills/demo", "account_id": "default"},
+ telemetry_id="telemetry-1",
+ task_id="task-1",
+ _task_work_id="work-1",
queue_enqueued_at=123.456,
)
- assert EmbeddingMsg.from_dict(msg.to_dict()).queue_enqueued_at == 123.456
+ restored = EmbeddingMsg.from_dict(msg.to_dict())
+
+ assert restored.id == msg.id
+ assert restored.task_id == "task-1"
+ assert restored._task_work_id == "work-1"
+ assert restored.queue_enqueued_at == 123.456
def test_embedding_msg_roundtrip_preserves_failure_requeue_count():
@@ -316,14 +299,10 @@ def test_legacy_upsert_discards_historically_ignored_fields():
assert restored.action is IndexAction.UPSERT
assert restored.record_ids == []
assert restored.field_patch is None
- assert set(restored.to_dict()) == {
- "id",
- "telemetry_id",
- "queue_enqueued_at",
- "failure_requeue_count",
- "action",
- "payload",
- }
+ serialized = restored.to_dict()
+ assert "record_ids" not in serialized
+ assert "update_fields" not in serialized
+ assert "initial_fields" not in serialized
def test_legacy_update_fields_message_normalizes_to_field_patch_payload():
diff --git a/tests/storage/test_memory_semantic_stall.py b/tests/storage/test_memory_semantic_stall.py
index 1e682013b..98572519f 100644
--- a/tests/storage/test_memory_semantic_stall.py
+++ b/tests/storage/test_memory_semantic_stall.py
@@ -17,7 +17,6 @@ import pytest
from openviking.storage.queuefs.process_result import ProcessOutcome
from openviking.storage.queuefs.semantic_msg import SemanticMsg
from openviking.storage.queuefs.semantic_processor import SemanticProcessor
-from openviking.telemetry.request_wait_tracker import get_request_wait_tracker
from openviking.utils.circuit_breaker import CircuitBreakerOpen
@@ -53,7 +52,17 @@ def _build_data(msg: SemanticMsg) -> dict:
@pytest.mark.parametrize("context_type", ["resource", "memory", "skill"])
async def test_retry_cancellation_keeps_skill_wait_isolated(monkeypatch, context_type):
+ from openviking.service.task_store import PersistentTaskStore
+ from openviking.service.task_tracker import TaskTracker
+ from tests.test_task_tracker import _FakeAgfs
+
+ monkeypatch.setattr(
+ "openviking.service.task_tracker._instance", TaskTracker(PersistentTaskStore(_FakeAgfs()))
+ )
processor = _processor()
+ processor._circuit_breakers["acc1"] = SimpleNamespace(
+ check=MagicMock(side_effect=CircuitBreakerOpen), retry_after=0
+ )
entered, release = asyncio.Event(), asyncio.Event()
written = []
@@ -88,7 +97,6 @@ async def test_retry_cancellation_keeps_skill_wait_isolated(monkeypatch, context
if not worker.done():
worker.cancel()
await asyncio.gather(worker, return_exceptions=True)
- get_request_wait_tracker().cleanup(msg.telemetry_id)
@pytest.mark.asyncio
diff --git a/tests/storage/test_process_result.py b/tests/storage/test_process_result.py
index 5d21e1c16..995e186f0 100644
--- a/tests/storage/test_process_result.py
+++ b/tests/storage/test_process_result.py
@@ -32,3 +32,9 @@ def test_result_preserves_value_and_empty_error():
payload = {"value": 1}
assert ProcessResult.success(payload).value is payload
assert ProcessResult.failed("").error == ""
+
+
[email protected]("errors", [(), ("valid", 1)])
+def test_result_rejects_invalid_structured_errors(errors):
+ with pytest.raises(ValueError):
+ ProcessResult.failed("failure", errors=errors)
diff --git a/tests/storage/test_queue_manager.py b/tests/storage/test_queue_manager.py
index b0aa487af..b691c8fe5 100644
--- a/tests/storage/test_queue_manager.py
+++ b/tests/storage/test_queue_manager.py
@@ -14,7 +14,11 @@ from unittest.mock import AsyncMock
import pytest
from openviking.pyagfs import AsyncAGFSClient
-from openviking.service.task_work_index import bind_task_context
+from openviking.service.task_work_index import (
+ QueueTaskMetadata,
+ bind_task_context,
+ record_enqueued_work,
+)
from openviking.storage.queuefs.named_queue import DequeueHandlerBase, NamedQueue
from openviking.storage.queuefs.process_result import ProcessResult
from openviking.storage.queuefs.queue_manager import QueueManager
@@ -86,17 +90,18 @@ async def test_skill_shutdown_releases_lock_after_embedding_worker_exits(
from openviking.storage.queuefs.semantic_executor import SemanticTreeStats
from openviking.storage.queuefs.semantic_msg import SemanticMsg
from openviking.storage.queuefs.semantic_processor import SemanticProcessor
- from openviking.telemetry import OperationTelemetry
- from openviking.telemetry.request_wait_tracker import get_request_wait_tracker
- tracker = get_request_wait_tracker()
- telemetry_id = OperationTelemetry("skill-shutdown").telemetry_id
- tracker.register_request(telemetry_id)
msg = SemanticMsg(
- uri="viking://agent/skills/demo", context_type="skill", telemetry_id=telemetry_id
+ uri="viking://agent/skills/demo",
+ context_type="skill",
+ task_id="skill-task",
+ _task_work_id="semantic",
)
started, queued, release = threading.Event(), threading.Event(), threading.Event()
events = []
+ work_index = None
+ active_work = QueueTaskMetadata("skill-task", "active")
+ queued_work = QueueTaskMetadata("skill-task", "queued")
class Lease:
lock = {"lease_ref": "demo"}
@@ -111,8 +116,11 @@ async def test_skill_shutdown_releases_lock_after_embedding_worker_exits(
pass
async def run(self, uri):
- tracker.register_embedding_root(telemetry_id, "active")
- tracker.register_embedding_root(telemetry_id, "queued")
+ assert work_index is not None
+ assert work_index.register("Embedding", active_work)
+ assert work_index.register("Embedding", queued_work)
+ record_enqueued_work("Embedding", active_work.work_id)
+ record_enqueued_work("Embedding", queued_work.work_id)
queued.set()
def get_stats(self):
@@ -124,7 +132,8 @@ async def test_skill_shutdown_releases_lock_after_embedding_worker_exits(
while not release.is_set():
await asyncio.sleep(0.01)
events.append("write-finished")
- tracker.mark_embedding_done(telemetry_id, "active")
+ assert work_index is not None
+ work_index.record_result(active_work, "completed")
await run_to_completion(finish)
return ProcessResult.success()
@@ -146,6 +155,9 @@ async def test_skill_shutdown_releases_lock_after_embedding_worker_exits(
return {"id": self.name, "data": msg.to_json()}
async def process_dequeued(self, data):
+ if self.name == QueueManager.SEMANTIC:
+ with bind_task_context("skill-task", "", "", enqueued_work=[]):
+ return await self.handler(data)
return await self.handler(data)
async def ack(self, *args):
@@ -171,6 +183,11 @@ async def test_skill_shutdown_releases_lock_after_embedding_worker_exits(
manager = QueueManager(
object(), max_concurrent_semantic=concurrency, max_concurrent_embedding=concurrency
)
+ work_index = manager._task_work_index
+ monkeypatch.setattr(
+ "openviking.storage.queuefs.semantic_work.get_task_tracker",
+ lambda: SimpleNamespace(wait_for_work=work_index.wait_for_work),
+ )
manager._poll_interval = 0.001
manager.set_vlm_resolver(SimpleNamespace(get_vlm=AsyncMock(return_value=SimpleNamespace())))
manager.setup_standard_queues(object(), start=False)
@@ -197,14 +214,13 @@ async def test_skill_shutdown_releases_lock_after_embedding_worker_exits(
assert all(not thread.is_alive() for thread in threads)
assert events.index("write-finished") < events.index("lock-released")
assert f"ack-{manager.SEMANTIC}" not in events
- assert not tracker.is_complete(telemetry_id) # Queued vectors await restart.
+ assert work_index.has_work("skill-task") # Queued vectors await restart.
finally:
release.set()
if stopping is not None:
await stopping
else:
await asyncio.to_thread(manager.stop)
- tracker.cleanup(telemetry_id)
async def test_status_waits_for_processing_messages_from_other_workers(monkeypatch) -> None:
diff --git a/tests/storage/test_reindex_msg.py b/tests/storage/test_reindex_msg.py
index 578982af6..5bb6883d3 100644
--- a/tests/storage/test_reindex_msg.py
+++ b/tests/storage/test_reindex_msg.py
@@ -1,3 +1,4 @@
+import asyncio
from types import SimpleNamespace
from unittest.mock import AsyncMock, Mock
@@ -62,15 +63,16 @@ def test_reindex_msg_rejects_missing_descriptor_fields():
@pytest.mark.asyncio
async def test_reindex_processor_terminalizes_root_after_descendants(monkeypatch):
+ from openviking.service.task_store import PersistentTaskStore
+ from openviking.service.task_tracker import TaskTracker
+ from openviking.service.task_work_index import QueueTaskMetadata, TaskWorkIndex
from openviking.storage.queuefs.reindex_processor import ReindexProcessor
+ from tests.test_task_tracker import _FakeAgfs
- tracker = SimpleNamespace(
- get=AsyncMock(return_value=SimpleNamespace(status=TaskStatus.RUNNING)),
- start=AsyncMock(),
- complete=AsyncMock(),
- fail=AsyncMock(),
- wait_for_descendants=AsyncMock(),
- )
+ owner = {"account_id": "account-1", "user_id": "user-1"}
+ tracker = TaskTracker(PersistentTaskStore(_FakeAgfs()))
+ index = TaskWorkIndex()
+ tracker.attach_work_index(index)
monkeypatch.setattr(
"openviking.storage.queuefs.reindex_processor.get_task_tracker", Mock(return_value=tracker)
)
@@ -86,25 +88,49 @@ async def test_reindex_processor_terminalizes_root_after_descendants(monkeypatch
viking_fs = SimpleNamespace(_async_agfs=SimpleNamespace(pathlock_release=AsyncMock()))
processor = ReindexProcessor(viking_fs)
processor._adopt_or_reacquire = AsyncMock(return_value={"lease": "root"})
+ task = await tracker.create("admin_reindex", resource_id="viking://resources/demo", **owner)
+ root_work = QueueTaskMetadata(task.task_id, "root-work", **owner)
+ child_work = QueueTaskMetadata(task.task_id, "child-work", **owner)
+ assert index.register("Reindex", root_work)
+ assert index.register("Embedding", child_work)
+ entered_descendant_wait = asyncio.Event()
+ original_wait_for_task_work = tracker.wait_for_task_work
+
+ async def wait_for_task_work(*args, **kwargs):
+ entered_descendant_wait.set()
+ await original_wait_for_task_work(*args, **kwargs)
+
+ monkeypatch.setattr(tracker, "wait_for_task_work", wait_for_task_work)
message = ReindexMsg(
- task_id="task-1",
+ task_id=task.task_id,
uri="viking://resources/demo",
object_type="resource",
mode="vectors_only",
- account_id="account-1",
- user_id="user-1",
+ **owner,
role="admin",
)
- outcome = await processor.on_dequeue(
- {"id": "queue-1", "data": {**message.to_dict(), "_task_work_id": "root-work"}}
+ processing = asyncio.create_task(
+ processor.on_dequeue(
+ {"id": "queue-1", "data": {**message.to_dict(), "_task_work_id": "root-work"}}
+ )
)
+ await asyncio.wait_for(entered_descendant_wait.wait(), 1)
+ assert not processing.done()
+ assert (await tracker.get(task.task_id, **owner)).status is TaskStatus.RUNNING
+ index.record_result(child_work, "completed")
+ await index.prepare_ack("Embedding", child_work)
+ outcome = await asyncio.wait_for(processing, 1)
assert outcome.outcome.value == "success"
- tracker.wait_for_descendants.assert_awaited_once_with("task-1", "root-work")
- tracker.complete.assert_awaited_once_with(
- "task-1", result, account_id="account-1", user_id="user-1"
- )
+ assert (await tracker.get(task.task_id, **owner)).status is TaskStatus.RUNNING
+
+ index.record_result(root_work, "completed")
+ await index.prepare_ack("Reindex", root_work)
+ completed = await tracker.wait(task.task_id, **owner, timeout=0)
+ assert completed.status is TaskStatus.COMPLETED
+ assert completed.result["rebuilt_records"] == 3
+ assert completed.result["queue_status"]["Embedding"]["processed"] == 1
@pytest.mark.asyncio
@@ -122,7 +148,7 @@ async def test_reindex_processor_restores_root_telemetry_for_executor(monkeypatc
start=AsyncMock(),
complete=AsyncMock(),
fail=AsyncMock(),
- wait_for_descendants=AsyncMock(),
+ wait_for_task_work=AsyncMock(),
)
monkeypatch.setattr(
"openviking.storage.queuefs.reindex_processor.get_task_tracker", Mock(return_value=tracker)
@@ -177,7 +203,7 @@ async def test_reindex_processor_replay_of_completed_task_skips_force_rebuild(mo
start=AsyncMock(),
complete=AsyncMock(),
fail=AsyncMock(),
- wait_for_descendants=AsyncMock(),
+ wait_for_task_work=AsyncMock(),
)
monkeypatch.setattr(
"openviking.storage.queuefs.reindex_processor.get_task_tracker", Mock(return_value=tracker)
@@ -225,7 +251,7 @@ async def test_reindex_processor_requeues_lock_handoff_failure_before_ack(monkey
start=AsyncMock(),
complete=AsyncMock(),
fail=AsyncMock(),
- wait_for_descendants=AsyncMock(),
+ wait_for_task_work=AsyncMock(),
)
queue_manager = SimpleNamespace(REINDEX="Reindex", enqueue=AsyncMock(return_value="retry-1"))
monkeypatch.setattr(
@@ -250,7 +276,12 @@ async def test_reindex_processor_requeues_lock_handoff_failure_before_ack(monkey
assert outcome.outcome.value == "requeued"
queue_manager.enqueue.assert_awaited_once_with(
- "Reindex", {**message.to_dict(), "lock_handoff_retry": 1}
+ "Reindex",
+ {
+ **message.to_dict(),
+ "_task_work_id": "root-work",
+ "lock_handoff_retry": 1,
+ },
)
tracker.fail.assert_not_awaited()
@@ -264,7 +295,7 @@ async def test_reindex_processor_fails_after_lock_handoff_retry_limit(monkeypatc
start=AsyncMock(),
complete=AsyncMock(),
fail=AsyncMock(),
- wait_for_descendants=AsyncMock(),
+ wait_for_task_work=AsyncMock(),
)
monkeypatch.setattr(
"openviking.storage.queuefs.reindex_processor.get_task_tracker", Mock(return_value=tracker)
@@ -324,7 +355,7 @@ async def test_reindex_processor_does_not_release_lease_owned_by_failed_executor
start=AsyncMock(),
complete=AsyncMock(),
fail=AsyncMock(),
- wait_for_descendants=AsyncMock(),
+ wait_for_task_work=AsyncMock(),
)
monkeypatch.setattr(
"openviking.storage.queuefs.reindex_processor.get_task_tracker", Mock(return_value=tracker)
diff --git a/tests/storage/test_semantic_processor_lock_ownership.py b/tests/storage/test_semantic_processor_lock_ownership.py
index 0e94ba804..534042adf 100644
--- a/tests/storage/test_semantic_processor_lock_ownership.py
+++ b/tests/storage/test_semantic_processor_lock_ownership.py
@@ -3,17 +3,19 @@
from __future__ import annotations
+import asyncio
+import json
+from contextlib import nullcontext
from types import SimpleNamespace
+from typing import cast
from unittest.mock import AsyncMock
import pytest
from openviking.storage.queuefs.process_result import ProcessOutcome
from openviking.storage.queuefs.semantic_executor import SemanticTreeStats
-from openviking.storage.queuefs.semantic_lock import SemanticLockScope
from openviking.storage.queuefs.semantic_msg import SemanticMsg
from openviking.storage.queuefs.semantic_processor import SemanticProcessor
-from openviking.storage.queuefs.semantic_work import SemanticMessageWork
def _processor():
@@ -71,87 +73,6 @@ async def test_memory_semantic_directory_does_not_release_borrowed_lock(monkeypa
assert pathlock.release_calls == []
-async def test_durable_plan_waits_for_embeddings_before_releasing_handoff_lock(monkeypatch):
- from openviking.storage.context_update_plan import (
- SemanticPlan,
- SemanticTreeEntry,
- SemanticTreeSnapshot,
- )
-
- events = []
- pathlock = _FakePathLock()
- lease = {"id": "durable-plan-lock"}
- plan = SemanticPlan(
- "viking://resources/demo",
- "resource",
- SemanticTreeSnapshot((SemanticTreeEntry("", "directory", "unchanged", "aggregate"),)),
- )
- msg = SemanticMsg(
- uri=plan.root_uri,
- context_type="resource",
- telemetry_id="durable-plan",
- lock_handoff={"owner_id": "producer"},
- plan=plan,
- )
-
- class Tracker:
- async def wait_for_embeddings(self, telemetry_id, **kwargs):
- assert telemetry_id == msg.telemetry_id
- events.append("embeddings-settled")
-
- monkeypatch.setattr(
- "openviking.storage.queuefs.semantic_work.get_request_wait_tracker", lambda: Tracker()
- )
- monkeypatch.setattr(
- "openviking.storage.queuefs.semantic_lock.get_viking_fs",
- lambda: _FakeVikingFS(pathlock),
- )
-
- work = SemanticMessageWork(SimpleNamespace(), msg, caller_lock=None)
- work.scope = SemanticLockScope(lease, _owned=True)
-
- await work.finish_processing(True)
-
- assert events == ["embeddings-settled"]
- assert pathlock.release_calls == ["durable-plan-lock"]
-
-
-async def test_content_write_waits_for_embeddings_before_releasing_handoff_lock(monkeypatch):
- events = []
- pathlock = _FakePathLock()
- lease = {"id": "content-write-lock"}
- msg = SemanticMsg(
- uri="viking://resources/demo",
- context_type="resource",
- telemetry_id="content-write",
- lock_handoff={"owner_id": "producer"},
- generation_trigger="content_write",
- )
-
- class Tracker:
- async def wait_for_embeddings(self, telemetry_id, **kwargs):
- assert telemetry_id == msg.telemetry_id
- events.append("embeddings-settled")
-
- monkeypatch.setattr(
- "openviking.storage.queuefs.semantic_work.get_request_wait_tracker", lambda: Tracker()
- )
- monkeypatch.setattr(
- "openviking.storage.queuefs.semantic_lock.get_viking_fs",
- lambda: _FakeVikingFS(pathlock),
- )
-
- work = SemanticMessageWork(SimpleNamespace(), msg, caller_lock=None)
- work.scope = SemanticLockScope(lease, _owned=True)
-
- await work.finish_processing(True)
-
- assert events == ["embeddings-settled"]
- assert pathlock.release_calls == ["content-write-lock"]
-
-
def test_semantic_tree_stats_aggregate_multiple_plans_for_one_request():
telemetry_id = "reindex-request-stats"
SemanticProcessor._cache_tree_stats(
@@ -177,6 +98,13 @@ def test_semantic_tree_stats_aggregate_multiple_plans_for_one_request():
@pytest.mark.parametrize("context_type", ["resource", "memory", "skill"])
async def test_missing_root_is_acked_before_lock_scope_is_resolved(monkeypatch, context_type):
"""Resolving the lock for a deleted root would recreate it to hold lock metadata."""
+ from openviking.service.task_store import PersistentTaskStore
+ from openviking.service.task_tracker import TaskTracker
+ from tests.test_task_tracker import _FakeAgfs
+
+ monkeypatch.setattr(
+ "openviking.service.task_tracker._instance", TaskTracker(PersistentTaskStore(_FakeAgfs()))
+ )
processor = _processor()
fs = _FakeVikingFS()
@@ -218,3 +146,179 @@ async def test_missing_root_is_acked_before_lock_scope_is_resolved(monkeypatch,
assert fs._async_agfs.release_calls == (
["queued-skill-lock"] if context_type == "skill" else []
)
+
+
+ ("context_type", "tracked", "fail_generation"),
+ [
+ ("resource", True, False),
+ ("skill", True, False),
+ ("skill", False, False),
+ ("skill", False, True),
+ ],
+)
+async def test_shared_scheduler_drains_each_messages_embeddings_before_unlock(
+ monkeypatch, context_type, tracked, fail_generation
+):
+ from openviking.pyagfs import AGFSSyncClientProtocol
+ from openviking.server.identity import RequestContext, Role
+ from openviking.service.task_queue_middleware import TaskWorkQueueMiddleware
+ from openviking.service.task_store import PersistentTaskStore
+ from openviking.service.task_tracker import TaskStatus, TaskTracker
+ from openviking.service.task_work_index import (
+ TaskWorkIndex,
+ bind_task_context,
+ extract_task_metadata,
+ )
+ from openviking.storage.context_update_plan import (
+ ContentState,
+ IndexAction,
+ IndexSlot,
+ SemanticAction,
+ SemanticPlan,
+ SemanticTreeEntry,
+ SemanticTreeSnapshot,
+ )
+ from openviking.storage.queuefs.named_queue import NamedQueue
+ from openviking.storage.queuefs.semantic_executor import SemanticTreeExecutor
+ from openviking_cli.session.user_id import UserIdentifier
+ from tests.storage.test_semantic_executor_skip_files import _FakeVikingFS as TreeFS
+ from tests.test_task_tracker import _FakeAgfs, _owner_kwargs
+
+ root = (
+ "viking://resources/demo"
+ if context_type == "resource"
+ else "viking://agent/skills/demo/reference"
+ )
+ warm_root = "viking://resources/warm"
+ released = []
+
+ class FS(TreeFS):
+ async def exists(self, uri, ctx=None):
+ return True
+
+ async def pathlock_adopt(self, lease):
+ return lease
+
+ async def pathlock_release(self, lease):
+ if "id" in lease:
+ released.append(lease["id"])
+
+ fs = FS({root: [], warm_root: []})
+ for module in ("semantic_processor", "semantic_executor", "semantic_lock"):
+ monkeypatch.setattr(f"openviking.storage.queuefs.{module}.get_viking_fs", lambda: fs)
+ agfs = _FakeAgfs()
+ tracker = TaskTracker(PersistentTaskStore(agfs))
+ index = TaskWorkIndex()
+ tracker.attach_work_index(index)
+ monkeypatch.setattr("openviking.service.task_tracker._instance", tracker)
+ entered_embedding_drain = asyncio.Event()
+ original_wait_for_work = tracker.wait_for_work
+
+ async def wait_for_work(*args, **kwargs):
+ entered_embedding_drain.set()
+ await original_wait_for_work(*args, **kwargs)
+
+ monkeypatch.setattr(tracker, "wait_for_work", wait_for_work)
+ owner = _owner_kwargs("acc", "alice")
+ source = await tracker.create("content_write", **owner)
+ queue = NamedQueue(
+ cast(AGFSSyncClientProtocol, agfs),
+ "/queue",
+ "Embedding",
+ middlewares=[TaskWorkQueueMiddleware(index)],
+ )
+ enqueued = asyncio.Event()
+
+ async def vectorize(uri, **kwargs):
+ await queue.enqueue({"context_data": {"uri": uri, **owner}})
+ enqueued.set()
+ if fail_generation:
+ raise RuntimeError("second embedding enqueue failed")
+ return {0, 1}
+
+ processor = _processor()
+ processor._generate_overview = AsyncMock(return_value="overview")
+ processor._vectorize_directory = vectorize
+ ctx = RequestContext(UserIdentifier("acc", "alice"), Role(Role.USER))
+ # Create the shared workers under a different Task, as happens across requests.
+ with bind_task_context(source.task_id, **owner, enqueued_work=[]):
+ await SemanticTreeExecutor(processor, "resource", 1, ctx, skip_vectorization=True).run(
+ warm_root
+ )
+ await tracker.complete(source.task_id, {"ok": True}, **owner)
+
+ plan = None
+ if context_type == "resource":
+ plan = SemanticPlan(
+ root,
+ context_type,
+ SemanticTreeSnapshot(
+ (
+ SemanticTreeEntry(
+ "",
+ "directory",
+ ContentState.UNCHANGED,
+ SemanticAction.AGGREGATE,
+ index_slots=(IndexSlot(0, "root-l0", action=IndexAction.UPSERT),),
+ ),
+ )
+ ),
+ )
+ msg = SemanticMsg(
+ uri=root,
+ context_type=context_type,
+ account_id="acc",
+ user_id="alice",
+ recursive=False,
+ propagate_to_parent=False,
+ generation_trigger="parent_refresh",
+ lock_handoff={"id": "message-lock"},
+ plan=plan,
+ )
+ task = await tracker.create("content_write", **owner) if tracked else None
+ task_context = (
+ bind_task_context(task.task_id, **owner, enqueued_work=[], work_id="semantic")
+ if task
+ else nullcontext()
+ )
+ with task_context:
+ worker = asyncio.create_task(processor.on_dequeue({"data": msg.to_json()}))
+ try:
+ await asyncio.wait_for(enqueued.wait(), 1)
+ await asyncio.wait_for(entered_embedding_drain.wait(), 1)
+ assert not worker.done()
+ assert released == []
+ payload = json.loads(agfs.files["/queue/Embedding/enqueue"])
+ metadata = extract_task_metadata(payload)
+ assert metadata is not None
+ assert metadata.task_id != source.task_id
+ if task:
+ assert metadata.task_id == task.task_id
+ index.record_result(metadata, "completed")
+ await index.prepare_ack("Embedding", metadata)
+ result = await asyncio.wait_for(worker, 1)
+ assert result.outcome is (
+ ProcessOutcome.FAILED if fail_generation else ProcessOutcome.SUCCESS
+ )
+ assert released == ["message-lock"]
+ if not tracked:
+ completed = await tracker.wait(metadata.task_id, **owner, timeout=0)
+ assert completed.status is (
+ TaskStatus.FAILED if fail_generation else TaskStatus.COMPLETED
+ )
+ if fail_generation:
+ assert completed.error is not None
+ assert "second embedding enqueue failed" in completed.error
+ source_result = await tracker.get(source.task_id, **owner)
+ assert source_result is not None
+ assert source_result.status is TaskStatus.COMPLETED
+ finally:
+ payload = agfs.files.get("/queue/Embedding/enqueue")
+ if payload:
+ metadata = extract_task_metadata(json.loads(payload))
+ if metadata:
+ index.record_result(metadata, "completed")
+ worker.cancel()
+ await asyncio.gather(worker, return_exceptions=True)
diff --git a/tests/storage/test_semantic_processor_permanent_storage_error.py b/tests/storage/test_semantic_processor_permanent_storage_error.py
index af31296ad..859af3855 100644
--- a/tests/storage/test_semantic_processor_permanent_storage_error.py
+++ b/tests/storage/test_semantic_processor_permanent_storage_error.py
@@ -5,7 +5,7 @@
import asyncio
import json
from types import SimpleNamespace
-from unittest.mock import AsyncMock, Mock, call
+from unittest.mock import AsyncMock, Mock
from uuid import uuid4
import pytest
@@ -21,7 +21,6 @@ from openviking.storage.queuefs.semantic_executor import SemanticTreeStats
from openviking.storage.queuefs.semantic_msg import SemanticMsg
from openviking.storage.queuefs.semantic_processor import SemanticProcessor
from openviking.storage.viking_fs import SyncDiff
-from openviking.telemetry.request_wait_tracker import get_request_wait_tracker
from openviking.utils.circuit_breaker import CircuitBreaker, CircuitBreakerOpen
@@ -49,21 +48,15 @@ def semantic_env(monkeypatch):
resolver = SimpleNamespace(get_vlm=AsyncMock(return_value=SimpleNamespace()))
processor = SemanticProcessor(vlm_resolver=resolver)
processor._sync_topdown_recursive = AsyncMock(return_value=SyncDiff())
- tracker = get_request_wait_tracker()
- telemetry_ids = []
def message(**kwargs):
- msg = SemanticMsg(
+ return SemanticMsg(
uri="viking://resources/notes.txt",
context_type="resource",
telemetry_id=str(uuid4()),
propagate_to_parent=False,
**kwargs,
)
- telemetry_ids.append(msg.telemetry_id)
- tracker.register_request(msg.telemetry_id)
- tracker.register_semantic_root(msg.telemetry_id, msg.id)
- return msg
yield SimpleNamespace(
processor=processor,
@@ -71,11 +64,8 @@ def semantic_env(monkeypatch):
lease=lease,
dag=dag,
retry_queue=retry_queue,
- tracker=tracker,
message=message,
)
- for telemetry_id in telemetry_ids:
- tracker.cleanup(telemetry_id)
def _storage_error(wrapped):
@@ -87,16 +77,6 @@ def _storage_error(wrapped):
return error
-async def _assert_request_failed(env, msg, error):
- await asyncio.wait_for(env.tracker.wait_for_request(msg.telemetry_id), timeout=1)
- assert env.tracker.build_queue_status(msg.telemetry_id)["Semantic"] == {
- "processed": 0,
- "requeue_count": 0,
- "error_count": 1,
- "errors": [{"message": str(error)}],
- }
-
-
@pytest.mark.asyncio
@pytest.mark.parametrize("wrapped", [False, True])
@pytest.mark.parametrize("boundary", ["sync", "dag"])
@@ -122,7 +102,6 @@ async def test_storage_error_finishes_request_and_respects_lock_ownership(
assert result.error == str(error)
operation.assert_awaited_once()
env.retry_queue.enqueue.assert_not_awaited()
- await _assert_request_failed(env, msg, error)
env.processor._account_breaker(msg.account_id).check()
if owned:
env.pathlock.pathlock_adopt.assert_awaited_once_with(msg.lock_handoff)
@@ -171,10 +150,6 @@ async def test_queue_acks_failed_semantic_work(semantic_env, monkeypatch, wrappe
await asyncio.wait_for(queue.dequeue(), timeout=1)
- assert backend.write.await_args_list == [
- call("/queue/Semantic/enqueue", queued_payload.encode()),
- call("/queue/Semantic/ack", b"queue-message"),
- ]
status = await queue.get_status()
assert status.is_complete
assert status.in_progress == status.processed == status.requeue_count == 0
@@ -189,7 +164,6 @@ async def test_queue_acks_failed_semantic_work(semantic_env, monkeypatch, wrappe
finalize.assert_not_awaited()
env.retry_queue.enqueue.assert_not_awaited()
env.pathlock.pathlock_release.assert_awaited_once_with(env.lease)
- await _assert_request_failed(env, msg, error)
@pytest.mark.asyncio
@@ -202,17 +176,14 @@ async def test_malformed_resources_do_not_pause_following_valid_work(semantic_en
result = await asyncio.wait_for(env.processor.on_dequeue(msg.to_dict()), timeout=1)
assert result.outcome is ProcessOutcome.FAILED
assert result.error == str(error)
- await _assert_request_failed(env, msg, error)
env.dag.run.side_effect = None
valid = env.message()
result = await asyncio.wait_for(env.processor.on_dequeue(valid.to_dict()), timeout=1)
assert result.outcome is ProcessOutcome.SUCCESS
- await asyncio.wait_for(env.tracker.wait_for_request(valid.telemetry_id), timeout=1)
env.retry_queue.enqueue.assert_not_awaited()
assert env.dag.run.await_count == 5
- assert env.tracker.build_queue_status(valid.telemetry_id)["Semantic"]["processed"] == 1
env.processor._account_breaker(valid.account_id).check()
@@ -233,8 +204,6 @@ async def test_lock_error_requeues_without_consuming_failure_budget(semantic_env
requeued = env.retry_queue.enqueue.call_args.args[0]
assert requeued.id == msg.id
assert requeued.failure_requeue_count == 0
- assert not env.tracker.is_complete(msg.telemetry_id)
- assert env.tracker.build_queue_status(msg.telemetry_id)["Semantic"]["requeue_count"] == 1
breaker.check()
sleep.assert_not_awaited()
@@ -251,8 +220,6 @@ async def test_semantic_model_error_fails_without_requeue(semantic_env):
assert result.outcome is ProcessOutcome.FAILED
assert result.error == str(error)
env.retry_queue.enqueue.assert_not_awaited()
- await _assert_request_failed(env, msg, error)
- env.processor._account_breaker(msg.account_id).check()
@pytest.mark.asyncio
@@ -267,10 +234,7 @@ async def test_invalid_request_does_not_open_breaker(semantic_env):
assert result.outcome is ProcessOutcome.FAILED
assert result.error == str(error)
env.retry_queue.enqueue.assert_not_awaited()
- await _assert_request_failed(env, msg, error)
- breaker = env.processor._account_breaker(msg.account_id)
- breaker.check()
- assert breaker._failure_count == 0
+ env.processor._account_breaker(msg.account_id).check()
@pytest.mark.asyncio
@@ -290,14 +254,12 @@ async def test_open_breaker_requeues_then_recovers(semantic_env, monkeypatch):
env.dag.run.assert_not_awaited()
env.retry_queue.enqueue.assert_awaited_once()
sleep.assert_awaited_once_with(30)
- assert not env.tracker.is_complete(msg.telemetry_id)
with pytest.raises(CircuitBreakerOpen):
breaker.check()
now[0] += 301
result = await asyncio.wait_for(env.processor.on_dequeue(msg.to_dict()), timeout=1)
assert result.outcome is ProcessOutcome.SUCCESS
- await asyncio.wait_for(env.tracker.wait_for_request(msg.telemetry_id), timeout=1)
env.dag.run.assert_awaited_once_with(msg.uri)
assert breaker.retry_after == 0
diff --git a/tests/storage/test_semantic_processor_target_preexisting.py b/tests/storage/test_semantic_processor_target_preexisting.py
index e3dc12e4b..278a67a82 100644
--- a/tests/storage/test_semantic_processor_target_preexisting.py
+++ b/tests/storage/test_semantic_processor_target_preexisting.py
@@ -6,7 +6,6 @@ import pytest
from openviking.storage.queuefs.semantic_msg import SemanticMsg
from openviking.storage.queuefs.semantic_processor import SemanticProcessor
from openviking.storage.viking_fs import SyncDiff
-from openviking.telemetry.request_wait_tracker import RequestWaitTracker
def _processor(**kwargs):
@@ -259,107 +258,6 @@ async def test_embedding_lock_wait_skips_semantic_execution(monkeypatch):
scope.close.assert_awaited_once()
-async def test_batch_lock_coordinator_requeues_while_sibling_work_is_pending(monkeypatch):
- monkeypatch.setattr(
- "openviking.storage.queuefs.semantic_processor.get_viking_fs",
- lambda: _FakeVikingFS(),
- )
- monkeypatch.setattr(
- "openviking.storage.queuefs.semantic_processor.is_semantic_msg_stale",
- lambda msg: False,
- )
- monkeypatch.setattr(
- "openviking.storage.queuefs.semantic_processor.asyncio.sleep",
- AsyncMock(),
- )
- tracker = RequestWaitTracker()
- telemetry_id = "tm_batch_coordinator_requeue"
- tracker.register_request(telemetry_id)
- tracker.register_semantic_root(telemetry_id, "coordinator")
- tracker.register_semantic_root(telemetry_id, "semantic-1")
- monkeypatch.setattr(
- "openviking.storage.queuefs.semantic_processor.get_request_wait_tracker",
- lambda: tracker,
- )
-
- work = SimpleNamespace(
- scope=SimpleNamespace(lock={"id": "batch-lock"}),
- start=lambda: None,
- resolve_telemetry=lambda: None,
- acquire_lock=AsyncMock(return_value=True),
- reenqueue=AsyncMock(),
- close=AsyncMock(),
- )
- processor = SemanticProcessor(vlm_resolver=None)
- monkeypatch.setattr(processor, "_message_work", lambda msg, lock=None: work)
- msg = SemanticMsg(
- uri="viking://resources/wiki",
- context_type="resource",
- telemetry_id=telemetry_id,
- lock_handoff={"owner_id": "producer"},
- wait_for_request_completion=True,
- )
- msg.id = "coordinator"
-
- result = await processor.on_dequeue(msg.to_dict())
-
- assert result.outcome.value == "requeued"
- work.reenqueue.assert_awaited_once()
- work.close.assert_awaited_once()
- assert tracker.build_queue_status(telemetry_id)["Semantic"]["requeue_count"] == 0
-
- tracker.mark_semantic_done(telemetry_id, "semantic-1")
- tracker.mark_semantic_done(telemetry_id, "coordinator")
- tracker.cleanup(telemetry_id)
-
-
-async def test_batch_lock_coordinator_releases_after_sibling_work_settles(monkeypatch):
- monkeypatch.setattr(
- "openviking.storage.queuefs.semantic_processor.get_viking_fs",
- lambda: _FakeVikingFS(),
- )
- monkeypatch.setattr(
- "openviking.storage.queuefs.semantic_processor.SemanticTreeExecutor",
- _FakeTreeExecutor,
- )
- scope = SimpleNamespace(lock={"id": "batch-lock"}, close=AsyncMock())
- monkeypatch.setattr(
- "openviking.storage.queuefs.semantic_processor.SemanticLockScope.resolve",
- AsyncMock(return_value=scope),
- )
- tracker = RequestWaitTracker()
- telemetry_id = "tm_batch_coordinator_release"
- tracker.register_request(telemetry_id)
- tracker.register_semantic_root(telemetry_id, "coordinator")
- monkeypatch.setattr(
- "openviking.storage.queuefs.semantic_processor.get_request_wait_tracker",
- lambda: tracker,
- )
-
- _FakeTreeExecutor.calls = []
- _FakeTreeExecutor.runs = []
- processor = SemanticProcessor(vlm_resolver=None)
- msg = SemanticMsg(
- uri="viking://resources/wiki",
- context_type="resource",
- telemetry_id=telemetry_id,
- lock_handoff={"owner_id": "producer"},
- wait_for_request_completion=True,
- )
- msg.id = "coordinator"
-
- result = await processor.on_dequeue(msg.to_dict())
-
- assert result.outcome.value == "success"
- assert _FakeTreeExecutor.calls == []
- scope.close.assert_awaited_once()
- assert tracker.is_complete(telemetry_id) is True
- assert tracker.build_queue_status(telemetry_id)["Semantic"]["processed"] == 0
- tracker.cleanup(telemetry_id)
-
-
@pytest.mark.asyncio
@pytest.mark.parametrize("recursive", [False, True])
async def test_memory_reindex_uses_semantic_executor(monkeypatch, recursive):
diff --git a/tests/storage/test_skill_package_cancellation.py b/tests/storage/test_skill_package_cancellation.py
index 7b2c3171a..a23ddcdaf 100644
--- a/tests/storage/test_skill_package_cancellation.py
+++ b/tests/storage/test_skill_package_cancellation.py
@@ -6,7 +6,6 @@ import json
import threading
from types import SimpleNamespace
from unittest.mock import AsyncMock
-from uuid import uuid4
import pytest
@@ -18,6 +17,7 @@ from openviking.service.task_work_index import (
bind_task_context,
get_task_context,
prepare_task_payload,
+ record_enqueued_work,
)
from openviking.storage.collection_schemas import TextEmbeddingHandler
from openviking.storage.queuefs.embedding_msg import EmbeddingMsg
@@ -26,7 +26,6 @@ from openviking.storage.queuefs.process_result import ProcessOutcome
from openviking.storage.queuefs.semantic_executor import SemanticTreeExecutor, SemanticTreeStats
from openviking.storage.queuefs.semantic_msg import SemanticMsg
from openviking.storage.queuefs.semantic_processor import SemanticProcessor
-from openviking.telemetry.request_wait_tracker import get_request_wait_tracker
from openviking_cli.session.user_id import UserIdentifier
from tests.storage.test_collection_schemas import _DummyConfig, _DummyEmbedder
from tests.storage.test_semantic_executor_skip_files import _FakeVikingFS
@@ -182,9 +181,6 @@ async def test_active_skill_embedding_cancel_settles_only_after_write_exit(
),
SimpleNamespace(bind=lambda _account_id: Embedder()),
)
- telemetry_id = str(uuid4())
- tracker = get_request_wait_tracker()
- tracker.register_request(telemetry_id)
msg = EmbeddingMsg(
"content",
{
@@ -192,15 +188,17 @@ async def test_active_skill_embedding_cancel_settles_only_after_write_exit(
"context_type": "skill",
"account_id": "acc",
},
- telemetry_id=telemetry_id,
)
- tracker.register_embedding_root(telemetry_id, msg.id)
index = TaskWorkIndex()
cancelling = set()
index.set_callbacks(
finalize_before_ack=AsyncMock(),
is_cancellation_requested=lambda task_id: task_id in cancelling,
)
+ monkeypatch.setattr(
+ "openviking.storage.collection_schemas.get_task_tracker",
+ lambda: SimpleNamespace(record_context_count=index.record_context_count),
+ )
with bind_task_context("skill-task", "acc", "alice"):
payload, metadata = prepare_task_payload(msg.to_dict())
queue = NamedQueue(
@@ -220,7 +218,6 @@ async def test_active_skill_embedding_cancel_settles_only_after_write_exit(
await asyncio.sleep(0.03)
if during_write:
assert not worker.done()
- assert not tracker.is_complete(telemetry_id)
assert index.has_work("skill-task")
assert not writes
else:
@@ -232,23 +229,26 @@ async def test_active_skill_embedding_cancel_settles_only_after_write_exit(
assert result.outcome is ProcessOutcome.CANCELLED
await index.prepare_ack(queue.name, envelope)
assert not index.has_work("skill-task")
- assert tracker.is_complete(telemetry_id)
- tracker.cleanup(telemetry_id)
assert len(writes) == int(during_write)
@pytest.mark.asyncio
async def test_semantic_cancel_drains_embeddings_before_releasing_package(monkeypatch):
- tracker = get_request_wait_tracker()
- telemetry_id = str(uuid4())
- tracker.register_request(telemetry_id)
+ index = TaskWorkIndex()
+ child = None
+ index.set_callbacks(
+ finalize_before_ack=AsyncMock(),
+ is_cancellation_requested=lambda _task_id: False,
+ )
+ monkeypatch.setattr(
+ "openviking.storage.queuefs.semantic_work.get_task_tracker",
+ lambda: SimpleNamespace(wait_for_work=index.wait_for_work),
+ )
msg = SemanticMsg(
uri="viking://agent/skills/demo",
context_type="skill",
- telemetry_id=telemetry_id,
propagate_to_parent=False,
)
- tracker.register_semantic_root(telemetry_id, msg.id)
started = asyncio.Event()
events = []
@@ -257,7 +257,12 @@ async def test_semantic_cancel_drains_embeddings_before_releasing_package(monkey
self.stale = False
async def run(self, uri):
- tracker.register_embedding_root(telemetry_id, "embedding")
+ nonlocal child
+ from openviking.service.task_work_index import QueueTaskMetadata
+
+ child = QueueTaskMetadata("skill-task", "embedding")
+ assert index.register("Embedding", child)
+ record_enqueued_work("Embedding", child.work_id)
started.set()
def get_stats(self):
@@ -280,9 +285,10 @@ async def test_semantic_cancel_drains_embeddings_before_releasing_package(monkey
lambda: SimpleNamespace(exists=AsyncMock(return_value=True)),
)
resolver = SimpleNamespace(get_vlm=AsyncMock())
- worker = asyncio.create_task(
- SemanticProcessor(vlm_resolver=resolver).on_dequeue({"data": msg.to_json()})
- )
+ with bind_task_context("skill-task", "acc", "alice", enqueued_work=[]):
+ worker = asyncio.create_task(
+ SemanticProcessor(vlm_resolver=resolver).on_dequeue({"data": msg.to_json()})
+ )
await asyncio.wait_for(started.wait(), 1)
worker.cancel()
await asyncio.sleep(0)
@@ -290,11 +296,9 @@ async def test_semantic_cancel_drains_embeddings_before_releasing_package(monkey
await asyncio.sleep(0.03)
assert not worker.done()
assert events == []
- assert not tracker.is_complete(telemetry_id)
events.append("embedding-settled")
- tracker.mark_embedding_done(telemetry_id, "embedding")
+ assert child is not None
+ index.record_result(child, "completed")
with pytest.raises(asyncio.CancelledError):
await asyncio.wait_for(worker, 1)
assert events == ["embedding-settled", "released"]
- assert tracker.is_complete(telemetry_id)
- tracker.cleanup(telemetry_id)
diff --git a/tests/storage/test_skill_package_semantics.py b/tests/storage/test_skill_package_semantics.py
deleted file mode 100644
index 7acf0d729..000000000
--- a/tests/storage/test_skill_package_semantics.py
+++ /dev/null
@@ -1,79 +0,0 @@
-import asyncio
-from types import SimpleNamespace
-from unittest.mock import AsyncMock
-
-import pytest
-
-from openviking.storage.queuefs.semantic_executor import SemanticTreeStats
-from openviking.storage.queuefs.semantic_msg import SemanticMsg
-from openviking.storage.queuefs.semantic_processor import SemanticProcessor
-from openviking.telemetry import get_current_telemetry, register_telemetry
-from openviking.telemetry.request_wait_tracker import get_request_wait_tracker
-
-
-async def test_skill_worker_keeps_package_locked_until_embeddings_finish(monkeypatch):
- root = "viking://agent/skills/demo"
- emitted = asyncio.Event()
- locked = True
- events = []
- tracker = get_request_wait_tracker()
- telemetry = get_current_telemetry()
- register_telemetry(telemetry)
- tracker.register_request(telemetry.telemetry_id)
- msg = SemanticMsg(
- uri=root,
- context_type="skill",
- telemetry_id=telemetry.telemetry_id,
- generation_trigger="skill_ingest",
- propagate_to_parent=False,
- )
- tracker.register_semantic_root(msg.telemetry_id, msg.id)
-
- class Lease:
- lock = {"lease_ref": "package"}
-
- async def close(self):
- nonlocal locked
- locked = False
- events.append("released")
-
- class Executor:
- def __init__(self, **kwargs):
- self.stale = False
-
- async def run(self, uri):
- tracker.register_embedding_root(msg.telemetry_id, "file-embedding")
- emitted.set()
-
- def get_stats(self):
- return SemanticTreeStats()
-
- monkeypatch.setattr(
- "openviking.storage.queuefs.semantic_processor.SemanticTreeExecutor", Executor
- )
- monkeypatch.setattr(
- "openviking.storage.queuefs.semantic_processor.SemanticLockScope.resolve",
- AsyncMock(return_value=Lease()),
- )
- monkeypatch.setattr(
- "openviking.storage.queuefs.semantic_processor.get_viking_fs",
- lambda: SimpleNamespace(exists=AsyncMock(return_value=True)),
- )
- resolver = SimpleNamespace(get_vlm=AsyncMock(return_value=SimpleNamespace()))
- worker = asyncio.create_task(
- SemanticProcessor(vlm_resolver=resolver).on_dequeue({"data": msg.to_json()})
- )
- await asyncio.wait_for(emitted.wait(), 1)
- assert locked and not worker.done() and not tracker.is_complete(msg.telemetry_id)
- # The HTTP timeout cleanup must not release the package while embedding runs.
- tracker.cleanup(msg.telemetry_id)
- await asyncio.sleep(0.06)
- assert locked and not worker.done() and not tracker.is_complete(msg.telemetry_id)
- # Deletion cannot pass the package lease while a file vector can still write.
- events.append("embedding-written")
- tracker.mark_embedding_done(msg.telemetry_id, "file-embedding", vector_written=True)
- await asyncio.wait_for(worker, 1)
- assert not locked and tracker.is_complete(msg.telemetry_id)
- assert events == ["embedding-written", "released"]
- tracker.cleanup(msg.telemetry_id)
diff --git a/tests/telemetry/test_request_wait_tracker.py b/tests/telemetry/test_request_wait_tracker.py
deleted file mode 100644
index e72a13b44..000000000
--- a/tests/telemetry/test_request_wait_tracker.py
+++ /dev/null
@@ -1,140 +0,0 @@
-# Copyright (c) 2026 Beijing Volcano Engine Technology Co., Ltd.
-# SPDX-License-Identifier: AGPL-3.0
-
-import pytest
-
-from openviking.telemetry.request_wait_tracker import RequestWaitTracker
-
-
-def test_request_wait_tracker_cleanup_prevents_state_recreation():
- tracker = RequestWaitTracker()
- telemetry_id = "tm_cleanup"
-
- tracker.register_request(telemetry_id)
- tracker.register_semantic_root(telemetry_id, "semantic-1")
- tracker.cleanup(telemetry_id)
-
- tracker.mark_semantic_done(telemetry_id, "semantic-1")
- tracker.mark_embedding_done(telemetry_id, "embedding-1")
-
- assert tracker.build_queue_status(telemetry_id) == {
- "Semantic": {"processed": 0, "requeue_count": 0, "error_count": 0, "errors": []},
- "Embedding": {"processed": 0, "requeue_count": 0, "error_count": 0, "errors": []},
- }
-
-
-def test_request_wait_tracker_cleanup_prevents_root_recreation():
- tracker = RequestWaitTracker()
- telemetry_id = "tm_late_root"
-
- tracker.register_request(telemetry_id)
- tracker.cleanup(telemetry_id)
-
- tracker.register_semantic_root(telemetry_id, "semantic-1")
- tracker.register_embedding_root(telemetry_id, "embedding-1")
-
- assert tracker.is_complete(telemetry_id) is True
- assert tracker.build_queue_status(telemetry_id) == {
- "Semantic": {"processed": 0, "requeue_count": 0, "error_count": 0, "errors": []},
- "Embedding": {"processed": 0, "requeue_count": 0, "error_count": 0, "errors": []},
- }
-
-
-def test_request_wait_tracker_records_requeues():
- tracker = RequestWaitTracker()
- telemetry_id = "tm_requeue"
-
- tracker.register_request(telemetry_id)
- tracker.record_semantic_requeue(telemetry_id)
- tracker.record_embedding_requeue(telemetry_id, delta=2)
-
- assert tracker.build_queue_status(telemetry_id) == {
- "Semantic": {"processed": 0, "requeue_count": 1, "error_count": 0, "errors": []},
- "Embedding": {"processed": 0, "requeue_count": 2, "error_count": 0, "errors": []},
- }
-
-
-def test_request_wait_tracker_completes_except_coordinator_after_siblings_settle():
- tracker = RequestWaitTracker()
- telemetry_id = "tm_batch_coordinator"
-
- tracker.register_request(telemetry_id)
- tracker.register_semantic_root(telemetry_id, "coordinator")
- tracker.register_semantic_root(telemetry_id, "semantic-1")
- tracker.register_embedding_root(telemetry_id, "embedding-1")
-
- assert tracker.is_complete_except(telemetry_id, "coordinator") is False
-
- tracker.mark_semantic_done(telemetry_id, "semantic-1")
- assert tracker.is_complete_except(telemetry_id, "coordinator") is False
-
- tracker.mark_embedding_done(telemetry_id, "embedding-1")
- assert tracker.is_complete_except(telemetry_id, "coordinator") is True
- assert tracker.is_complete(telemetry_id) is False
-
- tracker.mark_semantic_done(telemetry_id, "coordinator")
- tracker.cleanup(telemetry_id)
-
-
-def test_request_wait_tracker_retains_coordinator_after_request_cleanup():
- tracker = RequestWaitTracker()
- telemetry_id = "tm_batch_cleanup"
-
- tracker.register_request(telemetry_id)
- tracker.register_semantic_root(telemetry_id, "coordinator")
- tracker.retain_request(telemetry_id, "coordinator")
- tracker.cleanup(telemetry_id)
-
- assert tracker.has_request(telemetry_id) is True
-
- tracker.mark_semantic_done(telemetry_id, "coordinator")
-
- assert tracker.has_request(telemetry_id) is False
-
-
-def test_request_wait_tracker_counts_contexts_indexed_by_request():
- tracker = RequestWaitTracker()
- telemetry_id = "tm_vectors"
-
- tracker.register_request(telemetry_id)
- for root_id in ("embedding-1", "embedding-2", "embedding-3"):
- tracker.register_embedding_root(telemetry_id, root_id)
- tracker.mark_embedding_done(telemetry_id, root_id, vector_written=True)
- tracker.mark_embedding_done(telemetry_id, "embedding-3", vector_written=True)
-
- assert tracker.get_embedding_context_count(telemetry_id) == 3
-
- tracker.cleanup(telemetry_id)
-
- assert tracker.get_embedding_context_count(telemetry_id) == 0
-
-
-def test_request_wait_tracker_accumulates_queue_wait_and_execution_durations():
- tracker = RequestWaitTracker()
- telemetry_id = "tm_queue_timing"
-
- tracker.register_request(telemetry_id)
- tracker.record_semantic_timing(telemetry_id, queue_wait_ms=12.5, execute_ms=30.0)
- tracker.record_semantic_timing(telemetry_id, queue_wait_ms=7.5, execute_ms=10.0)
- tracker.record_embedding_timing(telemetry_id, queue_wait_ms=5.0, execute_ms=8.0)
-
- assert tracker.get_queue_timing(telemetry_id) == {
- "semantic": {"queue_wait_ms": 20.0, "execute_ms": 40.0},
- "embedding": {"queue_wait_ms": 5.0, "execute_ms": 8.0},
- }
-
-
-async def test_wait_for_request_timeout_preserves_pending_work():
- tracker = RequestWaitTracker()
- telemetry_id = "tm_wait_timeout"
-
- tracker.register_request(telemetry_id)
- tracker.register_embedding_root(telemetry_id, "embedding-1")
-
- with pytest.raises(TimeoutError):
- await tracker.wait_for_request(telemetry_id, timeout=0.01)
-
- assert not tracker.is_complete(telemetry_id)
- tracker.mark_embedding_done(telemetry_id, "embedding-1")
- await tracker.wait_for_request(telemetry_id)
- assert tracker.build_queue_status(telemetry_id)["Embedding"]["processed"] == 1
diff --git a/tests/telemetry/test_resource_summary.py b/tests/telemetry/test_resource_summary.py
deleted file mode 100644
index 076c644fb..000000000
--- a/tests/telemetry/test_resource_summary.py
+++ /dev/null
@@ -1,69 +0,0 @@
-from openviking.telemetry.operation import OperationTelemetry
-
-
-def test_record_resource_queue_metrics_collects_queue_and_tree_stats(monkeypatch):
- from openviking.telemetry.resource_summary import record_resource_queue_metrics
-
- telemetry = OperationTelemetry(operation="resources.add_resource", enabled=True)
- telemetry_id = telemetry.telemetry_id
-
- class _SemanticStats:
- processed = 7
- requeue_count = 0
- error_count = 2
-
- class _EmbeddingStats:
- processed = 11
- requeue_count = 0
- error_count = 1
-
- class _Timing:
- @staticmethod
- def get_queue_timing(_tid):
- return {
- "semantic": {"queue_wait_ms": 12.5, "execute_ms": 30.0},
- "embedding": {"queue_wait_ms": 8.0, "execute_ms": 20.0},
- }
-
- class _SemanticTreeStats:
- total_nodes = 9
- done_nodes = 8
- pending_nodes = 1
- in_progress_nodes = 0
-
- monkeypatch.setattr(
- "openviking.telemetry.resource_summary._consume_semantic_request_stats",
- lambda _tid: _SemanticStats(),
- )
- monkeypatch.setattr(
- "openviking.telemetry.resource_summary._consume_embedding_request_stats",
- lambda _tid: _EmbeddingStats(),
- )
- monkeypatch.setattr(
- "openviking.telemetry.resource_summary._consume_semantic_tree_stats",
- lambda _tid, _uri: _SemanticTreeStats(),
- )
- monkeypatch.setattr(
- "openviking.telemetry.resource_summary.get_request_wait_tracker",
- lambda: _Timing(),
- )
-
- record_resource_queue_metrics(
- telemetry=telemetry,
- telemetry_id=telemetry_id,
- root_uri="viking://resources/demo",
- )
-
- summary = telemetry.finish().summary
- assert summary["queue"]["semantic"]["processed"] == 7
- assert summary["queue"]["semantic"]["error_count"] == 2
- assert summary["queue"]["embedding"]["processed"] == 11
- assert summary["queue"]["embedding"]["error_count"] == 1
- assert summary["queue"]["semantic"]["queue_wait"]["duration_ms"] == 12.5
- assert summary["queue"]["semantic"]["execute"]["duration_ms"] == 30.0
- assert summary["queue"]["embedding"]["queue_wait"]["duration_ms"] == 8.0
- assert summary["queue"]["embedding"]["execute"]["duration_ms"] == 20.0
- assert summary["semantic_nodes"]["total"] == 9
- assert summary["semantic_nodes"]["done"] == 8
- assert summary["semantic_nodes"]["pending"] == 1
- assert "running" not in summary["semantic_nodes"]
diff --git a/tests/test_task_tracker.py b/tests/test_task_tracker.py
index f7ecfa33c..b322bfad3 100644
--- a/tests/test_task_tracker.py
+++ b/tests/test_task_tracker.py
@@ -5,6 +5,7 @@
import asyncio
import json
+import threading
import time
from copy import deepcopy
@@ -19,8 +20,15 @@ from openviking.service.task_tracker import (
TaskTracker,
_sanitize_error,
get_task_tracker,
+ run_tracked_operation,
set_task_tracker,
)
+from openviking.service.task_work_index import (
+ QueueTaskMetadata,
+ TaskWorkIndex,
+ get_task_context,
+)
+from openviking_cli.exceptions import DeadlineExceededError
from openviking_cli.session.user_id import UserIdentifier
pytestmark = pytest.mark.asyncio
@@ -620,6 +628,294 @@ async def test_get_task_tracker_requires_service_initialization():
get_task_tracker()
+async def test_wait_timeout_preserves_recorded_operation_result(tracker):
+ index = TaskWorkIndex()
+ tracker.attach_work_index(index)
+ set_task_tracker(tracker)
+ state = {}
+
+ async def operation():
+ context = get_task_context()
+ state["task_id"] = context.task_id
+ work = QueueTaskMetadata(
+ context.task_id,
+ "child-work",
+ context.account_id,
+ context.user_id,
+ )
+ assert index.register("Semantic", work)
+ return {"status": "success", "root_uri": "resource-result"}
+
+ with pytest.raises(DeadlineExceededError) as caught:
+ await run_tracked_operation(
+ "content_write",
+ account_id="acme",
+ user_id="alice",
+ operation=operation,
+ wait=True,
+ timeout=0,
+ )
+
+ task = await tracker.get(state["task_id"], **_owner_kwargs())
+ assert "task_id" not in caught.value.details
+ assert task.status is TaskStatus.RUNNING
+ assert task.result == {
+ "status": "success",
+ "root_uri": "resource-result",
+ "task_id": state["task_id"],
+ }
+
+
+async def test_accept_failed_result_returns_structured_partial_result(tracker):
+ index = TaskWorkIndex()
+ tracker.attach_work_index(index)
+ set_task_tracker(tracker)
+ workers = []
+
+ async def operation():
+ context = get_task_context()
+ work = QueueTaskMetadata(
+ context.task_id,
+ "embedding-work",
+ context.account_id,
+ context.user_id,
+ )
+ assert index.register("Embedding", work)
+
+ async def fail_work():
+ await asyncio.sleep(0)
+ index.record_result(work, "failed", error="embedding failed")
+ await index.prepare_ack("Embedding", work)
+
+ workers.append(asyncio.create_task(fail_work()))
+ return {
+ "status": "completed",
+ "failed_records": 1,
+ "warnings": ["Embedding queue failed during reindex: embedding failed"],
+ }
+
+ result, task_id = await run_tracked_operation(
+ "admin_reindex",
+ account_id="acme",
+ user_id="alice",
+ operation=operation,
+ wait=True,
+ accept_failed_result=lambda value: (
+ isinstance(value, dict) and value.get("failed_records", 0) > 0
+ ),
+ )
+
+ assert result["failed_records"] == 1
+ task = await tracker.wait(task_id, **_owner_kwargs(), timeout=1)
+ assert task.status is TaskStatus.FAILED
+ assert task.result["failed_records"] == 1
+ await asyncio.gather(*workers)
+
+
+ ("raised", "expected_message"),
+ [
+ (TimeoutError("operation timed out"), "operation timed out"),
+ (DeadlineExceededError("queue processing", 0.1), "Queue processing timed out after 0.1s"),
+ ],
+)
+async def test_operation_timeout_marks_task_failed(tracker, raised, expected_message):
+ set_task_tracker(tracker)
+ state = {}
+
+ async def operation():
+ state["task_id"] = get_task_context().task_id
+ raise raised
+
+ with pytest.raises(type(raised)):
+ await run_tracked_operation(
+ "content_write",
+ account_id="acme",
+ user_id="alice",
+ operation=operation,
+ wait=True,
+ timeout=0.1,
+ )
+
+ task = await tracker.get(state["task_id"], **_owner_kwargs())
+ assert task.status is TaskStatus.FAILED
+ assert task.error == expected_message
+
+
+async def test_wait_false_still_finishes_top_level_operation(tracker):
+ set_task_tracker(tracker)
+ started = asyncio.Event()
+ release = asyncio.Event()
+
+ async def operation():
+ started.set()
+ await release.wait()
+ return {"status": "success", "value": 1}
+
+ running = asyncio.create_task(
+ run_tracked_operation(
+ "content_write",
+ account_id="acme",
+ user_id="alice",
+ operation=operation,
+ wait=False,
+ )
+ )
+ await asyncio.wait_for(started.wait(), timeout=1)
+ assert not running.done()
+
+ release.set()
+ (result, task_id) = await asyncio.wait_for(running, timeout=1)
+ assert result == {"status": "success", "value": 1, "task_id": task_id}
+ completed = await tracker.get(task_id, **_owner_kwargs())
+ assert completed.status is TaskStatus.COMPLETED
+ assert completed.result == {
+ "status": "success",
+ "value": 1,
+ "task_id": task_id,
+ }
+
+
[email protected]("cancel_status", ["pending", "running"])
+async def test_cancel_during_operation_startup_settles_persisted_task(cancel_status):
+ """Cancellation must not orphan a task whose store write has already started."""
+ started = asyncio.Event()
+ release = threading.Event()
+ loop = asyncio.get_running_loop()
+
+ class BlockingAgfs(_FakeAgfs):
+ def write(self, path, data, *args, **kwargs):
+ if json.loads(data)["status"] == cancel_status:
+ loop.call_soon_threadsafe(started.set)
+ assert release.wait(3)
+ return super().write(path, data, *args, **kwargs)
+
+ agfs = BlockingAgfs()
+ tracker = TaskTracker(PersistentTaskStore(agfs))
+ set_task_tracker(tracker)
+ operation_called = False
+
+ async def operation():
+ nonlocal operation_called
+ operation_called = True
+ return {"ok": True}
+
+ running = asyncio.create_task(
+ run_tracked_operation("content_write", **_owner_kwargs(), operation=operation, wait=False)
+ )
+ try:
+ await asyncio.wait_for(started.wait(), 1)
+ running.cancel()
+ await asyncio.sleep(0)
+ running.cancel()
+ finally:
+ release.set()
+ with pytest.raises(asyncio.CancelledError):
+ await asyncio.wait_for(running, 1)
+
+ assert not operation_called
+ persisted = await TaskTracker(PersistentTaskStore(agfs)).list_tasks(**_owner_kwargs())
+ assert len(persisted) == 1
+ assert persisted[0].status is TaskStatus.FAILED
+ assert persisted[0].error == "CancelledError"
+ assert (
+ await tracker.wait(persisted[0].task_id, **_owner_kwargs(), timeout=0)
+ ).status is TaskStatus.FAILED
+
+
+async def test_operation_start_failure_settles_created_task():
+ class FailingAgfs(_FakeAgfs):
+ def write(self, path, data, *args, **kwargs):
+ if json.loads(data)["status"] == "running":
+ raise OSError("start write failed")
+ return super().write(path, data, *args, **kwargs)
+
+ tracker = TaskTracker(PersistentTaskStore(FailingAgfs()))
+ set_task_tracker(tracker)
+
+ async def operation():
+ pytest.fail("operation must not run after start fails")
+
+ with pytest.raises(OSError, match="start write failed"):
+ await run_tracked_operation(
+ "content_write", **_owner_kwargs(), operation=operation, wait=False
+ )
+
+ tasks = await tracker.list_tasks(**_owner_kwargs())
+ assert len(tasks) == 1
+ assert tasks[0].status is TaskStatus.FAILED
+ assert tasks[0].error == "start write failed"
+
+
[email protected]("timeout", [0, 0.01])
+async def test_wait_timeout_excludes_operation_time_for_completed_task(tracker, timeout):
+ set_task_tracker(tracker)
+
+ async def operation():
+ await asyncio.sleep(0.02)
+ return {"status": "success"}
+
+ result, task_id = await run_tracked_operation(
+ "content_write",
+ account_id="acme",
+ user_id="alice",
+ operation=operation,
+ wait=True,
+ timeout=timeout,
+ )
+
+ assert result == {"status": "success"}
+ completed = await tracker.wait(task_id, **_owner_kwargs(), timeout=0)
+ assert completed.status is TaskStatus.COMPLETED
+ assert completed.result["status"] == "success"
+ assert completed.result["task_id"] == task_id
+
+
+async def test_multiple_queue_waits_share_budget_and_leave_task_pollable(tracker):
+ index = TaskWorkIndex()
+ tracker.attach_work_index(index)
+ set_task_tracker(tracker)
+ state = {}
+
+ async def operation():
+ task_id = get_task_context().task_id
+ state["task_id"] = task_id
+ first = QueueTaskMetadata(task_id, "first", "acme", "alice")
+ assert index.register("Semantic", first)
+ # Exhaust the budget on a caught wait, then make the first Work complete.
+ with pytest.raises(TimeoutError):
+ await tracker.wait_for_work(task_id, [first.work_id], timeout=0.01)
+ index.record_result(first, "completed")
+ await index.prepare_ack("Semantic", first)
+ second = QueueTaskMetadata(task_id, "second", "acme", "alice")
+ assert index.register("Embedding", second)
+ state["second"] = second
+ with pytest.raises(TimeoutError):
+ await tracker.wait_for_task_work(task_id, timeout=60)
+ return {"status": "success"}
+
+ with pytest.raises(DeadlineExceededError) as caught:
+ await asyncio.wait_for(
+ run_tracked_operation(
+ "content_write",
+ **_owner_kwargs(),
+ operation=operation,
+ wait=True,
+ timeout=0.01,
+ ),
+ timeout=1,
+ )
+ assert "task_id" not in caught.value.details
+ task = await tracker.get(state["task_id"], **_owner_kwargs())
+ assert task.status is TaskStatus.RUNNING
+ assert task.error is None
+ index.record_result(state["second"], "completed")
+ await index.prepare_ack("Embedding", state["second"])
+ assert (
+ await tracker.wait(task.task_id, **_owner_kwargs(), timeout=0)
+ ).status is TaskStatus.COMPLETED
+
+
async def test_persistent_store_cross_tracker_visibility():
agfs = _FakeAgfs()
store = PersistentTaskStore(agfs)
@@ -896,27 +1192,11 @@ async def test_terminal_event_waits_for_owned_work(tracker, outcome):
)
snapshot = await tracker.get(task.task_id, **owner)
assert snapshot.status == TaskStatus.RUNNING
- waiting = asyncio.create_task(tracker.wait_for_descendants(task.task_id, "parent-work"))
-
- # Observe the public history while work remains, not a helper call sequence.
- async def observed_wait():
- while True:
- record = await tracker.get(task.task_id, **owner)
- if record.execution_events["items"][-1]["kind"] == "waiting_for_descendants":
- return
- await asyncio.sleep(0)
-
- try:
- await asyncio.wait_for(observed_wait(), timeout=2)
- finally:
- await work_index.prepare_ack("Semantic", child)
- await waiting
+ await work_index.prepare_ack("Semantic", child)
snapshot = await tracker.get(task.task_id, **owner)
last = snapshot.execution_events["items"][-1]
assert last["kind"] == "status_changed"
assert last["status"] == ("completed" if outcome == "complete" else "failed")
- await tracker.wait_for_descendants(task.task_id, "parent-work")
- assert (await tracker.get(task.task_id, **owner)).execution_events == snapshot.execution_events
async def test_legacy_tasks_do_not_get_invented_history():
diff --git a/tests/unit/test_summarizer_resources_root_split.py b/tests/unit/test_summarizer_resources_root_split.py
index cab5eed99..905525dc9 100644
--- a/tests/unit/test_summarizer_resources_root_split.py
+++ b/tests/unit/test_summarizer_resources_root_split.py
@@ -35,18 +35,6 @@ class _DummyQueueManager:
return self._queue
-class _DummyWaitTracker:
- def __init__(self):
- self.registered = []
- self.failed = []
-
- def register_semantic_root(self, telemetry_id, msg_id):
- self.registered.append((telemetry_id, msg_id))
-
- def mark_semantic_failed(self, telemetry_id, msg_id, error):
- self.failed.append((telemetry_id, msg_id, error))
-
-
class _DummyVikingFS:
def __init__(self, entries_by_uri):
self.entries_by_uri = entries_by_uri
@@ -63,7 +51,7 @@ async def test_resources_root_is_split_into_children():
{
"viking://temp/import_root": [
{"name": "existing_a", "isDir": True},
- {"name": "new_c", "isDir": True},
+ {"name": "file.txt", "isDir": False},
]
}
)
@@ -75,14 +63,11 @@ async def test_resources_root_is_split_into_children():
"openviking.utils.summarizer.get_current_telemetry",
return_value=SimpleNamespace(telemetry_id="tid"),
),
- patch(
- "openviking.utils.summarizer.get_request_wait_tracker", return_value=_DummyWaitTracker()
- ),
patch("openviking.utils.summarizer.get_viking_fs", return_value=vfs),
):
summarizer = Summarizer(vlm_processor=None)
res = await summarizer.summarize(
- resource_uris=["viking://resources"],
+ resource_uris=["viking://resources/"],
ctx=ctx,
temp_uris=["viking://temp/import_root"],
)
@@ -91,45 +76,14 @@ async def test_resources_root_is_split_into_children():
assert res["enqueued_count"] == 2
assert [m.target_uri for m in queue.msgs] == [
"viking://resources/existing_a",
- "viking://resources/new_c",
+ "viking://resources/file.txt",
]
assert [m.uri for m in queue.msgs] == [
"viking://temp/import_root/existing_a",
- "viking://temp/import_root/new_c",
+ "viking://temp/import_root/file.txt",
]
-async def test_resources_root_single_file_child():
- queue = _DummyQueue()
- qm = _DummyQueueManager(queue)
- vfs = _DummyVikingFS({"viking://temp/import_root": [{"name": "file.txt", "isDir": False}]})
- ctx = RequestContext(user=UserIdentifier.the_default_user(), role=Role.ROOT)
-
- with (
- patch("openviking.utils.summarizer.get_queue_manager", return_value=qm),
- patch(
- "openviking.utils.summarizer.get_current_telemetry",
- return_value=SimpleNamespace(telemetry_id="tid"),
- ),
- patch(
- "openviking.utils.summarizer.get_request_wait_tracker", return_value=_DummyWaitTracker()
- ),
- patch("openviking.utils.summarizer.get_viking_fs", return_value=vfs),
- ):
- summarizer = Summarizer(vlm_processor=None)
- res = await summarizer.summarize(
- resource_uris=["viking://resources/"],
- ctx=ctx,
- temp_uris=["viking://temp/import_root"],
- )
-
- assert res["status"] == "success"
- assert res["enqueued_count"] == 1
- assert queue.msgs[0].target_uri == "viking://resources/file.txt"
- assert queue.msgs[0].uri == "viking://temp/import_root/file.txt"
-
-
@pytest.mark.asyncio
async def test_explicit_subpath_not_split():
queue = _DummyQueue()
@@ -143,9 +97,6 @@ async def test_explicit_subpath_not_split():
"openviking.utils.summarizer.get_current_telemetry",
return_value=SimpleNamespace(telemetry_id="tid"),
),
- patch(
- "openviking.utils.summarizer.get_request_wait_tracker", return_value=_DummyWaitTracker()
- ),
patch("openviking.utils.summarizer.get_viking_fs", return_value=vfs),
):
summarizer = Summarizer(vlm_processor=None)
@@ -181,9 +132,6 @@ async def test_resources_root_empty_import_is_error():
"openviking.utils.summarizer.get_current_telemetry",
return_value=SimpleNamespace(telemetry_id="tid"),
),
- patch(
- "openviking.utils.summarizer.get_request_wait_tracker", return_value=_DummyWaitTracker()
- ),
patch("openviking.utils.summarizer.get_viking_fs", return_value=vfs),
):
summarizer = Summarizer(vlm_processor=None)
@@ -201,7 +149,6 @@ async def test_resources_root_empty_import_is_error():
async def test_flat_file_refresh_enqueues_incremental_parent_summary():
queue = _DummyQueue()
qm = _DummyQueueManager(queue)
- wait_tracker = _DummyWaitTracker()
ctx = RequestContext(user=UserIdentifier.the_default_user(), role=Role.ROOT)
with (
@@ -210,10 +157,6 @@ async def test_flat_file_refresh_enqueues_incremental_parent_summary():
"openviking.utils.summarizer.get_current_telemetry",
return_value=SimpleNamespace(telemetry_id="tid"),
),
- patch(
- "openviking.utils.summarizer.get_request_wait_tracker",
- return_value=wait_tracker,
- ),
):
result = await Summarizer(vlm_processor=None).refresh_file_parent(
file_uri="viking://resources/神雕.md",
@@ -234,14 +177,12 @@ async def test_flat_file_refresh_enqueues_incremental_parent_summary():
assert msg.skip_vectorization is False
assert msg.role == Role.ROOT
assert msg.telemetry_id == "tid"
- assert wait_tracker.registered == [("tid", msg.id)]
@pytest.mark.asyncio
async def test_grouped_file_refresh_coalesces_only_direct_parents():
queue = _DummyQueue()
qm = _DummyQueueManager(queue)
- wait_tracker = _DummyWaitTracker()
ctx = RequestContext(user=UserIdentifier.the_default_user(), role=Role.ROOT)
with (
@@ -250,10 +191,6 @@ async def test_grouped_file_refresh_coalesces_only_direct_parents():
"openviking.utils.summarizer.get_current_telemetry",
return_value=SimpleNamespace(telemetry_id="tid"),
),
- patch(
- "openviking.utils.summarizer.get_request_wait_tracker",
- return_value=wait_tracker,
- ),
):
result = await Summarizer(vlm_processor=None).refresh_file_parents(
changes=[
@@ -298,9 +235,8 @@ async def test_grouped_file_refresh_coalesces_only_direct_parents():
@pytest.mark.asyncio
-async def test_flat_file_refresh_marks_wait_failed_when_enqueue_fails():
+async def test_flat_file_refresh_propagates_enqueue_failure():
qm = _DummyQueueManager(_FailingQueue())
- wait_tracker = _DummyWaitTracker()
ctx = RequestContext(user=UserIdentifier.the_default_user(), role=Role.ROOT)
with (
@@ -309,18 +245,9 @@ async def test_flat_file_refresh_marks_wait_failed_when_enqueue_fails():
"openviking.utils.summarizer.get_current_telemetry",
return_value=SimpleNamespace(telemetry_id="tid"),
),
- patch(
- "openviking.utils.summarizer.get_request_wait_tracker",
- return_value=wait_tracker,
- ),
):
with pytest.raises(RuntimeError, match="queue unavailable"):
await Summarizer(vlm_processor=None).refresh_file_parent(
file_uri="viking://resources/神雕.md",
ctx=ctx,
)
-
- assert len(wait_tracker.registered) == 1
- assert wait_tracker.failed == [
- (*wait_tracker.registered[0], "queue unavailable"),
- ]
diff --git a/tests/unit/test_vectorize_file_strategy.py b/tests/unit/test_vectorize_file_strategy.py
index 059439063..1f92b93b8 100644
--- a/tests/unit/test_vectorize_file_strategy.py
+++ b/tests/unit/test_vectorize_file_strategy.py
@@ -356,82 +356,6 @@ async def test_vectorize_image_uses_supplied_bytes_without_second_read(monkeypat
assert fs.read_file_bytes_calls == 0
-async def test_vectorize_file_registers_request_wait_with_embedding_msg_id(monkeypatch):
- queue = DummyQueueWithId()
- registered = []
- monkeypatch.setattr(embedding_utils, "get_queue_manager", lambda: DummyQueueManager(queue))
- monkeypatch.setattr(embedding_utils, "get_viking_fs", lambda: DummyFS("hello"))
- monkeypatch.setattr(
- embedding_utils,
- "get_openviking_config",
- lambda: types.SimpleNamespace(
- embedding=types.SimpleNamespace(text_source="content_only", max_input_tokens=1000)
- ),
- )
- monkeypatch.setattr(
- embedding_utils,
- "get_request_wait_tracker",
- lambda: types.SimpleNamespace(
- register_embedding_root=lambda telemetry_id, root_id: registered.append(
- (telemetry_id, root_id)
- )
- ),
- )
-
- await embedding_utils.vectorize_file(
- file_path="viking://user/default/resources/test.md",
- summary_dict={"name": "test.md", "summary": ""},
- parent_uri="viking://user/default/resources",
- ctx=DummyReq(),
- )
-
- assert len(queue.items) == 1
- assert registered == [(queue.items[0].telemetry_id, queue.items[0].id)]
- assert registered[0][1] != "queue-message-id"
-
-
-async def test_vectorize_file_marks_registered_wait_root_failed_when_enqueue_raises(monkeypatch):
- queue = FailingQueue()
- registered = []
- failed = []
- monkeypatch.setattr(embedding_utils, "get_queue_manager", lambda: DummyQueueManager(queue))
- monkeypatch.setattr(embedding_utils, "get_viking_fs", lambda: DummyFS("hello"))
- monkeypatch.setattr(
- embedding_utils,
- "get_openviking_config",
- lambda: types.SimpleNamespace(
- embedding=types.SimpleNamespace(text_source="content_only", max_input_tokens=1000)
- ),
- )
- monkeypatch.setattr(
- embedding_utils,
- "get_request_wait_tracker",
- lambda: types.SimpleNamespace(
- register_embedding_root=lambda telemetry_id, root_id: registered.append(
- (telemetry_id, root_id)
- ),
- mark_embedding_failed=lambda telemetry_id, root_id, message: failed.append(
- (telemetry_id, root_id, message)
- ),
- ),
- )
-
- with pytest.raises(RuntimeError, match="queue unavailable"):
- await embedding_utils.vectorize_file(
- file_path="viking://user/default/resources/test.md",
- summary_dict={"name": "test.md", "summary": ""},
- parent_uri="viking://user/default/resources",
- ctx=DummyReq(),
- )
-
- assert len(queue.items) == 1
- assert registered == [(queue.items[0].telemetry_id, queue.items[0].id)]
- assert failed
- assert failed[0][0:2] == registered[0]
-
-
@pytest.mark.asyncio
async def test_vectorize_file_propagates_enqueue_failure(monkeypatch):
monkeypatch.setattr(