Self-evolving Context Database for AI Agents. Unify Agent Memory, Knowledge RAG and Skills.
Python agent-contextagent-memoryagent-pluginsagentic-ragcontext-databasedsh-pluginknowledge-basememorymemory-managementself-evolvingvector-database
refactor(queuefs): centralize queue processing lifecycle (#5019)
runyunzhou192b813e7e31
diff --git a/openviking/service/task_queue_middleware.py b/openviking/service/task_queue_middleware.pynew file mode 100644index 000000000..e7381cda6--- /dev/null+++ b/openviking/service/task_queue_middleware.py@@ -0,0 +1,100 @@+# Copyright (c) 2026 Beijing Volcano Engine Technology Co., Ltd.+# SPDX-License-Identifier: AGPL-3.0+"""Adapt the existing in-memory TaskWorkIndex to queue lifecycle middleware."""++import asyncio++from openviking.service.task_work_index import (+ TaskWorkIndex,+ TaskWorkRejected,+ bind_task_context,+ extract_task_metadata,+ prepare_task_payload,+)+from openviking.storage.queuefs.process_result import ProcessOutcome, ProcessResult+from openviking.storage.queuefs.queue_middleware import (+ AckContext,+ AckNext,+ ClearContext,+ ClearNext,+ EnqueueContext,+ EnqueueNext,+ ProcessContext,+ ProcessNext,+ QueueMiddleware,+)+++class TaskWorkQueueMiddleware(QueueMiddleware):+ """Keep QueueFS as the durable source of truth; no new task state or store."""++ def __init__(self, index: TaskWorkIndex) -> None:+ self._index = index++ async def enqueue(self, ctx: EnqueueContext, call_next: EnqueueNext) -> str:+ if not isinstance(ctx.payload, dict):+ return await call_next(ctx)+ ctx.payload, metadata = prepare_task_payload(ctx.payload)+ if not self._index.register(ctx.queue, metadata):+ raise TaskWorkRejected(+ f"Task {metadata.task_id} is cancelling; rejected work for {ctx.queue}"+ )+ try:+ return await call_next(ctx)+ finally:+ if metadata is not None and not ctx.committed:+ await self._index.discard(ctx.queue, metadata)++ async def process(self, ctx: ProcessContext, call_next: ProcessNext) -> ProcessResult:+ metadata = extract_task_metadata(ctx.message)+ if metadata is None:+ return await call_next(ctx)++ active_task = asyncio.current_task()+ with bind_task_context(metadata.task_id, metadata.account_id, metadata.user_id):+ if self._index.cancellation_requested(metadata.task_id):+ result = await ctx.cancel()+ if result.outcome is ProcessOutcome.FAILED:+ self._index.record_failure(metadata.task_id, result.error)+ return result+ if active_task is not None:+ self._index.register_active(metadata.task_id, active_task)+ try:+ result = await call_next(ctx)+ except asyncio.CancelledError:+ if self._index.cancellation_requested(metadata.task_id):+ # Handler unwinding must finish before this delivery is settled.+ return ProcessResult.cancelled()+ raise+ except Exception as exc:+ self._index.record_failure(metadata.task_id, str(exc))+ raise+ finally:+ if active_task is not None:+ self._index.unregister_active(metadata.task_id, active_task)+ if result.outcome is ProcessOutcome.FAILED:+ self._index.record_failure(metadata.task_id, result.error)+ return result++ async def ack(self, ctx: AckContext, call_next: AckNext) -> None:+ prepared = None+ try:+ if ctx.message is not None:+ prepared = await self._index.prepare_ack(ctx.queue, ctx.message)+ await call_next(ctx)+ finally:+ if prepared is not None and not ctx.committed:+ self._index.rollback_ack(ctx.queue, prepared)++ async def clear(self, ctx: ClearContext, call_next: ClearNext) -> None:+ prepared = []+ try:+ for message in ctx.messages:+ metadata = await self._index.prepare_ack(ctx.queue, message)+ if metadata is not None:+ prepared.append(metadata)+ await call_next(ctx)+ finally:+ if not ctx.committed:+ for metadata in prepared:+ self._index.rollback_ack(ctx.queue, metadata)diff --git a/openviking/service/task_tracker_concurrency.py b/openviking/service/task_tracker_concurrency.pyindex 4a6499e43..c79cd7c37 100644--- a/openviking/service/task_tracker_concurrency.py+++ b/openviking/service/task_tracker_concurrency.py@@ -63,8 +63,8 @@ async def run_to_completion(class OwnerLoopDispatcher:"""Run coroutine factories on one event loop, including foreign callers."""- def __init__(self) -> None:- self._owner_loop: asyncio.AbstractEventLoop | None = None+ def __init__(self, owner_loop: asyncio.AbstractEventLoop | None = None) -> None:+ self._owner_loop = owner_loopself._bind_lock = threading.Lock()def bind_current_loop(self) -> asyncio.AbstractEventLoop:diff --git a/openviking/service/user_deletion.py b/openviking/service/user_deletion.pyindex 7e8423d28..78bcc3ab6 100644--- a/openviking/service/user_deletion.py+++ b/openviking/service/user_deletion.py@@ -14,8 +14,10 @@ from openviking.core.namespace import canonical_user_rootfrom openviking.server.identity import RequestContext, Rolefrom openviking.service.task_store import SYSTEM_TASK_ACCOUNT_ID, SYSTEM_TASK_USER_IDfrom openviking.service.task_tracker import TaskStatus, get_task_tracker+from openviking.service.task_tracker_concurrency import OwnerLoopDispatcherfrom openviking.service.task_work_index import extract_task_metadatafrom openviking.storage.queuefs.named_queue import DequeueHandlerBase+from openviking.storage.queuefs.process_result import ProcessResultfrom openviking.storage.viking_fs import LS_ALL_NODESfrom openviking_cli.exceptions import NotFoundErrorfrom openviking_cli.session.user_id import UserIdentifier@@ -471,7 +473,7 @@ class _UserDeletionProcessor(DequeueHandlerBase):service_loop: asyncio.AbstractEventLoop,) -> None:self._deletion_service = deletion_service- self._service_loop = service_loop+ self._dispatcher = OwnerLoopDispatcher(service_loop)@staticmethoddef _parse_message(data: dict[str, Any]) -> dict[str, Any]:@@ -498,29 +500,16 @@ class _UserDeletionProcessor(DequeueHandlerBase):},}- async def on_dequeue(self, data: Optional[dict[str, Any]]) -> Optional[dict[str, Any]]:+ async def on_dequeue(self, data: Optional[dict[str, Any]]) -> ProcessResult:if not data:- return None+ return ProcessResult.success()try:message = self._parse_message(data)except (json.JSONDecodeError, TypeError, ValueError) as exc:- self.report_error(str(exc), data)- return None+ return ProcessResult.failed(str(exc))- future = asyncio.run_coroutine_threadsafe(- self._deletion_service._process(message),- self._service_loop,- )- try:- error = await asyncio.wrap_future(future)- except asyncio.CancelledError:- future.cancel()- raise- if error is None:- self.report_success()- else:- self.report_error(error, data)- return None+ error = await self._dispatcher.run(lambda: self._deletion_service._process(message))+ return ProcessResult.success() if error is None else ProcessResult.failed(error)async def setup_user_deletion(diff --git a/openviking/storage/collection_schemas.py b/openviking/storage/collection_schemas.pyindex b56c58046..80bd3f572 100644--- a/openviking/storage/collection_schemas.py+++ b/openviking/storage/collection_schemas.py@@ -27,6 +27,7 @@ from openviking.storage.errors import ()from openviking.storage.queuefs.embedding_msg import EmbeddingMsgfrom openviking.storage.queuefs.named_queue import DequeueHandlerBase+from openviking.storage.queuefs.process_result import ProcessResultfrom openviking.storage.vector_ids import vector_record_idfrom openviking.storage.viking_vector_index_backend import (VIKINGDB_CONTENT_MAX_SIZE,@@ -599,14 +600,12 @@ class TextEmbeddingHandler(DequeueHandlerBase):else inserted_data["abstract"][:VIKINGDB_CONTENT_MAX_SIZE])- async def on_dequeue(self, data: Optional[Dict[str, Any]]) -> Optional[Dict[str, Any]]:+ async def on_dequeue(self, data: Optional[Dict[str, Any]]) -> ProcessResult:"""Process dequeued message and add embedding vector(s)."""if not data:- return None+ return ProcessResult.success()embedding_msg: Optional[EmbeddingMsg] = None- report_success = False- report_error_args: Optional[tuple[str, Optional[Dict[str, Any]]]] = Nonerequest_failed_message: Optional[str] = Nonetry:embedding_msg = EmbeddingMsg.from_json(data["data"])@@ -628,8 +627,7 @@ class TextEmbeddingHandler(DequeueHandlerBase):logger.debug("Skip embedding dequeue during shutdown")self._merge_request_stats(embedding_msg.telemetry_id, processed=1)self._record_request_success(embedding_msg)- report_success = True- return None+ return ProcessResult.success()if not isinstance(embedding_msg.message, (str, list)):logger.debug(@@ -637,8 +635,7 @@ class TextEmbeddingHandler(DequeueHandlerBase):)self._merge_request_stats(embedding_msg.telemetry_id, processed=1)self._record_request_success(embedding_msg)- report_success = True- return data+ return ProcessResult.success(data)# Circuit breaker: if API is known-broken, re-enqueue and waittry:@@ -659,17 +656,14 @@ class TextEmbeddingHandler(DequeueHandlerBase):get_request_wait_tracker().record_embedding_requeue(embedding_msg.telemetry_id)- self.report_requeue()- report_success = True- return None+ return ProcessResult.requeued()# No queue manager — cannot re-enqueue, drop with errorerror_msg = self._embedding_error_msg(embedding_msg,"Circuit breaker open and no queue manager",)request_failed_message = error_msg- report_error_args = (error_msg, data)- return None+ return ProcessResult.failed(error_msg)# Initialize embedder if not already initializedif not self._embedder:@@ -719,16 +713,14 @@ class TextEmbeddingHandler(DequeueHandlerBase):logger.error(error_msg)self._merge_request_stats(embedding_msg.telemetry_id, error_count=1)request_failed_message = error_msg- report_error_args = (error_msg, data)- return None+ return ProcessResult.failed(error_msg)if error_class == ERROR_CLASS_PERMANENT:logger.critical(error_msg)self._circuit_breaker.record_failure(embed_err)self._merge_request_stats(embedding_msg.telemetry_id, error_count=1)request_failed_message = error_msg- report_error_args = (error_msg, data)- return None+ return ProcessResult.failed(error_msg)if error_class == ERROR_CLASS_AUTH:# Bad/expired credential: retrying cannot succeed. Fail@@ -740,8 +732,7 @@ class TextEmbeddingHandler(DequeueHandlerBase):logger.error(error_msg)self._merge_request_stats(embedding_msg.telemetry_id, error_count=1)request_failed_message = error_msg- report_error_args = (error_msg, data)- return None+ return ProcessResult.failed(error_msg)# Transient or unknown — re-enqueue for retrylogger.warning(error_msg)@@ -756,13 +747,11 @@ class TextEmbeddingHandler(DequeueHandlerBase):get_request_wait_tracker().record_embedding_requeue(embedding_msg.telemetry_id)- self.report_requeue()logger.info("Re-enqueued embedding message after transient error "f"({self._embedding_msg_log_context(embedding_msg)})")- report_success = True- return None+ return ProcessResult.requeued()except Exception as requeue_err:logger.error(self._embedding_error_msg(@@ -773,8 +762,7 @@ class TextEmbeddingHandler(DequeueHandlerBase):self._merge_request_stats(embedding_msg.telemetry_id, error_count=1)request_failed_message = error_msg- report_error_args = (error_msg, data)- return None+ return ProcessResult.failed(error_msg)# Add dense vectorif result.dense_vector:@@ -789,8 +777,7 @@ class TextEmbeddingHandler(DequeueHandlerBase):logger.error(error_msg)self._merge_request_stats(embedding_msg.telemetry_id, error_count=1)request_failed_message = error_msg- report_error_args = (error_msg, data)- return None+ return ProcessResult.failed(error_msg)# Add sparse vector if presentif result.sparse_vector:@@ -812,8 +799,7 @@ class TextEmbeddingHandler(DequeueHandlerBase):passself._merge_request_stats(embedding_msg.telemetry_id, error_count=1)request_failed_message = error_msg- report_error_args = (error_msg, data)- return None+ return ProcessResult.failed(error_msg)# Write to vector databasetry:@@ -847,8 +833,7 @@ class TextEmbeddingHandler(DequeueHandlerBase):logger.debug(f"Skip embedding write during shutdown: {db_err}")self._merge_request_stats(embedding_msg.telemetry_id, processed=1)self._record_request_success(embedding_msg)- report_success = True- return None+ return ProcessResult.success()error_msg = self._embedding_error_msg(embedding_msg,f"Failed to write to vector database: {db_err}",@@ -859,15 +844,13 @@ class TextEmbeddingHandler(DequeueHandlerBase):traceback.print_exc()self._merge_request_stats(embedding_msg.telemetry_id, error_count=1)request_failed_message = error_msg- report_error_args = (error_msg, data)- return None+ return ProcessResult.failed(error_msg)except Exception as db_err:if self._vikingdb.is_closing:logger.debug(f"Skip embedding write during shutdown: {db_err}")self._merge_request_stats(embedding_msg.telemetry_id, processed=1)self._record_request_success(embedding_msg)- report_success = True- return None+ return ProcessResult.success()error_msg = self._embedding_error_msg(embedding_msg,f"Failed to write to vector database: {db_err}",@@ -875,17 +858,15 @@ class TextEmbeddingHandler(DequeueHandlerBase):logger.error(error_msg)self._merge_request_stats(embedding_msg.telemetry_id, error_count=1)request_failed_message = error_msg- report_error_args = (error_msg, data)- return None+ return ProcessResult.failed(error_msg)self._merge_request_stats(embedding_msg.telemetry_id, processed=1)self._record_request_success(embedding_msg,vector_written=bool(record_id),)- report_success = Trueself._circuit_breaker.record_success()- return inserted_data+ return ProcessResult.success(inserted_data)except Exception as e:error_msg = self._embedding_error_msg(@@ -899,17 +880,12 @@ class TextEmbeddingHandler(DequeueHandlerBase):if embedding_msg is not None:self._merge_request_stats(embedding_msg.telemetry_id, error_count=1)request_failed_message = error_msg- report_error_args = (error_msg, data)- return None+ return ProcessResult.failed(error_msg)finally:if embedding_msg is not None and request_failed_message is not None:self._record_request_failure(embedding_msg, request_failed_message)- if report_error_args is not None:- self.report_error(*report_error_args)- elif report_success:- self.report_success()- async def on_cancelled(self, data: Optional[Dict[str, Any]]) -> Optional[Dict[str, Any]]:+ async def on_cancelled(self, data: Optional[Dict[str, Any]]) -> ProcessResult:"""Settle request-scoped waiting when a queued embedding is cancelled."""embedding_msg: Optional[EmbeddingMsg] = Nonetry:@@ -919,13 +895,11 @@ class TextEmbeddingHandler(DequeueHandlerBase):payload = json.loads(payload)embedding_msg = EmbeddingMsg.from_dict(payload)except (KeyError, TypeError, ValueError) as exc:- self.report_error(str(exc), data)- return None+ return ProcessResult.failed(str(exc))if embedding_msg is not None:self._record_request_success(embedding_msg)- self.report_success()- return None+ return ProcessResult.cancelled()@staticmethoddef _record_request_success(diff --git a/openviking/storage/queuefs/__init__.py b/openviking/storage/queuefs/__init__.pyindex 10a4d0ffa..241a9b3a3 100644--- a/openviking/storage/queuefs/__init__.py+++ b/openviking/storage/queuefs/__init__.py@@ -3,7 +3,9 @@from .embedding_msg import EmbeddingMsgfrom .embedding_queue import EmbeddingQueuefrom .named_queue import NamedQueue, QueueError, QueueStatus+from .process_result import ProcessOutcome, ProcessResultfrom .queue_manager import QueueManager, get_queue_manager, init_queue_manager+from .queue_middleware import QueueMiddlewarefrom .semantic_dag import SemanticDagExecutorfrom .semantic_msg import SemanticMsgfrom .semantic_processor import SemanticProcessor@@ -17,6 +19,9 @@ __all__ = ["NamedQueue","QueueStatus","QueueError",+ "QueueMiddleware",+ "ProcessOutcome",+ "ProcessResult","EmbeddingQueue","EmbeddingMsg","SemanticQueue",diff --git a/openviking/storage/queuefs/add_resource_processor.py b/openviking/storage/queuefs/add_resource_processor.pyindex 263c3f8ce..61e39fb26 100644--- a/openviking/storage/queuefs/add_resource_processor.py+++ b/openviking/storage/queuefs/add_resource_processor.py@@ -3,7 +3,6 @@"""Durable add-resource queue consumer."""import asyncio-import concurrent.futuresimport jsonfrom contextlib import suppressfrom copy import deepcopy@@ -12,9 +11,11 @@ from typing import Any, Dict, Optionalfrom openviking.observability.context import bind_execution_contextfrom openviking.server.identity import RequestContext, Rolefrom openviking.service.task_tracker import TaskStatus, get_task_tracker+from openviking.service.task_tracker_concurrency import OwnerLoopDispatcherfrom openviking.service.task_work_index import bind_task_context, extract_task_metadatafrom openviking.storage.queuefs.add_resource_msg import AddResourceMsgfrom openviking.storage.queuefs.named_queue import DequeueHandlerBase+from openviking.storage.queuefs.process_result import ProcessResultfrom openviking.telemetry import (OperationTelemetry,bind_telemetry,@@ -42,7 +43,7 @@ class AddResourceProcessor(DequeueHandlerBase):viking_fs: Any,):self._resource_service = resource_service- self._service_loop = service_loop+ self._dispatcher = OwnerLoopDispatcher(service_loop)self._queue_name = queue_nameself._viking_fs = viking_fs@@ -125,11 +126,9 @@ class AddResourceProcessor(DequeueHandlerBase):msg.task_id,exc,)- self.report_requeue()- self.report_success()return True- async def _process(self, msg: AddResourceMsg, data: Dict[str, Any]) -> None:+ async def _process(self, msg: AddResourceMsg, data: Dict[str, Any]) -> ProcessResult:telemetry_id = msg.telemetry_id or ""ctx = RequestContext(user=UserIdentifier(msg.account_id, msg.user_id),@@ -165,8 +164,7 @@ class AddResourceProcessor(DequeueHandlerBase):)await self._record_watch_execution(msg, status, getattr(task, "error", None))unregister_telemetry(telemetry_id)- self.report_success()- return None+ return ProcessResult.success()metadata = extract_task_metadata(data)replay_result = getattr(task, "result", None)@@ -176,7 +174,7 @@ class AddResourceProcessor(DequeueHandlerBase):resource_lock = await self._load_lock(msg, ctx)except Exception as exc:if await self._requeue_lock_handoff(msg, exc):- return None+ return ProcessResult.requeued()await tracker.fail(msg.task_id,f"Invalid lock_handoff: {exc}",@@ -188,11 +186,10 @@ class AddResourceProcessor(DequeueHandlerBase):"failed",f"Invalid lock_handoff: {exc}",)- self.report_error(f"Invalid lock_handoff: {exc}", data)unregister_telemetry(telemetry_id)with suppress(Exception):await self._cleanup_staged_source(msg, ctx)- return None+ return ProcessResult.failed(f"Invalid lock_handoff: {exc}")telemetry = resolve_telemetry(telemetry_id) if telemetry_id else Noneif telemetry is None:@@ -252,8 +249,7 @@ class AddResourceProcessor(DequeueHandlerBase):)await self._record_watch_execution(msg, "failed", error)terminal = True- self.report_error("resource processing failed", data)- return None+ return ProcessResult.failed("resource processing failed")if not msg.watch_task_id:await tracker.complete(msg.task_id,@@ -302,8 +298,7 @@ class AddResourceProcessor(DequeueHandlerBase):resource_id=result.get("root_uri"),)terminal = True- self.report_success()- return None+ return ProcessResult.success()except asyncio.CancelledError:await self._record_watch_execution(msg, "cancelled")terminal = True@@ -325,8 +320,7 @@ class AddResourceProcessor(DequeueHandlerBase):result=failure_result,)terminal = True- self.report_error(str(exc), data)- return None+ return ProcessResult.failed(str(exc))finally:request_wait_tracker.cleanup(telemetry_id)unregister_telemetry(telemetry_id)@@ -337,7 +331,7 @@ class AddResourceProcessor(DequeueHandlerBase):with suppress(Exception):await self._cleanup_staged_source(msg, ctx)- async def on_cancelled(self, data: Optional[Dict[str, Any]]) -> Optional[Dict[str, Any]]:+ async def on_cancelled(self, data: Optional[Dict[str, Any]]) -> ProcessResult:"""Release an enqueue-time lock before ACKing cancelled work."""try:payload = data.get("data", data) if isinstance(data, dict) else data@@ -345,10 +339,9 @@ class AddResourceProcessor(DequeueHandlerBase):payload = json.loads(payload)msg = AddResourceMsg.from_dict(payload)except (json.JSONDecodeError, TypeError, ValueError) as exc:- self.report_error(str(exc), data)- return None- future = asyncio.run_coroutine_threadsafe(- self._handle_cancelled(+ return ProcessResult.failed(str(exc))+ await self._dispatcher.run(+ lambda: self._handle_cancelled(msg,RequestContext(user=UserIdentifier(msg.account_id, msg.user_id),@@ -358,16 +351,13 @@ class AddResourceProcessor(DequeueHandlerBase):bypass_acl=msg.bypass_acl,),),- self._service_loop,)- await asyncio.wrap_future(future)unregister_telemetry(msg.telemetry_id or "")- self.report_success()- return None+ return ProcessResult.cancelled()- async def on_dequeue(self, data: Optional[Dict[str, Any]]) -> Optional[Dict[str, Any]]:+ async def on_dequeue(self, data: Optional[Dict[str, Any]]) -> ProcessResult:if not data:- return None+ return ProcessResult.success()try:if not isinstance(data, dict):raise ValueError("Queue message must be an object")@@ -376,16 +366,6 @@ class AddResourceProcessor(DequeueHandlerBase):payload = json.loads(payload)msg = AddResourceMsg.from_dict(payload)except (json.JSONDecodeError, TypeError, ValueError) as exc:- self.report_error(str(exc), data)- return None+ return ProcessResult.failed(str(exc))- future: concurrent.futures.Future[None] = asyncio.run_coroutine_threadsafe(- self._process(msg, data),- self._service_loop,- )- try:- await asyncio.wrap_future(future)- except asyncio.CancelledError:- future.cancel()- raise- return None+ return await self._dispatcher.run(lambda: self._process(msg, data))diff --git a/openviking/storage/queuefs/external_task_processor.py b/openviking/storage/queuefs/external_task_processor.pyindex 2480e5f66..7a97a8077 100644--- a/openviking/storage/queuefs/external_task_processor.py+++ b/openviking/storage/queuefs/external_task_processor.py@@ -13,6 +13,7 @@ from openviking.service.task_tracker_concurrency import OwnerLoopDispatcherfrom openviking.service.task_work_index import TaskWorkRejectedfrom openviking.storage.queuefs import QueueManager, get_queue_managerfrom openviking.storage.queuefs.named_queue import DequeueHandlerBase+from openviking.storage.queuefs.process_result import ProcessResultclass ExternalTaskProcessor(DequeueHandlerBase):@@ -40,14 +41,13 @@ class ExternalTaskProcessor(DequeueHandlerBase):raise ValueError("External task queue payload is missing task ownership")return task_id, account_id, user_id- async def on_dequeue(self, data: Optional[Dict[str, Any]]) -> Optional[Dict[str, Any]]:+ async def on_dequeue(self, data: Optional[Dict[str, Any]]) -> ProcessResult:if not data:- return None+ return ProcessResult.success()try:task_id, account_id, user_id = self._parse(data)except (json.JSONDecodeError, TypeError, ValueError) as exc:- self.report_error(str(exc), data)- return None+ return ProcessResult.failed(str(exc))processed = await self._dispatcher.run(lambda: self._service.execute(task_id, account_id, user_id))@@ -61,25 +61,22 @@ class ExternalTaskProcessor(DequeueHandlerBase):)except TaskWorkRejected:# Cancellation only needs the current delivery to be ACKed.- pass+ return ProcessResult.cancelled()else:- self.report_requeue()- self.report_success()- return None+ return ProcessResult.requeued()+ return ProcessResult.success()- async def on_cancelled(self, data: Optional[Dict[str, Any]]) -> Optional[Dict[str, Any]]:+ async def on_cancelled(self, data: Optional[Dict[str, Any]]) -> ProcessResult:if not data:- return None+ return ProcessResult.cancelled()try:task_id, account_id, user_id = self._parse(data)except (json.JSONDecodeError, TypeError, ValueError) as exc:- self.report_error(str(exc), data)- return None+ return ProcessResult.failed(str(exc))await self._dispatcher.run(lambda: self._service.cancel_recovered(task_id, account_id, user_id))- self.report_success()- return None+ return ProcessResult.cancelled()__all__ = ["ExternalTaskProcessor"]diff --git a/openviking/storage/queuefs/named_queue.py b/openviking/storage/queuefs/named_queue.pyindex 7090ddaa9..ee08d3449 100644--- a/openviking/storage/queuefs/named_queue.py+++ b/openviking/storage/queuefs/named_queue.py@@ -6,20 +6,24 @@ import jsonimport threadingfrom dataclasses import dataclass, fieldfrom datetime import datetime-from typing import Any, Callable, Dict, List, Optional, Union+from functools import partial+from typing import Any, Awaitable, Callable, Dict, List, Optional, Sequence, TypeVar, Unionfrom openviking.pyagfs import AGFSSyncClientProtocol, AsyncAGFSClientfrom openviking.pyagfs.exceptions import AGFSAlreadyExistsError, AGFSNotFoundError-from openviking.service.task_work_index import (- TaskWorkIndex,- TaskWorkRejected,- bind_task_context,- extract_task_metadata,- prepare_task_payload,+from openviking.storage.queuefs.process_result import ProcessOutcome, ProcessResult+from openviking.storage.queuefs.queue_middleware import (+ AckContext,+ ClearContext,+ EnqueueContext,+ ProcessContext,+ QueueMiddleware,)from openviking_cli.utils.logger import get_loggerlogger = get_logger(__name__)+_ContextT = TypeVar("_ContextT")+_ResultT = TypeVar("_ResultT")@dataclass@@ -51,63 +55,17 @@ class QueueStatus:return self.pending == 0 and self.in_progress == 0-class EnqueueHookBase(abc.ABC):- """Enqueue hook base class.-- All custom enqueue logic should inherit from this base class.- Provides on_enqueue method for custom processing before message enqueue.- """-- @abc.abstractmethod- async def on_enqueue(self, data: Union[str, Dict[str, Any]]) -> Union[str, Dict[str, Any]]:- """Called before message enqueue. Can modify data or perform validation."""- return data--class DequeueHandlerBase(abc.ABC):- """Dequeue handler base class, supports callback mechanism to report processing results."""-- _success_callback: Optional[Callable[[], None]] = None- _requeue_callback: Optional[Callable[[], None]] = None- _error_callback: Optional[Callable[[str, Optional[Dict[str, Any]]], None]] = None+ """Return a delivery outcome; queue statistics are never updated by handlers."""- def set_callbacks(- self,- on_success: Callable[[], None],- on_requeue: Callable[[], None],- on_error: Callable[[str, Optional[Dict[str, Any]]], None],- ) -> None:- """Set callback functions."""- self._success_callback = on_success- self._requeue_callback = on_requeue- self._error_callback = on_error-- def report_success(self) -> None:- """Report processing success."""- if self._success_callback:- self._success_callback()-- def report_requeue(self) -> None:- """Report that the current message was re-enqueued for later retry."""- if self._requeue_callback:- self._requeue_callback()-- def report_error(self, error_msg: str, data: Optional[Dict[str, Any]] = None) -> None:- """Report processing error."""- if self._error_callback:- self._error_callback(error_msg, data)-- async def on_cancelled(self, data: Optional[Dict[str, Any]]) -> Optional[Dict[str, Any]]:+ async def on_cancelled(self, data: Optional[Dict[str, Any]]) -> ProcessResult:"""Discard task work cancelled before its handler starts."""- self.report_success()- return None+ return ProcessResult.cancelled()@abc.abstractmethod- async def on_dequeue(self, data: Optional[Dict[str, Any]]) -> Optional[Dict[str, Any]]:- """Called after message dequeue. Returns None to discard message."""- if not data:- return None- return data+ async def on_dequeue(self, data: Optional[Dict[str, Any]]) -> ProcessResult:+ """Complete this delivery, or raise to leave it unacknowledged."""+ raise NotImplementedErrorclass NamedQueue:@@ -120,17 +78,15 @@ class NamedQueue:agfs: AGFSSyncClientProtocol,mount_point: str,name: str,- enqueue_hook: Optional[EnqueueHookBase] = None,dequeue_handler: Optional[DequeueHandlerBase] = None,- task_work_index: Optional[TaskWorkIndex] = None,+ middlewares: Sequence[QueueMiddleware] = (),):self.name = nameself.path = f"{mount_point}/{name}"self._agfs = agfsself._async_agfs = AsyncAGFSClient(agfs)- self._enqueue_hook = enqueue_hookself._dequeue_handler = dequeue_handler- self._task_work_index = task_work_index+ self._middlewares = tuple(middlewares)self._initialized = False# Status tracking@@ -140,35 +96,33 @@ class NamedQueue:self._error_count = 0self._errors: List[QueueError] = []- # Inject callbacks to handler- if self._dequeue_handler:- self.set_dequeue_handler(self._dequeue_handler)+ async def _run_middleware(+ self,+ operation: str,+ ctx: _ContextT,+ terminal: Callable[[_ContextT], Awaitable[_ResultT]],+ ) -> _ResultT:+ call_next = terminal+ for middleware in reversed(self._middlewares):+ call_next = partial(getattr(middleware, operation), call_next=call_next)+ return await call_next(ctx)def set_dequeue_handler(self, handler: DequeueHandlerBase) -> None:"""Bind the consumer after its runtime dependencies are initialized."""self._dequeue_handler = handler- handler.set_callbacks(- on_success=self._on_process_success,- on_requeue=self._on_process_requeue,- on_error=self._on_process_error,- )-- def _on_process_success(self) -> None:- """Called on processing success."""++ def _record_result(self, result: ProcessResult, data: Dict[str, Any]) -> None:+ if result.outcome is ProcessOutcome.FAILED:+ self._record_error(result.error, data)+ returnwith self._lock:+ # Keep the existing processed count: settled non-error deliveries,+ # including cancellations and successful re-enqueues.self._processed += 1+ if result.outcome is ProcessOutcome.REQUEUED:+ self._requeue_count += 1- def _on_process_requeue(self) -> None:- """Called when a dequeued message is re-enqueued for later retry."""- with self._lock:- self._requeue_count += 1-- def _on_process_error(self, error_msg: str, data: Optional[Dict[str, Any]] = None) -> None:- """Called on processing failure."""- if self._task_work_index is not None and data is not None:- metadata = extract_task_metadata(data)- if metadata is not None:- self._task_work_index.record_failure(metadata.task_id, error_msg)+ def _record_error(self, error_msg: str, data: Optional[Dict[str, Any]] = None) -> None:with self._lock:self._error_count += 1self._errors.append(@@ -218,66 +172,35 @@ class NamedQueue:async def enqueue(self, data: Union[str, Dict[str, Any]]) -> str:"""Send message to queue (enqueue)."""await self._ensure_initialized()- enqueue_file = f"{self.path}/enqueue"+ return await self._run_middleware("enqueue", EnqueueContext(self.name, data), self._enqueue)- # Execute enqueue hook- if self._enqueue_hook:- data = await self._enqueue_hook.on_enqueue(data)-- if isinstance(data, dict):- data, task_metadata = prepare_task_payload(data)- else:- task_metadata = None-- if self._task_work_index is not None and not self._task_work_index.register(- self.name, task_metadata- ):- logger.info(- "[NamedQueue] Skip enqueue for cancelling task %s on %s",- task_metadata.task_id,- self.name,- )- raise TaskWorkRejected(- f"Task {task_metadata.task_id} is cancelling; rejected work for {self.name}"- )-- try:- if isinstance(data, dict):- data = json.dumps(data)-- msg_id = await self._async_agfs.write(enqueue_file, data.encode("utf-8"))- except BaseException:- if self._task_work_index is not None and task_metadata is not None:- await self._task_work_index.discard(self.name, task_metadata)- raise+ async def _enqueue(self, ctx: EnqueueContext) -> str:+ data = json.dumps(ctx.payload) if isinstance(ctx.payload, dict) else ctx.payload+ msg_id = await self._async_agfs.write(f"{self.path}/enqueue", data.encode("utf-8"))+ ctx.committed = Truereturn msg_id if isinstance(msg_id, str) else str(msg_id)async def ack(self, msg_id: str, message: Optional[Dict[str, Any]] = None) -> None:"""Acknowledge successful processing of a message (deletes it from persistent storage).Must be called after the dequeue handler finishes processing a message.- Task-owned work is provisionally removed from the runtime index first so- the last message can persist its task's terminal state before deletion.+ Middleware can provisionally settle application state before deletion.If not called (e.g. process crashes), the message will be automaticallyre-queued on the next startup via RecoverStale."""if not msg_id:return- ack_file = f"{self.path}/ack"- prepared = Nonetry:- if self._task_work_index is not None and message is not None:- prepared = await self._task_work_index.prepare_ack(self.name, message)- await self._async_agfs.write(ack_file, msg_id.encode("utf-8"))+ await self._run_middleware("ack", AckContext(self.name, msg_id, message), self._ack)except asyncio.CancelledError:- if self._task_work_index is not None and prepared is not None:- self._task_work_index.rollback_ack(self.name, prepared)raiseexcept Exception as e:- if self._task_work_index is not None and prepared is not None:- self._task_work_index.rollback_ack(self.name, prepared)logger.warning(f"[NamedQueue] Ack failed for {self.name} msg_id={msg_id}: {e}")+ async def _ack(self, ctx: AckContext) -> None:+ await self._async_agfs.write(f"{self.path}/ack", ctx.message_id.encode("utf-8"))+ ctx.committed = True+async def _read_queue_message(self) -> Optional[Dict[str, Any]]:"""Read and remove one message from the AGFS queue; return parsed dict or None.@@ -312,11 +235,11 @@ class NamedQueue:data = await self._read_queue_message()if data is None:return None- # Capture message ID before passing data to handler (handler may modify it)msg_id = data.get("id", "") if isinstance(data, dict) else ""raw_data = dataif self._dequeue_handler:- data = await self.process_dequeued(data)+ result = await self.process_dequeued(raw_data)+ data = result.value# Ack unconditionally after handler returns (success or handled error).# If on_dequeue raises, the exception propagates and ack is skipped —# the message will be recovered on next startup.@@ -335,31 +258,32 @@ class NamedQueue:logger.debug(f"[NamedQueue] Dequeue raw failed for {self.name}: {e}")return None- async def process_dequeued(self, data: Dict[str, Any]) -> Optional[Dict[str, Any]]:- """Invoke the dequeue handler on already-fetched raw data."""- if self._dequeue_handler is None:- return data+ async def process_dequeued(self, data: Dict[str, Any]) -> ProcessResult:+ """Process one fetched delivery and settle local status exactly once."""+ handler = self._dequeue_handler+ if handler is None:+ return ProcessResult.success(data)- metadata = extract_task_metadata(data)- if metadata is None or self._task_work_index is None:- return await self._dequeue_handler.on_dequeue(data)+ async def process(ctx: ProcessContext) -> ProcessResult:+ return self._validate_result(await handler.on_dequeue(ctx.message))- active_task = asyncio.current_task()- with bind_task_context(metadata.task_id, metadata.account_id, metadata.user_id):- if self._task_work_index.cancellation_requested(metadata.task_id):- return await self._dequeue_handler.on_cancelled(data)- if active_task is not None:- self._task_work_index.register_active(metadata.task_id, active_task)- try:- return await self._dequeue_handler.on_dequeue(data)- except asyncio.CancelledError:- if self._task_work_index.cancellation_requested(metadata.task_id):- self._on_process_success()- return None- raise- finally:- if active_task is not None:- self._task_work_index.unregister_active(metadata.task_id, active_task)+ async def cancel() -> ProcessResult:+ return self._validate_result(await handler.on_cancelled(ctx.message))++ ctx = ProcessContext(self.name, data, cancel=cancel)+ try:+ result = self._validate_result(await self._run_middleware("process", ctx, process))+ except Exception as exc:+ self._record_error(str(exc), data)+ raise+ self._record_result(result, data)+ return result++ @staticmethod+ def _validate_result(result: ProcessResult) -> ProcessResult:+ if not isinstance(result, ProcessResult):+ raise TypeError("queue handlers and process middleware must return ProcessResult")+ return resultasync def peek(self) -> Optional[Dict[str, Any]]:"""Peek at head message without removing."""@@ -432,26 +356,17 @@ class NamedQueue:async def clear(self) -> bool:"""Clear queue."""await self._ensure_initialized()- clear_file = f"{self.path}/clear"-- messages = await self.snapshot() if self._task_work_index is not None else []- prepared = []+ messages = await self.snapshot() if self._middlewares else []+ ctx = ClearContext(self.name, messages)try:- if self._task_work_index is not None:- for message in messages:- metadata = await self._task_work_index.prepare_ack(self.name, message)- if metadata is not None:- prepared.append(metadata)- await self._async_agfs.write(clear_file, b"")- return True+ await self._run_middleware("clear", ctx, self._clear)+ return ctx.committedexcept asyncio.CancelledError:- if self._task_work_index is not None:- for metadata in prepared:- self._task_work_index.rollback_ack(self.name, metadata)raiseexcept Exception as e:- if self._task_work_index is not None:- for metadata in prepared:- self._task_work_index.rollback_ack(self.name, metadata)logger.error(f"[NamedQueue] Clear failed for {self.name}: {e}")return False++ async def _clear(self, ctx: ClearContext) -> None:+ await self._async_agfs.write(f"{self.path}/clear", b"")+ ctx.committed = Truediff --git a/openviking/storage/queuefs/process_result.py b/openviking/storage/queuefs/process_result.pynew file mode 100644index 000000000..455f4ab6c--- /dev/null+++ b/openviking/storage/queuefs/process_result.py@@ -0,0 +1,52 @@+# Copyright (c) 2026 Beijing Volcano Engine Technology Co., Ltd.+# SPDX-License-Identifier: AGPL-3.0+"""The outcome of one delivery, not the outcome of its owning business task."""++from dataclasses import dataclass+from enum import Enum+from typing import Any, Optional+++class ProcessOutcome(str, Enum):+ SUCCESS = "success"+ FAILED = "failed"+ REQUEUED = "requeued"+ CANCELLED = "cancelled"+++@dataclass(frozen=True)+class ProcessResult:+ """A settled delivery that may be ACKed after middleware completes.++ REQUEUED means the handler already enqueued a replacement. Exceptions,+ including unhandled cancellation, leave the current delivery unacknowledged.+ """++ outcome: ProcessOutcome+ value: Any = None+ error: Optional[str] = None++ def __post_init__(self) -> None:+ if not isinstance(self.outcome, ProcessOutcome):+ raise ValueError("outcome must be a ProcessOutcome")+ if self.outcome is ProcessOutcome.FAILED:+ if not isinstance(self.error, str):+ raise ValueError("failed results require an error string")+ elif self.error is not None:+ raise ValueError("only failed results may contain an error")++ @classmethod+ def success(cls, value: Any = None) -> "ProcessResult":+ return cls(ProcessOutcome.SUCCESS, value=value)++ @classmethod+ def failed(cls, error: str) -> "ProcessResult":+ return cls(ProcessOutcome.FAILED, error=error)++ @classmethod+ def requeued(cls) -> "ProcessResult":+ return cls(ProcessOutcome.REQUEUED)++ @classmethod+ def cancelled(cls) -> "ProcessResult":+ return cls(ProcessOutcome.CANCELLED)diff --git a/openviking/storage/queuefs/queue_manager.py b/openviking/storage/queuefs/queue_manager.pyindex 91cbb6a18..3249b3ca2 100644--- a/openviking/storage/queuefs/queue_manager.py+++ b/openviking/storage/queuefs/queue_manager.py@@ -10,13 +10,14 @@ import atexitimport threadingimport timeimport traceback-from typing import Any, Dict, Optional, Set, Union+from typing import Any, Dict, Optional, Sequence, Set, Unionfrom openviking.service.task_work_index import TaskWorkIndexfrom openviking_cli.utils.logger import get_loggerfrom .embedding_queue import EmbeddingQueue-from .named_queue import DequeueHandlerBase, EnqueueHookBase, NamedQueue, QueueStatus+from .named_queue import DequeueHandlerBase, NamedQueue, QueueStatus+from .queue_middleware import QueueMiddlewarefrom .semantic_queue import SemanticQueuelogger = get_logger(__name__)@@ -37,6 +38,8 @@ def init_queue_manager(max_concurrent_add_resource: int = 4,max_concurrent_session_commit: int = DEFAULT_MAX_CONCURRENT_SESSION_COMMIT,max_concurrent_external_task: int = 10,+ *,+ middlewares: Sequence[QueueMiddleware] = (),) -> "QueueManager":"""Initialize QueueManager singleton.@@ -49,6 +52,7 @@ def init_queue_manager(max_concurrent_external_parse: Max concurrent ExternalParse tasks.max_concurrent_add_resource: Max concurrent AddResource tasks.max_concurrent_session_commit: Max concurrent SessionCommit tasks.+ middlewares: Additional middleware, fixed at construction for all queues."""global _instance_instance = QueueManager(@@ -61,6 +65,7 @@ def init_queue_manager(max_concurrent_add_resource=max_concurrent_add_resource,max_concurrent_session_commit=max_concurrent_session_commit,max_concurrent_external_task=max_concurrent_external_task,+ middlewares=middlewares,)return _instance@@ -101,6 +106,8 @@ class QueueManager:max_concurrent_add_resource: int = 4,max_concurrent_session_commit: int = DEFAULT_MAX_CONCURRENT_SESSION_COMMIT,max_concurrent_external_task: int = 10,+ *,+ middlewares: Sequence[QueueMiddleware] = (),):"""Initialize QueueManager."""self._agfs = agfs@@ -118,6 +125,13 @@ class QueueManager:self._queue_stop_events: Dict[str, threading.Event] = {}self._poll_interval = 0.2self._task_work_index = TaskWorkIndex()+ # Import at composition time to avoid a service <-> queue package cycle.+ from openviking.service.task_queue_middleware import TaskWorkQueueMiddleware++ self._middlewares: tuple[QueueMiddleware, ...] = (+ TaskWorkQueueMiddleware(self._task_work_index),+ *middlewares,+ )atexit.register(self.stop)logger.info(@@ -276,8 +290,7 @@ class QueueManager:# Ack after successful processing (delete from persistent storage).await queue.ack(msg_id, data)except Exception as e:- # Do NOT ack — let RecoverStale re-queue on next startup.- queue._on_process_error(str(e), data)+ # The message remains in processing and will be recovered.logger.error(f"[QueueManager] Concurrent worker error for {queue.name}: {e}")while not stop_event.is_set():@@ -348,7 +361,6 @@ class QueueManager:def get_queue(self,name: str,- enqueue_hook: Optional[EnqueueHookBase] = None,dequeue_handler: Optional[DequeueHandlerBase] = None,allow_create: bool = False,) -> NamedQueue:@@ -361,27 +373,24 @@ class QueueManager:self._agfs,self.mount_point,name,- enqueue_hook=enqueue_hook,dequeue_handler=dequeue_handler,- task_work_index=self._task_work_index,+ middlewares=self._middlewares,)elif name == self.SEMANTIC:self._queues[name] = SemanticQueue(self._agfs,self.mount_point,name,- enqueue_hook=enqueue_hook,dequeue_handler=dequeue_handler,- task_work_index=self._task_work_index,+ middlewares=self._middlewares,)else:self._queues[name] = NamedQueue(self._agfs,self.mount_point,name,- enqueue_hook=enqueue_hook,dequeue_handler=dequeue_handler,- task_work_index=self._task_work_index,+ middlewares=self._middlewares,)if self._started:self._start_queue_worker(self._queues[name])diff --git a/openviking/storage/queuefs/queue_middleware.py b/openviking/storage/queuefs/queue_middleware.pynew file mode 100644index 000000000..c105b0ddd--- /dev/null+++ b/openviking/storage/queuefs/queue_middleware.py@@ -0,0 +1,80 @@+# Copyright (c) 2026 Beijing Volcano Engine Technology Co., Ltd.+# SPDX-License-Identifier: AGPL-3.0+"""Queue lifecycle extensions, independent of task tracking and storage backends."""++from dataclasses import dataclass+from typing import Any, Awaitable, Callable, Dict, List, Optional, Union++from .process_result import ProcessResult++Payload = Union[str, Dict[str, Any]]+Message = Dict[str, Any]+++@dataclass+class EnqueueContext:+ queue: str+ payload: Payload+ committed: bool = False+++@dataclass+class ProcessContext:+ queue: str+ message: Message+ cancel: Callable[[], Awaitable[ProcessResult]]+++@dataclass+class AckContext:+ queue: str+ message_id: str+ message: Optional[Message] = None+ committed: bool = False+++@dataclass+class ClearContext:+ queue: str+ messages: List[Message]+ committed: bool = False+++EnqueueNext = Callable[[EnqueueContext], Awaitable[str]]+ProcessNext = Callable[[ProcessContext], Awaitable[ProcessResult]]+AckNext = Callable[[AckContext], Awaitable[None]]+ClearNext = Callable[[ClearContext], Awaitable[None]]+++class QueueMiddleware:+ """Wrap operations in constructor-supplied order (first item is outermost).++ Queues copy the sequence to a tuple at construction; there is no runtime+ registration or reconfiguration.++ Override only the operations needed. Call ``call_next(ctx)`` once with the same+ context to continue, or return/raise to short-circuit. Exceptions unwind before+ the queue applies its existing error policy. Do not change message ownership+ after an outer middleware has registered it.+ Process must return a ProcessResult even when short-circuiting; the queue+ owns all delivery statistics and ACKs settled results.++ Contexts are invocation-local; middleware instances may be shared by queues+ and worker threads and must not store per-message state on themselves.+ Only the transport sets ``committed``. An outer middleware can compensate+ uncommitted operations, including a normal return from a short-circuit.+ It must not compensate an operation that already committed. Clear receives+ a pre-operation snapshot; it does not promise isolation from concurrent enqueue.+ """++ async def enqueue(self, ctx: EnqueueContext, call_next: EnqueueNext) -> str:+ return await call_next(ctx)++ async def process(self, ctx: ProcessContext, call_next: ProcessNext) -> ProcessResult:+ return await call_next(ctx)++ async def ack(self, ctx: AckContext, call_next: AckNext) -> None:+ await call_next(ctx)++ async def clear(self, ctx: ClearContext, call_next: ClearNext) -> None:+ await call_next(ctx)diff --git a/openviking/storage/queuefs/semantic_processor.py b/openviking/storage/queuefs/semantic_processor.pyindex 633b83bc7..d459c8da7 100644--- a/openviking/storage/queuefs/semantic_processor.py+++ b/openviking/storage/queuefs/semantic_processor.py@@ -45,6 +45,7 @@ from openviking.storage.abstract_overview import (from openviking.storage.acl import CreatorAclGrantfrom openviking.storage.errors import LockAcquisitionErrorfrom openviking.storage.queuefs.named_queue import DequeueHandlerBase+from openviking.storage.queuefs.process_result import ProcessResultfrom openviking.storage.queuefs.semantic_dag import DagStats, SemanticDagExecutorfrom openviking.storage.queuefs.semantic_lock import SemanticLockScopefrom openviking.storage.queuefs.semantic_msg import SemanticMsg, build_semantic_coalesce_key@@ -228,21 +229,18 @@ class SemanticProcessor(DequeueHandlerBase):async def _requeue_semantic_msg_after_error(self,msg: SemanticMsg,- data: Optional[Dict[str, Any]],error: Exception,- ) -> None:+ ) -> ProcessResult:try:await self._reenqueue_semantic_msg(msg)self._merge_request_stats(msg.telemetry_id, requeue_count=1)get_request_wait_tracker().record_semantic_requeue(msg.telemetry_id)- self.report_requeue()except Exception as requeue_err:logger.error(f"Failed to re-enqueue semantic message: {requeue_err}")self._merge_request_stats(msg.telemetry_id, error_count=1)get_request_wait_tracker().mark_semantic_failed(msg.telemetry_id, msg.id, str(error))- self.report_error(str(error), data)- return- self.report_success()+ return ProcessResult.failed(str(error))+ return ProcessResult.requeued()async def _enqueue_parent_refresh(self, msg: SemanticMsg, uri: str, *, l0_body_changed: bool@@ -329,7 +327,7 @@ class SemanticProcessor(DequeueHandlerBase):self,data: Optional[Dict[str, Any]],lock: Optional[Dict[str, Any]] = None,- ) -> Optional[Dict[str, Any]]:+ ) -> ProcessResult:"""Process dequeued SemanticMsg, recursively process all subdirectories."""msg: Optional[SemanticMsg] = Nonecollector = None@@ -337,7 +335,7 @@ class SemanticProcessor(DequeueHandlerBase):import jsonif not data:- return None+ return ProcessResult.success()if "data" in data and isinstance(data["data"], str):data = json.loads(data["data"])@@ -348,8 +346,7 @@ class SemanticProcessor(DequeueHandlerBase):logger.warning("Skipping semantic generation for root URI: %s", msg.uri)if msg.telemetry_id and msg.id:get_request_wait_tracker().mark_semantic_done(msg.telemetry_id, msg.id)- self.report_success()- return None+ return ProcessResult.success()if is_semantic_msg_stale(msg):live_file_changes = {kind: list(msg.changes.get(kind, []))@@ -377,8 +374,7 @@ class SemanticProcessor(DequeueHandlerBase):)if msg.telemetry_id and msg.id:get_request_wait_tracker().mark_semantic_done(msg.telemetry_id, msg.id)- self.report_success()- return None+ return ProcessResult.success()# Circuit breaker: if API is known-broken, re-enqueue and waittry:self._circuit_breaker.check()@@ -389,9 +385,7 @@ class SemanticProcessor(DequeueHandlerBase):await self._reenqueue_semantic_msg(msg)self._merge_request_stats(msg.telemetry_id, requeue_count=1)get_request_wait_tracker().record_semantic_requeue(msg.telemetry_id)- self.report_requeue()- self.report_success()- return None+ return ProcessResult.requeued()collector = resolve_telemetry(msg.telemetry_id)telemetry_ctx = bind_telemetry(collector) if collector is not None else nullcontext()with telemetry_ctx:@@ -519,9 +513,8 @@ class SemanticProcessor(DequeueHandlerBase):get_request_wait_tracker().mark_semantic_done(msg.telemetry_id, msg.id)self._merge_request_stats(msg.telemetry_id, processed=1)logger.info(f"Completed semantic generation for: {msg.uri}")- self.report_success()self._circuit_breaker.record_success()- return None+ return ProcessResult.success()finally:reset_root_observability_context(root_context_token)@@ -534,10 +527,8 @@ class SemanticProcessor(DequeueHandlerBase):exc_info=True,)if msg is not None:- await self._requeue_semantic_msg_after_error(msg, data, e)- else:- self.report_error(str(e), data)- return None+ return await self._requeue_semantic_msg_after_error(msg, e)+ return ProcessResult.failed(str(e))error_class = classify_api_error(e)if error_class == ERROR_CLASS_INPUT_TOO_LARGE:@@ -550,7 +541,7 @@ class SemanticProcessor(DequeueHandlerBase):get_request_wait_tracker().mark_semantic_failed(msg.telemetry_id, msg.id, str(e))- self.report_error(str(e), data)+ return ProcessResult.failed(str(e))elif error_class == ERROR_CLASS_PERMANENT:logger.critical(f"Permanent API error processing semantic message, dropping: {e}",@@ -562,7 +553,7 @@ class SemanticProcessor(DequeueHandlerBase):get_request_wait_tracker().mark_semantic_failed(msg.telemetry_id, msg.id, str(e))- self.report_error(str(e), data)+ return ProcessResult.failed(str(e))else:# Transient or unknown — re-enqueue for retrylogger.warning(@@ -571,12 +562,10 @@ class SemanticProcessor(DequeueHandlerBase):)self._circuit_breaker.record_failure(e)if msg is not None:- await self._requeue_semantic_msg_after_error(msg, data, e)- else:- self.report_error(str(e), data)- return None+ return await self._requeue_semantic_msg_after_error(msg, e)+ return ProcessResult.failed(str(e))- async def on_cancelled(self, data: Optional[Dict[str, Any]]) -> Optional[Dict[str, Any]]:+ async def on_cancelled(self, data: Optional[Dict[str, Any]]) -> ProcessResult:"""Release a queued semantic lock before cancelled work is ACKed."""try:import json@@ -586,8 +575,7 @@ class SemanticProcessor(DequeueHandlerBase):payload = json.loads(payload)msg = SemanticMsg.from_dict(payload)except (TypeError, ValueError) as exc:- self.report_error(str(exc), data)- return None+ return ProcessResult.failed(str(exc))if msg.telemetry_id and msg.id:get_request_wait_tracker().mark_semantic_done(msg.telemetry_id, msg.id)@@ -598,8 +586,7 @@ class SemanticProcessor(DequeueHandlerBase):await viking_fs._async_agfs.pathlock_release(lock)except Exception as exc:logger.warning("Failed to release cancelled semantic lock: %s", exc)- self.report_success()- return None+ return ProcessResult.cancelled()def get_dag_stats(self) -> Optional["DagStats"]:return SemanticDagExecutor.get_active_stats()diff --git a/openviking/storage/queuefs/session_commit_processor.py b/openviking/storage/queuefs/session_commit_processor.pyindex 9af90cbe6..8677f6400 100644--- a/openviking/storage/queuefs/session_commit_processor.py+++ b/openviking/storage/queuefs/session_commit_processor.py@@ -3,7 +3,6 @@"""Queue consumer for restart-safe Session Phase 2 work."""import asyncio-import concurrent.futuresimport jsonfrom typing import TYPE_CHECKING, Any, Dict, Optional@@ -13,8 +12,10 @@ from openviking.observability.context import ()from openviking.server.identity import RequestContext, Rolefrom openviking.service.task_tracker import get_task_tracker+from openviking.service.task_tracker_concurrency import OwnerLoopDispatcherfrom openviking.service.task_work_index import bind_task_contextfrom openviking.storage.queuefs.named_queue import DequeueHandlerBase+from openviking.storage.queuefs.process_result import ProcessResultfrom openviking.storage.queuefs.session_commit_msg import SessionCommitMsgfrom openviking.telemetry.span_models import create_root_span_attributesfrom openviking_cli.session.user_id import UserIdentifier@@ -30,7 +31,7 @@ class SessionCommitProcessor(DequeueHandlerBase):service_loop: asyncio.AbstractEventLoop,) -> None:self._session_service = session_service- self._service_loop = service_loop+ self._dispatcher = OwnerLoopDispatcher(service_loop)@staticmethoddef _parse_message(data: Dict[str, Any]) -> tuple[SessionCommitMsg, RequestContext]:@@ -47,9 +48,8 @@ class SessionCommitProcessor(DequeueHandlerBase):async def _process(self, msg: SessionCommitMsg, ctx: RequestContext) -> bool:# Bind a root observability context so Phase-2 extraction VLM/embedding# token events are attributed to the committing account/user rather than- # "__unknown__" (mirrors SemanticProcessor.on_dequeue). Must bind inside- # this coroutine: on_dequeue hops loops via run_coroutine_threadsafe, so- # a context bound there would not propagate here.+ # "__unknown__" (mirrors SemanticProcessor.on_dequeue). Bind inside the+ # dispatched operation to cover direct calls and restore the caller context.root_attrs = create_root_span_attributes(http_method="QUEUE",http_route="/queuefs/session_commit",@@ -92,7 +92,6 @@ class SessionCommitProcessor(DequeueHandlerBase):QueueManager.SESSION_COMMIT,msg.to_dict(),)- self.report_requeue()return processedfinally:reset_root_observability_context(root_context_token)@@ -106,45 +105,25 @@ class SessionCommitProcessor(DequeueHandlerBase):if await session.exists():await session.finalize_cancelled_commit(msg.archive_uri)- async def on_cancelled(self, data: Optional[Dict[str, Any]]) -> Optional[Dict[str, Any]]:+ async def on_cancelled(self, data: Optional[Dict[str, Any]]) -> ProcessResult:if not data:- return None+ return ProcessResult.cancelled()try:msg, ctx = self._parse_message(data)except (json.JSONDecodeError, KeyError, TypeError, ValueError) as exc:- self.report_error(str(exc), data)- return None+ return ProcessResult.failed(str(exc))- future = asyncio.run_coroutine_threadsafe(- self._finalize_cancelled(msg, ctx),- self._service_loop,- )- try:- await asyncio.wrap_future(future)- self.report_success()- except asyncio.CancelledError:- future.cancel()- raise- return None+ await self._dispatcher.run(lambda: self._finalize_cancelled(msg, ctx))+ return ProcessResult.cancelled()- async def on_dequeue(self, data: Optional[Dict[str, Any]]) -> Optional[Dict[str, Any]]:+ async def on_dequeue(self, data: Optional[Dict[str, Any]]) -> ProcessResult:if not data:- return None+ return ProcessResult.success()try:msg, ctx = self._parse_message(data)except (json.JSONDecodeError, KeyError, TypeError, ValueError) as exc:- self.report_error(str(exc), data)- return None- future: concurrent.futures.Future[bool] = asyncio.run_coroutine_threadsafe(- self._process(msg, ctx),- self._service_loop,- )- try:- await asyncio.wrap_future(future)- self.report_success()- except asyncio.CancelledError:- future.cancel()- raise- return None+ return ProcessResult.failed(str(exc))+ processed = await self._dispatcher.run(lambda: self._process(msg, ctx))+ return ProcessResult.success() if processed else ProcessResult.requeued()diff --git a/tests/server/test_admin_rebuild_api.py b/tests/server/test_admin_rebuild_api.pyindex a703d595c..b69be2a4b 100644--- a/tests/server/test_admin_rebuild_api.py+++ b/tests/server/test_admin_rebuild_api.py@@ -8,6 +8,7 @@ import pytestfrom openviking.core.context import ContextLevelfrom openviking.server.identity import RequestContext, Role+from openviking.storage.queuefs.process_result import ProcessResultfrom openviking_cli.exceptions import OpenVikingError, PermissionDeniedErrorfrom openviking_cli.session.user_id import UserIdentifierfrom tests.server.test_admin_api import ROOT_KEY@@ -25,6 +26,14 @@ ROOT_ACCOUNT_HEADERS = {}+def semantic_config(monkeypatch):+ monkeypatch.setattr(+ "openviking.service.reindex_executor.get_openviking_config",+ lambda: SimpleNamespace(vlm=SimpleNamespace(max_concurrent=2)),+ )++def _make_reindex_run(ctx, counters):from openviking.service.reindex_executor import _ReindexRunContext@@ -748,7 +757,9 @@ async def test_reindex_upsert_uses_uri_owner_for_user_scoped_records(monkeypatch@pytest.mark.asyncio-async def test_reindex_semantic_processor_uses_uri_owner_for_user_scoped_records(monkeypatch):+async def test_reindex_semantic_processor_uses_uri_owner_for_user_scoped_records(+ monkeypatch, semantic_config+):from openviking.service.reindex_executor import ReindexExecutorcaptured = {}@@ -760,6 +771,7 @@ async def test_reindex_semantic_processor_uses_uri_owner_for_user_scoped_recordsasync def on_dequeue(self, payload, lock=None):captured["payload"] = payloadcaptured["lock"] = lock+ return ProcessResult.success()monkeypatch.setattr("openviking.service.reindex_executor.SemanticProcessor",@@ -1363,6 +1375,7 @@ async def test_reindex_semantic_processor_uses_configured_vlm_concurrency(monkeyasync def on_dequeue(self, data, lock=None):seen["data"] = data+ return ProcessResult.success()monkeypatch.setattr(reindex_mod, "SemanticProcessor", FakeSemanticProcessor)monkeypatch.setattr(@@ -2131,7 +2144,9 @@ async def test_reindex_resource_vectors_parallelize_files_and_isolate_failures(m@pytest.mark.asyncio-async def test_reindex_semantic_processor_runs_with_skip_vectorization(monkeypatch):+async def test_reindex_semantic_processor_runs_with_skip_vectorization(+ monkeypatch, semantic_config+):from openviking.service.reindex_executor import ReindexExecutorseen = {}@@ -2142,6 +2157,7 @@ async def test_reindex_semantic_processor_runs_with_skip_vectorization(monkeypatasync def on_dequeue(self, payload, lock=None):seen["payload"] = payload+ return ProcessResult.success()monkeypatch.setattr("openviking.service.reindex_executor.SemanticProcessor",@@ -2170,7 +2186,9 @@ async def test_reindex_semantic_processor_runs_with_skip_vectorization(monkeypat@pytest.mark.asyncio-async def test_reindex_semantic_processor_passes_non_recursive_message(monkeypatch):+async def test_reindex_semantic_processor_passes_non_recursive_message(+ monkeypatch, semantic_config+):from openviking.service.reindex_executor import ReindexExecutorseen = {}@@ -2181,6 +2199,7 @@ async def test_reindex_semantic_processor_passes_non_recursive_message(monkeypatasync def on_dequeue(self, payload, lock=None):seen["payload"] = payload+ return ProcessResult.success()monkeypatch.setattr("openviking.service.reindex_executor.SemanticProcessor",@@ -2206,7 +2225,9 @@ async def test_reindex_semantic_processor_passes_non_recursive_message(monkeypat@pytest.mark.asyncio-async def test_reindex_semantic_processor_sets_memory_aggregation_policy(monkeypatch):+async def test_reindex_semantic_processor_sets_memory_aggregation_policy(+ monkeypatch, semantic_config+):from openviking.service.reindex_executor import ReindexExecutorseen = {}@@ -2217,6 +2238,7 @@ async def test_reindex_semantic_processor_sets_memory_aggregation_policy(monkeypasync def on_dequeue(self, payload, lock=None):seen["payload"] = payload+ return ProcessResult.success()monkeypatch.setattr("openviking.service.reindex_executor.SemanticProcessor",diff --git a/tests/service/test_task_queue_middleware.py b/tests/service/test_task_queue_middleware.pynew file mode 100644index 000000000..4a36bb19e--- /dev/null+++ b/tests/service/test_task_queue_middleware.py@@ -0,0 +1,488 @@+# Copyright (c) 2026 Beijing Volcano Engine Technology Co., Ltd.+# SPDX-License-Identifier: AGPL-3.0+"""Legacy task-index behavior through the middleware boundary."""++import asyncio+import json+from unittest.mock import AsyncMock++import pytest++from openviking.pyagfs import AsyncAGFSClient+from openviking.service.task_queue_middleware import TaskWorkQueueMiddleware+from openviking.service.task_work_index import (+ TaskWorkIndex,+ TaskWorkRejected,+ bind_task_context,+ extract_task_metadata,+ get_task_context,+)+from openviking.storage.queuefs.named_queue import DequeueHandlerBase, NamedQueue+from openviking.storage.queuefs.process_result import ProcessOutcome, ProcessResult+from openviking.storage.queuefs.queue_middleware import QueueMiddleware+from openviking.storage.queuefs.session_commit_processor import SessionCommitProcessor+++def tracked_queue(monkeypatch):+ transport = AsyncMock(spec=AsyncAGFSClient)+ transport.write.return_value = "message-1"+ # Tests override transport.read.return_value to feed /messages snapshots;+ # status reads must still return the backend JSON shape.+ transport.read.return_value = b"0"++ async def read(path):+ if path.endswith("/status"):+ return b'{"pending":0,"processing":0}'+ value = transport.read.return_value+ return value++ transport.read.side_effect = read+ monkeypatch.setattr(+ "openviking.storage.queuefs.named_queue.AsyncAGFSClient", lambda _client: transport+ )+ index = TaskWorkIndex()+ finalize = AsyncMock()+ cancelled = set()+ index.set_callbacks(+ finalize_before_ack=finalize,+ is_cancellation_requested=lambda task_id: task_id in cancelled,+ )+ queue = NamedQueue(object(), "/queue", "Test", middlewares=[TaskWorkQueueMiddleware(index)])+ return queue, index, transport, finalize, cancelled+++async def enqueue_task(queue, transport, task_id="task-1"):+ with bind_task_context(task_id, "account", "user"):+ await queue.enqueue({"value": 1})+ return {"id": "message-1", "data": transport.write.await_args.args[1].decode()}+++async def test_enqueue_registers_before_write_and_ack_finalizes_before_delete(tracked_queue):+ queue, index, transport, finalize, _ = tracked_queue++ async def write(path, data):+ assert path.endswith("/enqueue")+ assert index.has_work("task-1")+ return "message-1"++ transport.write.side_effect = write+ message = await enqueue_task(queue, transport)+ metadata = extract_task_metadata(message)+ assert metadata.task_id == "task-1"+ assert metadata.account_id == "account"+ assert metadata.user_id == "user"+ assert get_task_context() is None++ async def ack_write(path, data):+ assert path.endswith("/ack")+ assert not index.has_work("task-1")+ finalize.assert_awaited_once_with(metadata)+ return ""++ transport.write.side_effect = ack_write+ await queue.ack("message-1", message)+ assert not index.has_work("task-1")++[email protected]("failure", [OSError("write failed"), asyncio.CancelledError()])+async def test_enqueue_failure_discards_registered_work(tracked_queue, failure):+ queue, index, transport, finalize, _ = tracked_queue+ transport.write.side_effect = failure+ with bind_task_context("task-1", "account", "user"):+ with pytest.raises(type(failure)):+ await queue.enqueue({})+ assert not index.has_work("task-1")+ finalize.assert_awaited_once()+++async def test_cancelled_task_rejects_enqueue_without_write(tracked_queue):+ queue, index, transport, _, cancelled = tracked_queue+ cancelled.add("task-1")+ with bind_task_context("task-1", "account", "user"):+ with pytest.raises(TaskWorkRejected, match="cancelling"):+ await queue.enqueue({})+ transport.write.assert_not_awaited()+ assert not index.has_work("task-1")++[email protected]("operation", ["ack", "clear"])[email protected]("failure", [OSError("write failed"), asyncio.CancelledError()])+async def test_failed_ack_or_clear_restores_work(tracked_queue, operation, failure):+ queue, index, transport, finalize, _ = tracked_queue+ message = await enqueue_task(queue, transport)+ transport.read.return_value = json.dumps([message]).encode()+ transport.write.reset_mock()+ transport.write.side_effect = failure++ async def invoke():+ if operation == "ack":+ return await queue.ack("message-1", message)+ return await queue.clear()++ if isinstance(failure, asyncio.CancelledError):+ with pytest.raises(asyncio.CancelledError):+ await invoke()+ elif operation == "clear":+ assert await invoke() is False+ else:+ assert await invoke() is None+ finalize.assert_awaited_once()+ assert index.has_work("task-1")++[email protected]("operation", ["ack", "clear"])+async def test_finalization_failure_prevents_transport_delete(tracked_queue, operation):+ queue, index, transport, finalize, _ = tracked_queue+ message = await enqueue_task(queue, transport)+ transport.read.return_value = json.dumps([message]).encode()+ transport.write.reset_mock()+ finalize.side_effect = OSError("task store unavailable")+ if operation == "ack":+ await queue.ack("message-1", message)+ else:+ assert await queue.clear() is False+ transport.write.assert_not_awaited()+ assert index.has_work("task-1")++[email protected]("operation", ["enqueue", "ack", "clear"])+async def test_inner_short_circuit_compensates_uncommitted_index_change(tracked_queue, operation):+ queue, index, transport, _, _ = tracked_queue++ class ShortCircuit(QueueMiddleware):+ async def invoke(self, ctx, call_next):+ return "deduplicated"++ setattr(ShortCircuit, operation, ShortCircuit.invoke)+ queue = NamedQueue(+ object(), "/queue", "Test", middlewares=[TaskWorkQueueMiddleware(index), ShortCircuit()]+ )+ message = await enqueue_task(queue, transport) if operation != "enqueue" else None+ transport.write.reset_mock()+ if operation == "enqueue":+ with bind_task_context("task-1", "account", "user"):+ assert await queue.enqueue({}) == "deduplicated"+ assert not index.has_work("task-1")+ elif operation == "ack":+ await queue.ack("message-1", message)+ assert index.has_work("task-1")+ else:+ transport.read.return_value = json.dumps([message]).encode()+ await queue.clear()+ assert index.has_work("task-1")+ transport.write.assert_not_awaited()++[email protected]("operation", ["enqueue", "ack", "clear"])+async def test_post_commit_error_does_not_undo_index_change(tracked_queue, operation):+ queue, index, transport, _, _ = tracked_queue++ class FailAfter(QueueMiddleware):+ async def invoke(self, ctx, call_next):+ await call_next(ctx)+ raise RuntimeError("after commit")++ setattr(FailAfter, operation, FailAfter.invoke)+ queue = NamedQueue(+ object(), "/queue", "Test", middlewares=[TaskWorkQueueMiddleware(index), FailAfter()]+ )+ if operation == "enqueue":+ with bind_task_context("task-1", "account", "user"):+ with pytest.raises(RuntimeError, match="after commit"):+ await queue.enqueue({})+ assert index.has_work("task-1")+ else:+ message = await enqueue_task(queue, transport)+ transport.read.return_value = json.dumps([message]).encode()+ if operation == "ack":+ await queue.ack("message-1", message)+ else:+ assert await queue.clear() is False+ assert not index.has_work("task-1")+++async def test_clear_rolls_back_all_prepared_messages(tracked_queue):+ queue, index, transport, finalize, _ = tracked_queue+ first = await enqueue_task(queue, transport, "first")+ second = await enqueue_task(queue, transport, "second")+ transport.read.return_value = json.dumps([first, second]).encode()++ async def finish(metadata):+ if metadata.task_id == "second":+ raise OSError("second task failed")++ finalize.side_effect = finish+ transport.write.reset_mock()+ assert await queue.clear() is False+ transport.write.assert_not_awaited()+ assert index.has_work("first") and index.has_work("second")+ finalize.side_effect = None+ assert await queue.clear()+ assert not index.has_work("first") and not index.has_work("second")+++async def test_process_result_tracks_children_and_errors(tracked_queue):+ queue, index, transport, _, _ = tracked_queue+ message = await enqueue_task(queue, transport)+ contexts = []++ class Handler(DequeueHandlerBase):+ async def on_dequeue(self, data):+ contexts.append(get_task_context())+ await queue.enqueue({"child": True})+ return ProcessResult.failed("failed work")++ queue.set_dequeue_handler(Handler())+ assert (await queue.process_dequeued(message)).outcome is ProcessOutcome.FAILED+ assert contexts[0].task_id == "task-1"+ assert get_task_context() is None+ assert index.failure("task-1") == "failed work"+ child_payload = next(+ call.args[1]+ for call in reversed(transport.write.await_args_list)+ if call.args[0].endswith("/enqueue")+ )+ child = {"id": "child", "data": child_payload.decode()}+ assert index.has_work("task-1")+ await queue.ack("message-1", message)+ assert index.has_work("task-1")+ await queue.ack("child", child)+ assert not index.has_work("task-1")+ status = await queue.get_status()+ assert status.error_count == 1+ assert status.in_progress == 0+++async def test_handler_exception_records_task_failure_and_propagates(tracked_queue):+ queue, index, transport, _, _ = tracked_queue+ message = await enqueue_task(queue, transport)++ class Handler(DequeueHandlerBase):+ async def on_dequeue(self, data):+ raise RuntimeError("handler failed")++ queue.set_dequeue_handler(Handler())+ with pytest.raises(RuntimeError, match="handler failed"):+ await queue.process_dequeued(message)+ assert index.failure("task-1") == "handler failed"+ assert index.has_work("task-1")+++async def test_concurrent_results_update_each_task(tracked_queue):+ queue, index, transport, _, _ = tracked_queue+ first = await enqueue_task(queue, transport, "first")+ second = await enqueue_task(queue, transport, "second")+ both_started = asyncio.Event()+ started = 0++ class Handler(DequeueHandlerBase):+ async def on_dequeue(self, data):+ nonlocal started+ started += 1+ if started == 2:+ both_started.set()+ await both_started.wait()+ task_id = get_task_context().task_id+ return ProcessResult.failed(f"{task_id} failed")++ queue.set_dequeue_handler(Handler())+ await asyncio.wait_for(+ asyncio.gather(queue.process_dequeued(first), queue.process_dequeued(second)),+ timeout=1,+ )+ assert index.failure("first") == "first failed"+ assert index.failure("second") == "second failed"+++async def test_cancel_before_handler_uses_legacy_discard_callback(tracked_queue):+ queue, index, transport, _, cancelled = tracked_queue+ message = await enqueue_task(queue, transport)+ cancelled.add("task-1")+ calls = []++ class Handler(DequeueHandlerBase):+ async def on_dequeue(self, data):+ pytest.fail("cancelled task reached handler")++ async def on_cancelled(self, data):+ calls.append(data)+ return await super().on_cancelled(data)++ queue.set_dequeue_handler(Handler())+ assert (await queue.process_dequeued(message)).outcome is ProcessOutcome.CANCELLED+ assert calls == [message]+ await queue.ack("message-1", message)+ assert not index.has_work("task-1")+ status = await queue.get_status()+ assert status.processed == 1 and status.in_progress == 0++[email protected]("user_cancel", [True, False])+async def test_active_cancellation_distinguishes_user_from_shutdown(tracked_queue, user_cancel):+ queue, index, transport, _, cancelled = tracked_queue+ message = await enqueue_task(queue, transport)+ started = asyncio.Event()++ class Handler(DequeueHandlerBase):+ async def on_dequeue(self, data):+ started.set()+ await asyncio.Event().wait()++ queue.set_dequeue_handler(Handler())+ running = asyncio.create_task(queue.process_dequeued(message))+ await asyncio.wait_for(started.wait(), timeout=1)+ try:+ if user_cancel:+ cancelled.add("task-1")+ index.cancel_active("task-1")+ result = await asyncio.wait_for(running, timeout=1)+ assert result.outcome is ProcessOutcome.CANCELLED+ await queue.ack("message-1", message)+ assert not index.has_work("task-1")+ assert (await queue.get_status()).processed == 1+ else:+ running.cancel()+ with pytest.raises(asyncio.CancelledError):+ await running+ assert index.has_work("task-1")+ finally:+ if not running.done():+ running.cancel()+ await asyncio.gather(running, return_exceptions=True)+++async def test_shared_middleware_keeps_concurrent_contexts_isolated(tracked_queue):+ queue, index, transport, _, _ = tracked_queue+ first = await enqueue_task(queue, transport, "first")+ second = await enqueue_task(queue, transport, "second")+ both_started = asyncio.Event()+ seen = []++ class Handler(DequeueHandlerBase):+ async def on_dequeue(self, data):+ before = get_task_context().task_id+ seen.append(before)+ if len(seen) == 2:+ both_started.set()+ await both_started.wait()+ assert get_task_context().task_id == before+ return ProcessResult.success(data)++ queue.set_dequeue_handler(Handler())+ await asyncio.wait_for(+ asyncio.gather(queue.process_dequeued(first), queue.process_dequeued(second)), timeout=1+ )+ assert set(seen) == {"first", "second"}+ assert get_task_context() is None+ await queue.ack("first", first)+ await queue.ack("second", second)+ assert not index.has_work("first") and not index.has_work("second")+++async def test_awaited_child_result_is_recorded_before_ack(tracked_queue):+ queue, index, transport, finalize, _ = tracked_queue+ message = await enqueue_task(queue, transport)++ class Handler(DequeueHandlerBase):+ async def on_dequeue(self, data):+ async def child():+ await asyncio.sleep(0)+ return ProcessResult.failed("child failure")++ return await asyncio.create_task(child())++ async def finish(metadata):+ assert index.failure(metadata.task_id) == "child failure"++ finalize.side_effect = finish+ queue.set_dequeue_handler(Handler())+ await queue.process_dequeued(message)+ finalize.assert_not_awaited()+ await queue.ack("message-1", message)+ finalize.assert_awaited_once()+++async def test_cancel_cleanup_interruption_does_not_ack(tracked_queue):+ queue, index, transport, _, cancelled = tracked_queue+ message = await enqueue_task(queue, transport)+ cancelled.add("task-1")+ transport.write.reset_mock()++ class Handler(DequeueHandlerBase):+ async def on_dequeue(self, data):+ pytest.fail("cancelled work must not execute")++ async def on_cancelled(self, data):+ raise asyncio.CancelledError++ queue.set_dequeue_handler(Handler())+ with pytest.raises(asyncio.CancelledError):+ await queue.process_dequeued(message)+ assert index.has_work("task-1")+ transport.write.assert_not_awaited()+++async def test_cross_loop_cancellation_waits_for_handler_cleanup(tracked_queue):+ queue, index, transport, _, cancelled = tracked_queue+ started = asyncio.Event()+ cleanup_started = asyncio.Event()+ cleanup_release = asyncio.Event()++ class Session:+ async def exists(self):+ return True++ async def load(self):+ pass++ async def resume_queued_commit(self, msg):+ started.set()+ try:+ await asyncio.Event().wait()+ finally:+ cleanup_started.set()+ await cleanup_release.wait()++ class Service:+ def session(self, ctx, session_id, session_uri=None):+ return Session()++ from openviking.storage.queuefs.session_commit_msg import SessionCommitMsg+ from openviking_cli.session.user_id import UserIdentifier++ msg = SessionCommitMsg(+ task_id="task-1",+ session_id="session-1",+ session_uri="viking://session/session-1",+ archive_uri="viking://session/session-1/history/archive-1",+ user=UserIdentifier("account", "user").to_dict(),+ )+ with bind_task_context("task-1", "account", "user"):+ await queue.enqueue(msg.to_dict())+ message = {"id": "m", "data": transport.write.await_args.args[1].decode()}+ transport.write.reset_mock()+ queue.set_dequeue_handler(SessionCommitProcessor(Service(), asyncio.get_running_loop()))++ def consume():+ return asyncio.run(queue.process_dequeued(message))++ worker = asyncio.create_task(asyncio.to_thread(consume))+ try:+ await asyncio.wait_for(started.wait(), timeout=3)+ cancelled.add("task-1")+ index.cancel_active("task-1")+ await asyncio.wait_for(cleanup_started.wait(), timeout=3)+ assert not worker.done()+ transport.write.assert_not_awaited()+ finally:+ cleanup_release.set()+ if not cleanup_started.is_set():+ cancelled.add("task-1")+ index.cancel_active("task-1")+ result = await asyncio.wait_for(worker, timeout=3)+ assert result.outcome is ProcessOutcome.CANCELLED+ transport.write.assert_not_awaited()+ await queue.ack("m", message)+ assert not index.has_work("task-1")+ transport.write.assert_awaited_once_with("/queue/Test/ack", b"m")diff --git a/tests/storage/test_collection_schemas.py b/tests/storage/test_collection_schemas.pyindex 8f9e0a4ce..c62aba5b8 100644--- a/tests/storage/test_collection_schemas.py+++ b/tests/storage/test_collection_schemas.py@@ -26,6 +26,8 @@ from openviking.storage.errors import ()from openviking.storage.expr import Eqfrom openviking.storage.queuefs.embedding_msg import EmbeddingMsg+from openviking.storage.queuefs.process_result import ProcessOutcome+from openviking.storage.vector_ids import vector_record_idfrom openviking.storage.vectordb import engine as vectordb_enginefrom openviking.storage.vectordb.collection.result import UpsertDataResultfrom openviking.storage.vectordb.collection.vikingdb_clients import VikingDBClient@@ -189,12 +191,8 @@ async def test_init_context_collection_backfills_metadata_for_empty_legacy_colleschema_updates = []config = _DummyConfig(_DummyEmbedder(), backend="local")existing_schema = CollectionSchemas.context_collection("context", config.embedding.dimension)- existing_fields = [- field for field in existing_schema["Fields"] if field["FieldName"] != "tags"- ]- existing_scalar_index = [- field for field in existing_schema["ScalarIndex"] if field != "tags"- ]+ existing_fields = [field for field in existing_schema["Fields"] if field["FieldName"] != "tags"]+ existing_scalar_index = [field for field in existing_schema["ScalarIndex"] if field != "tags"]class _FakeStorage:async def create_collection(self, name, schema):@@ -302,20 +300,13 @@ async def test_embedding_handler_skip_all_work_when_manager_is_closing(monkeypat)handler = TextEmbeddingHandler(_ClosingVikingDB())- status = {"success": 0, "requeue": 0, "error": 0}- handler.set_callbacks(- on_success=lambda: status.__setitem__("success", status["success"] + 1),- on_requeue=lambda: status.__setitem__("requeue", status["requeue"] + 1),- on_error=lambda *_: status.__setitem__("error", status["error"] + 1),- )result = await handler.on_dequeue(_build_queue_payload())- assert result is None+ assert result.outcome is ProcessOutcome.SUCCESS+ assert result.value is None+ assert result.error is Noneassert embedder.calls == 0- assert status["success"] == 1- assert status["requeue"] == 0- assert status["error"] == 0@pytest.mark.asyncio@@ -342,12 +333,6 @@ async def test_embedding_handler_open_breaker_logs_summary_instead_of_per_item_w)handler = TextEmbeddingHandler(_QueueingVikingDB())- status = {"success": 0, "requeue": 0, "error": 0}- handler.set_callbacks(- on_success=lambda: status.__setitem__("success", status["success"] + 1),- on_requeue=lambda: status.__setitem__("requeue", status["requeue"] + 1),- on_error=lambda *_: status.__setitem__("error", status["error"] + 1),- )monkeypatch.setattr(handler._circuit_breaker,"check",@@ -356,18 +341,22 @@ async def test_embedding_handler_open_breaker_logs_summary_instead_of_per_item_wimport openviking.storage.collection_schemas as collection_schemas+ monkeypatch.setattr(collection_schemas.logger, "propagate", False)collection_schemas.logger.addHandler(caplog.handler)collection_schemas.logger.setLevel(logging.WARNING)try:with caplog.at_level(logging.WARNING):- await handler.on_dequeue(_build_queue_payload())- await handler.on_dequeue(_build_queue_payload())+ first_result = await handler.on_dequeue(_build_queue_payload())+ second_result = await handler.on_dequeue(_build_queue_payload())finally:collection_schemas.logger.removeHandler(caplog.handler)warnings = [record.message for record in caplog.records if record.levelno == logging.WARNING]assert warnings.count("Embedding circuit breaker is open; re-enqueueing messages") == 1- assert status == {"success": 2, "requeue": 2, "error": 0}+ for result in (first_result, second_result):+ assert result.outcome is ProcessOutcome.REQUEUED+ assert result.value is None+ assert result.error is None@pytest.mark.asyncio@@ -397,18 +386,17 @@ async def test_embedding_auth_error_fails_terminally_without_reenqueue(monkeypatlambda: _DummyConfig(_AuthErrorEmbedder()),)handler = TextEmbeddingHandler(vikingdb)- status = {"success": 0, "requeue": 0, "error": 0}- handler.set_callbacks(- on_success=lambda: status.__setitem__("success", status["success"] + 1),- on_requeue=lambda: status.__setitem__("requeue", status["requeue"] + 1),- on_error=lambda *_: status.__setitem__("error", status["error"] + 1),- )result = await handler.on_dequeue(_build_queue_payload_for_account("acct"))- assert result is None+ assert result.outcome is ProcessOutcome.FAILED+ assert result.value is None+ assert result.error == (+ "Failed to generate embedding: "+ "Error code: 401 - {'code': 'AuthenticationError'} Unauthorized "+ "(uri=viking://resources/sample)"+ )assert vikingdb.enqueued == [] # terminal: not re-enqueued- assert status == {"success": 0, "requeue": 0, "error": 1}handler._circuit_breaker.check() # breaker not tripped (would raise if open)@@ -435,21 +423,14 @@ async def test_embedding_handler_treats_shutdown_write_lock_as_success(monkeypatvikingdb = _ClosingDuringUpsertVikingDB()handler = TextEmbeddingHandler(vikingdb)- status = {"success": 0, "requeue": 0, "error": 0}- handler.set_callbacks(- on_success=lambda: status.__setitem__("success", status["success"] + 1),- on_requeue=lambda: status.__setitem__("requeue", status["requeue"] + 1),- on_error=lambda *_: status.__setitem__("error", status["error"] + 1),- )result = await handler.on_dequeue(_build_queue_payload())- assert result is None+ assert result.outcome is ProcessOutcome.SUCCESS+ assert result.value is None+ assert result.error is Noneassert vikingdb.calls == 1assert embedder.calls == 1- assert status["success"] == 1- assert status["requeue"] == 0- assert status["error"] == 0@pytest.mark.asyncio@@ -639,18 +620,17 @@ async def test_embedding_handler_drops_input_too_large_without_requeue(monkeypat)handler = TextEmbeddingHandler(vikingdb)- status = {"success": 0, "requeue": 0, "error": 0}- handler.set_callbacks(- on_success=lambda: status.__setitem__("success", status["success"] + 1),- on_requeue=lambda: status.__setitem__("requeue", status["requeue"] + 1),- on_error=lambda *_: status.__setitem__("error", status["error"] + 1),- )result = await handler.on_dequeue(_build_queue_payload())- assert result is None+ assert result.outcome is ProcessOutcome.FAILED+ assert result.value is None+ assert result.error == (+ "Failed to generate embedding: "+ "Malformed input request: expected maxLength: 50000, actual: 75000 "+ "(uri=viking://resources/sample)"+ )assert vikingdb.enqueued == []- assert status == {"success": 0, "requeue": 0, "error": 1}assert handler._circuit_breaker._failure_count == 0@@ -682,7 +662,13 @@ async def test_embedding_handler_preserves_parent_uri_for_backend_upsert_logic(mresult = await handler.on_dequeue(payload)- assert result is not None+ assert result.outcome is ProcessOutcome.SUCCESS+ assert result.error is None+ assert result.value == {+ **queue_data["context_data"],+ "id": vector_record_id("default", "viking://resources/sample", 2),+ "vector": [0.1, 0.2],+ }assert "data" in capturedassert captured["data"]["parent_uri"] == "viking://resources"diff --git a/tests/storage/test_memory_semantic_stall.py b/tests/storage/test_memory_semantic_stall.pyindex 3dbf9fb1e..7267e1bf8 100644--- a/tests/storage/test_memory_semantic_stall.py+++ b/tests/storage/test_memory_semantic_stall.py@@ -4,7 +4,7 @@"""Tests for memory semantic queue stall fix (issue #864).Ensures that _process_memory_directory() error paths propagate exceptions-so that on_dequeue() always calls report_success() or report_error().+so that on_dequeue() returns an explicit processing outcome."""from types import SimpleNamespace@@ -12,6 +12,7 @@ from unittest.mock import AsyncMock, MagicMock, patchimport pytest+from openviking.storage.queuefs.process_result import ProcessOutcomefrom openviking.storage.queuefs.semantic_msg import SemanticMsgfrom openviking.storage.queuefs.semantic_processor import SemanticProcessor@@ -44,17 +45,19 @@ def _build_data(msg: SemanticMsg) -> dict:@pytest.mark.asyncioasync def test_root_semantic_message_is_acknowledged_without_processing():processor = SemanticProcessor()- success = MagicMock()- processor.set_callbacks(success, MagicMock(), MagicMock())- await processor.on_dequeue(_build_data(_make_msg(uri="viking://", context_type="resource")))+ result = await processor.on_dequeue(+ _build_data(_make_msg(uri="viking://", context_type="resource"))+ )- success.assert_called_once_with()+ assert result.outcome is ProcessOutcome.SUCCESS+ assert result.value is None+ assert result.error is None@pytest.mark.asyncio-async def test_memory_empty_dir_still_reports_success():- """When viking_fs.ls returns an empty list, report_success() must be called."""+async def test_memory_empty_dir_still_returns_success():+ """An empty memory directory must return a successful processing result."""processor = SemanticProcessor()fake_fs = MagicMock()@@ -63,20 +66,6 @@ async def test_memory_empty_dir_still_reports_success():msg = _make_msg()data = _build_data(msg)- success_called = False-- def on_success():- nonlocal success_called- success_called = True-- error_called = False-- def on_error(error_msg, error_data=None):- nonlocal error_called- error_called = True-- processor.set_callbacks(on_success, lambda: None, on_error)-with (patch("openviking.storage.queuefs.semantic_processor.get_viking_fs",@@ -87,18 +76,19 @@ async def test_memory_empty_dir_still_reports_success():return_value=None,),):- await processor.on_dequeue(data)+ result = await processor.on_dequeue(data)- assert success_called, "report_success() was not called for empty memory directory"- assert not error_called, "report_error() should not be called for empty directory"+ assert result.outcome is ProcessOutcome.SUCCESS+ assert result.value is None+ assert result.error is None@pytest.mark.asyncio-async def test_memory_ls_error_reports_error():- """When viking_fs.ls raises a filesystem error, report_error() must be called.+async def test_memory_ls_error_returns_failed():+ """A permanent filesystem error must return a failed processing result.- Uses a real classify_api_error (no mock) — FileNotFoundError is classified- as permanent by the real classifier, so the processor calls report_error().+ Uses a real classify_api_error (no mock): FileNotFoundError is classified+ as permanent by the real classifier, so the processor returns FAILED."""processor = SemanticProcessor()@@ -108,22 +98,6 @@ async def test_memory_ls_error_reports_error():msg = _make_msg()data = _build_data(msg)- success_called = False-- def on_success():- nonlocal success_called- success_called = True-- error_called = False- error_info = {}-- def on_error(error_msg, error_data=None):- nonlocal error_called, error_info- error_called = True- error_info["msg"] = error_msg-- processor.set_callbacks(on_success, lambda: None, on_error)-with (patch("openviking.storage.queuefs.semantic_processor.get_viking_fs",@@ -134,11 +108,11 @@ async def test_memory_ls_error_reports_error():return_value=None,),):- await processor.on_dequeue(data)+ result = await processor.on_dequeue(data)- assert error_called, "report_error() was not called when ls() raised an exception"- assert not success_called, "report_success() should not be called on ls() error"- assert "/memories not found" in error_info["msg"]+ assert result.outcome is ProcessOutcome.FAILED+ assert result.value is None+ assert result.error == (f"Failed to list memory directory {msg.uri}: /memories not found")@pytest.mark.asyncio@@ -147,8 +121,8 @@ async def test_memory_ls_transient_error_requeues():A 500-class error wrapped by the processor's `raise RuntimeError(...) from e`is classified as `transient`. The outer on_dequeue() path must call- _reenqueue_semantic_msg(), bump requeue_count, and fire both report_requeue()- and report_success() — not report_error().+ _reenqueue_semantic_msg(), bump requeue_count, and return REQUEUED without+ a terminal error."""processor = SemanticProcessor()@@ -158,24 +132,6 @@ async def test_memory_ls_transient_error_requeues():msg = _make_msg(telemetry_id="tel-1")data = _build_data(msg)- success_called = False- requeue_called = False- error_called = False-- def on_success():- nonlocal success_called- success_called = True-- def on_requeue():- nonlocal requeue_called- requeue_called = True-- def on_error(error_msg, error_data=None):- nonlocal error_called- error_called = True-- processor.set_callbacks(on_success, on_requeue, on_error)-reenqueue_mock = AsyncMock()with (@@ -189,20 +145,20 @@ async def test_memory_ls_transient_error_requeues():),patch.object(processor, "_reenqueue_semantic_msg", new=reenqueue_mock),):- await processor.on_dequeue(data)+ result = await processor.on_dequeue(data)- assert requeue_called, "report_requeue() must fire for transient errors"- assert success_called, "report_success() must fire after successful re-enqueue"- assert not error_called, "report_error() must NOT fire for transient errors"+ assert result.outcome is ProcessOutcome.REQUEUED+ assert result.value is None+ assert result.error is Nonereenqueue_mock.assert_awaited_once()@pytest.mark.asyncio-async def test_memory_write_error_reports_error():- """When abstract/overview write raises PermissionError, report_error() is called.+async def test_memory_write_error_returns_failed():+ """An abstract/overview PermissionError must return a failed processing result.- Exercises the write failure path with real classify_api_error — PermissionError- is classified as permanent, so the processor calls report_error().+ Exercises the write failure path with real classify_api_error: PermissionError+ is classified as permanent, so the processor returns FAILED."""processor = SemanticProcessor()@@ -210,9 +166,7 @@ async def test_memory_write_error_reports_error():fake_fs.ls = AsyncMock(return_value=[{"name": "file1.md", "isDir": False}])fake_fs.read_file = AsyncMock(return_value="some content")fake_fs.write_file = AsyncMock(side_effect=PermissionError("Permission denied"))- fake_fs._async_agfs.pathlock_acquire_exact_batch = AsyncMock(- return_value={"lease_ref": "test"}- )+ fake_fs._async_agfs.pathlock_acquire_exact_batch = AsyncMock(return_value={"lease_ref": "test"})fake_fs._async_agfs.pathlock_release = AsyncMock()fake_fs._uri_to_path = MagicMock(side_effect=lambda uri, ctx=None: f"/local/acc1/{uri.removeprefix('viking://')}"@@ -221,22 +175,6 @@ async def test_memory_write_error_reports_error():msg = _make_msg(skip_vectorization=True)data = _build_data(msg)- success_called = False-- def on_success():- nonlocal success_called- success_called = True-- error_called = False- error_info = {}-- def on_error(error_msg, error_data=None):- nonlocal error_called, error_info- error_called = True- error_info["msg"] = error_msg-- processor.set_callbacks(on_success, lambda: None, on_error)-with (patch("openviking.storage.queuefs.semantic_processor.get_viking_fs",@@ -266,8 +204,8 @@ async def test_memory_write_error_reports_error():new=AsyncMock(return_value="# Overview\ntest overview"),),):- await processor.on_dequeue(data)+ result = await processor.on_dequeue(data)- assert error_called, "report_error() was not called when write() raised PermissionError"- assert not success_called, "report_success() should not be called on write error"- assert "Permission denied" in error_info["msg"]+ assert result.outcome is ProcessOutcome.FAILED+ assert result.value is None+ assert result.error == (f"Failed to write abstract/overview for {msg.uri}: Permission denied")diff --git a/tests/storage/test_named_queue_middleware.py b/tests/storage/test_named_queue_middleware.pynew file mode 100644index 000000000..1ac31f2d9--- /dev/null+++ b/tests/storage/test_named_queue_middleware.py@@ -0,0 +1,262 @@+# Copyright (c) 2026 Beijing Volcano Engine Technology Co., Ltd.+# SPDX-License-Identifier: AGPL-3.0+"""Queue middleware ordering and transport compatibility."""++import asyncio+import json+from unittest.mock import AsyncMock++import pytest++from openviking.pyagfs import AsyncAGFSClient+from openviking.storage.queuefs.named_queue import (+ DequeueHandlerBase,+ NamedQueue,+)+from openviking.storage.queuefs.process_result import ProcessOutcome, ProcessResult+from openviking.storage.queuefs.queue_middleware import QueueMiddleware+++def transport(monkeypatch):+ client = AsyncMock(spec=AsyncAGFSClient)+ client.write.return_value = "message-1"+ client.read.return_value = b"[]"+ monkeypatch.setattr(+ "openviking.storage.queuefs.named_queue.AsyncAGFSClient", lambda _client: client+ )+ return client+++class Handler(DequeueHandlerBase):+ async def on_dequeue(self, data):+ return ProcessResult.success(data)++[email protected]("operation", ["enqueue", "process", "ack", "clear"])+async def test_middleware_constructor_order_and_commit(transport, operation):+ events = []++ class Recorder(QueueMiddleware):+ def __init__(self, name):+ self.name = name++ async def invoke(self, ctx, call_next):+ events.append((self.name, "enter"))+ result = await call_next(ctx)+ if operation != "process":+ assert ctx.committed+ events.append((self.name, "exit"))+ return result++ enqueue = process = ack = clear = invoke++ middlewares = [Recorder("outer"), Recorder("inner")]+ queue = NamedQueue(+ object(), "/queue", "Test", dequeue_handler=Handler(), middlewares=middlewares+ )+ middlewares.clear()+ message = {"id": "message-1", "data": "{}"}+ if operation == "enqueue":+ assert await queue.enqueue({"value": 1}) == "message-1"+ transport.write.assert_awaited_once_with("/queue/Test/enqueue", b'{"value": 1}')+ elif operation == "process":+ assert (await queue.process_dequeued(message)).value == message+ transport.write.assert_not_awaited()+ elif operation == "ack":+ await queue.ack("message-1", message)+ transport.write.assert_awaited_once_with("/queue/Test/ack", b"message-1")+ else:+ assert await queue.clear()+ transport.write.assert_awaited_once_with("/queue/Test/clear", b"")+ assert events == [("outer", "enter"), ("inner", "enter"), ("inner", "exit"), ("outer", "exit")]++[email protected]("failure", [RuntimeError("write failed"), asyncio.CancelledError()])+async def test_enqueue_failure_unwinds_middlewares(transport, failure):+ events = []++ class Recorder(QueueMiddleware):+ def __init__(self, name):+ self.name = name++ async def enqueue(self, ctx, call_next):+ events.append(self.name)+ try:+ return await call_next(ctx)+ finally:+ assert not ctx.committed+ events.append(f"{self.name}-cleanup")++ transport.write.side_effect = failure+ queue = NamedQueue(+ object(), "/queue", "Test", middlewares=[Recorder("outer"), Recorder("inner")]+ )+ with pytest.raises(type(failure)):+ await queue.enqueue("payload")+ assert events == ["outer", "inner", "inner-cleanup", "outer-cleanup"]+++async def test_enqueue_middleware_rewrites_payload(transport):+ class Rewrite(QueueMiddleware):+ async def enqueue(self, ctx, call_next):+ assert ctx.payload == "input"+ ctx.payload = {"input": ctx.payload, "middleware": True}+ return await call_next(ctx)++ queue = NamedQueue(object(), "/queue", "Test", middlewares=[Rewrite()])+ await queue.enqueue("input")+ assert json.loads(transport.write.await_args.args[1]) == {"input": "input", "middleware": True}++[email protected]("failure", [RuntimeError("handler failed"), asyncio.CancelledError()])+async def test_handler_exception_still_skips_ack(transport, failure):+ class FailingHandler(DequeueHandlerBase):+ async def on_dequeue(self, data):+ raise failure++ transport.read.return_value = b'{"id":"message-1","data":"{}"}'+ queue = NamedQueue(+ object(),+ "/queue",+ "Test",+ dequeue_handler=FailingHandler(),+ middlewares=[QueueMiddleware()],+ )+ if isinstance(failure, asyncio.CancelledError):+ with pytest.raises(asyncio.CancelledError):+ await queue.dequeue()+ else:+ assert await queue.dequeue() is None+ transport.write.assert_not_awaited()+++async def test_empty_success_result_still_acks(transport):+ class DiscardHandler(DequeueHandlerBase):+ async def on_dequeue(self, data):+ return ProcessResult.success()++ transport.read.return_value = b'{"id":"message-1","data":"{}"}'+ queue = NamedQueue(+ object(),+ "/queue",+ "Test",+ dequeue_handler=DiscardHandler(),+ middlewares=[QueueMiddleware()],+ )+ assert await queue.dequeue() is None+ transport.write.assert_awaited_once_with("/queue/Test/ack", b"message-1")+++async def test_clear_snapshot_error_is_not_converted_to_success(transport):+ transport.read.side_effect = OSError("snapshot unavailable")+ queue = NamedQueue(object(), "/queue", "Test", middlewares=[QueueMiddleware()])+ with pytest.raises(OSError, match="snapshot unavailable"):+ await queue.clear()+ transport.write.assert_not_awaited()+++async def test_queue_without_middleware_remains_generic(transport):+ queue = NamedQueue(object(), "/queue", "Test")+ payload = {"task_id": "opaque-business-field"}+ await queue.enqueue(payload)+ assert json.loads(transport.write.await_args.args[1]) == payload+ assert (await queue.process_dequeued(payload)).value == payload+ transport.write.reset_mock()+ await queue.ack("")+ transport.write.assert_not_awaited()+ assert await queue.clear()+ transport.read.assert_not_awaited()+++ "result",+ [+ ProcessResult.success(),+ ProcessResult.failed("bad input"),+ ProcessResult.requeued(),+ ProcessResult.cancelled(),+ ],+)+async def test_process_short_circuit_settles_once_without_acking(transport, result):+ class ShortCircuit(QueueMiddleware):+ async def process(self, ctx, call_next):+ return result++ queue = NamedQueue(+ object(),+ "/queue",+ "Test",+ dequeue_handler=Handler(),+ middlewares=[ShortCircuit()],+ )+ message = {"id": "m", "data": "{}"}+ assert await queue.process_dequeued(message) == result+ assert queue._error_count == (result.outcome is ProcessOutcome.FAILED)+ assert queue._processed == (result.outcome is not ProcessOutcome.FAILED)+ assert queue._requeue_count == (result.outcome is ProcessOutcome.REQUEUED)+ transport.write.assert_not_awaited()+++async def test_cancel_before_process_starts_leaves_no_local_inflight_state(transport):+ queue = NamedQueue(object(), "/queue", "Test", dequeue_handler=Handler())+ task = asyncio.create_task(queue.process_dequeued({"id": "m"}))+ task.cancel()+ with pytest.raises(asyncio.CancelledError):+ await task+ transport.write.assert_not_awaited()+++async def test_invalid_process_result_does_not_ack(transport):+ class Invalid(QueueMiddleware):+ async def process(self, ctx, call_next):+ return None++ queue = NamedQueue(+ object(),+ "/queue",+ "Test",+ dequeue_handler=Handler(),+ middlewares=[Invalid()],+ )+ with pytest.raises(TypeError, match="ProcessResult"):+ await queue.process_dequeued({"id": "m"})+ assert queue._error_count == 1+ transport.write.assert_not_awaited()+++async def test_ack_error_does_not_double_count_processing(transport):+ transport.read.return_value = b'{"id":"m","data":"{}"}'+ transport.write.side_effect = OSError("ack failed")+ queue = NamedQueue(object(), "/queue", "Test", dequeue_handler=Handler())+ await queue.dequeue()+ assert queue._processed == 1+ assert queue._error_count == 0+++async def test_clear_short_circuit_reports_no_commit(transport):+ class SkipClear(QueueMiddleware):+ async def clear(self, ctx, call_next):+ return None++ queue = NamedQueue(object(), "/queue", "Test", middlewares=[SkipClear()])+ assert await queue.clear() is False+ transport.write.assert_not_awaited()+++async def test_dequeue_without_handler_bypasses_process_middleware_and_acks(transport):+ calls = []++ class Recorder(QueueMiddleware):+ async def process(self, ctx, call_next):+ calls.append(ctx.message)+ return await call_next(ctx)++ message = {"id": "m", "data": "{}"}+ transport.read.return_value = json.dumps(message).encode()+ queue = NamedQueue(object(), "/queue", "Test", middlewares=[Recorder()])++ assert await queue.dequeue() == message+ assert calls == []+ assert queue._processed == 0+ transport.write.assert_awaited_once_with("/queue/Test/ack", b"m")diff --git a/tests/storage/test_process_result.py b/tests/storage/test_process_result.pynew file mode 100644index 000000000..5d21e1c16--- /dev/null+++ b/tests/storage/test_process_result.py@@ -0,0 +1,34 @@+# Copyright (c) 2026 Beijing Volcano Engine Technology Co., Ltd.+# SPDX-License-Identifier: AGPL-3.0++import pytest++from openviking.storage.queuefs.process_result import ProcessOutcome, ProcessResult++[email protected]("outcome", list(ProcessOutcome))+def test_result_outcome_and_payload(outcome):+ error = "failure" if outcome is ProcessOutcome.FAILED else None+ result = ProcessResult(outcome, error=error)+ assert result.outcome is outcome+ assert result.error == error+ assert result.value is None+++ "outcome,error",+ [+ (ProcessOutcome.FAILED, None),+ (ProcessOutcome.SUCCESS, "failure"),+ ("success", None),+ ],+)+def test_result_rejects_inconsistent_state(outcome, error):+ with pytest.raises(ValueError):+ ProcessResult(outcome, error=error)+++def test_result_preserves_value_and_empty_error():+ payload = {"value": 1}+ assert ProcessResult.success(payload).value is payload+ assert ProcessResult.failed("").error == ""diff --git a/tests/storage/test_queue_manager.py b/tests/storage/test_queue_manager.pyindex 1511ce447..83eaccead 100644--- a/tests/storage/test_queue_manager.py+++ b/tests/storage/test_queue_manager.py@@ -2,13 +2,21 @@# SPDX-License-Identifier: AGPL-3.0"""Focused tests for QueueManager concurrency selection."""+import jsonimport osimport subprocessimport sys+import threadingfrom unittest.mock import AsyncMock-from openviking.storage.queuefs.named_queue import NamedQueue+import pytest++from openviking.pyagfs import AsyncAGFSClient+from openviking.service.task_work_index import bind_task_context+from openviking.storage.queuefs.named_queue import DequeueHandlerBase, NamedQueue+from openviking.storage.queuefs.process_result import ProcessResultfrom openviking.storage.queuefs.queue_manager import QueueManager+from openviking.storage.queuefs.queue_middleware import QueueMiddlewaredef test_queuefs_package_imports_in_a_clean_process(tmp_path) -> None:@@ -57,3 +65,119 @@ async def test_status_waits_for_processing_messages_from_other_workers(monkeypatassert status.pending == 0assert status.in_progress == 1assert not status.is_complete+++def transport(monkeypatch):+ client = AsyncMock(spec=AsyncAGFSClient)+ client.write.return_value = "message-1"+ client.read.return_value = b"0"+ monkeypatch.setattr(+ "openviking.storage.queuefs.named_queue.AsyncAGFSClient", lambda _client: client+ )+ return client+++async def test_constructor_middlewares_apply_to_all_queues(transport):+ calls = []++ class Recorder(QueueMiddleware):+ async def enqueue(self, ctx, call_next):+ calls.append(ctx.queue)+ return await call_next(ctx)++ middlewares = [Recorder()]+ manager = QueueManager(object(), middlewares=middlewares)+ middlewares.clear()+ existing = manager.get_queue("existing", allow_create=True)+ with bind_task_context("task", "account", "user"):+ await existing.enqueue({})+ middlewares.append(Recorder())+ future = manager.get_queue("future", allow_create=True)+ await future.enqueue({})+ assert calls == ["existing", "future"]+ assert manager._task_work_index.has_work("task")+ assert transport.write.await_count == 2+++async def test_init_queue_manager_forwards_constructor_middlewares(transport, monkeypatch):+ from openviking.storage.queuefs import queue_manager as queue_module++ calls = []++ class Recorder(QueueMiddleware):+ async def enqueue(self, ctx, call_next):+ calls.append(ctx.queue)+ return await call_next(ctx)++ monkeypatch.setattr(queue_module, "_instance", None)+ manager = queue_module.init_queue_manager(object(), middlewares=[Recorder()])+ assert queue_module.get_queue_manager() is manager+ await manager.get_queue("Test", allow_create=True).enqueue({})+ assert calls == ["Test"]+ transport.write.assert_awaited_once_with("/queue/Test/enqueue", b"{}")++[email protected]("fail", [False, True])+async def test_concurrent_worker_uses_process_and_ack_middleware(transport, fail):+ events = []+ stop = threading.Event()++ class Recorder(QueueMiddleware):+ async def process(self, ctx, call_next):+ events.append("process")+ return await call_next(ctx)++ async def ack(self, ctx, call_next):+ events.append("ack")+ return await call_next(ctx)++ class Handler(DequeueHandlerBase):+ async def on_dequeue(self, data):+ stop.set()+ if fail:+ raise RuntimeError("worker failure")+ return ProcessResult.success(data)++ manager = QueueManager(object(), middlewares=[Recorder()])+ queue = manager.get_queue("Test", dequeue_handler=Handler(), allow_create=True)+ reads = [b'{"id":"message-1","data":"{}"}']++ async def read(path):+ return reads.pop(0) if reads else b"{}"++ transport.read.side_effect = read+ await manager._worker_async_concurrent(queue, stop, 2)+ assert queue._processed == (not fail)+ assert queue._error_count == fail+ assert events == (["process"] if fail else ["process", "ack"])+ if fail:+ transport.write.assert_not_awaited()+ else:+ transport.write.assert_awaited_once_with("/queue/Test/ack", b"message-1")+++async def test_bootstrap_restores_legacy_messages_into_middleware_index(transport):+ manager = QueueManager(object())+ queue = manager.get_queue("Test", allow_create=True)+ message = {+ "id": "legacy-message",+ "data": json.dumps({"task_id": "task", "account_id": "account", "user_id": "user"}),+ }+ transport.read.return_value = json.dumps([message]).encode()+ attached = []++ class Tracker:+ def attach_work_index(self, index):+ attached.append(index)+ assert index.has_work("task")++ async def restore_work_tasks(self, owners):+ assert owners == {"task": ("account", "user")}+ return ["restored"]++ assert await manager.prepare_task_tracking(Tracker()) == ["restored"]+ assert attached == [manager._task_work_index]+ await queue.ack("legacy-message", message)+ assert not attached[0].has_work("task")+ transport.write.assert_awaited_once_with("/queue/Test/ack", b"legacy-message")diff --git a/tests/storage/test_semantic_dag_stats.py b/tests/storage/test_semantic_dag_stats.pyindex 145597e43..fca2b6ae9 100644--- a/tests/storage/test_semantic_dag_stats.py+++ b/tests/storage/test_semantic_dag_stats.py@@ -8,6 +8,7 @@ from unittest.mock import AsyncMockimport pytestfrom openviking.server.identity import RequestContext, Role+from openviking.service.task_queue_middleware import TaskWorkQueueMiddlewarefrom openviking.service.task_work_index import (TaskWorkIndex,TaskWorkRejected,@@ -424,7 +425,7 @@ async def test_task_work_rejection_does_not_stop_shared_semantic_worker():None,"/queue","Embedding",- task_work_index=work_index,+ middlewares=[TaskWorkQueueMiddleware(work_index)],)embedding_queue._initialized = Trueunrelated_ran = asyncio.Event()diff --git a/tests/storage/test_semantic_queue_memory_dedupe.py b/tests/storage/test_semantic_queue_memory_dedupe.pyindex f3782bb16..9109dc156 100644--- a/tests/storage/test_semantic_queue_memory_dedupe.py+++ b/tests/storage/test_semantic_queue_memory_dedupe.py@@ -152,7 +152,7 @@ def _patch_semantic_config(monkeypatch, *, overview_sample_limit=32):@pytest.mark.asyncio-async def test_stale_memory_semantic_write_is_skipped(monkeypatch):+async def test_stale_memory_semantic_write_is_success(monkeypatch):pathlock = _FakePathLock()viking_fs = _FakeVikingFS(pathlock)processor = SemanticProcessor()diff --git a/tests/storage/test_session_commit_processor_identity.py b/tests/storage/test_session_commit_processor_identity.pyindex f8a9e9df4..14c1c2be3 100644--- a/tests/storage/test_session_commit_processor_identity.py+++ b/tests/storage/test_session_commit_processor_identity.py@@ -9,13 +9,12 @@ worker binds the committing account/user (so tokens are not attributed to"""import asyncio-import concurrent.futuresimport json-from unittest.mock import Mockfrom openviking.observability.context import get_root_observability_contextfrom openviking.server.identity import RequestContext, Rolefrom openviking.session.session import Session+from openviking.storage.queuefs.process_result import ProcessOutcomefrom openviking.storage.queuefs.session_commit_msg import SessionCommitMsgfrom openviking.storage.queuefs.session_commit_processor import SessionCommitProcessorfrom openviking_cli.session.user_id import UserIdentifier@@ -52,7 +51,7 @@ class _MemoryVikingFS:def __init__(self) -> None:self.files: dict[str, str] = {}- async def stat(self, uri, ctx=None):+ async def stat(self, uri, ctx=None, skip_count=False):return {"path": uri}async def write_file(self, uri, content, ctx=None, lease_ref=None):@@ -85,8 +84,9 @@ async def test_process_binds_committing_identity_to_root_context():)ctx = RequestContext(user=UserIdentifier("acme", "alice"), role=Role.USER)- await processor._process(_make_msg(), ctx)+ result = await processor._process(_make_msg(), ctx)+ assert result is Trueassert captured["account_id"] == "acme"assert captured["user_id"] == "alice"@@ -108,13 +108,14 @@ async def test_process_requeues_deferred_commit_and_resets_root_context(monkeypa)ctx = RequestContext(user=UserIdentifier("acme", "alice"), role=Role.USER)- await processor._process(_make_msg(), ctx)+ result = await processor._process(_make_msg(), ctx)+ assert result is Falseassert queued == [("SessionCommit", _make_msg().to_dict())]assert get_root_observability_context() is None-async def test_cancelled_queued_commit_writes_terminal_marker_before_success(monkeypatch):+async def test_cancelled_queued_commit_writes_terminal_marker_before_returning():msg = _make_msg()viking_fs = _MemoryVikingFS()session = Session(@@ -127,34 +128,12 @@ async def test_cancelled_queued_commit_writes_terminal_marker_before_success(monasyncio.get_running_loop(),)marker_uri = f"{msg.archive_uri}/.failed.json"- on_success = Mock(side_effect=lambda: viking_fs.files[marker_uri])- processor.set_callbacks(on_success, Mock(), Mock())-- def run_on_current_loop(coro, _loop):- task = asyncio.create_task(coro)- future: concurrent.futures.Future[None] = concurrent.futures.Future()-- def complete(completed: asyncio.Task) -> None:- if completed.cancelled():- future.cancel()- return- error = completed.exception()- if error is not None:- future.set_exception(error)- else:- future.set_result(completed.result())-- task.add_done_callback(complete)- return future- monkeypatch.setattr(- "openviking.storage.queuefs.session_commit_processor.asyncio.run_coroutine_threadsafe",- run_on_current_loop,- )-- await processor.on_cancelled({"data": json.dumps(msg.to_dict())})+ result = await processor.on_cancelled({"data": json.dumps(msg.to_dict())})+ assert result.outcome is ProcessOutcome.CANCELLED+ assert result.value is None+ assert result.error is Nonemarker = json.loads(viking_fs.files[marker_uri])assert marker["stage"] == "cancelled"assert marker["error"] == "session commit cancelled"- on_success.assert_called_once_with()diff --git a/tests/test_task_tracker_concurrency.py b/tests/test_task_tracker_concurrency.pyindex 5e2f784dd..66a28b294 100644--- a/tests/test_task_tracker_concurrency.py+++ b/tests/test_task_tracker_concurrency.py@@ -171,6 +171,23 @@ async def test_owner_loop_dispatcher_runs_foreign_loop_work_on_owner_loop():assert work_loop is owner_loop+async def test_owner_loop_dispatcher_uses_explicit_owner_loop():+ from openviking.service.task_tracker_concurrency import OwnerLoopDispatcher++ owner_loop = asyncio.get_running_loop()+ dispatcher = await asyncio.to_thread(OwnerLoopDispatcher, owner_loop)+ work_loop: asyncio.AbstractEventLoop | None = None++ async def work() -> None:+ nonlocal work_loop+ work_loop = asyncio.get_running_loop()++ await asyncio.to_thread(lambda: asyncio.run(dispatcher.run(work)))++ assert work_loop is owner_loop++@pytest.mark.asyncioasync def test_owner_loop_dispatcher_propagates_foreign_cancellation():from openviking.service.task_tracker_concurrency import OwnerLoopDispatcher