Skip to content

Self-evolving Context Database for AI Agents. Unify Agent Memory, Knowledge RAG and Skills.

refactor(queuefs): centralize queue processing lifecycle (#5019)

runyunzhou192b813e7e31
diff --git a/openviking/service/task_queue_middleware.py b/openviking/service/task_queue_middleware.py
new file mode 100644
index 000000000..e7381cda6
--- /dev/null
+++ b/openviking/service/task_queue_middleware.py
@@ -0,0 +1,100 @@
+# Copyright (c) 2026 Beijing Volcano Engine Technology Co., Ltd.
+# SPDX-License-Identifier: AGPL-3.0
+"""Adapt the existing in-memory TaskWorkIndex to queue lifecycle middleware."""
+
+import asyncio
+
+from openviking.service.task_work_index import (
+ TaskWorkIndex,
+ TaskWorkRejected,
+ bind_task_context,
+ extract_task_metadata,
+ prepare_task_payload,
+)
+from openviking.storage.queuefs.process_result import ProcessOutcome, ProcessResult
+from openviking.storage.queuefs.queue_middleware import (
+ AckContext,
+ AckNext,
+ ClearContext,
+ ClearNext,
+ EnqueueContext,
+ EnqueueNext,
+ ProcessContext,
+ ProcessNext,
+ QueueMiddleware,
+)
+
+
+class TaskWorkQueueMiddleware(QueueMiddleware):
+ """Keep QueueFS as the durable source of truth; no new task state or store."""
+
+ def __init__(self, index: TaskWorkIndex) -> None:
+ self._index = index
+
+ async def enqueue(self, ctx: EnqueueContext, call_next: EnqueueNext) -> str:
+ if not isinstance(ctx.payload, dict):
+ return await call_next(ctx)
+ ctx.payload, metadata = prepare_task_payload(ctx.payload)
+ if not self._index.register(ctx.queue, metadata):
+ raise TaskWorkRejected(
+ f"Task {metadata.task_id} is cancelling; rejected work for {ctx.queue}"
+ )
+ try:
+ return await call_next(ctx)
+ finally:
+ if metadata is not None and not ctx.committed:
+ await self._index.discard(ctx.queue, metadata)
+
+ async def process(self, ctx: ProcessContext, call_next: ProcessNext) -> ProcessResult:
+ metadata = extract_task_metadata(ctx.message)
+ if metadata is None:
+ return await call_next(ctx)
+
+ active_task = asyncio.current_task()
+ with bind_task_context(metadata.task_id, metadata.account_id, metadata.user_id):
+ if self._index.cancellation_requested(metadata.task_id):
+ result = await ctx.cancel()
+ if result.outcome is ProcessOutcome.FAILED:
+ self._index.record_failure(metadata.task_id, result.error)
+ return result
+ if active_task is not None:
+ self._index.register_active(metadata.task_id, active_task)
+ try:
+ result = await call_next(ctx)
+ except asyncio.CancelledError:
+ if self._index.cancellation_requested(metadata.task_id):
+ # Handler unwinding must finish before this delivery is settled.
+ return ProcessResult.cancelled()
+ raise
+ except Exception as exc:
+ self._index.record_failure(metadata.task_id, str(exc))
+ raise
+ finally:
+ if active_task is not None:
+ self._index.unregister_active(metadata.task_id, active_task)
+ if result.outcome is ProcessOutcome.FAILED:
+ self._index.record_failure(metadata.task_id, result.error)
+ return result
+
+ async def ack(self, ctx: AckContext, call_next: AckNext) -> None:
+ prepared = None
+ try:
+ if ctx.message is not None:
+ prepared = await self._index.prepare_ack(ctx.queue, ctx.message)
+ await call_next(ctx)
+ finally:
+ if prepared is not None and not ctx.committed:
+ self._index.rollback_ack(ctx.queue, prepared)
+
+ async def clear(self, ctx: ClearContext, call_next: ClearNext) -> None:
+ prepared = []
+ try:
+ for message in ctx.messages:
+ metadata = await self._index.prepare_ack(ctx.queue, message)
+ if metadata is not None:
+ prepared.append(metadata)
+ await call_next(ctx)
+ finally:
+ if not ctx.committed:
+ for metadata in prepared:
+ self._index.rollback_ack(ctx.queue, metadata)
diff --git a/openviking/service/task_tracker_concurrency.py b/openviking/service/task_tracker_concurrency.py
index 4a6499e43..c79cd7c37 100644
--- a/openviking/service/task_tracker_concurrency.py
+++ b/openviking/service/task_tracker_concurrency.py
@@ -63,8 +63,8 @@ async def run_to_completion(
class OwnerLoopDispatcher:
"""Run coroutine factories on one event loop, including foreign callers."""
- def __init__(self) -> None:
- self._owner_loop: asyncio.AbstractEventLoop | None = None
+ def __init__(self, owner_loop: asyncio.AbstractEventLoop | None = None) -> None:
+ self._owner_loop = owner_loop
self._bind_lock = threading.Lock()
def bind_current_loop(self) -> asyncio.AbstractEventLoop:
diff --git a/openviking/service/user_deletion.py b/openviking/service/user_deletion.py
index 7e8423d28..78bcc3ab6 100644
--- a/openviking/service/user_deletion.py
+++ b/openviking/service/user_deletion.py
@@ -14,8 +14,10 @@ from openviking.core.namespace import canonical_user_root
from openviking.server.identity import RequestContext, Role
from openviking.service.task_store import SYSTEM_TASK_ACCOUNT_ID, SYSTEM_TASK_USER_ID
from openviking.service.task_tracker import TaskStatus, get_task_tracker
+from openviking.service.task_tracker_concurrency import OwnerLoopDispatcher
from openviking.service.task_work_index import extract_task_metadata
from openviking.storage.queuefs.named_queue import DequeueHandlerBase
+from openviking.storage.queuefs.process_result import ProcessResult
from openviking.storage.viking_fs import LS_ALL_NODES
from openviking_cli.exceptions import NotFoundError
from openviking_cli.session.user_id import UserIdentifier
@@ -471,7 +473,7 @@ class _UserDeletionProcessor(DequeueHandlerBase):
service_loop: asyncio.AbstractEventLoop,
) -> None:
self._deletion_service = deletion_service
- self._service_loop = service_loop
+ self._dispatcher = OwnerLoopDispatcher(service_loop)
@staticmethod
def _parse_message(data: dict[str, Any]) -> dict[str, Any]:
@@ -498,29 +500,16 @@ class _UserDeletionProcessor(DequeueHandlerBase):
},
}
- async def on_dequeue(self, data: Optional[dict[str, Any]]) -> Optional[dict[str, Any]]:
+ async def on_dequeue(self, data: Optional[dict[str, Any]]) -> ProcessResult:
if not data:
- return None
+ return ProcessResult.success()
try:
message = self._parse_message(data)
except (json.JSONDecodeError, TypeError, ValueError) as exc:
- self.report_error(str(exc), data)
- return None
+ return ProcessResult.failed(str(exc))
- future = asyncio.run_coroutine_threadsafe(
- self._deletion_service._process(message),
- self._service_loop,
- )
- try:
- error = await asyncio.wrap_future(future)
- except asyncio.CancelledError:
- future.cancel()
- raise
- if error is None:
- self.report_success()
- else:
- self.report_error(error, data)
- return None
+ error = await self._dispatcher.run(lambda: self._deletion_service._process(message))
+ return ProcessResult.success() if error is None else ProcessResult.failed(error)
async def setup_user_deletion(
diff --git a/openviking/storage/collection_schemas.py b/openviking/storage/collection_schemas.py
index b56c58046..80bd3f572 100644
--- a/openviking/storage/collection_schemas.py
+++ b/openviking/storage/collection_schemas.py
@@ -27,6 +27,7 @@ from openviking.storage.errors import (
)
from openviking.storage.queuefs.embedding_msg import EmbeddingMsg
from openviking.storage.queuefs.named_queue import DequeueHandlerBase
+from openviking.storage.queuefs.process_result import ProcessResult
from openviking.storage.vector_ids import vector_record_id
from openviking.storage.viking_vector_index_backend import (
VIKINGDB_CONTENT_MAX_SIZE,
@@ -599,14 +600,12 @@ class TextEmbeddingHandler(DequeueHandlerBase):
else inserted_data["abstract"][:VIKINGDB_CONTENT_MAX_SIZE]
)
- async def on_dequeue(self, data: Optional[Dict[str, Any]]) -> Optional[Dict[str, Any]]:
+ async def on_dequeue(self, data: Optional[Dict[str, Any]]) -> ProcessResult:
"""Process dequeued message and add embedding vector(s)."""
if not data:
- return None
+ return ProcessResult.success()
embedding_msg: Optional[EmbeddingMsg] = None
- report_success = False
- report_error_args: Optional[tuple[str, Optional[Dict[str, Any]]]] = None
request_failed_message: Optional[str] = None
try:
embedding_msg = EmbeddingMsg.from_json(data["data"])
@@ -628,8 +627,7 @@ class TextEmbeddingHandler(DequeueHandlerBase):
logger.debug("Skip embedding dequeue during shutdown")
self._merge_request_stats(embedding_msg.telemetry_id, processed=1)
self._record_request_success(embedding_msg)
- report_success = True
- return None
+ return ProcessResult.success()
if not isinstance(embedding_msg.message, (str, list)):
logger.debug(
@@ -637,8 +635,7 @@ class TextEmbeddingHandler(DequeueHandlerBase):
)
self._merge_request_stats(embedding_msg.telemetry_id, processed=1)
self._record_request_success(embedding_msg)
- report_success = True
- return data
+ return ProcessResult.success(data)
# Circuit breaker: if API is known-broken, re-enqueue and wait
try:
@@ -659,17 +656,14 @@ class TextEmbeddingHandler(DequeueHandlerBase):
get_request_wait_tracker().record_embedding_requeue(
embedding_msg.telemetry_id
)
- self.report_requeue()
- report_success = True
- return None
+ return ProcessResult.requeued()
# No queue manager — cannot re-enqueue, drop with error
error_msg = self._embedding_error_msg(
embedding_msg,
"Circuit breaker open and no queue manager",
)
request_failed_message = error_msg
- report_error_args = (error_msg, data)
- return None
+ return ProcessResult.failed(error_msg)
# Initialize embedder if not already initialized
if not self._embedder:
@@ -719,16 +713,14 @@ class TextEmbeddingHandler(DequeueHandlerBase):
logger.error(error_msg)
self._merge_request_stats(embedding_msg.telemetry_id, error_count=1)
request_failed_message = error_msg
- report_error_args = (error_msg, data)
- return None
+ return ProcessResult.failed(error_msg)
if error_class == ERROR_CLASS_PERMANENT:
logger.critical(error_msg)
self._circuit_breaker.record_failure(embed_err)
self._merge_request_stats(embedding_msg.telemetry_id, error_count=1)
request_failed_message = error_msg
- report_error_args = (error_msg, data)
- return None
+ return ProcessResult.failed(error_msg)
if error_class == ERROR_CLASS_AUTH:
# Bad/expired credential: retrying cannot succeed. Fail
@@ -740,8 +732,7 @@ class TextEmbeddingHandler(DequeueHandlerBase):
logger.error(error_msg)
self._merge_request_stats(embedding_msg.telemetry_id, error_count=1)
request_failed_message = error_msg
- report_error_args = (error_msg, data)
- return None
+ return ProcessResult.failed(error_msg)
# Transient or unknown — re-enqueue for retry
logger.warning(error_msg)
@@ -756,13 +747,11 @@ class TextEmbeddingHandler(DequeueHandlerBase):
get_request_wait_tracker().record_embedding_requeue(
embedding_msg.telemetry_id
)
- self.report_requeue()
logger.info(
"Re-enqueued embedding message after transient error "
f"({self._embedding_msg_log_context(embedding_msg)})"
)
- report_success = True
- return None
+ return ProcessResult.requeued()
except Exception as requeue_err:
logger.error(
self._embedding_error_msg(
@@ -773,8 +762,7 @@ class TextEmbeddingHandler(DequeueHandlerBase):
self._merge_request_stats(embedding_msg.telemetry_id, error_count=1)
request_failed_message = error_msg
- report_error_args = (error_msg, data)
- return None
+ return ProcessResult.failed(error_msg)
# Add dense vector
if result.dense_vector:
@@ -789,8 +777,7 @@ class TextEmbeddingHandler(DequeueHandlerBase):
logger.error(error_msg)
self._merge_request_stats(embedding_msg.telemetry_id, error_count=1)
request_failed_message = error_msg
- report_error_args = (error_msg, data)
- return None
+ return ProcessResult.failed(error_msg)
# Add sparse vector if present
if result.sparse_vector:
@@ -812,8 +799,7 @@ class TextEmbeddingHandler(DequeueHandlerBase):
pass
self._merge_request_stats(embedding_msg.telemetry_id, error_count=1)
request_failed_message = error_msg
- report_error_args = (error_msg, data)
- return None
+ return ProcessResult.failed(error_msg)
# Write to vector database
try:
@@ -847,8 +833,7 @@ class TextEmbeddingHandler(DequeueHandlerBase):
logger.debug(f"Skip embedding write during shutdown: {db_err}")
self._merge_request_stats(embedding_msg.telemetry_id, processed=1)
self._record_request_success(embedding_msg)
- report_success = True
- return None
+ return ProcessResult.success()
error_msg = self._embedding_error_msg(
embedding_msg,
f"Failed to write to vector database: {db_err}",
@@ -859,15 +844,13 @@ class TextEmbeddingHandler(DequeueHandlerBase):
traceback.print_exc()
self._merge_request_stats(embedding_msg.telemetry_id, error_count=1)
request_failed_message = error_msg
- report_error_args = (error_msg, data)
- return None
+ return ProcessResult.failed(error_msg)
except Exception as db_err:
if self._vikingdb.is_closing:
logger.debug(f"Skip embedding write during shutdown: {db_err}")
self._merge_request_stats(embedding_msg.telemetry_id, processed=1)
self._record_request_success(embedding_msg)
- report_success = True
- return None
+ return ProcessResult.success()
error_msg = self._embedding_error_msg(
embedding_msg,
f"Failed to write to vector database: {db_err}",
@@ -875,17 +858,15 @@ class TextEmbeddingHandler(DequeueHandlerBase):
logger.error(error_msg)
self._merge_request_stats(embedding_msg.telemetry_id, error_count=1)
request_failed_message = error_msg
- report_error_args = (error_msg, data)
- return None
+ return ProcessResult.failed(error_msg)
self._merge_request_stats(embedding_msg.telemetry_id, processed=1)
self._record_request_success(
embedding_msg,
vector_written=bool(record_id),
)
- report_success = True
self._circuit_breaker.record_success()
- return inserted_data
+ return ProcessResult.success(inserted_data)
except Exception as e:
error_msg = self._embedding_error_msg(
@@ -899,17 +880,12 @@ class TextEmbeddingHandler(DequeueHandlerBase):
if embedding_msg is not None:
self._merge_request_stats(embedding_msg.telemetry_id, error_count=1)
request_failed_message = error_msg
- report_error_args = (error_msg, data)
- return None
+ return ProcessResult.failed(error_msg)
finally:
if embedding_msg is not None and request_failed_message is not None:
self._record_request_failure(embedding_msg, request_failed_message)
- if report_error_args is not None:
- self.report_error(*report_error_args)
- elif report_success:
- self.report_success()
- async def on_cancelled(self, data: Optional[Dict[str, Any]]) -> Optional[Dict[str, Any]]:
+ async def on_cancelled(self, data: Optional[Dict[str, Any]]) -> ProcessResult:
"""Settle request-scoped waiting when a queued embedding is cancelled."""
embedding_msg: Optional[EmbeddingMsg] = None
try:
@@ -919,13 +895,11 @@ class TextEmbeddingHandler(DequeueHandlerBase):
payload = json.loads(payload)
embedding_msg = EmbeddingMsg.from_dict(payload)
except (KeyError, TypeError, ValueError) as exc:
- self.report_error(str(exc), data)
- return None
+ return ProcessResult.failed(str(exc))
if embedding_msg is not None:
self._record_request_success(embedding_msg)
- self.report_success()
- return None
+ return ProcessResult.cancelled()
@staticmethod
def _record_request_success(
diff --git a/openviking/storage/queuefs/__init__.py b/openviking/storage/queuefs/__init__.py
index 10a4d0ffa..241a9b3a3 100644
--- a/openviking/storage/queuefs/__init__.py
+++ b/openviking/storage/queuefs/__init__.py
@@ -3,7 +3,9 @@
from .embedding_msg import EmbeddingMsg
from .embedding_queue import EmbeddingQueue
from .named_queue import NamedQueue, QueueError, QueueStatus
+from .process_result import ProcessOutcome, ProcessResult
from .queue_manager import QueueManager, get_queue_manager, init_queue_manager
+from .queue_middleware import QueueMiddleware
from .semantic_dag import SemanticDagExecutor
from .semantic_msg import SemanticMsg
from .semantic_processor import SemanticProcessor
@@ -17,6 +19,9 @@ __all__ = [
"NamedQueue",
"QueueStatus",
"QueueError",
+ "QueueMiddleware",
+ "ProcessOutcome",
+ "ProcessResult",
"EmbeddingQueue",
"EmbeddingMsg",
"SemanticQueue",
diff --git a/openviking/storage/queuefs/add_resource_processor.py b/openviking/storage/queuefs/add_resource_processor.py
index 263c3f8ce..61e39fb26 100644
--- a/openviking/storage/queuefs/add_resource_processor.py
+++ b/openviking/storage/queuefs/add_resource_processor.py
@@ -3,7 +3,6 @@
"""Durable add-resource queue consumer."""
import asyncio
-import concurrent.futures
import json
from contextlib import suppress
from copy import deepcopy
@@ -12,9 +11,11 @@ from typing import Any, Dict, Optional
from openviking.observability.context import bind_execution_context
from openviking.server.identity import RequestContext, Role
from openviking.service.task_tracker import TaskStatus, get_task_tracker
+from openviking.service.task_tracker_concurrency import OwnerLoopDispatcher
from openviking.service.task_work_index import bind_task_context, extract_task_metadata
from openviking.storage.queuefs.add_resource_msg import AddResourceMsg
from openviking.storage.queuefs.named_queue import DequeueHandlerBase
+from openviking.storage.queuefs.process_result import ProcessResult
from openviking.telemetry import (
OperationTelemetry,
bind_telemetry,
@@ -42,7 +43,7 @@ class AddResourceProcessor(DequeueHandlerBase):
viking_fs: Any,
):
self._resource_service = resource_service
- self._service_loop = service_loop
+ self._dispatcher = OwnerLoopDispatcher(service_loop)
self._queue_name = queue_name
self._viking_fs = viking_fs
@@ -125,11 +126,9 @@ class AddResourceProcessor(DequeueHandlerBase):
msg.task_id,
exc,
)
- self.report_requeue()
- self.report_success()
return True
- async def _process(self, msg: AddResourceMsg, data: Dict[str, Any]) -> None:
+ async def _process(self, msg: AddResourceMsg, data: Dict[str, Any]) -> ProcessResult:
telemetry_id = msg.telemetry_id or ""
ctx = RequestContext(
user=UserIdentifier(msg.account_id, msg.user_id),
@@ -165,8 +164,7 @@ class AddResourceProcessor(DequeueHandlerBase):
)
await self._record_watch_execution(msg, status, getattr(task, "error", None))
unregister_telemetry(telemetry_id)
- self.report_success()
- return None
+ return ProcessResult.success()
metadata = extract_task_metadata(data)
replay_result = getattr(task, "result", None)
@@ -176,7 +174,7 @@ class AddResourceProcessor(DequeueHandlerBase):
resource_lock = await self._load_lock(msg, ctx)
except Exception as exc:
if await self._requeue_lock_handoff(msg, exc):
- return None
+ return ProcessResult.requeued()
await tracker.fail(
msg.task_id,
f"Invalid lock_handoff: {exc}",
@@ -188,11 +186,10 @@ class AddResourceProcessor(DequeueHandlerBase):
"failed",
f"Invalid lock_handoff: {exc}",
)
- self.report_error(f"Invalid lock_handoff: {exc}", data)
unregister_telemetry(telemetry_id)
with suppress(Exception):
await self._cleanup_staged_source(msg, ctx)
- return None
+ return ProcessResult.failed(f"Invalid lock_handoff: {exc}")
telemetry = resolve_telemetry(telemetry_id) if telemetry_id else None
if telemetry is None:
@@ -252,8 +249,7 @@ class AddResourceProcessor(DequeueHandlerBase):
)
await self._record_watch_execution(msg, "failed", error)
terminal = True
- self.report_error("resource processing failed", data)
- return None
+ return ProcessResult.failed("resource processing failed")
if not msg.watch_task_id:
await tracker.complete(
msg.task_id,
@@ -302,8 +298,7 @@ class AddResourceProcessor(DequeueHandlerBase):
resource_id=result.get("root_uri"),
)
terminal = True
- self.report_success()
- return None
+ return ProcessResult.success()
except asyncio.CancelledError:
await self._record_watch_execution(msg, "cancelled")
terminal = True
@@ -325,8 +320,7 @@ class AddResourceProcessor(DequeueHandlerBase):
result=failure_result,
)
terminal = True
- self.report_error(str(exc), data)
- return None
+ return ProcessResult.failed(str(exc))
finally:
request_wait_tracker.cleanup(telemetry_id)
unregister_telemetry(telemetry_id)
@@ -337,7 +331,7 @@ class AddResourceProcessor(DequeueHandlerBase):
with suppress(Exception):
await self._cleanup_staged_source(msg, ctx)
- async def on_cancelled(self, data: Optional[Dict[str, Any]]) -> Optional[Dict[str, Any]]:
+ async def on_cancelled(self, data: Optional[Dict[str, Any]]) -> ProcessResult:
"""Release an enqueue-time lock before ACKing cancelled work."""
try:
payload = data.get("data", data) if isinstance(data, dict) else data
@@ -345,10 +339,9 @@ class AddResourceProcessor(DequeueHandlerBase):
payload = json.loads(payload)
msg = AddResourceMsg.from_dict(payload)
except (json.JSONDecodeError, TypeError, ValueError) as exc:
- self.report_error(str(exc), data)
- return None
- future = asyncio.run_coroutine_threadsafe(
- self._handle_cancelled(
+ return ProcessResult.failed(str(exc))
+ await self._dispatcher.run(
+ lambda: self._handle_cancelled(
msg,
RequestContext(
user=UserIdentifier(msg.account_id, msg.user_id),
@@ -358,16 +351,13 @@ class AddResourceProcessor(DequeueHandlerBase):
bypass_acl=msg.bypass_acl,
),
),
- self._service_loop,
)
- await asyncio.wrap_future(future)
unregister_telemetry(msg.telemetry_id or "")
- self.report_success()
- return None
+ return ProcessResult.cancelled()
- async def on_dequeue(self, data: Optional[Dict[str, Any]]) -> Optional[Dict[str, Any]]:
+ async def on_dequeue(self, data: Optional[Dict[str, Any]]) -> ProcessResult:
if not data:
- return None
+ return ProcessResult.success()
try:
if not isinstance(data, dict):
raise ValueError("Queue message must be an object")
@@ -376,16 +366,6 @@ class AddResourceProcessor(DequeueHandlerBase):
payload = json.loads(payload)
msg = AddResourceMsg.from_dict(payload)
except (json.JSONDecodeError, TypeError, ValueError) as exc:
- self.report_error(str(exc), data)
- return None
+ return ProcessResult.failed(str(exc))
- future: concurrent.futures.Future[None] = asyncio.run_coroutine_threadsafe(
- self._process(msg, data),
- self._service_loop,
- )
- try:
- await asyncio.wrap_future(future)
- except asyncio.CancelledError:
- future.cancel()
- raise
- return None
+ return await self._dispatcher.run(lambda: self._process(msg, data))
diff --git a/openviking/storage/queuefs/external_task_processor.py b/openviking/storage/queuefs/external_task_processor.py
index 2480e5f66..7a97a8077 100644
--- a/openviking/storage/queuefs/external_task_processor.py
+++ b/openviking/storage/queuefs/external_task_processor.py
@@ -13,6 +13,7 @@ from openviking.service.task_tracker_concurrency import OwnerLoopDispatcher
from openviking.service.task_work_index import TaskWorkRejected
from openviking.storage.queuefs import QueueManager, get_queue_manager
from openviking.storage.queuefs.named_queue import DequeueHandlerBase
+from openviking.storage.queuefs.process_result import ProcessResult
class ExternalTaskProcessor(DequeueHandlerBase):
@@ -40,14 +41,13 @@ class ExternalTaskProcessor(DequeueHandlerBase):
raise ValueError("External task queue payload is missing task ownership")
return task_id, account_id, user_id
- async def on_dequeue(self, data: Optional[Dict[str, Any]]) -> Optional[Dict[str, Any]]:
+ async def on_dequeue(self, data: Optional[Dict[str, Any]]) -> ProcessResult:
if not data:
- return None
+ return ProcessResult.success()
try:
task_id, account_id, user_id = self._parse(data)
except (json.JSONDecodeError, TypeError, ValueError) as exc:
- self.report_error(str(exc), data)
- return None
+ return ProcessResult.failed(str(exc))
processed = await self._dispatcher.run(
lambda: self._service.execute(task_id, account_id, user_id)
)
@@ -61,25 +61,22 @@ class ExternalTaskProcessor(DequeueHandlerBase):
)
except TaskWorkRejected:
# Cancellation only needs the current delivery to be ACKed.
- pass
+ return ProcessResult.cancelled()
else:
- self.report_requeue()
- self.report_success()
- return None
+ return ProcessResult.requeued()
+ return ProcessResult.success()
- async def on_cancelled(self, data: Optional[Dict[str, Any]]) -> Optional[Dict[str, Any]]:
+ async def on_cancelled(self, data: Optional[Dict[str, Any]]) -> ProcessResult:
if not data:
- return None
+ return ProcessResult.cancelled()
try:
task_id, account_id, user_id = self._parse(data)
except (json.JSONDecodeError, TypeError, ValueError) as exc:
- self.report_error(str(exc), data)
- return None
+ return ProcessResult.failed(str(exc))
await self._dispatcher.run(
lambda: self._service.cancel_recovered(task_id, account_id, user_id)
)
- self.report_success()
- return None
+ return ProcessResult.cancelled()
__all__ = ["ExternalTaskProcessor"]
diff --git a/openviking/storage/queuefs/named_queue.py b/openviking/storage/queuefs/named_queue.py
index 7090ddaa9..ee08d3449 100644
--- a/openviking/storage/queuefs/named_queue.py
+++ b/openviking/storage/queuefs/named_queue.py
@@ -6,20 +6,24 @@ import json
import threading
from dataclasses import dataclass, field
from datetime import datetime
-from typing import Any, Callable, Dict, List, Optional, Union
+from functools import partial
+from typing import Any, Awaitable, Callable, Dict, List, Optional, Sequence, TypeVar, Union
from openviking.pyagfs import AGFSSyncClientProtocol, AsyncAGFSClient
from openviking.pyagfs.exceptions import AGFSAlreadyExistsError, AGFSNotFoundError
-from openviking.service.task_work_index import (
- TaskWorkIndex,
- TaskWorkRejected,
- bind_task_context,
- extract_task_metadata,
- prepare_task_payload,
+from openviking.storage.queuefs.process_result import ProcessOutcome, ProcessResult
+from openviking.storage.queuefs.queue_middleware import (
+ AckContext,
+ ClearContext,
+ EnqueueContext,
+ ProcessContext,
+ QueueMiddleware,
)
from openviking_cli.utils.logger import get_logger
logger = get_logger(__name__)
+_ContextT = TypeVar("_ContextT")
+_ResultT = TypeVar("_ResultT")
@dataclass
@@ -51,63 +55,17 @@ class QueueStatus:
return self.pending == 0 and self.in_progress == 0
-class EnqueueHookBase(abc.ABC):
- """Enqueue hook base class.
-
- All custom enqueue logic should inherit from this base class.
- Provides on_enqueue method for custom processing before message enqueue.
- """
-
- @abc.abstractmethod
- async def on_enqueue(self, data: Union[str, Dict[str, Any]]) -> Union[str, Dict[str, Any]]:
- """Called before message enqueue. Can modify data or perform validation."""
- return data
-
-
class DequeueHandlerBase(abc.ABC):
- """Dequeue handler base class, supports callback mechanism to report processing results."""
-
- _success_callback: Optional[Callable[[], None]] = None
- _requeue_callback: Optional[Callable[[], None]] = None
- _error_callback: Optional[Callable[[str, Optional[Dict[str, Any]]], None]] = None
+ """Return a delivery outcome; queue statistics are never updated by handlers."""
- def set_callbacks(
- self,
- on_success: Callable[[], None],
- on_requeue: Callable[[], None],
- on_error: Callable[[str, Optional[Dict[str, Any]]], None],
- ) -> None:
- """Set callback functions."""
- self._success_callback = on_success
- self._requeue_callback = on_requeue
- self._error_callback = on_error
-
- def report_success(self) -> None:
- """Report processing success."""
- if self._success_callback:
- self._success_callback()
-
- def report_requeue(self) -> None:
- """Report that the current message was re-enqueued for later retry."""
- if self._requeue_callback:
- self._requeue_callback()
-
- def report_error(self, error_msg: str, data: Optional[Dict[str, Any]] = None) -> None:
- """Report processing error."""
- if self._error_callback:
- self._error_callback(error_msg, data)
-
- async def on_cancelled(self, data: Optional[Dict[str, Any]]) -> Optional[Dict[str, Any]]:
+ async def on_cancelled(self, data: Optional[Dict[str, Any]]) -> ProcessResult:
"""Discard task work cancelled before its handler starts."""
- self.report_success()
- return None
+ return ProcessResult.cancelled()
@abc.abstractmethod
- async def on_dequeue(self, data: Optional[Dict[str, Any]]) -> Optional[Dict[str, Any]]:
- """Called after message dequeue. Returns None to discard message."""
- if not data:
- return None
- return data
+ async def on_dequeue(self, data: Optional[Dict[str, Any]]) -> ProcessResult:
+ """Complete this delivery, or raise to leave it unacknowledged."""
+ raise NotImplementedError
class NamedQueue:
@@ -120,17 +78,15 @@ class NamedQueue:
agfs: AGFSSyncClientProtocol,
mount_point: str,
name: str,
- enqueue_hook: Optional[EnqueueHookBase] = None,
dequeue_handler: Optional[DequeueHandlerBase] = None,
- task_work_index: Optional[TaskWorkIndex] = None,
+ middlewares: Sequence[QueueMiddleware] = (),
):
self.name = name
self.path = f"{mount_point}/{name}"
self._agfs = agfs
self._async_agfs = AsyncAGFSClient(agfs)
- self._enqueue_hook = enqueue_hook
self._dequeue_handler = dequeue_handler
- self._task_work_index = task_work_index
+ self._middlewares = tuple(middlewares)
self._initialized = False
# Status tracking
@@ -140,35 +96,33 @@ class NamedQueue:
self._error_count = 0
self._errors: List[QueueError] = []
- # Inject callbacks to handler
- if self._dequeue_handler:
- self.set_dequeue_handler(self._dequeue_handler)
+ async def _run_middleware(
+ self,
+ operation: str,
+ ctx: _ContextT,
+ terminal: Callable[[_ContextT], Awaitable[_ResultT]],
+ ) -> _ResultT:
+ call_next = terminal
+ for middleware in reversed(self._middlewares):
+ call_next = partial(getattr(middleware, operation), call_next=call_next)
+ return await call_next(ctx)
def set_dequeue_handler(self, handler: DequeueHandlerBase) -> None:
"""Bind the consumer after its runtime dependencies are initialized."""
self._dequeue_handler = handler
- handler.set_callbacks(
- on_success=self._on_process_success,
- on_requeue=self._on_process_requeue,
- on_error=self._on_process_error,
- )
-
- def _on_process_success(self) -> None:
- """Called on processing success."""
+
+ def _record_result(self, result: ProcessResult, data: Dict[str, Any]) -> None:
+ if result.outcome is ProcessOutcome.FAILED:
+ self._record_error(result.error, data)
+ return
with self._lock:
+ # Keep the existing processed count: settled non-error deliveries,
+ # including cancellations and successful re-enqueues.
self._processed += 1
+ if result.outcome is ProcessOutcome.REQUEUED:
+ self._requeue_count += 1
- def _on_process_requeue(self) -> None:
- """Called when a dequeued message is re-enqueued for later retry."""
- with self._lock:
- self._requeue_count += 1
-
- def _on_process_error(self, error_msg: str, data: Optional[Dict[str, Any]] = None) -> None:
- """Called on processing failure."""
- if self._task_work_index is not None and data is not None:
- metadata = extract_task_metadata(data)
- if metadata is not None:
- self._task_work_index.record_failure(metadata.task_id, error_msg)
+ def _record_error(self, error_msg: str, data: Optional[Dict[str, Any]] = None) -> None:
with self._lock:
self._error_count += 1
self._errors.append(
@@ -218,66 +172,35 @@ class NamedQueue:
async def enqueue(self, data: Union[str, Dict[str, Any]]) -> str:
"""Send message to queue (enqueue)."""
await self._ensure_initialized()
- enqueue_file = f"{self.path}/enqueue"
+ return await self._run_middleware("enqueue", EnqueueContext(self.name, data), self._enqueue)
- # Execute enqueue hook
- if self._enqueue_hook:
- data = await self._enqueue_hook.on_enqueue(data)
-
- if isinstance(data, dict):
- data, task_metadata = prepare_task_payload(data)
- else:
- task_metadata = None
-
- if self._task_work_index is not None and not self._task_work_index.register(
- self.name, task_metadata
- ):
- logger.info(
- "[NamedQueue] Skip enqueue for cancelling task %s on %s",
- task_metadata.task_id,
- self.name,
- )
- raise TaskWorkRejected(
- f"Task {task_metadata.task_id} is cancelling; rejected work for {self.name}"
- )
-
- try:
- if isinstance(data, dict):
- data = json.dumps(data)
-
- msg_id = await self._async_agfs.write(enqueue_file, data.encode("utf-8"))
- except BaseException:
- if self._task_work_index is not None and task_metadata is not None:
- await self._task_work_index.discard(self.name, task_metadata)
- raise
+ async def _enqueue(self, ctx: EnqueueContext) -> str:
+ data = json.dumps(ctx.payload) if isinstance(ctx.payload, dict) else ctx.payload
+ msg_id = await self._async_agfs.write(f"{self.path}/enqueue", data.encode("utf-8"))
+ ctx.committed = True
return msg_id if isinstance(msg_id, str) else str(msg_id)
async def ack(self, msg_id: str, message: Optional[Dict[str, Any]] = None) -> None:
"""Acknowledge successful processing of a message (deletes it from persistent storage).
Must be called after the dequeue handler finishes processing a message.
- Task-owned work is provisionally removed from the runtime index first so
- the last message can persist its task's terminal state before deletion.
+ Middleware can provisionally settle application state before deletion.
If not called (e.g. process crashes), the message will be automatically
re-queued on the next startup via RecoverStale.
"""
if not msg_id:
return
- ack_file = f"{self.path}/ack"
- prepared = None
try:
- if self._task_work_index is not None and message is not None:
- prepared = await self._task_work_index.prepare_ack(self.name, message)
- await self._async_agfs.write(ack_file, msg_id.encode("utf-8"))
+ await self._run_middleware("ack", AckContext(self.name, msg_id, message), self._ack)
except asyncio.CancelledError:
- if self._task_work_index is not None and prepared is not None:
- self._task_work_index.rollback_ack(self.name, prepared)
raise
except Exception as e:
- if self._task_work_index is not None and prepared is not None:
- self._task_work_index.rollback_ack(self.name, prepared)
logger.warning(f"[NamedQueue] Ack failed for {self.name} msg_id={msg_id}: {e}")
+ async def _ack(self, ctx: AckContext) -> None:
+ await self._async_agfs.write(f"{self.path}/ack", ctx.message_id.encode("utf-8"))
+ ctx.committed = True
+
async def _read_queue_message(self) -> Optional[Dict[str, Any]]:
"""Read and remove one message from the AGFS queue; return parsed dict or None.
@@ -312,11 +235,11 @@ class NamedQueue:
data = await self._read_queue_message()
if data is None:
return None
- # Capture message ID before passing data to handler (handler may modify it)
msg_id = data.get("id", "") if isinstance(data, dict) else ""
raw_data = data
if self._dequeue_handler:
- data = await self.process_dequeued(data)
+ result = await self.process_dequeued(raw_data)
+ data = result.value
# Ack unconditionally after handler returns (success or handled error).
# If on_dequeue raises, the exception propagates and ack is skipped —
# the message will be recovered on next startup.
@@ -335,31 +258,32 @@ class NamedQueue:
logger.debug(f"[NamedQueue] Dequeue raw failed for {self.name}: {e}")
return None
- async def process_dequeued(self, data: Dict[str, Any]) -> Optional[Dict[str, Any]]:
- """Invoke the dequeue handler on already-fetched raw data."""
- if self._dequeue_handler is None:
- return data
+ async def process_dequeued(self, data: Dict[str, Any]) -> ProcessResult:
+ """Process one fetched delivery and settle local status exactly once."""
+ handler = self._dequeue_handler
+ if handler is None:
+ return ProcessResult.success(data)
- metadata = extract_task_metadata(data)
- if metadata is None or self._task_work_index is None:
- return await self._dequeue_handler.on_dequeue(data)
+ async def process(ctx: ProcessContext) -> ProcessResult:
+ return self._validate_result(await handler.on_dequeue(ctx.message))
- active_task = asyncio.current_task()
- with bind_task_context(metadata.task_id, metadata.account_id, metadata.user_id):
- if self._task_work_index.cancellation_requested(metadata.task_id):
- return await self._dequeue_handler.on_cancelled(data)
- if active_task is not None:
- self._task_work_index.register_active(metadata.task_id, active_task)
- try:
- return await self._dequeue_handler.on_dequeue(data)
- except asyncio.CancelledError:
- if self._task_work_index.cancellation_requested(metadata.task_id):
- self._on_process_success()
- return None
- raise
- finally:
- if active_task is not None:
- self._task_work_index.unregister_active(metadata.task_id, active_task)
+ async def cancel() -> ProcessResult:
+ return self._validate_result(await handler.on_cancelled(ctx.message))
+
+ ctx = ProcessContext(self.name, data, cancel=cancel)
+ try:
+ result = self._validate_result(await self._run_middleware("process", ctx, process))
+ except Exception as exc:
+ self._record_error(str(exc), data)
+ raise
+ self._record_result(result, data)
+ return result
+
+ @staticmethod
+ def _validate_result(result: ProcessResult) -> ProcessResult:
+ if not isinstance(result, ProcessResult):
+ raise TypeError("queue handlers and process middleware must return ProcessResult")
+ return result
async def peek(self) -> Optional[Dict[str, Any]]:
"""Peek at head message without removing."""
@@ -432,26 +356,17 @@ class NamedQueue:
async def clear(self) -> bool:
"""Clear queue."""
await self._ensure_initialized()
- clear_file = f"{self.path}/clear"
-
- messages = await self.snapshot() if self._task_work_index is not None else []
- prepared = []
+ messages = await self.snapshot() if self._middlewares else []
+ ctx = ClearContext(self.name, messages)
try:
- if self._task_work_index is not None:
- for message in messages:
- metadata = await self._task_work_index.prepare_ack(self.name, message)
- if metadata is not None:
- prepared.append(metadata)
- await self._async_agfs.write(clear_file, b"")
- return True
+ await self._run_middleware("clear", ctx, self._clear)
+ return ctx.committed
except asyncio.CancelledError:
- if self._task_work_index is not None:
- for metadata in prepared:
- self._task_work_index.rollback_ack(self.name, metadata)
raise
except Exception as e:
- if self._task_work_index is not None:
- for metadata in prepared:
- self._task_work_index.rollback_ack(self.name, metadata)
logger.error(f"[NamedQueue] Clear failed for {self.name}: {e}")
return False
+
+ async def _clear(self, ctx: ClearContext) -> None:
+ await self._async_agfs.write(f"{self.path}/clear", b"")
+ ctx.committed = True
diff --git a/openviking/storage/queuefs/process_result.py b/openviking/storage/queuefs/process_result.py
new file mode 100644
index 000000000..455f4ab6c
--- /dev/null
+++ b/openviking/storage/queuefs/process_result.py
@@ -0,0 +1,52 @@
+# Copyright (c) 2026 Beijing Volcano Engine Technology Co., Ltd.
+# SPDX-License-Identifier: AGPL-3.0
+"""The outcome of one delivery, not the outcome of its owning business task."""
+
+from dataclasses import dataclass
+from enum import Enum
+from typing import Any, Optional
+
+
+class ProcessOutcome(str, Enum):
+ SUCCESS = "success"
+ FAILED = "failed"
+ REQUEUED = "requeued"
+ CANCELLED = "cancelled"
+
+
+@dataclass(frozen=True)
+class ProcessResult:
+ """A settled delivery that may be ACKed after middleware completes.
+
+ REQUEUED means the handler already enqueued a replacement. Exceptions,
+ including unhandled cancellation, leave the current delivery unacknowledged.
+ """
+
+ outcome: ProcessOutcome
+ value: Any = None
+ error: Optional[str] = None
+
+ def __post_init__(self) -> None:
+ if not isinstance(self.outcome, ProcessOutcome):
+ raise ValueError("outcome must be a ProcessOutcome")
+ if self.outcome is ProcessOutcome.FAILED:
+ if not isinstance(self.error, str):
+ raise ValueError("failed results require an error string")
+ elif self.error is not None:
+ raise ValueError("only failed results may contain an error")
+
+ @classmethod
+ def success(cls, value: Any = None) -> "ProcessResult":
+ return cls(ProcessOutcome.SUCCESS, value=value)
+
+ @classmethod
+ def failed(cls, error: str) -> "ProcessResult":
+ return cls(ProcessOutcome.FAILED, error=error)
+
+ @classmethod
+ def requeued(cls) -> "ProcessResult":
+ return cls(ProcessOutcome.REQUEUED)
+
+ @classmethod
+ def cancelled(cls) -> "ProcessResult":
+ return cls(ProcessOutcome.CANCELLED)
diff --git a/openviking/storage/queuefs/queue_manager.py b/openviking/storage/queuefs/queue_manager.py
index 91cbb6a18..3249b3ca2 100644
--- a/openviking/storage/queuefs/queue_manager.py
+++ b/openviking/storage/queuefs/queue_manager.py
@@ -10,13 +10,14 @@ import atexit
import threading
import time
import traceback
-from typing import Any, Dict, Optional, Set, Union
+from typing import Any, Dict, Optional, Sequence, Set, Union
from openviking.service.task_work_index import TaskWorkIndex
from openviking_cli.utils.logger import get_logger
from .embedding_queue import EmbeddingQueue
-from .named_queue import DequeueHandlerBase, EnqueueHookBase, NamedQueue, QueueStatus
+from .named_queue import DequeueHandlerBase, NamedQueue, QueueStatus
+from .queue_middleware import QueueMiddleware
from .semantic_queue import SemanticQueue
logger = get_logger(__name__)
@@ -37,6 +38,8 @@ def init_queue_manager(
max_concurrent_add_resource: int = 4,
max_concurrent_session_commit: int = DEFAULT_MAX_CONCURRENT_SESSION_COMMIT,
max_concurrent_external_task: int = 10,
+ *,
+ middlewares: Sequence[QueueMiddleware] = (),
) -> "QueueManager":
"""Initialize QueueManager singleton.
@@ -49,6 +52,7 @@ def init_queue_manager(
max_concurrent_external_parse: Max concurrent ExternalParse tasks.
max_concurrent_add_resource: Max concurrent AddResource tasks.
max_concurrent_session_commit: Max concurrent SessionCommit tasks.
+ middlewares: Additional middleware, fixed at construction for all queues.
"""
global _instance
_instance = QueueManager(
@@ -61,6 +65,7 @@ def init_queue_manager(
max_concurrent_add_resource=max_concurrent_add_resource,
max_concurrent_session_commit=max_concurrent_session_commit,
max_concurrent_external_task=max_concurrent_external_task,
+ middlewares=middlewares,
)
return _instance
@@ -101,6 +106,8 @@ class QueueManager:
max_concurrent_add_resource: int = 4,
max_concurrent_session_commit: int = DEFAULT_MAX_CONCURRENT_SESSION_COMMIT,
max_concurrent_external_task: int = 10,
+ *,
+ middlewares: Sequence[QueueMiddleware] = (),
):
"""Initialize QueueManager."""
self._agfs = agfs
@@ -118,6 +125,13 @@ class QueueManager:
self._queue_stop_events: Dict[str, threading.Event] = {}
self._poll_interval = 0.2
self._task_work_index = TaskWorkIndex()
+ # Import at composition time to avoid a service <-> queue package cycle.
+ from openviking.service.task_queue_middleware import TaskWorkQueueMiddleware
+
+ self._middlewares: tuple[QueueMiddleware, ...] = (
+ TaskWorkQueueMiddleware(self._task_work_index),
+ *middlewares,
+ )
atexit.register(self.stop)
logger.info(
@@ -276,8 +290,7 @@ class QueueManager:
# Ack after successful processing (delete from persistent storage).
await queue.ack(msg_id, data)
except Exception as e:
- # Do NOT ack — let RecoverStale re-queue on next startup.
- queue._on_process_error(str(e), data)
+ # The message remains in processing and will be recovered.
logger.error(f"[QueueManager] Concurrent worker error for {queue.name}: {e}")
while not stop_event.is_set():
@@ -348,7 +361,6 @@ class QueueManager:
def get_queue(
self,
name: str,
- enqueue_hook: Optional[EnqueueHookBase] = None,
dequeue_handler: Optional[DequeueHandlerBase] = None,
allow_create: bool = False,
) -> NamedQueue:
@@ -361,27 +373,24 @@ class QueueManager:
self._agfs,
self.mount_point,
name,
- enqueue_hook=enqueue_hook,
dequeue_handler=dequeue_handler,
- task_work_index=self._task_work_index,
+ middlewares=self._middlewares,
)
elif name == self.SEMANTIC:
self._queues[name] = SemanticQueue(
self._agfs,
self.mount_point,
name,
- enqueue_hook=enqueue_hook,
dequeue_handler=dequeue_handler,
- task_work_index=self._task_work_index,
+ middlewares=self._middlewares,
)
else:
self._queues[name] = NamedQueue(
self._agfs,
self.mount_point,
name,
- enqueue_hook=enqueue_hook,
dequeue_handler=dequeue_handler,
- task_work_index=self._task_work_index,
+ middlewares=self._middlewares,
)
if self._started:
self._start_queue_worker(self._queues[name])
diff --git a/openviking/storage/queuefs/queue_middleware.py b/openviking/storage/queuefs/queue_middleware.py
new file mode 100644
index 000000000..c105b0ddd
--- /dev/null
+++ b/openviking/storage/queuefs/queue_middleware.py
@@ -0,0 +1,80 @@
+# Copyright (c) 2026 Beijing Volcano Engine Technology Co., Ltd.
+# SPDX-License-Identifier: AGPL-3.0
+"""Queue lifecycle extensions, independent of task tracking and storage backends."""
+
+from dataclasses import dataclass
+from typing import Any, Awaitable, Callable, Dict, List, Optional, Union
+
+from .process_result import ProcessResult
+
+Payload = Union[str, Dict[str, Any]]
+Message = Dict[str, Any]
+
+
+@dataclass
+class EnqueueContext:
+ queue: str
+ payload: Payload
+ committed: bool = False
+
+
+@dataclass
+class ProcessContext:
+ queue: str
+ message: Message
+ cancel: Callable[[], Awaitable[ProcessResult]]
+
+
+@dataclass
+class AckContext:
+ queue: str
+ message_id: str
+ message: Optional[Message] = None
+ committed: bool = False
+
+
+@dataclass
+class ClearContext:
+ queue: str
+ messages: List[Message]
+ committed: bool = False
+
+
+EnqueueNext = Callable[[EnqueueContext], Awaitable[str]]
+ProcessNext = Callable[[ProcessContext], Awaitable[ProcessResult]]
+AckNext = Callable[[AckContext], Awaitable[None]]
+ClearNext = Callable[[ClearContext], Awaitable[None]]
+
+
+class QueueMiddleware:
+ """Wrap operations in constructor-supplied order (first item is outermost).
+
+ Queues copy the sequence to a tuple at construction; there is no runtime
+ registration or reconfiguration.
+
+ Override only the operations needed. Call ``call_next(ctx)`` once with the same
+ context to continue, or return/raise to short-circuit. Exceptions unwind before
+ the queue applies its existing error policy. Do not change message ownership
+ after an outer middleware has registered it.
+ Process must return a ProcessResult even when short-circuiting; the queue
+ owns all delivery statistics and ACKs settled results.
+
+ Contexts are invocation-local; middleware instances may be shared by queues
+ and worker threads and must not store per-message state on themselves.
+ Only the transport sets ``committed``. An outer middleware can compensate
+ uncommitted operations, including a normal return from a short-circuit.
+ It must not compensate an operation that already committed. Clear receives
+ a pre-operation snapshot; it does not promise isolation from concurrent enqueue.
+ """
+
+ async def enqueue(self, ctx: EnqueueContext, call_next: EnqueueNext) -> str:
+ return await call_next(ctx)
+
+ async def process(self, ctx: ProcessContext, call_next: ProcessNext) -> ProcessResult:
+ return await call_next(ctx)
+
+ async def ack(self, ctx: AckContext, call_next: AckNext) -> None:
+ await call_next(ctx)
+
+ async def clear(self, ctx: ClearContext, call_next: ClearNext) -> None:
+ await call_next(ctx)
diff --git a/openviking/storage/queuefs/semantic_processor.py b/openviking/storage/queuefs/semantic_processor.py
index 633b83bc7..d459c8da7 100644
--- a/openviking/storage/queuefs/semantic_processor.py
+++ b/openviking/storage/queuefs/semantic_processor.py
@@ -45,6 +45,7 @@ from openviking.storage.abstract_overview import (
from openviking.storage.acl import CreatorAclGrant
from openviking.storage.errors import LockAcquisitionError
from openviking.storage.queuefs.named_queue import DequeueHandlerBase
+from openviking.storage.queuefs.process_result import ProcessResult
from openviking.storage.queuefs.semantic_dag import DagStats, SemanticDagExecutor
from openviking.storage.queuefs.semantic_lock import SemanticLockScope
from openviking.storage.queuefs.semantic_msg import SemanticMsg, build_semantic_coalesce_key
@@ -228,21 +229,18 @@ class SemanticProcessor(DequeueHandlerBase):
async def _requeue_semantic_msg_after_error(
self,
msg: SemanticMsg,
- data: Optional[Dict[str, Any]],
error: Exception,
- ) -> None:
+ ) -> ProcessResult:
try:
await self._reenqueue_semantic_msg(msg)
self._merge_request_stats(msg.telemetry_id, requeue_count=1)
get_request_wait_tracker().record_semantic_requeue(msg.telemetry_id)
- self.report_requeue()
except Exception as requeue_err:
logger.error(f"Failed to re-enqueue semantic message: {requeue_err}")
self._merge_request_stats(msg.telemetry_id, error_count=1)
get_request_wait_tracker().mark_semantic_failed(msg.telemetry_id, msg.id, str(error))
- self.report_error(str(error), data)
- return
- self.report_success()
+ return ProcessResult.failed(str(error))
+ return ProcessResult.requeued()
async def _enqueue_parent_refresh(
self, msg: SemanticMsg, uri: str, *, l0_body_changed: bool
@@ -329,7 +327,7 @@ class SemanticProcessor(DequeueHandlerBase):
self,
data: Optional[Dict[str, Any]],
lock: Optional[Dict[str, Any]] = None,
- ) -> Optional[Dict[str, Any]]:
+ ) -> ProcessResult:
"""Process dequeued SemanticMsg, recursively process all subdirectories."""
msg: Optional[SemanticMsg] = None
collector = None
@@ -337,7 +335,7 @@ class SemanticProcessor(DequeueHandlerBase):
import json
if not data:
- return None
+ return ProcessResult.success()
if "data" in data and isinstance(data["data"], str):
data = json.loads(data["data"])
@@ -348,8 +346,7 @@ class SemanticProcessor(DequeueHandlerBase):
logger.warning("Skipping semantic generation for root URI: %s", msg.uri)
if msg.telemetry_id and msg.id:
get_request_wait_tracker().mark_semantic_done(msg.telemetry_id, msg.id)
- self.report_success()
- return None
+ return ProcessResult.success()
if is_semantic_msg_stale(msg):
live_file_changes = {
kind: list(msg.changes.get(kind, []))
@@ -377,8 +374,7 @@ class SemanticProcessor(DequeueHandlerBase):
)
if msg.telemetry_id and msg.id:
get_request_wait_tracker().mark_semantic_done(msg.telemetry_id, msg.id)
- self.report_success()
- return None
+ return ProcessResult.success()
# Circuit breaker: if API is known-broken, re-enqueue and wait
try:
self._circuit_breaker.check()
@@ -389,9 +385,7 @@ class SemanticProcessor(DequeueHandlerBase):
await self._reenqueue_semantic_msg(msg)
self._merge_request_stats(msg.telemetry_id, requeue_count=1)
get_request_wait_tracker().record_semantic_requeue(msg.telemetry_id)
- self.report_requeue()
- self.report_success()
- return None
+ return ProcessResult.requeued()
collector = resolve_telemetry(msg.telemetry_id)
telemetry_ctx = bind_telemetry(collector) if collector is not None else nullcontext()
with telemetry_ctx:
@@ -519,9 +513,8 @@ class SemanticProcessor(DequeueHandlerBase):
get_request_wait_tracker().mark_semantic_done(msg.telemetry_id, msg.id)
self._merge_request_stats(msg.telemetry_id, processed=1)
logger.info(f"Completed semantic generation for: {msg.uri}")
- self.report_success()
self._circuit_breaker.record_success()
- return None
+ return ProcessResult.success()
finally:
reset_root_observability_context(root_context_token)
@@ -534,10 +527,8 @@ class SemanticProcessor(DequeueHandlerBase):
exc_info=True,
)
if msg is not None:
- await self._requeue_semantic_msg_after_error(msg, data, e)
- else:
- self.report_error(str(e), data)
- return None
+ return await self._requeue_semantic_msg_after_error(msg, e)
+ return ProcessResult.failed(str(e))
error_class = classify_api_error(e)
if error_class == ERROR_CLASS_INPUT_TOO_LARGE:
@@ -550,7 +541,7 @@ class SemanticProcessor(DequeueHandlerBase):
get_request_wait_tracker().mark_semantic_failed(
msg.telemetry_id, msg.id, str(e)
)
- self.report_error(str(e), data)
+ return ProcessResult.failed(str(e))
elif error_class == ERROR_CLASS_PERMANENT:
logger.critical(
f"Permanent API error processing semantic message, dropping: {e}",
@@ -562,7 +553,7 @@ class SemanticProcessor(DequeueHandlerBase):
get_request_wait_tracker().mark_semantic_failed(
msg.telemetry_id, msg.id, str(e)
)
- self.report_error(str(e), data)
+ return ProcessResult.failed(str(e))
else:
# Transient or unknown — re-enqueue for retry
logger.warning(
@@ -571,12 +562,10 @@ class SemanticProcessor(DequeueHandlerBase):
)
self._circuit_breaker.record_failure(e)
if msg is not None:
- await self._requeue_semantic_msg_after_error(msg, data, e)
- else:
- self.report_error(str(e), data)
- return None
+ return await self._requeue_semantic_msg_after_error(msg, e)
+ return ProcessResult.failed(str(e))
- async def on_cancelled(self, data: Optional[Dict[str, Any]]) -> Optional[Dict[str, Any]]:
+ async def on_cancelled(self, data: Optional[Dict[str, Any]]) -> ProcessResult:
"""Release a queued semantic lock before cancelled work is ACKed."""
try:
import json
@@ -586,8 +575,7 @@ class SemanticProcessor(DequeueHandlerBase):
payload = json.loads(payload)
msg = SemanticMsg.from_dict(payload)
except (TypeError, ValueError) as exc:
- self.report_error(str(exc), data)
- return None
+ return ProcessResult.failed(str(exc))
if msg.telemetry_id and msg.id:
get_request_wait_tracker().mark_semantic_done(msg.telemetry_id, msg.id)
@@ -598,8 +586,7 @@ class SemanticProcessor(DequeueHandlerBase):
await viking_fs._async_agfs.pathlock_release(lock)
except Exception as exc:
logger.warning("Failed to release cancelled semantic lock: %s", exc)
- self.report_success()
- return None
+ return ProcessResult.cancelled()
def get_dag_stats(self) -> Optional["DagStats"]:
return SemanticDagExecutor.get_active_stats()
diff --git a/openviking/storage/queuefs/session_commit_processor.py b/openviking/storage/queuefs/session_commit_processor.py
index 9af90cbe6..8677f6400 100644
--- a/openviking/storage/queuefs/session_commit_processor.py
+++ b/openviking/storage/queuefs/session_commit_processor.py
@@ -3,7 +3,6 @@
"""Queue consumer for restart-safe Session Phase 2 work."""
import asyncio
-import concurrent.futures
import json
from typing import TYPE_CHECKING, Any, Dict, Optional
@@ -13,8 +12,10 @@ from openviking.observability.context import (
)
from openviking.server.identity import RequestContext, Role
from openviking.service.task_tracker import get_task_tracker
+from openviking.service.task_tracker_concurrency import OwnerLoopDispatcher
from openviking.service.task_work_index import bind_task_context
from openviking.storage.queuefs.named_queue import DequeueHandlerBase
+from openviking.storage.queuefs.process_result import ProcessResult
from openviking.storage.queuefs.session_commit_msg import SessionCommitMsg
from openviking.telemetry.span_models import create_root_span_attributes
from openviking_cli.session.user_id import UserIdentifier
@@ -30,7 +31,7 @@ class SessionCommitProcessor(DequeueHandlerBase):
service_loop: asyncio.AbstractEventLoop,
) -> None:
self._session_service = session_service
- self._service_loop = service_loop
+ self._dispatcher = OwnerLoopDispatcher(service_loop)
@staticmethod
def _parse_message(data: Dict[str, Any]) -> tuple[SessionCommitMsg, RequestContext]:
@@ -47,9 +48,8 @@ class SessionCommitProcessor(DequeueHandlerBase):
async def _process(self, msg: SessionCommitMsg, ctx: RequestContext) -> bool:
# Bind a root observability context so Phase-2 extraction VLM/embedding
# token events are attributed to the committing account/user rather than
- # "__unknown__" (mirrors SemanticProcessor.on_dequeue). Must bind inside
- # this coroutine: on_dequeue hops loops via run_coroutine_threadsafe, so
- # a context bound there would not propagate here.
+ # "__unknown__" (mirrors SemanticProcessor.on_dequeue). Bind inside the
+ # dispatched operation to cover direct calls and restore the caller context.
root_attrs = create_root_span_attributes(
http_method="QUEUE",
http_route="/queuefs/session_commit",
@@ -92,7 +92,6 @@ class SessionCommitProcessor(DequeueHandlerBase):
QueueManager.SESSION_COMMIT,
msg.to_dict(),
)
- self.report_requeue()
return processed
finally:
reset_root_observability_context(root_context_token)
@@ -106,45 +105,25 @@ class SessionCommitProcessor(DequeueHandlerBase):
if await session.exists():
await session.finalize_cancelled_commit(msg.archive_uri)
- async def on_cancelled(self, data: Optional[Dict[str, Any]]) -> Optional[Dict[str, Any]]:
+ async def on_cancelled(self, data: Optional[Dict[str, Any]]) -> ProcessResult:
if not data:
- return None
+ return ProcessResult.cancelled()
try:
msg, ctx = self._parse_message(data)
except (json.JSONDecodeError, KeyError, TypeError, ValueError) as exc:
- self.report_error(str(exc), data)
- return None
+ return ProcessResult.failed(str(exc))
- future = asyncio.run_coroutine_threadsafe(
- self._finalize_cancelled(msg, ctx),
- self._service_loop,
- )
- try:
- await asyncio.wrap_future(future)
- self.report_success()
- except asyncio.CancelledError:
- future.cancel()
- raise
- return None
+ await self._dispatcher.run(lambda: self._finalize_cancelled(msg, ctx))
+ return ProcessResult.cancelled()
- async def on_dequeue(self, data: Optional[Dict[str, Any]]) -> Optional[Dict[str, Any]]:
+ async def on_dequeue(self, data: Optional[Dict[str, Any]]) -> ProcessResult:
if not data:
- return None
+ return ProcessResult.success()
try:
msg, ctx = self._parse_message(data)
except (json.JSONDecodeError, KeyError, TypeError, ValueError) as exc:
- self.report_error(str(exc), data)
- return None
- future: concurrent.futures.Future[bool] = asyncio.run_coroutine_threadsafe(
- self._process(msg, ctx),
- self._service_loop,
- )
- try:
- await asyncio.wrap_future(future)
- self.report_success()
- except asyncio.CancelledError:
- future.cancel()
- raise
- return None
+ return ProcessResult.failed(str(exc))
+ processed = await self._dispatcher.run(lambda: self._process(msg, ctx))
+ return ProcessResult.success() if processed else ProcessResult.requeued()
diff --git a/tests/server/test_admin_rebuild_api.py b/tests/server/test_admin_rebuild_api.py
index a703d595c..b69be2a4b 100644
--- a/tests/server/test_admin_rebuild_api.py
+++ b/tests/server/test_admin_rebuild_api.py
@@ -8,6 +8,7 @@ import pytest
from openviking.core.context import ContextLevel
from openviking.server.identity import RequestContext, Role
+from openviking.storage.queuefs.process_result import ProcessResult
from openviking_cli.exceptions import OpenVikingError, PermissionDeniedError
from openviking_cli.session.user_id import UserIdentifier
from tests.server.test_admin_api import ROOT_KEY
@@ -25,6 +26,14 @@ ROOT_ACCOUNT_HEADERS = {
}
+def semantic_config(monkeypatch):
+ monkeypatch.setattr(
+ "openviking.service.reindex_executor.get_openviking_config",
+ lambda: SimpleNamespace(vlm=SimpleNamespace(max_concurrent=2)),
+ )
+
+
def _make_reindex_run(ctx, counters):
from openviking.service.reindex_executor import _ReindexRunContext
@@ -748,7 +757,9 @@ async def test_reindex_upsert_uses_uri_owner_for_user_scoped_records(monkeypatch
@pytest.mark.asyncio
-async def test_reindex_semantic_processor_uses_uri_owner_for_user_scoped_records(monkeypatch):
+async def test_reindex_semantic_processor_uses_uri_owner_for_user_scoped_records(
+ monkeypatch, semantic_config
+):
from openviking.service.reindex_executor import ReindexExecutor
captured = {}
@@ -760,6 +771,7 @@ async def test_reindex_semantic_processor_uses_uri_owner_for_user_scoped_records
async def on_dequeue(self, payload, lock=None):
captured["payload"] = payload
captured["lock"] = lock
+ return ProcessResult.success()
monkeypatch.setattr(
"openviking.service.reindex_executor.SemanticProcessor",
@@ -1363,6 +1375,7 @@ async def test_reindex_semantic_processor_uses_configured_vlm_concurrency(monkey
async def on_dequeue(self, data, lock=None):
seen["data"] = data
+ return ProcessResult.success()
monkeypatch.setattr(reindex_mod, "SemanticProcessor", FakeSemanticProcessor)
monkeypatch.setattr(
@@ -2131,7 +2144,9 @@ async def test_reindex_resource_vectors_parallelize_files_and_isolate_failures(m
@pytest.mark.asyncio
-async def test_reindex_semantic_processor_runs_with_skip_vectorization(monkeypatch):
+async def test_reindex_semantic_processor_runs_with_skip_vectorization(
+ monkeypatch, semantic_config
+):
from openviking.service.reindex_executor import ReindexExecutor
seen = {}
@@ -2142,6 +2157,7 @@ async def test_reindex_semantic_processor_runs_with_skip_vectorization(monkeypat
async def on_dequeue(self, payload, lock=None):
seen["payload"] = payload
+ return ProcessResult.success()
monkeypatch.setattr(
"openviking.service.reindex_executor.SemanticProcessor",
@@ -2170,7 +2186,9 @@ async def test_reindex_semantic_processor_runs_with_skip_vectorization(monkeypat
@pytest.mark.asyncio
-async def test_reindex_semantic_processor_passes_non_recursive_message(monkeypatch):
+async def test_reindex_semantic_processor_passes_non_recursive_message(
+ monkeypatch, semantic_config
+):
from openviking.service.reindex_executor import ReindexExecutor
seen = {}
@@ -2181,6 +2199,7 @@ async def test_reindex_semantic_processor_passes_non_recursive_message(monkeypat
async def on_dequeue(self, payload, lock=None):
seen["payload"] = payload
+ return ProcessResult.success()
monkeypatch.setattr(
"openviking.service.reindex_executor.SemanticProcessor",
@@ -2206,7 +2225,9 @@ async def test_reindex_semantic_processor_passes_non_recursive_message(monkeypat
@pytest.mark.asyncio
-async def test_reindex_semantic_processor_sets_memory_aggregation_policy(monkeypatch):
+async def test_reindex_semantic_processor_sets_memory_aggregation_policy(
+ monkeypatch, semantic_config
+):
from openviking.service.reindex_executor import ReindexExecutor
seen = {}
@@ -2217,6 +2238,7 @@ async def test_reindex_semantic_processor_sets_memory_aggregation_policy(monkeyp
async def on_dequeue(self, payload, lock=None):
seen["payload"] = payload
+ return ProcessResult.success()
monkeypatch.setattr(
"openviking.service.reindex_executor.SemanticProcessor",
diff --git a/tests/service/test_task_queue_middleware.py b/tests/service/test_task_queue_middleware.py
new file mode 100644
index 000000000..4a36bb19e
--- /dev/null
+++ b/tests/service/test_task_queue_middleware.py
@@ -0,0 +1,488 @@
+# Copyright (c) 2026 Beijing Volcano Engine Technology Co., Ltd.
+# SPDX-License-Identifier: AGPL-3.0
+"""Legacy task-index behavior through the middleware boundary."""
+
+import asyncio
+import json
+from unittest.mock import AsyncMock
+
+import pytest
+
+from openviking.pyagfs import AsyncAGFSClient
+from openviking.service.task_queue_middleware import TaskWorkQueueMiddleware
+from openviking.service.task_work_index import (
+ TaskWorkIndex,
+ TaskWorkRejected,
+ bind_task_context,
+ extract_task_metadata,
+ get_task_context,
+)
+from openviking.storage.queuefs.named_queue import DequeueHandlerBase, NamedQueue
+from openviking.storage.queuefs.process_result import ProcessOutcome, ProcessResult
+from openviking.storage.queuefs.queue_middleware import QueueMiddleware
+from openviking.storage.queuefs.session_commit_processor import SessionCommitProcessor
+
+
+def tracked_queue(monkeypatch):
+ transport = AsyncMock(spec=AsyncAGFSClient)
+ transport.write.return_value = "message-1"
+ # Tests override transport.read.return_value to feed /messages snapshots;
+ # status reads must still return the backend JSON shape.
+ transport.read.return_value = b"0"
+
+ async def read(path):
+ if path.endswith("/status"):
+ return b'{"pending":0,"processing":0}'
+ value = transport.read.return_value
+ return value
+
+ transport.read.side_effect = read
+ monkeypatch.setattr(
+ "openviking.storage.queuefs.named_queue.AsyncAGFSClient", lambda _client: transport
+ )
+ index = TaskWorkIndex()
+ finalize = AsyncMock()
+ cancelled = set()
+ index.set_callbacks(
+ finalize_before_ack=finalize,
+ is_cancellation_requested=lambda task_id: task_id in cancelled,
+ )
+ queue = NamedQueue(object(), "/queue", "Test", middlewares=[TaskWorkQueueMiddleware(index)])
+ return queue, index, transport, finalize, cancelled
+
+
+async def enqueue_task(queue, transport, task_id="task-1"):
+ with bind_task_context(task_id, "account", "user"):
+ await queue.enqueue({"value": 1})
+ return {"id": "message-1", "data": transport.write.await_args.args[1].decode()}
+
+
+async def test_enqueue_registers_before_write_and_ack_finalizes_before_delete(tracked_queue):
+ queue, index, transport, finalize, _ = tracked_queue
+
+ async def write(path, data):
+ assert path.endswith("/enqueue")
+ assert index.has_work("task-1")
+ return "message-1"
+
+ transport.write.side_effect = write
+ message = await enqueue_task(queue, transport)
+ metadata = extract_task_metadata(message)
+ assert metadata.task_id == "task-1"
+ assert metadata.account_id == "account"
+ assert metadata.user_id == "user"
+ assert get_task_context() is None
+
+ async def ack_write(path, data):
+ assert path.endswith("/ack")
+ assert not index.has_work("task-1")
+ finalize.assert_awaited_once_with(metadata)
+ return ""
+
+ transport.write.side_effect = ack_write
+ await queue.ack("message-1", message)
+ assert not index.has_work("task-1")
+
+
[email protected]("failure", [OSError("write failed"), asyncio.CancelledError()])
+async def test_enqueue_failure_discards_registered_work(tracked_queue, failure):
+ queue, index, transport, finalize, _ = tracked_queue
+ transport.write.side_effect = failure
+ with bind_task_context("task-1", "account", "user"):
+ with pytest.raises(type(failure)):
+ await queue.enqueue({})
+ assert not index.has_work("task-1")
+ finalize.assert_awaited_once()
+
+
+async def test_cancelled_task_rejects_enqueue_without_write(tracked_queue):
+ queue, index, transport, _, cancelled = tracked_queue
+ cancelled.add("task-1")
+ with bind_task_context("task-1", "account", "user"):
+ with pytest.raises(TaskWorkRejected, match="cancelling"):
+ await queue.enqueue({})
+ transport.write.assert_not_awaited()
+ assert not index.has_work("task-1")
+
+
[email protected]("operation", ["ack", "clear"])
[email protected]("failure", [OSError("write failed"), asyncio.CancelledError()])
+async def test_failed_ack_or_clear_restores_work(tracked_queue, operation, failure):
+ queue, index, transport, finalize, _ = tracked_queue
+ message = await enqueue_task(queue, transport)
+ transport.read.return_value = json.dumps([message]).encode()
+ transport.write.reset_mock()
+ transport.write.side_effect = failure
+
+ async def invoke():
+ if operation == "ack":
+ return await queue.ack("message-1", message)
+ return await queue.clear()
+
+ if isinstance(failure, asyncio.CancelledError):
+ with pytest.raises(asyncio.CancelledError):
+ await invoke()
+ elif operation == "clear":
+ assert await invoke() is False
+ else:
+ assert await invoke() is None
+ finalize.assert_awaited_once()
+ assert index.has_work("task-1")
+
+
[email protected]("operation", ["ack", "clear"])
+async def test_finalization_failure_prevents_transport_delete(tracked_queue, operation):
+ queue, index, transport, finalize, _ = tracked_queue
+ message = await enqueue_task(queue, transport)
+ transport.read.return_value = json.dumps([message]).encode()
+ transport.write.reset_mock()
+ finalize.side_effect = OSError("task store unavailable")
+ if operation == "ack":
+ await queue.ack("message-1", message)
+ else:
+ assert await queue.clear() is False
+ transport.write.assert_not_awaited()
+ assert index.has_work("task-1")
+
+
[email protected]("operation", ["enqueue", "ack", "clear"])
+async def test_inner_short_circuit_compensates_uncommitted_index_change(tracked_queue, operation):
+ queue, index, transport, _, _ = tracked_queue
+
+ class ShortCircuit(QueueMiddleware):
+ async def invoke(self, ctx, call_next):
+ return "deduplicated"
+
+ setattr(ShortCircuit, operation, ShortCircuit.invoke)
+ queue = NamedQueue(
+ object(), "/queue", "Test", middlewares=[TaskWorkQueueMiddleware(index), ShortCircuit()]
+ )
+ message = await enqueue_task(queue, transport) if operation != "enqueue" else None
+ transport.write.reset_mock()
+ if operation == "enqueue":
+ with bind_task_context("task-1", "account", "user"):
+ assert await queue.enqueue({}) == "deduplicated"
+ assert not index.has_work("task-1")
+ elif operation == "ack":
+ await queue.ack("message-1", message)
+ assert index.has_work("task-1")
+ else:
+ transport.read.return_value = json.dumps([message]).encode()
+ await queue.clear()
+ assert index.has_work("task-1")
+ transport.write.assert_not_awaited()
+
+
[email protected]("operation", ["enqueue", "ack", "clear"])
+async def test_post_commit_error_does_not_undo_index_change(tracked_queue, operation):
+ queue, index, transport, _, _ = tracked_queue
+
+ class FailAfter(QueueMiddleware):
+ async def invoke(self, ctx, call_next):
+ await call_next(ctx)
+ raise RuntimeError("after commit")
+
+ setattr(FailAfter, operation, FailAfter.invoke)
+ queue = NamedQueue(
+ object(), "/queue", "Test", middlewares=[TaskWorkQueueMiddleware(index), FailAfter()]
+ )
+ if operation == "enqueue":
+ with bind_task_context("task-1", "account", "user"):
+ with pytest.raises(RuntimeError, match="after commit"):
+ await queue.enqueue({})
+ assert index.has_work("task-1")
+ else:
+ message = await enqueue_task(queue, transport)
+ transport.read.return_value = json.dumps([message]).encode()
+ if operation == "ack":
+ await queue.ack("message-1", message)
+ else:
+ assert await queue.clear() is False
+ assert not index.has_work("task-1")
+
+
+async def test_clear_rolls_back_all_prepared_messages(tracked_queue):
+ queue, index, transport, finalize, _ = tracked_queue
+ first = await enqueue_task(queue, transport, "first")
+ second = await enqueue_task(queue, transport, "second")
+ transport.read.return_value = json.dumps([first, second]).encode()
+
+ async def finish(metadata):
+ if metadata.task_id == "second":
+ raise OSError("second task failed")
+
+ finalize.side_effect = finish
+ transport.write.reset_mock()
+ assert await queue.clear() is False
+ transport.write.assert_not_awaited()
+ assert index.has_work("first") and index.has_work("second")
+ finalize.side_effect = None
+ assert await queue.clear()
+ assert not index.has_work("first") and not index.has_work("second")
+
+
+async def test_process_result_tracks_children_and_errors(tracked_queue):
+ queue, index, transport, _, _ = tracked_queue
+ message = await enqueue_task(queue, transport)
+ contexts = []
+
+ class Handler(DequeueHandlerBase):
+ async def on_dequeue(self, data):
+ contexts.append(get_task_context())
+ await queue.enqueue({"child": True})
+ return ProcessResult.failed("failed work")
+
+ queue.set_dequeue_handler(Handler())
+ assert (await queue.process_dequeued(message)).outcome is ProcessOutcome.FAILED
+ assert contexts[0].task_id == "task-1"
+ assert get_task_context() is None
+ assert index.failure("task-1") == "failed work"
+ child_payload = next(
+ call.args[1]
+ for call in reversed(transport.write.await_args_list)
+ if call.args[0].endswith("/enqueue")
+ )
+ child = {"id": "child", "data": child_payload.decode()}
+ assert index.has_work("task-1")
+ await queue.ack("message-1", message)
+ assert index.has_work("task-1")
+ await queue.ack("child", child)
+ assert not index.has_work("task-1")
+ status = await queue.get_status()
+ assert status.error_count == 1
+ assert status.in_progress == 0
+
+
+async def test_handler_exception_records_task_failure_and_propagates(tracked_queue):
+ queue, index, transport, _, _ = tracked_queue
+ message = await enqueue_task(queue, transport)
+
+ class Handler(DequeueHandlerBase):
+ async def on_dequeue(self, data):
+ raise RuntimeError("handler failed")
+
+ queue.set_dequeue_handler(Handler())
+ with pytest.raises(RuntimeError, match="handler failed"):
+ await queue.process_dequeued(message)
+ assert index.failure("task-1") == "handler failed"
+ assert index.has_work("task-1")
+
+
+async def test_concurrent_results_update_each_task(tracked_queue):
+ queue, index, transport, _, _ = tracked_queue
+ first = await enqueue_task(queue, transport, "first")
+ second = await enqueue_task(queue, transport, "second")
+ both_started = asyncio.Event()
+ started = 0
+
+ class Handler(DequeueHandlerBase):
+ async def on_dequeue(self, data):
+ nonlocal started
+ started += 1
+ if started == 2:
+ both_started.set()
+ await both_started.wait()
+ task_id = get_task_context().task_id
+ return ProcessResult.failed(f"{task_id} failed")
+
+ queue.set_dequeue_handler(Handler())
+ await asyncio.wait_for(
+ asyncio.gather(queue.process_dequeued(first), queue.process_dequeued(second)),
+ timeout=1,
+ )
+ assert index.failure("first") == "first failed"
+ assert index.failure("second") == "second failed"
+
+
+async def test_cancel_before_handler_uses_legacy_discard_callback(tracked_queue):
+ queue, index, transport, _, cancelled = tracked_queue
+ message = await enqueue_task(queue, transport)
+ cancelled.add("task-1")
+ calls = []
+
+ class Handler(DequeueHandlerBase):
+ async def on_dequeue(self, data):
+ pytest.fail("cancelled task reached handler")
+
+ async def on_cancelled(self, data):
+ calls.append(data)
+ return await super().on_cancelled(data)
+
+ queue.set_dequeue_handler(Handler())
+ assert (await queue.process_dequeued(message)).outcome is ProcessOutcome.CANCELLED
+ assert calls == [message]
+ await queue.ack("message-1", message)
+ assert not index.has_work("task-1")
+ status = await queue.get_status()
+ assert status.processed == 1 and status.in_progress == 0
+
+
[email protected]("user_cancel", [True, False])
+async def test_active_cancellation_distinguishes_user_from_shutdown(tracked_queue, user_cancel):
+ queue, index, transport, _, cancelled = tracked_queue
+ message = await enqueue_task(queue, transport)
+ started = asyncio.Event()
+
+ class Handler(DequeueHandlerBase):
+ async def on_dequeue(self, data):
+ started.set()
+ await asyncio.Event().wait()
+
+ queue.set_dequeue_handler(Handler())
+ running = asyncio.create_task(queue.process_dequeued(message))
+ await asyncio.wait_for(started.wait(), timeout=1)
+ try:
+ if user_cancel:
+ cancelled.add("task-1")
+ index.cancel_active("task-1")
+ result = await asyncio.wait_for(running, timeout=1)
+ assert result.outcome is ProcessOutcome.CANCELLED
+ await queue.ack("message-1", message)
+ assert not index.has_work("task-1")
+ assert (await queue.get_status()).processed == 1
+ else:
+ running.cancel()
+ with pytest.raises(asyncio.CancelledError):
+ await running
+ assert index.has_work("task-1")
+ finally:
+ if not running.done():
+ running.cancel()
+ await asyncio.gather(running, return_exceptions=True)
+
+
+async def test_shared_middleware_keeps_concurrent_contexts_isolated(tracked_queue):
+ queue, index, transport, _, _ = tracked_queue
+ first = await enqueue_task(queue, transport, "first")
+ second = await enqueue_task(queue, transport, "second")
+ both_started = asyncio.Event()
+ seen = []
+
+ class Handler(DequeueHandlerBase):
+ async def on_dequeue(self, data):
+ before = get_task_context().task_id
+ seen.append(before)
+ if len(seen) == 2:
+ both_started.set()
+ await both_started.wait()
+ assert get_task_context().task_id == before
+ return ProcessResult.success(data)
+
+ queue.set_dequeue_handler(Handler())
+ await asyncio.wait_for(
+ asyncio.gather(queue.process_dequeued(first), queue.process_dequeued(second)), timeout=1
+ )
+ assert set(seen) == {"first", "second"}
+ assert get_task_context() is None
+ await queue.ack("first", first)
+ await queue.ack("second", second)
+ assert not index.has_work("first") and not index.has_work("second")
+
+
+async def test_awaited_child_result_is_recorded_before_ack(tracked_queue):
+ queue, index, transport, finalize, _ = tracked_queue
+ message = await enqueue_task(queue, transport)
+
+ class Handler(DequeueHandlerBase):
+ async def on_dequeue(self, data):
+ async def child():
+ await asyncio.sleep(0)
+ return ProcessResult.failed("child failure")
+
+ return await asyncio.create_task(child())
+
+ async def finish(metadata):
+ assert index.failure(metadata.task_id) == "child failure"
+
+ finalize.side_effect = finish
+ queue.set_dequeue_handler(Handler())
+ await queue.process_dequeued(message)
+ finalize.assert_not_awaited()
+ await queue.ack("message-1", message)
+ finalize.assert_awaited_once()
+
+
+async def test_cancel_cleanup_interruption_does_not_ack(tracked_queue):
+ queue, index, transport, _, cancelled = tracked_queue
+ message = await enqueue_task(queue, transport)
+ cancelled.add("task-1")
+ transport.write.reset_mock()
+
+ class Handler(DequeueHandlerBase):
+ async def on_dequeue(self, data):
+ pytest.fail("cancelled work must not execute")
+
+ async def on_cancelled(self, data):
+ raise asyncio.CancelledError
+
+ queue.set_dequeue_handler(Handler())
+ with pytest.raises(asyncio.CancelledError):
+ await queue.process_dequeued(message)
+ assert index.has_work("task-1")
+ transport.write.assert_not_awaited()
+
+
+async def test_cross_loop_cancellation_waits_for_handler_cleanup(tracked_queue):
+ queue, index, transport, _, cancelled = tracked_queue
+ started = asyncio.Event()
+ cleanup_started = asyncio.Event()
+ cleanup_release = asyncio.Event()
+
+ class Session:
+ async def exists(self):
+ return True
+
+ async def load(self):
+ pass
+
+ async def resume_queued_commit(self, msg):
+ started.set()
+ try:
+ await asyncio.Event().wait()
+ finally:
+ cleanup_started.set()
+ await cleanup_release.wait()
+
+ class Service:
+ def session(self, ctx, session_id, session_uri=None):
+ return Session()
+
+ from openviking.storage.queuefs.session_commit_msg import SessionCommitMsg
+ from openviking_cli.session.user_id import UserIdentifier
+
+ msg = SessionCommitMsg(
+ task_id="task-1",
+ session_id="session-1",
+ session_uri="viking://session/session-1",
+ archive_uri="viking://session/session-1/history/archive-1",
+ user=UserIdentifier("account", "user").to_dict(),
+ )
+ with bind_task_context("task-1", "account", "user"):
+ await queue.enqueue(msg.to_dict())
+ message = {"id": "m", "data": transport.write.await_args.args[1].decode()}
+ transport.write.reset_mock()
+ queue.set_dequeue_handler(SessionCommitProcessor(Service(), asyncio.get_running_loop()))
+
+ def consume():
+ return asyncio.run(queue.process_dequeued(message))
+
+ worker = asyncio.create_task(asyncio.to_thread(consume))
+ try:
+ await asyncio.wait_for(started.wait(), timeout=3)
+ cancelled.add("task-1")
+ index.cancel_active("task-1")
+ await asyncio.wait_for(cleanup_started.wait(), timeout=3)
+ assert not worker.done()
+ transport.write.assert_not_awaited()
+ finally:
+ cleanup_release.set()
+ if not cleanup_started.is_set():
+ cancelled.add("task-1")
+ index.cancel_active("task-1")
+ result = await asyncio.wait_for(worker, timeout=3)
+ assert result.outcome is ProcessOutcome.CANCELLED
+ transport.write.assert_not_awaited()
+ await queue.ack("m", message)
+ assert not index.has_work("task-1")
+ transport.write.assert_awaited_once_with("/queue/Test/ack", b"m")
diff --git a/tests/storage/test_collection_schemas.py b/tests/storage/test_collection_schemas.py
index 8f9e0a4ce..c62aba5b8 100644
--- a/tests/storage/test_collection_schemas.py
+++ b/tests/storage/test_collection_schemas.py
@@ -26,6 +26,8 @@ from openviking.storage.errors import (
)
from openviking.storage.expr import Eq
from openviking.storage.queuefs.embedding_msg import EmbeddingMsg
+from openviking.storage.queuefs.process_result import ProcessOutcome
+from openviking.storage.vector_ids import vector_record_id
from openviking.storage.vectordb import engine as vectordb_engine
from openviking.storage.vectordb.collection.result import UpsertDataResult
from openviking.storage.vectordb.collection.vikingdb_clients import VikingDBClient
@@ -189,12 +191,8 @@ async def test_init_context_collection_backfills_metadata_for_empty_legacy_colle
schema_updates = []
config = _DummyConfig(_DummyEmbedder(), backend="local")
existing_schema = CollectionSchemas.context_collection("context", config.embedding.dimension)
- existing_fields = [
- field for field in existing_schema["Fields"] if field["FieldName"] != "tags"
- ]
- existing_scalar_index = [
- field for field in existing_schema["ScalarIndex"] if field != "tags"
- ]
+ existing_fields = [field for field in existing_schema["Fields"] if field["FieldName"] != "tags"]
+ existing_scalar_index = [field for field in existing_schema["ScalarIndex"] if field != "tags"]
class _FakeStorage:
async def create_collection(self, name, schema):
@@ -302,20 +300,13 @@ async def test_embedding_handler_skip_all_work_when_manager_is_closing(monkeypat
)
handler = TextEmbeddingHandler(_ClosingVikingDB())
- status = {"success": 0, "requeue": 0, "error": 0}
- handler.set_callbacks(
- on_success=lambda: status.__setitem__("success", status["success"] + 1),
- on_requeue=lambda: status.__setitem__("requeue", status["requeue"] + 1),
- on_error=lambda *_: status.__setitem__("error", status["error"] + 1),
- )
result = await handler.on_dequeue(_build_queue_payload())
- assert result is None
+ assert result.outcome is ProcessOutcome.SUCCESS
+ assert result.value is None
+ assert result.error is None
assert embedder.calls == 0
- assert status["success"] == 1
- assert status["requeue"] == 0
- assert status["error"] == 0
@pytest.mark.asyncio
@@ -342,12 +333,6 @@ async def test_embedding_handler_open_breaker_logs_summary_instead_of_per_item_w
)
handler = TextEmbeddingHandler(_QueueingVikingDB())
- status = {"success": 0, "requeue": 0, "error": 0}
- handler.set_callbacks(
- on_success=lambda: status.__setitem__("success", status["success"] + 1),
- on_requeue=lambda: status.__setitem__("requeue", status["requeue"] + 1),
- on_error=lambda *_: status.__setitem__("error", status["error"] + 1),
- )
monkeypatch.setattr(
handler._circuit_breaker,
"check",
@@ -356,18 +341,22 @@ async def test_embedding_handler_open_breaker_logs_summary_instead_of_per_item_w
import openviking.storage.collection_schemas as collection_schemas
+ monkeypatch.setattr(collection_schemas.logger, "propagate", False)
collection_schemas.logger.addHandler(caplog.handler)
collection_schemas.logger.setLevel(logging.WARNING)
try:
with caplog.at_level(logging.WARNING):
- await handler.on_dequeue(_build_queue_payload())
- await handler.on_dequeue(_build_queue_payload())
+ first_result = await handler.on_dequeue(_build_queue_payload())
+ second_result = await handler.on_dequeue(_build_queue_payload())
finally:
collection_schemas.logger.removeHandler(caplog.handler)
warnings = [record.message for record in caplog.records if record.levelno == logging.WARNING]
assert warnings.count("Embedding circuit breaker is open; re-enqueueing messages") == 1
- assert status == {"success": 2, "requeue": 2, "error": 0}
+ for result in (first_result, second_result):
+ assert result.outcome is ProcessOutcome.REQUEUED
+ assert result.value is None
+ assert result.error is None
@pytest.mark.asyncio
@@ -397,18 +386,17 @@ async def test_embedding_auth_error_fails_terminally_without_reenqueue(monkeypat
lambda: _DummyConfig(_AuthErrorEmbedder()),
)
handler = TextEmbeddingHandler(vikingdb)
- status = {"success": 0, "requeue": 0, "error": 0}
- handler.set_callbacks(
- on_success=lambda: status.__setitem__("success", status["success"] + 1),
- on_requeue=lambda: status.__setitem__("requeue", status["requeue"] + 1),
- on_error=lambda *_: status.__setitem__("error", status["error"] + 1),
- )
result = await handler.on_dequeue(_build_queue_payload_for_account("acct"))
- assert result is None
+ assert result.outcome is ProcessOutcome.FAILED
+ assert result.value is None
+ assert result.error == (
+ "Failed to generate embedding: "
+ "Error code: 401 - {'code': 'AuthenticationError'} Unauthorized "
+ "(uri=viking://resources/sample)"
+ )
assert vikingdb.enqueued == [] # terminal: not re-enqueued
- assert status == {"success": 0, "requeue": 0, "error": 1}
handler._circuit_breaker.check() # breaker not tripped (would raise if open)
@@ -435,21 +423,14 @@ async def test_embedding_handler_treats_shutdown_write_lock_as_success(monkeypat
vikingdb = _ClosingDuringUpsertVikingDB()
handler = TextEmbeddingHandler(vikingdb)
- status = {"success": 0, "requeue": 0, "error": 0}
- handler.set_callbacks(
- on_success=lambda: status.__setitem__("success", status["success"] + 1),
- on_requeue=lambda: status.__setitem__("requeue", status["requeue"] + 1),
- on_error=lambda *_: status.__setitem__("error", status["error"] + 1),
- )
result = await handler.on_dequeue(_build_queue_payload())
- assert result is None
+ assert result.outcome is ProcessOutcome.SUCCESS
+ assert result.value is None
+ assert result.error is None
assert vikingdb.calls == 1
assert embedder.calls == 1
- assert status["success"] == 1
- assert status["requeue"] == 0
- assert status["error"] == 0
@pytest.mark.asyncio
@@ -639,18 +620,17 @@ async def test_embedding_handler_drops_input_too_large_without_requeue(monkeypat
)
handler = TextEmbeddingHandler(vikingdb)
- status = {"success": 0, "requeue": 0, "error": 0}
- handler.set_callbacks(
- on_success=lambda: status.__setitem__("success", status["success"] + 1),
- on_requeue=lambda: status.__setitem__("requeue", status["requeue"] + 1),
- on_error=lambda *_: status.__setitem__("error", status["error"] + 1),
- )
result = await handler.on_dequeue(_build_queue_payload())
- assert result is None
+ assert result.outcome is ProcessOutcome.FAILED
+ assert result.value is None
+ assert result.error == (
+ "Failed to generate embedding: "
+ "Malformed input request: expected maxLength: 50000, actual: 75000 "
+ "(uri=viking://resources/sample)"
+ )
assert vikingdb.enqueued == []
- assert status == {"success": 0, "requeue": 0, "error": 1}
assert handler._circuit_breaker._failure_count == 0
@@ -682,7 +662,13 @@ async def test_embedding_handler_preserves_parent_uri_for_backend_upsert_logic(m
result = await handler.on_dequeue(payload)
- assert result is not None
+ assert result.outcome is ProcessOutcome.SUCCESS
+ assert result.error is None
+ assert result.value == {
+ **queue_data["context_data"],
+ "id": vector_record_id("default", "viking://resources/sample", 2),
+ "vector": [0.1, 0.2],
+ }
assert "data" in captured
assert captured["data"]["parent_uri"] == "viking://resources"
diff --git a/tests/storage/test_memory_semantic_stall.py b/tests/storage/test_memory_semantic_stall.py
index 3dbf9fb1e..7267e1bf8 100644
--- a/tests/storage/test_memory_semantic_stall.py
+++ b/tests/storage/test_memory_semantic_stall.py
@@ -4,7 +4,7 @@
"""Tests for memory semantic queue stall fix (issue #864).
Ensures that _process_memory_directory() error paths propagate exceptions
-so that on_dequeue() always calls report_success() or report_error().
+so that on_dequeue() returns an explicit processing outcome.
"""
from types import SimpleNamespace
@@ -12,6 +12,7 @@ from unittest.mock import AsyncMock, MagicMock, patch
import pytest
+from openviking.storage.queuefs.process_result import ProcessOutcome
from openviking.storage.queuefs.semantic_msg import SemanticMsg
from openviking.storage.queuefs.semantic_processor import SemanticProcessor
@@ -44,17 +45,19 @@ def _build_data(msg: SemanticMsg) -> dict:
@pytest.mark.asyncio
async def test_root_semantic_message_is_acknowledged_without_processing():
processor = SemanticProcessor()
- success = MagicMock()
- processor.set_callbacks(success, MagicMock(), MagicMock())
- await processor.on_dequeue(_build_data(_make_msg(uri="viking://", context_type="resource")))
+ result = await processor.on_dequeue(
+ _build_data(_make_msg(uri="viking://", context_type="resource"))
+ )
- success.assert_called_once_with()
+ assert result.outcome is ProcessOutcome.SUCCESS
+ assert result.value is None
+ assert result.error is None
@pytest.mark.asyncio
-async def test_memory_empty_dir_still_reports_success():
- """When viking_fs.ls returns an empty list, report_success() must be called."""
+async def test_memory_empty_dir_still_returns_success():
+ """An empty memory directory must return a successful processing result."""
processor = SemanticProcessor()
fake_fs = MagicMock()
@@ -63,20 +66,6 @@ async def test_memory_empty_dir_still_reports_success():
msg = _make_msg()
data = _build_data(msg)
- success_called = False
-
- def on_success():
- nonlocal success_called
- success_called = True
-
- error_called = False
-
- def on_error(error_msg, error_data=None):
- nonlocal error_called
- error_called = True
-
- processor.set_callbacks(on_success, lambda: None, on_error)
-
with (
patch(
"openviking.storage.queuefs.semantic_processor.get_viking_fs",
@@ -87,18 +76,19 @@ async def test_memory_empty_dir_still_reports_success():
return_value=None,
),
):
- await processor.on_dequeue(data)
+ result = await processor.on_dequeue(data)
- assert success_called, "report_success() was not called for empty memory directory"
- assert not error_called, "report_error() should not be called for empty directory"
+ assert result.outcome is ProcessOutcome.SUCCESS
+ assert result.value is None
+ assert result.error is None
@pytest.mark.asyncio
-async def test_memory_ls_error_reports_error():
- """When viking_fs.ls raises a filesystem error, report_error() must be called.
+async def test_memory_ls_error_returns_failed():
+ """A permanent filesystem error must return a failed processing result.
- Uses a real classify_api_error (no mock) — FileNotFoundError is classified
- as permanent by the real classifier, so the processor calls report_error().
+ Uses a real classify_api_error (no mock): FileNotFoundError is classified
+ as permanent by the real classifier, so the processor returns FAILED.
"""
processor = SemanticProcessor()
@@ -108,22 +98,6 @@ async def test_memory_ls_error_reports_error():
msg = _make_msg()
data = _build_data(msg)
- success_called = False
-
- def on_success():
- nonlocal success_called
- success_called = True
-
- error_called = False
- error_info = {}
-
- def on_error(error_msg, error_data=None):
- nonlocal error_called, error_info
- error_called = True
- error_info["msg"] = error_msg
-
- processor.set_callbacks(on_success, lambda: None, on_error)
-
with (
patch(
"openviking.storage.queuefs.semantic_processor.get_viking_fs",
@@ -134,11 +108,11 @@ async def test_memory_ls_error_reports_error():
return_value=None,
),
):
- await processor.on_dequeue(data)
+ result = await processor.on_dequeue(data)
- assert error_called, "report_error() was not called when ls() raised an exception"
- assert not success_called, "report_success() should not be called on ls() error"
- assert "/memories not found" in error_info["msg"]
+ assert result.outcome is ProcessOutcome.FAILED
+ assert result.value is None
+ assert result.error == (f"Failed to list memory directory {msg.uri}: /memories not found")
@pytest.mark.asyncio
@@ -147,8 +121,8 @@ async def test_memory_ls_transient_error_requeues():
A 500-class error wrapped by the processor's `raise RuntimeError(...) from e`
is classified as `transient`. The outer on_dequeue() path must call
- _reenqueue_semantic_msg(), bump requeue_count, and fire both report_requeue()
- and report_success() — not report_error().
+ _reenqueue_semantic_msg(), bump requeue_count, and return REQUEUED without
+ a terminal error.
"""
processor = SemanticProcessor()
@@ -158,24 +132,6 @@ async def test_memory_ls_transient_error_requeues():
msg = _make_msg(telemetry_id="tel-1")
data = _build_data(msg)
- success_called = False
- requeue_called = False
- error_called = False
-
- def on_success():
- nonlocal success_called
- success_called = True
-
- def on_requeue():
- nonlocal requeue_called
- requeue_called = True
-
- def on_error(error_msg, error_data=None):
- nonlocal error_called
- error_called = True
-
- processor.set_callbacks(on_success, on_requeue, on_error)
-
reenqueue_mock = AsyncMock()
with (
@@ -189,20 +145,20 @@ async def test_memory_ls_transient_error_requeues():
),
patch.object(processor, "_reenqueue_semantic_msg", new=reenqueue_mock),
):
- await processor.on_dequeue(data)
+ result = await processor.on_dequeue(data)
- assert requeue_called, "report_requeue() must fire for transient errors"
- assert success_called, "report_success() must fire after successful re-enqueue"
- assert not error_called, "report_error() must NOT fire for transient errors"
+ assert result.outcome is ProcessOutcome.REQUEUED
+ assert result.value is None
+ assert result.error is None
reenqueue_mock.assert_awaited_once()
@pytest.mark.asyncio
-async def test_memory_write_error_reports_error():
- """When abstract/overview write raises PermissionError, report_error() is called.
+async def test_memory_write_error_returns_failed():
+ """An abstract/overview PermissionError must return a failed processing result.
- Exercises the write failure path with real classify_api_error — PermissionError
- is classified as permanent, so the processor calls report_error().
+ Exercises the write failure path with real classify_api_error: PermissionError
+ is classified as permanent, so the processor returns FAILED.
"""
processor = SemanticProcessor()
@@ -210,9 +166,7 @@ async def test_memory_write_error_reports_error():
fake_fs.ls = AsyncMock(return_value=[{"name": "file1.md", "isDir": False}])
fake_fs.read_file = AsyncMock(return_value="some content")
fake_fs.write_file = AsyncMock(side_effect=PermissionError("Permission denied"))
- fake_fs._async_agfs.pathlock_acquire_exact_batch = AsyncMock(
- return_value={"lease_ref": "test"}
- )
+ fake_fs._async_agfs.pathlock_acquire_exact_batch = AsyncMock(return_value={"lease_ref": "test"})
fake_fs._async_agfs.pathlock_release = AsyncMock()
fake_fs._uri_to_path = MagicMock(
side_effect=lambda uri, ctx=None: f"/local/acc1/{uri.removeprefix('viking://')}"
@@ -221,22 +175,6 @@ async def test_memory_write_error_reports_error():
msg = _make_msg(skip_vectorization=True)
data = _build_data(msg)
- success_called = False
-
- def on_success():
- nonlocal success_called
- success_called = True
-
- error_called = False
- error_info = {}
-
- def on_error(error_msg, error_data=None):
- nonlocal error_called, error_info
- error_called = True
- error_info["msg"] = error_msg
-
- processor.set_callbacks(on_success, lambda: None, on_error)
-
with (
patch(
"openviking.storage.queuefs.semantic_processor.get_viking_fs",
@@ -266,8 +204,8 @@ async def test_memory_write_error_reports_error():
new=AsyncMock(return_value="# Overview\ntest overview"),
),
):
- await processor.on_dequeue(data)
+ result = await processor.on_dequeue(data)
- assert error_called, "report_error() was not called when write() raised PermissionError"
- assert not success_called, "report_success() should not be called on write error"
- assert "Permission denied" in error_info["msg"]
+ assert result.outcome is ProcessOutcome.FAILED
+ assert result.value is None
+ assert result.error == (f"Failed to write abstract/overview for {msg.uri}: Permission denied")
diff --git a/tests/storage/test_named_queue_middleware.py b/tests/storage/test_named_queue_middleware.py
new file mode 100644
index 000000000..1ac31f2d9
--- /dev/null
+++ b/tests/storage/test_named_queue_middleware.py
@@ -0,0 +1,262 @@
+# Copyright (c) 2026 Beijing Volcano Engine Technology Co., Ltd.
+# SPDX-License-Identifier: AGPL-3.0
+"""Queue middleware ordering and transport compatibility."""
+
+import asyncio
+import json
+from unittest.mock import AsyncMock
+
+import pytest
+
+from openviking.pyagfs import AsyncAGFSClient
+from openviking.storage.queuefs.named_queue import (
+ DequeueHandlerBase,
+ NamedQueue,
+)
+from openviking.storage.queuefs.process_result import ProcessOutcome, ProcessResult
+from openviking.storage.queuefs.queue_middleware import QueueMiddleware
+
+
+def transport(monkeypatch):
+ client = AsyncMock(spec=AsyncAGFSClient)
+ client.write.return_value = "message-1"
+ client.read.return_value = b"[]"
+ monkeypatch.setattr(
+ "openviking.storage.queuefs.named_queue.AsyncAGFSClient", lambda _client: client
+ )
+ return client
+
+
+class Handler(DequeueHandlerBase):
+ async def on_dequeue(self, data):
+ return ProcessResult.success(data)
+
+
[email protected]("operation", ["enqueue", "process", "ack", "clear"])
+async def test_middleware_constructor_order_and_commit(transport, operation):
+ events = []
+
+ class Recorder(QueueMiddleware):
+ def __init__(self, name):
+ self.name = name
+
+ async def invoke(self, ctx, call_next):
+ events.append((self.name, "enter"))
+ result = await call_next(ctx)
+ if operation != "process":
+ assert ctx.committed
+ events.append((self.name, "exit"))
+ return result
+
+ enqueue = process = ack = clear = invoke
+
+ middlewares = [Recorder("outer"), Recorder("inner")]
+ queue = NamedQueue(
+ object(), "/queue", "Test", dequeue_handler=Handler(), middlewares=middlewares
+ )
+ middlewares.clear()
+ message = {"id": "message-1", "data": "{}"}
+ if operation == "enqueue":
+ assert await queue.enqueue({"value": 1}) == "message-1"
+ transport.write.assert_awaited_once_with("/queue/Test/enqueue", b'{"value": 1}')
+ elif operation == "process":
+ assert (await queue.process_dequeued(message)).value == message
+ transport.write.assert_not_awaited()
+ elif operation == "ack":
+ await queue.ack("message-1", message)
+ transport.write.assert_awaited_once_with("/queue/Test/ack", b"message-1")
+ else:
+ assert await queue.clear()
+ transport.write.assert_awaited_once_with("/queue/Test/clear", b"")
+ assert events == [("outer", "enter"), ("inner", "enter"), ("inner", "exit"), ("outer", "exit")]
+
+
[email protected]("failure", [RuntimeError("write failed"), asyncio.CancelledError()])
+async def test_enqueue_failure_unwinds_middlewares(transport, failure):
+ events = []
+
+ class Recorder(QueueMiddleware):
+ def __init__(self, name):
+ self.name = name
+
+ async def enqueue(self, ctx, call_next):
+ events.append(self.name)
+ try:
+ return await call_next(ctx)
+ finally:
+ assert not ctx.committed
+ events.append(f"{self.name}-cleanup")
+
+ transport.write.side_effect = failure
+ queue = NamedQueue(
+ object(), "/queue", "Test", middlewares=[Recorder("outer"), Recorder("inner")]
+ )
+ with pytest.raises(type(failure)):
+ await queue.enqueue("payload")
+ assert events == ["outer", "inner", "inner-cleanup", "outer-cleanup"]
+
+
+async def test_enqueue_middleware_rewrites_payload(transport):
+ class Rewrite(QueueMiddleware):
+ async def enqueue(self, ctx, call_next):
+ assert ctx.payload == "input"
+ ctx.payload = {"input": ctx.payload, "middleware": True}
+ return await call_next(ctx)
+
+ queue = NamedQueue(object(), "/queue", "Test", middlewares=[Rewrite()])
+ await queue.enqueue("input")
+ assert json.loads(transport.write.await_args.args[1]) == {"input": "input", "middleware": True}
+
+
[email protected]("failure", [RuntimeError("handler failed"), asyncio.CancelledError()])
+async def test_handler_exception_still_skips_ack(transport, failure):
+ class FailingHandler(DequeueHandlerBase):
+ async def on_dequeue(self, data):
+ raise failure
+
+ transport.read.return_value = b'{"id":"message-1","data":"{}"}'
+ queue = NamedQueue(
+ object(),
+ "/queue",
+ "Test",
+ dequeue_handler=FailingHandler(),
+ middlewares=[QueueMiddleware()],
+ )
+ if isinstance(failure, asyncio.CancelledError):
+ with pytest.raises(asyncio.CancelledError):
+ await queue.dequeue()
+ else:
+ assert await queue.dequeue() is None
+ transport.write.assert_not_awaited()
+
+
+async def test_empty_success_result_still_acks(transport):
+ class DiscardHandler(DequeueHandlerBase):
+ async def on_dequeue(self, data):
+ return ProcessResult.success()
+
+ transport.read.return_value = b'{"id":"message-1","data":"{}"}'
+ queue = NamedQueue(
+ object(),
+ "/queue",
+ "Test",
+ dequeue_handler=DiscardHandler(),
+ middlewares=[QueueMiddleware()],
+ )
+ assert await queue.dequeue() is None
+ transport.write.assert_awaited_once_with("/queue/Test/ack", b"message-1")
+
+
+async def test_clear_snapshot_error_is_not_converted_to_success(transport):
+ transport.read.side_effect = OSError("snapshot unavailable")
+ queue = NamedQueue(object(), "/queue", "Test", middlewares=[QueueMiddleware()])
+ with pytest.raises(OSError, match="snapshot unavailable"):
+ await queue.clear()
+ transport.write.assert_not_awaited()
+
+
+async def test_queue_without_middleware_remains_generic(transport):
+ queue = NamedQueue(object(), "/queue", "Test")
+ payload = {"task_id": "opaque-business-field"}
+ await queue.enqueue(payload)
+ assert json.loads(transport.write.await_args.args[1]) == payload
+ assert (await queue.process_dequeued(payload)).value == payload
+ transport.write.reset_mock()
+ await queue.ack("")
+ transport.write.assert_not_awaited()
+ assert await queue.clear()
+ transport.read.assert_not_awaited()
+
+
+ "result",
+ [
+ ProcessResult.success(),
+ ProcessResult.failed("bad input"),
+ ProcessResult.requeued(),
+ ProcessResult.cancelled(),
+ ],
+)
+async def test_process_short_circuit_settles_once_without_acking(transport, result):
+ class ShortCircuit(QueueMiddleware):
+ async def process(self, ctx, call_next):
+ return result
+
+ queue = NamedQueue(
+ object(),
+ "/queue",
+ "Test",
+ dequeue_handler=Handler(),
+ middlewares=[ShortCircuit()],
+ )
+ message = {"id": "m", "data": "{}"}
+ assert await queue.process_dequeued(message) == result
+ assert queue._error_count == (result.outcome is ProcessOutcome.FAILED)
+ assert queue._processed == (result.outcome is not ProcessOutcome.FAILED)
+ assert queue._requeue_count == (result.outcome is ProcessOutcome.REQUEUED)
+ transport.write.assert_not_awaited()
+
+
+async def test_cancel_before_process_starts_leaves_no_local_inflight_state(transport):
+ queue = NamedQueue(object(), "/queue", "Test", dequeue_handler=Handler())
+ task = asyncio.create_task(queue.process_dequeued({"id": "m"}))
+ task.cancel()
+ with pytest.raises(asyncio.CancelledError):
+ await task
+ transport.write.assert_not_awaited()
+
+
+async def test_invalid_process_result_does_not_ack(transport):
+ class Invalid(QueueMiddleware):
+ async def process(self, ctx, call_next):
+ return None
+
+ queue = NamedQueue(
+ object(),
+ "/queue",
+ "Test",
+ dequeue_handler=Handler(),
+ middlewares=[Invalid()],
+ )
+ with pytest.raises(TypeError, match="ProcessResult"):
+ await queue.process_dequeued({"id": "m"})
+ assert queue._error_count == 1
+ transport.write.assert_not_awaited()
+
+
+async def test_ack_error_does_not_double_count_processing(transport):
+ transport.read.return_value = b'{"id":"m","data":"{}"}'
+ transport.write.side_effect = OSError("ack failed")
+ queue = NamedQueue(object(), "/queue", "Test", dequeue_handler=Handler())
+ await queue.dequeue()
+ assert queue._processed == 1
+ assert queue._error_count == 0
+
+
+async def test_clear_short_circuit_reports_no_commit(transport):
+ class SkipClear(QueueMiddleware):
+ async def clear(self, ctx, call_next):
+ return None
+
+ queue = NamedQueue(object(), "/queue", "Test", middlewares=[SkipClear()])
+ assert await queue.clear() is False
+ transport.write.assert_not_awaited()
+
+
+async def test_dequeue_without_handler_bypasses_process_middleware_and_acks(transport):
+ calls = []
+
+ class Recorder(QueueMiddleware):
+ async def process(self, ctx, call_next):
+ calls.append(ctx.message)
+ return await call_next(ctx)
+
+ message = {"id": "m", "data": "{}"}
+ transport.read.return_value = json.dumps(message).encode()
+ queue = NamedQueue(object(), "/queue", "Test", middlewares=[Recorder()])
+
+ assert await queue.dequeue() == message
+ assert calls == []
+ assert queue._processed == 0
+ transport.write.assert_awaited_once_with("/queue/Test/ack", b"m")
diff --git a/tests/storage/test_process_result.py b/tests/storage/test_process_result.py
new file mode 100644
index 000000000..5d21e1c16
--- /dev/null
+++ b/tests/storage/test_process_result.py
@@ -0,0 +1,34 @@
+# Copyright (c) 2026 Beijing Volcano Engine Technology Co., Ltd.
+# SPDX-License-Identifier: AGPL-3.0
+
+import pytest
+
+from openviking.storage.queuefs.process_result import ProcessOutcome, ProcessResult
+
+
[email protected]("outcome", list(ProcessOutcome))
+def test_result_outcome_and_payload(outcome):
+ error = "failure" if outcome is ProcessOutcome.FAILED else None
+ result = ProcessResult(outcome, error=error)
+ assert result.outcome is outcome
+ assert result.error == error
+ assert result.value is None
+
+
+ "outcome,error",
+ [
+ (ProcessOutcome.FAILED, None),
+ (ProcessOutcome.SUCCESS, "failure"),
+ ("success", None),
+ ],
+)
+def test_result_rejects_inconsistent_state(outcome, error):
+ with pytest.raises(ValueError):
+ ProcessResult(outcome, error=error)
+
+
+def test_result_preserves_value_and_empty_error():
+ payload = {"value": 1}
+ assert ProcessResult.success(payload).value is payload
+ assert ProcessResult.failed("").error == ""
diff --git a/tests/storage/test_queue_manager.py b/tests/storage/test_queue_manager.py
index 1511ce447..83eaccead 100644
--- a/tests/storage/test_queue_manager.py
+++ b/tests/storage/test_queue_manager.py
@@ -2,13 +2,21 @@
# SPDX-License-Identifier: AGPL-3.0
"""Focused tests for QueueManager concurrency selection."""
+import json
import os
import subprocess
import sys
+import threading
from unittest.mock import AsyncMock
-from openviking.storage.queuefs.named_queue import NamedQueue
+import pytest
+
+from openviking.pyagfs import AsyncAGFSClient
+from openviking.service.task_work_index import bind_task_context
+from openviking.storage.queuefs.named_queue import DequeueHandlerBase, NamedQueue
+from openviking.storage.queuefs.process_result import ProcessResult
from openviking.storage.queuefs.queue_manager import QueueManager
+from openviking.storage.queuefs.queue_middleware import QueueMiddleware
def test_queuefs_package_imports_in_a_clean_process(tmp_path) -> None:
@@ -57,3 +65,119 @@ async def test_status_waits_for_processing_messages_from_other_workers(monkeypat
assert status.pending == 0
assert status.in_progress == 1
assert not status.is_complete
+
+
+def transport(monkeypatch):
+ client = AsyncMock(spec=AsyncAGFSClient)
+ client.write.return_value = "message-1"
+ client.read.return_value = b"0"
+ monkeypatch.setattr(
+ "openviking.storage.queuefs.named_queue.AsyncAGFSClient", lambda _client: client
+ )
+ return client
+
+
+async def test_constructor_middlewares_apply_to_all_queues(transport):
+ calls = []
+
+ class Recorder(QueueMiddleware):
+ async def enqueue(self, ctx, call_next):
+ calls.append(ctx.queue)
+ return await call_next(ctx)
+
+ middlewares = [Recorder()]
+ manager = QueueManager(object(), middlewares=middlewares)
+ middlewares.clear()
+ existing = manager.get_queue("existing", allow_create=True)
+ with bind_task_context("task", "account", "user"):
+ await existing.enqueue({})
+ middlewares.append(Recorder())
+ future = manager.get_queue("future", allow_create=True)
+ await future.enqueue({})
+ assert calls == ["existing", "future"]
+ assert manager._task_work_index.has_work("task")
+ assert transport.write.await_count == 2
+
+
+async def test_init_queue_manager_forwards_constructor_middlewares(transport, monkeypatch):
+ from openviking.storage.queuefs import queue_manager as queue_module
+
+ calls = []
+
+ class Recorder(QueueMiddleware):
+ async def enqueue(self, ctx, call_next):
+ calls.append(ctx.queue)
+ return await call_next(ctx)
+
+ monkeypatch.setattr(queue_module, "_instance", None)
+ manager = queue_module.init_queue_manager(object(), middlewares=[Recorder()])
+ assert queue_module.get_queue_manager() is manager
+ await manager.get_queue("Test", allow_create=True).enqueue({})
+ assert calls == ["Test"]
+ transport.write.assert_awaited_once_with("/queue/Test/enqueue", b"{}")
+
+
[email protected]("fail", [False, True])
+async def test_concurrent_worker_uses_process_and_ack_middleware(transport, fail):
+ events = []
+ stop = threading.Event()
+
+ class Recorder(QueueMiddleware):
+ async def process(self, ctx, call_next):
+ events.append("process")
+ return await call_next(ctx)
+
+ async def ack(self, ctx, call_next):
+ events.append("ack")
+ return await call_next(ctx)
+
+ class Handler(DequeueHandlerBase):
+ async def on_dequeue(self, data):
+ stop.set()
+ if fail:
+ raise RuntimeError("worker failure")
+ return ProcessResult.success(data)
+
+ manager = QueueManager(object(), middlewares=[Recorder()])
+ queue = manager.get_queue("Test", dequeue_handler=Handler(), allow_create=True)
+ reads = [b'{"id":"message-1","data":"{}"}']
+
+ async def read(path):
+ return reads.pop(0) if reads else b"{}"
+
+ transport.read.side_effect = read
+ await manager._worker_async_concurrent(queue, stop, 2)
+ assert queue._processed == (not fail)
+ assert queue._error_count == fail
+ assert events == (["process"] if fail else ["process", "ack"])
+ if fail:
+ transport.write.assert_not_awaited()
+ else:
+ transport.write.assert_awaited_once_with("/queue/Test/ack", b"message-1")
+
+
+async def test_bootstrap_restores_legacy_messages_into_middleware_index(transport):
+ manager = QueueManager(object())
+ queue = manager.get_queue("Test", allow_create=True)
+ message = {
+ "id": "legacy-message",
+ "data": json.dumps({"task_id": "task", "account_id": "account", "user_id": "user"}),
+ }
+ transport.read.return_value = json.dumps([message]).encode()
+ attached = []
+
+ class Tracker:
+ def attach_work_index(self, index):
+ attached.append(index)
+ assert index.has_work("task")
+
+ async def restore_work_tasks(self, owners):
+ assert owners == {"task": ("account", "user")}
+ return ["restored"]
+
+ assert await manager.prepare_task_tracking(Tracker()) == ["restored"]
+ assert attached == [manager._task_work_index]
+ await queue.ack("legacy-message", message)
+ assert not attached[0].has_work("task")
+ transport.write.assert_awaited_once_with("/queue/Test/ack", b"legacy-message")
diff --git a/tests/storage/test_semantic_dag_stats.py b/tests/storage/test_semantic_dag_stats.py
index 145597e43..fca2b6ae9 100644
--- a/tests/storage/test_semantic_dag_stats.py
+++ b/tests/storage/test_semantic_dag_stats.py
@@ -8,6 +8,7 @@ from unittest.mock import AsyncMock
import pytest
from openviking.server.identity import RequestContext, Role
+from openviking.service.task_queue_middleware import TaskWorkQueueMiddleware
from openviking.service.task_work_index import (
TaskWorkIndex,
TaskWorkRejected,
@@ -424,7 +425,7 @@ async def test_task_work_rejection_does_not_stop_shared_semantic_worker():
None,
"/queue",
"Embedding",
- task_work_index=work_index,
+ middlewares=[TaskWorkQueueMiddleware(work_index)],
)
embedding_queue._initialized = True
unrelated_ran = asyncio.Event()
diff --git a/tests/storage/test_semantic_queue_memory_dedupe.py b/tests/storage/test_semantic_queue_memory_dedupe.py
index f3782bb16..9109dc156 100644
--- a/tests/storage/test_semantic_queue_memory_dedupe.py
+++ b/tests/storage/test_semantic_queue_memory_dedupe.py
@@ -152,7 +152,7 @@ def _patch_semantic_config(monkeypatch, *, overview_sample_limit=32):
@pytest.mark.asyncio
-async def test_stale_memory_semantic_write_is_skipped(monkeypatch):
+async def test_stale_memory_semantic_write_is_success(monkeypatch):
pathlock = _FakePathLock()
viking_fs = _FakeVikingFS(pathlock)
processor = SemanticProcessor()
diff --git a/tests/storage/test_session_commit_processor_identity.py b/tests/storage/test_session_commit_processor_identity.py
index f8a9e9df4..14c1c2be3 100644
--- a/tests/storage/test_session_commit_processor_identity.py
+++ b/tests/storage/test_session_commit_processor_identity.py
@@ -9,13 +9,12 @@ worker binds the committing account/user (so tokens are not attributed to
"""
import asyncio
-import concurrent.futures
import json
-from unittest.mock import Mock
from openviking.observability.context import get_root_observability_context
from openviking.server.identity import RequestContext, Role
from openviking.session.session import Session
+from openviking.storage.queuefs.process_result import ProcessOutcome
from openviking.storage.queuefs.session_commit_msg import SessionCommitMsg
from openviking.storage.queuefs.session_commit_processor import SessionCommitProcessor
from openviking_cli.session.user_id import UserIdentifier
@@ -52,7 +51,7 @@ class _MemoryVikingFS:
def __init__(self) -> None:
self.files: dict[str, str] = {}
- async def stat(self, uri, ctx=None):
+ async def stat(self, uri, ctx=None, skip_count=False):
return {"path": uri}
async def write_file(self, uri, content, ctx=None, lease_ref=None):
@@ -85,8 +84,9 @@ async def test_process_binds_committing_identity_to_root_context():
)
ctx = RequestContext(user=UserIdentifier("acme", "alice"), role=Role.USER)
- await processor._process(_make_msg(), ctx)
+ result = await processor._process(_make_msg(), ctx)
+ assert result is True
assert captured["account_id"] == "acme"
assert captured["user_id"] == "alice"
@@ -108,13 +108,14 @@ async def test_process_requeues_deferred_commit_and_resets_root_context(monkeypa
)
ctx = RequestContext(user=UserIdentifier("acme", "alice"), role=Role.USER)
- await processor._process(_make_msg(), ctx)
+ result = await processor._process(_make_msg(), ctx)
+ assert result is False
assert queued == [("SessionCommit", _make_msg().to_dict())]
assert get_root_observability_context() is None
-async def test_cancelled_queued_commit_writes_terminal_marker_before_success(monkeypatch):
+async def test_cancelled_queued_commit_writes_terminal_marker_before_returning():
msg = _make_msg()
viking_fs = _MemoryVikingFS()
session = Session(
@@ -127,34 +128,12 @@ async def test_cancelled_queued_commit_writes_terminal_marker_before_success(mon
asyncio.get_running_loop(),
)
marker_uri = f"{msg.archive_uri}/.failed.json"
- on_success = Mock(side_effect=lambda: viking_fs.files[marker_uri])
- processor.set_callbacks(on_success, Mock(), Mock())
-
- def run_on_current_loop(coro, _loop):
- task = asyncio.create_task(coro)
- future: concurrent.futures.Future[None] = concurrent.futures.Future()
-
- def complete(completed: asyncio.Task) -> None:
- if completed.cancelled():
- future.cancel()
- return
- error = completed.exception()
- if error is not None:
- future.set_exception(error)
- else:
- future.set_result(completed.result())
-
- task.add_done_callback(complete)
- return future
- monkeypatch.setattr(
- "openviking.storage.queuefs.session_commit_processor.asyncio.run_coroutine_threadsafe",
- run_on_current_loop,
- )
-
- await processor.on_cancelled({"data": json.dumps(msg.to_dict())})
+ result = await processor.on_cancelled({"data": json.dumps(msg.to_dict())})
+ assert result.outcome is ProcessOutcome.CANCELLED
+ assert result.value is None
+ assert result.error is None
marker = json.loads(viking_fs.files[marker_uri])
assert marker["stage"] == "cancelled"
assert marker["error"] == "session commit cancelled"
- on_success.assert_called_once_with()
diff --git a/tests/test_task_tracker_concurrency.py b/tests/test_task_tracker_concurrency.py
index 5e2f784dd..66a28b294 100644
--- a/tests/test_task_tracker_concurrency.py
+++ b/tests/test_task_tracker_concurrency.py
@@ -171,6 +171,23 @@ async def test_owner_loop_dispatcher_runs_foreign_loop_work_on_owner_loop():
assert work_loop is owner_loop
+async def test_owner_loop_dispatcher_uses_explicit_owner_loop():
+ from openviking.service.task_tracker_concurrency import OwnerLoopDispatcher
+
+ owner_loop = asyncio.get_running_loop()
+ dispatcher = await asyncio.to_thread(OwnerLoopDispatcher, owner_loop)
+ work_loop: asyncio.AbstractEventLoop | None = None
+
+ async def work() -> None:
+ nonlocal work_loop
+ work_loop = asyncio.get_running_loop()
+
+ await asyncio.to_thread(lambda: asyncio.run(dispatcher.run(work)))
+
+ assert work_loop is owner_loop
+
+
@pytest.mark.asyncio
async def test_owner_loop_dispatcher_propagates_foreign_cancellation():
from openviking.service.task_tracker_concurrency import OwnerLoopDispatcher