Self-evolving Context Database for AI Agents. Unify Agent Memory, Knowledge RAG and Skills.
Python agent-contextagent-memoryagent-pluginsagentic-ragcontext-databasedsh-pluginknowledge-basememorymemory-managementself-evolvingvector-database
# Copyright (c) 2026 Beijing Volcano Engine Technology Co., Ltd.
# SPDX-License-Identifier: AGPL-3.0
"""
QueueManager: Encapsulates AGFS QueueFS plugin operations.
All queues are managed through NamedQueue.
"""
import asyncio
import atexit
import threading
import traceback
from typing import TYPE_CHECKING, Any, Dict, Optional, Sequence, Set, Union
from openviking.utils.async_client_cache import LoopScopedAsyncClientCache
from openviking_cli.utils.logger import get_logger
from .embedding_queue import EmbeddingQueue
from .named_queue import DequeueHandlerBase, NamedQueue, QueueStatus
from .queue_middleware import QueueMiddleware
from .semantic_queue import SemanticQueue
if TYPE_CHECKING:
from openviking.config.vlm import VLMResolver
logger = get_logger(__name__)
DEFAULT_MAX_CONCURRENT_SESSION_COMMIT = 8
# ========== Singleton Pattern ==========
_instance: Optional["QueueManager"] = None
def init_queue_manager(
agfs: Any,
timeout: int = 10,
mount_point: str = "/queue",
max_concurrent_embedding: int = 10,
max_concurrent_semantic: int = 32,
max_concurrent_external_parse: int = 4,
max_concurrent_add_resource: int = 4,
max_concurrent_reindex: int = 4,
max_concurrent_session_commit: int = DEFAULT_MAX_CONCURRENT_SESSION_COMMIT,
max_concurrent_external_task: int = 10,
*,
middlewares: Sequence[QueueMiddleware] = (),
) -> "QueueManager":
"""Initialize QueueManager singleton.
Args:
agfs: Pre-initialized AGFS client (HTTP or Binding).
timeout: Request timeout in seconds.
mount_point: Path where QueueFS is mounted.
max_concurrent_embedding: Max concurrent embedding tasks.
max_concurrent_semantic: Max concurrent semantic node work.
max_concurrent_external_parse: Max concurrent ExternalParse tasks.
max_concurrent_add_resource: Max concurrent AddResource tasks.
max_concurrent_reindex: Max concurrent Reindex tasks.
max_concurrent_session_commit: Max concurrent SessionCommit tasks.
middlewares: Additional middleware, fixed at construction for all queues.
"""
global _instance
_instance = QueueManager(
agfs=agfs,
timeout=timeout,
mount_point=mount_point,
max_concurrent_embedding=max_concurrent_embedding,
max_concurrent_semantic=max_concurrent_semantic,
max_concurrent_external_parse=max_concurrent_external_parse,
max_concurrent_add_resource=max_concurrent_add_resource,
max_concurrent_reindex=max_concurrent_reindex,
max_concurrent_session_commit=max_concurrent_session_commit,
max_concurrent_external_task=max_concurrent_external_task,
middlewares=middlewares,
)
return _instance
def get_queue_manager() -> "QueueManager":
"""Get QueueManager singleton."""
if _instance is None:
raise RuntimeError("QueueManager is not initialized. Call init_queue_manager() first.")
return _instance
class QueueManager:
"""
QueueManager: Encapsulates AGFS QueueFS plugin operations.
Integrates NamedQueue to manage multiple named queues.
"""
# Standard queue names
EMBEDDING = "Embedding"
SEMANTIC = "Semantic"
# Keep the on-disk name stable so pre-upgrade jobs remain recoverable.
EXTERNAL_PARSE = "ExternalParse"
ADD_RESOURCE = "AddResource"
REINDEX = "Reindex"
CONTENT_WRITE = "ContentWrite"
SESSION_COMMIT = "SessionCommit"
EXTERNAL_TASK = "ExternalTask"
# Account and user cleanup share one consumer. Retain the persisted name
# so user cleanup messages queued before this change resume in place.
DATA_CLEANUP = "UserDeletion"
# Deferred work re-enqueues itself; throttle the next scheduling round.
_REQUEUE_POLL_INTERVAL = 1.0
def __init__(
self,
agfs: Any,
timeout: int = 10,
mount_point: str = "/queue",
max_concurrent_embedding: int = 10,
max_concurrent_semantic: int = 32,
max_concurrent_external_parse: int = 4,
max_concurrent_add_resource: int = 4,
max_concurrent_reindex: int = 4,
max_concurrent_session_commit: int = DEFAULT_MAX_CONCURRENT_SESSION_COMMIT,
max_concurrent_external_task: int = 10,
*,
middlewares: Sequence[QueueMiddleware] = (),
):
"""Initialize QueueManager."""
self._agfs = agfs
self.timeout = timeout
self.mount_point = mount_point
self._max_concurrent_embedding = max_concurrent_embedding
self._max_concurrent_semantic = max_concurrent_semantic
self._max_concurrent_external_parse = max_concurrent_external_parse
self._max_concurrent_add_resource = max_concurrent_add_resource
self._max_concurrent_reindex = max_concurrent_reindex
self._max_concurrent_session_commit = max_concurrent_session_commit
self._max_concurrent_external_task = max_concurrent_external_task
self._queues: Dict[str, NamedQueue] = {}
self._started = False
self._queue_threads: Dict[str, threading.Thread] = {}
self._queue_stop_events: Dict[str, threading.Event] = {}
self._embedding_worker_stopped = threading.Event()
self._poll_interval = 0.2
self._vlm_resolver: Optional["VLMResolver"] = None
# Import at composition time to avoid a service <-> queue package cycle.
from openviking.service.task_queue_middleware import TaskWorkQueueMiddleware
self._middlewares: tuple[QueueMiddleware, ...] = (
TaskWorkQueueMiddleware(),
*middlewares,
)
atexit.register(self.stop)
logger.info(
f"[QueueManager] Initialized with agfs={type(agfs).__name__}, mount_point={mount_point}"
)
def start(self) -> None:
"""Start QueueManager workers."""
if self._started:
return
if self.SEMANTIC in self._queues and self._vlm_resolver is None:
raise RuntimeError("QueueManager requires a VLM resolver before semantic workers start")
self._started = True
# Start queue workers for existing queues
for queue in list(self._queues.values()):
self._start_queue_worker(queue)
logger.info(f"[QueueManager] mount_point={self.mount_point} Started")
async def prepare_task_tracking(self, tracker: Any) -> list[Any]:
"""Bootstrap Work only for stores that reconstruct it from QueueFS."""
from openviking.service.task_domain import WorkRecoveryMode
from openviking.service.task_work_index import extract_task_metadata
if tracker.work_recovery_mode is WorkRecoveryMode.DURABLE_STORE:
return []
snapshots = {name: await queue.snapshot() for name, queue in self._queues.items()}
owners: Dict[str, tuple[str, str]] = {}
for queue_name, messages in snapshots.items():
for message in messages:
metadata = extract_task_metadata(message)
if metadata is None or not metadata.account_id or not metadata.user_id:
continue
owners[metadata.task_id] = (metadata.account_id, metadata.user_id)
await tracker.reconcile_delivery(
metadata.task_id,
metadata.work_id,
queue_name,
account_id=metadata.account_id,
user_id=metadata.user_id,
)
return await tracker.restore_work_tasks(owners)
async def get_queued_tasks(self, tracker: Any, queue_name: str) -> list[Any]:
"""Read Tasks referenced by a queue's unacknowledged deliveries."""
from openviking.service.task_work_index import extract_task_metadata
queue = self._queues.get(queue_name)
if queue is None:
return []
tasks = []
seen: set[tuple[str, str, str]] = set()
for message in await queue.snapshot():
metadata = extract_task_metadata(message)
if metadata is None or not metadata.account_id or not metadata.user_id:
continue
key = (metadata.task_id, metadata.account_id, metadata.user_id)
if key in seen:
continue
seen.add(key)
task = await tracker.get(
metadata.task_id,
account_id=metadata.account_id,
user_id=metadata.user_id,
)
if task is not None:
tasks.append(task)
return tasks
def setup_standard_queues(
self,
vector_store: Any,
start: bool = True,
*,
embedding_provider: Any = None,
) -> None:
"""
Setup standard queues (Embedding and Semantic) with their handlers.
Args:
vector_store: Vector store instance for handlers to write results.
start: Whether to start worker threads immediately (default True).
Pass False when the consumer depends on resources that are not
yet initialized.
embedding_provider: Account-aware embedding provider for the handler.
"""
# Import handlers here to avoid circular dependencies
from openviking.storage.collection_schemas import TextEmbeddingHandler
from openviking.storage.queuefs import SemanticProcessor
# Embedding Queue
embedding_handler = TextEmbeddingHandler(vector_store, embedding_provider)
self.get_queue(
self.EMBEDDING,
dequeue_handler=embedding_handler,
allow_create=True,
)
logger.info("Embedding queue initialized with TextEmbeddingHandler")
# Semantic Queue
semantic_processor = SemanticProcessor(
max_concurrent_llm=self._max_concurrent_semantic,
embedding_worker_stopped=self._embedding_worker_stopped.is_set,
vlm_resolver=self._vlm_resolver,
)
self.get_queue(
self.SEMANTIC,
dequeue_handler=semantic_processor,
allow_create=True,
)
logger.info("Semantic queue initialized with SemanticProcessor")
if start:
self.start()
def set_vlm_resolver(self, resolver: "VLMResolver") -> None:
"""Bind the owning service's resolver before queue workers start."""
from openviking.storage.queuefs import SemanticProcessor
if self._started:
raise RuntimeError("Cannot replace the VLM resolver after queue workers start")
self._vlm_resolver = resolver
queue = self._queues.get(self.SEMANTIC)
if queue is not None:
queue.set_dequeue_handler(
SemanticProcessor(
max_concurrent_llm=self._max_concurrent_semantic,
embedding_worker_stopped=self._embedding_worker_stopped.is_set,
vlm_resolver=resolver,
)
)
def _start_queue_worker(self, queue: NamedQueue) -> None:
"""Start a dedicated worker thread for a queue if not already running."""
if queue.name in self._queue_threads:
thread = self._queue_threads[queue.name]
if thread.is_alive():
return
max_concurrent = self._max_concurrent_for_queue(queue.name)
stop_event = threading.Event()
self._queue_stop_events[queue.name] = stop_event
if queue.name == self.EMBEDDING:
self._embedding_worker_stopped.clear()
thread = threading.Thread(
target=self._queue_worker_loop,
args=(queue, stop_event, max_concurrent),
daemon=True,
)
self._queue_threads[queue.name] = thread
thread.start()
def _max_concurrent_for_queue(self, queue_name: str) -> int:
"""Return the worker concurrency limit for a named queue."""
if queue_name == self.DATA_CLEANUP:
return 1
if queue_name == self.EMBEDDING:
return self._max_concurrent_embedding
if queue_name == self.EXTERNAL_PARSE:
return self._max_concurrent_external_parse
if queue_name == self.ADD_RESOURCE:
return self._max_concurrent_add_resource
if queue_name == self.REINDEX:
return self._max_concurrent_reindex
if queue_name == self.SESSION_COMMIT:
return self._max_concurrent_session_commit
if queue_name == self.EXTERNAL_TASK:
return self._max_concurrent_external_task
return self._max_concurrent_semantic
def _queue_worker_loop(
self, queue: NamedQueue, stop_event: threading.Event, max_concurrent: int = 1
) -> None:
"""Worker loop for a single queue.
When max_concurrent > 1, items are fetched and processed in parallel
(up to max_concurrent at a time). Otherwise items are processed one by one.
"""
loop = asyncio.new_event_loop()
asyncio.set_event_loop(loop)
poll_interval = (
self._REQUEUE_POLL_INTERVAL
if queue.name in {self.SESSION_COMMIT, self.EXTERNAL_TASK}
else self._poll_interval
)
try:
if max_concurrent > 1:
loop.run_until_complete(
self._worker_async_concurrent(queue, stop_event, max_concurrent)
)
else:
while not stop_event.is_set():
try:
queue_size = loop.run_until_complete(queue.size())
if queue.has_dequeue_handler() and queue_size > 0:
data = loop.run_until_complete(queue.dequeue())
if data is not None:
logger.debug("[QueueManager] Dequeued message from %s", queue.name)
if queue.name in {self.SESSION_COMMIT, self.EXTERNAL_TASK}:
stop_event.wait(poll_interval)
else:
stop_event.wait(poll_interval)
except asyncio.CancelledError:
if not stop_event.is_set():
raise
break
except Exception as e:
logger.error(f"[QueueManager] Worker error for {queue.name}: {e}")
traceback.print_exc()
stop_event.wait(poll_interval)
finally:
# Consumers may own timers and async generators in addition to
# their active queue deliveries. Finish them on the worker loop.
pending = asyncio.all_tasks(loop)
for task in pending:
task.cancel()
if pending:
loop.run_until_complete(asyncio.gather(*pending, return_exceptions=True))
loop.run_until_complete(loop.shutdown_asyncgens())
loop.run_until_complete(loop.shutdown_default_executor())
try:
loop.run_until_complete(LoopScopedAsyncClientCache.close_current_loop_clients())
except Exception:
logger.warning(
"[QueueManager] Failed to close async clients for %s",
queue.name,
exc_info=True,
)
loop.close()
if queue.name == self.EMBEDDING:
# No more deliveries can start and active handlers have exited,
# including protected writes. Pending messages remain durable.
self._embedding_worker_stopped.set()
async def _worker_async_concurrent(
self, queue: NamedQueue, stop_event: threading.Event, max_concurrent: int
) -> None:
"""Concurrent worker: drains the queue and processes items in parallel.
A Semaphore caps inflight tasks at max_concurrent.
"""
poll_interval = (
self._REQUEUE_POLL_INTERVAL
if queue.name in {self.SESSION_COMMIT, self.EXTERNAL_TASK}
else self._poll_interval
)
sem = asyncio.Semaphore(max_concurrent)
active_tasks: Set[asyncio.Task] = set()
async def process_one(data: Dict[str, Any]) -> None:
async with sem:
msg_id = data.get("id", "") if isinstance(data, dict) else ""
try:
await queue.process_dequeued(data)
# Ack after successful processing (delete from persistent storage).
await queue.ack(msg_id, data)
except Exception as e:
# The message remains in processing and will be recovered.
logger.error(f"[QueueManager] Concurrent worker error for {queue.name}: {e}")
while not stop_event.is_set():
# Prune completed tasks
active_tasks = {t for t in active_tasks if not t.done()}
# While capacity remains, keep draining the queue
while len(active_tasks) < max_concurrent:
if not queue.has_dequeue_handler():
break
data = await queue.dequeue_raw()
if data is None:
break
task = asyncio.create_task(process_one(data))
active_tasks.add(task)
logger.debug(
f"[QueueManager] Dispatched concurrent task for {queue.name} "
f"(active={len(active_tasks)})"
)
await asyncio.sleep(poll_interval)
# Drain remaining in-flight tasks on shutdown (with timeout)
if active_tasks:
try:
await asyncio.wait_for(
asyncio.gather(*active_tasks, return_exceptions=True),
timeout=5.0,
)
except asyncio.TimeoutError:
logger.warning(
f"[QueueManager] Drain timeout for {queue.name}, "
f"cancelling {len(active_tasks)} in-flight task(s)"
)
for t in active_tasks:
t.cancel()
await asyncio.gather(*active_tasks, return_exceptions=True)
def stop(self) -> None:
"""Stop QueueManager and release resources."""
global _instance
if not self._started:
return
# Stop queue workers
for stop_event in self._queue_stop_events.values():
stop_event.set()
if self.EMBEDDING not in self._queue_threads:
self._embedding_worker_stopped.set()
alive_workers = []
for name, thread in self._queue_threads.items():
thread.join(timeout=10.0)
if thread.is_alive():
logger.warning(f"[QueueManager] Worker thread {name} did not exit in time")
alive_workers.append(name)
if alive_workers:
# Keep queues and dependencies available for pending finalizers and
# let the caller retry shutdown once the workers have terminated.
raise RuntimeError(f"Queue workers did not stop: {', '.join(alive_workers)}")
self._queue_threads.clear()
self._queue_stop_events.clear()
self._agfs = None
self._queues.clear()
self._started = False
if _instance is self:
_instance = None
logger.info("[QueueManager] Stopped")
def is_running(self) -> bool:
"""Check if QueueManager is running."""
return self._started
def get_queue(
self,
name: str,
dequeue_handler: Optional[DequeueHandlerBase] = None,
allow_create: bool = False,
) -> NamedQueue:
"""Get or create a named queue object."""
if name not in self._queues:
if not allow_create:
raise RuntimeError(f"Queue {name} does not exist and allow_create is False")
if name == self.EMBEDDING:
self._queues[name] = EmbeddingQueue(
self._agfs,
self.mount_point,
name,
dequeue_handler=dequeue_handler,
middlewares=self._middlewares,
)
elif name == self.SEMANTIC:
self._queues[name] = SemanticQueue(
self._agfs,
self.mount_point,
name,
dequeue_handler=dequeue_handler,
middlewares=self._middlewares,
)
else:
self._queues[name] = NamedQueue(
self._agfs,
self.mount_point,
name,
dequeue_handler=dequeue_handler,
middlewares=self._middlewares,
)
if self._started:
self._start_queue_worker(self._queues[name])
else:
if dequeue_handler is not None:
self._queues[name].set_dequeue_handler(dequeue_handler)
if self._started:
# Ensure existing queue has a worker running
self._start_queue_worker(self._queues[name])
return self._queues[name]
# ========== Compatibility convenience methods ==========
async def enqueue(self, queue_name: str, data: Union[str, Dict[str, Any]]) -> str:
"""Send message to queue (enqueue)."""
return await self.get_queue(queue_name).enqueue(data)
async def dequeue(self, queue_name: str) -> Optional[Dict[str, Any]]:
"""Get message from specified queue."""
return await self.get_queue(queue_name).dequeue()
async def peek(self, queue_name: str) -> Optional[Dict[str, Any]]:
"""Peek at the head message of specified queue."""
return await self.get_queue(queue_name).peek()
async def size(self, queue_name: str) -> int:
"""Get the size of specified queue."""
return await self.get_queue(queue_name).size()
async def clear(self, queue_name: str) -> bool:
"""Clear specified queue."""
return await self.get_queue(queue_name).clear()
# ========== Status check interface ==========
async def check_status(self, queue_name: Optional[str] = None) -> Dict[str, QueueStatus]:
"""Check queue status."""
if queue_name:
if queue_name not in self._queues:
return {}
return {queue_name: await self._queues[queue_name].get_status()}
return {name: await q.get_status() for name, q in self._queues.items()}
def has_errors(self, queue_name: Optional[str] = None) -> bool:
"""Check if there are errors."""
if queue_name:
if queue_name not in self._queues:
return False
return self._queues[queue_name]._error_count > 0
return any(q._error_count > 0 for q in self._queues.values())
async def is_all_complete(self, queue_name: Optional[str] = None) -> bool:
"""Check if all processing is complete."""
statuses = await self.check_status(queue_name)
return all(s.is_complete for s in statuses.values())
async def wait_complete(self) -> Dict[str, QueueStatus]:
"""Wait for all queues to complete and return their final status."""
while not await self.is_all_complete():
await asyncio.sleep(0.5)
return await self.check_status()