Self-evolving Context Database for AI Agents. Unify Agent Memory, Knowledge RAG and Skills.
Python agent-contextagent-memoryagent-pluginsagentic-ragcontext-databasedsh-pluginknowledge-basememorymemory-managementself-evolvingvector-database
# Copyright (c) 2026 Beijing Volcano Engine Technology Co., Ltd.
# SPDX-License-Identifier: AGPL-3.0
"""
VikingDB Manager class that extends VikingVectorIndexBackend with queue management functionality.
"""
from contextlib import asynccontextmanager
from datetime import datetime
from typing import Any, AsyncIterator, Dict, List, Mapping, Optional, Tuple
from openviking.server.identity import RequestContext
from openviking.storage.expr import FilterExpr
from openviking.storage.queuefs.embedding_msg import EmbeddingMsg
from openviking.storage.queuefs.embedding_queue import EmbeddingQueue
from openviking.storage.queuefs.queue_manager import QueueManager
from openviking.storage.viking_vector_index_backend import (
UpsertOptions,
VectorTransferResult,
VikingVectorIndexBackend,
normalize_upsert_options,
)
from openviking_cli.utils import get_logger
from openviking_cli.utils.config.vectordb_config import VectorDBBackendConfig
logger = get_logger(__name__)
class VikingDBManager(VikingVectorIndexBackend):
"""
VikingDB Manager that extends VikingVectorIndexBackend with queue management capabilities.
This class provides all the functionality of VikingVectorIndexBackend plus:
- Queue manager integration (via injection)
- Embedding queue integration
- Background processing capabilities
Usage:
# In-memory mode with queue management
manager = VikingDBManager(vectordb_config=..., queue_manager=qm)
"""
def __init__(
self,
vectordb_config: VectorDBBackendConfig,
queue_manager: Optional[QueueManager] = None,
):
"""
Initialize VikingDB Manager.
Args:
vectordb_config: Configuration object for VectorDB backend.
queue_manager: QueueManager instance.
"""
# Initialize the base VikingVectorIndexBackend without queue management
super().__init__(config=vectordb_config)
# Queue management specific attributes
self._queue_manager = queue_manager
self._closing = False
def mark_closing(self) -> None:
"""Mark the manager as entering shutdown flow.
Queue workers may still be draining messages before the backend is
finally closed. Handlers should check ``is_closing`` and stop writing
into vector storage to avoid lock contention during rapid restart.
"""
self._closing = True
async def close(self) -> None:
"""Close storage connection and release resources."""
self.mark_closing()
try:
# We do NOT stop the queue manager here as it is an injected dependency
# and should be managed by the creator (OpenVikingService).
# Then close the base backend
await super().close()
except Exception as e:
logger.error(f"Error closing VikingDB manager: {e}")
@property
def is_closing(self) -> bool:
"""Whether the manager is in shutdown flow."""
return self._closing
# =========================================================================
# Queue Management Properties
# =========================================================================
@property
def queue_manager(self):
"""Get the queue manager instance."""
return self._queue_manager
@property
def embedding_queue(self) -> Optional["EmbeddingQueue"]:
"""Get the embedding queue instance."""
if not self._queue_manager:
return None
# get_queue returns EmbeddingQueue when name is QueueManager.EMBEDDING
queue = self._queue_manager.get_queue(self._queue_manager.EMBEDDING)
return queue if isinstance(queue, EmbeddingQueue) else None
@property
def has_queue_manager(self) -> bool:
"""Check if queue manager is initialized."""
return self._queue_manager is not None
# =========================================================================
# Convenience Methods for Queue Operations
# =========================================================================
async def enqueue_embedding_msg(self, embedding_msg: "EmbeddingMsg") -> bool:
"""
Enqueue an embedding message for processing.
Args:
embedding_msg: The EmbeddingMsg object to enqueue
Returns:
True if enqueued successfully, False otherwise
"""
if not embedding_msg:
logger.warning("Embedding message is None, skipping enqueuing")
return False
if not self._queue_manager:
raise RuntimeError("Queue manager not initialized, cannot enqueue embedding")
try:
embedding_queue = self.embedding_queue
if not embedding_queue:
raise RuntimeError("Embedding queue not initialized")
await embedding_queue.enqueue(embedding_msg)
logger.debug(f"Enqueued embedding message: {embedding_msg.id}")
return True
except Exception as e:
logger.error(f"Error enqueuing embedding message: {e}")
return False
async def get_embedding_queue_size(self) -> int:
"""
Get the current size of the embedding queue.
Returns:
The number of messages in the embedding queue
"""
if not self._queue_manager:
return 0
try:
embedding_queue = self._queue_manager.get_queue("embedding")
return await embedding_queue.size()
except Exception as e:
logger.error(f"Error getting embedding queue size: {e}")
return 0
class VikingDBManagerProxy:
"""
租户绑定的 VikingDBManager 代理。
使用 RequestContext 初始化后,所有方法调用自动携带 ctx,
无需在每次调用时显式传入。API 与 VikingDBManager 完全兼容。
示例:
```python
# 初始化
manager = VikingDBManager(...)
proxy = VikingDBManagerProxy(manager, ctx)
# 使用(无需传 ctx;仅在需要保留未显式传入字段时开启 partial_update)
await proxy.upsert(data, options=UpsertOptions(partial_update=True))
```
"""
def __init__(
self,
manager: VikingDBManager,
ctx: RequestContext,
):
"""
初始化租户绑定的 VikingDBManager 代理。
Args:
manager: 底层的 VikingDBManager 实例
ctx: 请求上下文,包含租户信息
"""
self._manager = manager
self._ctx = ctx
@property
def ctx(self) -> RequestContext:
"""获取绑定的请求上下文。"""
return self._ctx
@property
def manager(self) -> VikingDBManager:
"""获取底层的 VikingDBManager 实例。"""
return self._manager
@property
def collection_name(self) -> str:
return self._manager.collection_name
@property
def mode(self) -> str:
return self._manager.mode
# =========================================================================
# Queue Management Properties(透传)
# =========================================================================
@property
def queue_manager(self):
return self._manager.queue_manager
@property
def embedding_queue(self) -> Optional["EmbeddingQueue"]:
return self._manager.embedding_queue
@property
def has_queue_manager(self) -> bool:
return self._manager.has_queue_manager
def mark_closing(self) -> None:
return self._manager.mark_closing()
@property
def is_closing(self) -> bool:
return self._manager.is_closing
# =========================================================================
# Queue Operations(透传)
# =========================================================================
async def enqueue_embedding_msg(self, embedding_msg: "EmbeddingMsg") -> bool:
return await self._manager.enqueue_embedding_msg(embedding_msg)
async def get_embedding_queue_size(self) -> int:
return await self._manager.get_embedding_queue_size()
# =========================================================================
# Collection Management(透传)
# =========================================================================
async def create_collection(self, name: str, schema: Dict[str, Any]) -> bool:
backend = await self._manager.get_account_backend(self._ctx.account_id)
return await backend.create_collection(name, schema)
async def drop_collection(self) -> bool:
backend = await self._manager.get_account_backend(self._ctx.account_id)
return await backend.drop_collection()
async def collection_exists(self) -> bool:
backend = await self._manager.get_account_backend(self._ctx.account_id)
return await backend.collection_exists()
async def collection_exists_bound(self) -> bool:
return await self.collection_exists()
async def get_collection_info(self) -> Optional[Dict[str, Any]]:
backend = await self._manager.get_account_backend(self._ctx.account_id)
return await backend.get_collection_info()
async def get_collection_meta(self) -> Optional[Dict[str, Any]]:
return await self._manager.get_collection_meta(ctx=self._ctx)
async def update_collection_description(self, description: str) -> bool:
backend = await self._manager.get_account_backend(self._ctx.account_id)
return await backend.update_collection_description(description)
# =========================================================================
# 数据操作 API(自动携带 ctx)
# =========================================================================
async def upsert(
self,
data: Dict[str, Any],
options: UpsertOptions | Mapping[str, Any] | None = None,
):
"""Bound write entrypoint.
``options.partial_update=False`` keeps the legacy full-record upsert semantics.
``options.partial_update=True`` reads the current record first and preserves
fields that are omitted from ``data`` before writing.
"""
options = normalize_upsert_options(options)
return await self._manager.upsert(
data,
ctx=self._ctx,
options=options,
)
async def upsert_many(self, data_list: List[Dict[str, Any]]) -> List[str]:
"""Bulk full-record upsert with the proxy's bound request context."""
return await self._manager.upsert_many(data_list, ctx=self._ctx)
@asynccontextmanager
async def bulk_ingest(self) -> AsyncIterator[None]:
"""Bind the proxy context to a bulk-ingest maintenance scope."""
async with self._manager.bulk_ingest(ctx=self._ctx):
yield
async def get(self, ids: List[str]) -> List[Dict[str, Any]]:
return await self._manager.get(ids, ctx=self._ctx)
async def delete(self, ids: List[str]) -> int:
return await self._manager.delete(ids, ctx=self._ctx)
async def exists(self, id: str) -> bool:
return await self._manager.exists(id, ctx=self._ctx)
async def fetch_by_uri(self, uri: str) -> Optional[Dict[str, Any]]:
return await self._manager.fetch_by_uri(uri, ctx=self._ctx)
async def query(
self,
query_vector: Optional[List[float]] = None,
sparse_query_vector: Optional[Dict[str, float]] = None,
filter: Optional[Dict[str, Any] | FilterExpr] = None,
limit: int = 10,
offset: int = 0,
output_fields: Optional[List[str]] = None,
order_by: Optional[str] = None,
order_desc: bool = False,
) -> List[Dict[str, Any]]:
return await self._manager.query(
query_vector=query_vector,
sparse_query_vector=sparse_query_vector,
filter=filter,
limit=limit,
offset=offset,
output_fields=output_fields,
order_by=order_by,
order_desc=order_desc,
ctx=self._ctx,
)
async def search(
self,
query_vector: Optional[List[float]] = None,
sparse_query_vector: Optional[Dict[str, float]] = None,
filter: Optional[Dict[str, Any] | FilterExpr] = None,
limit: int = 10,
offset: int = 0,
output_fields: Optional[List[str]] = None,
) -> List[Dict[str, Any]]:
return await self._manager.search(
query_vector=query_vector,
sparse_query_vector=sparse_query_vector,
filter=filter,
limit=limit,
offset=offset,
output_fields=output_fields,
ctx=self._ctx,
)
async def filter(
self,
filter: Dict[str, Any] | FilterExpr,
limit: int = 10,
offset: int = 0,
output_fields: Optional[List[str]] = None,
order_by: Optional[str] = None,
order_desc: bool = False,
) -> List[Dict[str, Any]]:
return await self._manager.filter(
filter=filter,
limit=limit,
offset=offset,
output_fields=output_fields,
order_by=order_by,
order_desc=order_desc,
ctx=self._ctx,
)
async def remove_by_uri(self, uri: str) -> int:
return await self._manager.remove_by_uri(uri, ctx=self._ctx)
async def scroll(
self,
filter: Optional[Dict[str, Any] | FilterExpr] = None,
limit: int = 100,
cursor: Optional[str] = None,
output_fields: Optional[List[str]] = None,
) -> Tuple[List[Dict[str, Any]], Optional[str]]:
return await self._manager.scroll(
filter=filter,
limit=limit,
cursor=cursor,
output_fields=output_fields,
ctx=self._ctx,
)
async def count(
self,
filter: Optional[Dict[str, Any] | FilterExpr] = None,
) -> int:
return await self._manager.count(filter=filter, ctx=self._ctx)
async def clear(self) -> bool:
return await self._manager.clear(ctx=self._ctx)
async def optimize(self) -> bool:
backend = await self._manager.get_account_backend(self._ctx.account_id)
return await backend.optimize()
async def close(self) -> None:
return await self._manager.close()
async def health_check(self) -> bool:
backend = await self._manager.get_account_backend(self._ctx.account_id)
return await backend.health_check()
async def get_stats(self) -> Dict[str, Any]:
backend = await self._manager.get_account_backend(self._ctx.account_id)
return await backend.get_stats()
# =========================================================================
# Tenant-Aware 方法(自动携带 ctx)
# =========================================================================
async def search_in_tenant(
self,
query_vector: Optional[List[float]],
sparse_query_vector: Optional[Dict[str, float]] = None,
context_type: Optional[str] = None,
target_directories: Optional[List[str]] = None,
extra_filter: Optional[FilterExpr | Dict[str, Any]] = None,
level: Optional[List[int]] = None,
limit: int = 10,
offset: int = 0,
events_time_decay_protection: Optional[str] = None,
request_now: Optional[datetime] = None,
) -> List[Dict[str, Any]]:
kwargs: Dict[str, Any] = {
"query_vector": query_vector,
"sparse_query_vector": sparse_query_vector,
"context_type": context_type,
"target_directories": target_directories,
"extra_filter": extra_filter,
"level": level,
"limit": limit,
"offset": offset,
}
if events_time_decay_protection is not None:
kwargs.update(
events_time_decay_protection=events_time_decay_protection,
request_now=request_now,
)
return await self._manager.search_in_tenant(self._ctx, **kwargs)
async def search_by_keywords_in_tenant(
self,
query: str,
context_type: Optional[str] = None,
target_directories: Optional[List[str]] = None,
extra_filter: Optional[FilterExpr | Dict[str, Any]] = None,
level: Optional[List[int]] = None,
limit: int = 10,
offset: int = 0,
) -> List[Dict[str, Any]]:
return await self._manager.search_by_keywords_in_tenant(
self._ctx,
query=query,
context_type=context_type,
target_directories=target_directories,
extra_filter=extra_filter,
level=level,
limit=limit,
offset=offset,
)
async def filter_in_tenant(
self,
context_type: Optional[str] = None,
target_directories: Optional[List[str]] = None,
extra_filter: Optional[FilterExpr | Dict[str, Any]] = None,
level: Optional[List[int]] = None,
limit: int = 10,
offset: int = 0,
) -> List[Dict[str, Any]]:
return await self._manager.filter_in_tenant(
self._ctx,
context_type=context_type,
target_directories=target_directories,
extra_filter=extra_filter,
level=level,
limit=limit,
offset=offset,
)
async def get_context_by_uri(
self,
uri: str,
owner_space: Optional[str] = None,
level: Optional[int] = None,
limit: int = 1,
*,
ctx: Optional[RequestContext] = None,
) -> List[Dict[str, Any]]:
return await self._manager.get_context_by_uri(
uri=uri,
owner_space=owner_space,
level=level,
limit=limit,
ctx=ctx if ctx is not None else self._ctx,
)
async def delete_account_data(self, account_id: str) -> int:
return await self._manager.delete_account_data(account_id, ctx=self._ctx)
async def delete_uris(self, uris: List[str]) -> None:
return await self._manager.delete_uris(self._ctx, uris)
async def copy_uri_mapping(
self,
source_uri: str,
target_uri: str,
recursive: bool = False,
) -> VectorTransferResult:
return await self._manager.copy_uri_mapping(
self._ctx,
source_uri=source_uri,
target_uri=target_uri,
recursive=recursive,
)
async def update_uri_mapping(
self,
source_uri: str,
target_uri: str,
recursive: bool = False,
) -> VectorTransferResult:
return await self._manager.update_uri_mapping(
self._ctx,
source_uri=source_uri,
target_uri=target_uri,
recursive=recursive,
)