Self-evolving Context Database for AI Agents. Unify Agent Memory, Knowledge RAG and Skills.
Python agent-contextagent-memoryagent-pluginsagentic-ragcontext-databasedsh-pluginknowledge-basememorymemory-managementself-evolvingvector-database
Parallel add (#121)
kkkwjx31455e1a79cb
diff --git a/openviking/storage/observers/queue_observer.py b/openviking/storage/observers/queue_observer.pyindex 8946233f3..6665e6869 100644--- a/openviking/storage/observers/queue_observer.py+++ b/openviking/storage/observers/queue_observer.py@@ -6,7 +6,7 @@ QueueObserver: Queue system observability tool.Provides methods to observe and report queue status in various formats."""-from typing import Dict+from typing import Dict, Optionalfrom openviking.storage.observers.base_observer import BaseObserverfrom openviking.storage.queuefs.named_queue import QueueStatus@@ -29,7 +29,8 @@ class QueueObserver(BaseObserver):async def get_status_table_async(self) -> str:statuses = await self._queue_manager.check_status()- return self._format_status_as_table(statuses)+ dag_stats = self._get_semantic_dag_stats()+ return self._format_status_as_table(statuses, dag_stats)def get_status_table(self) -> str:return run_async(self.get_status_table_async())@@ -37,7 +38,9 @@ class QueueObserver(BaseObserver):def __str__(self) -> str:return self.get_status_table()- def _format_status_as_table(self, statuses: Dict[str, QueueStatus]) -> str:+ def _format_status_as_table(+ self, statuses: Dict[str, QueueStatus], dag_stats: Optional[object]+ ) -> str:"""Format queue statuses as a table using tabulate.@@ -75,6 +78,19 @@ class QueueObserver(BaseObserver):total_processed += status.processedtotal_errors += status.error_count+ data.append(+ {+ "Queue": "Semantic-Nodes",+ "Pending": getattr(dag_stats, "pending_nodes", 0) if dag_stats else 0,+ "In Progress": getattr(dag_stats, "overview_scheduled_nodes", 0)+ if dag_stats+ else 0,+ "Processed": getattr(dag_stats, "done_nodes", 0) if dag_stats else 0,+ "Errors": 0,+ "Total": getattr(dag_stats, "total_nodes", 0) if dag_stats else 0,+ }+ )+# Add total rowtotal_total = total_pending + total_in_progress + total_processeddata.append(@@ -90,6 +106,15 @@ class QueueObserver(BaseObserver):return tabulate(data, headers="keys", tablefmt="pretty")+ def _get_semantic_dag_stats(self) -> Optional[object]:+ semantic_queue = self._queue_manager._queues.get(self._queue_manager.SEMANTIC)+ if not semantic_queue:+ return None+ handler = getattr(semantic_queue, "_dequeue_handler", None)+ if handler and hasattr(handler, "get_dag_stats"):+ return handler.get_dag_stats()+ return None+def is_healthy(self) -> bool:return not self.has_errors()diff --git a/openviking/storage/queuefs/__init__.py b/openviking/storage/queuefs/__init__.pyindex 7016da0ec..b73a01d7f 100644--- a/openviking/storage/queuefs/__init__.py+++ b/openviking/storage/queuefs/__init__.py@@ -4,6 +4,7 @@ from .embedding_msg import EmbeddingMsgfrom .embedding_queue import EmbeddingQueuefrom .named_queue import NamedQueue, QueueError, QueueStatusfrom .queue_manager import QueueManager, get_queue_manager, init_queue_manager+from .semantic_dag import SemanticDagExecutorfrom .semantic_msg import SemanticMsgfrom .semantic_processor import SemanticProcessorfrom .semantic_queue import SemanticQueue@@ -18,6 +19,7 @@ __all__ = ["EmbeddingQueue","EmbeddingMsg","SemanticQueue",+ "SemanticDagExecutor","SemanticMsg","SemanticProcessor",]diff --git a/openviking/storage/queuefs/queue_manager.py b/openviking/storage/queuefs/queue_manager.pyindex 7d3a76ff8..9effe35c7 100644--- a/openviking/storage/queuefs/queue_manager.py+++ b/openviking/storage/queuefs/queue_manager.py@@ -69,8 +69,9 @@ class QueueManager:self._agfs: Optional[Any] = Noneself._queues: Dict[str, NamedQueue] = {}self._started = False- self._heartbeat_thread: Optional[threading.Thread] = None- self._stop_event = threading.Event()+ self._queue_threads: Dict[str, threading.Thread] = {}+ self._queue_stop_events: Dict[str, threading.Event] = {}+ self._poll_interval = 0.2atexit.register(self.stop)logger.info(@@ -92,51 +93,66 @@ class QueueManager:self._agfs = AGFSClient(api_base_url=self._agfs_url, timeout=self.timeout)self._started = True- # Start heartbeat thread- self._stop_event.clear()- self._heartbeat_thread = threading.Thread(target=self._heartbeat_loop, daemon=True)- self._heartbeat_thread.start()+ # Start queue workers for existing queues+ for queue in list(self._queues.values()):+ self._start_queue_worker(queue)logger.info("[QueueManager] Started")- def _heartbeat_loop(self) -> None:- """Heartbeat thread main loop, reuses the same event loop to avoid httpx client cleanup issues."""+ def _start_queue_worker(self, queue: NamedQueue) -> None:+ """Start a dedicated worker thread for a queue if not already running."""+ if queue.name in self._queue_threads:+ thread = self._queue_threads[queue.name]+ if thread.is_alive():+ return++ stop_event = threading.Event()+ self._queue_stop_events[queue.name] = stop_event+ thread = threading.Thread(+ target=self._queue_worker_loop,+ args=(queue, stop_event),+ daemon=True,+ )+ self._queue_threads[queue.name] = thread+ thread.start()++ def _queue_worker_loop(self, queue: NamedQueue, stop_event: threading.Event) -> None:+ """Worker loop for a single queue, processes items sequentially."""loop = asyncio.new_event_loop()asyncio.set_event_loop(loop)try:- while not self._stop_event.is_set():+ while not stop_event.is_set():try:- loop.run_until_complete(self.heartbeat())+ queue_size = loop.run_until_complete(queue.size())+ if queue.has_dequeue_handler() and queue_size > 0:+ data = loop.run_until_complete(queue.dequeue())+ if data is not None:+ logger.debug(+ f"[QueueManager] Dequeued message from {queue.name}: {data}"+ )+ else:+ stop_event.wait(self._poll_interval)except Exception as e:- logger.error(f"[QueueManager] Heartbeat error: {e}")+ logger.error(f"[QueueManager] Worker error for {queue.name}: {e}")import tracebacktraceback.print_exc()-- # Call every 0.2 seconds, using wait to respond faster to stop signal- self._stop_event.wait(0.2)+ stop_event.wait(self._poll_interval)finally:loop.close()- async def heartbeat(self) -> None:- """Async entry point for dequeue and dispatch."""- for queue in list(self._queues.values()):- queue_size = await queue.size()- if queue.has_dequeue_handler() and queue_size > 0:- data = await queue.dequeue()- if data is not None:- logger.debug(f"[QueueManager] Dequeued message from {queue.name}: {data}")-def stop(self) -> None:"""Stop QueueManager and release resources."""if not self._started:return- # Stop heartbeat thread- self._stop_event.set()- if self._heartbeat_thread:- self._heartbeat_thread.join()- self._heartbeat_thread = None+ # Stop queue workers+ for stop_event in self._queue_stop_events.values():+ stop_event.set()+ for thread in self._queue_threads.values():+ thread.join()+ self._queue_threads.clear()+ self._queue_stop_events.clear()self._agfs = Noneself._queues.clear()@@ -185,6 +201,11 @@ class QueueManager:enqueue_hook=enqueue_hook,dequeue_handler=dequeue_handler,)+ if self._started:+ self._start_queue_worker(self._queues[name])+ elif self._started:+ # Ensure existing queue has a worker running+ self._start_queue_worker(self._queues[name])return self._queues[name]# ========== Compatibility convenience methods ==========diff --git a/openviking/storage/queuefs/semantic_dag.py b/openviking/storage/queuefs/semantic_dag.pynew file mode 100644index 000000000..98c432173--- /dev/null+++ b/openviking/storage/queuefs/semantic_dag.py@@ -0,0 +1,273 @@+# Copyright (c) 2026 Beijing Volcano Engine Technology Co., Ltd.+# SPDX-License-Identifier: Apache-2.0+"""Semantic DAG executor with event-driven lazy dispatch."""++import asyncio+from dataclasses import dataclass, field+from typing import Dict, List, Optional++from openviking.storage.viking_fs import get_viking_fs+from openviking.utils import VikingURI+from openviking.utils.logger import get_logger++logger = get_logger(__name__)+++@dataclass+class DirNode:+ """Directory node state for DAG execution."""++ uri: str+ children_dirs: List[str]+ file_paths: List[str]+ file_index: Dict[str, int]+ child_index: Dict[str, int]+ file_summaries: List[Optional[Dict[str, str]]]+ children_abstracts: List[Optional[Dict[str, str]]]+ pending: int+ dispatched: bool = False+ overview_scheduled: bool = False+ lock: asyncio.Lock = field(default_factory=asyncio.Lock)+++@dataclass+class DagStats:+ total_nodes: int = 0+ pending_nodes: int = 0+ overview_scheduled_nodes: int = 0+ done_nodes: int = 0+++class SemanticDagExecutor:+ """Execute semantic generation with DAG-style, event-driven lazy dispatch."""++ def __init__(self, processor: "SemanticProcessor", context_type: str, max_concurrent_llm: int):+ self._processor = processor+ self._context_type = context_type+ self._max_concurrent_llm = max_concurrent_llm+ self._llm_sem = asyncio.Semaphore(max_concurrent_llm)+ self._viking_fs = get_viking_fs()+ self._nodes: Dict[str, DirNode] = {}+ self._parent: Dict[str, Optional[str]] = {}+ self._root_uri: Optional[str] = None+ self._root_done: Optional[asyncio.Event] = None+ self._stats = DagStats()++ async def run(self, root_uri: str) -> None:+ """Run DAG execution starting from root_uri."""+ self._root_uri = root_uri+ self._root_done = asyncio.Event()+ await self._dispatch_dir(root_uri, parent_uri=None)+ await self._root_done.wait()++ async def _dispatch_dir(self, dir_uri: str, parent_uri: Optional[str]) -> None:+ """Lazy-dispatch tasks for a directory when it is triggered."""+ if dir_uri in self._nodes:+ return++ self._parent[dir_uri] = parent_uri++ try:+ children_dirs, file_paths = await self._list_dir(dir_uri)+ file_index = {path: idx for idx, path in enumerate(file_paths)}+ child_index = {path: idx for idx, path in enumerate(children_dirs)}+ pending = len(children_dirs) + len(file_paths)++ node = DirNode(+ uri=dir_uri,+ children_dirs=children_dirs,+ file_paths=file_paths,+ file_index=file_index,+ child_index=child_index,+ file_summaries=[None] * len(file_paths),+ children_abstracts=[None] * len(children_dirs),+ pending=pending,+ dispatched=True,+ )+ self._nodes[dir_uri] = node+ self._stats.total_nodes += 1+ self._stats.pending_nodes += 1++ if pending == 0:+ self._schedule_overview(dir_uri)+ return++ for file_path in file_paths:+ self._stats.total_nodes += 1+ self._stats.pending_nodes += 1+ asyncio.create_task(self._file_summary_task(dir_uri, file_path))++ for child_uri in children_dirs:+ asyncio.create_task(self._dispatch_dir(child_uri, dir_uri))+ except Exception as e:+ logger.error(f"Failed to dispatch directory {dir_uri}: {e}", exc_info=True)+ if parent_uri:+ await self._on_child_done(parent_uri, dir_uri, "")+ elif self._root_done:+ self._root_done.set()++ async def _list_dir(self, uri: str) -> tuple[list[str], list[str]]:+ """List directory entries and return (child_dirs, file_paths)."""+ try:+ entries = await self._viking_fs.ls(uri)+ except Exception as e:+ logger.warning(f"Failed to list directory {uri}: {e}")+ return [], []++ children_dirs: List[str] = []+ file_paths: List[str] = []++ for entry in entries:+ name = entry.get("name", "")+ if not name or name.startswith(".") or name in [".", ".."]:+ continue++ item_uri = VikingURI(uri).join(name).uri+ if entry.get("isDir", False):+ children_dirs.append(item_uri)+ else:+ file_paths.append(item_uri)++ return children_dirs, file_paths++ async def _file_summary_task(self, parent_uri: str, file_path: str) -> None:+ """Generate file summary and notify parent completion."""+ file_name = file_path.split("/")[-1]+ try:+ summary_dict = await self._processor._generate_single_file_summary(+ file_path, llm_sem=self._llm_sem+ )+ except Exception as e:+ logger.warning(f"Failed to generate summary for {file_path}: {e}")+ summary_dict = {"name": file_name, "summary": ""}+ finally:+ self._stats.done_nodes += 1+ self._stats.pending_nodes = max(0, self._stats.pending_nodes - 1)++ await self._on_file_done(parent_uri, file_path, summary_dict)++ async def _on_file_done(+ self, parent_uri: str, file_path: str, summary_dict: Dict[str, str]+ ) -> None:+ node = self._nodes.get(parent_uri)+ if not node:+ return++ async with node.lock:+ idx = node.file_index.get(file_path)+ if idx is not None:+ node.file_summaries[idx] = summary_dict+ node.pending -= 1+ if node.pending == 0 and not node.overview_scheduled:+ node.overview_scheduled = True+ self._stats.overview_scheduled_nodes += 1+ asyncio.create_task(self._overview_task(parent_uri))++ async def _on_child_done(self, parent_uri: str, child_uri: str, abstract: str) -> None:+ node = self._nodes.get(parent_uri)+ if not node:+ return++ child_name = child_uri.split("/")[-1]+ async with node.lock:+ idx = node.child_index.get(child_uri)+ if idx is not None:+ node.children_abstracts[idx] = {"name": child_name, "abstract": abstract}+ node.pending -= 1+ if node.pending == 0 and not node.overview_scheduled:+ node.overview_scheduled = True+ self._stats.overview_scheduled_nodes += 1+ asyncio.create_task(self._overview_task(parent_uri))++ def _schedule_overview(self, dir_uri: str) -> None:+ node = self._nodes.get(dir_uri)+ if not node:+ return+ if node.overview_scheduled:+ return+ node.overview_scheduled = True+ self._stats.overview_scheduled_nodes += 1+ asyncio.create_task(self._overview_task(dir_uri))++ def _finalize_file_summaries(self, node: DirNode) -> List[Dict[str, str]]:+ summaries: List[Dict[str, str]] = []+ for idx, file_path in enumerate(node.file_paths):+ item = node.file_summaries[idx]+ if item is None:+ summaries.append({"name": file_path.split("/")[-1], "summary": ""})+ else:+ summaries.append(item)+ return summaries++ def _finalize_children_abstracts(self, node: DirNode) -> List[Dict[str, str]]:+ results: List[Dict[str, str]] = []+ for idx, child_uri in enumerate(node.children_dirs):+ item = node.children_abstracts[idx]+ if item is None:+ results.append({"name": child_uri.split("/")[-1], "abstract": ""})+ else:+ results.append(item)+ return results++ async def _overview_task(self, dir_uri: str) -> None:+ node = self._nodes.get(dir_uri)+ if not node:+ return++ async with node.lock:+ file_summaries = self._finalize_file_summaries(node)+ children_abstracts = self._finalize_children_abstracts(node)++ try:+ async with self._llm_sem:+ overview = await self._processor._generate_overview(+ dir_uri, file_summaries, children_abstracts+ )+ abstract = self._processor._extract_abstract_from_overview(overview)++ try:+ await self._viking_fs.write_file(f"{dir_uri}/.overview.md", overview)+ await self._viking_fs.write_file(f"{dir_uri}/.abstract.md", abstract)+ except Exception as e:+ logger.warning(f"Failed to write overview/abstract for {dir_uri}: {e}")++ try:+ await self._processor._vectorize_directory_simple(+ dir_uri, self._context_type, abstract, overview+ )+ except Exception as e:+ logger.error(f"Failed to vectorize directory {dir_uri}: {e}", exc_info=True)++ try:+ await self._processor._vectorize_files(+ dir_uri, self._context_type, node.file_paths, file_summaries+ )+ except Exception as e:+ logger.error(f"Failed to vectorize files in {dir_uri}: {e}", exc_info=True)++ except Exception as e:+ logger.error(f"Failed to generate overview for {dir_uri}: {e}", exc_info=True)+ abstract = ""+ finally:+ self._stats.done_nodes += 1+ self._stats.pending_nodes = max(0, self._stats.pending_nodes - 1)++ parent_uri = self._parent.get(dir_uri)+ if parent_uri is None:+ if self._root_done:+ self._root_done.set()+ return++ await self._on_child_done(parent_uri, dir_uri, abstract)++ def get_stats(self) -> DagStats:+ return DagStats(+ total_nodes=self._stats.total_nodes,+ pending_nodes=self._stats.pending_nodes,+ overview_scheduled_nodes=self._stats.overview_scheduled_nodes,+ done_nodes=self._stats.done_nodes,+ )+++if False: # pragma: no cover - for type checkers only+ from openviking.storage.queuefs.semantic_processor import SemanticProcessordiff --git a/openviking/storage/queuefs/semantic_processor.py b/openviking/storage/queuefs/semantic_processor.pyindex db493abff..9922b6f2a 100644--- a/openviking/storage/queuefs/semantic_processor.py+++ b/openviking/storage/queuefs/semantic_processor.py@@ -15,6 +15,7 @@ from openviking.parse.parsers.constants import ()from openviking.prompts import render_promptfrom openviking.storage.queuefs.named_queue import DequeueHandlerBase+from openviking.storage.queuefs.semantic_dag import DagStats, SemanticDagExecutorfrom openviking.storage.queuefs.semantic_msg import SemanticMsgfrom openviking.storage.viking_fs import get_viking_fsfrom openviking.utils import VikingURI@@ -35,7 +36,7 @@ class SemanticProcessor(DequeueHandlerBase):4. Enqueue to EmbeddingQueue for vectorization"""- def __init__(self, max_concurrent_llm: int = 10):+ def __init__(self, max_concurrent_llm: int = 100):"""Initialize SemanticProcessor.@@ -43,6 +44,7 @@ class SemanticProcessor(DequeueHandlerBase):max_concurrent_llm: Maximum concurrent LLM calls"""self.max_concurrent_llm = max_concurrent_llm+ self._dag_executor: Optional[SemanticDagExecutor] = Nonedef _detect_file_type(self, file_name: str) -> str:"""@@ -135,26 +137,14 @@ class SemanticProcessor(DequeueHandlerBase):)if msg.recursive:- # For recursive processing, collect all directory info- dir_info_list: List[Tuple[str, List[str], List[str]]] = []- await self._collect_directory_info(msg.uri, dir_info_list)-- # Enqueue each directory for processing (leaves first)- # For re-enqueued messages, set recursive=False to avoid repeated recursion- for uri, _, _ in dir_info_list:- # Create a new message for this directory- sub_msg = SemanticMsg(- uri=uri,- context_type=msg.context_type,- recursive=False, # Avoid repeated recursion- )-- # Enqueue for processing- await self._enqueue_semantic_msg(sub_msg)-- logger.info(- f"Enqueued {len(dir_info_list)} directories for processing from: {msg.uri}"+ executor = SemanticDagExecutor(+ processor=self,+ context_type=msg.context_type,+ max_concurrent_llm=self.max_concurrent_llm,)+ self._dag_executor = executor+ await executor.run(msg.uri)+ logger.info(f"Completed semantic generation for: {msg.uri}")self.report_success()return Noneelse:@@ -197,6 +187,11 @@ class SemanticProcessor(DequeueHandlerBase):self.report_error(str(e), data)return None+ def get_dag_stats(self) -> Optional["DagStats"]:+ if not self._dag_executor:+ return None+ return self._dag_executor.get_stats()+async def _process_single_directory(self,uri: str,@@ -263,7 +258,9 @@ class SemanticProcessor(DequeueHandlerBase):tasks = [generate_one_summary(fp) for fp in file_paths]return await asyncio.gather(*tasks)- async def _generate_single_file_summary(self, file_path: str) -> Dict[str, str]:+ async def _generate_single_file_summary(+ self, file_path: str, llm_sem: Optional[asyncio.Semaphore] = None+ ) -> Dict[str, str]:"""Generate summary for a single file.Args:@@ -307,7 +304,11 @@ class SemanticProcessor(DequeueHandlerBase):{"file_name": file_name, "content": content},)- summary = await vlm.get_completion_async(prompt)+ if llm_sem:+ async with llm_sem:+ summary = await vlm.get_completion_async(prompt)+ else:+ summary = await vlm.get_completion_async(prompt)return {"name": file_name, "summary": summary.strip()}except Exception as e: