Skip to content

Self-evolving Context Database for AI Agents. Unify Agent Memory, Knowledge RAG and Skills.

Parallel add (#121)

kkkwjx31455e1a79cb
diff --git a/openviking/storage/observers/queue_observer.py b/openviking/storage/observers/queue_observer.py
index 8946233f3..6665e6869 100644
--- a/openviking/storage/observers/queue_observer.py
+++ b/openviking/storage/observers/queue_observer.py
@@ -6,7 +6,7 @@ QueueObserver: Queue system observability tool.
Provides methods to observe and report queue status in various formats.
"""
-from typing import Dict
+from typing import Dict, Optional
from openviking.storage.observers.base_observer import BaseObserver
from openviking.storage.queuefs.named_queue import QueueStatus
@@ -29,7 +29,8 @@ class QueueObserver(BaseObserver):
async def get_status_table_async(self) -> str:
statuses = await self._queue_manager.check_status()
- return self._format_status_as_table(statuses)
+ dag_stats = self._get_semantic_dag_stats()
+ return self._format_status_as_table(statuses, dag_stats)
def get_status_table(self) -> str:
return run_async(self.get_status_table_async())
@@ -37,7 +38,9 @@ class QueueObserver(BaseObserver):
def __str__(self) -> str:
return self.get_status_table()
- def _format_status_as_table(self, statuses: Dict[str, QueueStatus]) -> str:
+ def _format_status_as_table(
+ self, statuses: Dict[str, QueueStatus], dag_stats: Optional[object]
+ ) -> str:
"""
Format queue statuses as a table using tabulate.
@@ -75,6 +78,19 @@ class QueueObserver(BaseObserver):
total_processed += status.processed
total_errors += status.error_count
+ data.append(
+ {
+ "Queue": "Semantic-Nodes",
+ "Pending": getattr(dag_stats, "pending_nodes", 0) if dag_stats else 0,
+ "In Progress": getattr(dag_stats, "overview_scheduled_nodes", 0)
+ if dag_stats
+ else 0,
+ "Processed": getattr(dag_stats, "done_nodes", 0) if dag_stats else 0,
+ "Errors": 0,
+ "Total": getattr(dag_stats, "total_nodes", 0) if dag_stats else 0,
+ }
+ )
+
# Add total row
total_total = total_pending + total_in_progress + total_processed
data.append(
@@ -90,6 +106,15 @@ class QueueObserver(BaseObserver):
return tabulate(data, headers="keys", tablefmt="pretty")
+ def _get_semantic_dag_stats(self) -> Optional[object]:
+ semantic_queue = self._queue_manager._queues.get(self._queue_manager.SEMANTIC)
+ if not semantic_queue:
+ return None
+ handler = getattr(semantic_queue, "_dequeue_handler", None)
+ if handler and hasattr(handler, "get_dag_stats"):
+ return handler.get_dag_stats()
+ return None
+
def is_healthy(self) -> bool:
return not self.has_errors()
diff --git a/openviking/storage/queuefs/__init__.py b/openviking/storage/queuefs/__init__.py
index 7016da0ec..b73a01d7f 100644
--- a/openviking/storage/queuefs/__init__.py
+++ b/openviking/storage/queuefs/__init__.py
@@ -4,6 +4,7 @@ from .embedding_msg import EmbeddingMsg
from .embedding_queue import EmbeddingQueue
from .named_queue import NamedQueue, QueueError, QueueStatus
from .queue_manager import QueueManager, get_queue_manager, init_queue_manager
+from .semantic_dag import SemanticDagExecutor
from .semantic_msg import SemanticMsg
from .semantic_processor import SemanticProcessor
from .semantic_queue import SemanticQueue
@@ -18,6 +19,7 @@ __all__ = [
"EmbeddingQueue",
"EmbeddingMsg",
"SemanticQueue",
+ "SemanticDagExecutor",
"SemanticMsg",
"SemanticProcessor",
]
diff --git a/openviking/storage/queuefs/queue_manager.py b/openviking/storage/queuefs/queue_manager.py
index 7d3a76ff8..9effe35c7 100644
--- a/openviking/storage/queuefs/queue_manager.py
+++ b/openviking/storage/queuefs/queue_manager.py
@@ -69,8 +69,9 @@ class QueueManager:
self._agfs: Optional[Any] = None
self._queues: Dict[str, NamedQueue] = {}
self._started = False
- self._heartbeat_thread: Optional[threading.Thread] = None
- self._stop_event = threading.Event()
+ self._queue_threads: Dict[str, threading.Thread] = {}
+ self._queue_stop_events: Dict[str, threading.Event] = {}
+ self._poll_interval = 0.2
atexit.register(self.stop)
logger.info(
@@ -92,51 +93,66 @@ class QueueManager:
self._agfs = AGFSClient(api_base_url=self._agfs_url, timeout=self.timeout)
self._started = True
- # Start heartbeat thread
- self._stop_event.clear()
- self._heartbeat_thread = threading.Thread(target=self._heartbeat_loop, daemon=True)
- self._heartbeat_thread.start()
+ # Start queue workers for existing queues
+ for queue in list(self._queues.values()):
+ self._start_queue_worker(queue)
logger.info("[QueueManager] Started")
- def _heartbeat_loop(self) -> None:
- """Heartbeat thread main loop, reuses the same event loop to avoid httpx client cleanup issues."""
+ def _start_queue_worker(self, queue: NamedQueue) -> None:
+ """Start a dedicated worker thread for a queue if not already running."""
+ if queue.name in self._queue_threads:
+ thread = self._queue_threads[queue.name]
+ if thread.is_alive():
+ return
+
+ stop_event = threading.Event()
+ self._queue_stop_events[queue.name] = stop_event
+ thread = threading.Thread(
+ target=self._queue_worker_loop,
+ args=(queue, stop_event),
+ daemon=True,
+ )
+ self._queue_threads[queue.name] = thread
+ thread.start()
+
+ def _queue_worker_loop(self, queue: NamedQueue, stop_event: threading.Event) -> None:
+ """Worker loop for a single queue, processes items sequentially."""
loop = asyncio.new_event_loop()
asyncio.set_event_loop(loop)
try:
- while not self._stop_event.is_set():
+ while not stop_event.is_set():
try:
- loop.run_until_complete(self.heartbeat())
+ queue_size = loop.run_until_complete(queue.size())
+ if queue.has_dequeue_handler() and queue_size > 0:
+ data = loop.run_until_complete(queue.dequeue())
+ if data is not None:
+ logger.debug(
+ f"[QueueManager] Dequeued message from {queue.name}: {data}"
+ )
+ else:
+ stop_event.wait(self._poll_interval)
except Exception as e:
- logger.error(f"[QueueManager] Heartbeat error: {e}")
+ logger.error(f"[QueueManager] Worker error for {queue.name}: {e}")
import traceback
traceback.print_exc()
-
- # Call every 0.2 seconds, using wait to respond faster to stop signal
- self._stop_event.wait(0.2)
+ stop_event.wait(self._poll_interval)
finally:
loop.close()
- async def heartbeat(self) -> None:
- """Async entry point for dequeue and dispatch."""
- for queue in list(self._queues.values()):
- queue_size = await queue.size()
- if queue.has_dequeue_handler() and queue_size > 0:
- data = await queue.dequeue()
- if data is not None:
- logger.debug(f"[QueueManager] Dequeued message from {queue.name}: {data}")
-
def stop(self) -> None:
"""Stop QueueManager and release resources."""
if not self._started:
return
- # Stop heartbeat thread
- self._stop_event.set()
- if self._heartbeat_thread:
- self._heartbeat_thread.join()
- self._heartbeat_thread = None
+ # Stop queue workers
+ for stop_event in self._queue_stop_events.values():
+ stop_event.set()
+ for thread in self._queue_threads.values():
+ thread.join()
+ self._queue_threads.clear()
+ self._queue_stop_events.clear()
self._agfs = None
self._queues.clear()
@@ -185,6 +201,11 @@ class QueueManager:
enqueue_hook=enqueue_hook,
dequeue_handler=dequeue_handler,
)
+ if self._started:
+ self._start_queue_worker(self._queues[name])
+ elif self._started:
+ # Ensure existing queue has a worker running
+ self._start_queue_worker(self._queues[name])
return self._queues[name]
# ========== Compatibility convenience methods ==========
diff --git a/openviking/storage/queuefs/semantic_dag.py b/openviking/storage/queuefs/semantic_dag.py
new file mode 100644
index 000000000..98c432173
--- /dev/null
+++ b/openviking/storage/queuefs/semantic_dag.py
@@ -0,0 +1,273 @@
+# Copyright (c) 2026 Beijing Volcano Engine Technology Co., Ltd.
+# SPDX-License-Identifier: Apache-2.0
+"""Semantic DAG executor with event-driven lazy dispatch."""
+
+import asyncio
+from dataclasses import dataclass, field
+from typing import Dict, List, Optional
+
+from openviking.storage.viking_fs import get_viking_fs
+from openviking.utils import VikingURI
+from openviking.utils.logger import get_logger
+
+logger = get_logger(__name__)
+
+
+@dataclass
+class DirNode:
+ """Directory node state for DAG execution."""
+
+ uri: str
+ children_dirs: List[str]
+ file_paths: List[str]
+ file_index: Dict[str, int]
+ child_index: Dict[str, int]
+ file_summaries: List[Optional[Dict[str, str]]]
+ children_abstracts: List[Optional[Dict[str, str]]]
+ pending: int
+ dispatched: bool = False
+ overview_scheduled: bool = False
+ lock: asyncio.Lock = field(default_factory=asyncio.Lock)
+
+
+@dataclass
+class DagStats:
+ total_nodes: int = 0
+ pending_nodes: int = 0
+ overview_scheduled_nodes: int = 0
+ done_nodes: int = 0
+
+
+class SemanticDagExecutor:
+ """Execute semantic generation with DAG-style, event-driven lazy dispatch."""
+
+ def __init__(self, processor: "SemanticProcessor", context_type: str, max_concurrent_llm: int):
+ self._processor = processor
+ self._context_type = context_type
+ self._max_concurrent_llm = max_concurrent_llm
+ self._llm_sem = asyncio.Semaphore(max_concurrent_llm)
+ self._viking_fs = get_viking_fs()
+ self._nodes: Dict[str, DirNode] = {}
+ self._parent: Dict[str, Optional[str]] = {}
+ self._root_uri: Optional[str] = None
+ self._root_done: Optional[asyncio.Event] = None
+ self._stats = DagStats()
+
+ async def run(self, root_uri: str) -> None:
+ """Run DAG execution starting from root_uri."""
+ self._root_uri = root_uri
+ self._root_done = asyncio.Event()
+ await self._dispatch_dir(root_uri, parent_uri=None)
+ await self._root_done.wait()
+
+ async def _dispatch_dir(self, dir_uri: str, parent_uri: Optional[str]) -> None:
+ """Lazy-dispatch tasks for a directory when it is triggered."""
+ if dir_uri in self._nodes:
+ return
+
+ self._parent[dir_uri] = parent_uri
+
+ try:
+ children_dirs, file_paths = await self._list_dir(dir_uri)
+ file_index = {path: idx for idx, path in enumerate(file_paths)}
+ child_index = {path: idx for idx, path in enumerate(children_dirs)}
+ pending = len(children_dirs) + len(file_paths)
+
+ node = DirNode(
+ uri=dir_uri,
+ children_dirs=children_dirs,
+ file_paths=file_paths,
+ file_index=file_index,
+ child_index=child_index,
+ file_summaries=[None] * len(file_paths),
+ children_abstracts=[None] * len(children_dirs),
+ pending=pending,
+ dispatched=True,
+ )
+ self._nodes[dir_uri] = node
+ self._stats.total_nodes += 1
+ self._stats.pending_nodes += 1
+
+ if pending == 0:
+ self._schedule_overview(dir_uri)
+ return
+
+ for file_path in file_paths:
+ self._stats.total_nodes += 1
+ self._stats.pending_nodes += 1
+ asyncio.create_task(self._file_summary_task(dir_uri, file_path))
+
+ for child_uri in children_dirs:
+ asyncio.create_task(self._dispatch_dir(child_uri, dir_uri))
+ except Exception as e:
+ logger.error(f"Failed to dispatch directory {dir_uri}: {e}", exc_info=True)
+ if parent_uri:
+ await self._on_child_done(parent_uri, dir_uri, "")
+ elif self._root_done:
+ self._root_done.set()
+
+ async def _list_dir(self, uri: str) -> tuple[list[str], list[str]]:
+ """List directory entries and return (child_dirs, file_paths)."""
+ try:
+ entries = await self._viking_fs.ls(uri)
+ except Exception as e:
+ logger.warning(f"Failed to list directory {uri}: {e}")
+ return [], []
+
+ children_dirs: List[str] = []
+ file_paths: List[str] = []
+
+ for entry in entries:
+ name = entry.get("name", "")
+ if not name or name.startswith(".") or name in [".", ".."]:
+ continue
+
+ item_uri = VikingURI(uri).join(name).uri
+ if entry.get("isDir", False):
+ children_dirs.append(item_uri)
+ else:
+ file_paths.append(item_uri)
+
+ return children_dirs, file_paths
+
+ async def _file_summary_task(self, parent_uri: str, file_path: str) -> None:
+ """Generate file summary and notify parent completion."""
+ file_name = file_path.split("/")[-1]
+ try:
+ summary_dict = await self._processor._generate_single_file_summary(
+ file_path, llm_sem=self._llm_sem
+ )
+ except Exception as e:
+ logger.warning(f"Failed to generate summary for {file_path}: {e}")
+ summary_dict = {"name": file_name, "summary": ""}
+ finally:
+ self._stats.done_nodes += 1
+ self._stats.pending_nodes = max(0, self._stats.pending_nodes - 1)
+
+ await self._on_file_done(parent_uri, file_path, summary_dict)
+
+ async def _on_file_done(
+ self, parent_uri: str, file_path: str, summary_dict: Dict[str, str]
+ ) -> None:
+ node = self._nodes.get(parent_uri)
+ if not node:
+ return
+
+ async with node.lock:
+ idx = node.file_index.get(file_path)
+ if idx is not None:
+ node.file_summaries[idx] = summary_dict
+ node.pending -= 1
+ if node.pending == 0 and not node.overview_scheduled:
+ node.overview_scheduled = True
+ self._stats.overview_scheduled_nodes += 1
+ asyncio.create_task(self._overview_task(parent_uri))
+
+ async def _on_child_done(self, parent_uri: str, child_uri: str, abstract: str) -> None:
+ node = self._nodes.get(parent_uri)
+ if not node:
+ return
+
+ child_name = child_uri.split("/")[-1]
+ async with node.lock:
+ idx = node.child_index.get(child_uri)
+ if idx is not None:
+ node.children_abstracts[idx] = {"name": child_name, "abstract": abstract}
+ node.pending -= 1
+ if node.pending == 0 and not node.overview_scheduled:
+ node.overview_scheduled = True
+ self._stats.overview_scheduled_nodes += 1
+ asyncio.create_task(self._overview_task(parent_uri))
+
+ def _schedule_overview(self, dir_uri: str) -> None:
+ node = self._nodes.get(dir_uri)
+ if not node:
+ return
+ if node.overview_scheduled:
+ return
+ node.overview_scheduled = True
+ self._stats.overview_scheduled_nodes += 1
+ asyncio.create_task(self._overview_task(dir_uri))
+
+ def _finalize_file_summaries(self, node: DirNode) -> List[Dict[str, str]]:
+ summaries: List[Dict[str, str]] = []
+ for idx, file_path in enumerate(node.file_paths):
+ item = node.file_summaries[idx]
+ if item is None:
+ summaries.append({"name": file_path.split("/")[-1], "summary": ""})
+ else:
+ summaries.append(item)
+ return summaries
+
+ def _finalize_children_abstracts(self, node: DirNode) -> List[Dict[str, str]]:
+ results: List[Dict[str, str]] = []
+ for idx, child_uri in enumerate(node.children_dirs):
+ item = node.children_abstracts[idx]
+ if item is None:
+ results.append({"name": child_uri.split("/")[-1], "abstract": ""})
+ else:
+ results.append(item)
+ return results
+
+ async def _overview_task(self, dir_uri: str) -> None:
+ node = self._nodes.get(dir_uri)
+ if not node:
+ return
+
+ async with node.lock:
+ file_summaries = self._finalize_file_summaries(node)
+ children_abstracts = self._finalize_children_abstracts(node)
+
+ try:
+ async with self._llm_sem:
+ overview = await self._processor._generate_overview(
+ dir_uri, file_summaries, children_abstracts
+ )
+ abstract = self._processor._extract_abstract_from_overview(overview)
+
+ try:
+ await self._viking_fs.write_file(f"{dir_uri}/.overview.md", overview)
+ await self._viking_fs.write_file(f"{dir_uri}/.abstract.md", abstract)
+ except Exception as e:
+ logger.warning(f"Failed to write overview/abstract for {dir_uri}: {e}")
+
+ try:
+ await self._processor._vectorize_directory_simple(
+ dir_uri, self._context_type, abstract, overview
+ )
+ except Exception as e:
+ logger.error(f"Failed to vectorize directory {dir_uri}: {e}", exc_info=True)
+
+ try:
+ await self._processor._vectorize_files(
+ dir_uri, self._context_type, node.file_paths, file_summaries
+ )
+ except Exception as e:
+ logger.error(f"Failed to vectorize files in {dir_uri}: {e}", exc_info=True)
+
+ except Exception as e:
+ logger.error(f"Failed to generate overview for {dir_uri}: {e}", exc_info=True)
+ abstract = ""
+ finally:
+ self._stats.done_nodes += 1
+ self._stats.pending_nodes = max(0, self._stats.pending_nodes - 1)
+
+ parent_uri = self._parent.get(dir_uri)
+ if parent_uri is None:
+ if self._root_done:
+ self._root_done.set()
+ return
+
+ await self._on_child_done(parent_uri, dir_uri, abstract)
+
+ def get_stats(self) -> DagStats:
+ return DagStats(
+ total_nodes=self._stats.total_nodes,
+ pending_nodes=self._stats.pending_nodes,
+ overview_scheduled_nodes=self._stats.overview_scheduled_nodes,
+ done_nodes=self._stats.done_nodes,
+ )
+
+
+if False: # pragma: no cover - for type checkers only
+ from openviking.storage.queuefs.semantic_processor import SemanticProcessor
diff --git a/openviking/storage/queuefs/semantic_processor.py b/openviking/storage/queuefs/semantic_processor.py
index db493abff..9922b6f2a 100644
--- a/openviking/storage/queuefs/semantic_processor.py
+++ b/openviking/storage/queuefs/semantic_processor.py
@@ -15,6 +15,7 @@ from openviking.parse.parsers.constants import (
)
from openviking.prompts import render_prompt
from openviking.storage.queuefs.named_queue import DequeueHandlerBase
+from openviking.storage.queuefs.semantic_dag import DagStats, SemanticDagExecutor
from openviking.storage.queuefs.semantic_msg import SemanticMsg
from openviking.storage.viking_fs import get_viking_fs
from openviking.utils import VikingURI
@@ -35,7 +36,7 @@ class SemanticProcessor(DequeueHandlerBase):
4. Enqueue to EmbeddingQueue for vectorization
"""
- def __init__(self, max_concurrent_llm: int = 10):
+ def __init__(self, max_concurrent_llm: int = 100):
"""
Initialize SemanticProcessor.
@@ -43,6 +44,7 @@ class SemanticProcessor(DequeueHandlerBase):
max_concurrent_llm: Maximum concurrent LLM calls
"""
self.max_concurrent_llm = max_concurrent_llm
+ self._dag_executor: Optional[SemanticDagExecutor] = None
def _detect_file_type(self, file_name: str) -> str:
"""
@@ -135,26 +137,14 @@ class SemanticProcessor(DequeueHandlerBase):
)
if msg.recursive:
- # For recursive processing, collect all directory info
- dir_info_list: List[Tuple[str, List[str], List[str]]] = []
- await self._collect_directory_info(msg.uri, dir_info_list)
-
- # Enqueue each directory for processing (leaves first)
- # For re-enqueued messages, set recursive=False to avoid repeated recursion
- for uri, _, _ in dir_info_list:
- # Create a new message for this directory
- sub_msg = SemanticMsg(
- uri=uri,
- context_type=msg.context_type,
- recursive=False, # Avoid repeated recursion
- )
-
- # Enqueue for processing
- await self._enqueue_semantic_msg(sub_msg)
-
- logger.info(
- f"Enqueued {len(dir_info_list)} directories for processing from: {msg.uri}"
+ executor = SemanticDagExecutor(
+ processor=self,
+ context_type=msg.context_type,
+ max_concurrent_llm=self.max_concurrent_llm,
)
+ self._dag_executor = executor
+ await executor.run(msg.uri)
+ logger.info(f"Completed semantic generation for: {msg.uri}")
self.report_success()
return None
else:
@@ -197,6 +187,11 @@ class SemanticProcessor(DequeueHandlerBase):
self.report_error(str(e), data)
return None
+ def get_dag_stats(self) -> Optional["DagStats"]:
+ if not self._dag_executor:
+ return None
+ return self._dag_executor.get_stats()
+
async def _process_single_directory(
self,
uri: str,
@@ -263,7 +258,9 @@ class SemanticProcessor(DequeueHandlerBase):
tasks = [generate_one_summary(fp) for fp in file_paths]
return await asyncio.gather(*tasks)
- async def _generate_single_file_summary(self, file_path: str) -> Dict[str, str]:
+ async def _generate_single_file_summary(
+ self, file_path: str, llm_sem: Optional[asyncio.Semaphore] = None
+ ) -> Dict[str, str]:
"""Generate summary for a single file.
Args:
@@ -307,7 +304,11 @@ class SemanticProcessor(DequeueHandlerBase):
{"file_name": file_name, "content": content},
)
- summary = await vlm.get_completion_async(prompt)
+ if llm_sem:
+ async with llm_sem:
+ summary = await vlm.get_completion_async(prompt)
+ else:
+ summary = await vlm.get_completion_async(prompt)
return {"name": file_name, "summary": summary.strip()}
except Exception as e: