Skip to content

Self-evolving Context Database for AI Agents. Unify Agent Memory, Knowledge RAG and Skills.

fix(queuefs): bound semantic node scheduling (#2343)

Qin Haojie294810dfb3a3
diff --git a/openviking/service/core.py b/openviking/service/core.py
index 7edd3ecd7..f2428cb05 100644
--- a/openviking/service/core.py
+++ b/openviking/service/core.py
@@ -115,7 +115,7 @@ class OpenVikingService:
self,
config: StorageConfig,
max_concurrent_embedding: int = 10,
- max_concurrent_semantic: int = 100,
+ max_concurrent_semantic: int = 64,
) -> None:
"""Initialize storage resources."""
from openviking.utils.agfs_utils import create_agfs_client
diff --git a/openviking/storage/queuefs/queue_manager.py b/openviking/storage/queuefs/queue_manager.py
index 222ec7c2f..962e62291 100644
--- a/openviking/storage/queuefs/queue_manager.py
+++ b/openviking/storage/queuefs/queue_manager.py
@@ -29,7 +29,7 @@ def init_queue_manager(
timeout: int = 10,
mount_point: str = "/queue",
max_concurrent_embedding: int = 10,
- max_concurrent_semantic: int = 100,
+ max_concurrent_semantic: int = 64,
) -> "QueueManager":
"""Initialize QueueManager singleton.
@@ -38,7 +38,7 @@ def init_queue_manager(
timeout: Request timeout in seconds.
mount_point: Path where QueueFS is mounted.
max_concurrent_embedding: Max concurrent embedding tasks.
- max_concurrent_semantic: Max concurrent semantic tasks.
+ max_concurrent_semantic: Max concurrent semantic node work.
"""
global _instance
_instance = QueueManager(
@@ -74,7 +74,7 @@ class QueueManager:
timeout: int = 10,
mount_point: str = "/queue",
max_concurrent_embedding: int = 10,
- max_concurrent_semantic: int = 100,
+ max_concurrent_semantic: int = 64,
):
"""Initialize QueueManager."""
self._agfs = agfs
diff --git a/openviking/storage/queuefs/semantic_dag.py b/openviking/storage/queuefs/semantic_dag.py
index cd3143ffa..a5b9921a3 100644
--- a/openviking/storage/queuefs/semantic_dag.py
+++ b/openviking/storage/queuefs/semantic_dag.py
@@ -3,8 +3,10 @@
"""Semantic DAG executor with event-driven lazy dispatch."""
import asyncio
+import threading
+from weakref import WeakKeyDictionary
from dataclasses import dataclass, field
-from typing import Dict, List, Optional
+from typing import ClassVar, Dict, List, Optional, Set
from openviking.server.identity import RequestContext
from openviking.storage.queuefs.semantic_sidecar import write_semantic_sidecars
@@ -66,9 +68,94 @@ class VectorizeTask:
overview: Optional[str] = None
+@dataclass(frozen=True)
+class DagWork:
+ """A scheduled unit of DAG work."""
+
+ kind: str
+ dir_uri: str
+ parent_uri: Optional[str] = None
+ file_path: Optional[str] = None
+ vectorize_task: Optional[VectorizeTask] = None
+
+
+@dataclass(frozen=True)
+class ScheduledDagWork:
+ executor: "SemanticDagExecutor"
+ work: DagWork
+
+
+class SemanticNodeScheduler:
+ """Shared node executor for semantic DAG work in one event loop."""
+
+ _idle_timeout = 0.05
+
+ def __init__(self, max_workers: int):
+ self._max_workers = max(1, max_workers)
+ self._queue: asyncio.Queue[ScheduledDagWork] = asyncio.Queue()
+ self._workers: Set[asyncio.Task] = set()
+
+ def configure(self, max_workers: int) -> None:
+ self._max_workers = max(1, max_workers)
+ self._ensure_workers()
+
+ def submit(self, executor: "SemanticDagExecutor", work: DagWork) -> None:
+ self._queue.put_nowait(ScheduledDagWork(executor=executor, work=work))
+ self._ensure_workers()
+
+ def _ensure_workers(self) -> None:
+ self._workers = {task for task in self._workers if not task.done()}
+ target = min(self._max_workers, self._queue.qsize())
+ while len(self._workers) < target:
+ task = asyncio.create_task(self._worker())
+ task.add_done_callback(self._workers.discard)
+ self._workers.add(task)
+
+ async def _worker(self) -> None:
+ while True:
+ try:
+ item = await asyncio.wait_for(
+ self._queue.get(),
+ timeout=self._idle_timeout,
+ )
+ except asyncio.TimeoutError:
+ if self._queue.empty():
+ return
+ continue
+
+ try:
+ if not item.executor.closed:
+ await item.executor._run_work(item.work)
+ except asyncio.CancelledError:
+ raise
+ except Exception as exc:
+ item.executor.fail(exc)
+ finally:
+ self._queue.task_done()
+
+
+_node_schedulers: "WeakKeyDictionary[asyncio.AbstractEventLoop, SemanticNodeScheduler]" = (
+ WeakKeyDictionary()
+)
+
+
+def get_semantic_node_scheduler(max_workers: int) -> SemanticNodeScheduler:
+ loop = asyncio.get_running_loop()
+ scheduler = _node_schedulers.get(loop)
+ if scheduler is None:
+ scheduler = SemanticNodeScheduler(max_workers=max_workers)
+ _node_schedulers[loop] = scheduler
+ else:
+ scheduler.configure(max_workers)
+ return scheduler
+
+
class SemanticDagExecutor:
"""Execute semantic generation with DAG-style, event-driven lazy dispatch."""
+ _active_lock: ClassVar[threading.Lock] = threading.Lock()
+ _active_executors: ClassVar[Set["SemanticDagExecutor"]] = set()
+
def __init__(
self,
processor: "SemanticProcessor",
@@ -89,7 +176,6 @@ class SemanticDagExecutor:
):
self._processor = processor
self._context_type = context_type
- self._max_concurrent_llm = max_concurrent_llm
self._ctx = ctx
self._incremental_update = incremental_update
self._target_uri = target_uri
@@ -106,15 +192,21 @@ class SemanticDagExecutor:
self._changed_paths = {
path for key in ("added", "modified", "deleted") for path in self._changes.get(key, [])
}
+ self._node_concurrency = max(1, max_concurrent_llm)
self._llm_sem = asyncio.Semaphore(max_concurrent_llm)
self._viking_fs = get_viking_fs()
self._nodes: Dict[str, DirNode] = {}
self._parent: Dict[str, Optional[str]] = {}
self._root_uri: Optional[str] = None
self._root_done: Optional[asyncio.Event] = None
+ self._scheduler: Optional[SemanticNodeScheduler] = None
+ self._closed = False
+ self._failure: Optional[Exception] = None
self._stats = DagStats()
self._vectorize_task_count: int = 0
self._pending_vectorize_tasks: List[VectorizeTask] = []
+ self._pending_vectorize_work = 0
+ self._vectorize_done: Optional[asyncio.Event] = None
self._vectorize_lock = asyncio.Lock()
self._file_change_status: Dict[str, bool] = {}
self._dir_change_status: Dict[str, bool] = {}
@@ -125,74 +217,211 @@ class SemanticDagExecutor:
"""Run DAG execution starting from root_uri."""
self._root_uri = root_uri
self._root_done = asyncio.Event()
+ self._scheduler = get_semantic_node_scheduler(self._node_concurrency)
try:
- await self._dispatch_dir(root_uri, parent_uri=None)
+ self._register_active()
+ self._schedule_dir(root_uri, parent_uri=None)
await self._root_done.wait()
- except Exception:
- await self._lock.close()
+ if self._failure:
+ raise self._failure
+
+ # Release owned semantic locks after downstream vectorization finishes.
+ async def wrapped_on_complete() -> None:
+ try:
+ if self._telemetry_id and self._semantic_msg_id:
+ get_request_wait_tracker().mark_semantic_done(
+ self._telemetry_id, self._semantic_msg_id
+ )
+ finally:
+ await self._lock.close()
+
+ async with self._vectorize_lock:
+ task_count = self._vectorize_task_count
+ tasks = list(self._pending_vectorize_tasks)
+
+ if task_count > 0:
+ from .embedding_tracker import EmbeddingTaskTracker
+
+ tracker = EmbeddingTaskTracker.get_instance()
+ await tracker.register(
+ semantic_msg_id=self._semantic_msg_id,
+ total_count=task_count,
+ on_complete=wrapped_on_complete,
+ metadata={"uri": root_uri},
+ )
+
+ await self._dispatch_vectorize_tasks(tasks)
+ else:
+ # No vectorize tasks — release lock immediately (via wrapped callback)
+ try:
+ await wrapped_on_complete()
+ except Exception as e:
+ logger.error(f"Error in on_complete callback: {e}", exc_info=True)
+ except BaseException:
+ self._closed = True
+ try:
+ await self._lock.close()
+ except Exception:
+ pass
raise
+ finally:
+ self._closed = True
+ self._unregister_active()
+
+ def _schedule_work(self, work: DagWork) -> None:
+ if self._closed:
+ return
+ if self._scheduler is None:
+ self._scheduler = get_semantic_node_scheduler(self._node_concurrency)
+ self._scheduler.submit(self, work)
+
+ def _schedule_dir(self, dir_uri: str, parent_uri: Optional[str]) -> None:
+ if self._closed:
+ return
+ self._stats.total_nodes += 1
+ self._stats.pending_nodes += 1
+ self._schedule_work(DagWork(kind="dir", dir_uri=dir_uri, parent_uri=parent_uri))
- # Release owned semantic locks after downstream vectorization finishes.
- async def wrapped_on_complete() -> None:
+ def _schedule_file(self, parent_uri: str, file_path: str) -> None:
+ if self._closed:
+ return
+ self._stats.total_nodes += 1
+ self._stats.pending_nodes += 1
+ self._schedule_work(DagWork(kind="file", dir_uri=parent_uri, file_path=file_path))
+
+ def _mark_node_started(self) -> None:
+ self._stats.pending_nodes = max(0, self._stats.pending_nodes - 1)
+ self._stats.in_progress_nodes += 1
+
+ def _mark_node_waiting(self) -> None:
+ self._stats.in_progress_nodes = max(0, self._stats.in_progress_nodes - 1)
+ self._stats.pending_nodes += 1
+
+ def _mark_node_done(self) -> None:
+ self._stats.done_nodes += 1
+ self._stats.in_progress_nodes = max(0, self._stats.in_progress_nodes - 1)
+
+ def _release_dir_node(self, dir_uri: str) -> None:
+ self._nodes.pop(dir_uri, None)
+ self._parent.pop(dir_uri, None)
+
+ @property
+ def closed(self) -> bool:
+ return self._closed
+
+ def fail(self, exc: Exception) -> None:
+ if self._failure is None:
+ self._failure = exc
+ self._closed = True
+ if self._root_done:
+ self._root_done.set()
+ if self._vectorize_done:
+ self._vectorize_done.set()
+
+ def _register_active(self) -> None:
+ with self._active_lock:
+ self._active_executors.add(self)
+
+ def _unregister_active(self) -> None:
+ with self._active_lock:
+ self._active_executors.discard(self)
+
+ @classmethod
+ def get_active_stats(cls) -> DagStats:
+ stats = DagStats()
+ with cls._active_lock:
+ executors = list(cls._active_executors)
+ for executor in executors:
+ current = executor.get_stats()
+ stats.total_nodes += current.total_nodes
+ stats.pending_nodes += current.pending_nodes
+ stats.in_progress_nodes += current.in_progress_nodes
+ stats.done_nodes += current.done_nodes
+ return stats
+
+ async def _run_work(self, work: DagWork) -> None:
+ if work.kind == "vectorize":
+ await self._run_vectorize_work(work.vectorize_task)
+ return
+
+ self._mark_node_started()
+
+ if work.kind == "dir":
+ terminal = False
try:
- if self._telemetry_id and self._semantic_msg_id:
- get_request_wait_tracker().mark_semantic_done(
- self._telemetry_id, self._semantic_msg_id
- )
+ terminal = await self._dispatch_dir(work.dir_uri, work.parent_uri)
finally:
- await self._lock.close()
+ if terminal:
+ self._mark_node_done()
+ else:
+ self._mark_node_waiting()
+ return
- async with self._vectorize_lock:
- task_count = self._vectorize_task_count
- tasks = list(self._pending_vectorize_tasks)
-
- if task_count > 0:
- from .embedding_tracker import EmbeddingTaskTracker
-
- tracker = EmbeddingTaskTracker.get_instance()
- await tracker.register(
- semantic_msg_id=self._semantic_msg_id,
- total_count=task_count,
- on_complete=wrapped_on_complete,
- metadata={"uri": root_uri},
+ if work.kind == "file":
+ if work.file_path is None:
+ self._mark_node_done()
+ return
+ await self._file_summary_task(work.dir_uri, work.file_path)
+ return
+
+ if work.kind == "overview":
+ await self._overview_task(work.dir_uri)
+ return
+
+ self._mark_node_done()
+ logger.warning("Unknown semantic DAG work kind: %s", work.kind)
+
+ async def _dispatch_vectorize_tasks(self, tasks: List[VectorizeTask]) -> None:
+ self._vectorize_done = asyncio.Event()
+ self._pending_vectorize_work = len(tasks)
+ for task in tasks:
+ self._schedule_work(
+ DagWork(
+ kind="vectorize",
+ dir_uri=task.uri,
+ vectorize_task=task,
+ )
)
+ await self._vectorize_done.wait()
- for task in tasks:
- if task.task_type == "file":
- asyncio.create_task(
- self._processor._vectorize_single_file(
- parent_uri=task.parent_uri,
- context_type=task.context_type,
- file_path=task.file_path,
- summary_dict=task.summary_dict,
- ctx=task.ctx,
- semantic_msg_id=task.semantic_msg_id,
- use_summary=task.use_summary,
- )
- )
- else:
- asyncio.create_task(
- self._processor._vectorize_directory(
- task.uri,
- task.context_type,
- task.abstract,
- task.overview,
- ctx=task.ctx,
- semantic_msg_id=task.semantic_msg_id,
- )
- )
- else:
- # No vectorize tasks — release lock immediately (via wrapped callback)
- try:
- await wrapped_on_complete()
- except Exception as e:
- logger.error(f"Error in on_complete callback: {e}", exc_info=True)
+ async def _run_vectorize_work(self, task: Optional[VectorizeTask]) -> None:
+ try:
+ if task is not None:
+ await self._run_vectorize_task(task)
+ except Exception as exc:
+ logger.error("Vectorization dispatch task failed: %s", exc, exc_info=True)
+ finally:
+ self._pending_vectorize_work = max(0, self._pending_vectorize_work - 1)
+ if self._pending_vectorize_work == 0 and self._vectorize_done:
+ self._vectorize_done.set()
+
+ async def _run_vectorize_task(self, task: VectorizeTask) -> None:
+ if task.task_type == "file":
+ await self._processor._vectorize_single_file(
+ parent_uri=task.parent_uri,
+ context_type=task.context_type,
+ file_path=task.file_path,
+ summary_dict=task.summary_dict,
+ ctx=task.ctx,
+ semantic_msg_id=task.semantic_msg_id,
+ use_summary=task.use_summary,
+ )
+ return
- async def _dispatch_dir(self, dir_uri: str, parent_uri: Optional[str]) -> None:
+ await self._processor._vectorize_directory(
+ task.uri,
+ task.context_type,
+ task.abstract,
+ task.overview,
+ ctx=task.ctx,
+ semantic_msg_id=task.semantic_msg_id,
+ )
+
+ async def _dispatch_dir(self, dir_uri: str, parent_uri: Optional[str]) -> bool:
"""Lazy-dispatch tasks for a directory when it is triggered."""
if dir_uri in self._nodes:
- return
+ return True
self._parent[dir_uri] = parent_uri
@@ -217,31 +446,26 @@ class SemanticDagExecutor:
dispatched=True,
)
self._nodes[dir_uri] = node
- self._stats.total_nodes += 1
- self._stats.pending_nodes += 1
if pending == 0:
self._schedule_overview(dir_uri)
- return
+ return False
for file_path in file_paths:
- self._stats.total_nodes += 1
- # File nodes are scheduled immediately: pending -> in_progress.
- self._stats.pending_nodes += 1
- self._stats.pending_nodes = max(0, self._stats.pending_nodes - 1)
- self._stats.in_progress_nodes += 1
- asyncio.create_task(self._file_summary_task(dir_uri, file_path))
+ self._schedule_file(dir_uri, file_path)
if children_dirs:
if self._recursive:
for child_uri in children_dirs:
- asyncio.create_task(self._dispatch_dir(child_uri, dir_uri))
+ self._schedule_dir(child_uri, dir_uri)
+ return False
except Exception as e:
logger.error(f"Failed to dispatch directory {dir_uri}: {e}", exc_info=True)
if parent_uri:
await self._on_child_done(parent_uri, dir_uri, "")
elif self._root_done:
self._root_done.set()
+ return True
async def _list_dir(self, uri: str, from_hint: str) -> tuple[list[str], list[str]]:
"""List directory entries and return (child_dirs, file_paths)."""
@@ -272,7 +496,8 @@ class SemanticDagExecutor:
def _get_target_file_path(self, current_uri: str) -> Optional[str]:
if not self._incremental_update or not self._target_uri or not self._root_uri:
logger.warning(
- f"Invalid target_uri or root_uri for incremental update: target_uri={self._target_uri}, root_uri={self._root_uri}"
+ "Invalid target_uri or root_uri for incremental update: "
+ f"target_uri={self._target_uri}, root_uri={self._root_uri}"
)
return None
if self._target_uri != self._root_uri:
@@ -485,10 +710,7 @@ class SemanticDagExecutor:
node.file_summaries[idx] = summary_dict
node.pending -= 1
if node.pending == 0 and not node.overview_scheduled:
- node.overview_scheduled = True
- self._stats.pending_nodes = max(0, self._stats.pending_nodes - 1)
- self._stats.in_progress_nodes += 1
- asyncio.create_task(self._overview_task(parent_uri))
+ self._schedule_overview(parent_uri)
async def _on_child_done(self, parent_uri: str, child_uri: str, abstract: str) -> None:
node = self._nodes.get(parent_uri)
@@ -502,21 +724,14 @@ class SemanticDagExecutor:
node.children_abstracts[idx] = {"name": child_name, "abstract": abstract}
node.pending -= 1
if node.pending == 0 and not node.overview_scheduled:
- node.overview_scheduled = True
- self._stats.pending_nodes = max(0, self._stats.pending_nodes - 1)
- self._stats.in_progress_nodes += 1
- asyncio.create_task(self._overview_task(parent_uri))
+ self._schedule_overview(parent_uri)
def _schedule_overview(self, dir_uri: str) -> None:
node = self._nodes.get(dir_uri)
- if not node:
- return
- if node.overview_scheduled:
+ if not node or node.overview_scheduled:
return
node.overview_scheduled = True
- self._stats.pending_nodes = max(0, self._stats.pending_nodes - 1)
- self._stats.in_progress_nodes += 1
- asyncio.create_task(self._overview_task(dir_uri))
+ self._schedule_work(DagWork(kind="overview", dir_uri=dir_uri))
def _finalize_file_summaries(self, node: DirNode) -> List[Dict[str, str]]:
summaries: List[Dict[str, str]] = []
@@ -633,11 +848,13 @@ class SemanticDagExecutor:
parent_uri = self._parent.get(dir_uri)
if parent_uri is None:
+ self._release_dir_node(dir_uri)
if self._root_done:
self._root_done.set()
return
await self._on_child_done(parent_uri, dir_uri, abstract)
+ self._release_dir_node(dir_uri)
async def _add_vectorize_task(self, task: VectorizeTask) -> None:
"""Add a vectorize task to the pending list."""
diff --git a/openviking/storage/queuefs/semantic_processor.py b/openviking/storage/queuefs/semantic_processor.py
index 6b5d0ee29..c4d0d6fe4 100644
--- a/openviking/storage/queuefs/semantic_processor.py
+++ b/openviking/storage/queuefs/semantic_processor.py
@@ -96,7 +96,7 @@ class SemanticProcessor(DequeueHandlerBase):
_request_stats_order: List[str] = []
_max_cached_stats = 256
- def __init__(self, max_concurrent_llm: int = 100):
+ def __init__(self, max_concurrent_llm: int = 64):
"""
Initialize SemanticProcessor.
@@ -104,9 +104,7 @@ class SemanticProcessor(DequeueHandlerBase):
max_concurrent_llm: Maximum concurrent LLM calls
"""
self.max_concurrent_llm = max_concurrent_llm
- self._dag_executor: Optional[SemanticDagExecutor] = None
- self._current_ctx = RequestContext(user=UserIdentifier.the_default_user(), role=Role.ROOT)
- self._current_msg: Optional[SemanticMsg] = None
+ self._default_ctx = RequestContext(user=UserIdentifier.the_default_user(), role=Role.ROOT)
self._circuit_breaker = CircuitBreaker()
@classmethod
@@ -356,8 +354,7 @@ class SemanticProcessor(DequeueHandlerBase):
root_attrs.agent_id = msg.agent_id
root_context_token = bind_root_observability_context(root_attrs)
try:
- self._current_msg = msg
- self._current_ctx = self._ctx_from_semantic_msg(msg)
+ current_ctx = self._ctx_from_semantic_msg(msg)
logger.info(
f"Processing semantic generation for: {msg.uri} (recursive={msg.recursive})"
)
@@ -374,6 +371,7 @@ class SemanticProcessor(DequeueHandlerBase):
lock_transferred = True
await self._process_memory_directory(
msg,
+ ctx=current_ctx,
lock=semantic_lock.lock,
)
else:
@@ -384,7 +382,7 @@ class SemanticProcessor(DequeueHandlerBase):
viking_fs = get_viking_fs()
if msg.target_uri:
target_exists = await viking_fs.exists(
- msg.target_uri, ctx=self._current_ctx
+ msg.target_uri, ctx=current_ctx
)
if msg.uri != msg.target_uri:
logger.info(
@@ -394,7 +392,7 @@ class SemanticProcessor(DequeueHandlerBase):
diff = await self._sync_topdown_recursive(
msg.uri,
msg.target_uri,
- ctx=self._current_ctx,
+ ctx=current_ctx,
lock=semantic_lock.lock,
)
logger.info(
@@ -425,7 +423,7 @@ class SemanticProcessor(DequeueHandlerBase):
processor=self,
context_type=msg.context_type,
max_concurrent_llm=self.max_concurrent_llm,
- ctx=self._current_ctx,
+ ctx=current_ctx,
incremental_update=is_incremental,
target_uri=target_uri,
semantic_msg_id=msg.id,
@@ -438,7 +436,6 @@ class SemanticProcessor(DequeueHandlerBase):
coalesce_key=msg.coalesce_key,
coalesce_version=msg.coalesce_version,
)
- self._dag_executor = executor
lock_transferred = True
await executor.run(run_uri)
self._cache_dag_stats(
@@ -509,16 +506,15 @@ class SemanticProcessor(DequeueHandlerBase):
else:
self.report_error(str(e), data)
return None
- finally:
- self._current_msg = None
- self._current_ctx = None
-
def get_dag_stats(self) -> Optional["DagStats"]:
- if not self._dag_executor:
- return None
- return self._dag_executor.get_stats()
+ return SemanticDagExecutor.get_active_stats()
- async def _process_memory_directory(self, msg: SemanticMsg, lock: LockLease = NO_LOCK) -> None:
+ async def _process_memory_directory(
+ self,
+ msg: SemanticMsg,
+ ctx: Optional[RequestContext] = None,
+ lock: LockLease = NO_LOCK,
+ ) -> None:
"""Process a memory directory with special handling.
For memory directories:
@@ -531,7 +527,7 @@ class SemanticProcessor(DequeueHandlerBase):
"""
viking_fs = get_viking_fs()
dir_uri = msg.uri
- ctx = self._current_ctx
+ ctx = ctx or self._default_ctx
llm_sem = asyncio.Semaphore(self.max_concurrent_llm)
request_wait_tracker = get_request_wait_tracker()
@@ -959,7 +955,7 @@ class SemanticProcessor(DequeueHandlerBase):
"""Generate summary for a single text file (code, documentation, or other text)."""
viking_fs = get_viking_fs()
vlm = get_openviking_config().vlm
- active_ctx = ctx or self._current_ctx
+ active_ctx = ctx or self._default_ctx
content = await viking_fs.read_file(file_path, ctx=active_ctx)
if isinstance(content, bytes):
@@ -1437,13 +1433,9 @@ class SemanticProcessor(DequeueHandlerBase):
) -> None:
"""Create directory Context and enqueue to EmbeddingQueue."""
- if self._current_msg and getattr(self._current_msg, "skip_vectorization", False):
- logger.info(f"Skipping vectorization for {uri} (requested via SemanticMsg)")
- return
-
from openviking.utils.embedding_utils import vectorize_directory_meta
- active_ctx = ctx or self._current_ctx
+ active_ctx = ctx or self._default_ctx
await vectorize_directory_meta(
uri=uri,
abstract=abstract,
@@ -1467,7 +1459,7 @@ class SemanticProcessor(DequeueHandlerBase):
"""Vectorize a single file using its content or summary."""
from openviking.utils.embedding_utils import vectorize_file
- active_ctx = ctx or self._current_ctx
+ active_ctx = ctx or self._default_ctx
await vectorize_file(
file_path=file_path,
summary_dict=summary_dict,
diff --git a/openviking_cli/utils/config/vlm_config.py b/openviking_cli/utils/config/vlm_config.py
index a3ae470ef..884ced649 100644
--- a/openviking_cli/utils/config/vlm_config.py
+++ b/openviking_cli/utils/config/vlm_config.py
@@ -65,7 +65,7 @@ class VLMConfig(BaseModel):
thinking: bool = Field(default=False, description="Enable thinking mode for VolcEngine models")
max_concurrent: int = Field(
- default=100, description="Maximum number of concurrent LLM calls for semantic processing"
+ default=64, description="Maximum number of concurrent LLM calls for semantic processing"
)
api_version: Optional[str] = Field(
diff --git a/tests/storage/test_semantic_dag_stats.py b/tests/storage/test_semantic_dag_stats.py
index 751fa3dfc..ff6ad534a 100644
--- a/tests/storage/test_semantic_dag_stats.py
+++ b/tests/storage/test_semantic_dag_stats.py
@@ -64,6 +64,22 @@ class _FakeProcessor:
await self._vectorize_directory(uri, context_type, abstract, overview, ctx=ctx)
+class _TrackingProcessor(_FakeProcessor):
+ def __init__(self):
+ super().__init__()
+ self.active_summaries = 0
+ self.max_active_summaries = 0
+
+ async def _generate_single_file_summary(self, file_path, llm_sem=None, ctx=None):
+ self.active_summaries += 1
+ self.max_active_summaries = max(self.max_active_summaries, self.active_summaries)
+ try:
+ await asyncio.sleep(0.01)
+ return {"name": file_path.split("/")[-1], "summary": "summary"}
+ finally:
+ self.active_summaries -= 1
+
+
class _DummyTracker:
def __init__(self):
self.register_calls = []
@@ -131,6 +147,84 @@ async def test_semantic_dag_stats_collects_nodes(monkeypatch):
)
+async def test_semantic_dag_bounds_active_node_work(monkeypatch):
+ root_uri = "viking://resources/root"
+ tree = {
+ root_uri: [{"name": f"file-{idx}.txt", "isDir": False} for idx in range(40)],
+ }
+ fake_fs = _FakeVikingFS(tree)
+ monkeypatch.setattr("openviking.storage.queuefs.semantic_dag.get_viking_fs", lambda: fake_fs)
+ monkeypatch.setattr(
+ "openviking.storage.queuefs.embedding_tracker.EmbeddingTaskTracker.get_instance",
+ lambda: _DummyTracker(),
+ )
+
+ processor = _TrackingProcessor()
+ ctx = RequestContext(user=UserIdentifier("acc1", "user1", "agent1"), role=Role.USER)
+ executor = SemanticDagExecutor(
+ processor=processor,
+ context_type="resource",
+ max_concurrent_llm=3,
+ ctx=ctx,
+ skip_vectorization=True,
+ )
+
+ max_running = 0
+ run_task = asyncio.create_task(executor.run(root_uri))
+ while not run_task.done():
+ max_running = max(max_running, executor.get_stats().in_progress_nodes)
+ await asyncio.sleep(0)
+ await run_task
+
+ stats = executor.get_stats()
+ assert stats.total_nodes == 41
+ assert stats.done_nodes == 41
+ assert stats.pending_nodes == 0
+ assert stats.in_progress_nodes == 0
+ assert processor.max_active_summaries <= 3
+ assert max_running <= 3
+
+
+async def test_semantic_dag_shares_node_scheduler_across_roots(monkeypatch):
+ root_a = "viking://resources/root-a"
+ root_b = "viking://resources/root-b"
+ tree = {
+ root_a: [{"name": f"a-{idx}.txt", "isDir": False} for idx in range(20)],
+ root_b: [{"name": f"b-{idx}.txt", "isDir": False} for idx in range(20)],
+ }
+ fake_fs = _FakeVikingFS(tree)
+ monkeypatch.setattr("openviking.storage.queuefs.semantic_dag.get_viking_fs", lambda: fake_fs)
+ monkeypatch.setattr(
+ "openviking.storage.queuefs.embedding_tracker.EmbeddingTaskTracker.get_instance",
+ lambda: _DummyTracker(),
+ )
+
+ processor = _TrackingProcessor()
+ ctx = RequestContext(user=UserIdentifier("acc1", "user1", "agent1"), role=Role.USER)
+ executor_a = SemanticDagExecutor(
+ processor=processor,
+ context_type="resource",
+ max_concurrent_llm=4,
+ ctx=ctx,
+ skip_vectorization=True,
+ )
+ executor_b = SemanticDagExecutor(
+ processor=processor,
+ context_type="resource",
+ max_concurrent_llm=4,
+ ctx=ctx,
+ skip_vectorization=True,
+ )
+
+ await asyncio.gather(executor_a.run(root_a), executor_b.run(root_b))
+
+ assert processor.max_active_summaries <= 4
+ assert executor_a.get_stats().done_nodes == 21
+ assert executor_b.get_stats().done_nodes == 21
+
+
@pytest.mark.asyncio
async def test_semantic_dag_skip_vectorization_does_not_schedule_tasks(monkeypatch):
root_uri = "viking://resources/root"