Self-evolving Context Database for AI Agents. Unify Agent Memory, Knowledge RAG and Skills.
Python agent-contextagent-memoryagent-pluginsagentic-ragcontext-databasedsh-pluginknowledge-basememorymemory-managementself-evolvingvector-database
fix(queuefs): bound semantic node scheduling (#2343)
Qin Haojie294810dfb3a3
diff --git a/openviking/service/core.py b/openviking/service/core.pyindex 7edd3ecd7..f2428cb05 100644--- a/openviking/service/core.py+++ b/openviking/service/core.py@@ -115,7 +115,7 @@ class OpenVikingService:self,config: StorageConfig,max_concurrent_embedding: int = 10,- max_concurrent_semantic: int = 100,+ max_concurrent_semantic: int = 64,) -> None:"""Initialize storage resources."""from openviking.utils.agfs_utils import create_agfs_clientdiff --git a/openviking/storage/queuefs/queue_manager.py b/openviking/storage/queuefs/queue_manager.pyindex 222ec7c2f..962e62291 100644--- a/openviking/storage/queuefs/queue_manager.py+++ b/openviking/storage/queuefs/queue_manager.py@@ -29,7 +29,7 @@ def init_queue_manager(timeout: int = 10,mount_point: str = "/queue",max_concurrent_embedding: int = 10,- max_concurrent_semantic: int = 100,+ max_concurrent_semantic: int = 64,) -> "QueueManager":"""Initialize QueueManager singleton.@@ -38,7 +38,7 @@ def init_queue_manager(timeout: Request timeout in seconds.mount_point: Path where QueueFS is mounted.max_concurrent_embedding: Max concurrent embedding tasks.- max_concurrent_semantic: Max concurrent semantic tasks.+ max_concurrent_semantic: Max concurrent semantic node work."""global _instance_instance = QueueManager(@@ -74,7 +74,7 @@ class QueueManager:timeout: int = 10,mount_point: str = "/queue",max_concurrent_embedding: int = 10,- max_concurrent_semantic: int = 100,+ max_concurrent_semantic: int = 64,):"""Initialize QueueManager."""self._agfs = agfsdiff --git a/openviking/storage/queuefs/semantic_dag.py b/openviking/storage/queuefs/semantic_dag.pyindex cd3143ffa..a5b9921a3 100644--- a/openviking/storage/queuefs/semantic_dag.py+++ b/openviking/storage/queuefs/semantic_dag.py@@ -3,8 +3,10 @@"""Semantic DAG executor with event-driven lazy dispatch."""import asyncio+import threading+from weakref import WeakKeyDictionaryfrom dataclasses import dataclass, field-from typing import Dict, List, Optional+from typing import ClassVar, Dict, List, Optional, Setfrom openviking.server.identity import RequestContextfrom openviking.storage.queuefs.semantic_sidecar import write_semantic_sidecars@@ -66,9 +68,94 @@ class VectorizeTask:overview: Optional[str] = None+@dataclass(frozen=True)+class DagWork:+ """A scheduled unit of DAG work."""++ kind: str+ dir_uri: str+ parent_uri: Optional[str] = None+ file_path: Optional[str] = None+ vectorize_task: Optional[VectorizeTask] = None+++@dataclass(frozen=True)+class ScheduledDagWork:+ executor: "SemanticDagExecutor"+ work: DagWork+++class SemanticNodeScheduler:+ """Shared node executor for semantic DAG work in one event loop."""++ _idle_timeout = 0.05++ def __init__(self, max_workers: int):+ self._max_workers = max(1, max_workers)+ self._queue: asyncio.Queue[ScheduledDagWork] = asyncio.Queue()+ self._workers: Set[asyncio.Task] = set()++ def configure(self, max_workers: int) -> None:+ self._max_workers = max(1, max_workers)+ self._ensure_workers()++ def submit(self, executor: "SemanticDagExecutor", work: DagWork) -> None:+ self._queue.put_nowait(ScheduledDagWork(executor=executor, work=work))+ self._ensure_workers()++ def _ensure_workers(self) -> None:+ self._workers = {task for task in self._workers if not task.done()}+ target = min(self._max_workers, self._queue.qsize())+ while len(self._workers) < target:+ task = asyncio.create_task(self._worker())+ task.add_done_callback(self._workers.discard)+ self._workers.add(task)++ async def _worker(self) -> None:+ while True:+ try:+ item = await asyncio.wait_for(+ self._queue.get(),+ timeout=self._idle_timeout,+ )+ except asyncio.TimeoutError:+ if self._queue.empty():+ return+ continue++ try:+ if not item.executor.closed:+ await item.executor._run_work(item.work)+ except asyncio.CancelledError:+ raise+ except Exception as exc:+ item.executor.fail(exc)+ finally:+ self._queue.task_done()+++_node_schedulers: "WeakKeyDictionary[asyncio.AbstractEventLoop, SemanticNodeScheduler]" = (+ WeakKeyDictionary()+)+++def get_semantic_node_scheduler(max_workers: int) -> SemanticNodeScheduler:+ loop = asyncio.get_running_loop()+ scheduler = _node_schedulers.get(loop)+ if scheduler is None:+ scheduler = SemanticNodeScheduler(max_workers=max_workers)+ _node_schedulers[loop] = scheduler+ else:+ scheduler.configure(max_workers)+ return scheduler++class SemanticDagExecutor:"""Execute semantic generation with DAG-style, event-driven lazy dispatch."""+ _active_lock: ClassVar[threading.Lock] = threading.Lock()+ _active_executors: ClassVar[Set["SemanticDagExecutor"]] = set()+def __init__(self,processor: "SemanticProcessor",@@ -89,7 +176,6 @@ class SemanticDagExecutor:):self._processor = processorself._context_type = context_type- self._max_concurrent_llm = max_concurrent_llmself._ctx = ctxself._incremental_update = incremental_updateself._target_uri = target_uri@@ -106,15 +192,21 @@ class SemanticDagExecutor:self._changed_paths = {path for key in ("added", "modified", "deleted") for path in self._changes.get(key, [])}+ self._node_concurrency = max(1, max_concurrent_llm)self._llm_sem = asyncio.Semaphore(max_concurrent_llm)self._viking_fs = get_viking_fs()self._nodes: Dict[str, DirNode] = {}self._parent: Dict[str, Optional[str]] = {}self._root_uri: Optional[str] = Noneself._root_done: Optional[asyncio.Event] = None+ self._scheduler: Optional[SemanticNodeScheduler] = None+ self._closed = False+ self._failure: Optional[Exception] = Noneself._stats = DagStats()self._vectorize_task_count: int = 0self._pending_vectorize_tasks: List[VectorizeTask] = []+ self._pending_vectorize_work = 0+ self._vectorize_done: Optional[asyncio.Event] = Noneself._vectorize_lock = asyncio.Lock()self._file_change_status: Dict[str, bool] = {}self._dir_change_status: Dict[str, bool] = {}@@ -125,74 +217,211 @@ class SemanticDagExecutor:"""Run DAG execution starting from root_uri."""self._root_uri = root_uriself._root_done = asyncio.Event()+ self._scheduler = get_semantic_node_scheduler(self._node_concurrency)try:- await self._dispatch_dir(root_uri, parent_uri=None)+ self._register_active()+ self._schedule_dir(root_uri, parent_uri=None)await self._root_done.wait()- except Exception:- await self._lock.close()+ if self._failure:+ raise self._failure++ # Release owned semantic locks after downstream vectorization finishes.+ async def wrapped_on_complete() -> None:+ try:+ if self._telemetry_id and self._semantic_msg_id:+ get_request_wait_tracker().mark_semantic_done(+ self._telemetry_id, self._semantic_msg_id+ )+ finally:+ await self._lock.close()++ async with self._vectorize_lock:+ task_count = self._vectorize_task_count+ tasks = list(self._pending_vectorize_tasks)++ if task_count > 0:+ from .embedding_tracker import EmbeddingTaskTracker++ tracker = EmbeddingTaskTracker.get_instance()+ await tracker.register(+ semantic_msg_id=self._semantic_msg_id,+ total_count=task_count,+ on_complete=wrapped_on_complete,+ metadata={"uri": root_uri},+ )++ await self._dispatch_vectorize_tasks(tasks)+ else:+ # No vectorize tasks — release lock immediately (via wrapped callback)+ try:+ await wrapped_on_complete()+ except Exception as e:+ logger.error(f"Error in on_complete callback: {e}", exc_info=True)+ except BaseException:+ self._closed = True+ try:+ await self._lock.close()+ except Exception:+ passraise+ finally:+ self._closed = True+ self._unregister_active()++ def _schedule_work(self, work: DagWork) -> None:+ if self._closed:+ return+ if self._scheduler is None:+ self._scheduler = get_semantic_node_scheduler(self._node_concurrency)+ self._scheduler.submit(self, work)++ def _schedule_dir(self, dir_uri: str, parent_uri: Optional[str]) -> None:+ if self._closed:+ return+ self._stats.total_nodes += 1+ self._stats.pending_nodes += 1+ self._schedule_work(DagWork(kind="dir", dir_uri=dir_uri, parent_uri=parent_uri))- # Release owned semantic locks after downstream vectorization finishes.- async def wrapped_on_complete() -> None:+ def _schedule_file(self, parent_uri: str, file_path: str) -> None:+ if self._closed:+ return+ self._stats.total_nodes += 1+ self._stats.pending_nodes += 1+ self._schedule_work(DagWork(kind="file", dir_uri=parent_uri, file_path=file_path))++ def _mark_node_started(self) -> None:+ self._stats.pending_nodes = max(0, self._stats.pending_nodes - 1)+ self._stats.in_progress_nodes += 1++ def _mark_node_waiting(self) -> None:+ self._stats.in_progress_nodes = max(0, self._stats.in_progress_nodes - 1)+ self._stats.pending_nodes += 1++ def _mark_node_done(self) -> None:+ self._stats.done_nodes += 1+ self._stats.in_progress_nodes = max(0, self._stats.in_progress_nodes - 1)++ def _release_dir_node(self, dir_uri: str) -> None:+ self._nodes.pop(dir_uri, None)+ self._parent.pop(dir_uri, None)++ @property+ def closed(self) -> bool:+ return self._closed++ def fail(self, exc: Exception) -> None:+ if self._failure is None:+ self._failure = exc+ self._closed = True+ if self._root_done:+ self._root_done.set()+ if self._vectorize_done:+ self._vectorize_done.set()++ def _register_active(self) -> None:+ with self._active_lock:+ self._active_executors.add(self)++ def _unregister_active(self) -> None:+ with self._active_lock:+ self._active_executors.discard(self)++ @classmethod+ def get_active_stats(cls) -> DagStats:+ stats = DagStats()+ with cls._active_lock:+ executors = list(cls._active_executors)+ for executor in executors:+ current = executor.get_stats()+ stats.total_nodes += current.total_nodes+ stats.pending_nodes += current.pending_nodes+ stats.in_progress_nodes += current.in_progress_nodes+ stats.done_nodes += current.done_nodes+ return stats++ async def _run_work(self, work: DagWork) -> None:+ if work.kind == "vectorize":+ await self._run_vectorize_work(work.vectorize_task)+ return++ self._mark_node_started()++ if work.kind == "dir":+ terminal = Falsetry:- if self._telemetry_id and self._semantic_msg_id:- get_request_wait_tracker().mark_semantic_done(- self._telemetry_id, self._semantic_msg_id- )+ terminal = await self._dispatch_dir(work.dir_uri, work.parent_uri)finally:- await self._lock.close()+ if terminal:+ self._mark_node_done()+ else:+ self._mark_node_waiting()+ return- async with self._vectorize_lock:- task_count = self._vectorize_task_count- tasks = list(self._pending_vectorize_tasks)-- if task_count > 0:- from .embedding_tracker import EmbeddingTaskTracker-- tracker = EmbeddingTaskTracker.get_instance()- await tracker.register(- semantic_msg_id=self._semantic_msg_id,- total_count=task_count,- on_complete=wrapped_on_complete,- metadata={"uri": root_uri},+ if work.kind == "file":+ if work.file_path is None:+ self._mark_node_done()+ return+ await self._file_summary_task(work.dir_uri, work.file_path)+ return++ if work.kind == "overview":+ await self._overview_task(work.dir_uri)+ return++ self._mark_node_done()+ logger.warning("Unknown semantic DAG work kind: %s", work.kind)++ async def _dispatch_vectorize_tasks(self, tasks: List[VectorizeTask]) -> None:+ self._vectorize_done = asyncio.Event()+ self._pending_vectorize_work = len(tasks)+ for task in tasks:+ self._schedule_work(+ DagWork(+ kind="vectorize",+ dir_uri=task.uri,+ vectorize_task=task,+ ))+ await self._vectorize_done.wait()- for task in tasks:- if task.task_type == "file":- asyncio.create_task(- self._processor._vectorize_single_file(- parent_uri=task.parent_uri,- context_type=task.context_type,- file_path=task.file_path,- summary_dict=task.summary_dict,- ctx=task.ctx,- semantic_msg_id=task.semantic_msg_id,- use_summary=task.use_summary,- )- )- else:- asyncio.create_task(- self._processor._vectorize_directory(- task.uri,- task.context_type,- task.abstract,- task.overview,- ctx=task.ctx,- semantic_msg_id=task.semantic_msg_id,- )- )- else:- # No vectorize tasks — release lock immediately (via wrapped callback)- try:- await wrapped_on_complete()- except Exception as e:- logger.error(f"Error in on_complete callback: {e}", exc_info=True)+ async def _run_vectorize_work(self, task: Optional[VectorizeTask]) -> None:+ try:+ if task is not None:+ await self._run_vectorize_task(task)+ except Exception as exc:+ logger.error("Vectorization dispatch task failed: %s", exc, exc_info=True)+ finally:+ self._pending_vectorize_work = max(0, self._pending_vectorize_work - 1)+ if self._pending_vectorize_work == 0 and self._vectorize_done:+ self._vectorize_done.set()++ async def _run_vectorize_task(self, task: VectorizeTask) -> None:+ if task.task_type == "file":+ await self._processor._vectorize_single_file(+ parent_uri=task.parent_uri,+ context_type=task.context_type,+ file_path=task.file_path,+ summary_dict=task.summary_dict,+ ctx=task.ctx,+ semantic_msg_id=task.semantic_msg_id,+ use_summary=task.use_summary,+ )+ return- async def _dispatch_dir(self, dir_uri: str, parent_uri: Optional[str]) -> None:+ await self._processor._vectorize_directory(+ task.uri,+ task.context_type,+ task.abstract,+ task.overview,+ ctx=task.ctx,+ semantic_msg_id=task.semantic_msg_id,+ )++ async def _dispatch_dir(self, dir_uri: str, parent_uri: Optional[str]) -> bool:"""Lazy-dispatch tasks for a directory when it is triggered."""if dir_uri in self._nodes:- return+ return Trueself._parent[dir_uri] = parent_uri@@ -217,31 +446,26 @@ class SemanticDagExecutor:dispatched=True,)self._nodes[dir_uri] = node- self._stats.total_nodes += 1- self._stats.pending_nodes += 1if pending == 0:self._schedule_overview(dir_uri)- return+ return Falsefor file_path in file_paths:- self._stats.total_nodes += 1- # File nodes are scheduled immediately: pending -> in_progress.- self._stats.pending_nodes += 1- self._stats.pending_nodes = max(0, self._stats.pending_nodes - 1)- self._stats.in_progress_nodes += 1- asyncio.create_task(self._file_summary_task(dir_uri, file_path))+ self._schedule_file(dir_uri, file_path)if children_dirs:if self._recursive:for child_uri in children_dirs:- asyncio.create_task(self._dispatch_dir(child_uri, dir_uri))+ self._schedule_dir(child_uri, dir_uri)+ return Falseexcept Exception as e:logger.error(f"Failed to dispatch directory {dir_uri}: {e}", exc_info=True)if parent_uri:await self._on_child_done(parent_uri, dir_uri, "")elif self._root_done:self._root_done.set()+ return Trueasync def _list_dir(self, uri: str, from_hint: str) -> tuple[list[str], list[str]]:"""List directory entries and return (child_dirs, file_paths)."""@@ -272,7 +496,8 @@ class SemanticDagExecutor:def _get_target_file_path(self, current_uri: str) -> Optional[str]:if not self._incremental_update or not self._target_uri or not self._root_uri:logger.warning(- f"Invalid target_uri or root_uri for incremental update: target_uri={self._target_uri}, root_uri={self._root_uri}"+ "Invalid target_uri or root_uri for incremental update: "+ f"target_uri={self._target_uri}, root_uri={self._root_uri}")return Noneif self._target_uri != self._root_uri:@@ -485,10 +710,7 @@ class SemanticDagExecutor:node.file_summaries[idx] = summary_dictnode.pending -= 1if node.pending == 0 and not node.overview_scheduled:- node.overview_scheduled = True- self._stats.pending_nodes = max(0, self._stats.pending_nodes - 1)- self._stats.in_progress_nodes += 1- asyncio.create_task(self._overview_task(parent_uri))+ self._schedule_overview(parent_uri)async def _on_child_done(self, parent_uri: str, child_uri: str, abstract: str) -> None:node = self._nodes.get(parent_uri)@@ -502,21 +724,14 @@ class SemanticDagExecutor:node.children_abstracts[idx] = {"name": child_name, "abstract": abstract}node.pending -= 1if node.pending == 0 and not node.overview_scheduled:- node.overview_scheduled = True- self._stats.pending_nodes = max(0, self._stats.pending_nodes - 1)- self._stats.in_progress_nodes += 1- asyncio.create_task(self._overview_task(parent_uri))+ self._schedule_overview(parent_uri)def _schedule_overview(self, dir_uri: str) -> None:node = self._nodes.get(dir_uri)- if not node:- return- if node.overview_scheduled:+ if not node or node.overview_scheduled:returnnode.overview_scheduled = True- self._stats.pending_nodes = max(0, self._stats.pending_nodes - 1)- self._stats.in_progress_nodes += 1- asyncio.create_task(self._overview_task(dir_uri))+ self._schedule_work(DagWork(kind="overview", dir_uri=dir_uri))def _finalize_file_summaries(self, node: DirNode) -> List[Dict[str, str]]:summaries: List[Dict[str, str]] = []@@ -633,11 +848,13 @@ class SemanticDagExecutor:parent_uri = self._parent.get(dir_uri)if parent_uri is None:+ self._release_dir_node(dir_uri)if self._root_done:self._root_done.set()returnawait self._on_child_done(parent_uri, dir_uri, abstract)+ self._release_dir_node(dir_uri)async def _add_vectorize_task(self, task: VectorizeTask) -> None:"""Add a vectorize task to the pending list."""diff --git a/openviking/storage/queuefs/semantic_processor.py b/openviking/storage/queuefs/semantic_processor.pyindex 6b5d0ee29..c4d0d6fe4 100644--- a/openviking/storage/queuefs/semantic_processor.py+++ b/openviking/storage/queuefs/semantic_processor.py@@ -96,7 +96,7 @@ class SemanticProcessor(DequeueHandlerBase):_request_stats_order: List[str] = []_max_cached_stats = 256- def __init__(self, max_concurrent_llm: int = 100):+ def __init__(self, max_concurrent_llm: int = 64):"""Initialize SemanticProcessor.@@ -104,9 +104,7 @@ class SemanticProcessor(DequeueHandlerBase):max_concurrent_llm: Maximum concurrent LLM calls"""self.max_concurrent_llm = max_concurrent_llm- self._dag_executor: Optional[SemanticDagExecutor] = None- self._current_ctx = RequestContext(user=UserIdentifier.the_default_user(), role=Role.ROOT)- self._current_msg: Optional[SemanticMsg] = None+ self._default_ctx = RequestContext(user=UserIdentifier.the_default_user(), role=Role.ROOT)self._circuit_breaker = CircuitBreaker()@classmethod@@ -356,8 +354,7 @@ class SemanticProcessor(DequeueHandlerBase):root_attrs.agent_id = msg.agent_idroot_context_token = bind_root_observability_context(root_attrs)try:- self._current_msg = msg- self._current_ctx = self._ctx_from_semantic_msg(msg)+ current_ctx = self._ctx_from_semantic_msg(msg)logger.info(f"Processing semantic generation for: {msg.uri} (recursive={msg.recursive})")@@ -374,6 +371,7 @@ class SemanticProcessor(DequeueHandlerBase):lock_transferred = Trueawait self._process_memory_directory(msg,+ ctx=current_ctx,lock=semantic_lock.lock,)else:@@ -384,7 +382,7 @@ class SemanticProcessor(DequeueHandlerBase):viking_fs = get_viking_fs()if msg.target_uri:target_exists = await viking_fs.exists(- msg.target_uri, ctx=self._current_ctx+ msg.target_uri, ctx=current_ctx)if msg.uri != msg.target_uri:logger.info(@@ -394,7 +392,7 @@ class SemanticProcessor(DequeueHandlerBase):diff = await self._sync_topdown_recursive(msg.uri,msg.target_uri,- ctx=self._current_ctx,+ ctx=current_ctx,lock=semantic_lock.lock,)logger.info(@@ -425,7 +423,7 @@ class SemanticProcessor(DequeueHandlerBase):processor=self,context_type=msg.context_type,max_concurrent_llm=self.max_concurrent_llm,- ctx=self._current_ctx,+ ctx=current_ctx,incremental_update=is_incremental,target_uri=target_uri,semantic_msg_id=msg.id,@@ -438,7 +436,6 @@ class SemanticProcessor(DequeueHandlerBase):coalesce_key=msg.coalesce_key,coalesce_version=msg.coalesce_version,)- self._dag_executor = executorlock_transferred = Trueawait executor.run(run_uri)self._cache_dag_stats(@@ -509,16 +506,15 @@ class SemanticProcessor(DequeueHandlerBase):else:self.report_error(str(e), data)return None- finally:- self._current_msg = None- self._current_ctx = None-def get_dag_stats(self) -> Optional["DagStats"]:- if not self._dag_executor:- return None- return self._dag_executor.get_stats()+ return SemanticDagExecutor.get_active_stats()- async def _process_memory_directory(self, msg: SemanticMsg, lock: LockLease = NO_LOCK) -> None:+ async def _process_memory_directory(+ self,+ msg: SemanticMsg,+ ctx: Optional[RequestContext] = None,+ lock: LockLease = NO_LOCK,+ ) -> None:"""Process a memory directory with special handling.For memory directories:@@ -531,7 +527,7 @@ class SemanticProcessor(DequeueHandlerBase):"""viking_fs = get_viking_fs()dir_uri = msg.uri- ctx = self._current_ctx+ ctx = ctx or self._default_ctxllm_sem = asyncio.Semaphore(self.max_concurrent_llm)request_wait_tracker = get_request_wait_tracker()@@ -959,7 +955,7 @@ class SemanticProcessor(DequeueHandlerBase):"""Generate summary for a single text file (code, documentation, or other text)."""viking_fs = get_viking_fs()vlm = get_openviking_config().vlm- active_ctx = ctx or self._current_ctx+ active_ctx = ctx or self._default_ctxcontent = await viking_fs.read_file(file_path, ctx=active_ctx)if isinstance(content, bytes):@@ -1437,13 +1433,9 @@ class SemanticProcessor(DequeueHandlerBase):) -> None:"""Create directory Context and enqueue to EmbeddingQueue."""- if self._current_msg and getattr(self._current_msg, "skip_vectorization", False):- logger.info(f"Skipping vectorization for {uri} (requested via SemanticMsg)")- return-from openviking.utils.embedding_utils import vectorize_directory_meta- active_ctx = ctx or self._current_ctx+ active_ctx = ctx or self._default_ctxawait vectorize_directory_meta(uri=uri,abstract=abstract,@@ -1467,7 +1459,7 @@ class SemanticProcessor(DequeueHandlerBase):"""Vectorize a single file using its content or summary."""from openviking.utils.embedding_utils import vectorize_file- active_ctx = ctx or self._current_ctx+ active_ctx = ctx or self._default_ctxawait vectorize_file(file_path=file_path,summary_dict=summary_dict,diff --git a/openviking_cli/utils/config/vlm_config.py b/openviking_cli/utils/config/vlm_config.pyindex a3ae470ef..884ced649 100644--- a/openviking_cli/utils/config/vlm_config.py+++ b/openviking_cli/utils/config/vlm_config.py@@ -65,7 +65,7 @@ class VLMConfig(BaseModel):thinking: bool = Field(default=False, description="Enable thinking mode for VolcEngine models")max_concurrent: int = Field(- default=100, description="Maximum number of concurrent LLM calls for semantic processing"+ default=64, description="Maximum number of concurrent LLM calls for semantic processing")api_version: Optional[str] = Field(diff --git a/tests/storage/test_semantic_dag_stats.py b/tests/storage/test_semantic_dag_stats.pyindex 751fa3dfc..ff6ad534a 100644--- a/tests/storage/test_semantic_dag_stats.py+++ b/tests/storage/test_semantic_dag_stats.py@@ -64,6 +64,22 @@ class _FakeProcessor:await self._vectorize_directory(uri, context_type, abstract, overview, ctx=ctx)+class _TrackingProcessor(_FakeProcessor):+ def __init__(self):+ super().__init__()+ self.active_summaries = 0+ self.max_active_summaries = 0++ async def _generate_single_file_summary(self, file_path, llm_sem=None, ctx=None):+ self.active_summaries += 1+ self.max_active_summaries = max(self.max_active_summaries, self.active_summaries)+ try:+ await asyncio.sleep(0.01)+ return {"name": file_path.split("/")[-1], "summary": "summary"}+ finally:+ self.active_summaries -= 1++class _DummyTracker:def __init__(self):self.register_calls = []@@ -131,6 +147,84 @@ async def test_semantic_dag_stats_collects_nodes(monkeypatch):)+async def test_semantic_dag_bounds_active_node_work(monkeypatch):+ root_uri = "viking://resources/root"+ tree = {+ root_uri: [{"name": f"file-{idx}.txt", "isDir": False} for idx in range(40)],+ }+ fake_fs = _FakeVikingFS(tree)+ monkeypatch.setattr("openviking.storage.queuefs.semantic_dag.get_viking_fs", lambda: fake_fs)+ monkeypatch.setattr(+ "openviking.storage.queuefs.embedding_tracker.EmbeddingTaskTracker.get_instance",+ lambda: _DummyTracker(),+ )++ processor = _TrackingProcessor()+ ctx = RequestContext(user=UserIdentifier("acc1", "user1", "agent1"), role=Role.USER)+ executor = SemanticDagExecutor(+ processor=processor,+ context_type="resource",+ max_concurrent_llm=3,+ ctx=ctx,+ skip_vectorization=True,+ )++ max_running = 0+ run_task = asyncio.create_task(executor.run(root_uri))+ while not run_task.done():+ max_running = max(max_running, executor.get_stats().in_progress_nodes)+ await asyncio.sleep(0)+ await run_task++ stats = executor.get_stats()+ assert stats.total_nodes == 41+ assert stats.done_nodes == 41+ assert stats.pending_nodes == 0+ assert stats.in_progress_nodes == 0+ assert processor.max_active_summaries <= 3+ assert max_running <= 3+++async def test_semantic_dag_shares_node_scheduler_across_roots(monkeypatch):+ root_a = "viking://resources/root-a"+ root_b = "viking://resources/root-b"+ tree = {+ root_a: [{"name": f"a-{idx}.txt", "isDir": False} for idx in range(20)],+ root_b: [{"name": f"b-{idx}.txt", "isDir": False} for idx in range(20)],+ }+ fake_fs = _FakeVikingFS(tree)+ monkeypatch.setattr("openviking.storage.queuefs.semantic_dag.get_viking_fs", lambda: fake_fs)+ monkeypatch.setattr(+ "openviking.storage.queuefs.embedding_tracker.EmbeddingTaskTracker.get_instance",+ lambda: _DummyTracker(),+ )++ processor = _TrackingProcessor()+ ctx = RequestContext(user=UserIdentifier("acc1", "user1", "agent1"), role=Role.USER)+ executor_a = SemanticDagExecutor(+ processor=processor,+ context_type="resource",+ max_concurrent_llm=4,+ ctx=ctx,+ skip_vectorization=True,+ )+ executor_b = SemanticDagExecutor(+ processor=processor,+ context_type="resource",+ max_concurrent_llm=4,+ ctx=ctx,+ skip_vectorization=True,+ )++ await asyncio.gather(executor_a.run(root_a), executor_b.run(root_b))++ assert processor.max_active_summaries <= 4+ assert executor_a.get_stats().done_nodes == 21+ assert executor_b.get_stats().done_nodes == 21++@pytest.mark.asyncioasync def test_semantic_dag_skip_vectorization_does_not_schedule_tasks(monkeypatch):root_uri = "viking://resources/root"