Self-evolving Context Database for AI Agents. Unify Agent Memory, Knowledge RAG and Skills.
Python agent-contextagent-memoryagent-pluginsagentic-ragcontext-databasedsh-pluginknowledge-basememorymemory-managementself-evolvingvector-database
perf(queue): 流式调度语义向量化任务 (#3636)
Qin Haojieb877ababa5ed
diff --git a/docs/en/configuration/01-server.md b/docs/en/configuration/01-server.mdindex ebe69d044..a137b97fa 100644--- a/docs/en/configuration/01-server.md+++ b/docs/en/configuration/01-server.md@@ -208,6 +208,18 @@ Remote backends also require endpoint, bucket/collection, credentials, and timeoThis setting controls queue-job concurrency. It is separate from `vlm.media.max_concurrent`, which limits audio/video VLM calls, and does not limit individual Understanding API HTTP requests.+### `queue_workers.add_resource`++| Field | Type | Default | Description |+|---|---|---:|---|+| `max_concurrent` | integer | `4` | Number of complete AddResource jobs consumed concurrently; must be greater than `0`; requires a server restart after changes |++### `queue_workers.session_commit`++| Field | Type | Default | Description |+|---|---|---:|---|+| `max_concurrent` | integer | `4` | Number of SessionCommit jobs consumed concurrently; must be greater than `0`; requires a server restart after changes |+## HTTP Server Settings```jsondiff --git a/docs/en/guides/01-configuration.md b/docs/en/guides/01-configuration.mdindex 3d5535e1e..53cea7820 100644--- a/docs/en/guides/01-configuration.md+++ b/docs/en/guides/01-configuration.md@@ -631,7 +631,7 @@ Vision Language Model for semantic extraction (L0/L1 generation).| `model` | str | Model name || `api_base` | str | API endpoint (optional) || `thinking` | bool | Enable thinking mode for VolcEngine models (default: `false`) |-| `max_concurrent` | int | Maximum concurrent semantic LLM calls (default: `64`) |+| `max_concurrent` | int | Maximum concurrent semantic LLM calls (default: `32`) || `max_retries` | int | Maximum retry attempts for transient VLM provider errors (default: `3`; `0` disables retry) || `credentials` | array | Ordered VLM credential/model list, with index 0 having the highest priority. Each item can override `provider`, `model`, `api_key`, `api_base`, `api_version`, `extra_headers`, `extra_request_body`, and `stream` || `failback_timeout_seconds` | float | Time threshold for attempting a step back toward a higher-priority credential after failover (default: `600`) |@@ -1778,7 +1778,7 @@ For detailed encryption explanations, see [Data Encryption](../concepts/10-encry"model": "string","api_base": "string","thinking": false,- "max_concurrent": 64,+ "max_concurrent": 32,"max_retries": 3,"extra_headers": {},"extra_request_body": {},diff --git a/docs/zh/configuration/01-server.md b/docs/zh/configuration/01-server.mdindex 3b9ebf163..c4a3192e4 100644--- a/docs/zh/configuration/01-server.md+++ b/docs/zh/configuration/01-server.md@@ -208,6 +208,18 @@ Search 和 Find 请求的默认 `limit` 为 `10`,可以在每次 API 或 SDK该配置控制队列作业并发,不等同于 `vlm.media.max_concurrent` 的音视频 VLM 调用并发,也不限制 Understanding API 的单独 HTTP 请求数。+### `queue_workers.add_resource`++| 字段 | 类型 | 默认值 | 说明 |+|---|---|---:|---|+| `max_concurrent` | integer | `4` | 同时消费的完整 AddResource 作业数,必须大于 `0`;修改后需重启服务 |++### `queue_workers.session_commit`++| 字段 | 类型 | 默认值 | 说明 |+|---|---|---:|---|+| `max_concurrent` | integer | `4` | 同时消费的 SessionCommit 作业数,必须大于 `0`;修改后需重启服务 |+## HTTP 服务配置```jsondiff --git a/docs/zh/guides/01-configuration.md b/docs/zh/guides/01-configuration.mdindex 00a155368..ea2ddc609 100644--- a/docs/zh/guides/01-configuration.md+++ b/docs/zh/guides/01-configuration.md@@ -600,7 +600,7 @@ provider,并设置 `storage.vectordb.sparse_weight > 0`。自托管模型的| `model` | str | 模型名称 || `api_base` | str | API 端点(可选) || `thinking` | bool | 启用思考模式(仅对部分火山模型生效,默认:`false`) |-| `max_concurrent` | int | 语义处理阶段 LLM 最大并发调用数(默认:`64`) |+| `max_concurrent` | int | 语义处理阶段 LLM 最大并发调用数(默认:`32`) || `max_retries` | int | VLM provider 瞬时错误的最大重试次数(默认:`3`;`0` 表示禁用重试) || `credentials` | array | 有序 VLM 凭据/模型列表,索引 0 优先级最高。每项可单独覆盖 `provider`、`model`、`api_key`、`api_base`、`api_version`、`extra_headers`、`extra_request_body` 和 `stream` || `failback_timeout_seconds` | float | 切换到低优先级 credential 后,尝试逐级切回的时间阈值(默认:`600`) |@@ -1859,7 +1859,7 @@ Task 记录文件位于所属账号的系统目录:"model": "string","api_base": "string","thinking": false,- "max_concurrent": 64,+ "max_concurrent": 32,"max_retries": 3,"extra_headers": {},"extra_request_body": {},diff --git a/examples/ov.conf.example b/examples/ov.conf.exampleindex 3b654718e..9056a7286 100644--- a/examples/ov.conf.example+++ b/examples/ov.conf.example@@ -82,7 +82,11 @@},},},- "queue_workers": {"external_parse": {"max_concurrent": 4}},+ "queue_workers": {+ "external_parse": {"max_concurrent": 4},+ "add_resource": {"max_concurrent": 4},+ "session_commit": {"max_concurrent": 4},+ },"embedding": {"dense": {"model": "doubao-embedding-vision-251215",diff --git a/openviking/service/core.py b/openviking/service/core.pyindex 86b9296aa..3650b90fd 100644--- a/openviking/service/core.py+++ b/openviking/service/core.py@@ -132,6 +132,8 @@ class OpenVikingService:max_concurrent_embedding=config.embedding.max_concurrent,max_concurrent_semantic=config.vlm.max_concurrent,max_concurrent_external_parse=config.queue_workers.external_parse.max_concurrent,+ max_concurrent_add_resource=config.queue_workers.add_resource.max_concurrent,+ max_concurrent_session_commit=config.queue_workers.session_commit.max_concurrent,binding_config=binding_config,git_config=config.git,)@@ -146,8 +148,10 @@ class OpenVikingService:self,config: StorageConfig,max_concurrent_embedding: int = 10,- max_concurrent_semantic: int = 64,+ max_concurrent_semantic: int = 32,max_concurrent_external_parse: int = 4,+ max_concurrent_add_resource: int = 4,+ max_concurrent_session_commit: int = 4,binding_config: Any = None,*,git_config: Optional[GitConfig] = None,@@ -169,6 +173,8 @@ class OpenVikingService:max_concurrent_embedding=max_concurrent_embedding,max_concurrent_semantic=max_concurrent_semantic,max_concurrent_external_parse=max_concurrent_external_parse,+ max_concurrent_add_resource=max_concurrent_add_resource,+ max_concurrent_session_commit=max_concurrent_session_commit,)else:logger.warning("RAGFS client not initialized, skipping queue manager")@@ -310,6 +316,12 @@ class OpenVikingService:max_concurrent_external_parse=(self._config.queue_workers.external_parse.max_concurrent),+ max_concurrent_add_resource=(+ self._config.queue_workers.add_resource.max_concurrent+ ),+ max_concurrent_session_commit=(+ self._config.queue_workers.session_commit.max_concurrent+ ),binding_config=self._build_ragfs_binding_config(),git_config=self._config.git,)diff --git a/openviking/storage/collection_schemas.py b/openviking/storage/collection_schemas.pyindex f9145bca7..77b48caa6 100644--- a/openviking/storage/collection_schemas.py+++ b/openviking/storage/collection_schemas.py@@ -870,21 +870,13 @@ class TextEmbeddingHandler(DequeueHandlerBase):finally:if embedding_msg is not None and request_failed_message is not None:self._record_request_failure(embedding_msg, request_failed_message)- if embedding_msg and embedding_msg.semantic_msg_id:- from openviking.storage.queuefs.embedding_tracker import EmbeddingTaskTracker-- tracker = EmbeddingTaskTracker.get_instance()- try:- await tracker.decrement(embedding_msg.semantic_msg_id)- except Exception as tracker_err:- logger.warning(f"Failed to decrement embedding tracker: {tracker_err}")if report_error_args is not None:self.report_error(*report_error_args)elif report_success:self.report_success()async def on_cancelled(self, data: Optional[Dict[str, Any]]) -> Optional[Dict[str, Any]]:- """Settle semantic joins when a queued embedding is cancelled."""+ """Settle request-scoped waiting when a queued embedding is cancelled."""embedding_msg: Optional[EmbeddingMsg] = Nonetry:if data:@@ -898,25 +890,20 @@ class TextEmbeddingHandler(DequeueHandlerBase):if embedding_msg is not None:self._record_request_success(embedding_msg)- if embedding_msg.semantic_msg_id:- from openviking.storage.queuefs.embedding_tracker import EmbeddingTaskTracker-- await EmbeddingTaskTracker.get_instance().decrement(embedding_msg.semantic_msg_id)self.report_success()return None@staticmethoddef _record_request_success(embedding_msg: EmbeddingMsg) -> None:- tracker = get_request_wait_tracker()- if embedding_msg.semantic_msg_id:- tracker.record_embedding_processed(embedding_msg.telemetry_id)- else:- tracker.mark_embedding_done(embedding_msg.telemetry_id, embedding_msg.id)+ get_request_wait_tracker().mark_embedding_done(+ embedding_msg.telemetry_id,+ embedding_msg.id,+ )@staticmethoddef _record_request_failure(embedding_msg: EmbeddingMsg, message: str) -> None:- tracker = get_request_wait_tracker()- if embedding_msg.semantic_msg_id:- tracker.record_embedding_error(embedding_msg.telemetry_id, message)- else:- tracker.mark_embedding_failed(embedding_msg.telemetry_id, embedding_msg.id, message)+ get_request_wait_tracker().mark_embedding_failed(+ embedding_msg.telemetry_id,+ embedding_msg.id,+ message,+ )diff --git a/openviking/storage/content_write.py b/openviking/storage/content_write.pyindex c177d0ce2..158d4b0c9 100644--- a/openviking/storage/content_write.py+++ b/openviking/storage/content_write.py@@ -829,7 +829,6 @@ class ContentWriteCoordinator:parent_uri=parent.uri,context_type=context_type,ctx=ctx,- register_request_wait=True,)def _validate_mode(self, mode: str) -> None:diff --git a/openviking/storage/queuefs/__init__.py b/openviking/storage/queuefs/__init__.pyindex 7302db2df..10a4d0ffa 100644--- a/openviking/storage/queuefs/__init__.py+++ b/openviking/storage/queuefs/__init__.py@@ -2,7 +2,6 @@# SPDX-License-Identifier: AGPL-3.0from .embedding_msg import EmbeddingMsgfrom .embedding_queue import EmbeddingQueue-from .embedding_tracker import EmbeddingTaskTrackerfrom .named_queue import NamedQueue, QueueError, QueueStatusfrom .queue_manager import QueueManager, get_queue_manager, init_queue_managerfrom .semantic_dag import SemanticDagExecutor@@ -20,7 +19,6 @@ __all__ = ["QueueError","EmbeddingQueue","EmbeddingMsg",- "EmbeddingTaskTracker","SemanticQueue","SemanticDagExecutor","SemanticMsg",diff --git a/openviking/storage/queuefs/embedding_msg.py b/openviking/storage/queuefs/embedding_msg.pyindex e8385c00d..4e529318e 100644--- a/openviking/storage/queuefs/embedding_msg.py+++ b/openviking/storage/queuefs/embedding_msg.py@@ -2,7 +2,7 @@# SPDX-License-Identifier: AGPL-3.0import jsonfrom dataclasses import asdict, dataclass, field-from typing import Any, Dict, List, Optional, Union+from typing import Any, Dict, List, Unionfrom uuid import uuid4@@ -12,20 +12,17 @@ class EmbeddingMsg:context_data: Dict[str, Any]id: str = field(default_factory=lambda: str(uuid4()))telemetry_id: str = ""- semantic_msg_id: Optional[str] = Nonedef __init__(self,message: Union[str, List[Dict[str, Any]]],context_data: Dict[str, Any],telemetry_id: str = "",- semantic_msg_id: Optional[str] = None,):self.id = str(uuid4())self.message = messageself.context_data = context_dataself.telemetry_id = telemetry_id- self.semantic_msg_id = semantic_msg_iddef to_dict(self) -> Dict[str, Any]:"""Convert embedding message to dictionary format."""@@ -42,7 +39,6 @@ class EmbeddingMsg:message=data["message"],context_data=data["context_data"],telemetry_id=data.get("telemetry_id", ""),- semantic_msg_id=data.get("semantic_msg_id"),)obj.id = data.get("id", obj.id)return objdiff --git a/openviking/storage/queuefs/embedding_tracker.py b/openviking/storage/queuefs/embedding_tracker.pydeleted file mode 100644index a993184c2..000000000--- a/openviking/storage/queuefs/embedding_tracker.py+++ /dev/null@@ -1,226 +0,0 @@-# Copyright (c) 2026 Beijing Volcano Engine Technology Co., Ltd.-# SPDX-License-Identifier: AGPL-3.0-"""Embedding Task Tracker for tracking embedding task completion status."""--import asyncio-import inspect-import threading-from dataclasses import dataclass-from typing import Any, Callable, Dict, Optional--from openviking_cli.utils.logger import get_logger--logger = get_logger(__name__)---@dataclass-class _EmbeddingTaskRecord:- """Coordinator state for a single semantic message."""-- remaining: int- total: int- on_complete: Optional[Callable[[], Any]]- metadata: Dict[str, Any]- owner_loop: Optional[asyncio.AbstractEventLoop]---class EmbeddingTaskTracker:- """Track embedding task completion status for each SemanticMsg.-- This tracker maintains a process-global registry of embedding tasks associated- with each SemanticMsg. Because semantic and embedding queues run on separate- worker threads with distinct event loops, its internal state must be guarded- by thread-safe primitives rather than loop-bound asyncio locks.-- When all embedding tasks for a SemanticMsg are completed, it triggers the- registered callback and removes the entry.- """-- _instance: Optional["EmbeddingTaskTracker"] = None- _initialized: bool = False-- def __new__(cls) -> "EmbeddingTaskTracker":- if cls._instance is None:- cls._instance = super().__new__(cls)- return cls._instance-- def __init__(self):- if self._initialized:- return- self._lock = threading.Lock()- self._tasks: Dict[str, _EmbeddingTaskRecord] = {}- self._initialized = True-- @staticmethod- async def _await_callback_result(result: Any) -> None:- """Await callback results when they are async."""- if inspect.isawaitable(result):- await result-- async def _execute_callback(self, on_complete: Callable[[], Any]) -> None:- """Invoke a completion callback and await async results."""- await self._await_callback_result(on_complete())-- async def _run_on_complete(- self,- semantic_msg_id: str,- record: _EmbeddingTaskRecord,- ) -> None:- """Execute the completion callback on the loop that registered it."""- on_complete = record.on_complete- owner_loop = record.owner_loop- if on_complete is None:- return-- try:- current_loop = asyncio.get_running_loop()- except RuntimeError:- current_loop = None-- owner_loop_running = bool(owner_loop and owner_loop.is_running())- owner_loop_available = bool(- owner_loop and not owner_loop.is_closed() and owner_loop_running- )-- try:- if owner_loop and owner_loop is not current_loop:- if not owner_loop_available:- logger.warning(- "Owner loop unavailable before completion callback for %s; "- "running callback in current loop",- semantic_msg_id,- )- else:- try:- fut = asyncio.run_coroutine_threadsafe(- self._execute_callback(on_complete),- owner_loop,- )- except RuntimeError:- logger.warning(- "Owner loop stopped before completion callback for %s; "- "running callback in current loop",- semantic_msg_id,- )- else:- await asyncio.wrap_future(fut)- return-- await self._execute_callback(on_complete)- except Exception as e:- logger.error(- f"Error in completion callback for {semantic_msg_id}: {e}",- exc_info=True,- )-- @classmethod- def get_instance(cls) -> "EmbeddingTaskTracker":- """Get the singleton instance of EmbeddingTaskTracker."""- if cls._instance is None:- cls._instance = cls()- return cls._instance-- async def register(- self,- semantic_msg_id: str,- total_count: int,- on_complete: Optional[Callable[[], Any]] = None,- metadata: Optional[Dict[str, Any]] = None,- ) -> None:- """Register a SemanticMsg with its total embedding task count.-- Args:- semantic_msg_id: The ID of the SemanticMsg- total_count: Total number of embedding tasks for this SemanticMsg- on_complete: Optional callback when all tasks complete- metadata: Optional metadata to store with the task- """- owner_loop = asyncio.get_running_loop()- record_to_finalize: Optional[_EmbeddingTaskRecord] = None-- with self._lock:- existing = self._tasks.get(semantic_msg_id)- if existing is not None:- logger.warning(- "Overwriting existing embedding tracker record for SemanticMsg %s",- semantic_msg_id,- )-- self._tasks[semantic_msg_id] = _EmbeddingTaskRecord(- remaining=total_count,- total=total_count,- on_complete=on_complete,- metadata=metadata or {},- owner_loop=owner_loop,- )- logger.info(- f"Registered embedding tracker for SemanticMsg {semantic_msg_id}: "- f"{total_count} tasks"- )-- if total_count <= 0:- record_to_finalize = self._tasks.pop(semantic_msg_id)- logger.info(- f"No embedding tasks for SemanticMsg {semantic_msg_id}, "- f"clearing tracker entry immediately"- )-- if record_to_finalize is not None:- await self._run_on_complete(semantic_msg_id, record_to_finalize)-- async def decrement(self, semantic_msg_id: str) -> Optional[int]:- """Decrement the remaining task count for a SemanticMsg.-- This method should be called when an embedding task is completed.- When the count reaches zero, the registered callback is executed- and the entry is removed from the tracker.-- Args:- semantic_msg_id: The ID of the SemanticMsg-- Returns:- The remaining count after decrement, or None if not found- """- record_to_finalize: Optional[_EmbeddingTaskRecord] = None-- with self._lock:- record = self._tasks.get(semantic_msg_id)- if record is None:- return None-- record.remaining -= 1- remaining = record.remaining-- if remaining <= 0:- record_to_finalize = self._tasks.pop(semantic_msg_id)- logger.info(- f"All embedding tasks({record.total}) completed for SemanticMsg {semantic_msg_id}"- )-- if record_to_finalize is not None:- await self._run_on_complete(semantic_msg_id, record_to_finalize)- return remaining-- async def abort(self, semantic_msg_id: str) -> bool:- """Retire one tracking attempt without running its completion callback.-- Late decrements for an aborted attempt are ignored because its record no- longer exists. Callers that may retry a logical semantic message must- register each attempt under a unique tracking ID.-- Returns:- True when an active record was removed, otherwise False.- """- with self._lock:- record = self._tasks.pop(semantic_msg_id, None)-- if record is None:- return False-- logger.warning(- "Aborted embedding tracker for SemanticMsg %s with %s/%s tasks remaining",- semantic_msg_id,- record.remaining,- record.total,- )- return Truediff --git a/openviking/storage/queuefs/queue_manager.py b/openviking/storage/queuefs/queue_manager.pyindex 41a01178b..5593eaea7 100644--- a/openviking/storage/queuefs/queue_manager.py+++ b/openviking/storage/queuefs/queue_manager.py@@ -21,8 +21,6 @@ from .semantic_queue import SemanticQueuelogger = get_logger(__name__)-DEFAULT_MAX_CONCURRENT_SESSION_COMMIT = 4-# ========== Singleton Pattern ==========_instance: Optional["QueueManager"] = None@@ -32,8 +30,10 @@ def init_queue_manager(timeout: int = 10,mount_point: str = "/queue",max_concurrent_embedding: int = 10,- max_concurrent_semantic: int = 64,+ max_concurrent_semantic: int = 32,max_concurrent_external_parse: int = 4,+ max_concurrent_add_resource: int = 4,+ max_concurrent_session_commit: int = 4,) -> "QueueManager":"""Initialize QueueManager singleton.@@ -44,6 +44,8 @@ def init_queue_manager(max_concurrent_embedding: Max concurrent embedding tasks.max_concurrent_semantic: Max concurrent semantic node work.max_concurrent_external_parse: Max concurrent ExternalParse tasks.+ max_concurrent_add_resource: Max concurrent AddResource tasks.+ max_concurrent_session_commit: Max concurrent SessionCommit tasks."""global _instance_instance = QueueManager(@@ -53,6 +55,8 @@ def init_queue_manager(max_concurrent_embedding=max_concurrent_embedding,max_concurrent_semantic=max_concurrent_semantic,max_concurrent_external_parse=max_concurrent_external_parse,+ max_concurrent_add_resource=max_concurrent_add_resource,+ max_concurrent_session_commit=max_concurrent_session_commit,)return _instance@@ -84,8 +88,10 @@ class QueueManager:timeout: int = 10,mount_point: str = "/queue",max_concurrent_embedding: int = 10,- max_concurrent_semantic: int = 64,+ max_concurrent_semantic: int = 32,max_concurrent_external_parse: int = 4,+ max_concurrent_add_resource: int = 4,+ max_concurrent_session_commit: int = 4,):"""Initialize QueueManager."""self._agfs = agfs@@ -94,6 +100,8 @@ class QueueManager:self._max_concurrent_embedding = max_concurrent_embeddingself._max_concurrent_semantic = max_concurrent_semanticself._max_concurrent_external_parse = max_concurrent_external_parse+ self._max_concurrent_add_resource = max_concurrent_add_resource+ self._max_concurrent_session_commit = max_concurrent_session_commitself._queues: Dict[str, NamedQueue] = {}self._started = Falseself._queue_threads: Dict[str, threading.Thread] = {}@@ -186,8 +194,10 @@ class QueueManager:return self._max_concurrent_embeddingif queue_name == self.EXTERNAL_PARSE:return self._max_concurrent_external_parse+ if queue_name == self.ADD_RESOURCE:+ return self._max_concurrent_add_resourceif queue_name == self.SESSION_COMMIT:- return DEFAULT_MAX_CONCURRENT_SESSION_COMMIT+ return self._max_concurrent_session_commitreturn self._max_concurrent_semanticdef _queue_worker_loop(diff --git a/openviking/storage/queuefs/semantic_dag.py b/openviking/storage/queuefs/semantic_dag.pyindex 9b30b3e7b..f8c24c4c9 100644--- a/openviking/storage/queuefs/semantic_dag.py+++ b/openviking/storage/queuefs/semantic_dag.py@@ -5,9 +5,9 @@import asyncioimport reimport threading+from contextlib import nullcontextfrom dataclasses import dataclass, fieldfrom typing import Any, ClassVar, Dict, List, Optional, Set-from uuid import uuid4from weakref import WeakKeyDictionaryfrom openviking.parse.parsers.media import get_media_type@@ -15,7 +15,7 @@ from openviking.server.identity import RequestContextfrom openviking.service.task_work_index import bind_task_context, get_task_contextfrom openviking.storage.queuefs.semantic_sidecar import write_semantic_sidecarsfrom openviking.storage.viking_fs import LS_ALL_NODES, get_viking_fs-from openviking.telemetry.request_wait_tracker import get_request_wait_tracker+from openviking.telemetry import bind_telemetry, get_current_telemetryfrom openviking.utils.ingest_options import IngestOptionsfrom openviking_cli.utils import VikingURIfrom openviking_cli.utils.logger import get_logger@@ -53,26 +53,6 @@ class DagStats:done_nodes: int = 0-@dataclass-class VectorizeTask:- """Vectorize task information."""-- task_type: str # "file" or "directory"- uri: str- context_type: str- ctx: "RequestContext"- semantic_msg_id: Optional[str] = None- # For file tasks- file_path: Optional[str] = None- summary_dict: Optional[Dict[str, str]] = None- parent_uri: Optional[str] = None- use_summary: bool = False- ingest_options: IngestOptions = field(default_factory=IngestOptions)- # For directory tasks- abstract: Optional[str] = None- overview: Optional[str] = None--@dataclass(frozen=True)class DagWork:"""A scheduled unit of DAG work."""@@ -81,7 +61,6 @@ class DagWork:dir_uri: strparent_uri: Optional[str] = Nonefile_path: Optional[str] = None- vectorize_task: Optional[VectorizeTask] = None@dataclass(frozen=True)@@ -171,8 +150,6 @@ class SemanticDagExecutor:ctx: RequestContext,incremental_update: bool = False,target_uri: Optional[str] = None,- semantic_msg_id: Optional[str] = None,- telemetry_id: str = "",recursive: bool = True,lock: Optional[Dict[str, Any]] = None,is_code_repo: bool = False,@@ -187,8 +164,6 @@ class SemanticDagExecutor:self._ctx = ctxself._incremental_update = incremental_updateself._target_uri = target_uri- self._semantic_msg_id = semantic_msg_id- self._telemetry_id = telemetry_idself._recursive = recursiveself._lock = lockself._is_code_repo = is_code_repo@@ -198,6 +173,7 @@ class SemanticDagExecutor:self._coalesce_key = coalesce_keyself._coalesce_version = coalesce_versionself._task_context = get_task_context()+ self._telemetry = get_current_telemetry()self._stale = Falseself._changed_paths = {path for key in ("added", "modified", "deleted") for path in self._changes.get(key, [])@@ -216,20 +192,6 @@ class SemanticDagExecutor:self._closed = Falseself._failure: Optional[Exception] = Noneself._stats = DagStats()- self._vectorize_task_count: int = 0- self._pending_vectorize_tasks: List[VectorizeTask] = []- self._pending_vectorize_work = 0- self._vectorize_done: Optional[asyncio.Event] = None- self._vectorize_failure: Optional[Exception] = None- self._vectorize_dispatch_complete = False- self._embedding_tracker_done: Optional[asyncio.Event] = None- self._embedding_tracker = None- # Retries preserve the logical SemanticMsg ID, so each execution needs- # a distinct tracker key to quarantine late decrements from old attempts.- self._embedding_tracker_id = f"{self._semantic_msg_id or 'semantic'}:{uuid4()}"- self._vectorize_finalized = False- self._vectorize_finalize_lock = asyncio.Lock()- self._vectorize_lock = asyncio.Lock()self._file_change_status: Dict[str, bool] = {}self._dir_change_status: Dict[str, bool] = {}self._overview_cache: Dict[str, Dict[str, str]] = {}@@ -247,56 +209,9 @@ class SemanticDagExecutor:await self._root_done.wait()if self._failure:raise self._failure-- # Mark semantic done after downstream vectorization finishes.- async with self._vectorize_lock:- task_count = self._vectorize_task_count- tasks = list(self._pending_vectorize_tasks)-- if task_count > 0:- from .embedding_tracker import EmbeddingTaskTracker-- embedding_tracker_done = asyncio.Event()- self._embedding_tracker_done = embedding_tracker_done- tracker = EmbeddingTaskTracker.get_instance()- self._embedding_tracker = tracker- await tracker.register(- semantic_msg_id=self._embedding_tracker_id,- total_count=task_count,- on_complete=self._on_embedding_tasks_complete,- metadata={- "uri": root_uri,- "semantic_msg_id": self._semantic_msg_id,- },- )-- await self._dispatch_vectorize_tasks(tasks)- self._vectorize_dispatch_complete = True-- if embedding_tracker_done.is_set():- await self._finalize_vectorization()- if self._vectorize_failure is not None:- raise self._vectorize_failure- else:- # No vectorize tasks — mark semantic work done immediately.- try:- await self._finalize_vectorization()- except Exception as e:- logger.error(f"Error in on_complete callback: {e}", exc_info=True)except BaseException:self._closed = Trueawait self._active_work_idle.wait()- if self._embedding_tracker is not None:- try:- # Pending embedding messages may still finish, but their- # attempt-specific decrements become harmless after abort.- await self._embedding_tracker.abort(self._embedding_tracker_id)- except Exception:- logger.warning(- "Failed to abort embedding tracker %s",- self._embedding_tracker_id,- exc_info=True,- )raisefinally:self._closed = True@@ -359,8 +274,6 @@ class SemanticDagExecutor:self._closed = Trueif self._root_done:self._root_done.set()- if self._vectorize_done:- self._vectorize_done.set()def _register_active(self) -> None:with self._active_lock:@@ -384,21 +297,19 @@ class SemanticDagExecutor:return statsasync def _run_work(self, work: DagWork) -> None:- if self._task_context is not None:- with bind_task_context(+ task_context = (+ bind_task_context(self._task_context.task_id,self._task_context.account_id,self._task_context.user_id,- ):- await self._run_work_bound(work)- return- await self._run_work_bound(work)+ )+ if self._task_context is not None+ else nullcontext()+ )+ with bind_telemetry(self._telemetry), task_context:+ await self._run_work_bound(work)async def _run_work_bound(self, work: DagWork) -> None:- if work.kind == "vectorize":- await self._run_vectorize_work(work.vectorize_task)- return-self._mark_node_started()if work.kind == "dir":@@ -426,72 +337,6 @@ class SemanticDagExecutor:self._mark_node_done()logger.warning("Unknown semantic DAG work kind: %s", work.kind)- async def _dispatch_vectorize_tasks(self, tasks: List[VectorizeTask]) -> None:- self._vectorize_done = asyncio.Event()- self._pending_vectorize_work = len(tasks)- for task in tasks:- self._schedule_work(- DagWork(- kind="vectorize",- dir_uri=task.uri,- vectorize_task=task,- )- )- await self._vectorize_done.wait()-- async def _run_vectorize_work(self, task: Optional[VectorizeTask]) -> None:- try:- if task is not None:- await self._run_vectorize_task(task)- except Exception as exc:- logger.error("Vectorization dispatch task failed: %s", exc, exc_info=True)- if self._vectorize_failure is None:- self._vectorize_failure = exc- finally:- self._pending_vectorize_work = max(0, self._pending_vectorize_work - 1)- if self._pending_vectorize_work == 0 and self._vectorize_done:- self._vectorize_done.set()-- async def _on_embedding_tasks_complete(self) -> None:- if self._embedding_tracker_done is not None:- self._embedding_tracker_done.set()- if self._vectorize_dispatch_complete:- await self._finalize_vectorization()-- async def _finalize_vectorization(self) -> None:- async with self._vectorize_finalize_lock:- if self._vectorize_finalized:- return- if self._vectorize_failure is None and self._telemetry_id and self._semantic_msg_id:- get_request_wait_tracker().mark_semantic_done(- self._telemetry_id, self._semantic_msg_id- )- self._vectorize_finalized = True-- async def _run_vectorize_task(self, task: VectorizeTask) -> None:- if task.task_type == "file":- await self._processor._vectorize_single_file(- parent_uri=task.parent_uri,- context_type=task.context_type,- file_path=task.file_path,- summary_dict=task.summary_dict,- ctx=task.ctx,- semantic_msg_id=task.semantic_msg_id,- use_summary=task.use_summary,- ingest_options=task.ingest_options,- )- return-- await self._processor._vectorize_directory(- task.uri,- task.context_type,- task.abstract,- task.overview,- ctx=task.ctx,- semantic_msg_id=task.semantic_msg_id,- ingest_options=task.ingest_options,- )-async def _dispatch_dir(self, dir_uri: str, parent_uri: Optional[str]) -> bool:"""Lazy-dispatch tasks for a directory when it is triggered."""if dir_uri in self._nodes:@@ -754,25 +599,34 @@ class SemanticDagExecutor:if self._closed:return- try:- if need_vectorize:- use_summary = self._is_code_repo and bool(summary_dict.get("summary"))- task = VectorizeTask(- task_type="file",- uri=file_path,+ if need_vectorize and not self._skip_vectorization:+ use_summary = self._is_code_repo and bool(summary_dict.get("summary"))+ try:+ await self._processor._vectorize_single_file(+ parent_uri=parent_uri,context_type=self._context_type,- ctx=self._ctx,- semantic_msg_id=self._embedding_tracker_id,file_path=file_path,summary_dict=summary_dict,- parent_uri=parent_uri,+ ctx=self._ctx,use_summary=use_summary,ingest_options=self._ingest_options,)- await self._add_vectorize_task(task)- except Exception as e:- logger.error(f"Failed to schedule vectorization for {file_path}: {e}", exc_info=True)- await self._on_file_done(parent_uri, file_path, summary_dict)+ except Exception as e:+ logger.error(+ "Failed to schedule vectorization for %s: %s",+ file_path,+ e,+ exc_info=True,+ )+ raise+ await self._on_file_done(+ parent_uri,+ file_path,+ {+ "name": str(summary_dict.get("name") or file_name),+ "summary": str(summary_dict.get("summary") or ""),+ },+ )async def _on_file_done(self, parent_uri: str, file_path: str, summary_dict: Dict[str, str]@@ -937,24 +791,27 @@ class SemanticDagExecutor:except Exception:logger.info(f"[SemanticDag] {dir_uri} write failed, skipping")- try:- if need_vectorize:- task = VectorizeTask(- task_type="directory",- uri=dir_uri,+ except Exception as e:+ logger.error(f"Failed to generate overview for {dir_uri}: {e}", exc_info=True)+ else:+ if need_vectorize and not self._skip_vectorization:+ try:+ await self._processor._vectorize_directory(+ dir_uri,context_type=self._context_type,- ctx=self._ctx,- semantic_msg_id=self._embedding_tracker_id,abstract=abstract,overview=overview,+ ctx=self._ctx,ingest_options=self._ingest_options,)- await self._add_vectorize_task(task)- except Exception as e:- logger.error(f"Failed to schedule vectorization for {dir_uri}: {e}", exc_info=True)-- except Exception as e:- logger.error(f"Failed to generate overview for {dir_uri}: {e}", exc_info=True)+ except Exception as e:+ logger.error(+ "Failed to schedule vectorization for %s: %s",+ dir_uri,+ e,+ exc_info=True,+ )+ raisefinally:self._stats.done_nodes += 1self._stats.in_progress_nodes = max(0, self._stats.in_progress_nodes - 1)@@ -971,21 +828,6 @@ class SemanticDagExecutor:await self._on_child_done(parent_uri, dir_uri, abstract)self._release_dir_node(dir_uri)- async def _add_vectorize_task(self, task: VectorizeTask) -> None:- """Add a vectorize task to the pending list."""- if self._skip_vectorization:- logger.info(- "Skipping vectorization task for %s (requested via SemanticMsg)",- task.uri,- )- return- async with self._vectorize_lock:- self._pending_vectorize_tasks.append(task)- if task.task_type == "file":- self._vectorize_task_count += 1- else: # directory- self._vectorize_task_count += 2-def get_stats(self) -> DagStats:return DagStats(total_nodes=self._stats.total_nodes,diff --git a/openviking/storage/queuefs/semantic_processor.py b/openviking/storage/queuefs/semantic_processor.pyindex d3da70d82..c73d6158c 100644--- a/openviking/storage/queuefs/semantic_processor.py+++ b/openviking/storage/queuefs/semantic_processor.py@@ -102,7 +102,7 @@ class SemanticProcessor(DequeueHandlerBase):_request_stats_order: List[str] = []_max_cached_stats = 256- def __init__(self, max_concurrent_llm: int = 64):+ def __init__(self, max_concurrent_llm: int = 32):"""Initialize SemanticProcessor.@@ -438,8 +438,6 @@ class SemanticProcessor(DequeueHandlerBase):ctx=current_ctx,incremental_update=is_incremental,target_uri=target_uri,- semantic_msg_id=msg.id,- telemetry_id=msg.telemetry_id,recursive=msg.recursive,lock=semantic_lock.lock,is_code_repo=msg.is_code_repo,@@ -459,6 +457,7 @@ class SemanticProcessor(DequeueHandlerBase):await self._enqueue_parent_refresh(msg, target_uri or msg.uri)finally:await semantic_lock.close()+ get_request_wait_tracker().mark_semantic_done(msg.telemetry_id, msg.id)self._merge_request_stats(msg.telemetry_id, processed=1)logger.info(f"Completed semantic generation for: {msg.uri}")self.report_success()@@ -555,8 +554,9 @@ class SemanticProcessor(DequeueHandlerBase):"""Process a memory directory with special handling.For memory directories:- - Memory files are already vectorized via embedding queue- - Only generate abstract.md and overview.md+ - Generate file summaries in bounded batches+ - Enqueue each changed file for vectorization as soon as its summary is ready+ - Retain only compact summaries for the directory overview- Vectorize the generated abstract.md and overview.mdArgs:@@ -566,185 +566,154 @@ class SemanticProcessor(DequeueHandlerBase):dir_uri = msg.urictx = ctx or self._default_ctxllm_sem = asyncio.Semaphore(self.max_concurrent_llm)- request_wait_tracker = get_request_wait_tracker()-- def _mark_done() -> None:- if msg.telemetry_id and msg.id:- request_wait_tracker.mark_semantic_done(msg.telemetry_id, msg.id)try:- try:- entries = await viking_fs.ls(dir_uri, node_limit=LS_ALL_NODES, ctx=ctx)- except Exception as e:- raise RuntimeError(f"Failed to list memory directory {dir_uri}: {e}") from e+ entries = await viking_fs.ls(dir_uri, node_limit=LS_ALL_NODES, ctx=ctx)+ except Exception as e:+ raise RuntimeError(f"Failed to list memory directory {dir_uri}: {e}") from e- file_paths: List[str] = []- for entry in entries:- name = entry.get("name", "")- if not name or name.startswith(".") or name in [".", ".."]:- continue- if not entry.get("isDir", False):- item_uri = VikingURI(dir_uri).join(name).uri- file_paths.append(item_uri)+ file_paths: List[str] = []+ for entry in entries:+ name = entry.get("name", "")+ if not name or name.startswith(".") or name in [".", ".."]:+ continue+ if not entry.get("isDir", False):+ item_uri = VikingURI(dir_uri).join(name).uri+ file_paths.append(item_uri)- if not file_paths:- logger.info(f"No memory files found in {dir_uri}")- _mark_done()- return+ if not file_paths:+ logger.info(f"No memory files found in {dir_uri}")+ return- existing_summaries: Dict[str, str] = {}- if msg.changes:- try:- old_overview = await viking_fs.read_file(f"{dir_uri}/.overview.md", ctx=ctx)- if old_overview:- existing_summaries = self._parse_overview_md(old_overview)- logger.info(- f"Parsed {len(existing_summaries)} existing summaries from overview.md"- )- except Exception as e:- logger.debug(f"No existing overview.md found for {dir_uri}: {e}")+ existing_summaries: Dict[str, str] = {}+ if msg.changes:+ try:+ old_overview = await viking_fs.read_file(f"{dir_uri}/.overview.md", ctx=ctx)+ if old_overview:+ existing_summaries = self._parse_overview_md(old_overview)+ logger.info(+ f"Parsed {len(existing_summaries)} existing summaries from overview.md"+ )+ except Exception as e:+ logger.debug(f"No existing overview.md found for {dir_uri}: {e}")- changed_files: Set[str] = set()- if msg.changes:- changed_files = set(msg.changes.get("added", []) + msg.changes.get("modified", []))- deleted_files = set(msg.changes.get("deleted", []))- logger.info(- f"Processing memory directory {dir_uri} with changes: "- f"added={len(msg.changes.get('added', []))}, "- f"modified={len(msg.changes.get('modified', []))}, "- f"deleted={len(deleted_files)}"- )+ changed_files: Set[str] = set()+ if msg.changes:+ changed_files = set(msg.changes.get("added", []) + msg.changes.get("modified", []))+ deleted_files = set(msg.changes.get("deleted", []))+ logger.info(+ f"Processing memory directory {dir_uri} with changes: "+ f"added={len(msg.changes.get('added', []))}, "+ f"modified={len(msg.changes.get('modified', []))}, "+ f"deleted={len(deleted_files)}"+ )- pending_indices: List[Tuple[int, str]] = []- file_summaries: List[Optional[Dict[str, str]]] = [None] * len(file_paths)+ pending_indices: List[Tuple[int, str]] = []+ file_summaries: List[Optional[Dict[str, str]]] = [None] * len(file_paths)+ paths_to_vectorize = changed_files if msg.changes else set(file_paths)++ for idx, file_path in enumerate(file_paths):+ file_name = file_path.split("/")[-1]+ if file_path not in changed_files and file_name in existing_summaries:+ file_summaries[idx] = {+ "name": file_name,+ "summary": existing_summaries[file_name],+ }+ logger.debug(f"Reused existing summary for {file_name}")+ else:+ pending_indices.append((idx, file_path))- for idx, file_path in enumerate(file_paths):- file_name = file_path.split("/")[-1]- if file_path not in changed_files and file_name in existing_summaries:- file_summaries[idx] = {- "name": file_name,- "summary": existing_summaries[file_name],- }- logger.debug(f"Reused existing summary for {file_name}")- else:- pending_indices.append((idx, file_path))+ if file_paths and not pending_indices:+ try:+ from openviking.metrics.datasources.cache import CacheEventDataSource- if file_paths and not pending_indices:- try:- from openviking.metrics.datasources.cache import CacheEventDataSource+ CacheEventDataSource.record_hit("L1")+ except Exception:+ pass+ elif file_paths and pending_indices:+ try:+ from openviking.metrics.datasources.cache import CacheEventDataSource+ if len(file_paths) > len(pending_indices):CacheEventDataSource.record_hit("L1")- except Exception:- pass- elif file_paths and pending_indices:- try:- from openviking.metrics.datasources.cache import CacheEventDataSource+ CacheEventDataSource.record_miss("L1")+ except Exception:+ pass- if len(file_paths) > len(pending_indices):- CacheEventDataSource.record_hit("L1")- CacheEventDataSource.record_miss("L1")- except Exception:- pass+ if pending_indices:+ logger.info(+ f"Generating summaries for {len(pending_indices)} changed files "+ f"(reused {len(file_paths) - len(pending_indices)} cached)"+ )- if pending_indices:+ async def _gen(idx: int, file_path: str) -> None:+ file_name = file_path.split("/")[-1]+ try:+ summary_dict = await self._generate_single_file_summary(+ file_path, llm_sem=llm_sem, ctx=ctx+ )+ logger.debug(f"Generated summary for {file_name}")+ except Exception as e:+ logger.warning(f"Failed to generate summary for {file_path}: {e}")+ summary_dict = {"name": file_name, "summary": ""}++ if file_path in paths_to_vectorize and not msg.skip_vectorization:+ await self._vectorize_single_file(+ parent_uri=dir_uri,+ context_type="memory",+ file_path=file_path,+ summary_dict=summary_dict,+ ctx=ctx,+ preserve_existing_created_at=True,+ )+ file_summaries[idx] = {+ "name": str(summary_dict.get("name") or file_name),+ "summary": str(summary_dict.get("summary") or ""),+ }++ batch_size = max(1, min(self.max_concurrent_llm, 10))+ for batch_start in range(0, len(pending_indices), batch_size):+ batch = pending_indices[batch_start : batch_start + batch_size]logger.info(- f"Generating summaries for {len(pending_indices)} changed files "- f"(reused {len(file_paths) - len(pending_indices)} cached)"+ f"[MemorySemantic] Processing batch {batch_start // batch_size + 1}/"+ f"{(len(pending_indices) + batch_size - 1) // batch_size} "+ f"({len(batch)} files)")+ await asyncio.gather(*[_gen(i, fp) for i, fp in batch])- async def _gen(idx: int, file_path: str) -> None:- file_name = file_path.split("/")[-1]- try:- summary_dict = await self._generate_single_file_summary(- file_path, llm_sem=llm_sem, ctx=ctx- )- file_summaries[idx] = summary_dict- logger.debug(f"Generated summary for {file_name}")- except Exception as e:- logger.warning(f"Failed to generate summary for {file_path}: {e}")- file_summaries[idx] = {"name": file_name, "summary": ""}-- batch_size = max(1, min(self.max_concurrent_llm, 10))- for batch_start in range(0, len(pending_indices), batch_size):- batch = pending_indices[batch_start : batch_start + batch_size]- logger.info(- f"[MemorySemantic] Processing batch {batch_start // batch_size + 1}/"- f"{(len(pending_indices) + batch_size - 1) // batch_size} "- f"({len(batch)} files)"- )- await asyncio.gather(*[_gen(i, fp) for i, fp in batch])-- completed_summaries = [s for s in file_summaries if s is not None]- # Incremental writes carry changes; full rebuild tasks do not.- if msg.changes:- paths_to_vectorize = changed_files- else:- paths_to_vectorize = set(file_paths)- file_vectorize_items = [- (file_path, summary)- for file_path, summary in zip(file_paths, file_summaries, strict=False)- if file_path in paths_to_vectorize and summary is not None- ]- generated_content = await self._generate_overview(- dir_uri, completed_summaries, [], llm_sem=llm_sem- )- overview, abstract = self._normalize_overview_generation(generated_content)-- try:- wrote_semantics = await self._write_memory_directory_semantics(- msg=msg,- viking_fs=viking_fs,- dir_uri=dir_uri,- overview=overview,- abstract=abstract,- ctx=ctx,- lock=lock,- )- except Exception as e:- raise RuntimeError(f"Failed to write abstract/overview for {dir_uri}: {e}") from e- if not wrote_semantics:- _mark_done()- return- logger.info(f"Generated abstract.md and overview.md for {dir_uri}")-- if msg.skip_vectorization:- logger.info(f"Skipping vectorization for {dir_uri} (requested via SemanticMsg)")- _mark_done()- return- if msg.telemetry_id and msg.id:- from openviking.storage.queuefs.embedding_tracker import EmbeddingTaskTracker-- async def _on_complete() -> None:- get_request_wait_tracker().mark_semantic_done(msg.telemetry_id, msg.id)+ completed_summaries = [s for s in file_summaries if s is not None]+ generated_content = await self._generate_overview(+ dir_uri, completed_summaries, [], llm_sem=llm_sem+ )+ overview, abstract = self._normalize_overview_generation(generated_content)- tracker = EmbeddingTaskTracker.get_instance()- await tracker.register(- semantic_msg_id=msg.id,- total_count=2 + len(file_vectorize_items),- on_complete=_on_complete,- metadata={"uri": dir_uri},- )- for file_path, summary_dict in file_vectorize_items:- await self._vectorize_single_file(- parent_uri=dir_uri,- context_type="memory",- file_path=file_path,- summary_dict=summary_dict,- ctx=ctx,- semantic_msg_id=msg.id,- preserve_existing_created_at=True,- )- await self._vectorize_directory(- uri=dir_uri,- context_type="memory",- abstract=abstract,+ try:+ wrote_semantics = await self._write_memory_directory_semantics(+ msg=msg,+ viking_fs=viking_fs,+ dir_uri=dir_uri,overview=overview,+ abstract=abstract,ctx=ctx,- semantic_msg_id=msg.id,+ lock=lock,)- logger.info(f"Vectorized abstract.md and overview.md for {dir_uri}")- finally:- pass+ except Exception as e:+ raise RuntimeError(f"Failed to write abstract/overview for {dir_uri}: {e}") from e+ if not wrote_semantics:+ return+ logger.info(f"Generated abstract.md and overview.md for {dir_uri}")++ if msg.skip_vectorization:+ logger.info(f"Skipping vectorization for {dir_uri} (requested via SemanticMsg)")+ return+ await self._vectorize_directory(+ uri=dir_uri,+ context_type="memory",+ abstract=abstract,+ overview=overview,+ ctx=ctx,+ )+ logger.info(f"Vectorized abstract.md and overview.md for {dir_uri}")async def _write_memory_directory_semantics(self,@@ -1563,7 +1532,6 @@ class SemanticProcessor(DequeueHandlerBase):abstract: str,overview: str,ctx: Optional[RequestContext] = None,- semantic_msg_id: Optional[str] = None,ingest_options: IngestOptions | None = None,) -> None:"""Create directory Context and enqueue to EmbeddingQueue."""@@ -1577,7 +1545,6 @@ class SemanticProcessor(DequeueHandlerBase):overview=overview,context_type=context_type,ctx=active_ctx,- semantic_msg_id=semantic_msg_id,ingest_options=ingest_options,)@@ -1588,7 +1555,6 @@ class SemanticProcessor(DequeueHandlerBase):file_path: str,summary_dict: Dict[str, str],ctx: Optional[RequestContext] = None,- semantic_msg_id: Optional[str] = None,use_summary: bool = False,preserve_existing_created_at: bool = False,ingest_options: IngestOptions | None = None,@@ -1603,7 +1569,6 @@ class SemanticProcessor(DequeueHandlerBase):parent_uri=parent_uri,context_type=context_type,ctx=active_ctx,- semantic_msg_id=semantic_msg_id,use_summary=use_summary,preserve_existing_created_at=preserve_existing_created_at,ingest_options=ingest_options,diff --git a/openviking/telemetry/request_wait_tracker.py b/openviking/telemetry/request_wait_tracker.pyindex b22fc1527..a26b84a18 100644--- a/openviking/telemetry/request_wait_tracker.py+++ b/openviking/telemetry/request_wait_tracker.py@@ -55,14 +55,14 @@ class RequestWaitTracker:def register_request(self, telemetry_id: str) -> None:self._create_state(telemetry_id)- def register_semantic_root(self, telemetry_id: str, semantic_msg_id: str) -> None:- if not telemetry_id or not semantic_msg_id:+ def register_semantic_root(self, telemetry_id: str, root_id: str) -> None:+ if not telemetry_id or not root_id:returnwith self._lock:state = self._states.get(telemetry_id)if state is None:return- state.pending_semantic_roots.add(semantic_msg_id)+ state.pending_semantic_roots.add(root_id)def register_embedding_root(self, telemetry_id: str, root_id: str) -> None:if not telemetry_id or not root_id:@@ -73,15 +73,6 @@ class RequestWaitTracker:returnstate.pending_embedding_roots.add(root_id)- def record_embedding_processed(self, telemetry_id: str, delta: int = 1) -> None:- if not telemetry_id:- return- with self._lock:- state = self._states.get(telemetry_id)- if state is None:- return- state.embedding_processed += max(delta, 0)-def record_embedding_requeue(self, telemetry_id: str, delta: int = 1) -> None:if not telemetry_id:return@@ -91,21 +82,10 @@ class RequestWaitTracker:returnstate.embedding_requeue_count += max(delta, 0)- def record_embedding_error(self, telemetry_id: str, message: str) -> None:- if not telemetry_id:- return- with self._lock:- state = self._states.get(telemetry_id)- if state is None:- return- state.embedding_error_count += 1- if message:- state.embedding_errors.append(message)-def mark_semantic_done(self,telemetry_id: str,- semantic_msg_id: str,+ root_id: str,processed_delta: int = 1,) -> None:if not telemetry_id:@@ -114,7 +94,7 @@ class RequestWaitTracker:state = self._states.get(telemetry_id)if state is None:return- state.pending_semantic_roots.discard(semantic_msg_id)+ state.pending_semantic_roots.discard(root_id)state.semantic_processed += max(processed_delta, 0)def record_semantic_requeue(self, telemetry_id: str, delta: int = 1) -> None:@@ -126,14 +106,14 @@ class RequestWaitTracker:returnstate.semantic_requeue_count += max(delta, 0)- def mark_semantic_failed(self, telemetry_id: str, semantic_msg_id: str, message: str) -> None:+ def mark_semantic_failed(self, telemetry_id: str, root_id: str, message: str) -> None:if not telemetry_id:returnwith self._lock:state = self._states.get(telemetry_id)if state is None:return- state.pending_semantic_roots.discard(semantic_msg_id)+ state.pending_semantic_roots.discard(root_id)state.semantic_error_count += 1if message:state.semantic_errors.append(message)diff --git a/openviking/utils/embedding_utils.py b/openviking/utils/embedding_utils.pyindex b5477ecf3..a34b463d3 100644--- a/openviking/utils/embedding_utils.py+++ b/openviking/utils/embedding_utils.py@@ -94,20 +94,34 @@ def _apply_ingest_options(}-async def _decrement_embedding_tracker(semantic_msg_id: Optional[str], count: int) -> None:- if not semantic_msg_id or count <= 0:- return+async def _enqueue_embedding_message(+ embedding_queue,+ embedding_msg,+ *,+ failure_message: str,+) -> bool:+ """Persist one embedding message and settle request tracking on enqueue failure."""+ wait_tracker = get_request_wait_tracker()+ wait_tracker.register_embedding_root(embedding_msg.telemetry_id, embedding_msg.id)+try:- from openviking.storage.queuefs.embedding_tracker import EmbeddingTaskTracker+ enqueue_id = await embedding_queue.enqueue(embedding_msg)+ except BaseException as exc:+ wait_tracker.mark_embedding_failed(+ embedding_msg.telemetry_id,+ embedding_msg.id,+ f"{failure_message}: {exc}",+ )+ raise- tracker = EmbeddingTaskTracker.get_instance()- for _ in range(count):- await tracker.decrement(semantic_msg_id)- except Exception as e:- logger.error(- f"Failed to decrement embedding tracker for semantic_msg_id={semantic_msg_id}: {e}",- exc_info=True,+ if not enqueue_id:+ wait_tracker.mark_embedding_failed(+ embedding_msg.telemetry_id,+ embedding_msg.id,+ failure_message,)+ return False+ return Truedef _coerce_datetime(value: object) -> Optional[datetime]:@@ -341,7 +355,6 @@ async def vectorize_directory_meta(overview: str,context_type: str = "resource",ctx: Optional[RequestContext] = None,- semantic_msg_id: Optional[str] = None,include_overview: bool = True,scalar_overrides: Optional[Dict[int, Dict[str, Any]]] = None,ingest_options: IngestOptions | None = None,@@ -351,8 +364,6 @@ async def vectorize_directory_meta(Creates Context objects for abstract and overview and enqueues them."""- enqueued = 0- expected = 2 if include_overview else 1first_enqueue_error: Optional[Exception] = Nonetry:if not ctx:@@ -394,11 +405,14 @@ async def vectorize_directory_meta()_apply_ingest_options(msg_abstract, ingest_options)if msg_abstract:- msg_abstract.semantic_msg_id = semantic_msg_idtry:- await embedding_queue.enqueue(msg_abstract)- enqueued += 1- logger.debug(f"Enqueued directory L0 (abstract) for vectorization: {uri}")+ enqueued = await _enqueue_embedding_message(+ embedding_queue,+ msg_abstract,+ failure_message=f"Failed to enqueue directory L0 vector for {uri}",+ )+ if enqueued:+ logger.debug(f"Enqueued directory L0 (abstract) for vectorization: {uri}")except TaskWorkRejected:logger.debug("Skipped directory vectorization for cancelling task: %s", uri)return@@ -434,11 +448,14 @@ async def vectorize_directory_meta()_apply_ingest_options(msg_overview, ingest_options)if msg_overview:- msg_overview.semantic_msg_id = semantic_msg_idtry:- await embedding_queue.enqueue(msg_overview)- enqueued += 1- logger.debug(f"Enqueued directory L1 (overview) for vectorization: {uri}")+ enqueued = await _enqueue_embedding_message(+ embedding_queue,+ msg_overview,+ failure_message=f"Failed to enqueue directory L1 vector for {uri}",+ )+ if enqueued:+ logger.debug(f"Enqueued directory L1 (overview) for vectorization: {uri}")except TaskWorkRejected:logger.debug("Skipped directory vectorization for cancelling task: %s", uri)return@@ -457,8 +474,6 @@ async def vectorize_directory_meta(exc_info=True,)raise- finally:- await _decrement_embedding_tracker(semantic_msg_id, expected - enqueued)async def vectorize_file(@@ -467,11 +482,9 @@ async def vectorize_file(parent_uri: str,context_type: str = "resource",ctx: Optional[RequestContext] = None,- semantic_msg_id: Optional[str] = None,use_summary: bool = False,preserve_existing_created_at: bool = False,scalar_override: Optional[Dict[str, Any]] = None,- register_request_wait: bool = False,ingest_options: IngestOptions | None = None,) -> bool:"""@@ -482,9 +495,6 @@ async def vectorize_file(`use_summary` flag (code path override) or the embedding config.Returns whether an embedding message was enqueued."""- enqueued = False- registered_wait_root: Optional[tuple[str, str]] = None-try:if not ctx:logger.warning("No context provided for vectorization")@@ -595,45 +605,22 @@ async def vectorize_file(_apply_scalar_overrides(embedding_msg, scalar_override)_apply_ingest_options(embedding_msg, ingest_options)- embedding_msg.semantic_msg_id = semantic_msg_id- if register_request_wait:- get_request_wait_tracker().register_embedding_root(- embedding_msg.telemetry_id,- embedding_msg.id,- )- registered_wait_root = (embedding_msg.telemetry_id, embedding_msg.id)- enqueued_id = await embedding_queue.enqueue(embedding_msg)- if register_request_wait and not enqueued_id:- get_request_wait_tracker().mark_embedding_failed(- embedding_msg.telemetry_id,- embedding_msg.id,- f"Failed to enqueue file vector for {file_path}",- )+ enqueued = await _enqueue_embedding_message(+ embedding_queue,+ embedding_msg,+ failure_message=f"Failed to enqueue file vector for {file_path}",+ )+ if not enqueued:return False- enqueued = Truelogger.debug(f"Enqueued file for vectorization: {file_path}")except TaskWorkRejected:logger.debug("Skipped file vectorization for cancelling task: %s", file_path)- if registered_wait_root is not None:- get_request_wait_tracker().mark_embedding_failed(- registered_wait_root[0],- registered_wait_root[1],- f"Task cancellation skipped file vector for {file_path}",- )+ return Falseexcept Exception as e:logger.error(f"Failed to vectorize file {file_path}: {e}", exc_info=True)- if registered_wait_root is not None:- get_request_wait_tracker().mark_embedding_failed(- registered_wait_root[0],- registered_wait_root[1],- f"Failed to enqueue file vector for {file_path}: {e}",- )raise- finally:- if not enqueued:- await _decrement_embedding_tracker(semantic_msg_id, 1)- return enqueued+ return Trueasync def index_resource(diff --git a/openviking/utils/resource_processor.py b/openviking/utils/resource_processor.pyindex ffdc9a287..608e9ecc2 100644--- a/openviking/utils/resource_processor.py+++ b/openviking/utils/resource_processor.py@@ -13,7 +13,6 @@ from collections.abc import Callablefrom typing import TYPE_CHECKING, Any, Dict, List, Optional, Unionfrom openviking.core.context import ContextLevel-from openviking.utils.ingest_options import IngestOptionsfrom openviking.core.namespace import context_type_for_urifrom openviking.parse.image_rewrite import rewrite_image_urisfrom openviking.parse.mode import ParseMode, normalize_parse_mode@@ -25,14 +24,15 @@ from openviking.resource.processing_mode import (normalize_processing_mode,)from openviking.server.identity import RequestContext-from openviking.storage.vikingdb_manager import VikingDBManagerfrom openviking.storage.errors import LockAcquisitionErrorfrom openviking.storage.expr import And, Eq, PathScopefrom openviking.storage.internal_names import STORAGE_INTERNAL_ENTRY_NAMESfrom openviking.storage.queuefs.semantic_processor import SemanticProcessorfrom openviking.storage.viking_fs import LS_ALL_NODES, get_viking_fs+from openviking.storage.vikingdb_manager import VikingDBManagerfrom openviking.telemetry import get_current_telemetryfrom openviking.utils.embedding_utils import index_resource, vectorize_file+from openviking.utils.ingest_options import IngestOptionsfrom openviking.utils.summarizer import Summarizerfrom openviking_cli.exceptions import OpenVikingErrorfrom openviking_cli.utils import VikingURI, get_logger@@ -669,7 +669,6 @@ class ResourceProcessor:context_type=context_type_for_uri(entry_uri),ctx=ctx,ingest_options=ingest_options,- register_request_wait=True,)async def _vectorize_resource_file(@@ -690,7 +689,6 @@ class ResourceProcessor:context_type=context_type_for_uri(file_uri),ctx=ctx,ingest_options=IngestOptions.from_value(ingest_options),- register_request_wait=True,)async def reserve_unique_candidate(diff --git a/openviking_cli/utils/config/open_viking_config.py b/openviking_cli/utils/config/open_viking_config.pyindex 95f014b75..c7fbb2451 100644--- a/openviking_cli/utils/config/open_viking_config.py+++ b/openviking_cli/utils/config/open_viking_config.py@@ -185,7 +185,6 @@ class OpenVikingConfig(BaseModel):default_factory=GitConfig, description="Git version control configuration")- # Parser configurationspdf: PDFConfig = Field(default_factory=PDFConfig, description="PDF parsing configuration")code: CodeConfig = Field(default_factory=CodeConfig, description="Code parsing configuration")diff --git a/openviking_cli/utils/config/queue_worker_config.py b/openviking_cli/utils/config/queue_worker_config.pyindex 497c26f77..a73f40e96 100644--- a/openviking_cli/utils/config/queue_worker_config.py+++ b/openviking_cli/utils/config/queue_worker_config.py@@ -21,5 +21,7 @@ class QueueWorkersConfig(BaseModel):"""Runtime limits for QueueFS consumers."""external_parse: QueueWorkerConfig = Field(default_factory=QueueWorkerConfig)+ add_resource: QueueWorkerConfig = Field(default_factory=QueueWorkerConfig)+ session_commit: QueueWorkerConfig = Field(default_factory=QueueWorkerConfig)model_config = {"extra": "forbid"}diff --git a/openviking_cli/utils/config/vlm_config.py b/openviking_cli/utils/config/vlm_config.pyindex 75914888c..45b5a0082 100644--- a/openviking_cli/utils/config/vlm_config.py+++ b/openviking_cli/utils/config/vlm_config.py@@ -134,7 +134,7 @@ class VLMConfig(BaseModel):thinking: bool = Field(default=False, description="Enable thinking mode for VolcEngine models")max_concurrent: int = Field(- default=64, description="Maximum number of concurrent LLM calls for semantic processing"+ default=32, description="Maximum number of concurrent LLM calls for semantic processing")media: VLMMediaConfig = Field(diff --git a/tests/service/test_fs_service.py b/tests/service/test_fs_service.pyindex fd55fd180..39f282d01 100644--- a/tests/service/test_fs_service.py+++ b/tests/service/test_fs_service.py@@ -86,11 +86,11 @@ class _FakeWaitTracker:def register_request(self, telemetry_id):self.registered_requests.append(telemetry_id)- def register_semantic_root(self, telemetry_id, semantic_msg_id):+ def register_semantic_root(self, telemetry_id, root_id):self.registered_roots.append({"telemetry_id": telemetry_id,- "semantic_msg_id": semantic_msg_id,+ "root_id": root_id,"request_was_registered": telemetry_id in self.registered_requests,})@@ -104,7 +104,7 @@ class _FakeWaitTracker:"Embedding": {"processed": 0, "error_count": 0, "errors": []},}- def mark_semantic_failed(self, telemetry_id, semantic_msg_id, message):+ def mark_semantic_failed(self, telemetry_id, root_id, message):passdef cleanup(self, telemetry_id):diff --git a/tests/session/test_memory_batching.py b/tests/session/test_memory_batching.pyindex 7cbaa7bb6..7e4f4d988 100644--- a/tests/session/test_memory_batching.py+++ b/tests/session/test_memory_batching.py@@ -4,7 +4,7 @@import asyncioimport loggingimport os-from unittest.mock import AsyncMock, MagicMock, patch+from unittest.mock import MagicMock, patchimport pytest@@ -58,6 +58,7 @@ class NorthwestTripMockVLM:return (content * 10)[:2000]+@skip_if_not_manual@pytest.mark.asyncioasync def test_manual_memory_batching_100_files(monkeypatch):"""@@ -112,11 +113,6 @@ async def test_manual_memory_batching_100_files(monkeypatch):mock_fs = MockVikingFS()- # 3. 模拟 Tracker 和 WaitTracker- mock_wait_tracker = MagicMock()- mock_embedding_tracker = MagicMock()- mock_embedding_tracker.register = AsyncMock()-# 使用 patch.multiple 来模拟多个 get_xxx 方法with (patch(@@ -124,14 +120,6 @@ async def test_manual_memory_batching_100_files(monkeypatch):return_value=mock_config,),patch("openviking.storage.queuefs.semantic_processor.get_viking_fs", return_value=mock_fs),- patch(- "openviking.storage.queuefs.semantic_processor.get_request_wait_tracker",- return_value=mock_wait_tracker,- ),- patch(- "openviking.storage.queuefs.embedding_tracker.EmbeddingTaskTracker.get_instance",- return_value=mock_embedding_tracker,- ),):# 4. 初始化 Processor 并设置并发processor = SemanticProcessor(max_concurrent_llm=10)@@ -183,7 +171,6 @@ async def test_manual_memory_batching_100_files(monkeypatch):# 100次 摘要生成 + 1次 overview(L1) + 1次 abstract(L0)# 因为 read_file 也被 mock 了,所以构造过程不再消耗 call_countassert mock_vlm.call_count >= 102- assert mock_embedding_tracker.register.calledprint("[Manual Test] 分批逻辑压力测试及并发验证成功。")diff --git a/tests/storage/test_collection_schemas.py b/tests/storage/test_collection_schemas.pyindex 96bae5aee..22f98658c 100644--- a/tests/storage/test_collection_schemas.py+++ b/tests/storage/test_collection_schemas.py@@ -1,7 +1,6 @@# Copyright (c) 2026 Beijing Volcano Engine Technology Co., Ltd.# SPDX-License-Identifier: AGPL-3.0-import asyncioimport hashlibimport inspectimport json@@ -22,20 +21,20 @@ from openviking.storage.errors import EmbeddingRebuildRequiredErrorfrom openviking.storage.expr import Eqfrom openviking.storage.queuefs.embedding_msg import EmbeddingMsgfrom openviking.storage.vectordb import engine as vectordb_engine+from openviking.storage.vectordb.collection.result import UpsertDataResult+from openviking.storage.vectordb.collection.vikingdb_collection import VikingDBCollectionfrom openviking.storage.vectordb.collection.volcengine_api_key_collection import (VolcengineApiKeyCollection,)-from openviking.storage.vectordb.collection.vikingdb_collection import VikingDBCollectionfrom openviking.storage.vectordb.collection.volcengine_collection import VolcengineCollection-from openviking.storage.vectordb.collection.result import UpsertDataResultfrom openviking.storage.vectordb_adapters.base import (VIKINGDB_TEXT_FIELD_BYTE_LIMIT,_truncate_text_field,)from openviking.storage.vectordb_adapters.local_adapter import LocalCollectionAdapterfrom openviking.storage.viking_vector_index_backend import (- UpsertOptions,VIKINGDB_CONTENT_MAX_SIZE,+ UpsertOptions,VikingVectorIndexBackend,_SingleAccountBackend,)@@ -660,7 +659,7 @@ async def test_embedding_handler_preserves_parent_uri_for_backend_upsert_logic(m@pytest.mark.asyncio-async def test_embedding_handler_marks_success_only_after_tracker_completion(monkeypatch):+async def test_embedding_handler_settles_request_wait_by_message_id(monkeypatch):class _CapturingVikingDB:is_closing = Falsemode = "local"@@ -676,46 +675,25 @@ async def test_embedding_handler_marks_success_only_after_tracker_completion(monlambda: _DummyConfig(embedder),)- decrement_started = asyncio.Event()- allow_decrement_finish = asyncio.Event()-- class _FakeTracker:- async def decrement(self, _semantic_msg_id):- decrement_started.set()- await allow_decrement_finish.wait()- return 0-+ completed = []monkeypatch.setattr(- "openviking.storage.queuefs.embedding_tracker.EmbeddingTaskTracker.get_instance",- lambda: _FakeTracker(),+ "openviking.storage.collection_schemas.get_request_wait_tracker",+ lambda: SimpleNamespace(+ mark_embedding_done=lambda telemetry_id, root_id: completed.append(+ (telemetry_id, root_id)+ )+ ),)handler = TextEmbeddingHandler(_CapturingVikingDB())- status = {"success": 0, "requeue": 0, "error": 0}- handler.set_callbacks(- on_success=lambda: status.__setitem__("success", status["success"] + 1),- on_requeue=lambda: status.__setitem__("requeue", status["requeue"] + 1),- on_error=lambda *_: status.__setitem__("error", status["error"] + 1),- )-payload = _build_queue_payload()queue_data = json.loads(payload["data"])- queue_data["semantic_msg_id"] = "semantic-1"+ queue_data["telemetry_id"] = "request-1"payload["data"] = json.dumps(queue_data)- task = asyncio.create_task(handler.on_dequeue(payload))- await decrement_started.wait()-- assert status["success"] == 0- assert status["requeue"] == 0- assert status["error"] == 0-- allow_decrement_finish.set()- await task+ await handler.on_dequeue(payload)- assert status["success"] == 1- assert status["requeue"] == 0- assert status["error"] == 0+ assert completed == [("request-1", queue_data["id"])]def test_context_collection_excludes_parent_uri():diff --git a/tests/storage/test_content_write_processing_mode.py b/tests/storage/test_content_write_processing_mode.pyindex 394d8db12..25c582999 100644--- a/tests/storage/test_content_write_processing_mode.py+++ b/tests/storage/test_content_write_processing_mode.py@@ -73,7 +73,7 @@ async def test_vectors_only_write_skips_semantic_refresh_and_vectorizes_file(mon"name": "demo.md","summary": "",}- assert vectorize_file.await_args.kwargs["register_request_wait"] is True+ assert "register_request_wait" not in vectorize_file.await_args.kwargsassert result["semantic_status"] == "skipped"assert result["vector_status"] == "queued"diff --git a/tests/storage/test_embedding_tracker.py b/tests/storage/test_embedding_tracker.pydeleted file mode 100644index ea6ad0b72..000000000--- a/tests/storage/test_embedding_tracker.py+++ /dev/null@@ -1,213 +0,0 @@-# Copyright (c) 2026 Beijing Volcano Engine Technology Co., Ltd.-# SPDX-License-Identifier: AGPL-3.0--import asyncio-import concurrent.futures-import threading-import time--import pytest--from openviking.storage.queuefs.embedding_tracker import EmbeddingTaskTracker---class _LoopThread:- def __init__(self, close_delay: float = 0) -> None:- self.loop = asyncio.new_event_loop()- self._ready = threading.Event()- self._close_delay = close_delay- self.thread = threading.Thread(target=self._run, daemon=True)- self.thread.start()- self._ready.wait(timeout=2)-- def _run(self) -> None:- asyncio.set_event_loop(self.loop)- self._ready.set()- self.loop.run_forever()- if self._close_delay:- time.sleep(self._close_delay)- pending = asyncio.all_tasks(self.loop)- for task in pending:- task.cancel()- if pending:- self.loop.run_until_complete(asyncio.gather(*pending, return_exceptions=True))- self.loop.close()-- def submit(self, coro):- return asyncio.run_coroutine_threadsafe(coro, self.loop)-- def stop(self) -> None:- if self.loop.is_closed():- return- self.loop.call_soon_threadsafe(self.loop.stop)- self.thread.join(timeout=3)-- def stop_without_join(self) -> None:- if self.loop.is_closed():- return- self.loop.call_soon_threadsafe(self.loop.stop)-- def join(self) -> None:- self.thread.join(timeout=3)--[email protected](autouse=True)-def _reset_tracker_singleton():- EmbeddingTaskTracker._instance = None- EmbeddingTaskTracker._initialized = False- yield- EmbeddingTaskTracker._instance = None- EmbeddingTaskTracker._initialized = False---def test_tracker_runs_completion_callback_on_register_loop():- tracker = EmbeddingTaskTracker.get_instance()- owner = _LoopThread()- worker = _LoopThread()- callback_info = concurrent.futures.Future()-- async def on_complete():- callback_info.set_result((threading.get_ident(), asyncio.get_running_loop()))- await asyncio.sleep(0)-- async def register():- await tracker.register("semantic-msg", 1, on_complete=on_complete)-- async def decrement():- return await tracker.decrement("semantic-msg")-- try:- owner.submit(register()).result(timeout=2)- assert not callback_info.done()-- remaining = worker.submit(decrement()).result(timeout=2)- callback_thread_id, callback_loop = callback_info.result(timeout=2)- finally:- owner.stop()- worker.stop()-- assert remaining == 0- assert callback_thread_id == owner.thread.ident- assert callback_loop is owner.loop---def test_tracker_falls_back_to_current_loop_when_owner_loop_is_closed():- tracker = EmbeddingTaskTracker.get_instance()- owner = _LoopThread()- worker = _LoopThread()- callback_info = concurrent.futures.Future()-- async def on_complete():- callback_info.set_result((threading.get_ident(), asyncio.get_running_loop()))-- async def register():- await tracker.register("semantic-msg", 1, on_complete=on_complete)-- async def decrement():- return await tracker.decrement("semantic-msg")-- try:- owner.submit(register()).result(timeout=2)- owner.stop()-- remaining = worker.submit(decrement()).result(timeout=2)- callback_thread_id, callback_loop = callback_info.result(timeout=2)- finally:- worker.stop()-- assert remaining == 0- assert callback_thread_id == worker.thread.ident- assert callback_loop is worker.loop---def test_tracker_falls_back_to_current_loop_when_owner_loop_is_stopped():- tracker = EmbeddingTaskTracker.get_instance()- owner = _LoopThread(close_delay=1)- worker = _LoopThread()- callback_info = concurrent.futures.Future()-- async def on_complete():- callback_info.set_result((threading.get_ident(), asyncio.get_running_loop()))-- async def register():- await tracker.register("semantic-msg", 1, on_complete=on_complete)-- async def decrement():- return await tracker.decrement("semantic-msg")-- try:- owner.submit(register()).result(timeout=2)- owner.stop_without_join()- time.sleep(0.1)-- remaining = worker.submit(decrement()).result(timeout=2)- callback_thread_id, callback_loop = callback_info.result(timeout=2)- finally:- worker.stop()- owner.join()-- assert remaining == 0- assert callback_thread_id == worker.thread.ident- assert callback_loop is worker.loop---async def test_tracker_runs_zero_task_callback_immediately():- tracker = EmbeddingTaskTracker.get_instance()- callback_loop = None-- async def on_complete():- nonlocal callback_loop- callback_loop = asyncio.get_running_loop()-- await tracker.register("semantic-msg", 0, on_complete=on_complete)-- assert callback_loop is asyncio.get_running_loop()---async def test_tracker_supports_sync_callback_and_missing_task():- tracker = EmbeddingTaskTracker.get_instance()- callback_calls = []-- await tracker.register("semantic-msg", 1, on_complete=lambda: callback_calls.append("done"))- remaining = await tracker.decrement("semantic-msg")-- assert remaining == 0- assert callback_calls == ["done"]- assert await tracker.decrement("missing-semantic-msg") is None---async def test_tracker_clears_zero_task_entry_without_callback():- tracker = EmbeddingTaskTracker.get_instance()-- await tracker.register("semantic-msg", 0, on_complete=None)-- assert await tracker.decrement("semantic-msg") is None---async def test_tracker_abort_ignores_late_decrements_without_completing():- tracker = EmbeddingTaskTracker.get_instance()- callback_calls = []-- await tracker.register(- "semantic-msg:attempt-1",- 2,- on_complete=lambda: callback_calls.append("attempt-1"),- )-- assert await tracker.abort("semantic-msg:attempt-1") is True- assert await tracker.abort("semantic-msg:attempt-1") is False-- await tracker.register(- "semantic-msg:attempt-2",- 1,- on_complete=lambda: callback_calls.append("attempt-2"),- )-- assert await tracker.decrement("semantic-msg:attempt-1") is None- assert callback_calls == []- assert await tracker.decrement("semantic-msg:attempt-2") == 0- assert callback_calls == ["attempt-2"]diff --git a/tests/storage/test_memory_semantic_stall.py b/tests/storage/test_memory_semantic_stall.pyindex 099fa35f5..fb2000a7e 100644--- a/tests/storage/test_memory_semantic_stall.py+++ b/tests/storage/test_memory_semantic_stall.py@@ -7,6 +7,7 @@ Ensures that _process_memory_directory() error paths propagate exceptionsso that on_dequeue() always calls report_success() or report_error()."""+from types import SimpleNamespacefrom unittest.mock import AsyncMock, MagicMock, patchimport pytest@@ -206,7 +207,7 @@ async def test_memory_write_error_reports_error():side_effect=lambda uri, ctx=None: f"/local/acc1/{uri.removeprefix('viking://')}")- msg = _make_msg()+ msg = _make_msg(skip_vectorization=True)data = _build_data(msg)success_called = False@@ -234,6 +235,15 @@ async def test_memory_write_error_reports_error():"openviking.storage.queuefs.semantic_processor.resolve_telemetry",return_value=None,),+ patch(+ "openviking.storage.queuefs.semantic_processor.get_openviking_config",+ return_value=SimpleNamespace(+ semantic=SimpleNamespace(+ overview_max_chars=100_000,+ abstract_max_chars=10_000,+ )+ ),+ ),patch.object(processor,"_generate_single_file_summary",diff --git a/tests/storage/test_queue_manager.py b/tests/storage/test_queue_manager.pyindex 6621cffe4..30a65b3a8 100644--- a/tests/storage/test_queue_manager.py+++ b/tests/storage/test_queue_manager.py@@ -5,13 +5,14 @@from openviking.storage.queuefs.queue_manager import QueueManager-def test_external_parse_concurrent_uses_configured_value() -> None:- manager = QueueManager(agfs=object(), max_concurrent_external_parse=9)+def test_queue_concurrency_uses_separate_configured_values() -> None:+ manager = QueueManager(+ agfs=object(),+ max_concurrent_external_parse=9,+ max_concurrent_add_resource=7,+ max_concurrent_session_commit=5,+ )assert manager._max_concurrent_for_queue(manager.EXTERNAL_PARSE) == 9---def test_session_commit_concurrent_stays_at_four() -> None:- manager = QueueManager(agfs=object(), max_concurrent_external_parse=9)-- assert manager._max_concurrent_for_queue(manager.SESSION_COMMIT) == 4+ assert manager._max_concurrent_for_queue(manager.ADD_RESOURCE) == 7+ assert manager._max_concurrent_for_queue(manager.SESSION_COMMIT) == 5diff --git a/tests/storage/test_semantic_dag_incremental.py b/tests/storage/test_semantic_dag_incremental.pyindex 1d9e8ed73..67405030e 100644--- a/tests/storage/test_semantic_dag_incremental.py+++ b/tests/storage/test_semantic_dag_incremental.py@@ -2,7 +2,7 @@# SPDX-License-Identifier: AGPL-3.0import re-from unittest.mock import AsyncMock, MagicMock+from unittest.mock import MagicMockimport pytest@@ -11,20 +11,12 @@ from openviking.storage.queuefs.semantic_dag import SemanticDagExecutorfrom openviking_cli.session.user_id import UserIdentifier-class _FakeAgfs:- async def pathlock_acquire_exact_batch(self, _paths):- return {"lease_ref": "test"}-- async def pathlock_release(self, _lease):- return None--class _FakeVikingFS:def __init__(self, tree, file_contents):self._tree = {self._norm(k): v for k, v in tree.items()}self._file_contents = {self._norm(k): v for k, v in file_contents.items()}self.writes = []- self._async_agfs = _FakeAgfs()+ self._async_agfs = selfdef _norm(self, path):if "://" not in path:@@ -48,6 +40,12 @@ class _FakeVikingFS:self._file_contents[norm_path] = contentself.writes.append((norm_path, content))+ async def pathlock_acquire_exact_batch(self, paths):+ return {"paths": paths}++ async def pathlock_release(self, lease):+ return None+def _uri_to_path(self, uri, ctx=None):return uri.replace("viking://", "/local/acc1/")@@ -57,6 +55,7 @@ class _FakeProcessor:self._fs = viking_fsself.summarized_files = []self.sync_calls = []+ self.vectorized_files = []def _parse_overview_md(self, overview_content):results = {}@@ -82,6 +81,29 @@ class _FakeProcessor:def _normalize_overview_generation(self, overview):return overview, "abstract"+ async def _vectorize_single_file(+ self,+ parent_uri,+ context_type,+ file_path,+ summary_dict,+ ctx=None,+ use_summary=False,+ ingest_options=None,+ ):+ self.vectorized_files.append(file_path)++ async def _vectorize_directory(+ self,+ uri,+ context_type,+ abstract,+ overview,+ ctx=None,+ ingest_options=None,+ ):+ return None+async def _sync_topdown_recursive(self, root_uri, target_uri, ctx=None, file_change_status=None, lock=None):@@ -134,27 +156,15 @@ async def test_direct_incremental_update_uses_changes_without_temp_sync(monkeypatarget_uri=root_uri,changes={"modified": [f"{root_uri}/a.txt"]},)- add_vectorize_task = AsyncMock()- monkeypatch.setattr(executor, "_add_vectorize_task", add_vectorize_task)await executor.run(root_uri)assert processor.summarized_files == [f"{root_uri}/a.txt"]+ assert processor.vectorized_files == [f"{root_uri}/a.txt"]assert processor.sync_calls == []overview = fake_fs._file_contents[f"{root_uri}/.overview.md"]assert "- a.txt: summary" in overviewassert "- b.txt: old-b" in overview- changed_file_tasks = [- call.args[0]- for call in add_vectorize_task.await_args_list- if call.args[0].task_type == "file"- ]- assert len(changed_file_tasks) == 1- assert changed_file_tasks[0].file_path == f"{root_uri}/a.txt"- assert changed_file_tasks[0].summary_dict == {- "name": "a.txt",- "summary": "summary",- }if __name__ == "__main__":diff --git a/tests/storage/test_semantic_dag_skip_files.py b/tests/storage/test_semantic_dag_skip_files.pyindex 947a206ba..4a09b8041 100644--- a/tests/storage/test_semantic_dag_skip_files.py+++ b/tests/storage/test_semantic_dag_skip_files.py@@ -8,19 +8,11 @@ from openviking.storage.queuefs.semantic_dag import SemanticDagExecutorfrom openviking_cli.session.user_id import UserIdentifier-class _FakeAgfs:- async def pathlock_acquire_exact_batch(self, _paths):- return {"lease_ref": "test"}-- async def pathlock_release(self, _lease):- return None--class _FakeVikingFS:def __init__(self, tree):self._tree = treeself.writes = []- self._async_agfs = _FakeAgfs()+ self._async_agfs = selfasync def ls(self, uri, node_limit=None, ctx=None):return self._tree.get(uri, [])@@ -28,6 +20,12 @@ class _FakeVikingFS:async def write_file(self, path, content, ctx=None, lease_ref=None):self.writes.append((path, content))+ async def pathlock_acquire_exact_batch(self, paths):+ return {"paths": paths}++ async def pathlock_release(self, lease):+ return None+def _uri_to_path(self, uri, ctx=None):return uri.replace("viking://", "/local/acc1/")@@ -54,7 +52,6 @@ class _FakeProcessor:abstract,overview,ctx=None,- semantic_msg_id=None,ingest_options=None,):pass@@ -69,18 +66,12 @@ class _FakeProcessor:file_path,summary_dict,ctx=None,- semantic_msg_id=None,use_summary=False,ingest_options=None,):self.vectorized_files.append(file_path)-class _DummyTracker:- async def register(self, **_kwargs):- return None--@pytest.mark.asyncio@pytest.mark.parametrize("root_uri",@@ -100,10 +91,6 @@ async def test_messages_jsonl_excluded_from_summary(monkeypatch, root_uri):}fake_fs = _FakeVikingFS(tree)monkeypatch.setattr("openviking.storage.queuefs.semantic_dag.get_viking_fs", lambda: fake_fs)- monkeypatch.setattr(- "openviking.storage.queuefs.embedding_tracker.EmbeddingTaskTracker.get_instance",- lambda: _DummyTracker(),- )processor = _FakeProcessor()ctx = RequestContext(user=UserIdentifier("acc1", "user1"), role=Role.USER)@@ -142,10 +129,6 @@ async def test_messages_jsonl_excluded_in_subdirectory(monkeypatch, root_uri):}fake_fs = _FakeVikingFS(tree)monkeypatch.setattr("openviking.storage.queuefs.semantic_dag.get_viking_fs", lambda: fake_fs)- monkeypatch.setattr(- "openviking.storage.queuefs.embedding_tracker.EmbeddingTaskTracker.get_instance",- lambda: _DummyTracker(),- )processor = _FakeProcessor()ctx = RequestContext(user=UserIdentifier("acc1", "user1"), role=Role.USER)diff --git a/tests/storage/test_semantic_dag_stats.py b/tests/storage/test_semantic_dag_stats.pyindex b3619f762..d6f6d0a83 100644--- a/tests/storage/test_semantic_dag_stats.py+++ b/tests/storage/test_semantic_dag_stats.py@@ -6,7 +6,12 @@ import asyncioimport pytestfrom openviking.server.identity import RequestContext, Role-from openviking.service.task_work_index import TaskWorkIndex, TaskWorkRejected+from openviking.service.task_work_index import (+ TaskWorkIndex,+ TaskWorkRejected,+ bind_task_context,+ get_task_context,+)from openviking.storage.queuefs.named_queue import NamedQueuefrom openviking.storage.queuefs.semantic_dag import (DagStats,@@ -14,51 +19,56 @@ from openviking.storage.queuefs.semantic_dag import (SemanticDagExecutor,SemanticNodeScheduler,)+from openviking.telemetry import (+ OperationTelemetry,+ bind_telemetry,+ get_current_telemetry,+)from openviking_cli.session.user_id import UserIdentifier-class _FakeAgfs:- async def pathlock_acquire_exact_batch(self, _paths):- return {"lease_ref": "test"}-- async def pathlock_release(self, _lease):- return None--class _FakeVikingFS:def __init__(self, tree):self._tree = treeself.writes = []- self._async_agfs = _FakeAgfs()+ self._async_agfs = selfasync def ls(self, uri, node_limit=None, ctx=None):del node_limitreturn self._tree.get(uri, [])- async def write_file(- self,- path,- content,- ctx=None,- lease_ref=None,- lock_handle=None,- ):- del ctx, lease_ref, lock_handle+ async def write_file(self, path, content, ctx=None, lease_ref=None):self.writes.append((path, content))+ async def pathlock_acquire_exact_batch(self, paths):+ return {"paths": paths}++ async def pathlock_release(self, lease):+ return None+def _uri_to_path(self, uri, ctx=None):return uri.replace("viking://", "/local/acc1/")class _FakeProcessor:- def __init__(self):+ def __init__(self, verify_streaming=False):self.vectorized_dirs = []self.vectorized_files = []+ self.vectorized_contexts = {}+ self.verify_streaming = verify_streamingasync def _generate_single_file_summary(self, file_path, llm_sem=None, ctx=None):- return {"name": file_path.split("/")[-1], "summary": "summary"}+ result = {"name": file_path.split("/")[-1], "summary": "summary"}+ if self.verify_streaming:+ result["content"] = "x" * 100_000+ return resultasync def _generate_overview(self, dir_uri, file_summaries, children_abstracts):+ if self.verify_streaming:+ assert all("content" not in item for item in file_summaries)+ assert all(+ f"{dir_uri}/{item['name']}" in self.vectorized_files for item in file_summaries+ )return "overview"def _normalize_overview_generation(self, overview):@@ -71,7 +81,6 @@ class _FakeProcessor:abstract,overview,ctx=None,- semantic_msg_id=None,ingest_options=None,):self.vectorized_dirs.append(uri)@@ -83,11 +92,17 @@ class _FakeProcessor:file_path,summary_dict,ctx=None,- semantic_msg_id=None,use_summary=False,ingest_options=None,):+ if self.verify_streaming:+ assert summary_dict["content"]self.vectorized_files.append(file_path)+ task_context = get_task_context()+ self.vectorized_contexts[file_path] = (+ task_context.task_id if task_context is not None else None,+ get_current_telemetry().telemetry_id,+ )async def _vectorize_directory_simple(self, uri, context_type, abstract, overview, ctx=None):await self._vectorize_directory(uri, context_type, abstract, overview, ctx=ctx)@@ -109,168 +124,6 @@ class _TrackingProcessor(_FakeProcessor):self.active_summaries -= 1-class _DummyTracker:- def __init__(self):- self.register_calls = []-- async def register(self, **_kwargs):- self.register_calls.append(_kwargs)- return None---class _CompletingTracker:- def __init__(self):- self.remaining = 0- self.total = 0- self.on_complete = None- self.completed = False- self.active_id = None- self.aborted_ids = []-- async def register(self, *, semantic_msg_id, total_count, on_complete, **_kwargs):- self.active_id = semantic_msg_id- self.remaining = total_count- self.total = total_count- self.on_complete = on_complete-- async def decrement(self, semantic_msg_id):- if semantic_msg_id != self.active_id:- return None- self.remaining -= 1- if self.remaining == 0:- self.completed = True- self.active_id = None- await self.on_complete()- return self.remaining-- async def abort(self, semantic_msg_id):- if semantic_msg_id != self.active_id:- return False- self.aborted_ids.append(semantic_msg_id)- self.active_id = None- return True---class _FailingVectorizeProcessor(_FakeProcessor):- def __init__(self, tracker):- super().__init__()- self._tracker = tracker-- async def _vectorize_single_file(- self,- parent_uri,- context_type,- file_path,- summary_dict,- ctx=None,- semantic_msg_id=None,- use_summary=False,- ingest_options=None,- ):- self.vectorized_files.append(file_path)- try:- raise RuntimeError("embedding enqueue unavailable")- finally:- await self._tracker.decrement(semantic_msg_id)-- async def _vectorize_directory(- self,- uri,- context_type,- abstract,- overview,- ctx=None,- semantic_msg_id=None,- ingest_options=None,- ):- self.vectorized_dirs.append(uri)- await self._tracker.decrement(semantic_msg_id)- await self._tracker.decrement(semantic_msg_id)---class _StalledEmbeddingProcessor(_FailingVectorizeProcessor):- async def _vectorize_directory(- self,- uri,- context_type,- abstract,- overview,- ctx=None,- semantic_msg_id=None,- ingest_options=None,- ):- self.vectorized_dirs.append(uri)- # Both directory messages were accepted, but the embedding worker is- # unavailable and therefore never decrements their tracker slots.---async def test_semantic_dag_propagates_vectorization_failure_after_tracker_drains(monkeypatch):- root_uri = "viking://resources/root"- tree = {root_uri: [{"name": "a.txt", "isDir": False}]}- fake_fs = _FakeVikingFS(tree)- tracker = _CompletingTracker()- monkeypatch.setattr("openviking.storage.queuefs.semantic_dag.get_viking_fs", lambda: fake_fs)- monkeypatch.setattr(- "openviking.storage.queuefs.embedding_tracker.EmbeddingTaskTracker.get_instance",- lambda: tracker,- )-- processor = _FailingVectorizeProcessor(tracker)- ctx = RequestContext(user=UserIdentifier("acc1", "user1"), role=Role.USER)- executor = SemanticDagExecutor(- processor=processor,- context_type="resource",- max_concurrent_llm=2,- ctx=ctx,- semantic_msg_id="semantic-root",- )-- with pytest.raises(RuntimeError, match="embedding enqueue unavailable"):- await executor.run(root_uri)-- assert processor.vectorized_files == [f"{root_uri}/a.txt"]- assert processor.vectorized_dirs == [root_uri]- assert tracker.total == 3- assert tracker.remaining == 0- assert tracker.completed is True---async def test_semantic_dag_aborts_stalled_embedding_attempt_after_dispatch_failure(monkeypatch):- root_uri = "viking://resources/root"- tree = {root_uri: [{"name": "a.txt", "isDir": False}]}- fake_fs = _FakeVikingFS(tree)- tracker = _CompletingTracker()- monkeypatch.setattr("openviking.storage.queuefs.semantic_dag.get_viking_fs", lambda: fake_fs)- monkeypatch.setattr(- "openviking.storage.queuefs.embedding_tracker.EmbeddingTaskTracker.get_instance",- lambda: tracker,- )-- processor = _StalledEmbeddingProcessor(tracker)- ctx = RequestContext(user=UserIdentifier("acc1", "user1"), role=Role.USER)- executor = SemanticDagExecutor(- processor=processor,- context_type="resource",- max_concurrent_llm=2,- ctx=ctx,- semantic_msg_id="semantic-root",- )-- with pytest.raises(RuntimeError, match="embedding enqueue unavailable"):- await asyncio.wait_for(executor.run(root_uri), timeout=0.5)-- assert processor.vectorized_files == [f"{root_uri}/a.txt"]- assert processor.vectorized_dirs == [root_uri]- assert tracker.remaining == 2- assert tracker.completed is False- assert len(tracker.aborted_ids) == 1- aborted_id = tracker.aborted_ids[0]- assert aborted_id.startswith("semantic-root:")- assert await tracker.decrement(aborted_id) is None--class _ScheduledExecutor:def __init__(self, run) -> None:self.closed = False@@ -305,12 +158,8 @@ async def test_semantic_dag_stats_collects_nodes(monkeypatch):}fake_fs = _FakeVikingFS(tree)monkeypatch.setattr("openviking.storage.queuefs.semantic_dag.get_viking_fs", lambda: fake_fs)- monkeypatch.setattr(- "openviking.storage.queuefs.embedding_tracker.EmbeddingTaskTracker.get_instance",- lambda: _DummyTracker(),- )- processor = _FakeProcessor()+ processor = _FakeProcessor(verify_streaming=True)ctx = RequestContext(user=UserIdentifier("acc1", "user1"), role=Role.USER)executor = SemanticDagExecutor(processor=processor,@@ -345,10 +194,6 @@ async def test_semantic_dag_bounds_active_node_work(monkeypatch):}fake_fs = _FakeVikingFS(tree)monkeypatch.setattr("openviking.storage.queuefs.semantic_dag.get_viking_fs", lambda: fake_fs)- monkeypatch.setattr(- "openviking.storage.queuefs.embedding_tracker.EmbeddingTaskTracker.get_instance",- lambda: _DummyTracker(),- )processor = _TrackingProcessor()ctx = RequestContext(user=UserIdentifier("acc1", "user1"), role=Role.USER)@@ -386,33 +231,43 @@ async def test_semantic_dag_shares_node_scheduler_across_roots(monkeypatch):}fake_fs = _FakeVikingFS(tree)monkeypatch.setattr("openviking.storage.queuefs.semantic_dag.get_viking_fs", lambda: fake_fs)- monkeypatch.setattr(- "openviking.storage.queuefs.embedding_tracker.EmbeddingTaskTracker.get_instance",- lambda: _DummyTracker(),- )processor = _TrackingProcessor()ctx = RequestContext(user=UserIdentifier("acc1", "user1"), role=Role.USER)- executor_a = SemanticDagExecutor(- processor=processor,- context_type="resource",- max_concurrent_llm=4,- ctx=ctx,- skip_vectorization=True,- )- executor_b = SemanticDagExecutor(- processor=processor,- context_type="resource",- max_concurrent_llm=4,- ctx=ctx,- skip_vectorization=True,- )+ telemetry_a = OperationTelemetry("semantic-a")+ telemetry_b = OperationTelemetry("semantic-b")+ with (+ bind_task_context("task-a", "acc1", "user1"),+ bind_telemetry(telemetry_a),+ ):+ executor_a = SemanticDagExecutor(+ processor=processor,+ context_type="resource",+ max_concurrent_llm=1,+ ctx=ctx,+ )+ with (+ bind_task_context("task-b", "acc1", "user1"),+ bind_telemetry(telemetry_b),+ ):+ executor_b = SemanticDagExecutor(+ processor=processor,+ context_type="resource",+ max_concurrent_llm=1,+ ctx=ctx,+ )await asyncio.gather(executor_a.run(root_a), executor_b.run(root_b))- assert processor.max_active_summaries <= 4+ assert processor.max_active_summaries == 1assert executor_a.get_stats().done_nodes == 21assert executor_b.get_stats().done_nodes == 21+ assert {+ processor.vectorized_contexts[f"{root_a}/a-{idx}.txt"] for idx in range(20)+ } == {("task-a", telemetry_a.telemetry_id)}+ assert {+ processor.vectorized_contexts[f"{root_b}/b-{idx}.txt"] for idx in range(20)+ } == {("task-b", telemetry_b.telemetry_id)}@pytest.mark.asyncio@@ -476,12 +331,7 @@ async def test_semantic_dag_skip_vectorization_does_not_schedule_tasks(monkeypat],}fake_fs = _FakeVikingFS(tree)- tracker = _DummyTracker()monkeypatch.setattr("openviking.storage.queuefs.semantic_dag.get_viking_fs", lambda: fake_fs)- monkeypatch.setattr(- "openviking.storage.queuefs.embedding_tracker.EmbeddingTaskTracker.get_instance",- lambda: tracker,- )processor = _FakeProcessor()ctx = RequestContext(user=UserIdentifier("acc1", "user1"), role=Role.USER)@@ -503,8 +353,5 @@ async def test_semantic_dag_skip_vectorization_does_not_schedule_tasks(monkeypat]assert processor.vectorized_dirs == []assert processor.vectorized_files == []- assert tracker.register_calls == []--if __name__ == "__main__":pytest.main([__file__])diff --git a/tests/storage/test_semantic_queue_memory_dedupe.py b/tests/storage/test_semantic_queue_memory_dedupe.pyindex c69c4320b..508ff21a2 100644--- a/tests/storage/test_semantic_queue_memory_dedupe.py+++ b/tests/storage/test_semantic_queue_memory_dedupe.py@@ -125,8 +125,8 @@ class _FakeVikingFS:del ctxreturn f"/fake/{uri.replace('://', '/').strip('/')}"- async def write_file(self, uri, content, ctx=None):- del ctx+ async def write_file(self, uri, content, ctx=None, lease_ref=None):+ del ctx, lease_refself.writes.append((uri, content))@@ -246,10 +246,12 @@ async def test_memory_directory_vectorizes_changed_files_with_generated_summary(async def generate_file_summary(file_path, llm_sem=None, ctx=None):del llm_sem, ctxname = file_path.rsplit("/", 1)[-1]- return {"name": name, "summary": f"summary:{name}"}+ return {"name": name, "summary": f"summary:{name}", "content": "raw content"}async def generate_overview(dir_uri, file_summaries, children_abstracts, llm_sem=None):- del dir_uri, file_summaries, children_abstracts, llm_sem+ del dir_uri, children_abstracts, llm_sem+ assert len(captured_file_vectorize) == 1+ assert all("content" not in summary for summary in file_summaries)return "overview"async def write_semantics(**kwargs):@@ -292,6 +294,7 @@ async def test_memory_directory_vectorizes_changed_files_with_generated_summary(assert captured_file_vectorize[0]["summary_dict"] == {"name": "first.md","summary": "summary:first.md",+ "content": "raw content",}assert captured_file_vectorize[0]["preserve_existing_created_at"] is Trueassert len(captured_directory_vectorize) == 1diff --git a/tests/test_config_loader.py b/tests/test_config_loader.pyindex c319c64db..9cf26591c 100644--- a/tests/test_config_loader.py+++ b/tests/test_config_loader.py@@ -121,24 +121,34 @@ class TestRequireConfig:require_config(None, "TEST_MISSING_ENV", "nonexistent_file.conf", "test")-def test_external_parse_worker_max_concurrent_defaults_to_four():+def test_queue_worker_concurrency_defaults_to_four():config = OpenVikingConfig.from_dict({})assert config.queue_workers.external_parse.max_concurrent == 4+ assert config.queue_workers.add_resource.max_concurrent == 4+ assert config.queue_workers.session_commit.max_concurrent == 4-def test_external_parse_worker_max_concurrent_accepts_positive_value():+def test_queue_worker_concurrency_accepts_separate_values():config = OpenVikingConfig.from_dict(- {"queue_workers": {"external_parse": {"max_concurrent": 9}}}+ {+ "queue_workers": {+ "external_parse": {"max_concurrent": 9},+ "add_resource": {"max_concurrent": 7},+ "session_commit": {"max_concurrent": 5},+ }+ })assert config.queue_workers.external_parse.max_concurrent == 9+ assert config.queue_workers.add_resource.max_concurrent == 7+ assert config.queue_workers.session_commit.max_concurrent == 5@pytest.mark.parametrize("value", [0, -1])-def test_external_parse_worker_max_concurrent_rejects_non_positive_value(value):+def test_queue_worker_concurrency_rejects_non_positive_value(value):with pytest.raises(ValueError) as exc_info:- QueueWorkersConfig(external_parse={"max_concurrent": value})+ QueueWorkersConfig(add_resource={"max_concurrent": value})assert exc_info.value.errors()[0]["type"] == "greater_than"diff --git a/tests/unit/service/test_core_consistency.py b/tests/unit/service/test_core_consistency.pyindex bf616c0c6..9a43c23fe 100644--- a/tests/unit/service/test_core_consistency.py+++ b/tests/unit/service/test_core_consistency.py@@ -26,10 +26,12 @@ def test_service_passes_queue_worker_concurrency_to_storage(monkeypatch) -> Nonedimension=1024,get_embedder=lambda: embedder,),- vlm=SimpleNamespace(max_concurrent=64),+ vlm=SimpleNamespace(max_concurrent=32),parser_api=SimpleNamespace(),queue_workers=SimpleNamespace(external_parse=SimpleNamespace(max_concurrent=9),+ add_resource=SimpleNamespace(max_concurrent=7),+ session_commit=SimpleNamespace(max_concurrent=5),),git=object(),)@@ -53,6 +55,8 @@ def test_service_passes_queue_worker_concurrency_to_storage(monkeypatch) -> NoneOpenVikingService()assert storage_calls[0][1]["max_concurrent_external_parse"] == 9+ assert storage_calls[0][1]["max_concurrent_add_resource"] == 7+ assert storage_calls[0][1]["max_concurrent_session_commit"] == 5def _service_with_fs(stat_result: dict) -> tuple[OpenVikingService, AsyncMock]:diff --git a/tests/unit/test_vectorize_file_strategy.py b/tests/unit/test_vectorize_file_strategy.pyindex 482972585..0e8c5716e 100644--- a/tests/unit/test_vectorize_file_strategy.py+++ b/tests/unit/test_vectorize_file_strategy.py@@ -1,5 +1,4 @@import types-from unittest.mock import AsyncMockimport pytest@@ -18,6 +17,7 @@ class DummyQueue:async def enqueue(self, msg):self.items.append(msg)+ return "queue-message-id"class DummyQueueWithId(DummyQueue):@@ -200,7 +200,6 @@ async def test_vectorize_file_registers_request_wait_with_embedding_msg_id(monkesummary_dict={"name": "test.md", "summary": ""},parent_uri="viking://user/default/resources",ctx=DummyReq(),- register_request_wait=True,)assert len(queue.items) == 1@@ -241,7 +240,6 @@ async def test_vectorize_file_marks_registered_wait_root_failed_when_enqueue_raisummary_dict={"name": "test.md", "summary": ""},parent_uri="viking://user/default/resources",ctx=DummyReq(),- register_request_wait=True,)assert len(queue.items) == 1@@ -273,12 +271,10 @@ async def test_vectorize_file_propagates_enqueue_failure(monkeypatch):@pytest.mark.asyncio-async def test_vectorize_directory_propagates_enqueue_failures_and_drains_tracker(monkeypatch):- decrement = AsyncMock()+async def test_vectorize_directory_propagates_enqueue_failures(monkeypatch):monkeypatch.setattr(embedding_utils, "get_queue_manager", lambda: DummyQueueManager(FailingQueue()))- monkeypatch.setattr(embedding_utils, "_decrement_embedding_tracker", decrement)with pytest.raises(RuntimeError, match="queue unavailable"):await embedding_utils.vectorize_directory_meta(@@ -286,11 +282,8 @@ async def test_vectorize_directory_propagates_enqueue_failures_and_drains_trackeabstract="abstract",overview="overview",ctx=DummyReq(),- semantic_msg_id="semantic-root",)- decrement.assert_awaited_once_with("semantic-root", 2)-@pytest.mark.asyncioasync def test_vectorize_unknown_text_file_embeds_summary_without_reading_content(monkeypatch):diff --git a/tests/utils/test_resource_processor_processing_mode.py b/tests/utils/test_resource_processor_processing_mode.pyindex 58a92bcd8..13c36224f 100644--- a/tests/utils/test_resource_processor_processing_mode.py+++ b/tests/utils/test_resource_processor_processing_mode.py@@ -6,8 +6,8 @@ from unittest.mock import AsyncMock, Mockimport pytest-from openviking.utils.ingest_options import IngestOptionsfrom openviking.server.identity import RequestContext, Role+from openviking.utils.ingest_options import IngestOptionsfrom openviking.utils.resource_processor import ResourceProcessorfrom openviking_cli.session.user_id import UserIdentifier@@ -260,7 +260,6 @@ async def test_vectors_only_persists_tree_and_vectorizes_files_only(monkeypatch,search_tags=["team=search"],search_tag_mode="append",)- assert vectorize_file.await_args.kwargs["register_request_wait"] is Trueprocessor._delete_resource_semantic_markers.assert_not_awaited()processor._delete_resource_semantic_vectors.assert_not_awaited()viking_fs._async_agfs.pathlock_release.assert_awaited_once_with(lock)