Skip to content

Self-evolving Context Database for AI Agents. Unify Agent Memory, Knowledge RAG and Skills.

perf(queue): 流式调度语义向量化任务 (#3636)

Qin Haojieb877ababa5ed
diff --git a/docs/en/configuration/01-server.md b/docs/en/configuration/01-server.md
index ebe69d044..a137b97fa 100644
--- a/docs/en/configuration/01-server.md
+++ b/docs/en/configuration/01-server.md
@@ -208,6 +208,18 @@ Remote backends also require endpoint, bucket/collection, credentials, and timeo
This setting controls queue-job concurrency. It is separate from `vlm.media.max_concurrent`, which limits audio/video VLM calls, and does not limit individual Understanding API HTTP requests.
+### `queue_workers.add_resource`
+
+| Field | Type | Default | Description |
+|---|---|---:|---|
+| `max_concurrent` | integer | `4` | Number of complete AddResource jobs consumed concurrently; must be greater than `0`; requires a server restart after changes |
+
+### `queue_workers.session_commit`
+
+| Field | Type | Default | Description |
+|---|---|---:|---|
+| `max_concurrent` | integer | `4` | Number of SessionCommit jobs consumed concurrently; must be greater than `0`; requires a server restart after changes |
+
## HTTP Server Settings
```json
diff --git a/docs/en/guides/01-configuration.md b/docs/en/guides/01-configuration.md
index 3d5535e1e..53cea7820 100644
--- a/docs/en/guides/01-configuration.md
+++ b/docs/en/guides/01-configuration.md
@@ -631,7 +631,7 @@ Vision Language Model for semantic extraction (L0/L1 generation).
| `model` | str | Model name |
| `api_base` | str | API endpoint (optional) |
| `thinking` | bool | Enable thinking mode for VolcEngine models (default: `false`) |
-| `max_concurrent` | int | Maximum concurrent semantic LLM calls (default: `64`) |
+| `max_concurrent` | int | Maximum concurrent semantic LLM calls (default: `32`) |
| `max_retries` | int | Maximum retry attempts for transient VLM provider errors (default: `3`; `0` disables retry) |
| `credentials` | array | Ordered VLM credential/model list, with index 0 having the highest priority. Each item can override `provider`, `model`, `api_key`, `api_base`, `api_version`, `extra_headers`, `extra_request_body`, and `stream` |
| `failback_timeout_seconds` | float | Time threshold for attempting a step back toward a higher-priority credential after failover (default: `600`) |
@@ -1778,7 +1778,7 @@ For detailed encryption explanations, see [Data Encryption](../concepts/10-encry
"model": "string",
"api_base": "string",
"thinking": false,
- "max_concurrent": 64,
+ "max_concurrent": 32,
"max_retries": 3,
"extra_headers": {},
"extra_request_body": {},
diff --git a/docs/zh/configuration/01-server.md b/docs/zh/configuration/01-server.md
index 3b9ebf163..c4a3192e4 100644
--- a/docs/zh/configuration/01-server.md
+++ b/docs/zh/configuration/01-server.md
@@ -208,6 +208,18 @@ Search 和 Find 请求的默认 `limit` 为 `10`,可以在每次 API 或 SDK
该配置控制队列作业并发,不等同于 `vlm.media.max_concurrent` 的音视频 VLM 调用并发,也不限制 Understanding API 的单独 HTTP 请求数。
+### `queue_workers.add_resource`
+
+| 字段 | 类型 | 默认值 | 说明 |
+|---|---|---:|---|
+| `max_concurrent` | integer | `4` | 同时消费的完整 AddResource 作业数,必须大于 `0`;修改后需重启服务 |
+
+### `queue_workers.session_commit`
+
+| 字段 | 类型 | 默认值 | 说明 |
+|---|---|---:|---|
+| `max_concurrent` | integer | `4` | 同时消费的 SessionCommit 作业数,必须大于 `0`;修改后需重启服务 |
+
## HTTP 服务配置
```json
diff --git a/docs/zh/guides/01-configuration.md b/docs/zh/guides/01-configuration.md
index 00a155368..ea2ddc609 100644
--- a/docs/zh/guides/01-configuration.md
+++ b/docs/zh/guides/01-configuration.md
@@ -600,7 +600,7 @@ provider,并设置 `storage.vectordb.sparse_weight > 0`。自托管模型的
| `model` | str | 模型名称 |
| `api_base` | str | API 端点(可选) |
| `thinking` | bool | 启用思考模式(仅对部分火山模型生效,默认:`false`) |
-| `max_concurrent` | int | 语义处理阶段 LLM 最大并发调用数(默认:`64`) |
+| `max_concurrent` | int | 语义处理阶段 LLM 最大并发调用数(默认:`32`) |
| `max_retries` | int | VLM provider 瞬时错误的最大重试次数(默认:`3`;`0` 表示禁用重试) |
| `credentials` | array | 有序 VLM 凭据/模型列表,索引 0 优先级最高。每项可单独覆盖 `provider`、`model`、`api_key`、`api_base`、`api_version`、`extra_headers`、`extra_request_body` 和 `stream` |
| `failback_timeout_seconds` | float | 切换到低优先级 credential 后,尝试逐级切回的时间阈值(默认:`600`) |
@@ -1859,7 +1859,7 @@ Task 记录文件位于所属账号的系统目录:
"model": "string",
"api_base": "string",
"thinking": false,
- "max_concurrent": 64,
+ "max_concurrent": 32,
"max_retries": 3,
"extra_headers": {},
"extra_request_body": {},
diff --git a/examples/ov.conf.example b/examples/ov.conf.example
index 3b654718e..9056a7286 100644
--- a/examples/ov.conf.example
+++ b/examples/ov.conf.example
@@ -82,7 +82,11 @@
},
},
},
- "queue_workers": {"external_parse": {"max_concurrent": 4}},
+ "queue_workers": {
+ "external_parse": {"max_concurrent": 4},
+ "add_resource": {"max_concurrent": 4},
+ "session_commit": {"max_concurrent": 4},
+ },
"embedding": {
"dense": {
"model": "doubao-embedding-vision-251215",
diff --git a/openviking/service/core.py b/openviking/service/core.py
index 86b9296aa..3650b90fd 100644
--- a/openviking/service/core.py
+++ b/openviking/service/core.py
@@ -132,6 +132,8 @@ class OpenVikingService:
max_concurrent_embedding=config.embedding.max_concurrent,
max_concurrent_semantic=config.vlm.max_concurrent,
max_concurrent_external_parse=config.queue_workers.external_parse.max_concurrent,
+ max_concurrent_add_resource=config.queue_workers.add_resource.max_concurrent,
+ max_concurrent_session_commit=config.queue_workers.session_commit.max_concurrent,
binding_config=binding_config,
git_config=config.git,
)
@@ -146,8 +148,10 @@ class OpenVikingService:
self,
config: StorageConfig,
max_concurrent_embedding: int = 10,
- max_concurrent_semantic: int = 64,
+ max_concurrent_semantic: int = 32,
max_concurrent_external_parse: int = 4,
+ max_concurrent_add_resource: int = 4,
+ max_concurrent_session_commit: int = 4,
binding_config: Any = None,
*,
git_config: Optional[GitConfig] = None,
@@ -169,6 +173,8 @@ class OpenVikingService:
max_concurrent_embedding=max_concurrent_embedding,
max_concurrent_semantic=max_concurrent_semantic,
max_concurrent_external_parse=max_concurrent_external_parse,
+ max_concurrent_add_resource=max_concurrent_add_resource,
+ max_concurrent_session_commit=max_concurrent_session_commit,
)
else:
logger.warning("RAGFS client not initialized, skipping queue manager")
@@ -310,6 +316,12 @@ class OpenVikingService:
max_concurrent_external_parse=(
self._config.queue_workers.external_parse.max_concurrent
),
+ max_concurrent_add_resource=(
+ self._config.queue_workers.add_resource.max_concurrent
+ ),
+ max_concurrent_session_commit=(
+ self._config.queue_workers.session_commit.max_concurrent
+ ),
binding_config=self._build_ragfs_binding_config(),
git_config=self._config.git,
)
diff --git a/openviking/storage/collection_schemas.py b/openviking/storage/collection_schemas.py
index f9145bca7..77b48caa6 100644
--- a/openviking/storage/collection_schemas.py
+++ b/openviking/storage/collection_schemas.py
@@ -870,21 +870,13 @@ class TextEmbeddingHandler(DequeueHandlerBase):
finally:
if embedding_msg is not None and request_failed_message is not None:
self._record_request_failure(embedding_msg, request_failed_message)
- if embedding_msg and embedding_msg.semantic_msg_id:
- from openviking.storage.queuefs.embedding_tracker import EmbeddingTaskTracker
-
- tracker = EmbeddingTaskTracker.get_instance()
- try:
- await tracker.decrement(embedding_msg.semantic_msg_id)
- except Exception as tracker_err:
- logger.warning(f"Failed to decrement embedding tracker: {tracker_err}")
if report_error_args is not None:
self.report_error(*report_error_args)
elif report_success:
self.report_success()
async def on_cancelled(self, data: Optional[Dict[str, Any]]) -> Optional[Dict[str, Any]]:
- """Settle semantic joins when a queued embedding is cancelled."""
+ """Settle request-scoped waiting when a queued embedding is cancelled."""
embedding_msg: Optional[EmbeddingMsg] = None
try:
if data:
@@ -898,25 +890,20 @@ class TextEmbeddingHandler(DequeueHandlerBase):
if embedding_msg is not None:
self._record_request_success(embedding_msg)
- if embedding_msg.semantic_msg_id:
- from openviking.storage.queuefs.embedding_tracker import EmbeddingTaskTracker
-
- await EmbeddingTaskTracker.get_instance().decrement(embedding_msg.semantic_msg_id)
self.report_success()
return None
@staticmethod
def _record_request_success(embedding_msg: EmbeddingMsg) -> None:
- tracker = get_request_wait_tracker()
- if embedding_msg.semantic_msg_id:
- tracker.record_embedding_processed(embedding_msg.telemetry_id)
- else:
- tracker.mark_embedding_done(embedding_msg.telemetry_id, embedding_msg.id)
+ get_request_wait_tracker().mark_embedding_done(
+ embedding_msg.telemetry_id,
+ embedding_msg.id,
+ )
@staticmethod
def _record_request_failure(embedding_msg: EmbeddingMsg, message: str) -> None:
- tracker = get_request_wait_tracker()
- if embedding_msg.semantic_msg_id:
- tracker.record_embedding_error(embedding_msg.telemetry_id, message)
- else:
- tracker.mark_embedding_failed(embedding_msg.telemetry_id, embedding_msg.id, message)
+ get_request_wait_tracker().mark_embedding_failed(
+ embedding_msg.telemetry_id,
+ embedding_msg.id,
+ message,
+ )
diff --git a/openviking/storage/content_write.py b/openviking/storage/content_write.py
index c177d0ce2..158d4b0c9 100644
--- a/openviking/storage/content_write.py
+++ b/openviking/storage/content_write.py
@@ -829,7 +829,6 @@ class ContentWriteCoordinator:
parent_uri=parent.uri,
context_type=context_type,
ctx=ctx,
- register_request_wait=True,
)
def _validate_mode(self, mode: str) -> None:
diff --git a/openviking/storage/queuefs/__init__.py b/openviking/storage/queuefs/__init__.py
index 7302db2df..10a4d0ffa 100644
--- a/openviking/storage/queuefs/__init__.py
+++ b/openviking/storage/queuefs/__init__.py
@@ -2,7 +2,6 @@
# SPDX-License-Identifier: AGPL-3.0
from .embedding_msg import EmbeddingMsg
from .embedding_queue import EmbeddingQueue
-from .embedding_tracker import EmbeddingTaskTracker
from .named_queue import NamedQueue, QueueError, QueueStatus
from .queue_manager import QueueManager, get_queue_manager, init_queue_manager
from .semantic_dag import SemanticDagExecutor
@@ -20,7 +19,6 @@ __all__ = [
"QueueError",
"EmbeddingQueue",
"EmbeddingMsg",
- "EmbeddingTaskTracker",
"SemanticQueue",
"SemanticDagExecutor",
"SemanticMsg",
diff --git a/openviking/storage/queuefs/embedding_msg.py b/openviking/storage/queuefs/embedding_msg.py
index e8385c00d..4e529318e 100644
--- a/openviking/storage/queuefs/embedding_msg.py
+++ b/openviking/storage/queuefs/embedding_msg.py
@@ -2,7 +2,7 @@
# SPDX-License-Identifier: AGPL-3.0
import json
from dataclasses import asdict, dataclass, field
-from typing import Any, Dict, List, Optional, Union
+from typing import Any, Dict, List, Union
from uuid import uuid4
@@ -12,20 +12,17 @@ class EmbeddingMsg:
context_data: Dict[str, Any]
id: str = field(default_factory=lambda: str(uuid4()))
telemetry_id: str = ""
- semantic_msg_id: Optional[str] = None
def __init__(
self,
message: Union[str, List[Dict[str, Any]]],
context_data: Dict[str, Any],
telemetry_id: str = "",
- semantic_msg_id: Optional[str] = None,
):
self.id = str(uuid4())
self.message = message
self.context_data = context_data
self.telemetry_id = telemetry_id
- self.semantic_msg_id = semantic_msg_id
def to_dict(self) -> Dict[str, Any]:
"""Convert embedding message to dictionary format."""
@@ -42,7 +39,6 @@ class EmbeddingMsg:
message=data["message"],
context_data=data["context_data"],
telemetry_id=data.get("telemetry_id", ""),
- semantic_msg_id=data.get("semantic_msg_id"),
)
obj.id = data.get("id", obj.id)
return obj
diff --git a/openviking/storage/queuefs/embedding_tracker.py b/openviking/storage/queuefs/embedding_tracker.py
deleted file mode 100644
index a993184c2..000000000
--- a/openviking/storage/queuefs/embedding_tracker.py
+++ /dev/null
@@ -1,226 +0,0 @@
-# Copyright (c) 2026 Beijing Volcano Engine Technology Co., Ltd.
-# SPDX-License-Identifier: AGPL-3.0
-"""Embedding Task Tracker for tracking embedding task completion status."""
-
-import asyncio
-import inspect
-import threading
-from dataclasses import dataclass
-from typing import Any, Callable, Dict, Optional
-
-from openviking_cli.utils.logger import get_logger
-
-logger = get_logger(__name__)
-
-
-@dataclass
-class _EmbeddingTaskRecord:
- """Coordinator state for a single semantic message."""
-
- remaining: int
- total: int
- on_complete: Optional[Callable[[], Any]]
- metadata: Dict[str, Any]
- owner_loop: Optional[asyncio.AbstractEventLoop]
-
-
-class EmbeddingTaskTracker:
- """Track embedding task completion status for each SemanticMsg.
-
- This tracker maintains a process-global registry of embedding tasks associated
- with each SemanticMsg. Because semantic and embedding queues run on separate
- worker threads with distinct event loops, its internal state must be guarded
- by thread-safe primitives rather than loop-bound asyncio locks.
-
- When all embedding tasks for a SemanticMsg are completed, it triggers the
- registered callback and removes the entry.
- """
-
- _instance: Optional["EmbeddingTaskTracker"] = None
- _initialized: bool = False
-
- def __new__(cls) -> "EmbeddingTaskTracker":
- if cls._instance is None:
- cls._instance = super().__new__(cls)
- return cls._instance
-
- def __init__(self):
- if self._initialized:
- return
- self._lock = threading.Lock()
- self._tasks: Dict[str, _EmbeddingTaskRecord] = {}
- self._initialized = True
-
- @staticmethod
- async def _await_callback_result(result: Any) -> None:
- """Await callback results when they are async."""
- if inspect.isawaitable(result):
- await result
-
- async def _execute_callback(self, on_complete: Callable[[], Any]) -> None:
- """Invoke a completion callback and await async results."""
- await self._await_callback_result(on_complete())
-
- async def _run_on_complete(
- self,
- semantic_msg_id: str,
- record: _EmbeddingTaskRecord,
- ) -> None:
- """Execute the completion callback on the loop that registered it."""
- on_complete = record.on_complete
- owner_loop = record.owner_loop
- if on_complete is None:
- return
-
- try:
- current_loop = asyncio.get_running_loop()
- except RuntimeError:
- current_loop = None
-
- owner_loop_running = bool(owner_loop and owner_loop.is_running())
- owner_loop_available = bool(
- owner_loop and not owner_loop.is_closed() and owner_loop_running
- )
-
- try:
- if owner_loop and owner_loop is not current_loop:
- if not owner_loop_available:
- logger.warning(
- "Owner loop unavailable before completion callback for %s; "
- "running callback in current loop",
- semantic_msg_id,
- )
- else:
- try:
- fut = asyncio.run_coroutine_threadsafe(
- self._execute_callback(on_complete),
- owner_loop,
- )
- except RuntimeError:
- logger.warning(
- "Owner loop stopped before completion callback for %s; "
- "running callback in current loop",
- semantic_msg_id,
- )
- else:
- await asyncio.wrap_future(fut)
- return
-
- await self._execute_callback(on_complete)
- except Exception as e:
- logger.error(
- f"Error in completion callback for {semantic_msg_id}: {e}",
- exc_info=True,
- )
-
- @classmethod
- def get_instance(cls) -> "EmbeddingTaskTracker":
- """Get the singleton instance of EmbeddingTaskTracker."""
- if cls._instance is None:
- cls._instance = cls()
- return cls._instance
-
- async def register(
- self,
- semantic_msg_id: str,
- total_count: int,
- on_complete: Optional[Callable[[], Any]] = None,
- metadata: Optional[Dict[str, Any]] = None,
- ) -> None:
- """Register a SemanticMsg with its total embedding task count.
-
- Args:
- semantic_msg_id: The ID of the SemanticMsg
- total_count: Total number of embedding tasks for this SemanticMsg
- on_complete: Optional callback when all tasks complete
- metadata: Optional metadata to store with the task
- """
- owner_loop = asyncio.get_running_loop()
- record_to_finalize: Optional[_EmbeddingTaskRecord] = None
-
- with self._lock:
- existing = self._tasks.get(semantic_msg_id)
- if existing is not None:
- logger.warning(
- "Overwriting existing embedding tracker record for SemanticMsg %s",
- semantic_msg_id,
- )
-
- self._tasks[semantic_msg_id] = _EmbeddingTaskRecord(
- remaining=total_count,
- total=total_count,
- on_complete=on_complete,
- metadata=metadata or {},
- owner_loop=owner_loop,
- )
- logger.info(
- f"Registered embedding tracker for SemanticMsg {semantic_msg_id}: "
- f"{total_count} tasks"
- )
-
- if total_count <= 0:
- record_to_finalize = self._tasks.pop(semantic_msg_id)
- logger.info(
- f"No embedding tasks for SemanticMsg {semantic_msg_id}, "
- f"clearing tracker entry immediately"
- )
-
- if record_to_finalize is not None:
- await self._run_on_complete(semantic_msg_id, record_to_finalize)
-
- async def decrement(self, semantic_msg_id: str) -> Optional[int]:
- """Decrement the remaining task count for a SemanticMsg.
-
- This method should be called when an embedding task is completed.
- When the count reaches zero, the registered callback is executed
- and the entry is removed from the tracker.
-
- Args:
- semantic_msg_id: The ID of the SemanticMsg
-
- Returns:
- The remaining count after decrement, or None if not found
- """
- record_to_finalize: Optional[_EmbeddingTaskRecord] = None
-
- with self._lock:
- record = self._tasks.get(semantic_msg_id)
- if record is None:
- return None
-
- record.remaining -= 1
- remaining = record.remaining
-
- if remaining <= 0:
- record_to_finalize = self._tasks.pop(semantic_msg_id)
- logger.info(
- f"All embedding tasks({record.total}) completed for SemanticMsg {semantic_msg_id}"
- )
-
- if record_to_finalize is not None:
- await self._run_on_complete(semantic_msg_id, record_to_finalize)
- return remaining
-
- async def abort(self, semantic_msg_id: str) -> bool:
- """Retire one tracking attempt without running its completion callback.
-
- Late decrements for an aborted attempt are ignored because its record no
- longer exists. Callers that may retry a logical semantic message must
- register each attempt under a unique tracking ID.
-
- Returns:
- True when an active record was removed, otherwise False.
- """
- with self._lock:
- record = self._tasks.pop(semantic_msg_id, None)
-
- if record is None:
- return False
-
- logger.warning(
- "Aborted embedding tracker for SemanticMsg %s with %s/%s tasks remaining",
- semantic_msg_id,
- record.remaining,
- record.total,
- )
- return True
diff --git a/openviking/storage/queuefs/queue_manager.py b/openviking/storage/queuefs/queue_manager.py
index 41a01178b..5593eaea7 100644
--- a/openviking/storage/queuefs/queue_manager.py
+++ b/openviking/storage/queuefs/queue_manager.py
@@ -21,8 +21,6 @@ from .semantic_queue import SemanticQueue
logger = get_logger(__name__)
-DEFAULT_MAX_CONCURRENT_SESSION_COMMIT = 4
-
# ========== Singleton Pattern ==========
_instance: Optional["QueueManager"] = None
@@ -32,8 +30,10 @@ def init_queue_manager(
timeout: int = 10,
mount_point: str = "/queue",
max_concurrent_embedding: int = 10,
- max_concurrent_semantic: int = 64,
+ max_concurrent_semantic: int = 32,
max_concurrent_external_parse: int = 4,
+ max_concurrent_add_resource: int = 4,
+ max_concurrent_session_commit: int = 4,
) -> "QueueManager":
"""Initialize QueueManager singleton.
@@ -44,6 +44,8 @@ def init_queue_manager(
max_concurrent_embedding: Max concurrent embedding tasks.
max_concurrent_semantic: Max concurrent semantic node work.
max_concurrent_external_parse: Max concurrent ExternalParse tasks.
+ max_concurrent_add_resource: Max concurrent AddResource tasks.
+ max_concurrent_session_commit: Max concurrent SessionCommit tasks.
"""
global _instance
_instance = QueueManager(
@@ -53,6 +55,8 @@ def init_queue_manager(
max_concurrent_embedding=max_concurrent_embedding,
max_concurrent_semantic=max_concurrent_semantic,
max_concurrent_external_parse=max_concurrent_external_parse,
+ max_concurrent_add_resource=max_concurrent_add_resource,
+ max_concurrent_session_commit=max_concurrent_session_commit,
)
return _instance
@@ -84,8 +88,10 @@ class QueueManager:
timeout: int = 10,
mount_point: str = "/queue",
max_concurrent_embedding: int = 10,
- max_concurrent_semantic: int = 64,
+ max_concurrent_semantic: int = 32,
max_concurrent_external_parse: int = 4,
+ max_concurrent_add_resource: int = 4,
+ max_concurrent_session_commit: int = 4,
):
"""Initialize QueueManager."""
self._agfs = agfs
@@ -94,6 +100,8 @@ class QueueManager:
self._max_concurrent_embedding = max_concurrent_embedding
self._max_concurrent_semantic = max_concurrent_semantic
self._max_concurrent_external_parse = max_concurrent_external_parse
+ self._max_concurrent_add_resource = max_concurrent_add_resource
+ self._max_concurrent_session_commit = max_concurrent_session_commit
self._queues: Dict[str, NamedQueue] = {}
self._started = False
self._queue_threads: Dict[str, threading.Thread] = {}
@@ -186,8 +194,10 @@ class QueueManager:
return self._max_concurrent_embedding
if queue_name == self.EXTERNAL_PARSE:
return self._max_concurrent_external_parse
+ if queue_name == self.ADD_RESOURCE:
+ return self._max_concurrent_add_resource
if queue_name == self.SESSION_COMMIT:
- return DEFAULT_MAX_CONCURRENT_SESSION_COMMIT
+ return self._max_concurrent_session_commit
return self._max_concurrent_semantic
def _queue_worker_loop(
diff --git a/openviking/storage/queuefs/semantic_dag.py b/openviking/storage/queuefs/semantic_dag.py
index 9b30b3e7b..f8c24c4c9 100644
--- a/openviking/storage/queuefs/semantic_dag.py
+++ b/openviking/storage/queuefs/semantic_dag.py
@@ -5,9 +5,9 @@
import asyncio
import re
import threading
+from contextlib import nullcontext
from dataclasses import dataclass, field
from typing import Any, ClassVar, Dict, List, Optional, Set
-from uuid import uuid4
from weakref import WeakKeyDictionary
from openviking.parse.parsers.media import get_media_type
@@ -15,7 +15,7 @@ from openviking.server.identity import RequestContext
from openviking.service.task_work_index import bind_task_context, get_task_context
from openviking.storage.queuefs.semantic_sidecar import write_semantic_sidecars
from openviking.storage.viking_fs import LS_ALL_NODES, get_viking_fs
-from openviking.telemetry.request_wait_tracker import get_request_wait_tracker
+from openviking.telemetry import bind_telemetry, get_current_telemetry
from openviking.utils.ingest_options import IngestOptions
from openviking_cli.utils import VikingURI
from openviking_cli.utils.logger import get_logger
@@ -53,26 +53,6 @@ class DagStats:
done_nodes: int = 0
-@dataclass
-class VectorizeTask:
- """Vectorize task information."""
-
- task_type: str # "file" or "directory"
- uri: str
- context_type: str
- ctx: "RequestContext"
- semantic_msg_id: Optional[str] = None
- # For file tasks
- file_path: Optional[str] = None
- summary_dict: Optional[Dict[str, str]] = None
- parent_uri: Optional[str] = None
- use_summary: bool = False
- ingest_options: IngestOptions = field(default_factory=IngestOptions)
- # For directory tasks
- abstract: Optional[str] = None
- overview: Optional[str] = None
-
-
@dataclass(frozen=True)
class DagWork:
"""A scheduled unit of DAG work."""
@@ -81,7 +61,6 @@ class DagWork:
dir_uri: str
parent_uri: Optional[str] = None
file_path: Optional[str] = None
- vectorize_task: Optional[VectorizeTask] = None
@dataclass(frozen=True)
@@ -171,8 +150,6 @@ class SemanticDagExecutor:
ctx: RequestContext,
incremental_update: bool = False,
target_uri: Optional[str] = None,
- semantic_msg_id: Optional[str] = None,
- telemetry_id: str = "",
recursive: bool = True,
lock: Optional[Dict[str, Any]] = None,
is_code_repo: bool = False,
@@ -187,8 +164,6 @@ class SemanticDagExecutor:
self._ctx = ctx
self._incremental_update = incremental_update
self._target_uri = target_uri
- self._semantic_msg_id = semantic_msg_id
- self._telemetry_id = telemetry_id
self._recursive = recursive
self._lock = lock
self._is_code_repo = is_code_repo
@@ -198,6 +173,7 @@ class SemanticDagExecutor:
self._coalesce_key = coalesce_key
self._coalesce_version = coalesce_version
self._task_context = get_task_context()
+ self._telemetry = get_current_telemetry()
self._stale = False
self._changed_paths = {
path for key in ("added", "modified", "deleted") for path in self._changes.get(key, [])
@@ -216,20 +192,6 @@ class SemanticDagExecutor:
self._closed = False
self._failure: Optional[Exception] = None
self._stats = DagStats()
- self._vectorize_task_count: int = 0
- self._pending_vectorize_tasks: List[VectorizeTask] = []
- self._pending_vectorize_work = 0
- self._vectorize_done: Optional[asyncio.Event] = None
- self._vectorize_failure: Optional[Exception] = None
- self._vectorize_dispatch_complete = False
- self._embedding_tracker_done: Optional[asyncio.Event] = None
- self._embedding_tracker = None
- # Retries preserve the logical SemanticMsg ID, so each execution needs
- # a distinct tracker key to quarantine late decrements from old attempts.
- self._embedding_tracker_id = f"{self._semantic_msg_id or 'semantic'}:{uuid4()}"
- self._vectorize_finalized = False
- self._vectorize_finalize_lock = asyncio.Lock()
- self._vectorize_lock = asyncio.Lock()
self._file_change_status: Dict[str, bool] = {}
self._dir_change_status: Dict[str, bool] = {}
self._overview_cache: Dict[str, Dict[str, str]] = {}
@@ -247,56 +209,9 @@ class SemanticDagExecutor:
await self._root_done.wait()
if self._failure:
raise self._failure
-
- # Mark semantic done after downstream vectorization finishes.
- async with self._vectorize_lock:
- task_count = self._vectorize_task_count
- tasks = list(self._pending_vectorize_tasks)
-
- if task_count > 0:
- from .embedding_tracker import EmbeddingTaskTracker
-
- embedding_tracker_done = asyncio.Event()
- self._embedding_tracker_done = embedding_tracker_done
- tracker = EmbeddingTaskTracker.get_instance()
- self._embedding_tracker = tracker
- await tracker.register(
- semantic_msg_id=self._embedding_tracker_id,
- total_count=task_count,
- on_complete=self._on_embedding_tasks_complete,
- metadata={
- "uri": root_uri,
- "semantic_msg_id": self._semantic_msg_id,
- },
- )
-
- await self._dispatch_vectorize_tasks(tasks)
- self._vectorize_dispatch_complete = True
-
- if embedding_tracker_done.is_set():
- await self._finalize_vectorization()
- if self._vectorize_failure is not None:
- raise self._vectorize_failure
- else:
- # No vectorize tasks — mark semantic work done immediately.
- try:
- await self._finalize_vectorization()
- except Exception as e:
- logger.error(f"Error in on_complete callback: {e}", exc_info=True)
except BaseException:
self._closed = True
await self._active_work_idle.wait()
- if self._embedding_tracker is not None:
- try:
- # Pending embedding messages may still finish, but their
- # attempt-specific decrements become harmless after abort.
- await self._embedding_tracker.abort(self._embedding_tracker_id)
- except Exception:
- logger.warning(
- "Failed to abort embedding tracker %s",
- self._embedding_tracker_id,
- exc_info=True,
- )
raise
finally:
self._closed = True
@@ -359,8 +274,6 @@ class SemanticDagExecutor:
self._closed = True
if self._root_done:
self._root_done.set()
- if self._vectorize_done:
- self._vectorize_done.set()
def _register_active(self) -> None:
with self._active_lock:
@@ -384,21 +297,19 @@ class SemanticDagExecutor:
return stats
async def _run_work(self, work: DagWork) -> None:
- if self._task_context is not None:
- with bind_task_context(
+ task_context = (
+ bind_task_context(
self._task_context.task_id,
self._task_context.account_id,
self._task_context.user_id,
- ):
- await self._run_work_bound(work)
- return
- await self._run_work_bound(work)
+ )
+ if self._task_context is not None
+ else nullcontext()
+ )
+ with bind_telemetry(self._telemetry), task_context:
+ await self._run_work_bound(work)
async def _run_work_bound(self, work: DagWork) -> None:
- if work.kind == "vectorize":
- await self._run_vectorize_work(work.vectorize_task)
- return
-
self._mark_node_started()
if work.kind == "dir":
@@ -426,72 +337,6 @@ class SemanticDagExecutor:
self._mark_node_done()
logger.warning("Unknown semantic DAG work kind: %s", work.kind)
- async def _dispatch_vectorize_tasks(self, tasks: List[VectorizeTask]) -> None:
- self._vectorize_done = asyncio.Event()
- self._pending_vectorize_work = len(tasks)
- for task in tasks:
- self._schedule_work(
- DagWork(
- kind="vectorize",
- dir_uri=task.uri,
- vectorize_task=task,
- )
- )
- await self._vectorize_done.wait()
-
- async def _run_vectorize_work(self, task: Optional[VectorizeTask]) -> None:
- try:
- if task is not None:
- await self._run_vectorize_task(task)
- except Exception as exc:
- logger.error("Vectorization dispatch task failed: %s", exc, exc_info=True)
- if self._vectorize_failure is None:
- self._vectorize_failure = exc
- finally:
- self._pending_vectorize_work = max(0, self._pending_vectorize_work - 1)
- if self._pending_vectorize_work == 0 and self._vectorize_done:
- self._vectorize_done.set()
-
- async def _on_embedding_tasks_complete(self) -> None:
- if self._embedding_tracker_done is not None:
- self._embedding_tracker_done.set()
- if self._vectorize_dispatch_complete:
- await self._finalize_vectorization()
-
- async def _finalize_vectorization(self) -> None:
- async with self._vectorize_finalize_lock:
- if self._vectorize_finalized:
- return
- if self._vectorize_failure is None and self._telemetry_id and self._semantic_msg_id:
- get_request_wait_tracker().mark_semantic_done(
- self._telemetry_id, self._semantic_msg_id
- )
- self._vectorize_finalized = True
-
- async def _run_vectorize_task(self, task: VectorizeTask) -> None:
- if task.task_type == "file":
- await self._processor._vectorize_single_file(
- parent_uri=task.parent_uri,
- context_type=task.context_type,
- file_path=task.file_path,
- summary_dict=task.summary_dict,
- ctx=task.ctx,
- semantic_msg_id=task.semantic_msg_id,
- use_summary=task.use_summary,
- ingest_options=task.ingest_options,
- )
- return
-
- await self._processor._vectorize_directory(
- task.uri,
- task.context_type,
- task.abstract,
- task.overview,
- ctx=task.ctx,
- semantic_msg_id=task.semantic_msg_id,
- ingest_options=task.ingest_options,
- )
-
async def _dispatch_dir(self, dir_uri: str, parent_uri: Optional[str]) -> bool:
"""Lazy-dispatch tasks for a directory when it is triggered."""
if dir_uri in self._nodes:
@@ -754,25 +599,34 @@ class SemanticDagExecutor:
if self._closed:
return
- try:
- if need_vectorize:
- use_summary = self._is_code_repo and bool(summary_dict.get("summary"))
- task = VectorizeTask(
- task_type="file",
- uri=file_path,
+ if need_vectorize and not self._skip_vectorization:
+ use_summary = self._is_code_repo and bool(summary_dict.get("summary"))
+ try:
+ await self._processor._vectorize_single_file(
+ parent_uri=parent_uri,
context_type=self._context_type,
- ctx=self._ctx,
- semantic_msg_id=self._embedding_tracker_id,
file_path=file_path,
summary_dict=summary_dict,
- parent_uri=parent_uri,
+ ctx=self._ctx,
use_summary=use_summary,
ingest_options=self._ingest_options,
)
- await self._add_vectorize_task(task)
- except Exception as e:
- logger.error(f"Failed to schedule vectorization for {file_path}: {e}", exc_info=True)
- await self._on_file_done(parent_uri, file_path, summary_dict)
+ except Exception as e:
+ logger.error(
+ "Failed to schedule vectorization for %s: %s",
+ file_path,
+ e,
+ exc_info=True,
+ )
+ raise
+ await self._on_file_done(
+ parent_uri,
+ file_path,
+ {
+ "name": str(summary_dict.get("name") or file_name),
+ "summary": str(summary_dict.get("summary") or ""),
+ },
+ )
async def _on_file_done(
self, parent_uri: str, file_path: str, summary_dict: Dict[str, str]
@@ -937,24 +791,27 @@ class SemanticDagExecutor:
except Exception:
logger.info(f"[SemanticDag] {dir_uri} write failed, skipping")
- try:
- if need_vectorize:
- task = VectorizeTask(
- task_type="directory",
- uri=dir_uri,
+ except Exception as e:
+ logger.error(f"Failed to generate overview for {dir_uri}: {e}", exc_info=True)
+ else:
+ if need_vectorize and not self._skip_vectorization:
+ try:
+ await self._processor._vectorize_directory(
+ dir_uri,
context_type=self._context_type,
- ctx=self._ctx,
- semantic_msg_id=self._embedding_tracker_id,
abstract=abstract,
overview=overview,
+ ctx=self._ctx,
ingest_options=self._ingest_options,
)
- await self._add_vectorize_task(task)
- except Exception as e:
- logger.error(f"Failed to schedule vectorization for {dir_uri}: {e}", exc_info=True)
-
- except Exception as e:
- logger.error(f"Failed to generate overview for {dir_uri}: {e}", exc_info=True)
+ except Exception as e:
+ logger.error(
+ "Failed to schedule vectorization for %s: %s",
+ dir_uri,
+ e,
+ exc_info=True,
+ )
+ raise
finally:
self._stats.done_nodes += 1
self._stats.in_progress_nodes = max(0, self._stats.in_progress_nodes - 1)
@@ -971,21 +828,6 @@ class SemanticDagExecutor:
await self._on_child_done(parent_uri, dir_uri, abstract)
self._release_dir_node(dir_uri)
- async def _add_vectorize_task(self, task: VectorizeTask) -> None:
- """Add a vectorize task to the pending list."""
- if self._skip_vectorization:
- logger.info(
- "Skipping vectorization task for %s (requested via SemanticMsg)",
- task.uri,
- )
- return
- async with self._vectorize_lock:
- self._pending_vectorize_tasks.append(task)
- if task.task_type == "file":
- self._vectorize_task_count += 1
- else: # directory
- self._vectorize_task_count += 2
-
def get_stats(self) -> DagStats:
return DagStats(
total_nodes=self._stats.total_nodes,
diff --git a/openviking/storage/queuefs/semantic_processor.py b/openviking/storage/queuefs/semantic_processor.py
index d3da70d82..c73d6158c 100644
--- a/openviking/storage/queuefs/semantic_processor.py
+++ b/openviking/storage/queuefs/semantic_processor.py
@@ -102,7 +102,7 @@ class SemanticProcessor(DequeueHandlerBase):
_request_stats_order: List[str] = []
_max_cached_stats = 256
- def __init__(self, max_concurrent_llm: int = 64):
+ def __init__(self, max_concurrent_llm: int = 32):
"""
Initialize SemanticProcessor.
@@ -438,8 +438,6 @@ class SemanticProcessor(DequeueHandlerBase):
ctx=current_ctx,
incremental_update=is_incremental,
target_uri=target_uri,
- semantic_msg_id=msg.id,
- telemetry_id=msg.telemetry_id,
recursive=msg.recursive,
lock=semantic_lock.lock,
is_code_repo=msg.is_code_repo,
@@ -459,6 +457,7 @@ class SemanticProcessor(DequeueHandlerBase):
await self._enqueue_parent_refresh(msg, target_uri or msg.uri)
finally:
await semantic_lock.close()
+ get_request_wait_tracker().mark_semantic_done(msg.telemetry_id, msg.id)
self._merge_request_stats(msg.telemetry_id, processed=1)
logger.info(f"Completed semantic generation for: {msg.uri}")
self.report_success()
@@ -555,8 +554,9 @@ class SemanticProcessor(DequeueHandlerBase):
"""Process a memory directory with special handling.
For memory directories:
- - Memory files are already vectorized via embedding queue
- - Only generate abstract.md and overview.md
+ - Generate file summaries in bounded batches
+ - Enqueue each changed file for vectorization as soon as its summary is ready
+ - Retain only compact summaries for the directory overview
- Vectorize the generated abstract.md and overview.md
Args:
@@ -566,185 +566,154 @@ class SemanticProcessor(DequeueHandlerBase):
dir_uri = msg.uri
ctx = ctx or self._default_ctx
llm_sem = asyncio.Semaphore(self.max_concurrent_llm)
- request_wait_tracker = get_request_wait_tracker()
-
- def _mark_done() -> None:
- if msg.telemetry_id and msg.id:
- request_wait_tracker.mark_semantic_done(msg.telemetry_id, msg.id)
try:
- try:
- entries = await viking_fs.ls(dir_uri, node_limit=LS_ALL_NODES, ctx=ctx)
- except Exception as e:
- raise RuntimeError(f"Failed to list memory directory {dir_uri}: {e}") from e
+ entries = await viking_fs.ls(dir_uri, node_limit=LS_ALL_NODES, ctx=ctx)
+ except Exception as e:
+ raise RuntimeError(f"Failed to list memory directory {dir_uri}: {e}") from e
- file_paths: List[str] = []
- for entry in entries:
- name = entry.get("name", "")
- if not name or name.startswith(".") or name in [".", ".."]:
- continue
- if not entry.get("isDir", False):
- item_uri = VikingURI(dir_uri).join(name).uri
- file_paths.append(item_uri)
+ file_paths: List[str] = []
+ for entry in entries:
+ name = entry.get("name", "")
+ if not name or name.startswith(".") or name in [".", ".."]:
+ continue
+ if not entry.get("isDir", False):
+ item_uri = VikingURI(dir_uri).join(name).uri
+ file_paths.append(item_uri)
- if not file_paths:
- logger.info(f"No memory files found in {dir_uri}")
- _mark_done()
- return
+ if not file_paths:
+ logger.info(f"No memory files found in {dir_uri}")
+ return
- existing_summaries: Dict[str, str] = {}
- if msg.changes:
- try:
- old_overview = await viking_fs.read_file(f"{dir_uri}/.overview.md", ctx=ctx)
- if old_overview:
- existing_summaries = self._parse_overview_md(old_overview)
- logger.info(
- f"Parsed {len(existing_summaries)} existing summaries from overview.md"
- )
- except Exception as e:
- logger.debug(f"No existing overview.md found for {dir_uri}: {e}")
+ existing_summaries: Dict[str, str] = {}
+ if msg.changes:
+ try:
+ old_overview = await viking_fs.read_file(f"{dir_uri}/.overview.md", ctx=ctx)
+ if old_overview:
+ existing_summaries = self._parse_overview_md(old_overview)
+ logger.info(
+ f"Parsed {len(existing_summaries)} existing summaries from overview.md"
+ )
+ except Exception as e:
+ logger.debug(f"No existing overview.md found for {dir_uri}: {e}")
- changed_files: Set[str] = set()
- if msg.changes:
- changed_files = set(msg.changes.get("added", []) + msg.changes.get("modified", []))
- deleted_files = set(msg.changes.get("deleted", []))
- logger.info(
- f"Processing memory directory {dir_uri} with changes: "
- f"added={len(msg.changes.get('added', []))}, "
- f"modified={len(msg.changes.get('modified', []))}, "
- f"deleted={len(deleted_files)}"
- )
+ changed_files: Set[str] = set()
+ if msg.changes:
+ changed_files = set(msg.changes.get("added", []) + msg.changes.get("modified", []))
+ deleted_files = set(msg.changes.get("deleted", []))
+ logger.info(
+ f"Processing memory directory {dir_uri} with changes: "
+ f"added={len(msg.changes.get('added', []))}, "
+ f"modified={len(msg.changes.get('modified', []))}, "
+ f"deleted={len(deleted_files)}"
+ )
- pending_indices: List[Tuple[int, str]] = []
- file_summaries: List[Optional[Dict[str, str]]] = [None] * len(file_paths)
+ pending_indices: List[Tuple[int, str]] = []
+ file_summaries: List[Optional[Dict[str, str]]] = [None] * len(file_paths)
+ paths_to_vectorize = changed_files if msg.changes else set(file_paths)
+
+ for idx, file_path in enumerate(file_paths):
+ file_name = file_path.split("/")[-1]
+ if file_path not in changed_files and file_name in existing_summaries:
+ file_summaries[idx] = {
+ "name": file_name,
+ "summary": existing_summaries[file_name],
+ }
+ logger.debug(f"Reused existing summary for {file_name}")
+ else:
+ pending_indices.append((idx, file_path))
- for idx, file_path in enumerate(file_paths):
- file_name = file_path.split("/")[-1]
- if file_path not in changed_files and file_name in existing_summaries:
- file_summaries[idx] = {
- "name": file_name,
- "summary": existing_summaries[file_name],
- }
- logger.debug(f"Reused existing summary for {file_name}")
- else:
- pending_indices.append((idx, file_path))
+ if file_paths and not pending_indices:
+ try:
+ from openviking.metrics.datasources.cache import CacheEventDataSource
- if file_paths and not pending_indices:
- try:
- from openviking.metrics.datasources.cache import CacheEventDataSource
+ CacheEventDataSource.record_hit("L1")
+ except Exception:
+ pass
+ elif file_paths and pending_indices:
+ try:
+ from openviking.metrics.datasources.cache import CacheEventDataSource
+ if len(file_paths) > len(pending_indices):
CacheEventDataSource.record_hit("L1")
- except Exception:
- pass
- elif file_paths and pending_indices:
- try:
- from openviking.metrics.datasources.cache import CacheEventDataSource
+ CacheEventDataSource.record_miss("L1")
+ except Exception:
+ pass
- if len(file_paths) > len(pending_indices):
- CacheEventDataSource.record_hit("L1")
- CacheEventDataSource.record_miss("L1")
- except Exception:
- pass
+ if pending_indices:
+ logger.info(
+ f"Generating summaries for {len(pending_indices)} changed files "
+ f"(reused {len(file_paths) - len(pending_indices)} cached)"
+ )
- if pending_indices:
+ async def _gen(idx: int, file_path: str) -> None:
+ file_name = file_path.split("/")[-1]
+ try:
+ summary_dict = await self._generate_single_file_summary(
+ file_path, llm_sem=llm_sem, ctx=ctx
+ )
+ logger.debug(f"Generated summary for {file_name}")
+ except Exception as e:
+ logger.warning(f"Failed to generate summary for {file_path}: {e}")
+ summary_dict = {"name": file_name, "summary": ""}
+
+ if file_path in paths_to_vectorize and not msg.skip_vectorization:
+ await self._vectorize_single_file(
+ parent_uri=dir_uri,
+ context_type="memory",
+ file_path=file_path,
+ summary_dict=summary_dict,
+ ctx=ctx,
+ preserve_existing_created_at=True,
+ )
+ file_summaries[idx] = {
+ "name": str(summary_dict.get("name") or file_name),
+ "summary": str(summary_dict.get("summary") or ""),
+ }
+
+ batch_size = max(1, min(self.max_concurrent_llm, 10))
+ for batch_start in range(0, len(pending_indices), batch_size):
+ batch = pending_indices[batch_start : batch_start + batch_size]
logger.info(
- f"Generating summaries for {len(pending_indices)} changed files "
- f"(reused {len(file_paths) - len(pending_indices)} cached)"
+ f"[MemorySemantic] Processing batch {batch_start // batch_size + 1}/"
+ f"{(len(pending_indices) + batch_size - 1) // batch_size} "
+ f"({len(batch)} files)"
)
+ await asyncio.gather(*[_gen(i, fp) for i, fp in batch])
- async def _gen(idx: int, file_path: str) -> None:
- file_name = file_path.split("/")[-1]
- try:
- summary_dict = await self._generate_single_file_summary(
- file_path, llm_sem=llm_sem, ctx=ctx
- )
- file_summaries[idx] = summary_dict
- logger.debug(f"Generated summary for {file_name}")
- except Exception as e:
- logger.warning(f"Failed to generate summary for {file_path}: {e}")
- file_summaries[idx] = {"name": file_name, "summary": ""}
-
- batch_size = max(1, min(self.max_concurrent_llm, 10))
- for batch_start in range(0, len(pending_indices), batch_size):
- batch = pending_indices[batch_start : batch_start + batch_size]
- logger.info(
- f"[MemorySemantic] Processing batch {batch_start // batch_size + 1}/"
- f"{(len(pending_indices) + batch_size - 1) // batch_size} "
- f"({len(batch)} files)"
- )
- await asyncio.gather(*[_gen(i, fp) for i, fp in batch])
-
- completed_summaries = [s for s in file_summaries if s is not None]
- # Incremental writes carry changes; full rebuild tasks do not.
- if msg.changes:
- paths_to_vectorize = changed_files
- else:
- paths_to_vectorize = set(file_paths)
- file_vectorize_items = [
- (file_path, summary)
- for file_path, summary in zip(file_paths, file_summaries, strict=False)
- if file_path in paths_to_vectorize and summary is not None
- ]
- generated_content = await self._generate_overview(
- dir_uri, completed_summaries, [], llm_sem=llm_sem
- )
- overview, abstract = self._normalize_overview_generation(generated_content)
-
- try:
- wrote_semantics = await self._write_memory_directory_semantics(
- msg=msg,
- viking_fs=viking_fs,
- dir_uri=dir_uri,
- overview=overview,
- abstract=abstract,
- ctx=ctx,
- lock=lock,
- )
- except Exception as e:
- raise RuntimeError(f"Failed to write abstract/overview for {dir_uri}: {e}") from e
- if not wrote_semantics:
- _mark_done()
- return
- logger.info(f"Generated abstract.md and overview.md for {dir_uri}")
-
- if msg.skip_vectorization:
- logger.info(f"Skipping vectorization for {dir_uri} (requested via SemanticMsg)")
- _mark_done()
- return
- if msg.telemetry_id and msg.id:
- from openviking.storage.queuefs.embedding_tracker import EmbeddingTaskTracker
-
- async def _on_complete() -> None:
- get_request_wait_tracker().mark_semantic_done(msg.telemetry_id, msg.id)
+ completed_summaries = [s for s in file_summaries if s is not None]
+ generated_content = await self._generate_overview(
+ dir_uri, completed_summaries, [], llm_sem=llm_sem
+ )
+ overview, abstract = self._normalize_overview_generation(generated_content)
- tracker = EmbeddingTaskTracker.get_instance()
- await tracker.register(
- semantic_msg_id=msg.id,
- total_count=2 + len(file_vectorize_items),
- on_complete=_on_complete,
- metadata={"uri": dir_uri},
- )
- for file_path, summary_dict in file_vectorize_items:
- await self._vectorize_single_file(
- parent_uri=dir_uri,
- context_type="memory",
- file_path=file_path,
- summary_dict=summary_dict,
- ctx=ctx,
- semantic_msg_id=msg.id,
- preserve_existing_created_at=True,
- )
- await self._vectorize_directory(
- uri=dir_uri,
- context_type="memory",
- abstract=abstract,
+ try:
+ wrote_semantics = await self._write_memory_directory_semantics(
+ msg=msg,
+ viking_fs=viking_fs,
+ dir_uri=dir_uri,
overview=overview,
+ abstract=abstract,
ctx=ctx,
- semantic_msg_id=msg.id,
+ lock=lock,
)
- logger.info(f"Vectorized abstract.md and overview.md for {dir_uri}")
- finally:
- pass
+ except Exception as e:
+ raise RuntimeError(f"Failed to write abstract/overview for {dir_uri}: {e}") from e
+ if not wrote_semantics:
+ return
+ logger.info(f"Generated abstract.md and overview.md for {dir_uri}")
+
+ if msg.skip_vectorization:
+ logger.info(f"Skipping vectorization for {dir_uri} (requested via SemanticMsg)")
+ return
+ await self._vectorize_directory(
+ uri=dir_uri,
+ context_type="memory",
+ abstract=abstract,
+ overview=overview,
+ ctx=ctx,
+ )
+ logger.info(f"Vectorized abstract.md and overview.md for {dir_uri}")
async def _write_memory_directory_semantics(
self,
@@ -1563,7 +1532,6 @@ class SemanticProcessor(DequeueHandlerBase):
abstract: str,
overview: str,
ctx: Optional[RequestContext] = None,
- semantic_msg_id: Optional[str] = None,
ingest_options: IngestOptions | None = None,
) -> None:
"""Create directory Context and enqueue to EmbeddingQueue."""
@@ -1577,7 +1545,6 @@ class SemanticProcessor(DequeueHandlerBase):
overview=overview,
context_type=context_type,
ctx=active_ctx,
- semantic_msg_id=semantic_msg_id,
ingest_options=ingest_options,
)
@@ -1588,7 +1555,6 @@ class SemanticProcessor(DequeueHandlerBase):
file_path: str,
summary_dict: Dict[str, str],
ctx: Optional[RequestContext] = None,
- semantic_msg_id: Optional[str] = None,
use_summary: bool = False,
preserve_existing_created_at: bool = False,
ingest_options: IngestOptions | None = None,
@@ -1603,7 +1569,6 @@ class SemanticProcessor(DequeueHandlerBase):
parent_uri=parent_uri,
context_type=context_type,
ctx=active_ctx,
- semantic_msg_id=semantic_msg_id,
use_summary=use_summary,
preserve_existing_created_at=preserve_existing_created_at,
ingest_options=ingest_options,
diff --git a/openviking/telemetry/request_wait_tracker.py b/openviking/telemetry/request_wait_tracker.py
index b22fc1527..a26b84a18 100644
--- a/openviking/telemetry/request_wait_tracker.py
+++ b/openviking/telemetry/request_wait_tracker.py
@@ -55,14 +55,14 @@ class RequestWaitTracker:
def register_request(self, telemetry_id: str) -> None:
self._create_state(telemetry_id)
- def register_semantic_root(self, telemetry_id: str, semantic_msg_id: str) -> None:
- if not telemetry_id or not semantic_msg_id:
+ def register_semantic_root(self, telemetry_id: str, root_id: str) -> None:
+ if not telemetry_id or not root_id:
return
with self._lock:
state = self._states.get(telemetry_id)
if state is None:
return
- state.pending_semantic_roots.add(semantic_msg_id)
+ state.pending_semantic_roots.add(root_id)
def register_embedding_root(self, telemetry_id: str, root_id: str) -> None:
if not telemetry_id or not root_id:
@@ -73,15 +73,6 @@ class RequestWaitTracker:
return
state.pending_embedding_roots.add(root_id)
- def record_embedding_processed(self, telemetry_id: str, delta: int = 1) -> None:
- if not telemetry_id:
- return
- with self._lock:
- state = self._states.get(telemetry_id)
- if state is None:
- return
- state.embedding_processed += max(delta, 0)
-
def record_embedding_requeue(self, telemetry_id: str, delta: int = 1) -> None:
if not telemetry_id:
return
@@ -91,21 +82,10 @@ class RequestWaitTracker:
return
state.embedding_requeue_count += max(delta, 0)
- def record_embedding_error(self, telemetry_id: str, message: str) -> None:
- if not telemetry_id:
- return
- with self._lock:
- state = self._states.get(telemetry_id)
- if state is None:
- return
- state.embedding_error_count += 1
- if message:
- state.embedding_errors.append(message)
-
def mark_semantic_done(
self,
telemetry_id: str,
- semantic_msg_id: str,
+ root_id: str,
processed_delta: int = 1,
) -> None:
if not telemetry_id:
@@ -114,7 +94,7 @@ class RequestWaitTracker:
state = self._states.get(telemetry_id)
if state is None:
return
- state.pending_semantic_roots.discard(semantic_msg_id)
+ state.pending_semantic_roots.discard(root_id)
state.semantic_processed += max(processed_delta, 0)
def record_semantic_requeue(self, telemetry_id: str, delta: int = 1) -> None:
@@ -126,14 +106,14 @@ class RequestWaitTracker:
return
state.semantic_requeue_count += max(delta, 0)
- def mark_semantic_failed(self, telemetry_id: str, semantic_msg_id: str, message: str) -> None:
+ def mark_semantic_failed(self, telemetry_id: str, root_id: str, message: str) -> None:
if not telemetry_id:
return
with self._lock:
state = self._states.get(telemetry_id)
if state is None:
return
- state.pending_semantic_roots.discard(semantic_msg_id)
+ state.pending_semantic_roots.discard(root_id)
state.semantic_error_count += 1
if message:
state.semantic_errors.append(message)
diff --git a/openviking/utils/embedding_utils.py b/openviking/utils/embedding_utils.py
index b5477ecf3..a34b463d3 100644
--- a/openviking/utils/embedding_utils.py
+++ b/openviking/utils/embedding_utils.py
@@ -94,20 +94,34 @@ def _apply_ingest_options(
}
-async def _decrement_embedding_tracker(semantic_msg_id: Optional[str], count: int) -> None:
- if not semantic_msg_id or count <= 0:
- return
+async def _enqueue_embedding_message(
+ embedding_queue,
+ embedding_msg,
+ *,
+ failure_message: str,
+) -> bool:
+ """Persist one embedding message and settle request tracking on enqueue failure."""
+ wait_tracker = get_request_wait_tracker()
+ wait_tracker.register_embedding_root(embedding_msg.telemetry_id, embedding_msg.id)
+
try:
- from openviking.storage.queuefs.embedding_tracker import EmbeddingTaskTracker
+ enqueue_id = await embedding_queue.enqueue(embedding_msg)
+ except BaseException as exc:
+ wait_tracker.mark_embedding_failed(
+ embedding_msg.telemetry_id,
+ embedding_msg.id,
+ f"{failure_message}: {exc}",
+ )
+ raise
- tracker = EmbeddingTaskTracker.get_instance()
- for _ in range(count):
- await tracker.decrement(semantic_msg_id)
- except Exception as e:
- logger.error(
- f"Failed to decrement embedding tracker for semantic_msg_id={semantic_msg_id}: {e}",
- exc_info=True,
+ if not enqueue_id:
+ wait_tracker.mark_embedding_failed(
+ embedding_msg.telemetry_id,
+ embedding_msg.id,
+ failure_message,
)
+ return False
+ return True
def _coerce_datetime(value: object) -> Optional[datetime]:
@@ -341,7 +355,6 @@ async def vectorize_directory_meta(
overview: str,
context_type: str = "resource",
ctx: Optional[RequestContext] = None,
- semantic_msg_id: Optional[str] = None,
include_overview: bool = True,
scalar_overrides: Optional[Dict[int, Dict[str, Any]]] = None,
ingest_options: IngestOptions | None = None,
@@ -351,8 +364,6 @@ async def vectorize_directory_meta(
Creates Context objects for abstract and overview and enqueues them.
"""
- enqueued = 0
- expected = 2 if include_overview else 1
first_enqueue_error: Optional[Exception] = None
try:
if not ctx:
@@ -394,11 +405,14 @@ async def vectorize_directory_meta(
)
_apply_ingest_options(msg_abstract, ingest_options)
if msg_abstract:
- msg_abstract.semantic_msg_id = semantic_msg_id
try:
- await embedding_queue.enqueue(msg_abstract)
- enqueued += 1
- logger.debug(f"Enqueued directory L0 (abstract) for vectorization: {uri}")
+ enqueued = await _enqueue_embedding_message(
+ embedding_queue,
+ msg_abstract,
+ failure_message=f"Failed to enqueue directory L0 vector for {uri}",
+ )
+ if enqueued:
+ logger.debug(f"Enqueued directory L0 (abstract) for vectorization: {uri}")
except TaskWorkRejected:
logger.debug("Skipped directory vectorization for cancelling task: %s", uri)
return
@@ -434,11 +448,14 @@ async def vectorize_directory_meta(
)
_apply_ingest_options(msg_overview, ingest_options)
if msg_overview:
- msg_overview.semantic_msg_id = semantic_msg_id
try:
- await embedding_queue.enqueue(msg_overview)
- enqueued += 1
- logger.debug(f"Enqueued directory L1 (overview) for vectorization: {uri}")
+ enqueued = await _enqueue_embedding_message(
+ embedding_queue,
+ msg_overview,
+ failure_message=f"Failed to enqueue directory L1 vector for {uri}",
+ )
+ if enqueued:
+ logger.debug(f"Enqueued directory L1 (overview) for vectorization: {uri}")
except TaskWorkRejected:
logger.debug("Skipped directory vectorization for cancelling task: %s", uri)
return
@@ -457,8 +474,6 @@ async def vectorize_directory_meta(
exc_info=True,
)
raise
- finally:
- await _decrement_embedding_tracker(semantic_msg_id, expected - enqueued)
async def vectorize_file(
@@ -467,11 +482,9 @@ async def vectorize_file(
parent_uri: str,
context_type: str = "resource",
ctx: Optional[RequestContext] = None,
- semantic_msg_id: Optional[str] = None,
use_summary: bool = False,
preserve_existing_created_at: bool = False,
scalar_override: Optional[Dict[str, Any]] = None,
- register_request_wait: bool = False,
ingest_options: IngestOptions | None = None,
) -> bool:
"""
@@ -482,9 +495,6 @@ async def vectorize_file(
`use_summary` flag (code path override) or the embedding config.
Returns whether an embedding message was enqueued.
"""
- enqueued = False
- registered_wait_root: Optional[tuple[str, str]] = None
-
try:
if not ctx:
logger.warning("No context provided for vectorization")
@@ -595,45 +605,22 @@ async def vectorize_file(
_apply_scalar_overrides(embedding_msg, scalar_override)
_apply_ingest_options(embedding_msg, ingest_options)
- embedding_msg.semantic_msg_id = semantic_msg_id
- if register_request_wait:
- get_request_wait_tracker().register_embedding_root(
- embedding_msg.telemetry_id,
- embedding_msg.id,
- )
- registered_wait_root = (embedding_msg.telemetry_id, embedding_msg.id)
- enqueued_id = await embedding_queue.enqueue(embedding_msg)
- if register_request_wait and not enqueued_id:
- get_request_wait_tracker().mark_embedding_failed(
- embedding_msg.telemetry_id,
- embedding_msg.id,
- f"Failed to enqueue file vector for {file_path}",
- )
+ enqueued = await _enqueue_embedding_message(
+ embedding_queue,
+ embedding_msg,
+ failure_message=f"Failed to enqueue file vector for {file_path}",
+ )
+ if not enqueued:
return False
- enqueued = True
logger.debug(f"Enqueued file for vectorization: {file_path}")
except TaskWorkRejected:
logger.debug("Skipped file vectorization for cancelling task: %s", file_path)
- if registered_wait_root is not None:
- get_request_wait_tracker().mark_embedding_failed(
- registered_wait_root[0],
- registered_wait_root[1],
- f"Task cancellation skipped file vector for {file_path}",
- )
+ return False
except Exception as e:
logger.error(f"Failed to vectorize file {file_path}: {e}", exc_info=True)
- if registered_wait_root is not None:
- get_request_wait_tracker().mark_embedding_failed(
- registered_wait_root[0],
- registered_wait_root[1],
- f"Failed to enqueue file vector for {file_path}: {e}",
- )
raise
- finally:
- if not enqueued:
- await _decrement_embedding_tracker(semantic_msg_id, 1)
- return enqueued
+ return True
async def index_resource(
diff --git a/openviking/utils/resource_processor.py b/openviking/utils/resource_processor.py
index ffdc9a287..608e9ecc2 100644
--- a/openviking/utils/resource_processor.py
+++ b/openviking/utils/resource_processor.py
@@ -13,7 +13,6 @@ from collections.abc import Callable
from typing import TYPE_CHECKING, Any, Dict, List, Optional, Union
from openviking.core.context import ContextLevel
-from openviking.utils.ingest_options import IngestOptions
from openviking.core.namespace import context_type_for_uri
from openviking.parse.image_rewrite import rewrite_image_uris
from openviking.parse.mode import ParseMode, normalize_parse_mode
@@ -25,14 +24,15 @@ from openviking.resource.processing_mode import (
normalize_processing_mode,
)
from openviking.server.identity import RequestContext
-from openviking.storage.vikingdb_manager import VikingDBManager
from openviking.storage.errors import LockAcquisitionError
from openviking.storage.expr import And, Eq, PathScope
from openviking.storage.internal_names import STORAGE_INTERNAL_ENTRY_NAMES
from openviking.storage.queuefs.semantic_processor import SemanticProcessor
from openviking.storage.viking_fs import LS_ALL_NODES, get_viking_fs
+from openviking.storage.vikingdb_manager import VikingDBManager
from openviking.telemetry import get_current_telemetry
from openviking.utils.embedding_utils import index_resource, vectorize_file
+from openviking.utils.ingest_options import IngestOptions
from openviking.utils.summarizer import Summarizer
from openviking_cli.exceptions import OpenVikingError
from openviking_cli.utils import VikingURI, get_logger
@@ -669,7 +669,6 @@ class ResourceProcessor:
context_type=context_type_for_uri(entry_uri),
ctx=ctx,
ingest_options=ingest_options,
- register_request_wait=True,
)
async def _vectorize_resource_file(
@@ -690,7 +689,6 @@ class ResourceProcessor:
context_type=context_type_for_uri(file_uri),
ctx=ctx,
ingest_options=IngestOptions.from_value(ingest_options),
- register_request_wait=True,
)
async def reserve_unique_candidate(
diff --git a/openviking_cli/utils/config/open_viking_config.py b/openviking_cli/utils/config/open_viking_config.py
index 95f014b75..c7fbb2451 100644
--- a/openviking_cli/utils/config/open_viking_config.py
+++ b/openviking_cli/utils/config/open_viking_config.py
@@ -185,7 +185,6 @@ class OpenVikingConfig(BaseModel):
default_factory=GitConfig, description="Git version control configuration"
)
- # Parser configurations
pdf: PDFConfig = Field(default_factory=PDFConfig, description="PDF parsing configuration")
code: CodeConfig = Field(default_factory=CodeConfig, description="Code parsing configuration")
diff --git a/openviking_cli/utils/config/queue_worker_config.py b/openviking_cli/utils/config/queue_worker_config.py
index 497c26f77..a73f40e96 100644
--- a/openviking_cli/utils/config/queue_worker_config.py
+++ b/openviking_cli/utils/config/queue_worker_config.py
@@ -21,5 +21,7 @@ class QueueWorkersConfig(BaseModel):
"""Runtime limits for QueueFS consumers."""
external_parse: QueueWorkerConfig = Field(default_factory=QueueWorkerConfig)
+ add_resource: QueueWorkerConfig = Field(default_factory=QueueWorkerConfig)
+ session_commit: QueueWorkerConfig = Field(default_factory=QueueWorkerConfig)
model_config = {"extra": "forbid"}
diff --git a/openviking_cli/utils/config/vlm_config.py b/openviking_cli/utils/config/vlm_config.py
index 75914888c..45b5a0082 100644
--- a/openviking_cli/utils/config/vlm_config.py
+++ b/openviking_cli/utils/config/vlm_config.py
@@ -134,7 +134,7 @@ class VLMConfig(BaseModel):
thinking: bool = Field(default=False, description="Enable thinking mode for VolcEngine models")
max_concurrent: int = Field(
- default=64, description="Maximum number of concurrent LLM calls for semantic processing"
+ default=32, description="Maximum number of concurrent LLM calls for semantic processing"
)
media: VLMMediaConfig = Field(
diff --git a/tests/service/test_fs_service.py b/tests/service/test_fs_service.py
index fd55fd180..39f282d01 100644
--- a/tests/service/test_fs_service.py
+++ b/tests/service/test_fs_service.py
@@ -86,11 +86,11 @@ class _FakeWaitTracker:
def register_request(self, telemetry_id):
self.registered_requests.append(telemetry_id)
- def register_semantic_root(self, telemetry_id, semantic_msg_id):
+ def register_semantic_root(self, telemetry_id, root_id):
self.registered_roots.append(
{
"telemetry_id": telemetry_id,
- "semantic_msg_id": semantic_msg_id,
+ "root_id": root_id,
"request_was_registered": telemetry_id in self.registered_requests,
}
)
@@ -104,7 +104,7 @@ class _FakeWaitTracker:
"Embedding": {"processed": 0, "error_count": 0, "errors": []},
}
- def mark_semantic_failed(self, telemetry_id, semantic_msg_id, message):
+ def mark_semantic_failed(self, telemetry_id, root_id, message):
pass
def cleanup(self, telemetry_id):
diff --git a/tests/session/test_memory_batching.py b/tests/session/test_memory_batching.py
index 7cbaa7bb6..7e4f4d988 100644
--- a/tests/session/test_memory_batching.py
+++ b/tests/session/test_memory_batching.py
@@ -4,7 +4,7 @@
import asyncio
import logging
import os
-from unittest.mock import AsyncMock, MagicMock, patch
+from unittest.mock import MagicMock, patch
import pytest
@@ -58,6 +58,7 @@ class NorthwestTripMockVLM:
return (content * 10)[:2000]
+@skip_if_not_manual
@pytest.mark.asyncio
async def test_manual_memory_batching_100_files(monkeypatch):
"""
@@ -112,11 +113,6 @@ async def test_manual_memory_batching_100_files(monkeypatch):
mock_fs = MockVikingFS()
- # 3. 模拟 Tracker 和 WaitTracker
- mock_wait_tracker = MagicMock()
- mock_embedding_tracker = MagicMock()
- mock_embedding_tracker.register = AsyncMock()
-
# 使用 patch.multiple 来模拟多个 get_xxx 方法
with (
patch(
@@ -124,14 +120,6 @@ async def test_manual_memory_batching_100_files(monkeypatch):
return_value=mock_config,
),
patch("openviking.storage.queuefs.semantic_processor.get_viking_fs", return_value=mock_fs),
- patch(
- "openviking.storage.queuefs.semantic_processor.get_request_wait_tracker",
- return_value=mock_wait_tracker,
- ),
- patch(
- "openviking.storage.queuefs.embedding_tracker.EmbeddingTaskTracker.get_instance",
- return_value=mock_embedding_tracker,
- ),
):
# 4. 初始化 Processor 并设置并发
processor = SemanticProcessor(max_concurrent_llm=10)
@@ -183,7 +171,6 @@ async def test_manual_memory_batching_100_files(monkeypatch):
# 100次 摘要生成 + 1次 overview(L1) + 1次 abstract(L0)
# 因为 read_file 也被 mock 了,所以构造过程不再消耗 call_count
assert mock_vlm.call_count >= 102
- assert mock_embedding_tracker.register.called
print("[Manual Test] 分批逻辑压力测试及并发验证成功。")
diff --git a/tests/storage/test_collection_schemas.py b/tests/storage/test_collection_schemas.py
index 96bae5aee..22f98658c 100644
--- a/tests/storage/test_collection_schemas.py
+++ b/tests/storage/test_collection_schemas.py
@@ -1,7 +1,6 @@
# Copyright (c) 2026 Beijing Volcano Engine Technology Co., Ltd.
# SPDX-License-Identifier: AGPL-3.0
-import asyncio
import hashlib
import inspect
import json
@@ -22,20 +21,20 @@ from openviking.storage.errors import EmbeddingRebuildRequiredError
from openviking.storage.expr import Eq
from openviking.storage.queuefs.embedding_msg import EmbeddingMsg
from openviking.storage.vectordb import engine as vectordb_engine
+from openviking.storage.vectordb.collection.result import UpsertDataResult
+from openviking.storage.vectordb.collection.vikingdb_collection import VikingDBCollection
from openviking.storage.vectordb.collection.volcengine_api_key_collection import (
VolcengineApiKeyCollection,
)
-from openviking.storage.vectordb.collection.vikingdb_collection import VikingDBCollection
from openviking.storage.vectordb.collection.volcengine_collection import VolcengineCollection
-from openviking.storage.vectordb.collection.result import UpsertDataResult
from openviking.storage.vectordb_adapters.base import (
VIKINGDB_TEXT_FIELD_BYTE_LIMIT,
_truncate_text_field,
)
from openviking.storage.vectordb_adapters.local_adapter import LocalCollectionAdapter
from openviking.storage.viking_vector_index_backend import (
- UpsertOptions,
VIKINGDB_CONTENT_MAX_SIZE,
+ UpsertOptions,
VikingVectorIndexBackend,
_SingleAccountBackend,
)
@@ -660,7 +659,7 @@ async def test_embedding_handler_preserves_parent_uri_for_backend_upsert_logic(m
@pytest.mark.asyncio
-async def test_embedding_handler_marks_success_only_after_tracker_completion(monkeypatch):
+async def test_embedding_handler_settles_request_wait_by_message_id(monkeypatch):
class _CapturingVikingDB:
is_closing = False
mode = "local"
@@ -676,46 +675,25 @@ async def test_embedding_handler_marks_success_only_after_tracker_completion(mon
lambda: _DummyConfig(embedder),
)
- decrement_started = asyncio.Event()
- allow_decrement_finish = asyncio.Event()
-
- class _FakeTracker:
- async def decrement(self, _semantic_msg_id):
- decrement_started.set()
- await allow_decrement_finish.wait()
- return 0
-
+ completed = []
monkeypatch.setattr(
- "openviking.storage.queuefs.embedding_tracker.EmbeddingTaskTracker.get_instance",
- lambda: _FakeTracker(),
+ "openviking.storage.collection_schemas.get_request_wait_tracker",
+ lambda: SimpleNamespace(
+ mark_embedding_done=lambda telemetry_id, root_id: completed.append(
+ (telemetry_id, root_id)
+ )
+ ),
)
handler = TextEmbeddingHandler(_CapturingVikingDB())
- status = {"success": 0, "requeue": 0, "error": 0}
- handler.set_callbacks(
- on_success=lambda: status.__setitem__("success", status["success"] + 1),
- on_requeue=lambda: status.__setitem__("requeue", status["requeue"] + 1),
- on_error=lambda *_: status.__setitem__("error", status["error"] + 1),
- )
-
payload = _build_queue_payload()
queue_data = json.loads(payload["data"])
- queue_data["semantic_msg_id"] = "semantic-1"
+ queue_data["telemetry_id"] = "request-1"
payload["data"] = json.dumps(queue_data)
- task = asyncio.create_task(handler.on_dequeue(payload))
- await decrement_started.wait()
-
- assert status["success"] == 0
- assert status["requeue"] == 0
- assert status["error"] == 0
-
- allow_decrement_finish.set()
- await task
+ await handler.on_dequeue(payload)
- assert status["success"] == 1
- assert status["requeue"] == 0
- assert status["error"] == 0
+ assert completed == [("request-1", queue_data["id"])]
def test_context_collection_excludes_parent_uri():
diff --git a/tests/storage/test_content_write_processing_mode.py b/tests/storage/test_content_write_processing_mode.py
index 394d8db12..25c582999 100644
--- a/tests/storage/test_content_write_processing_mode.py
+++ b/tests/storage/test_content_write_processing_mode.py
@@ -73,7 +73,7 @@ async def test_vectors_only_write_skips_semantic_refresh_and_vectorizes_file(mon
"name": "demo.md",
"summary": "",
}
- assert vectorize_file.await_args.kwargs["register_request_wait"] is True
+ assert "register_request_wait" not in vectorize_file.await_args.kwargs
assert result["semantic_status"] == "skipped"
assert result["vector_status"] == "queued"
diff --git a/tests/storage/test_embedding_tracker.py b/tests/storage/test_embedding_tracker.py
deleted file mode 100644
index ea6ad0b72..000000000
--- a/tests/storage/test_embedding_tracker.py
+++ /dev/null
@@ -1,213 +0,0 @@
-# Copyright (c) 2026 Beijing Volcano Engine Technology Co., Ltd.
-# SPDX-License-Identifier: AGPL-3.0
-
-import asyncio
-import concurrent.futures
-import threading
-import time
-
-import pytest
-
-from openviking.storage.queuefs.embedding_tracker import EmbeddingTaskTracker
-
-
-class _LoopThread:
- def __init__(self, close_delay: float = 0) -> None:
- self.loop = asyncio.new_event_loop()
- self._ready = threading.Event()
- self._close_delay = close_delay
- self.thread = threading.Thread(target=self._run, daemon=True)
- self.thread.start()
- self._ready.wait(timeout=2)
-
- def _run(self) -> None:
- asyncio.set_event_loop(self.loop)
- self._ready.set()
- self.loop.run_forever()
- if self._close_delay:
- time.sleep(self._close_delay)
- pending = asyncio.all_tasks(self.loop)
- for task in pending:
- task.cancel()
- if pending:
- self.loop.run_until_complete(asyncio.gather(*pending, return_exceptions=True))
- self.loop.close()
-
- def submit(self, coro):
- return asyncio.run_coroutine_threadsafe(coro, self.loop)
-
- def stop(self) -> None:
- if self.loop.is_closed():
- return
- self.loop.call_soon_threadsafe(self.loop.stop)
- self.thread.join(timeout=3)
-
- def stop_without_join(self) -> None:
- if self.loop.is_closed():
- return
- self.loop.call_soon_threadsafe(self.loop.stop)
-
- def join(self) -> None:
- self.thread.join(timeout=3)
-
-
[email protected](autouse=True)
-def _reset_tracker_singleton():
- EmbeddingTaskTracker._instance = None
- EmbeddingTaskTracker._initialized = False
- yield
- EmbeddingTaskTracker._instance = None
- EmbeddingTaskTracker._initialized = False
-
-
-def test_tracker_runs_completion_callback_on_register_loop():
- tracker = EmbeddingTaskTracker.get_instance()
- owner = _LoopThread()
- worker = _LoopThread()
- callback_info = concurrent.futures.Future()
-
- async def on_complete():
- callback_info.set_result((threading.get_ident(), asyncio.get_running_loop()))
- await asyncio.sleep(0)
-
- async def register():
- await tracker.register("semantic-msg", 1, on_complete=on_complete)
-
- async def decrement():
- return await tracker.decrement("semantic-msg")
-
- try:
- owner.submit(register()).result(timeout=2)
- assert not callback_info.done()
-
- remaining = worker.submit(decrement()).result(timeout=2)
- callback_thread_id, callback_loop = callback_info.result(timeout=2)
- finally:
- owner.stop()
- worker.stop()
-
- assert remaining == 0
- assert callback_thread_id == owner.thread.ident
- assert callback_loop is owner.loop
-
-
-def test_tracker_falls_back_to_current_loop_when_owner_loop_is_closed():
- tracker = EmbeddingTaskTracker.get_instance()
- owner = _LoopThread()
- worker = _LoopThread()
- callback_info = concurrent.futures.Future()
-
- async def on_complete():
- callback_info.set_result((threading.get_ident(), asyncio.get_running_loop()))
-
- async def register():
- await tracker.register("semantic-msg", 1, on_complete=on_complete)
-
- async def decrement():
- return await tracker.decrement("semantic-msg")
-
- try:
- owner.submit(register()).result(timeout=2)
- owner.stop()
-
- remaining = worker.submit(decrement()).result(timeout=2)
- callback_thread_id, callback_loop = callback_info.result(timeout=2)
- finally:
- worker.stop()
-
- assert remaining == 0
- assert callback_thread_id == worker.thread.ident
- assert callback_loop is worker.loop
-
-
-def test_tracker_falls_back_to_current_loop_when_owner_loop_is_stopped():
- tracker = EmbeddingTaskTracker.get_instance()
- owner = _LoopThread(close_delay=1)
- worker = _LoopThread()
- callback_info = concurrent.futures.Future()
-
- async def on_complete():
- callback_info.set_result((threading.get_ident(), asyncio.get_running_loop()))
-
- async def register():
- await tracker.register("semantic-msg", 1, on_complete=on_complete)
-
- async def decrement():
- return await tracker.decrement("semantic-msg")
-
- try:
- owner.submit(register()).result(timeout=2)
- owner.stop_without_join()
- time.sleep(0.1)
-
- remaining = worker.submit(decrement()).result(timeout=2)
- callback_thread_id, callback_loop = callback_info.result(timeout=2)
- finally:
- worker.stop()
- owner.join()
-
- assert remaining == 0
- assert callback_thread_id == worker.thread.ident
- assert callback_loop is worker.loop
-
-
-async def test_tracker_runs_zero_task_callback_immediately():
- tracker = EmbeddingTaskTracker.get_instance()
- callback_loop = None
-
- async def on_complete():
- nonlocal callback_loop
- callback_loop = asyncio.get_running_loop()
-
- await tracker.register("semantic-msg", 0, on_complete=on_complete)
-
- assert callback_loop is asyncio.get_running_loop()
-
-
-async def test_tracker_supports_sync_callback_and_missing_task():
- tracker = EmbeddingTaskTracker.get_instance()
- callback_calls = []
-
- await tracker.register("semantic-msg", 1, on_complete=lambda: callback_calls.append("done"))
- remaining = await tracker.decrement("semantic-msg")
-
- assert remaining == 0
- assert callback_calls == ["done"]
- assert await tracker.decrement("missing-semantic-msg") is None
-
-
-async def test_tracker_clears_zero_task_entry_without_callback():
- tracker = EmbeddingTaskTracker.get_instance()
-
- await tracker.register("semantic-msg", 0, on_complete=None)
-
- assert await tracker.decrement("semantic-msg") is None
-
-
-async def test_tracker_abort_ignores_late_decrements_without_completing():
- tracker = EmbeddingTaskTracker.get_instance()
- callback_calls = []
-
- await tracker.register(
- "semantic-msg:attempt-1",
- 2,
- on_complete=lambda: callback_calls.append("attempt-1"),
- )
-
- assert await tracker.abort("semantic-msg:attempt-1") is True
- assert await tracker.abort("semantic-msg:attempt-1") is False
-
- await tracker.register(
- "semantic-msg:attempt-2",
- 1,
- on_complete=lambda: callback_calls.append("attempt-2"),
- )
-
- assert await tracker.decrement("semantic-msg:attempt-1") is None
- assert callback_calls == []
- assert await tracker.decrement("semantic-msg:attempt-2") == 0
- assert callback_calls == ["attempt-2"]
diff --git a/tests/storage/test_memory_semantic_stall.py b/tests/storage/test_memory_semantic_stall.py
index 099fa35f5..fb2000a7e 100644
--- a/tests/storage/test_memory_semantic_stall.py
+++ b/tests/storage/test_memory_semantic_stall.py
@@ -7,6 +7,7 @@ Ensures that _process_memory_directory() error paths propagate exceptions
so that on_dequeue() always calls report_success() or report_error().
"""
+from types import SimpleNamespace
from unittest.mock import AsyncMock, MagicMock, patch
import pytest
@@ -206,7 +207,7 @@ async def test_memory_write_error_reports_error():
side_effect=lambda uri, ctx=None: f"/local/acc1/{uri.removeprefix('viking://')}"
)
- msg = _make_msg()
+ msg = _make_msg(skip_vectorization=True)
data = _build_data(msg)
success_called = False
@@ -234,6 +235,15 @@ async def test_memory_write_error_reports_error():
"openviking.storage.queuefs.semantic_processor.resolve_telemetry",
return_value=None,
),
+ patch(
+ "openviking.storage.queuefs.semantic_processor.get_openviking_config",
+ return_value=SimpleNamespace(
+ semantic=SimpleNamespace(
+ overview_max_chars=100_000,
+ abstract_max_chars=10_000,
+ )
+ ),
+ ),
patch.object(
processor,
"_generate_single_file_summary",
diff --git a/tests/storage/test_queue_manager.py b/tests/storage/test_queue_manager.py
index 6621cffe4..30a65b3a8 100644
--- a/tests/storage/test_queue_manager.py
+++ b/tests/storage/test_queue_manager.py
@@ -5,13 +5,14 @@
from openviking.storage.queuefs.queue_manager import QueueManager
-def test_external_parse_concurrent_uses_configured_value() -> None:
- manager = QueueManager(agfs=object(), max_concurrent_external_parse=9)
+def test_queue_concurrency_uses_separate_configured_values() -> None:
+ manager = QueueManager(
+ agfs=object(),
+ max_concurrent_external_parse=9,
+ max_concurrent_add_resource=7,
+ max_concurrent_session_commit=5,
+ )
assert manager._max_concurrent_for_queue(manager.EXTERNAL_PARSE) == 9
-
-
-def test_session_commit_concurrent_stays_at_four() -> None:
- manager = QueueManager(agfs=object(), max_concurrent_external_parse=9)
-
- assert manager._max_concurrent_for_queue(manager.SESSION_COMMIT) == 4
+ assert manager._max_concurrent_for_queue(manager.ADD_RESOURCE) == 7
+ assert manager._max_concurrent_for_queue(manager.SESSION_COMMIT) == 5
diff --git a/tests/storage/test_semantic_dag_incremental.py b/tests/storage/test_semantic_dag_incremental.py
index 1d9e8ed73..67405030e 100644
--- a/tests/storage/test_semantic_dag_incremental.py
+++ b/tests/storage/test_semantic_dag_incremental.py
@@ -2,7 +2,7 @@
# SPDX-License-Identifier: AGPL-3.0
import re
-from unittest.mock import AsyncMock, MagicMock
+from unittest.mock import MagicMock
import pytest
@@ -11,20 +11,12 @@ from openviking.storage.queuefs.semantic_dag import SemanticDagExecutor
from openviking_cli.session.user_id import UserIdentifier
-class _FakeAgfs:
- async def pathlock_acquire_exact_batch(self, _paths):
- return {"lease_ref": "test"}
-
- async def pathlock_release(self, _lease):
- return None
-
-
class _FakeVikingFS:
def __init__(self, tree, file_contents):
self._tree = {self._norm(k): v for k, v in tree.items()}
self._file_contents = {self._norm(k): v for k, v in file_contents.items()}
self.writes = []
- self._async_agfs = _FakeAgfs()
+ self._async_agfs = self
def _norm(self, path):
if "://" not in path:
@@ -48,6 +40,12 @@ class _FakeVikingFS:
self._file_contents[norm_path] = content
self.writes.append((norm_path, content))
+ async def pathlock_acquire_exact_batch(self, paths):
+ return {"paths": paths}
+
+ async def pathlock_release(self, lease):
+ return None
+
def _uri_to_path(self, uri, ctx=None):
return uri.replace("viking://", "/local/acc1/")
@@ -57,6 +55,7 @@ class _FakeProcessor:
self._fs = viking_fs
self.summarized_files = []
self.sync_calls = []
+ self.vectorized_files = []
def _parse_overview_md(self, overview_content):
results = {}
@@ -82,6 +81,29 @@ class _FakeProcessor:
def _normalize_overview_generation(self, overview):
return overview, "abstract"
+ async def _vectorize_single_file(
+ self,
+ parent_uri,
+ context_type,
+ file_path,
+ summary_dict,
+ ctx=None,
+ use_summary=False,
+ ingest_options=None,
+ ):
+ self.vectorized_files.append(file_path)
+
+ async def _vectorize_directory(
+ self,
+ uri,
+ context_type,
+ abstract,
+ overview,
+ ctx=None,
+ ingest_options=None,
+ ):
+ return None
+
async def _sync_topdown_recursive(
self, root_uri, target_uri, ctx=None, file_change_status=None, lock=None
):
@@ -134,27 +156,15 @@ async def test_direct_incremental_update_uses_changes_without_temp_sync(monkeypa
target_uri=root_uri,
changes={"modified": [f"{root_uri}/a.txt"]},
)
- add_vectorize_task = AsyncMock()
- monkeypatch.setattr(executor, "_add_vectorize_task", add_vectorize_task)
await executor.run(root_uri)
assert processor.summarized_files == [f"{root_uri}/a.txt"]
+ assert processor.vectorized_files == [f"{root_uri}/a.txt"]
assert processor.sync_calls == []
overview = fake_fs._file_contents[f"{root_uri}/.overview.md"]
assert "- a.txt: summary" in overview
assert "- b.txt: old-b" in overview
- changed_file_tasks = [
- call.args[0]
- for call in add_vectorize_task.await_args_list
- if call.args[0].task_type == "file"
- ]
- assert len(changed_file_tasks) == 1
- assert changed_file_tasks[0].file_path == f"{root_uri}/a.txt"
- assert changed_file_tasks[0].summary_dict == {
- "name": "a.txt",
- "summary": "summary",
- }
if __name__ == "__main__":
diff --git a/tests/storage/test_semantic_dag_skip_files.py b/tests/storage/test_semantic_dag_skip_files.py
index 947a206ba..4a09b8041 100644
--- a/tests/storage/test_semantic_dag_skip_files.py
+++ b/tests/storage/test_semantic_dag_skip_files.py
@@ -8,19 +8,11 @@ from openviking.storage.queuefs.semantic_dag import SemanticDagExecutor
from openviking_cli.session.user_id import UserIdentifier
-class _FakeAgfs:
- async def pathlock_acquire_exact_batch(self, _paths):
- return {"lease_ref": "test"}
-
- async def pathlock_release(self, _lease):
- return None
-
-
class _FakeVikingFS:
def __init__(self, tree):
self._tree = tree
self.writes = []
- self._async_agfs = _FakeAgfs()
+ self._async_agfs = self
async def ls(self, uri, node_limit=None, ctx=None):
return self._tree.get(uri, [])
@@ -28,6 +20,12 @@ class _FakeVikingFS:
async def write_file(self, path, content, ctx=None, lease_ref=None):
self.writes.append((path, content))
+ async def pathlock_acquire_exact_batch(self, paths):
+ return {"paths": paths}
+
+ async def pathlock_release(self, lease):
+ return None
+
def _uri_to_path(self, uri, ctx=None):
return uri.replace("viking://", "/local/acc1/")
@@ -54,7 +52,6 @@ class _FakeProcessor:
abstract,
overview,
ctx=None,
- semantic_msg_id=None,
ingest_options=None,
):
pass
@@ -69,18 +66,12 @@ class _FakeProcessor:
file_path,
summary_dict,
ctx=None,
- semantic_msg_id=None,
use_summary=False,
ingest_options=None,
):
self.vectorized_files.append(file_path)
-class _DummyTracker:
- async def register(self, **_kwargs):
- return None
-
-
@pytest.mark.asyncio
@pytest.mark.parametrize(
"root_uri",
@@ -100,10 +91,6 @@ async def test_messages_jsonl_excluded_from_summary(monkeypatch, root_uri):
}
fake_fs = _FakeVikingFS(tree)
monkeypatch.setattr("openviking.storage.queuefs.semantic_dag.get_viking_fs", lambda: fake_fs)
- monkeypatch.setattr(
- "openviking.storage.queuefs.embedding_tracker.EmbeddingTaskTracker.get_instance",
- lambda: _DummyTracker(),
- )
processor = _FakeProcessor()
ctx = RequestContext(user=UserIdentifier("acc1", "user1"), role=Role.USER)
@@ -142,10 +129,6 @@ async def test_messages_jsonl_excluded_in_subdirectory(monkeypatch, root_uri):
}
fake_fs = _FakeVikingFS(tree)
monkeypatch.setattr("openviking.storage.queuefs.semantic_dag.get_viking_fs", lambda: fake_fs)
- monkeypatch.setattr(
- "openviking.storage.queuefs.embedding_tracker.EmbeddingTaskTracker.get_instance",
- lambda: _DummyTracker(),
- )
processor = _FakeProcessor()
ctx = RequestContext(user=UserIdentifier("acc1", "user1"), role=Role.USER)
diff --git a/tests/storage/test_semantic_dag_stats.py b/tests/storage/test_semantic_dag_stats.py
index b3619f762..d6f6d0a83 100644
--- a/tests/storage/test_semantic_dag_stats.py
+++ b/tests/storage/test_semantic_dag_stats.py
@@ -6,7 +6,12 @@ import asyncio
import pytest
from openviking.server.identity import RequestContext, Role
-from openviking.service.task_work_index import TaskWorkIndex, TaskWorkRejected
+from openviking.service.task_work_index import (
+ TaskWorkIndex,
+ TaskWorkRejected,
+ bind_task_context,
+ get_task_context,
+)
from openviking.storage.queuefs.named_queue import NamedQueue
from openviking.storage.queuefs.semantic_dag import (
DagStats,
@@ -14,51 +19,56 @@ from openviking.storage.queuefs.semantic_dag import (
SemanticDagExecutor,
SemanticNodeScheduler,
)
+from openviking.telemetry import (
+ OperationTelemetry,
+ bind_telemetry,
+ get_current_telemetry,
+)
from openviking_cli.session.user_id import UserIdentifier
-class _FakeAgfs:
- async def pathlock_acquire_exact_batch(self, _paths):
- return {"lease_ref": "test"}
-
- async def pathlock_release(self, _lease):
- return None
-
-
class _FakeVikingFS:
def __init__(self, tree):
self._tree = tree
self.writes = []
- self._async_agfs = _FakeAgfs()
+ self._async_agfs = self
async def ls(self, uri, node_limit=None, ctx=None):
del node_limit
return self._tree.get(uri, [])
- async def write_file(
- self,
- path,
- content,
- ctx=None,
- lease_ref=None,
- lock_handle=None,
- ):
- del ctx, lease_ref, lock_handle
+ async def write_file(self, path, content, ctx=None, lease_ref=None):
self.writes.append((path, content))
+ async def pathlock_acquire_exact_batch(self, paths):
+ return {"paths": paths}
+
+ async def pathlock_release(self, lease):
+ return None
+
def _uri_to_path(self, uri, ctx=None):
return uri.replace("viking://", "/local/acc1/")
class _FakeProcessor:
- def __init__(self):
+ def __init__(self, verify_streaming=False):
self.vectorized_dirs = []
self.vectorized_files = []
+ self.vectorized_contexts = {}
+ self.verify_streaming = verify_streaming
async def _generate_single_file_summary(self, file_path, llm_sem=None, ctx=None):
- return {"name": file_path.split("/")[-1], "summary": "summary"}
+ result = {"name": file_path.split("/")[-1], "summary": "summary"}
+ if self.verify_streaming:
+ result["content"] = "x" * 100_000
+ return result
async def _generate_overview(self, dir_uri, file_summaries, children_abstracts):
+ if self.verify_streaming:
+ assert all("content" not in item for item in file_summaries)
+ assert all(
+ f"{dir_uri}/{item['name']}" in self.vectorized_files for item in file_summaries
+ )
return "overview"
def _normalize_overview_generation(self, overview):
@@ -71,7 +81,6 @@ class _FakeProcessor:
abstract,
overview,
ctx=None,
- semantic_msg_id=None,
ingest_options=None,
):
self.vectorized_dirs.append(uri)
@@ -83,11 +92,17 @@ class _FakeProcessor:
file_path,
summary_dict,
ctx=None,
- semantic_msg_id=None,
use_summary=False,
ingest_options=None,
):
+ if self.verify_streaming:
+ assert summary_dict["content"]
self.vectorized_files.append(file_path)
+ task_context = get_task_context()
+ self.vectorized_contexts[file_path] = (
+ task_context.task_id if task_context is not None else None,
+ get_current_telemetry().telemetry_id,
+ )
async def _vectorize_directory_simple(self, uri, context_type, abstract, overview, ctx=None):
await self._vectorize_directory(uri, context_type, abstract, overview, ctx=ctx)
@@ -109,168 +124,6 @@ class _TrackingProcessor(_FakeProcessor):
self.active_summaries -= 1
-class _DummyTracker:
- def __init__(self):
- self.register_calls = []
-
- async def register(self, **_kwargs):
- self.register_calls.append(_kwargs)
- return None
-
-
-class _CompletingTracker:
- def __init__(self):
- self.remaining = 0
- self.total = 0
- self.on_complete = None
- self.completed = False
- self.active_id = None
- self.aborted_ids = []
-
- async def register(self, *, semantic_msg_id, total_count, on_complete, **_kwargs):
- self.active_id = semantic_msg_id
- self.remaining = total_count
- self.total = total_count
- self.on_complete = on_complete
-
- async def decrement(self, semantic_msg_id):
- if semantic_msg_id != self.active_id:
- return None
- self.remaining -= 1
- if self.remaining == 0:
- self.completed = True
- self.active_id = None
- await self.on_complete()
- return self.remaining
-
- async def abort(self, semantic_msg_id):
- if semantic_msg_id != self.active_id:
- return False
- self.aborted_ids.append(semantic_msg_id)
- self.active_id = None
- return True
-
-
-class _FailingVectorizeProcessor(_FakeProcessor):
- def __init__(self, tracker):
- super().__init__()
- self._tracker = tracker
-
- async def _vectorize_single_file(
- self,
- parent_uri,
- context_type,
- file_path,
- summary_dict,
- ctx=None,
- semantic_msg_id=None,
- use_summary=False,
- ingest_options=None,
- ):
- self.vectorized_files.append(file_path)
- try:
- raise RuntimeError("embedding enqueue unavailable")
- finally:
- await self._tracker.decrement(semantic_msg_id)
-
- async def _vectorize_directory(
- self,
- uri,
- context_type,
- abstract,
- overview,
- ctx=None,
- semantic_msg_id=None,
- ingest_options=None,
- ):
- self.vectorized_dirs.append(uri)
- await self._tracker.decrement(semantic_msg_id)
- await self._tracker.decrement(semantic_msg_id)
-
-
-class _StalledEmbeddingProcessor(_FailingVectorizeProcessor):
- async def _vectorize_directory(
- self,
- uri,
- context_type,
- abstract,
- overview,
- ctx=None,
- semantic_msg_id=None,
- ingest_options=None,
- ):
- self.vectorized_dirs.append(uri)
- # Both directory messages were accepted, but the embedding worker is
- # unavailable and therefore never decrements their tracker slots.
-
-
-async def test_semantic_dag_propagates_vectorization_failure_after_tracker_drains(monkeypatch):
- root_uri = "viking://resources/root"
- tree = {root_uri: [{"name": "a.txt", "isDir": False}]}
- fake_fs = _FakeVikingFS(tree)
- tracker = _CompletingTracker()
- monkeypatch.setattr("openviking.storage.queuefs.semantic_dag.get_viking_fs", lambda: fake_fs)
- monkeypatch.setattr(
- "openviking.storage.queuefs.embedding_tracker.EmbeddingTaskTracker.get_instance",
- lambda: tracker,
- )
-
- processor = _FailingVectorizeProcessor(tracker)
- ctx = RequestContext(user=UserIdentifier("acc1", "user1"), role=Role.USER)
- executor = SemanticDagExecutor(
- processor=processor,
- context_type="resource",
- max_concurrent_llm=2,
- ctx=ctx,
- semantic_msg_id="semantic-root",
- )
-
- with pytest.raises(RuntimeError, match="embedding enqueue unavailable"):
- await executor.run(root_uri)
-
- assert processor.vectorized_files == [f"{root_uri}/a.txt"]
- assert processor.vectorized_dirs == [root_uri]
- assert tracker.total == 3
- assert tracker.remaining == 0
- assert tracker.completed is True
-
-
-async def test_semantic_dag_aborts_stalled_embedding_attempt_after_dispatch_failure(monkeypatch):
- root_uri = "viking://resources/root"
- tree = {root_uri: [{"name": "a.txt", "isDir": False}]}
- fake_fs = _FakeVikingFS(tree)
- tracker = _CompletingTracker()
- monkeypatch.setattr("openviking.storage.queuefs.semantic_dag.get_viking_fs", lambda: fake_fs)
- monkeypatch.setattr(
- "openviking.storage.queuefs.embedding_tracker.EmbeddingTaskTracker.get_instance",
- lambda: tracker,
- )
-
- processor = _StalledEmbeddingProcessor(tracker)
- ctx = RequestContext(user=UserIdentifier("acc1", "user1"), role=Role.USER)
- executor = SemanticDagExecutor(
- processor=processor,
- context_type="resource",
- max_concurrent_llm=2,
- ctx=ctx,
- semantic_msg_id="semantic-root",
- )
-
- with pytest.raises(RuntimeError, match="embedding enqueue unavailable"):
- await asyncio.wait_for(executor.run(root_uri), timeout=0.5)
-
- assert processor.vectorized_files == [f"{root_uri}/a.txt"]
- assert processor.vectorized_dirs == [root_uri]
- assert tracker.remaining == 2
- assert tracker.completed is False
- assert len(tracker.aborted_ids) == 1
- aborted_id = tracker.aborted_ids[0]
- assert aborted_id.startswith("semantic-root:")
- assert await tracker.decrement(aborted_id) is None
-
-
class _ScheduledExecutor:
def __init__(self, run) -> None:
self.closed = False
@@ -305,12 +158,8 @@ async def test_semantic_dag_stats_collects_nodes(monkeypatch):
}
fake_fs = _FakeVikingFS(tree)
monkeypatch.setattr("openviking.storage.queuefs.semantic_dag.get_viking_fs", lambda: fake_fs)
- monkeypatch.setattr(
- "openviking.storage.queuefs.embedding_tracker.EmbeddingTaskTracker.get_instance",
- lambda: _DummyTracker(),
- )
- processor = _FakeProcessor()
+ processor = _FakeProcessor(verify_streaming=True)
ctx = RequestContext(user=UserIdentifier("acc1", "user1"), role=Role.USER)
executor = SemanticDagExecutor(
processor=processor,
@@ -345,10 +194,6 @@ async def test_semantic_dag_bounds_active_node_work(monkeypatch):
}
fake_fs = _FakeVikingFS(tree)
monkeypatch.setattr("openviking.storage.queuefs.semantic_dag.get_viking_fs", lambda: fake_fs)
- monkeypatch.setattr(
- "openviking.storage.queuefs.embedding_tracker.EmbeddingTaskTracker.get_instance",
- lambda: _DummyTracker(),
- )
processor = _TrackingProcessor()
ctx = RequestContext(user=UserIdentifier("acc1", "user1"), role=Role.USER)
@@ -386,33 +231,43 @@ async def test_semantic_dag_shares_node_scheduler_across_roots(monkeypatch):
}
fake_fs = _FakeVikingFS(tree)
monkeypatch.setattr("openviking.storage.queuefs.semantic_dag.get_viking_fs", lambda: fake_fs)
- monkeypatch.setattr(
- "openviking.storage.queuefs.embedding_tracker.EmbeddingTaskTracker.get_instance",
- lambda: _DummyTracker(),
- )
processor = _TrackingProcessor()
ctx = RequestContext(user=UserIdentifier("acc1", "user1"), role=Role.USER)
- executor_a = SemanticDagExecutor(
- processor=processor,
- context_type="resource",
- max_concurrent_llm=4,
- ctx=ctx,
- skip_vectorization=True,
- )
- executor_b = SemanticDagExecutor(
- processor=processor,
- context_type="resource",
- max_concurrent_llm=4,
- ctx=ctx,
- skip_vectorization=True,
- )
+ telemetry_a = OperationTelemetry("semantic-a")
+ telemetry_b = OperationTelemetry("semantic-b")
+ with (
+ bind_task_context("task-a", "acc1", "user1"),
+ bind_telemetry(telemetry_a),
+ ):
+ executor_a = SemanticDagExecutor(
+ processor=processor,
+ context_type="resource",
+ max_concurrent_llm=1,
+ ctx=ctx,
+ )
+ with (
+ bind_task_context("task-b", "acc1", "user1"),
+ bind_telemetry(telemetry_b),
+ ):
+ executor_b = SemanticDagExecutor(
+ processor=processor,
+ context_type="resource",
+ max_concurrent_llm=1,
+ ctx=ctx,
+ )
await asyncio.gather(executor_a.run(root_a), executor_b.run(root_b))
- assert processor.max_active_summaries <= 4
+ assert processor.max_active_summaries == 1
assert executor_a.get_stats().done_nodes == 21
assert executor_b.get_stats().done_nodes == 21
+ assert {
+ processor.vectorized_contexts[f"{root_a}/a-{idx}.txt"] for idx in range(20)
+ } == {("task-a", telemetry_a.telemetry_id)}
+ assert {
+ processor.vectorized_contexts[f"{root_b}/b-{idx}.txt"] for idx in range(20)
+ } == {("task-b", telemetry_b.telemetry_id)}
@pytest.mark.asyncio
@@ -476,12 +331,7 @@ async def test_semantic_dag_skip_vectorization_does_not_schedule_tasks(monkeypat
],
}
fake_fs = _FakeVikingFS(tree)
- tracker = _DummyTracker()
monkeypatch.setattr("openviking.storage.queuefs.semantic_dag.get_viking_fs", lambda: fake_fs)
- monkeypatch.setattr(
- "openviking.storage.queuefs.embedding_tracker.EmbeddingTaskTracker.get_instance",
- lambda: tracker,
- )
processor = _FakeProcessor()
ctx = RequestContext(user=UserIdentifier("acc1", "user1"), role=Role.USER)
@@ -503,8 +353,5 @@ async def test_semantic_dag_skip_vectorization_does_not_schedule_tasks(monkeypat
]
assert processor.vectorized_dirs == []
assert processor.vectorized_files == []
- assert tracker.register_calls == []
-
-
if __name__ == "__main__":
pytest.main([__file__])
diff --git a/tests/storage/test_semantic_queue_memory_dedupe.py b/tests/storage/test_semantic_queue_memory_dedupe.py
index c69c4320b..508ff21a2 100644
--- a/tests/storage/test_semantic_queue_memory_dedupe.py
+++ b/tests/storage/test_semantic_queue_memory_dedupe.py
@@ -125,8 +125,8 @@ class _FakeVikingFS:
del ctx
return f"/fake/{uri.replace('://', '/').strip('/')}"
- async def write_file(self, uri, content, ctx=None):
- del ctx
+ async def write_file(self, uri, content, ctx=None, lease_ref=None):
+ del ctx, lease_ref
self.writes.append((uri, content))
@@ -246,10 +246,12 @@ async def test_memory_directory_vectorizes_changed_files_with_generated_summary(
async def generate_file_summary(file_path, llm_sem=None, ctx=None):
del llm_sem, ctx
name = file_path.rsplit("/", 1)[-1]
- return {"name": name, "summary": f"summary:{name}"}
+ return {"name": name, "summary": f"summary:{name}", "content": "raw content"}
async def generate_overview(dir_uri, file_summaries, children_abstracts, llm_sem=None):
- del dir_uri, file_summaries, children_abstracts, llm_sem
+ del dir_uri, children_abstracts, llm_sem
+ assert len(captured_file_vectorize) == 1
+ assert all("content" not in summary for summary in file_summaries)
return "overview"
async def write_semantics(**kwargs):
@@ -292,6 +294,7 @@ async def test_memory_directory_vectorizes_changed_files_with_generated_summary(
assert captured_file_vectorize[0]["summary_dict"] == {
"name": "first.md",
"summary": "summary:first.md",
+ "content": "raw content",
}
assert captured_file_vectorize[0]["preserve_existing_created_at"] is True
assert len(captured_directory_vectorize) == 1
diff --git a/tests/test_config_loader.py b/tests/test_config_loader.py
index c319c64db..9cf26591c 100644
--- a/tests/test_config_loader.py
+++ b/tests/test_config_loader.py
@@ -121,24 +121,34 @@ class TestRequireConfig:
require_config(None, "TEST_MISSING_ENV", "nonexistent_file.conf", "test")
-def test_external_parse_worker_max_concurrent_defaults_to_four():
+def test_queue_worker_concurrency_defaults_to_four():
config = OpenVikingConfig.from_dict({})
assert config.queue_workers.external_parse.max_concurrent == 4
+ assert config.queue_workers.add_resource.max_concurrent == 4
+ assert config.queue_workers.session_commit.max_concurrent == 4
-def test_external_parse_worker_max_concurrent_accepts_positive_value():
+def test_queue_worker_concurrency_accepts_separate_values():
config = OpenVikingConfig.from_dict(
- {"queue_workers": {"external_parse": {"max_concurrent": 9}}}
+ {
+ "queue_workers": {
+ "external_parse": {"max_concurrent": 9},
+ "add_resource": {"max_concurrent": 7},
+ "session_commit": {"max_concurrent": 5},
+ }
+ }
)
assert config.queue_workers.external_parse.max_concurrent == 9
+ assert config.queue_workers.add_resource.max_concurrent == 7
+ assert config.queue_workers.session_commit.max_concurrent == 5
@pytest.mark.parametrize("value", [0, -1])
-def test_external_parse_worker_max_concurrent_rejects_non_positive_value(value):
+def test_queue_worker_concurrency_rejects_non_positive_value(value):
with pytest.raises(ValueError) as exc_info:
- QueueWorkersConfig(external_parse={"max_concurrent": value})
+ QueueWorkersConfig(add_resource={"max_concurrent": value})
assert exc_info.value.errors()[0]["type"] == "greater_than"
diff --git a/tests/unit/service/test_core_consistency.py b/tests/unit/service/test_core_consistency.py
index bf616c0c6..9a43c23fe 100644
--- a/tests/unit/service/test_core_consistency.py
+++ b/tests/unit/service/test_core_consistency.py
@@ -26,10 +26,12 @@ def test_service_passes_queue_worker_concurrency_to_storage(monkeypatch) -> None
dimension=1024,
get_embedder=lambda: embedder,
),
- vlm=SimpleNamespace(max_concurrent=64),
+ vlm=SimpleNamespace(max_concurrent=32),
parser_api=SimpleNamespace(),
queue_workers=SimpleNamespace(
external_parse=SimpleNamespace(max_concurrent=9),
+ add_resource=SimpleNamespace(max_concurrent=7),
+ session_commit=SimpleNamespace(max_concurrent=5),
),
git=object(),
)
@@ -53,6 +55,8 @@ def test_service_passes_queue_worker_concurrency_to_storage(monkeypatch) -> None
OpenVikingService()
assert storage_calls[0][1]["max_concurrent_external_parse"] == 9
+ assert storage_calls[0][1]["max_concurrent_add_resource"] == 7
+ assert storage_calls[0][1]["max_concurrent_session_commit"] == 5
def _service_with_fs(stat_result: dict) -> tuple[OpenVikingService, AsyncMock]:
diff --git a/tests/unit/test_vectorize_file_strategy.py b/tests/unit/test_vectorize_file_strategy.py
index 482972585..0e8c5716e 100644
--- a/tests/unit/test_vectorize_file_strategy.py
+++ b/tests/unit/test_vectorize_file_strategy.py
@@ -1,5 +1,4 @@
import types
-from unittest.mock import AsyncMock
import pytest
@@ -18,6 +17,7 @@ class DummyQueue:
async def enqueue(self, msg):
self.items.append(msg)
+ return "queue-message-id"
class DummyQueueWithId(DummyQueue):
@@ -200,7 +200,6 @@ async def test_vectorize_file_registers_request_wait_with_embedding_msg_id(monke
summary_dict={"name": "test.md", "summary": ""},
parent_uri="viking://user/default/resources",
ctx=DummyReq(),
- register_request_wait=True,
)
assert len(queue.items) == 1
@@ -241,7 +240,6 @@ async def test_vectorize_file_marks_registered_wait_root_failed_when_enqueue_rai
summary_dict={"name": "test.md", "summary": ""},
parent_uri="viking://user/default/resources",
ctx=DummyReq(),
- register_request_wait=True,
)
assert len(queue.items) == 1
@@ -273,12 +271,10 @@ async def test_vectorize_file_propagates_enqueue_failure(monkeypatch):
@pytest.mark.asyncio
-async def test_vectorize_directory_propagates_enqueue_failures_and_drains_tracker(monkeypatch):
- decrement = AsyncMock()
+async def test_vectorize_directory_propagates_enqueue_failures(monkeypatch):
monkeypatch.setattr(
embedding_utils, "get_queue_manager", lambda: DummyQueueManager(FailingQueue())
)
- monkeypatch.setattr(embedding_utils, "_decrement_embedding_tracker", decrement)
with pytest.raises(RuntimeError, match="queue unavailable"):
await embedding_utils.vectorize_directory_meta(
@@ -286,11 +282,8 @@ async def test_vectorize_directory_propagates_enqueue_failures_and_drains_tracke
abstract="abstract",
overview="overview",
ctx=DummyReq(),
- semantic_msg_id="semantic-root",
)
- decrement.assert_awaited_once_with("semantic-root", 2)
-
@pytest.mark.asyncio
async def test_vectorize_unknown_text_file_embeds_summary_without_reading_content(monkeypatch):
diff --git a/tests/utils/test_resource_processor_processing_mode.py b/tests/utils/test_resource_processor_processing_mode.py
index 58a92bcd8..13c36224f 100644
--- a/tests/utils/test_resource_processor_processing_mode.py
+++ b/tests/utils/test_resource_processor_processing_mode.py
@@ -6,8 +6,8 @@ from unittest.mock import AsyncMock, Mock
import pytest
-from openviking.utils.ingest_options import IngestOptions
from openviking.server.identity import RequestContext, Role
+from openviking.utils.ingest_options import IngestOptions
from openviking.utils.resource_processor import ResourceProcessor
from openviking_cli.session.user_id import UserIdentifier
@@ -260,7 +260,6 @@ async def test_vectors_only_persists_tree_and_vectorizes_files_only(monkeypatch,
search_tags=["team=search"],
search_tag_mode="append",
)
- assert vectorize_file.await_args.kwargs["register_request_wait"] is True
processor._delete_resource_semantic_markers.assert_not_awaited()
processor._delete_resource_semantic_vectors.assert_not_awaited()
viking_fs._async_agfs.pathlock_release.assert_awaited_once_with(lock)