Skip to content

Self-evolving Context Database for AI Agents. Unify Agent Memory, Knowledge RAG and Skills.

Perf/reindex vector enqueue concurrency (#4071)

Jiahui Zhou22f000332068
diff --git a/docs/en/configuration/01-server.md b/docs/en/configuration/01-server.md
index aedcc2bc0..647984257 100644
--- a/docs/en/configuration/01-server.md
+++ b/docs/en/configuration/01-server.md
@@ -213,6 +213,9 @@ This setting controls queue-job concurrency. It is separate from `vlm.media.max_
| Field | Type | Default | Description |
|---|---|---:|---|
| `max_concurrent` | integer | `4` | Number of complete AddResource jobs consumed concurrently; must be greater than `0`; requires a server restart after changes |
+| `file_vectorization_concurrency` | integer | `8` | Number of files concurrently read, prepared, and enqueued within one directory AddResource job when `processing_mode="vectors_only"`; must be greater than `0`; values above the internal safety limit of `64` are capped; requires a server restart after changes |
+
+`max_concurrent` controls independent AddResource jobs, while `file_vectorization_concurrency` controls files within one vectors-only directory job. It does not affect single-file resources or `semantic_and_vectors` processing.
### `queue_workers.session_commit`
@@ -220,6 +223,14 @@ This setting controls queue-job concurrency. It is separate from `vlm.media.max_
|---|---|---:|---|
| `max_concurrent` | integer | `8` | Number of SessionCommit jobs consumed concurrently; must be greater than `0`; requires a server restart after changes |
+## Reindex Settings
+
+### `reindex`
+
+| Field | Type | Default | Description |
+|---|---|---:|---|
+| `file_vectorization_concurrency` | integer | `8` | Number of files concurrently read, prepared, and enqueued by one `vectors_only` reindex task; must be greater than `0`; values above the internal safety limit of `64` are capped; requires a server restart after changes |
+
## HTTP Server Settings
```json
diff --git a/docs/zh/configuration/01-server.md b/docs/zh/configuration/01-server.md
index d9fa39698..2e6d98728 100644
--- a/docs/zh/configuration/01-server.md
+++ b/docs/zh/configuration/01-server.md
@@ -213,6 +213,9 @@ Search 和 Find 请求的默认 `limit` 为 `10`,可以在每次 API 或 SDK
| 字段 | 类型 | 默认值 | 说明 |
|---|---|---:|---|
| `max_concurrent` | integer | `4` | 同时消费的完整 AddResource 作业数,必须大于 `0`;修改后需重启服务 |
+| `file_vectorization_concurrency` | integer | `8` | 当目录 AddResource 使用 `processing_mode="vectors_only"` 时,单个作业内并发读取、准备并入队的文件数,必须大于 `0`;超过内部安全上限 `64` 的值会被截断;修改后需重启服务 |
+
+`max_concurrent` 控制相互独立的 AddResource 作业并发,`file_vectorization_concurrency` 控制单个 vectors-only 目录作业内的文件并发。该配置不影响单文件资源或 `semantic_and_vectors` 处理。
### `queue_workers.session_commit`
@@ -220,6 +223,14 @@ Search 和 Find 请求的默认 `limit` 为 `10`,可以在每次 API 或 SDK
|---|---|---:|---|
| `max_concurrent` | integer | `8` | 同时消费的 SessionCommit 作业数,必须大于 `0`;修改后需重启服务 |
+## Reindex 配置
+
+### `reindex`
+
+| 字段 | 类型 | 默认值 | 说明 |
+|---|---|---:|---|
+| `file_vectorization_concurrency` | integer | `8` | 单个 `vectors_only` reindex 任务内并发读取、准备并入队的文件数,必须大于 `0`;超过内部安全上限 `64` 的值会被截断;修改后需重启服务 |
+
## HTTP 服务配置
```json
diff --git a/examples/ov.conf.example b/examples/ov.conf.example
index 52a3d6a2c..e461e3e05 100644
--- a/examples/ov.conf.example
+++ b/examples/ov.conf.example
@@ -84,9 +84,15 @@
},
"queue_workers": {
"external_parse": {"max_concurrent": 4},
- "add_resource": {"max_concurrent": 4},
+ "add_resource": {
+ "max_concurrent": 4,
+ "file_vectorization_concurrency": 8
+ },
"session_commit": {"max_concurrent": 8},
},
+ "reindex": {
+ "file_vectorization_concurrency": 8
+ },
"embedding": {
"dense": {
"model": "doubao-embedding-vision-251215",
diff --git a/openviking/service/reindex_executor.py b/openviking/service/reindex_executor.py
index be323b0ec..e0f389301 100644
--- a/openviking/service/reindex_executor.py
+++ b/openviking/service/reindex_executor.py
@@ -55,6 +55,7 @@ logger = get_logger(__name__)
REINDEX_TASK_TYPE = "admin_reindex"
PRUNE_ORPHAN_CANDIDATE_LIMIT = 100000
PRUNE_OUTPUT_FIELDS = ["id", "uri", "level", "context_type", "account_id", "owner_user_id"]
+_MAX_FILE_VECTORIZATION_CONCURRENCY = 64
# Trailing markers VikingFS appends when a directory has no generated .abstract.md/.overview.md
@@ -101,6 +102,15 @@ class _ReindexCounters:
failed_records: int = 0
warnings: list[str] = field(default_factory=list)
+ def merge_from(self, other: "_ReindexCounters") -> None:
+ self.scanned_records += other.scanned_records
+ self.rebuilt_records += other.rebuilt_records
+ self.deleted_records += other.deleted_records
+ self.would_delete_records += other.would_delete_records
+ self.unsupported_records += other.unsupported_records
+ self.failed_records += other.failed_records
+ self.warnings.extend(other.warnings)
+
@dataclass
class _ReindexRunContext:
@@ -129,6 +139,30 @@ class ReindexExecutor:
"memory": {"vectors_only", "semantic_and_vectors", "prune_orphans"},
}
+ @staticmethod
+ def _effective_file_vectorization_concurrency() -> int:
+ config = get_openviking_config().reindex
+ return max(
+ 1,
+ min(
+ int(config.file_vectorization_concurrency),
+ _MAX_FILE_VECTORIZATION_CONCURRENCY,
+ ),
+ )
+
+ async def _run_ordered_counter_batches(
+ self,
+ items: list[str],
+ *,
+ concurrency: int,
+ processor: Any,
+ counters: _ReindexCounters,
+ ) -> None:
+ for start in range(0, len(items), concurrency):
+ batch = items[start : start + concurrency]
+ for item_counters in await asyncio.gather(*(processor(item) for item in batch)):
+ counters.merge_from(item_counters)
+
@staticmethod
def _content_owner_ctx(uri: str, ctx: RequestContext) -> RequestContext:
"""Return the content-owner context for user-scoped reindex writes.
@@ -1122,15 +1156,15 @@ class ReindexExecutor:
counters.failed_records += 1
counters.warnings.append(f"Failed to reindex {directory_uri} L1 vector: {exc}")
- for file_uri in deduped_files:
- counters.scanned_records += 1
+ async def process_file(file_uri: str) -> _ReindexCounters:
+ file_counters = _ReindexCounters(scanned_records=1)
parent_uri = VikingURI(file_uri).parent.uri
summary = await self._best_file_summary(file_uri, ctx=ctx)
vector_text = await self._best_resource_file_vector_text(file_uri, summary, ctx=ctx)
if not vector_text:
- counters.unsupported_records += 1
- counters.warnings.append(f"No vector source found for {file_uri}")
- continue
+ file_counters.unsupported_records += 1
+ file_counters.warnings.append(f"No vector source found for {file_uri}")
+ return file_counters
abstract = self._prefer_non_empty(summary, vector_text)
try:
await self._upsert_context(
@@ -1144,10 +1178,26 @@ class ReindexExecutor:
ctx=ctx,
ingest_options=ingest_options,
)
- counters.rebuilt_records += 1
+ file_counters.rebuilt_records += 1
except Exception as exc:
- counters.failed_records += 1
- counters.warnings.append(f"Failed to reindex {file_uri} vector: {exc}")
+ file_counters.failed_records += 1
+ file_counters.warnings.append(f"Failed to reindex {file_uri} vector: {exc}")
+ return file_counters
+
+ concurrency = self._effective_file_vectorization_concurrency()
+ if deduped_files:
+ logger.info(
+ "Reindex resource file vectorization: root=%s files=%d concurrency=%d",
+ root_uri,
+ len(deduped_files),
+ concurrency,
+ )
+ await self._run_ordered_counter_batches(
+ deduped_files,
+ concurrency=concurrency,
+ processor=process_file,
+ counters=counters,
+ )
async def reindex_directory_marker(
self, *, dir_uri: str, level: ContextLevel, ctx: RequestContext
@@ -1519,15 +1569,15 @@ class ReindexExecutor:
else:
raise NotFoundError(uri, "memory")
- for file_uri in file_uris:
- counters.scanned_records += 1
+ async def process_file(file_uri: str) -> _ReindexCounters:
+ file_counters = _ReindexCounters(scanned_records=1)
body_source = await self._read_memory_body(file_uri, ctx=ctx)
if body_source.error:
- counters.failed_records += 1
- counters.warnings.append(
+ file_counters.failed_records += 1
+ file_counters.warnings.append(
f"Skipped {file_uri}: failed to read memory body: {body_source.error}"
)
- continue
+ return file_counters
body = body_source.text if body_source.exists else ""
memory_content = MemoryFileUtils.read(body).content if body else ""
existing = await self._fetch_existing_record(
@@ -1540,9 +1590,9 @@ class ReindexExecutor:
await self._best_file_summary(file_uri, ctx=ctx),
)
if not body and existing is None:
- counters.unsupported_records += 1
- counters.warnings.append(f"No memory source found for {file_uri}")
- continue
+ file_counters.unsupported_records += 1
+ file_counters.warnings.append(f"No memory source found for {file_uri}")
+ return file_counters
parent_uri = VikingURI(file_uri.split("#", 1)[0]).parent.uri
if body:
@@ -1559,12 +1609,11 @@ class ReindexExecutor:
ctx=ctx,
ingest_options=ingest_options,
)
- counters.rebuilt_records += 1
+ file_counters.rebuilt_records += 1
except Exception as exc:
- counters.failed_records += 1
- counters.warnings.append(f"Failed to reindex {file_uri} vector: {exc}")
- continue
- continue
+ file_counters.failed_records += 1
+ file_counters.warnings.append(f"Failed to reindex {file_uri} vector: {exc}")
+ return file_counters
try:
await self._upsert_context(
@@ -1578,13 +1627,29 @@ class ReindexExecutor:
ctx=ctx,
ingest_options=ingest_options,
)
- counters.rebuilt_records += 1
- counters.warnings.append(
+ file_counters.rebuilt_records += 1
+ file_counters.warnings.append(
f"Reindexed {file_uri} from abstract fallback because original memory body is unavailable"
)
except Exception as exc:
- counters.failed_records += 1
- counters.warnings.append(f"Failed to reindex {file_uri} vector: {exc}")
+ file_counters.failed_records += 1
+ file_counters.warnings.append(f"Failed to reindex {file_uri} vector: {exc}")
+ return file_counters
+
+ concurrency = self._effective_file_vectorization_concurrency()
+ if file_uris:
+ logger.info(
+ "Reindex memory file vectorization: root=%s files=%d concurrency=%d",
+ uri,
+ len(file_uris),
+ concurrency,
+ )
+ await self._run_ordered_counter_batches(
+ file_uris,
+ concurrency=concurrency,
+ processor=process_file,
+ counters=counters,
+ )
async def _reindex_memory_directory_chain(
self,
diff --git a/openviking/utils/resource_processor.py b/openviking/utils/resource_processor.py
index 608e9ecc2..a5fd601c9 100644
--- a/openviking/utils/resource_processor.py
+++ b/openviking/utils/resource_processor.py
@@ -7,6 +7,7 @@ Handles coordinated writes and self-iteration processes
as described in the OpenViking design document.
"""
+import asyncio
import inspect
import time
from collections.abc import Callable
@@ -36,6 +37,7 @@ from openviking.utils.ingest_options import IngestOptions
from openviking.utils.summarizer import Summarizer
from openviking_cli.exceptions import OpenVikingError
from openviking_cli.utils import VikingURI, get_logger
+from openviking_cli.utils.config import get_openviking_config
from openviking_cli.utils.storage import StoragePath
if TYPE_CHECKING:
@@ -43,6 +45,7 @@ if TYPE_CHECKING:
from openviking.parse.vlm import VLMProcessor
logger = get_logger(__name__)
+_MAX_FILE_VECTORIZATION_CONCURRENCY = 64
VECTORDB_MAX_QUERY_LIMIT = 100_000
@@ -652,6 +655,7 @@ class ResourceProcessor:
level_limit=None,
ctx=ctx,
)
+ files: list[tuple[str, str, str]] = []
for entry in entries:
entry_uri = entry.get("uri") if isinstance(entry, dict) else None
if not entry_uri or entry.get("isDir"):
@@ -662,15 +666,40 @@ class ResourceProcessor:
parent = VikingURI(entry_uri).parent
if parent is None:
continue
+ files.append((entry_uri, str(name), parent.uri))
+
+ config = get_openviking_config().queue_workers.add_resource
+ concurrency = max(
+ 1,
+ min(
+ int(config.file_vectorization_concurrency),
+ _MAX_FILE_VECTORIZATION_CONCURRENCY,
+ ),
+ )
+
+ async def vectorize(entry_uri: str, name: str, parent_uri: str) -> None:
await vectorize_file(
file_path=entry_uri,
summary_dict={"name": name, "summary": ""},
- parent_uri=parent.uri,
+ parent_uri=parent_uri,
context_type=context_type_for_uri(entry_uri),
ctx=ctx,
ingest_options=ingest_options,
)
+ for start in range(0, len(files), concurrency):
+ tasks = [
+ asyncio.create_task(vectorize(entry_uri, name, parent_uri))
+ for entry_uri, name, parent_uri in files[start : start + concurrency]
+ ]
+ try:
+ await asyncio.gather(*tasks)
+ except BaseException:
+ for task in tasks:
+ task.cancel()
+ await asyncio.gather(*tasks, return_exceptions=True)
+ raise
+
async def _vectorize_resource_file(
self,
file_uri: str,
diff --git a/openviking_cli/utils/config/open_viking_config.py b/openviking_cli/utils/config/open_viking_config.py
index c7fbb2451..601c9eb42 100644
--- a/openviking_cli/utils/config/open_viking_config.py
+++ b/openviking_cli/utils/config/open_viking_config.py
@@ -44,6 +44,7 @@ from .parser_config import (
)
from .prompts_config import PromptsConfig
from .queue_worker_config import QueueWorkersConfig
+from .reindex_config import ReindexConfig
from .rerank_config import RerankConfig
from .retrieval_config import RetrievalConfig
from .storage_config import StorageConfig
@@ -241,6 +242,11 @@ class OpenVikingConfig(BaseModel):
description="Queue worker runtime configuration",
)
+ reindex: ReindexConfig = Field(
+ default_factory=ReindexConfig,
+ description="Admin reindex runtime configuration",
+ )
+
parser_api: ParserApiConfig = Field(
default_factory=ParserApiConfig,
description="Third-party parser API configuration (files/responses)",
diff --git a/openviking_cli/utils/config/queue_worker_config.py b/openviking_cli/utils/config/queue_worker_config.py
index 7abed5682..406ef513b 100644
--- a/openviking_cli/utils/config/queue_worker_config.py
+++ b/openviking_cli/utils/config/queue_worker_config.py
@@ -17,11 +17,23 @@ class QueueWorkerConfig(BaseModel):
model_config = {"extra": "forbid"}
+class AddResourceQueueWorkerConfig(QueueWorkerConfig):
+ """Runtime limits for add-resource queue workers."""
+
+ file_vectorization_concurrency: int = Field(
+ default=8,
+ gt=0,
+ description="Maximum number of files read, prepared, and enqueued concurrently within one vectors-only job",
+ )
+
+
class QueueWorkersConfig(BaseModel):
"""Runtime limits for QueueFS consumers."""
external_parse: QueueWorkerConfig = Field(default_factory=QueueWorkerConfig)
- add_resource: QueueWorkerConfig = Field(default_factory=QueueWorkerConfig)
+ add_resource: AddResourceQueueWorkerConfig = Field(
+ default_factory=AddResourceQueueWorkerConfig
+ )
session_commit: QueueWorkerConfig = Field(
default_factory=lambda: QueueWorkerConfig(max_concurrent=8)
)
diff --git a/openviking_cli/utils/config/reindex_config.py b/openviking_cli/utils/config/reindex_config.py
new file mode 100644
index 000000000..ec71ede6a
--- /dev/null
+++ b/openviking_cli/utils/config/reindex_config.py
@@ -0,0 +1,17 @@
+# Copyright (c) 2026 Beijing Volcano Engine Technology Co., Ltd.
+# SPDX-License-Identifier: AGPL-3.0
+"""Reindex executor runtime configuration."""
+
+from pydantic import BaseModel, Field
+
+
+class ReindexConfig(BaseModel):
+ """Runtime limits for admin reindex execution."""
+
+ file_vectorization_concurrency: int = Field(
+ default=8,
+ gt=0,
+ description="Maximum number of files read, prepared, and enqueued concurrently",
+ )
+
+ model_config = {"extra": "forbid"}
diff --git a/tests/server/test_admin_rebuild_api.py b/tests/server/test_admin_rebuild_api.py
index 97381e3c8..72c09da2a 100644
--- a/tests/server/test_admin_rebuild_api.py
+++ b/tests/server/test_admin_rebuild_api.py
@@ -1,5 +1,8 @@
"""Tests for reindex admin endpoint and executor behavior."""
+import asyncio
+from types import SimpleNamespace
+
import httpx
import pytest
@@ -1839,7 +1842,7 @@ async def test_reindex_upsert_context_omits_search_tags_without_ingest_options(m
@pytest.mark.asyncio
-async def test_reindex_resource_vectors_only_continues_after_single_record_failure(monkeypatch):
+async def test_reindex_resource_vectors_parallelize_files_and_isolate_failures(monkeypatch):
from openviking.service.reindex_executor import ReindexExecutor, _ReindexCounters
from openviking_cli.exceptions import OpenVikingError
@@ -1879,12 +1882,21 @@ async def test_reindex_resource_vectors_only_continues_after_single_record_failu
return summary
seen = []
+ both_entered = asyncio.Event()
+ release = asyncio.Event()
async def fake_upsert_context(self, **kwargs):
seen.append(kwargs["uri"])
+ if len(seen) == 2:
+ both_entered.set()
+ await release.wait()
if kwargs["uri"].endswith("bad.txt"):
raise OpenVikingError("boom", code="PROCESSING_ERROR")
+ monkeypatch.setattr(
+ "openviking.service.reindex_executor.get_openviking_config",
+ lambda: SimpleNamespace(reindex=SimpleNamespace(file_vectorization_concurrency=8)),
+ )
monkeypatch.setattr("openviking.service.reindex_executor.get_viking_fs", lambda: FakeVikingFS())
monkeypatch.setattr(ReindexExecutor, "_read_directory_abstract", fake_read_directory_abstract)
monkeypatch.setattr(ReindexExecutor, "_read_directory_overview", fake_read_directory_overview)
@@ -1903,16 +1915,23 @@ async def test_reindex_resource_vectors_only_continues_after_single_record_failu
role=Role.ROOT,
)
- await service._reindex_resource_vectors(
- uri="viking://resources/demo",
- counters=counters,
- ctx=ctx,
+ task = asyncio.create_task(
+ service._reindex_resource_vectors(
+ uri="viking://resources/demo",
+ counters=counters,
+ ctx=ctx,
+ )
)
+ try:
+ await asyncio.wait_for(both_entered.wait(), timeout=1)
+ finally:
+ release.set()
+ await task
- assert seen == [
+ assert set(seen) == {
"viking://resources/demo/bad.txt",
"viking://resources/demo/good.txt",
- ]
+ }
assert counters.failed_records == 1
assert counters.rebuilt_records == 1
diff --git a/tests/test_config_loader.py b/tests/test_config_loader.py
index 70dcebce8..9750c64f3 100644
--- a/tests/test_config_loader.py
+++ b/tests/test_config_loader.py
@@ -121,28 +121,36 @@ class TestRequireConfig:
require_config(None, "TEST_MISSING_ENV", "nonexistent_file.conf", "test")
-def test_queue_worker_concurrency_uses_queue_specific_defaults():
+def test_runtime_concurrency_uses_scope_specific_defaults():
config = OpenVikingConfig.from_dict({})
assert config.queue_workers.external_parse.max_concurrent == 4
assert config.queue_workers.add_resource.max_concurrent == 4
+ assert config.queue_workers.add_resource.file_vectorization_concurrency == 8
assert config.queue_workers.session_commit.max_concurrent == 8
+ assert config.reindex.file_vectorization_concurrency == 8
-def test_queue_worker_concurrency_accepts_separate_values():
+def test_runtime_concurrency_accepts_separate_values():
config = OpenVikingConfig.from_dict(
{
"queue_workers": {
"external_parse": {"max_concurrent": 9},
- "add_resource": {"max_concurrent": 7},
+ "add_resource": {
+ "max_concurrent": 7,
+ "file_vectorization_concurrency": 12,
+ },
"session_commit": {"max_concurrent": 50},
- }
+ },
+ "reindex": {"file_vectorization_concurrency": 16},
}
)
assert config.queue_workers.external_parse.max_concurrent == 9
assert config.queue_workers.add_resource.max_concurrent == 7
+ assert config.queue_workers.add_resource.file_vectorization_concurrency == 12
assert config.queue_workers.session_commit.max_concurrent == 50
+ assert config.reindex.file_vectorization_concurrency == 16
@pytest.mark.parametrize("value", [0, -1])
diff --git a/tests/utils/test_resource_processor_processing_mode.py b/tests/utils/test_resource_processor_processing_mode.py
index 13c36224f..86407ac15 100644
--- a/tests/utils/test_resource_processor_processing_mode.py
+++ b/tests/utils/test_resource_processor_processing_mode.py
@@ -1,6 +1,7 @@
# Copyright (c) 2026 Beijing Volcano Engine Technology Co., Ltd.
# SPDX-License-Identifier: AGPL-3.0
+import asyncio
from types import SimpleNamespace
from unittest.mock import AsyncMock, Mock
@@ -200,6 +201,11 @@ async def test_vectors_only_persists_tree_and_vectorizes_files_only(monkeypatch,
"isDir": False,
"name": "page.md",
},
+ {
+ "uri": "viking://resources/demo/section/notes.txt",
+ "isDir": False,
+ "name": "notes.txt",
+ },
{
"uri": "viking://resources/demo/section/.abstract.md",
"isDir": False,
@@ -208,11 +214,28 @@ async def test_vectors_only_persists_tree_and_vectorizes_files_only(monkeypatch,
]
),
)
- vectorize_file = AsyncMock()
+ vectorized = {}
+ both_entered = asyncio.Event()
+ release = asyncio.Event()
+
+ async def vectorize_file(**kwargs):
+ vectorized[kwargs["file_path"]] = kwargs
+ if len(vectorized) == 2:
+ both_entered.set()
+ await release.wait()
+
rewrite_image_uris = AsyncMock()
monkeypatch.setattr("openviking.utils.resource_processor.get_viking_fs", lambda: viking_fs)
monkeypatch.setattr("openviking.utils.resource_processor.rewrite_image_uris", rewrite_image_uris)
monkeypatch.setattr("openviking.utils.resource_processor.vectorize_file", vectorize_file)
+ monkeypatch.setattr(
+ "openviking.utils.resource_processor.get_openviking_config",
+ lambda: SimpleNamespace(
+ queue_workers=SimpleNamespace(
+ add_resource=SimpleNamespace(file_vectorization_concurrency=8)
+ )
+ ),
+ )
processor = ResourceProcessor(_FakeVikingDB())
processor._get_summarizer = Mock(side_effect=AssertionError("summarizer should not run"))
processor._delete_resource_semantic_markers = AsyncMock()
@@ -220,19 +243,27 @@ async def test_vectors_only_persists_tree_and_vectorizes_files_only(monkeypatch,
processor._delete_removed_resource_vectors = AsyncMock()
lock = {"lease_ref": "lock-1"}
- result = await processor.finish_prepared_resource(
- {
- "root_uri": "viking://resources/demo",
- "temp_uri": "viking://temp/demo",
- "temp_dir_path": "tmp/demo",
- "source_committed": False,
- },
- ctx=ctx,
- resource_lock=lock,
- build_index=True,
- processing_mode="vectors_only",
- ingest_options=IngestOptions.from_search_tags(["team=search"], mode="append"),
- )
+ task = asyncio.create_task(
+ processor.finish_prepared_resource(
+ {
+ "root_uri": "viking://resources/demo",
+ "temp_uri": "viking://temp/demo",
+ "temp_dir_path": "tmp/demo",
+ "source_committed": False,
+ },
+ ctx=ctx,
+ resource_lock=lock,
+ build_index=True,
+ processing_mode="vectors_only",
+ ingest_options=IngestOptions.from_search_tags(["team=search"], mode="append"),
+ )
+ )
+ try:
+ await asyncio.wait_for(both_entered.wait(), timeout=1)
+ finally:
+ release.set()
+ await task
+ result = task.result()
assert result == {"status": "success", "root_uri": "viking://resources/demo"}
viking_fs.persist_temp_tree.assert_awaited_once_with(
@@ -247,16 +278,17 @@ async def test_vectors_only_persists_tree_and_vectorizes_files_only(monkeypatch,
lease_ref=lock,
)
viking_fs.delete_temp.assert_awaited_once_with("tmp/demo", ctx=ctx)
- vectorize_file.assert_awaited_once()
- assert vectorize_file.await_args.kwargs["file_path"] == (
- "viking://resources/demo/section/page.md"
- )
- assert vectorize_file.await_args.kwargs["parent_uri"] == "viking://resources/demo/section"
- assert vectorize_file.await_args.kwargs["summary_dict"] == {
+ assert set(vectorized) == {
+ "viking://resources/demo/section/page.md",
+ "viking://resources/demo/section/notes.txt",
+ }
+ page = vectorized["viking://resources/demo/section/page.md"]
+ assert page["parent_uri"] == "viking://resources/demo/section"
+ assert page["summary_dict"] == {
"name": "page.md",
"summary": "",
}
- assert vectorize_file.await_args.kwargs["ingest_options"] == IngestOptions(
+ assert page["ingest_options"] == IngestOptions(
search_tags=["team=search"],
search_tag_mode="append",
)
@@ -344,34 +376,72 @@ async def test_vectors_only_syncs_preexisting_target_instead_of_merging(monkeypa
@pytest.mark.asyncio
-async def test_vectors_only_leaves_existing_semantic_vectors_untouched(monkeypatch, ctx):
+async def test_vectors_only_cancels_siblings_before_releasing_lock(monkeypatch, ctx):
+ healthy_started = asyncio.Event()
+ healthy_cancelled = asyncio.Event()
+ lock = {"lease_ref": "lock-1"}
+
+ async def release_lock(released_lock):
+ assert released_lock == lock
+ assert healthy_cancelled.is_set()
+
viking_fs = SimpleNamespace(
- _async_agfs=SimpleNamespace(pathlock_release=AsyncMock()),
- persist_temp_tree=AsyncMock(),
- delete_temp=AsyncMock(),
- tree=AsyncMock(return_value=[]),
+ _async_agfs=SimpleNamespace(pathlock_release=AsyncMock(side_effect=release_lock)),
+ tree=AsyncMock(
+ return_value=[
+ {
+ "uri": "viking://resources/demo/failing.md",
+ "isDir": False,
+ "name": "failing.md",
+ },
+ {
+ "uri": "viking://resources/demo/healthy.md",
+ "isDir": False,
+ "name": "healthy.md",
+ },
+ ]
+ ),
)
+
+ async def vectorize_file(*, file_path, **kwargs):
+ if file_path.endswith("failing.md"):
+ await healthy_started.wait()
+ raise RuntimeError("vector enqueue failed")
+ healthy_started.set()
+ try:
+ await asyncio.Event().wait()
+ except asyncio.CancelledError:
+ healthy_cancelled.set()
+ raise
+
monkeypatch.setattr("openviking.utils.resource_processor.get_viking_fs", lambda: viking_fs)
- monkeypatch.setattr("openviking.utils.resource_processor.rewrite_image_uris", AsyncMock())
- monkeypatch.setattr("openviking.utils.resource_processor.vectorize_file", AsyncMock())
+ monkeypatch.setattr("openviking.utils.resource_processor.vectorize_file", vectorize_file)
+ monkeypatch.setattr(
+ "openviking.utils.resource_processor.get_openviking_config",
+ lambda: SimpleNamespace(
+ queue_workers=SimpleNamespace(
+ add_resource=SimpleNamespace(file_vectorization_concurrency=8)
+ )
+ ),
+ )
processor = ResourceProcessor(_FakeVikingDB())
processor._delete_resource_semantic_vectors = AsyncMock()
- processor._delete_removed_resource_vectors = AsyncMock()
- lock = {"lease_ref": "lock-1"}
-
- await processor.finish_prepared_resource(
- {
- "root_uri": "viking://resources/demo",
- "temp_uri": "viking://temp/demo",
- "source_committed": False,
- },
- ctx=ctx,
- resource_lock=lock,
- build_index=True,
- processing_mode="vectors_only",
- )
+ with pytest.raises(RuntimeError, match="vector enqueue failed"):
+ await processor.finish_prepared_resource(
+ {
+ "root_uri": "viking://resources/demo",
+ "source_committed": True,
+ },
+ ctx=ctx,
+ resource_lock=lock,
+ build_index=True,
+ processing_mode="vectors_only",
+ )
+
+ assert healthy_cancelled.is_set()
processor._delete_resource_semantic_vectors.assert_not_awaited()
+ viking_fs._async_agfs.pathlock_release.assert_awaited_once_with(lock)
@pytest.mark.asyncio