Skip to content

Self-evolving Context Database for AI Agents. Unify Agent Memory, Knowledge RAG and Skills.

feat(queue): make ExternalParse worker concurrency configurable (#3710)

Kchenc7ee8e753a69
diff --git a/docs/en/configuration/01-server.md b/docs/en/configuration/01-server.md
index d1ebad2d0..bd1cfa4db 100644
--- a/docs/en/configuration/01-server.md
+++ b/docs/en/configuration/01-server.md
@@ -51,6 +51,7 @@ Optional sections use their defaults when omitted. Unknown fields are rejected.
| `retrieval` | object | see below | Ranking and intent-analysis behavior |
| `grep` | object | built-in defaults | Text search engine |
| `storage` | object | local | Workspace, file system, and vector database |
+| `queue_workers` | object | see below | Runtime concurrency for QueueFS consumer workers |
| `server` | object | local development | HTTP, authentication, uploads, and observability |
| `memory` | object | see below | Memory and skill extraction on session commit |
| `parsers` | object | parser defaults | PDF, code, image, audio, video, and text parsing |
@@ -197,6 +198,16 @@ Search and Find requests default to `limit: 10`; override the limit on each API
Remote backends also require endpoint, bucket/collection, credentials, and timeout fields. See [Configuration](../guides/01-configuration.md#storage) for complete examples.
+## Queue Worker Settings
+
+### `queue_workers.external_parse`
+
+| Field | Type | Default | Description |
+|---|---|---:|---|
+| `max_concurrent` | integer | `4` | Number of complete ExternalParse jobs consumed concurrently; must be greater than `0`; requires a server restart after changes |
+
+This setting controls queue-job concurrency. It is separate from `vlm.media.max_concurrent`, which limits audio/video VLM calls, and does not limit individual Understanding API HTTP requests.
+
## HTTP Server Settings
```json
diff --git a/docs/zh/configuration/01-server.md b/docs/zh/configuration/01-server.md
index f87bd22fe..ffce60be7 100644
--- a/docs/zh/configuration/01-server.md
+++ b/docs/zh/configuration/01-server.md
@@ -51,6 +51,7 @@ openviking-server --config /path/to/ov.conf
| `retrieval` | object | 见下表 | 检索排序和意图分析策略 |
| `grep` | object | 内置默认值 | 文本搜索引擎配置 |
| `storage` | object | 本地存储 | 工作目录、文件系统和向量数据库 |
+| `queue_workers` | object | 见下表 | QueueFS 消费 worker 的运行时并发配置 |
| `server` | object | 本地开发模式 | HTTP 服务、鉴权、上传和可观测性 |
| `memory` | object | 见下表 | 会话提交后的记忆与技能抽取 |
| `parsers` | object | 各解析器默认值 | PDF、代码、图片、音视频等解析行为 |
@@ -197,6 +198,16 @@ Search 和 Find 请求的默认 `limit` 为 `10`,可以在每次 API 或 SDK
远程存储后端还需要配置 endpoint、bucket/collection、鉴权和超时等字段。完整后端示例见[配置指南](../guides/01-configuration.md#storage)。
+## 队列 Worker 配置
+
+### `queue_workers.external_parse`
+
+| 字段 | 类型 | 默认值 | 说明 |
+|---|---|---:|---|
+| `max_concurrent` | integer | `4` | 同时消费的完整 ExternalParse 作业数,必须大于 `0`;修改后需重启服务 |
+
+该配置控制队列作业并发,不等同于 `vlm.media.max_concurrent` 的音视频 VLM 调用并发,也不限制 Understanding API 的单独 HTTP 请求数。
+
## HTTP 服务配置
```json
diff --git a/examples/ov.conf.example b/examples/ov.conf.example
index 301a0f48f..ac43a9b82 100644
--- a/examples/ov.conf.example
+++ b/examples/ov.conf.example
@@ -82,6 +82,7 @@
},
},
},
+ "queue_workers": {"external_parse": {"max_concurrent": 4}},
"embedding": {
"dense": {
"model": "doubao-embedding-vision-251215",
diff --git a/openviking/service/core.py b/openviking/service/core.py
index 0e7a3ae01..c05db0809 100644
--- a/openviking/service/core.py
+++ b/openviking/service/core.py
@@ -25,13 +25,13 @@ from openviking.service.search_service import SearchService
from openviking.service.session_service import SessionService
from openviking.service.task_tracker import get_task_tracker, set_task_tracker
from openviking.session import create_session_compressor
-from openviking.storage.vikingdb_manager import VikingDBManager
from openviking.storage.collection_schemas import init_context_collection
from openviking.storage.index_consistency import check_index_consistency
from openviking.storage.queuefs.add_resource_processor import AddResourceProcessor
from openviking.storage.queuefs.queue_manager import QueueManager, init_queue_manager
from openviking.storage.queuefs.session_commit_processor import SessionCommitProcessor
from openviking.storage.viking_fs import VikingFS, init_viking_fs
+from openviking.storage.vikingdb_manager import VikingDBManager
from openviking.utils.agfs_utils import (
build_runtime_ragfs_binding_config,
resolve_queuefs_mount_point,
@@ -120,8 +120,9 @@ class OpenVikingService:
# Initialize storage
self._init_storage(
config.storage,
- config.embedding.max_concurrent,
- config.vlm.max_concurrent,
+ max_concurrent_embedding=config.embedding.max_concurrent,
+ max_concurrent_semantic=config.vlm.max_concurrent,
+ max_concurrent_external_parse=config.queue_workers.external_parse.max_concurrent,
binding_config=binding_config,
git_config=config.git,
)
@@ -137,6 +138,7 @@ class OpenVikingService:
config: StorageConfig,
max_concurrent_embedding: int = 10,
max_concurrent_semantic: int = 64,
+ max_concurrent_external_parse: int = 4,
binding_config: Any = None,
*,
git_config: Optional[GitConfig] = None,
@@ -157,6 +159,7 @@ class OpenVikingService:
mount_point=queue_mount_point,
max_concurrent_embedding=max_concurrent_embedding,
max_concurrent_semantic=max_concurrent_semantic,
+ max_concurrent_external_parse=max_concurrent_external_parse,
)
else:
logger.warning("RAGFS client not initialized, skipping queue manager")
@@ -288,8 +291,11 @@ class OpenVikingService:
if self._vikingdb_manager is None:
self._init_storage(
self._config.storage,
- self._config.embedding.max_concurrent,
- self._config.vlm.max_concurrent,
+ max_concurrent_embedding=self._config.embedding.max_concurrent,
+ max_concurrent_semantic=self._config.vlm.max_concurrent,
+ max_concurrent_external_parse=(
+ self._config.queue_workers.external_parse.max_concurrent
+ ),
binding_config=self._build_ragfs_binding_config(),
git_config=self._config.git,
)
diff --git a/openviking/storage/queuefs/queue_manager.py b/openviking/storage/queuefs/queue_manager.py
index c8dd4a09a..447be8e83 100644
--- a/openviking/storage/queuefs/queue_manager.py
+++ b/openviking/storage/queuefs/queue_manager.py
@@ -21,6 +21,8 @@ from .semantic_queue import SemanticQueue
logger = get_logger(__name__)
+DEFAULT_MAX_CONCURRENT_SESSION_COMMIT = 4
+
# ========== Singleton Pattern ==========
_instance: Optional["QueueManager"] = None
@@ -41,6 +43,7 @@ def init_queue_manager(
mount_point: Path where QueueFS is mounted.
max_concurrent_embedding: Max concurrent embedding tasks.
max_concurrent_semantic: Max concurrent semantic node work.
+ max_concurrent_external_parse: Max concurrent ExternalParse tasks.
"""
global _instance
_instance = QueueManager(
@@ -166,12 +169,7 @@ class QueueManager:
if thread.is_alive():
return
- if queue.name == self.EMBEDDING:
- max_concurrent = self._max_concurrent_embedding
- elif queue.name in {self.EXTERNAL_PARSE, self.SESSION_COMMIT}:
- max_concurrent = self._max_concurrent_external_parse
- else:
- max_concurrent = self._max_concurrent_semantic
+ max_concurrent = self._max_concurrent_for_queue(queue.name)
stop_event = threading.Event()
self._queue_stop_events[queue.name] = stop_event
thread = threading.Thread(
@@ -182,6 +180,16 @@ class QueueManager:
self._queue_threads[queue.name] = thread
thread.start()
+ def _max_concurrent_for_queue(self, queue_name: str) -> int:
+ """Return the worker concurrency limit for a named queue."""
+ if queue_name == self.EMBEDDING:
+ return self._max_concurrent_embedding
+ if queue_name == self.EXTERNAL_PARSE:
+ return self._max_concurrent_external_parse
+ if queue_name == self.SESSION_COMMIT:
+ return DEFAULT_MAX_CONCURRENT_SESSION_COMMIT
+ return self._max_concurrent_semantic
+
def _queue_worker_loop(
self, queue: NamedQueue, stop_event: threading.Event, max_concurrent: int = 1
) -> None:
diff --git a/openviking_cli/utils/config/open_viking_config.py b/openviking_cli/utils/config/open_viking_config.py
index bb2f3edb7..9715c05b0 100644
--- a/openviking_cli/utils/config/open_viking_config.py
+++ b/openviking_cli/utils/config/open_viking_config.py
@@ -43,6 +43,7 @@ from .parser_config import (
WebFeedConfig,
)
from .prompts_config import PromptsConfig
+from .queue_worker_config import QueueWorkersConfig
from .rerank_config import RerankConfig
from .retrieval_config import RetrievalConfig
from .storage_config import StorageConfig
@@ -99,7 +100,6 @@ class ParserApiConfig(BaseModel):
http_timeout_seconds: float = 10.0
response_timeout_seconds: int = 1800
poll_interval_ms: int = 3000
-
model_config = {"extra": "forbid"}
@model_validator(mode="after")
@@ -237,6 +237,11 @@ class OpenVikingConfig(BaseModel):
description="Semantic processing configuration (overview/abstract limits)",
)
+ queue_workers: QueueWorkersConfig = Field(
+ default_factory=QueueWorkersConfig,
+ description="Queue worker runtime configuration",
+ )
+
parser_api: ParserApiConfig = Field(
default_factory=ParserApiConfig,
description="Third-party parser API configuration (files/responses)",
diff --git a/openviking_cli/utils/config/queue_worker_config.py b/openviking_cli/utils/config/queue_worker_config.py
new file mode 100644
index 000000000..497c26f77
--- /dev/null
+++ b/openviking_cli/utils/config/queue_worker_config.py
@@ -0,0 +1,25 @@
+# Copyright (c) 2026 Beijing Volcano Engine Technology Co., Ltd.
+# SPDX-License-Identifier: AGPL-3.0
+"""Queue worker runtime configuration."""
+
+from pydantic import BaseModel, Field
+
+
+class QueueWorkerConfig(BaseModel):
+ """Runtime limits for one queue worker."""
+
+ max_concurrent: int = Field(
+ default=4,
+ gt=0,
+ description="Maximum number of jobs processed concurrently",
+ )
+
+ model_config = {"extra": "forbid"}
+
+
+class QueueWorkersConfig(BaseModel):
+ """Runtime limits for QueueFS consumers."""
+
+ external_parse: QueueWorkerConfig = Field(default_factory=QueueWorkerConfig)
+
+ model_config = {"extra": "forbid"}
diff --git a/tests/storage/test_queue_manager.py b/tests/storage/test_queue_manager.py
new file mode 100644
index 000000000..6621cffe4
--- /dev/null
+++ b/tests/storage/test_queue_manager.py
@@ -0,0 +1,17 @@
+# Copyright (c) 2026 Beijing Volcano Engine Technology Co., Ltd.
+# SPDX-License-Identifier: AGPL-3.0
+"""Focused tests for QueueManager concurrency selection."""
+
+from openviking.storage.queuefs.queue_manager import QueueManager
+
+
+def test_external_parse_concurrent_uses_configured_value() -> None:
+ manager = QueueManager(agfs=object(), max_concurrent_external_parse=9)
+
+ assert manager._max_concurrent_for_queue(manager.EXTERNAL_PARSE) == 9
+
+
+def test_session_commit_concurrent_stays_at_four() -> None:
+ manager = QueueManager(agfs=object(), max_concurrent_external_parse=9)
+
+ assert manager._max_concurrent_for_queue(manager.SESSION_COMMIT) == 4
diff --git a/tests/test_config_loader.py b/tests/test_config_loader.py
index 71393826c..c319c64db 100644
--- a/tests/test_config_loader.py
+++ b/tests/test_config_loader.py
@@ -18,7 +18,9 @@ from openviking_cli.utils.config.config_loader import (
require_config,
resolve_config_path,
)
+from openviking_cli.utils.config.open_viking_config import OpenVikingConfig, ParserApiConfig
from openviking_cli.utils.config.parser_config import CodeHostingConfig
+from openviking_cli.utils.config.queue_worker_config import QueueWorkersConfig
class TestResolveConfigPath:
@@ -119,6 +121,35 @@ class TestRequireConfig:
require_config(None, "TEST_MISSING_ENV", "nonexistent_file.conf", "test")
+def test_external_parse_worker_max_concurrent_defaults_to_four():
+ config = OpenVikingConfig.from_dict({})
+
+ assert config.queue_workers.external_parse.max_concurrent == 4
+
+
+def test_external_parse_worker_max_concurrent_accepts_positive_value():
+ config = OpenVikingConfig.from_dict(
+ {"queue_workers": {"external_parse": {"max_concurrent": 9}}}
+ )
+
+ assert config.queue_workers.external_parse.max_concurrent == 9
+
+
[email protected]("value", [0, -1])
+def test_external_parse_worker_max_concurrent_rejects_non_positive_value(value):
+ with pytest.raises(ValueError) as exc_info:
+ QueueWorkersConfig(external_parse={"max_concurrent": value})
+
+ assert exc_info.value.errors()[0]["type"] == "greater_than"
+
+
+def test_parser_api_rejects_worker_max_concurrent():
+ with pytest.raises(ValueError) as exc_info:
+ ParserApiConfig(max_concurrent=9)
+
+ assert exc_info.value.errors()[0]["type"] == "extra_forbidden"
+
+
def test_generic_code_hosting_domains_include_supported_platforms():
config = CodeHostingConfig()
diff --git a/tests/unit/service/test_core_consistency.py b/tests/unit/service/test_core_consistency.py
index 6951b5433..bf616c0c6 100644
--- a/tests/unit/service/test_core_consistency.py
+++ b/tests/unit/service/test_core_consistency.py
@@ -1,7 +1,8 @@
# Copyright (c) 2026 Beijing Volcano Engine Technology Co., Ltd.
# SPDX-License-Identifier: Apache-2.0
-"""Focused tests for system consistency URI validation."""
+"""Focused tests for core service behavior."""
+from types import SimpleNamespace
from unittest.mock import AsyncMock, patch
import pytest
@@ -13,6 +14,47 @@ from openviking_cli.exceptions import InvalidArgumentError
from openviking_cli.session.user_id import UserIdentifier
+def test_service_passes_queue_worker_concurrency_to_storage(monkeypatch) -> None:
+ storage_calls = []
+ embedder = SimpleNamespace(is_sparse=False)
+ config = SimpleNamespace(
+ default_account="default",
+ default_user="default",
+ storage=object(),
+ embedding=SimpleNamespace(
+ max_concurrent=10,
+ dimension=1024,
+ get_embedder=lambda: embedder,
+ ),
+ vlm=SimpleNamespace(max_concurrent=64),
+ parser_api=SimpleNamespace(),
+ queue_workers=SimpleNamespace(
+ external_parse=SimpleNamespace(max_concurrent=9),
+ ),
+ git=object(),
+ )
+
+ monkeypatch.setattr(
+ "openviking.service.core.initialize_openviking_config",
+ lambda **_kwargs: config,
+ )
+ monkeypatch.setattr(OpenVikingService, "_ensure_data_dir_lock_acquired", lambda self: None)
+ monkeypatch.setattr(
+ OpenVikingService,
+ "_build_ragfs_binding_config",
+ lambda self: object(),
+ )
+
+ def _capture_storage_init(self, *args, **kwargs) -> None:
+ storage_calls.append((args, kwargs))
+
+ monkeypatch.setattr(OpenVikingService, "_init_storage", _capture_storage_init)
+
+ OpenVikingService()
+
+ assert storage_calls[0][1]["max_concurrent_external_parse"] == 9
+
+
def _service_with_fs(stat_result: dict) -> tuple[OpenVikingService, AsyncMock]:
service = OpenVikingService.__new__(OpenVikingService)
service._initialized = True