Self-evolving Context Database for AI Agents. Unify Agent Memory, Knowledge RAG and Skills.
Python agent-contextagent-memoryagent-pluginsagentic-ragcontext-databasedsh-pluginknowledge-basememorymemory-managementself-evolvingvector-database
feat(queue): make ExternalParse worker concurrency configurable (#3710)
Kchenc7ee8e753a69
diff --git a/docs/en/configuration/01-server.md b/docs/en/configuration/01-server.mdindex d1ebad2d0..bd1cfa4db 100644--- a/docs/en/configuration/01-server.md+++ b/docs/en/configuration/01-server.md@@ -51,6 +51,7 @@ Optional sections use their defaults when omitted. Unknown fields are rejected.| `retrieval` | object | see below | Ranking and intent-analysis behavior || `grep` | object | built-in defaults | Text search engine || `storage` | object | local | Workspace, file system, and vector database |+| `queue_workers` | object | see below | Runtime concurrency for QueueFS consumer workers || `server` | object | local development | HTTP, authentication, uploads, and observability || `memory` | object | see below | Memory and skill extraction on session commit || `parsers` | object | parser defaults | PDF, code, image, audio, video, and text parsing |@@ -197,6 +198,16 @@ Search and Find requests default to `limit: 10`; override the limit on each APIRemote backends also require endpoint, bucket/collection, credentials, and timeout fields. See [Configuration](../guides/01-configuration.md#storage) for complete examples.+## Queue Worker Settings++### `queue_workers.external_parse`++| Field | Type | Default | Description |+|---|---|---:|---|+| `max_concurrent` | integer | `4` | Number of complete ExternalParse jobs consumed concurrently; must be greater than `0`; requires a server restart after changes |++This setting controls queue-job concurrency. It is separate from `vlm.media.max_concurrent`, which limits audio/video VLM calls, and does not limit individual Understanding API HTTP requests.+## HTTP Server Settings```jsondiff --git a/docs/zh/configuration/01-server.md b/docs/zh/configuration/01-server.mdindex f87bd22fe..ffce60be7 100644--- a/docs/zh/configuration/01-server.md+++ b/docs/zh/configuration/01-server.md@@ -51,6 +51,7 @@ openviking-server --config /path/to/ov.conf| `retrieval` | object | 见下表 | 检索排序和意图分析策略 || `grep` | object | 内置默认值 | 文本搜索引擎配置 || `storage` | object | 本地存储 | 工作目录、文件系统和向量数据库 |+| `queue_workers` | object | 见下表 | QueueFS 消费 worker 的运行时并发配置 || `server` | object | 本地开发模式 | HTTP 服务、鉴权、上传和可观测性 || `memory` | object | 见下表 | 会话提交后的记忆与技能抽取 || `parsers` | object | 各解析器默认值 | PDF、代码、图片、音视频等解析行为 |@@ -197,6 +198,16 @@ Search 和 Find 请求的默认 `limit` 为 `10`,可以在每次 API 或 SDK远程存储后端还需要配置 endpoint、bucket/collection、鉴权和超时等字段。完整后端示例见[配置指南](../guides/01-configuration.md#storage)。+## 队列 Worker 配置++### `queue_workers.external_parse`++| 字段 | 类型 | 默认值 | 说明 |+|---|---|---:|---|+| `max_concurrent` | integer | `4` | 同时消费的完整 ExternalParse 作业数,必须大于 `0`;修改后需重启服务 |++该配置控制队列作业并发,不等同于 `vlm.media.max_concurrent` 的音视频 VLM 调用并发,也不限制 Understanding API 的单独 HTTP 请求数。+## HTTP 服务配置```jsondiff --git a/examples/ov.conf.example b/examples/ov.conf.exampleindex 301a0f48f..ac43a9b82 100644--- a/examples/ov.conf.example+++ b/examples/ov.conf.example@@ -82,6 +82,7 @@},},},+ "queue_workers": {"external_parse": {"max_concurrent": 4}},"embedding": {"dense": {"model": "doubao-embedding-vision-251215",diff --git a/openviking/service/core.py b/openviking/service/core.pyindex 0e7a3ae01..c05db0809 100644--- a/openviking/service/core.py+++ b/openviking/service/core.py@@ -25,13 +25,13 @@ from openviking.service.search_service import SearchServicefrom openviking.service.session_service import SessionServicefrom openviking.service.task_tracker import get_task_tracker, set_task_trackerfrom openviking.session import create_session_compressor-from openviking.storage.vikingdb_manager import VikingDBManagerfrom openviking.storage.collection_schemas import init_context_collectionfrom openviking.storage.index_consistency import check_index_consistencyfrom openviking.storage.queuefs.add_resource_processor import AddResourceProcessorfrom openviking.storage.queuefs.queue_manager import QueueManager, init_queue_managerfrom openviking.storage.queuefs.session_commit_processor import SessionCommitProcessorfrom openviking.storage.viking_fs import VikingFS, init_viking_fs+from openviking.storage.vikingdb_manager import VikingDBManagerfrom openviking.utils.agfs_utils import (build_runtime_ragfs_binding_config,resolve_queuefs_mount_point,@@ -120,8 +120,9 @@ class OpenVikingService:# Initialize storageself._init_storage(config.storage,- config.embedding.max_concurrent,- config.vlm.max_concurrent,+ max_concurrent_embedding=config.embedding.max_concurrent,+ max_concurrent_semantic=config.vlm.max_concurrent,+ max_concurrent_external_parse=config.queue_workers.external_parse.max_concurrent,binding_config=binding_config,git_config=config.git,)@@ -137,6 +138,7 @@ class OpenVikingService:config: StorageConfig,max_concurrent_embedding: int = 10,max_concurrent_semantic: int = 64,+ max_concurrent_external_parse: int = 4,binding_config: Any = None,*,git_config: Optional[GitConfig] = None,@@ -157,6 +159,7 @@ class OpenVikingService:mount_point=queue_mount_point,max_concurrent_embedding=max_concurrent_embedding,max_concurrent_semantic=max_concurrent_semantic,+ max_concurrent_external_parse=max_concurrent_external_parse,)else:logger.warning("RAGFS client not initialized, skipping queue manager")@@ -288,8 +291,11 @@ class OpenVikingService:if self._vikingdb_manager is None:self._init_storage(self._config.storage,- self._config.embedding.max_concurrent,- self._config.vlm.max_concurrent,+ max_concurrent_embedding=self._config.embedding.max_concurrent,+ max_concurrent_semantic=self._config.vlm.max_concurrent,+ max_concurrent_external_parse=(+ self._config.queue_workers.external_parse.max_concurrent+ ),binding_config=self._build_ragfs_binding_config(),git_config=self._config.git,)diff --git a/openviking/storage/queuefs/queue_manager.py b/openviking/storage/queuefs/queue_manager.pyindex c8dd4a09a..447be8e83 100644--- a/openviking/storage/queuefs/queue_manager.py+++ b/openviking/storage/queuefs/queue_manager.py@@ -21,6 +21,8 @@ from .semantic_queue import SemanticQueuelogger = get_logger(__name__)+DEFAULT_MAX_CONCURRENT_SESSION_COMMIT = 4+# ========== Singleton Pattern ==========_instance: Optional["QueueManager"] = None@@ -41,6 +43,7 @@ def init_queue_manager(mount_point: Path where QueueFS is mounted.max_concurrent_embedding: Max concurrent embedding tasks.max_concurrent_semantic: Max concurrent semantic node work.+ max_concurrent_external_parse: Max concurrent ExternalParse tasks."""global _instance_instance = QueueManager(@@ -166,12 +169,7 @@ class QueueManager:if thread.is_alive():return- if queue.name == self.EMBEDDING:- max_concurrent = self._max_concurrent_embedding- elif queue.name in {self.EXTERNAL_PARSE, self.SESSION_COMMIT}:- max_concurrent = self._max_concurrent_external_parse- else:- max_concurrent = self._max_concurrent_semantic+ max_concurrent = self._max_concurrent_for_queue(queue.name)stop_event = threading.Event()self._queue_stop_events[queue.name] = stop_eventthread = threading.Thread(@@ -182,6 +180,16 @@ class QueueManager:self._queue_threads[queue.name] = threadthread.start()+ def _max_concurrent_for_queue(self, queue_name: str) -> int:+ """Return the worker concurrency limit for a named queue."""+ if queue_name == self.EMBEDDING:+ return self._max_concurrent_embedding+ if queue_name == self.EXTERNAL_PARSE:+ return self._max_concurrent_external_parse+ if queue_name == self.SESSION_COMMIT:+ return DEFAULT_MAX_CONCURRENT_SESSION_COMMIT+ return self._max_concurrent_semantic+def _queue_worker_loop(self, queue: NamedQueue, stop_event: threading.Event, max_concurrent: int = 1) -> None:diff --git a/openviking_cli/utils/config/open_viking_config.py b/openviking_cli/utils/config/open_viking_config.pyindex bb2f3edb7..9715c05b0 100644--- a/openviking_cli/utils/config/open_viking_config.py+++ b/openviking_cli/utils/config/open_viking_config.py@@ -43,6 +43,7 @@ from .parser_config import (WebFeedConfig,)from .prompts_config import PromptsConfig+from .queue_worker_config import QueueWorkersConfigfrom .rerank_config import RerankConfigfrom .retrieval_config import RetrievalConfigfrom .storage_config import StorageConfig@@ -99,7 +100,6 @@ class ParserApiConfig(BaseModel):http_timeout_seconds: float = 10.0response_timeout_seconds: int = 1800poll_interval_ms: int = 3000-model_config = {"extra": "forbid"}@model_validator(mode="after")@@ -237,6 +237,11 @@ class OpenVikingConfig(BaseModel):description="Semantic processing configuration (overview/abstract limits)",)+ queue_workers: QueueWorkersConfig = Field(+ default_factory=QueueWorkersConfig,+ description="Queue worker runtime configuration",+ )+parser_api: ParserApiConfig = Field(default_factory=ParserApiConfig,description="Third-party parser API configuration (files/responses)",diff --git a/openviking_cli/utils/config/queue_worker_config.py b/openviking_cli/utils/config/queue_worker_config.pynew file mode 100644index 000000000..497c26f77--- /dev/null+++ b/openviking_cli/utils/config/queue_worker_config.py@@ -0,0 +1,25 @@+# Copyright (c) 2026 Beijing Volcano Engine Technology Co., Ltd.+# SPDX-License-Identifier: AGPL-3.0+"""Queue worker runtime configuration."""++from pydantic import BaseModel, Field+++class QueueWorkerConfig(BaseModel):+ """Runtime limits for one queue worker."""++ max_concurrent: int = Field(+ default=4,+ gt=0,+ description="Maximum number of jobs processed concurrently",+ )++ model_config = {"extra": "forbid"}+++class QueueWorkersConfig(BaseModel):+ """Runtime limits for QueueFS consumers."""++ external_parse: QueueWorkerConfig = Field(default_factory=QueueWorkerConfig)++ model_config = {"extra": "forbid"}diff --git a/tests/storage/test_queue_manager.py b/tests/storage/test_queue_manager.pynew file mode 100644index 000000000..6621cffe4--- /dev/null+++ b/tests/storage/test_queue_manager.py@@ -0,0 +1,17 @@+# Copyright (c) 2026 Beijing Volcano Engine Technology Co., Ltd.+# SPDX-License-Identifier: AGPL-3.0+"""Focused tests for QueueManager concurrency selection."""++from openviking.storage.queuefs.queue_manager import QueueManager+++def test_external_parse_concurrent_uses_configured_value() -> None:+ manager = QueueManager(agfs=object(), max_concurrent_external_parse=9)++ assert manager._max_concurrent_for_queue(manager.EXTERNAL_PARSE) == 9+++def test_session_commit_concurrent_stays_at_four() -> None:+ manager = QueueManager(agfs=object(), max_concurrent_external_parse=9)++ assert manager._max_concurrent_for_queue(manager.SESSION_COMMIT) == 4diff --git a/tests/test_config_loader.py b/tests/test_config_loader.pyindex 71393826c..c319c64db 100644--- a/tests/test_config_loader.py+++ b/tests/test_config_loader.py@@ -18,7 +18,9 @@ from openviking_cli.utils.config.config_loader import (require_config,resolve_config_path,)+from openviking_cli.utils.config.open_viking_config import OpenVikingConfig, ParserApiConfigfrom openviking_cli.utils.config.parser_config import CodeHostingConfig+from openviking_cli.utils.config.queue_worker_config import QueueWorkersConfigclass TestResolveConfigPath:@@ -119,6 +121,35 @@ class TestRequireConfig:require_config(None, "TEST_MISSING_ENV", "nonexistent_file.conf", "test")+def test_external_parse_worker_max_concurrent_defaults_to_four():+ config = OpenVikingConfig.from_dict({})++ assert config.queue_workers.external_parse.max_concurrent == 4+++def test_external_parse_worker_max_concurrent_accepts_positive_value():+ config = OpenVikingConfig.from_dict(+ {"queue_workers": {"external_parse": {"max_concurrent": 9}}}+ )++ assert config.queue_workers.external_parse.max_concurrent == 9++[email protected]("value", [0, -1])+def test_external_parse_worker_max_concurrent_rejects_non_positive_value(value):+ with pytest.raises(ValueError) as exc_info:+ QueueWorkersConfig(external_parse={"max_concurrent": value})++ assert exc_info.value.errors()[0]["type"] == "greater_than"+++def test_parser_api_rejects_worker_max_concurrent():+ with pytest.raises(ValueError) as exc_info:+ ParserApiConfig(max_concurrent=9)++ assert exc_info.value.errors()[0]["type"] == "extra_forbidden"++def test_generic_code_hosting_domains_include_supported_platforms():config = CodeHostingConfig()diff --git a/tests/unit/service/test_core_consistency.py b/tests/unit/service/test_core_consistency.pyindex 6951b5433..bf616c0c6 100644--- a/tests/unit/service/test_core_consistency.py+++ b/tests/unit/service/test_core_consistency.py@@ -1,7 +1,8 @@# Copyright (c) 2026 Beijing Volcano Engine Technology Co., Ltd.# SPDX-License-Identifier: Apache-2.0-"""Focused tests for system consistency URI validation."""+"""Focused tests for core service behavior."""+from types import SimpleNamespacefrom unittest.mock import AsyncMock, patchimport pytest@@ -13,6 +14,47 @@ from openviking_cli.exceptions import InvalidArgumentErrorfrom openviking_cli.session.user_id import UserIdentifier+def test_service_passes_queue_worker_concurrency_to_storage(monkeypatch) -> None:+ storage_calls = []+ embedder = SimpleNamespace(is_sparse=False)+ config = SimpleNamespace(+ default_account="default",+ default_user="default",+ storage=object(),+ embedding=SimpleNamespace(+ max_concurrent=10,+ dimension=1024,+ get_embedder=lambda: embedder,+ ),+ vlm=SimpleNamespace(max_concurrent=64),+ parser_api=SimpleNamespace(),+ queue_workers=SimpleNamespace(+ external_parse=SimpleNamespace(max_concurrent=9),+ ),+ git=object(),+ )++ monkeypatch.setattr(+ "openviking.service.core.initialize_openviking_config",+ lambda **_kwargs: config,+ )+ monkeypatch.setattr(OpenVikingService, "_ensure_data_dir_lock_acquired", lambda self: None)+ monkeypatch.setattr(+ OpenVikingService,+ "_build_ragfs_binding_config",+ lambda self: object(),+ )++ def _capture_storage_init(self, *args, **kwargs) -> None:+ storage_calls.append((args, kwargs))++ monkeypatch.setattr(OpenVikingService, "_init_storage", _capture_storage_init)++ OpenVikingService()++ assert storage_calls[0][1]["max_concurrent_external_parse"] == 9++def _service_with_fs(stat_result: dict) -> tuple[OpenVikingService, AsyncMock]:service = OpenVikingService.__new__(OpenVikingService)service._initialized = True