Self-evolving Context Database for AI Agents. Unify Agent Memory, Knowledge RAG and Skills.
Python agent-contextagent-memoryagent-pluginsagentic-ragcontext-databasedsh-pluginknowledge-basememorymemory-managementself-evolvingvector-database
Fix/remove case only conflict check (#6298)
Jiahui Zhoub51cba425487
diff --git a/docs/en/configuration/01-server.md b/docs/en/configuration/01-server.mdindex a002e200d..a8f2a9037 100644--- a/docs/en/configuration/01-server.md+++ b/docs/en/configuration/01-server.md@@ -196,6 +196,9 @@ Search and Find requests default to `limit: 10`; override the limit on each API"parse_output": {"mode": "agfs"}+ },+ "parsers": {+ "download_temp_dir": "/data/openviking/parser-tmp"}}```@@ -219,6 +222,14 @@ The same worker must commit the required bytes to the formal resource tree beforenqueueing downstream work. Artifacts are temporary and are removed after thecontent commit; provision `local_root` with enough space for concurrent imports.+### `parsers`++| Field | Type / common values | Default | Purpose |+|---|---|---|---|+| `download_temp_dir` | path or `null` | system temp directory | Temporary directory for built-in parser downloads, Git checkouts, and archive extraction |++When configured, parsers create temporary directories with prefixes such as `ov_git_` and `ov_zip_` below this directory. When unset, the system temporary directory is used.+## Queue Worker Settings### `queue_workers.external_parse`diff --git a/docs/zh/configuration/01-server.md b/docs/zh/configuration/01-server.mdindex c4b5bf611..34ca25932 100644--- a/docs/zh/configuration/01-server.md+++ b/docs/zh/configuration/01-server.md@@ -196,6 +196,9 @@ Search 和 Find 请求的默认 `limit` 为 `10`,可以在每次 API 或 SDK"parse_output": {"mode": "agfs"}+ },+ "parsers": {+ "download_temp_dir": "/data/openviking/parser-tmp"}}```@@ -218,6 +221,14 @@ Search 和 Find 请求的默认 `limit` 为 `10`,可以在每次 API 或 SDK必须在下游任务入队前把所需字节提交到正式资源树。产物会在内容提交后清理;请为 `local_root` 预留足够空间以容纳并发导入。+### `parsers`++| 字段 | 类型 / 常用值 | 默认值 | 作用 |+|---|---|---|---|+| `download_temp_dir` | 路径或 `null` | 系统临时目录 | 内置 parser 下载文件、Git checkout 和压缩包解压使用的临时目录 |++配置后,parser 会在该目录下创建带有 `ov_git_`、`ov_zip_` 等前缀的临时目录;未配置时继续使用系统临时目录。+## 队列 Worker 配置### `queue_workers.external_parse`diff --git a/openviking/parse/accessors/feishu_accessor.py b/openviking/parse/accessors/feishu_accessor.pyindex f849a3b71..6a78b8130 100644--- a/openviking/parse/accessors/feishu_accessor.py+++ b/openviking/parse/accessors/feishu_accessor.py@@ -26,6 +26,7 @@ from lark_oapi.core.cache import ICachefrom openviking.parse.base import format_table_to_markdownfrom openviking.parse.feishu_import import FeishuImportPlan, recursive_wiki+from openviking.parse.temp import get_parser_download_temp_dirfrom openviking.utils.exceptions import error_code_from_http_statusfrom openviking_cli.exceptions import InvalidArgumentError, OpenVikingErrorfrom openviking_cli.utils.logger import get_logger@@ -505,7 +506,11 @@ class FeishuAccessor(DataAccessor):)if doc_type == "folder":- temp_dir = Path(tempfile.mkdtemp(prefix="ov_feishu_folder_"))+ temp_dir = Path(+ tempfile.mkdtemp(+ prefix="ov_feishu_folder_", dir=get_parser_download_temp_dir()+ )+ )plan = self._new_import_plan(temp_dir, kwargs)plan.source_url = source_strskipped_items: list[dict[str, Any]] = []@@ -541,7 +546,11 @@ class FeishuAccessor(DataAccessor):)if doc_type == "wiki" and recursive_wiki(kwargs):- temp_dir = Path(tempfile.mkdtemp(prefix="ov_feishu_wiki_"))+ temp_dir = Path(+ tempfile.mkdtemp(+ prefix="ov_feishu_wiki_", dir=get_parser_download_temp_dir()+ )+ )plan = self._new_import_plan(temp_dir, kwargs)plan.source_url = source_strskipped_items: list[dict[str, Any]] = []@@ -621,7 +630,9 @@ class FeishuAccessor(DataAccessor):}if downloaded_images:- temp_dir = Path(tempfile.mkdtemp(prefix="ov_feishu_"))+ temp_dir = Path(+ tempfile.mkdtemp(prefix="ov_feishu_", dir=get_parser_download_temp_dir())+ )markdown_path = temp_dir / "document.md"markdown_path.write_text(markdown_content, encoding="utf-8")for rel_path, image_bytes in downloaded_images.items():@@ -637,6 +648,7 @@ class FeishuAccessor(DataAccessor):suffix=".md",prefix="ov_feishu_",delete=False,+ dir=get_parser_download_temp_dir(),encoding="utf-8",)temp_file.write(markdown_content)@@ -1530,7 +1542,9 @@ class FeishuAccessor(DataAccessor):content_type,filename_hint=filename_hint,)- temp_dir = Path(tempfile.mkdtemp(prefix="ov_feishu_file_"))+ temp_dir = Path(+ tempfile.mkdtemp(prefix="ov_feishu_file_", dir=get_parser_download_temp_dir())+ )path = temp_dir / filenamepath.write_bytes(content)return pathdiff --git a/openviking/parse/accessors/git_accessor.py b/openviking/parse/accessors/git_accessor.pyindex 6e34599e2..dad21ac81 100644--- a/openviking/parse/accessors/git_accessor.py+++ b/openviking/parse/accessors/git_accessor.py@@ -35,6 +35,7 @@ from openviking.utils.git_auth import (parse_git_http_auth_config,raise_git_auth_error,)+from openviking.parse.temp import get_parser_download_temp_dirfrom openviking_cli.utils.logger import get_loggerfrom .base import DataAccessor, LocalResource, SourceType@@ -113,7 +114,11 @@ class GitAccessor(DataAccessor):try:# Create local temp directory (non-blocking)- temp_local_dir = await asyncio.to_thread(tempfile.mkdtemp, prefix="ov_git_")+ temp_local_dir = await asyncio.to_thread(+ tempfile.mkdtemp,+ prefix="ov_git_",+ dir=get_parser_download_temp_dir(),+ )logger.info(f"[GitAccessor] Created local temp dir for git: {temp_local_dir}")# Fetch content (Clone or Extract)diff --git a/openviking/parse/accessors/http_accessor.py b/openviking/parse/accessors/http_accessor.pyindex b8f20b533..d94ba2034 100644--- a/openviking/parse/accessors/http_accessor.py+++ b/openviking/parse/accessors/http_accessor.py@@ -24,6 +24,7 @@ from typing import Any, Dict, Mapping, Optional, Tuple, Unionfrom urllib.parse import unquote, urlparsefrom openviking.parse.base import lazy_import+from openviking.parse.temp import get_parser_download_temp_dirfrom openviking.parse.parsers.constants import CODE_EXTENSIONSfrom openviking.parse.parsers.media.constants import (AUDIO_EXTENSIONS,@@ -626,7 +627,11 @@ class HTTPAccessor(DataAccessor):# Create temp file after GET headers/content have had a chance# to refine detection. This avoids routing extensionless signed# URLs as HTML when HEAD is rejected by object storage.- temp_file = tempfile.NamedTemporaryFile(delete=False, suffix=ext)+ temp_file = tempfile.NamedTemporaryFile(+ delete=False,+ suffix=ext,+ dir=get_parser_download_temp_dir(),+ )temp_path = temp_file.nametemp_file.close()diff --git a/openviking/parse/accessors/web_feed_accessor.py b/openviking/parse/accessors/web_feed_accessor.pyindex bcddf07ae..c9c9cc21c 100644--- a/openviking/parse/accessors/web_feed_accessor.py+++ b/openviking/parse/accessors/web_feed_accessor.py@@ -37,6 +37,7 @@ from urllib.parse import unquote, urljoin, urlparsefrom urllib.robotparser import RobotFileParserfrom openviking.parse.base import lazy_import+from openviking.parse.temp import get_parser_download_temp_dirfrom openviking.utils.network_guard import build_httpx_request_validation_hooksfrom openviking_cli.utils.logger import get_logger@@ -288,7 +289,11 @@ class WebFeedAccessor(DataAccessor):include = kwargs.get("include")exclude = kwargs.get("exclude")- temp_dir = await asyncio.to_thread(tempfile.mkdtemp, prefix="ov_webfeed_")+ temp_dir = await asyncio.to_thread(+ tempfile.mkdtemp,+ prefix="ov_webfeed_",+ dir=get_parser_download_temp_dir(),+ )try:client_kwargs = _build_httpx_client_kwargs(request_validator, cfg["request_timeout"])httpx = lazy_import("httpx")diff --git a/openviking/parse/accessors/web_importer.py b/openviking/parse/accessors/web_importer.pyindex 5b781cc30..77786c68a 100644--- a/openviking/parse/accessors/web_importer.py+++ b/openviking/parse/accessors/web_importer.py@@ -12,6 +12,7 @@ from typing import Any, Dict, Optionalfrom urllib.parse import unquote, urlsplit, urlunsplitfrom openviking.parse.accessors.http_accessor import HTTPAccessor+from openviking.parse.temp import get_parser_download_temp_dirfrom openviking.parse.accessors.web_crawler import CrawlConfig, ScrapyWebCrawlerfrom openviking.parse.accessors.web_feed_accessor import (FeedEntry,@@ -98,7 +99,9 @@ class WebImporter:raise FailedPreconditionError(message)raise RuntimeError(message)- temp_root = Path(tempfile.mkdtemp(prefix="ov_web_"))+ temp_root = Path(+ tempfile.mkdtemp(prefix="ov_web_", dir=get_parser_download_temp_dir())+ )temp_dir = temp_root / _host_name(root_url)try:temp_dir.mkdir(parents=True, exist_ok=True)diff --git a/openviking/parse/output.py b/openviking/parse/output.pyindex 4f9d13da8..59e08b8ba 100644--- a/openviking/parse/output.py+++ b/openviking/parse/output.py@@ -430,18 +430,6 @@ class LocalParseOutputStore(ParseOutputStore):raise ValueError(f"resolved path escapes the artifact root: {rel_path}")return target- @staticmethod- def _check_case_conflict(target: Path) -> None:- parent = target.parent- if not parent.is_dir():- return- lower = target.name.casefold()- for existing in parent.iterdir():- if existing.name != target.name and existing.name.casefold() == lower:- raise ValueError(- f"case-only name conflict: {target.name} vs existing {existing.name}"- )-async def create_artifact(self, *, root_type: str = "dir") -> ParseArtifactRef:if root_type not in _ROOT_TYPES:raise ValueError(f"root_type must be one of {sorted(_ROOT_TYPES)}")@@ -462,7 +450,6 @@ class LocalParseOutputStore(ParseOutputStore):def _write() -> None:target.parent.mkdir(parents=True, exist_ok=True)- self._check_case_conflict(target)target.write_bytes(content)await asyncio.to_thread(_write)@@ -508,7 +495,6 @@ class LocalParseOutputStore(ParseOutputStore):def _move() -> None:target.parent.mkdir(parents=True, exist_ok=True)- self._check_case_conflict(target)shutil.move(str(source), str(target))await asyncio.to_thread(_move)diff --git a/openviking/parse/parsers/media/audio.py b/openviking/parse/parsers/media/audio.pyindex 3b20dd1fb..64dce13ca 100644--- a/openviking/parse/parsers/media/audio.py+++ b/openviking/parse/parsers/media/audio.py@@ -35,6 +35,7 @@ from openviking.parse.output import create_parse_artifact_writerfrom openviking.parse.parsers.base_parser import BaseParserfrom openviking.parse.parsers.media.constants import AUDIO_EXTENSIONSfrom openviking.parse.parsers.media.naming import resolve_media_names+from openviking.parse.temp import get_parser_download_temp_dirfrom openviking_cli.utils.config.parser_config import AudioConfigfrom openviking_cli.utils.logger import get_logger@@ -197,7 +198,12 @@ class AudioParser(BaseParser):if not suffix:suffix = ".wav"- with tempfile.NamedTemporaryFile(mode="wb", suffix=suffix, delete=False) as temp_file:+ with tempfile.NamedTemporaryFile(+ mode="wb",+ suffix=suffix,+ delete=False,+ dir=get_parser_download_temp_dir(),+ ) as temp_file:temp_file.write(audio_bytes)temp_file_path = temp_file.namediff --git a/openviking/parse/parsers/media/utils.py b/openviking/parse/parsers/media/utils.pyindex d77808116..081e4d48f 100644--- a/openviking/parse/parsers/media/utils.py+++ b/openviking/parse/parsers/media/utils.py@@ -11,6 +11,7 @@ from typing import TYPE_CHECKING, Any, Dict, Optionalfrom openviking.core.path_variables import CalendarVariableProviderfrom openviking.parse.parsers.constants import TYPESCRIPT_MPEG_TS_EXTENSION+from openviking.parse.temp import get_parser_download_temp_dirfrom openviking.prompts import render_promptfrom openviking.storage.viking_fs import get_viking_fsfrom openviking.utils.image_search import prepare_image_bytes_for_model@@ -463,6 +464,7 @@ async def _generate_media_summary(with tempfile.NamedTemporaryFile(suffix=Path(original_filename).suffix,delete=False,+ dir=get_parser_download_temp_dir(),) as temporary:temporary_path = Path(temporary.name)diff --git a/openviking/parse/parsers/zip_parser.py b/openviking/parse/parsers/zip_parser.pyindex b045ce623..15ec23077 100644--- a/openviking/parse/parsers/zip_parser.py+++ b/openviking/parse/parsers/zip_parser.py@@ -15,6 +15,7 @@ from pathlib import Pathfrom typing import List, Optional, Unionfrom openviking.parse.base import ParseResult+from openviking.parse.temp import get_parser_download_temp_dirfrom openviking.parse.parsers.base_parser import BaseParserfrom openviking.utils.zip_safe import safe_extract_zipfrom openviking_cli.exceptions import InvalidArgumentError@@ -64,7 +65,13 @@ class ZipParser(BaseParser):raise FileNotFoundError(f"ZIP file not found: {path}")# Extract ZIP to temporary directory (non-blocking)- temp_dir = Path(await asyncio.to_thread(tempfile.mkdtemp, prefix="ov_zip_"))+ temp_dir = Path(+ await asyncio.to_thread(+ tempfile.mkdtemp,+ prefix="ov_zip_",+ dir=get_parser_download_temp_dir(),+ )+ )try:# Extract zip (non-blocking)def _extract_zip():diff --git a/openviking/parse/temp.py b/openviking/parse/temp.pynew file mode 100644index 000000000..e65a9495a--- /dev/null+++ b/openviking/parse/temp.py@@ -0,0 +1,22 @@+"""Helpers for parser-owned download and extraction temporary files."""++from __future__ import annotations++from pathlib import Path+from typing import Optional+++def get_parser_download_temp_dir() -> Optional[str]:+ """Return the configured parser download directory, if configured."""+ try:+ from openviking_cli.utils.config import get_openviking_config++ configured = get_openviking_config().parser.download_temp_dir+ except Exception:+ return None++ if not configured:+ return None+ root = Path(configured).expanduser()+ root.mkdir(parents=True, exist_ok=True)+ return str(root)diff --git a/openviking/parse/understanding_api.py b/openviking/parse/understanding_api.pyindex e5cedf32d..9d55a28e8 100644--- a/openviking/parse/understanding_api.py+++ b/openviking/parse/understanding_api.py@@ -39,6 +39,7 @@ from openviking.parse.parsers.media.constants import (IMAGE_EXTENSIONS,VIDEO_EXTENSIONS,)+from openviking.parse.temp import get_parser_download_temp_dirfrom openviking.storage.viking_fs import get_viking_fsfrom openviking.utils.zip_safe import normalize_zip_filenames, safe_extract_zipfrom openviking_cli.exceptions import InvalidArgumentError@@ -775,7 +776,11 @@ class UnderstandingAPI(BaseParser):async with httpx.AsyncClient(timeout=60.0, follow_redirects=True) as client:rsp = await client.get(zip_url)rsp.raise_for_status()- with tempfile.NamedTemporaryFile(delete=False, suffix=".zip") as f:+ with tempfile.NamedTemporaryFile(+ delete=False,+ suffix=".zip",+ dir=get_parser_download_temp_dir(),+ ) as f:f.write(rsp.content)return Path(f.name)@@ -794,7 +799,7 @@ class UnderstandingAPI(BaseParser):try:await writer.mkdir(resource_rel)- with tempfile.TemporaryDirectory() as extract_dir:+ with tempfile.TemporaryDirectory(dir=get_parser_download_temp_dir()) as extract_dir:with zipfile.ZipFile(zip_path, "r") as zf:safe_extract_zip(zf, extract_dir)extract_path = Path(extract_dir)diff --git a/openviking_cli/utils/config/open_viking_config.py b/openviking_cli/utils/config/open_viking_config.pyindex 60185bf45..faf905f20 100644--- a/openviking_cli/utils/config/open_viking_config.py+++ b/openviking_cli/utils/config/open_viking_config.py@@ -160,6 +160,18 @@ class ParserApiConfig(BaseModel):return self+class ParserRuntimeConfig(BaseModel):+ """Runtime directories used by built-in parser download/extraction paths."""++ download_temp_dir: Optional[str] = Field(+ default=None,+ description=(+ "Directory for parser-owned downloads, Git checkouts, and archive extraction. "+ "Defaults to the system temporary directory."+ ),+ )++class CompileApiConfig(BaseModel):"""Configuration for the external Compile task API."""@@ -213,6 +225,11 @@ class OpenVikingConfig(BaseModel):default_factory=StorageConfig, description="Storage configuration")+ parser: ParserRuntimeConfig = Field(+ default_factory=ParserRuntimeConfig,+ description="Runtime directories used by built-in parsers",+ )+embedding: EmbeddingConfig = Field(default_factory=EmbeddingConfig, description="Embedding configuration")@@ -548,6 +565,8 @@ class OpenVikingConfig(BaseModel):# Handle parser configurations from nested "parsers" sectionparser_configs = {}+ missing_parser_download_temp_dir = object()+ parser_download_temp_dir = missing_parser_download_temp_dirif "parsers" in config_copy:parser_configs = config_copy.pop("parsers")if parser_configs is None:@@ -561,9 +580,12 @@ class OpenVikingConfig(BaseModel):"Config field 'parsers.excel' was removed and is ignored; ""spreadsheet parsing now uses 'parsers.anydoc'.")+ parser_download_temp_dir = parser_configs.pop(+ "download_temp_dir", missing_parser_download_temp_dir+ )warn_unknown_fields(data=parser_configs,- valid_fields=set(parser_types),+ valid_fields=set(parser_types) | {"download_temp_dir"},path_prefix="parsers",logger=_get_config_logger(),)@@ -602,6 +624,9 @@ class OpenVikingConfig(BaseModel):instance._check_expiry_backend()# Apply parser configurations+ if parser_download_temp_dir is not missing_parser_download_temp_dir:+ instance.parser = ParserRuntimeConfig(download_temp_dir=parser_download_temp_dir)+for parser_type in parser_types:if parser_type in parser_configs:parser_data = parser_configs[parser_type]diff --git a/tests/parse/test_parse_output.py b/tests/parse/test_parse_output.pyindex 8517d0ca4..01511aeb9 100644--- a/tests/parse/test_parse_output.py+++ b/tests/parse/test_parse_output.py@@ -287,12 +287,13 @@ class TestLocalParseOutputStore:assert outside.exists()- async def test_case_only_conflict_is_detected(self, tmp_path) -> None:+ async def test_case_only_names_can_coexist(self, tmp_path) -> None:store = LocalParseOutputStore(local_root=str(tmp_path / "out"))ref = await store.create_artifact(root_type="dir")await store.write_bytes(ref, "Readme.md", b"a")- with pytest.raises(ValueError, match="case"):- await store.write_bytes(ref, "README.MD", b"b")+ await store.write_bytes(ref, "README.MD", b"b")+ assert await store.read_bytes(ref, "Readme.md") == b"a"+ assert await store.read_bytes(ref, "README.MD") == b"b"async def test_missing_artifact_read_raises(self, tmp_path) -> None:store = LocalParseOutputStore(local_root=str(tmp_path / "out"))diff --git a/tests/parse/test_parser_download_temp_dir.py b/tests/parse/test_parser_download_temp_dir.pynew file mode 100644index 000000000..084da170e--- /dev/null+++ b/tests/parse/test_parser_download_temp_dir.py@@ -0,0 +1,27 @@+from pathlib import Path++from openviking_cli.utils.config.open_viking_config import OpenVikingConfig+++def test_parser_download_temp_dir_is_configurable() -> None:+ config = OpenVikingConfig.from_dict(+ {"parsers": {"download_temp_dir": "/var/tmp/openviking-downloads"}}+ )++ assert config.parser.download_temp_dir == "/var/tmp/openviking-downloads"+++def test_parser_download_temp_dir_helper_creates_configured_directory(tmp_path, monkeypatch) -> None:+ from openviking.parse import temp++ configured = tmp_path / "downloads"+ config = OpenVikingConfig.from_dict(+ {"parsers": {"download_temp_dir": str(configured)}}+ )+ monkeypatch.setattr(+ "openviking_cli.utils.config.get_openviking_config",+ lambda: config,+ )++ assert temp.get_parser_download_temp_dir() == str(configured)+ assert Path(configured).is_dir()