Skip to content

Self-evolving Context Database for AI Agents. Unify Agent Memory, Knowledge RAG and Skills.

Fix/remove case only conflict check (#6298)

Jiahui Zhoub51cba425487
diff --git a/docs/en/configuration/01-server.md b/docs/en/configuration/01-server.md
index a002e200d..a8f2a9037 100644
--- a/docs/en/configuration/01-server.md
+++ b/docs/en/configuration/01-server.md
@@ -196,6 +196,9 @@ Search and Find requests default to `limit: 10`; override the limit on each API
"parse_output": {
"mode": "agfs"
}
+ },
+ "parsers": {
+ "download_temp_dir": "/data/openviking/parser-tmp"
}
}
```
@@ -219,6 +222,14 @@ The same worker must commit the required bytes to the formal resource tree befor
enqueueing downstream work. Artifacts are temporary and are removed after the
content commit; provision `local_root` with enough space for concurrent imports.
+### `parsers`
+
+| Field | Type / common values | Default | Purpose |
+|---|---|---|---|
+| `download_temp_dir` | path or `null` | system temp directory | Temporary directory for built-in parser downloads, Git checkouts, and archive extraction |
+
+When configured, parsers create temporary directories with prefixes such as `ov_git_` and `ov_zip_` below this directory. When unset, the system temporary directory is used.
+
## Queue Worker Settings
### `queue_workers.external_parse`
diff --git a/docs/zh/configuration/01-server.md b/docs/zh/configuration/01-server.md
index c4b5bf611..34ca25932 100644
--- a/docs/zh/configuration/01-server.md
+++ b/docs/zh/configuration/01-server.md
@@ -196,6 +196,9 @@ Search 和 Find 请求的默认 `limit` 为 `10`,可以在每次 API 或 SDK
"parse_output": {
"mode": "agfs"
}
+ },
+ "parsers": {
+ "download_temp_dir": "/data/openviking/parser-tmp"
}
}
```
@@ -218,6 +221,14 @@ Search 和 Find 请求的默认 `limit` 为 `10`,可以在每次 API 或 SDK
必须在下游任务入队前把所需字节提交到正式资源树。产物会在内容提交后清理;
请为 `local_root` 预留足够空间以容纳并发导入。
+### `parsers`
+
+| 字段 | 类型 / 常用值 | 默认值 | 作用 |
+|---|---|---|---|
+| `download_temp_dir` | 路径或 `null` | 系统临时目录 | 内置 parser 下载文件、Git checkout 和压缩包解压使用的临时目录 |
+
+配置后,parser 会在该目录下创建带有 `ov_git_`、`ov_zip_` 等前缀的临时目录;未配置时继续使用系统临时目录。
+
## 队列 Worker 配置
### `queue_workers.external_parse`
diff --git a/openviking/parse/accessors/feishu_accessor.py b/openviking/parse/accessors/feishu_accessor.py
index f849a3b71..6a78b8130 100644
--- a/openviking/parse/accessors/feishu_accessor.py
+++ b/openviking/parse/accessors/feishu_accessor.py
@@ -26,6 +26,7 @@ from lark_oapi.core.cache import ICache
from openviking.parse.base import format_table_to_markdown
from openviking.parse.feishu_import import FeishuImportPlan, recursive_wiki
+from openviking.parse.temp import get_parser_download_temp_dir
from openviking.utils.exceptions import error_code_from_http_status
from openviking_cli.exceptions import InvalidArgumentError, OpenVikingError
from openviking_cli.utils.logger import get_logger
@@ -505,7 +506,11 @@ class FeishuAccessor(DataAccessor):
)
if doc_type == "folder":
- temp_dir = Path(tempfile.mkdtemp(prefix="ov_feishu_folder_"))
+ temp_dir = Path(
+ tempfile.mkdtemp(
+ prefix="ov_feishu_folder_", dir=get_parser_download_temp_dir()
+ )
+ )
plan = self._new_import_plan(temp_dir, kwargs)
plan.source_url = source_str
skipped_items: list[dict[str, Any]] = []
@@ -541,7 +546,11 @@ class FeishuAccessor(DataAccessor):
)
if doc_type == "wiki" and recursive_wiki(kwargs):
- temp_dir = Path(tempfile.mkdtemp(prefix="ov_feishu_wiki_"))
+ temp_dir = Path(
+ tempfile.mkdtemp(
+ prefix="ov_feishu_wiki_", dir=get_parser_download_temp_dir()
+ )
+ )
plan = self._new_import_plan(temp_dir, kwargs)
plan.source_url = source_str
skipped_items: list[dict[str, Any]] = []
@@ -621,7 +630,9 @@ class FeishuAccessor(DataAccessor):
}
if downloaded_images:
- temp_dir = Path(tempfile.mkdtemp(prefix="ov_feishu_"))
+ temp_dir = Path(
+ tempfile.mkdtemp(prefix="ov_feishu_", dir=get_parser_download_temp_dir())
+ )
markdown_path = temp_dir / "document.md"
markdown_path.write_text(markdown_content, encoding="utf-8")
for rel_path, image_bytes in downloaded_images.items():
@@ -637,6 +648,7 @@ class FeishuAccessor(DataAccessor):
suffix=".md",
prefix="ov_feishu_",
delete=False,
+ dir=get_parser_download_temp_dir(),
encoding="utf-8",
)
temp_file.write(markdown_content)
@@ -1530,7 +1542,9 @@ class FeishuAccessor(DataAccessor):
content_type,
filename_hint=filename_hint,
)
- temp_dir = Path(tempfile.mkdtemp(prefix="ov_feishu_file_"))
+ temp_dir = Path(
+ tempfile.mkdtemp(prefix="ov_feishu_file_", dir=get_parser_download_temp_dir())
+ )
path = temp_dir / filename
path.write_bytes(content)
return path
diff --git a/openviking/parse/accessors/git_accessor.py b/openviking/parse/accessors/git_accessor.py
index 6e34599e2..dad21ac81 100644
--- a/openviking/parse/accessors/git_accessor.py
+++ b/openviking/parse/accessors/git_accessor.py
@@ -35,6 +35,7 @@ from openviking.utils.git_auth import (
parse_git_http_auth_config,
raise_git_auth_error,
)
+from openviking.parse.temp import get_parser_download_temp_dir
from openviking_cli.utils.logger import get_logger
from .base import DataAccessor, LocalResource, SourceType
@@ -113,7 +114,11 @@ class GitAccessor(DataAccessor):
try:
# Create local temp directory (non-blocking)
- temp_local_dir = await asyncio.to_thread(tempfile.mkdtemp, prefix="ov_git_")
+ temp_local_dir = await asyncio.to_thread(
+ tempfile.mkdtemp,
+ prefix="ov_git_",
+ dir=get_parser_download_temp_dir(),
+ )
logger.info(f"[GitAccessor] Created local temp dir for git: {temp_local_dir}")
# Fetch content (Clone or Extract)
diff --git a/openviking/parse/accessors/http_accessor.py b/openviking/parse/accessors/http_accessor.py
index b8f20b533..d94ba2034 100644
--- a/openviking/parse/accessors/http_accessor.py
+++ b/openviking/parse/accessors/http_accessor.py
@@ -24,6 +24,7 @@ from typing import Any, Dict, Mapping, Optional, Tuple, Union
from urllib.parse import unquote, urlparse
from openviking.parse.base import lazy_import
+from openviking.parse.temp import get_parser_download_temp_dir
from openviking.parse.parsers.constants import CODE_EXTENSIONS
from openviking.parse.parsers.media.constants import (
AUDIO_EXTENSIONS,
@@ -626,7 +627,11 @@ class HTTPAccessor(DataAccessor):
# Create temp file after GET headers/content have had a chance
# to refine detection. This avoids routing extensionless signed
# URLs as HTML when HEAD is rejected by object storage.
- temp_file = tempfile.NamedTemporaryFile(delete=False, suffix=ext)
+ temp_file = tempfile.NamedTemporaryFile(
+ delete=False,
+ suffix=ext,
+ dir=get_parser_download_temp_dir(),
+ )
temp_path = temp_file.name
temp_file.close()
diff --git a/openviking/parse/accessors/web_feed_accessor.py b/openviking/parse/accessors/web_feed_accessor.py
index bcddf07ae..c9c9cc21c 100644
--- a/openviking/parse/accessors/web_feed_accessor.py
+++ b/openviking/parse/accessors/web_feed_accessor.py
@@ -37,6 +37,7 @@ from urllib.parse import unquote, urljoin, urlparse
from urllib.robotparser import RobotFileParser
from openviking.parse.base import lazy_import
+from openviking.parse.temp import get_parser_download_temp_dir
from openviking.utils.network_guard import build_httpx_request_validation_hooks
from openviking_cli.utils.logger import get_logger
@@ -288,7 +289,11 @@ class WebFeedAccessor(DataAccessor):
include = kwargs.get("include")
exclude = kwargs.get("exclude")
- temp_dir = await asyncio.to_thread(tempfile.mkdtemp, prefix="ov_webfeed_")
+ temp_dir = await asyncio.to_thread(
+ tempfile.mkdtemp,
+ prefix="ov_webfeed_",
+ dir=get_parser_download_temp_dir(),
+ )
try:
client_kwargs = _build_httpx_client_kwargs(request_validator, cfg["request_timeout"])
httpx = lazy_import("httpx")
diff --git a/openviking/parse/accessors/web_importer.py b/openviking/parse/accessors/web_importer.py
index 5b781cc30..77786c68a 100644
--- a/openviking/parse/accessors/web_importer.py
+++ b/openviking/parse/accessors/web_importer.py
@@ -12,6 +12,7 @@ from typing import Any, Dict, Optional
from urllib.parse import unquote, urlsplit, urlunsplit
from openviking.parse.accessors.http_accessor import HTTPAccessor
+from openviking.parse.temp import get_parser_download_temp_dir
from openviking.parse.accessors.web_crawler import CrawlConfig, ScrapyWebCrawler
from openviking.parse.accessors.web_feed_accessor import (
FeedEntry,
@@ -98,7 +99,9 @@ class WebImporter:
raise FailedPreconditionError(message)
raise RuntimeError(message)
- temp_root = Path(tempfile.mkdtemp(prefix="ov_web_"))
+ temp_root = Path(
+ tempfile.mkdtemp(prefix="ov_web_", dir=get_parser_download_temp_dir())
+ )
temp_dir = temp_root / _host_name(root_url)
try:
temp_dir.mkdir(parents=True, exist_ok=True)
diff --git a/openviking/parse/output.py b/openviking/parse/output.py
index 4f9d13da8..59e08b8ba 100644
--- a/openviking/parse/output.py
+++ b/openviking/parse/output.py
@@ -430,18 +430,6 @@ class LocalParseOutputStore(ParseOutputStore):
raise ValueError(f"resolved path escapes the artifact root: {rel_path}")
return target
- @staticmethod
- def _check_case_conflict(target: Path) -> None:
- parent = target.parent
- if not parent.is_dir():
- return
- lower = target.name.casefold()
- for existing in parent.iterdir():
- if existing.name != target.name and existing.name.casefold() == lower:
- raise ValueError(
- f"case-only name conflict: {target.name} vs existing {existing.name}"
- )
-
async def create_artifact(self, *, root_type: str = "dir") -> ParseArtifactRef:
if root_type not in _ROOT_TYPES:
raise ValueError(f"root_type must be one of {sorted(_ROOT_TYPES)}")
@@ -462,7 +450,6 @@ class LocalParseOutputStore(ParseOutputStore):
def _write() -> None:
target.parent.mkdir(parents=True, exist_ok=True)
- self._check_case_conflict(target)
target.write_bytes(content)
await asyncio.to_thread(_write)
@@ -508,7 +495,6 @@ class LocalParseOutputStore(ParseOutputStore):
def _move() -> None:
target.parent.mkdir(parents=True, exist_ok=True)
- self._check_case_conflict(target)
shutil.move(str(source), str(target))
await asyncio.to_thread(_move)
diff --git a/openviking/parse/parsers/media/audio.py b/openviking/parse/parsers/media/audio.py
index 3b20dd1fb..64dce13ca 100644
--- a/openviking/parse/parsers/media/audio.py
+++ b/openviking/parse/parsers/media/audio.py
@@ -35,6 +35,7 @@ from openviking.parse.output import create_parse_artifact_writer
from openviking.parse.parsers.base_parser import BaseParser
from openviking.parse.parsers.media.constants import AUDIO_EXTENSIONS
from openviking.parse.parsers.media.naming import resolve_media_names
+from openviking.parse.temp import get_parser_download_temp_dir
from openviking_cli.utils.config.parser_config import AudioConfig
from openviking_cli.utils.logger import get_logger
@@ -197,7 +198,12 @@ class AudioParser(BaseParser):
if not suffix:
suffix = ".wav"
- with tempfile.NamedTemporaryFile(mode="wb", suffix=suffix, delete=False) as temp_file:
+ with tempfile.NamedTemporaryFile(
+ mode="wb",
+ suffix=suffix,
+ delete=False,
+ dir=get_parser_download_temp_dir(),
+ ) as temp_file:
temp_file.write(audio_bytes)
temp_file_path = temp_file.name
diff --git a/openviking/parse/parsers/media/utils.py b/openviking/parse/parsers/media/utils.py
index d77808116..081e4d48f 100644
--- a/openviking/parse/parsers/media/utils.py
+++ b/openviking/parse/parsers/media/utils.py
@@ -11,6 +11,7 @@ from typing import TYPE_CHECKING, Any, Dict, Optional
from openviking.core.path_variables import CalendarVariableProvider
from openviking.parse.parsers.constants import TYPESCRIPT_MPEG_TS_EXTENSION
+from openviking.parse.temp import get_parser_download_temp_dir
from openviking.prompts import render_prompt
from openviking.storage.viking_fs import get_viking_fs
from openviking.utils.image_search import prepare_image_bytes_for_model
@@ -463,6 +464,7 @@ async def _generate_media_summary(
with tempfile.NamedTemporaryFile(
suffix=Path(original_filename).suffix,
delete=False,
+ dir=get_parser_download_temp_dir(),
) as temporary:
temporary_path = Path(temporary.name)
diff --git a/openviking/parse/parsers/zip_parser.py b/openviking/parse/parsers/zip_parser.py
index b045ce623..15ec23077 100644
--- a/openviking/parse/parsers/zip_parser.py
+++ b/openviking/parse/parsers/zip_parser.py
@@ -15,6 +15,7 @@ from pathlib import Path
from typing import List, Optional, Union
from openviking.parse.base import ParseResult
+from openviking.parse.temp import get_parser_download_temp_dir
from openviking.parse.parsers.base_parser import BaseParser
from openviking.utils.zip_safe import safe_extract_zip
from openviking_cli.exceptions import InvalidArgumentError
@@ -64,7 +65,13 @@ class ZipParser(BaseParser):
raise FileNotFoundError(f"ZIP file not found: {path}")
# Extract ZIP to temporary directory (non-blocking)
- temp_dir = Path(await asyncio.to_thread(tempfile.mkdtemp, prefix="ov_zip_"))
+ temp_dir = Path(
+ await asyncio.to_thread(
+ tempfile.mkdtemp,
+ prefix="ov_zip_",
+ dir=get_parser_download_temp_dir(),
+ )
+ )
try:
# Extract zip (non-blocking)
def _extract_zip():
diff --git a/openviking/parse/temp.py b/openviking/parse/temp.py
new file mode 100644
index 000000000..e65a9495a
--- /dev/null
+++ b/openviking/parse/temp.py
@@ -0,0 +1,22 @@
+"""Helpers for parser-owned download and extraction temporary files."""
+
+from __future__ import annotations
+
+from pathlib import Path
+from typing import Optional
+
+
+def get_parser_download_temp_dir() -> Optional[str]:
+ """Return the configured parser download directory, if configured."""
+ try:
+ from openviking_cli.utils.config import get_openviking_config
+
+ configured = get_openviking_config().parser.download_temp_dir
+ except Exception:
+ return None
+
+ if not configured:
+ return None
+ root = Path(configured).expanduser()
+ root.mkdir(parents=True, exist_ok=True)
+ return str(root)
diff --git a/openviking/parse/understanding_api.py b/openviking/parse/understanding_api.py
index e5cedf32d..9d55a28e8 100644
--- a/openviking/parse/understanding_api.py
+++ b/openviking/parse/understanding_api.py
@@ -39,6 +39,7 @@ from openviking.parse.parsers.media.constants import (
IMAGE_EXTENSIONS,
VIDEO_EXTENSIONS,
)
+from openviking.parse.temp import get_parser_download_temp_dir
from openviking.storage.viking_fs import get_viking_fs
from openviking.utils.zip_safe import normalize_zip_filenames, safe_extract_zip
from openviking_cli.exceptions import InvalidArgumentError
@@ -775,7 +776,11 @@ class UnderstandingAPI(BaseParser):
async with httpx.AsyncClient(timeout=60.0, follow_redirects=True) as client:
rsp = await client.get(zip_url)
rsp.raise_for_status()
- with tempfile.NamedTemporaryFile(delete=False, suffix=".zip") as f:
+ with tempfile.NamedTemporaryFile(
+ delete=False,
+ suffix=".zip",
+ dir=get_parser_download_temp_dir(),
+ ) as f:
f.write(rsp.content)
return Path(f.name)
@@ -794,7 +799,7 @@ class UnderstandingAPI(BaseParser):
try:
await writer.mkdir(resource_rel)
- with tempfile.TemporaryDirectory() as extract_dir:
+ with tempfile.TemporaryDirectory(dir=get_parser_download_temp_dir()) as extract_dir:
with zipfile.ZipFile(zip_path, "r") as zf:
safe_extract_zip(zf, extract_dir)
extract_path = Path(extract_dir)
diff --git a/openviking_cli/utils/config/open_viking_config.py b/openviking_cli/utils/config/open_viking_config.py
index 60185bf45..faf905f20 100644
--- a/openviking_cli/utils/config/open_viking_config.py
+++ b/openviking_cli/utils/config/open_viking_config.py
@@ -160,6 +160,18 @@ class ParserApiConfig(BaseModel):
return self
+class ParserRuntimeConfig(BaseModel):
+ """Runtime directories used by built-in parser download/extraction paths."""
+
+ download_temp_dir: Optional[str] = Field(
+ default=None,
+ description=(
+ "Directory for parser-owned downloads, Git checkouts, and archive extraction. "
+ "Defaults to the system temporary directory."
+ ),
+ )
+
+
class CompileApiConfig(BaseModel):
"""Configuration for the external Compile task API."""
@@ -213,6 +225,11 @@ class OpenVikingConfig(BaseModel):
default_factory=StorageConfig, description="Storage configuration"
)
+ parser: ParserRuntimeConfig = Field(
+ default_factory=ParserRuntimeConfig,
+ description="Runtime directories used by built-in parsers",
+ )
+
embedding: EmbeddingConfig = Field(
default_factory=EmbeddingConfig, description="Embedding configuration"
)
@@ -548,6 +565,8 @@ class OpenVikingConfig(BaseModel):
# Handle parser configurations from nested "parsers" section
parser_configs = {}
+ missing_parser_download_temp_dir = object()
+ parser_download_temp_dir = missing_parser_download_temp_dir
if "parsers" in config_copy:
parser_configs = config_copy.pop("parsers")
if parser_configs is None:
@@ -561,9 +580,12 @@ class OpenVikingConfig(BaseModel):
"Config field 'parsers.excel' was removed and is ignored; "
"spreadsheet parsing now uses 'parsers.anydoc'."
)
+ parser_download_temp_dir = parser_configs.pop(
+ "download_temp_dir", missing_parser_download_temp_dir
+ )
warn_unknown_fields(
data=parser_configs,
- valid_fields=set(parser_types),
+ valid_fields=set(parser_types) | {"download_temp_dir"},
path_prefix="parsers",
logger=_get_config_logger(),
)
@@ -602,6 +624,9 @@ class OpenVikingConfig(BaseModel):
instance._check_expiry_backend()
# Apply parser configurations
+ if parser_download_temp_dir is not missing_parser_download_temp_dir:
+ instance.parser = ParserRuntimeConfig(download_temp_dir=parser_download_temp_dir)
+
for parser_type in parser_types:
if parser_type in parser_configs:
parser_data = parser_configs[parser_type]
diff --git a/tests/parse/test_parse_output.py b/tests/parse/test_parse_output.py
index 8517d0ca4..01511aeb9 100644
--- a/tests/parse/test_parse_output.py
+++ b/tests/parse/test_parse_output.py
@@ -287,12 +287,13 @@ class TestLocalParseOutputStore:
assert outside.exists()
- async def test_case_only_conflict_is_detected(self, tmp_path) -> None:
+ async def test_case_only_names_can_coexist(self, tmp_path) -> None:
store = LocalParseOutputStore(local_root=str(tmp_path / "out"))
ref = await store.create_artifact(root_type="dir")
await store.write_bytes(ref, "Readme.md", b"a")
- with pytest.raises(ValueError, match="case"):
- await store.write_bytes(ref, "README.MD", b"b")
+ await store.write_bytes(ref, "README.MD", b"b")
+ assert await store.read_bytes(ref, "Readme.md") == b"a"
+ assert await store.read_bytes(ref, "README.MD") == b"b"
async def test_missing_artifact_read_raises(self, tmp_path) -> None:
store = LocalParseOutputStore(local_root=str(tmp_path / "out"))
diff --git a/tests/parse/test_parser_download_temp_dir.py b/tests/parse/test_parser_download_temp_dir.py
new file mode 100644
index 000000000..084da170e
--- /dev/null
+++ b/tests/parse/test_parser_download_temp_dir.py
@@ -0,0 +1,27 @@
+from pathlib import Path
+
+from openviking_cli.utils.config.open_viking_config import OpenVikingConfig
+
+
+def test_parser_download_temp_dir_is_configurable() -> None:
+ config = OpenVikingConfig.from_dict(
+ {"parsers": {"download_temp_dir": "/var/tmp/openviking-downloads"}}
+ )
+
+ assert config.parser.download_temp_dir == "/var/tmp/openviking-downloads"
+
+
+def test_parser_download_temp_dir_helper_creates_configured_directory(tmp_path, monkeypatch) -> None:
+ from openviking.parse import temp
+
+ configured = tmp_path / "downloads"
+ config = OpenVikingConfig.from_dict(
+ {"parsers": {"download_temp_dir": str(configured)}}
+ )
+ monkeypatch.setattr(
+ "openviking_cli.utils.config.get_openviking_config",
+ lambda: config,
+ )
+
+ assert temp.get_parser_download_temp_dir() == str(configured)
+ assert Path(configured).is_dir()