Self-evolving Context Database for AI Agents. Unify Agent Memory, Knowledge RAG and Skills.
Python agent-contextagent-memoryagent-pluginsagentic-ragcontext-databasedsh-pluginknowledge-basememorymemory-managementself-evolvingvector-database
# Copyright (c) 2026 Beijing Volcano Engine Technology Co., Ltd.
# SPDX-License-Identifier: AGPL-3.0
"""
Directory parser for OpenViking.
Handles local directories containing mixed document types (PDF, Markdown,
Text, code, etc.). Follows the same three-phase pattern as
CodeRepositoryParser:
1. Scan → classify files with ``scan_directory()``
2. For each file:
- Files routed to UnderstandingAPI or a dedicated internal parser →
``parser.parse()`` handles conversion and VikingFS temp creation; results
are merged into the main temp.
- Files WITHOUT a parser (code, config, …) → written directly to VikingFS.
3. Return ``ParseResult`` so that ``TreeBuilder.finalize_from_temp``
can move the content to AGFS and enqueue semantic processing.
"""
import asyncio
import time
from pathlib import Path, PurePosixPath
from typing import TYPE_CHECKING, Any, Dict, List, Optional, Union
from weakref import WeakKeyDictionary
from openviking.parse.backend import ParserBackend, normalize_parser_backend
from openviking.parse.base import (
NodeType,
ParseResult,
ResourceNode,
create_parse_result,
)
from openviking.parse.gitignore import GitignoreMatcher
from openviking.parse.image_rewrite import IMAGE_MAPPINGS_FILENAME
from openviking.parse.output import (
ParseArtifactWriter,
copy_artifact_tree,
create_parse_artifact_writer,
store_for_artifact_ref,
)
from openviking.parse.parsers.base_parser import BaseParser
from openviking.parse.parsers.media.constants import MEDIA_EXTENSIONS
from openviking.parse.parsers.upload_utils import detect_and_convert_encoding, is_text_file
from openviking.storage.viking_fs import LS_ALL_NODES
from openviking.utils.path_safety import (
_normalize_storage_segments,
normalize_storage_target_uri,
)
from openviking_cli.exceptions import InvalidArgumentError
from openviking_cli.utils.logger import get_logger
if TYPE_CHECKING:
from openviking.parse.directory_scan import ClassifiedFile
from openviking.parse.parser_router import ParserRouter
from openviking.parse.registry import ParserRegistry
logger = get_logger(__name__)
# Hidden files a parser's temp tree is allowed to carry through the merge.
# Everything else hidden stays filtered, like a default ls.
_MERGE_SIDECAR_ALLOWLIST = frozenset({IMAGE_MAPPINGS_FILENAME})
# DirectoryParser instances share one limiter in the server event loop so
# concurrent directory imports cannot multiply UnderstandingAPI concurrency.
_UNDERSTANDING_LIMITERS: WeakKeyDictionary[
asyncio.AbstractEventLoop, tuple[int, asyncio.Semaphore]
] = WeakKeyDictionary()
def _normalize_output_target(path: str, writer: Optional[ParseArtifactWriter]) -> str:
"""Normalize a parser output path without assuming its artifact backend."""
if path.startswith("viking://"):
return normalize_storage_target_uri(path)
if writer is None:
raise ValueError("non-Viking output paths require an artifact writer")
root = writer.ref.root.rstrip("/")
relative_path = _normalize_storage_segments(writer.relative_path(path))
return f"{root}/{relative_path}" if relative_path else root
def _get_understanding_limiter(max_concurrent: int) -> asyncio.Semaphore:
"""Return the shared directory Understanding limiter for this event loop."""
loop = asyncio.get_running_loop()
configured = _UNDERSTANDING_LIMITERS.get(loop)
if configured is None:
limiter = asyncio.Semaphore(max_concurrent)
_UNDERSTANDING_LIMITERS[loop] = (max_concurrent, limiter)
return limiter
configured_limit, limiter = configured
if configured_limit != max_concurrent:
raise InvalidArgumentError(
"parsers.directory.max_concurrent changed while the service is running; "
"restart the service to apply the new global directory limit"
)
return limiter
class DirectoryParser(BaseParser):
"""
Parser for local directories.
Scans the directory, routes each file through ParserRouter when configured
for UnderstandingAPI, and otherwise delegates to its registered parser.
Files without any parser are written directly.
The resulting ``ParseResult.temp_dir_path`` is consumed by
``TreeBuilder.finalize_from_temp`` exactly like any other parser.
"""
@property
def supported_extensions(self) -> List[str]:
# Directories have no file extension; routing is handled
# by ``is_dir()`` checks in the registry / media processor.
return []
def can_parse(self, path: Union[str, Path]) -> bool: # type: ignore[override]
"""Return *True* when *path* is an existing directory."""
return Path(path).is_dir()
# ------------------------------------------------------------------
# Main entry point
# ------------------------------------------------------------------
async def parse(
self,
source: Union[str, Path],
instruction: str = "",
**kwargs,
) -> ParseResult:
"""Parse a local directory.
Args:
source: Path to the directory.
instruction: Processing instruction (forwarded where applicable).
**kwargs: Extra options forwarded to ``scan_directory``:
``strict``, ``ignore_dirs``, ``include``, ``exclude``,
``directly_upload_media``.
Returns:
``ParseResult`` with ``temp_dir_path`` pointing to VikingFS temp.
"""
start_time = time.time()
source_path = Path(source).resolve()
if not source_path.is_dir():
raise NotADirectoryError(f"Not a directory: {source_path}")
# Check if this is a git repository, delegate to CodeRepositoryParser
if await self._is_git_repository(source_path):
logger.debug(
f"Directory {source_path} is a git repository, delegating to CodeRepositoryParser"
)
from openviking.parse.parsers.code.code import CodeRepositoryParser
# Don't add git metadata if we already have _source_meta from DataAccessor
# This is crucial:
# 1. _source_meta already contains repo_name in org/repo format from GitAccessor
# 2. kwargs also has original_source with the full code-hosting URL
# 3. Calling _add_git_metadata would overwrite repo_name with just directory name
# and lose the org prefix!
if "_source_meta" not in kwargs:
await self._add_git_metadata(source_path, kwargs)
return await CodeRepositoryParser().parse(str(source_path), instruction, **kwargs)
dir_name = kwargs.get("source_name") or source_path.name
warnings: List[str] = []
temp_uri: Optional[str] = None
keep_temp = False
pending_results: list[ParseResult] = []
output_store = kwargs.get("parse_output_store")
writer: Optional[ParseArtifactWriter] = None
try:
# ── Phase 1: scan directory ───────────────────────────────
from openviking.parse.directory_scan import scan_directory
from openviking.parse.parser_router import ParserRouter
from openviking.parse.registry import get_registry
from openviking_cli.utils.config.open_viking_config import get_openviking_config
from openviking_cli.utils.config.parser_config import DirectoryConfig
registry = get_registry()
parser_router = ParserRouter(registry)
ov_config = get_openviking_config()
directory_config = getattr(ov_config, "directory", None) or DirectoryConfig()
split_content = kwargs.get("split_content", True)
plan = kwargs.get("_feishu_import_plan")
# Ordinary source jobs use INTERNAL for directory orchestration;
# their files still select a parser independently by extension.
backend = normalize_parser_backend(kwargs.get("parser_backend")) if plan else None
parser_api_config = getattr(ov_config, "parser_api", None)
understanding_limits_enabled = bool(
split_content
and backend is not ParserBackend.INTERNAL
and (
(plan and plan.use_understanding)
or backend is ParserBackend.UNDERSTANDING
or (
parser_router.understanding_api_enabled()
and getattr(parser_api_config, "extensions", None)
)
)
)
scan_result = scan_directory(
root=str(source_path),
registry=registry,
strict=kwargs.get("strict", False),
ignore_dirs=kwargs.get("ignore_dirs"),
include=kwargs.get("include"),
exclude=kwargs.get("exclude"),
additional_can_process=parser_router.should_use_understanding_api,
max_files=directory_config.max_files
if plan or understanding_limits_enabled
else None,
max_depth=directory_config.max_depth if understanding_limits_enabled else None,
)
directly_upload_media = kwargs.get("directly_upload_media", True)
preserve_structure = kwargs.get("preserve_structure")
if preserve_structure is None:
preserve_structure = directory_config.preserve_structure
processable_files = list(scan_result.all_processable_files())
entry_map = {entry.path.resolve(): entry for entry in plan.entries} if plan else {}
feishu_gitignore = GitignoreMatcher(source_path) if plan else None
if plan:
from openviking.parse.directory_scan import CLASS_PROCESSABLE, ClassifiedFile
for entry in plan.entries:
if entry.kind != "url":
continue
entry_path = entry.path.resolve()
relative = entry_path.relative_to(source_path).as_posix()
if not self._include_feishu_path(
entry_path, source_path, kwargs, gitignore=feishu_gitignore
):
scan_result.skipped.append(f"{relative} (excluded by source filter)")
continue
processable_files.append(
ClassifiedFile(entry_path, relative, CLASS_PROCESSABLE)
)
if (
directory_config.max_files is not None
and len(processable_files) > directory_config.max_files
):
raise InvalidArgumentError("Feishu directory file limit exceeded")
processable_files.sort(key=lambda item: item.rel_path)
warnings.extend(scan_result.warnings)
source_skipped_items = self._source_skipped_items(
kwargs.get("_source_meta"),
source_path,
)
warnings.extend(
f"Skipped Feishu Drive item {item['path']}: {item.get('reason', 'unknown error')}"
for item in source_skipped_items
)
file_jobs: List[Dict[str, Any]] = []
understanding_jobs: List[Dict[str, Any]] = []
for index, cf in enumerate(processable_files):
entry = entry_map.get(cf.path)
remote = entry is not None and entry.kind == "url"
normalized = entry is not None and entry.kind == "markdown"
configured_for_understanding = bool(
backend is not ParserBackend.INTERNAL
and not normalized
and (
remote
or backend is ParserBackend.UNDERSTANDING
or parser_router.should_use_understanding_api(cf.path)
)
)
use_understanding = bool(split_content and configured_for_understanding)
native_parser = None if use_understanding else self._assign_parser(cf, registry)
file_parser = parser_router if use_understanding else native_parser
native_parser_unavailable = bool(
not split_content and native_parser is None and not is_text_file(cf.path)
)
parser_name = (
"UnderstandingAPI"
if use_understanding
else "native"
if native_parser_unavailable
else type(file_parser).__name__
if file_parser
else "direct"
)
is_media_parser = file_parser and parser_name in {
"ImageParser",
"AudioParser",
"VideoParser",
}
is_media_file = Path(cf.path).suffix.lower() in MEDIA_EXTENSIONS
job = {
"index": index,
"classified_file": cf,
"file_parser": file_parser,
"parser_name": parser_name,
"use_understanding": use_understanding,
"native_parser_unavailable": native_parser_unavailable,
"direct_upload": bool(
directly_upload_media
and not use_understanding
and is_media_parser
and is_media_file
),
}
if use_understanding:
parse_options = {"parser_backend": ParserBackend.UNDERSTANDING}
if remote:
parse_options.update(
_source=entry.url,
resource_name=cf.path.name,
feishu_access_token=kwargs.get("feishu_access_token"),
feishu_config=kwargs.get("feishu_config"),
)
if kwargs.get("lark_file"):
parse_options["lark_file"] = kwargs["lark_file"]
checkpoint = kwargs.get("_feishu_checkpoint")
if plan and checkpoint is not None:
saved, save = checkpoint
key = entry.checkpoint_key(plan.root) if entry else cf.rel_path
if key in saved:
parse_options["understanding_response_id"] = saved[key]
async def record(response_id, key=key, save=save):
await save(key, response_id)
parse_options["_response_checkpoint"] = record
job["parse_options"] = parse_options
file_jobs.append(job)
if use_understanding:
understanding_jobs.append(job)
viking_fs = self._get_viking_fs() if output_store is None else None
writer = await create_parse_artifact_writer(output_store, viking_fs=viking_fs)
output_store = writer.store
temp_uri = writer.ref.root
target_uri = _normalize_output_target(f"{temp_uri}/{dir_name}", writer)
target_rel = writer.relative_path(target_uri)
await writer.mkdir()
await writer.mkdir(target_rel)
if not processable_files:
root = ResourceNode(
type=NodeType.ROOT,
title=dir_name,
meta={"file_count": 0, "type": "directory"},
)
result = create_parse_result(
root=root,
source_path=str(source_path),
source_format="directory",
parser_name="DirectoryParser",
parse_time=time.time() - start_time,
warnings=warnings,
)
result.temp_dir_path = temp_uri
result.artifact_ref = await writer.finalize(resource_rel=target_rel)
result.meta["file_count"] = 0
result.meta["dir_name"] = dir_name
result.meta["total_processable"] = 0
result.meta["processed_files"] = []
result.meta["failed_files"] = source_skipped_items
result.meta["unsupported_files"] = []
result.meta["skipped_files"] = self._parse_skipped(scan_result.skipped)
keep_temp = True
return result
# ── Phase 2: process each file ────────────────────────────
file_count = 0
processed_files: List[Dict[str, Any]] = []
failed_files: List[Dict[str, Any]] = []
understanding_results: Dict[int, Dict[str, Any]] = {}
if understanding_jobs:
parser_api = getattr(ov_config, "parser_api", None)
job_timeout = self._get_parser_api_job_timeout(parser_api)
logger.info(
"[DirectoryParser] Processing %d Understanding file(s) with "
"shared_max_concurrent=%d, job_timeout=%.1fs",
len(understanding_jobs),
directory_config.max_concurrent,
job_timeout,
)
understanding_results = await self._parse_understanding_jobs(
understanding_jobs,
preserve_structure=preserve_structure,
import_root=str(source_path),
split_content=split_content,
max_concurrent=directory_config.max_concurrent,
job_timeout=job_timeout,
output_store=output_store,
)
pending_results.extend(
parsed["result"]
for parsed in understanding_results.values()
if parsed.get("result") and parsed["result"].temp_dir_path
)
for job in file_jobs:
cf = job["classified_file"]
file_parser = job["file_parser"]
parser_name = job["parser_name"]
detail: Dict[str, Any]
if job["use_understanding"]:
parsed = understanding_results[job["index"]]
error = parsed.get("error")
sub_result = parsed.get("result")
if error is not None:
warnings.append(f"Failed to parse {cf.rel_path}: {error}")
error_meta = getattr(error, "meta", {})
detail = {
"ok": False,
"meta": error_meta if isinstance(error_meta, dict) else {},
"error": str(error),
}
else:
try:
# The merge helper owns this artifact once merging starts.
if sub_result in pending_results:
pending_results.remove(sub_result)
await self._merge_parser_result(
cf,
sub_result,
target_uri,
viking_fs,
target_writer=writer,
preserve_structure=preserve_structure,
split_content=split_content,
)
detail = {
"ok": True,
"meta": getattr(sub_result, "meta", {}) or {},
"error": None,
}
except Exception as exc:
warnings.append(f"Failed to parse {cf.rel_path}: {exc}")
detail = {
"ok": False,
"meta": getattr(sub_result, "meta", {}) or {},
"error": str(exc),
}
elif job["native_parser_unavailable"]:
error = (
"parse_mode='no_split' requires a native parser, but none is "
"available for this file type"
)
warnings.append(f"Failed to parse {cf.rel_path}: {error}")
detail = {"ok": False, "meta": {}, "error": error}
elif job["direct_upload"]:
detail = await self._upload_file_directly(
cf,
target_uri,
viking_fs,
warnings,
target_writer=writer,
preserve_structure=preserve_structure,
)
parser_name = "direct_upload"
else:
detail = await self._process_single_file(
cf,
file_parser,
target_uri,
viking_fs,
warnings,
target_writer=writer,
output_store=output_store,
preserve_structure=preserve_structure,
import_root=str(source_path),
split_content=split_content,
)
file_entry = self._file_status_entry(cf, parser_name, detail)
entry = entry_map.get(cf.path)
if entry:
file_entry["source_url"] = entry.url
file_entry["source_token"] = entry.token
if plan and not detail["ok"] and kwargs.get("strict", False):
raise InvalidArgumentError(
f"Failed to import {cf.rel_path}: {detail.get('error')}"
)
if detail["ok"]:
file_count += 1
processed_files.append(file_entry)
else:
failed_files.append(file_entry)
failed_files.extend(self._nested_failed_files(cf, parser_name, detail))
# Collect unsupported files from scan result
unsupported_files = [
{
"path": uf.rel_path,
"status": "unsupported",
"reason": uf.classification,
}
for uf in scan_result.unsupported
]
# Parse skipped entries: format is "path (reason)"
skipped_files = self._parse_skipped(scan_result.skipped)
# ── Phase 3: build ParseResult ────────────────────────────
root = ResourceNode(
type=NodeType.ROOT,
title=dir_name,
meta={
"file_count": file_count,
"type": "directory",
},
)
result = create_parse_result(
root=root,
source_path=str(source_path),
source_format="directory",
parser_name="DirectoryParser",
parse_time=time.time() - start_time,
warnings=warnings,
)
result.temp_dir_path = temp_uri
result.artifact_ref = await writer.finalize(resource_rel=target_rel)
result.meta["file_count"] = file_count
result.meta["dir_name"] = dir_name
result.meta["total_processable"] = len(processable_files)
result.meta["processed_files"] = processed_files
result.meta["failed_files"] = failed_files + source_skipped_items
result.meta["unsupported_files"] = unsupported_files
result.meta["skipped_files"] = skipped_files
keep_temp = True
return result
except InvalidArgumentError:
raise
except Exception as exc:
logger.error(
f"[DirectoryParser] Failed to parse directory {source_path}: {exc}",
exc_info=True,
)
return create_parse_result(
root=ResourceNode(type=NodeType.ROOT),
source_path=str(source_path),
source_format="directory",
parser_name="DirectoryParser",
parse_time=time.time() - start_time,
warnings=[f"Failed to parse directory: {exc}"],
)
finally:
if writer is not None and not keep_temp:
await writer.cleanup()
for pending_result in pending_results:
try:
pending_ref = getattr(pending_result, "artifact_ref", None)
if pending_ref is not None:
pending_store = (
output_store
if output_store is not None
and output_store.backend == pending_ref.backend
else store_for_artifact_ref(
pending_ref,
viking_fs=self._get_viking_fs(),
)
)
await pending_store.cleanup(pending_ref)
elif pending_result.temp_dir_path:
await self._get_viking_fs().delete_temp(pending_result.temp_dir_path)
except Exception as exc:
logger.warning(
"[DirectoryParser] Failed to clean temporary artifact %s: %s",
pending_result.temp_dir_path,
exc,
)
@staticmethod
def _include_feishu_path(
path: Path,
root: Path,
options: Dict[str, Any],
*,
gitignore: Optional[GitignoreMatcher] = None,
) -> bool:
from openviking.parse.directory_scan import (
_matches_exclude,
_matches_include,
_parse_patterns,
_should_skip_directory,
)
relative = path.relative_to(root).as_posix()
ignored = options.get("ignore_dirs") or set()
if isinstance(ignored, str):
ignored = set(_parse_patterns(ignored))
for parent in path.parents:
if parent == root:
break
if _should_skip_directory(parent, root, ignored)[0]:
return False
if gitignore and gitignore.is_ignored_dir(
parent, gitignore.spec_for_dir(parent.parent)
):
return False
if path.name.startswith("."):
return False
if gitignore and gitignore.is_ignored_file(path, gitignore.spec_for_dir(path.parent)):
return False
includes = _parse_patterns(options.get("include"))
excludes = _parse_patterns(options.get("exclude"))
return (not includes or _matches_include(path.name, includes)) and not _matches_exclude(
relative, path.name, excludes
)
# ------------------------------------------------------------------
# parse_content – not applicable for directories
# ------------------------------------------------------------------
async def parse_content(
self,
content: str,
source_path: Optional[str] = None,
instruction: str = "",
**kwargs,
) -> ParseResult:
raise NotImplementedError("DirectoryParser does not support parse_content")
# ------------------------------------------------------------------
# Skipped entries parsing
# ------------------------------------------------------------------
_REASON_TO_STATUS = {
"dot directory": "ignore",
"dot file": "ignore",
"symlink": "ignore",
"empty file": "ignore",
"os error": "ignore",
"IGNORE_DIRS": "ignore",
"ignore_dirs": "ignore",
"excluded by include filter": "exclude",
"excluded by exclude filter": "exclude",
}
@staticmethod
def _parse_skipped(skipped: List[str]) -> List[Dict[str, str]]:
"""Parse skipped entry strings into structured dicts.
Each entry has the format ``"rel_path (reason)"``.
Returns a list of ``{"path": ..., "status": ..., "reason": ...}``.
"""
result: List[Dict[str, str]] = []
for entry in skipped:
# Extract "path (reason)"
paren_idx = entry.rfind(" (")
if paren_idx != -1 and entry.endswith(")"):
path = entry[:paren_idx]
reason = entry[paren_idx + 2 : -1]
else:
path = entry
reason = "skip"
status = DirectoryParser._REASON_TO_STATUS.get(reason, "skip")
result.append({"path": path, "status": status, "reason": reason})
return result
@staticmethod
def _source_skipped_items(source_meta: Any, source_path: Path) -> List[Dict[str, str]]:
"""Normalize skipped items reported by a remote source accessor."""
if not isinstance(source_meta, dict):
return []
items = source_meta.get("feishu_folder_skipped_items") or []
if not isinstance(items, list):
return []
normalized: List[Dict[str, str]] = []
for item in items:
if not isinstance(item, dict):
continue
raw_path = str(item.get("path") or item.get("name") or item.get("token") or "unknown")
display_path = raw_path
try:
path = Path(raw_path).resolve(strict=False)
if path.is_absolute():
display_path = str(path.relative_to(source_path.resolve(strict=False)))
except Exception:
pass
display_path = display_path.replace("\\", "/")
normalized.append(
{
"path": display_path,
"parser": "feishu",
"status": "failed",
"type": str(item.get("type") or ""),
"token": str(item.get("token") or ""),
"reason": str(item.get("reason") or "unknown error"),
}
)
return normalized
@staticmethod
def _nested_failed_files(
classified_file: "ClassifiedFile",
parser_name: str,
detail: Dict[str, Any],
) -> List[Dict[str, Any]]:
"""Promote leaf failures from a parsed ZIP into the outer directory result."""
if parser_name != "ZipParser":
return []
meta = detail.get("meta")
if not isinstance(meta, dict) or not isinstance(meta.get("failed_files"), list):
return []
prefix = classified_file.rel_path.replace("\\", "/").rstrip("/")
promoted: List[Dict[str, Any]] = []
for item in meta["failed_files"]:
if not isinstance(item, dict):
continue
child = dict(item)
child_path = str(child.get("path") or "<unknown>").replace("\\", "/").lstrip("/")
child["path"] = f"{prefix}/{child_path}"
promoted.append(child)
return promoted
@staticmethod
def _file_status_entry(
classified_file: "ClassifiedFile",
parser_name: str,
detail: Dict[str, Any],
) -> Dict[str, Any]:
"""Build deterministic per-file status metadata."""
entry: Dict[str, Any] = {
"path": classified_file.rel_path,
"parser": parser_name,
}
meta = detail.get("meta")
if isinstance(meta, dict):
for key in (
"doc_name",
"doc_type",
"source_name",
"file_name",
"file_id",
"response_id",
):
if meta.get(key):
entry[key] = meta[key]
if detail.get("error"):
entry["error"] = str(detail["error"])
return entry
@staticmethod
def _get_parser_api_job_timeout(parser_api: Any, default: float = 1800.0) -> float:
"""Return a bounded end-to-end timeout for one Understanding file."""
try:
response_timeout = float(getattr(parser_api, "response_timeout_seconds", default))
http_timeout = float(getattr(parser_api, "http_timeout_seconds", 10.0))
except (TypeError, ValueError) as exc:
raise InvalidArgumentError(
"parser_api response/http timeouts must be positive numbers"
) from exc
if response_timeout <= 0 or http_timeout <= 0:
raise InvalidArgumentError("parser_api response/http timeouts must be positive numbers")
return response_timeout + max(60.0, http_timeout * 2.0)
# ------------------------------------------------------------------
# Parser assignment
# ------------------------------------------------------------------
@staticmethod
def _assign_parser(
classified_file: "ClassifiedFile",
registry: "ParserRegistry",
) -> Optional[BaseParser]:
"""Look up the parser for a file via the registry.
Returns:
The ``BaseParser`` instance for the file's extension, or
``None`` for text-fallback files with no dedicated parser.
"""
return registry.get_parser_for_file(classified_file.path)
# ------------------------------------------------------------------
# Per-file processing
# ------------------------------------------------------------------
@staticmethod
async def _parse_understanding_jobs(
jobs: List[Dict[str, Any]],
*,
preserve_structure: bool,
import_root: Optional[str],
split_content: bool,
max_concurrent: int,
job_timeout: float,
output_store: Any = None,
) -> Dict[int, Dict[str, Any]]:
"""Parse jobs with a fixed local pool and a shared service-loop limit."""
limiter = _get_understanding_limiter(max_concurrent)
queue: asyncio.Queue[Dict[str, Any]] = asyncio.Queue()
for job in jobs:
queue.put_nowait(job)
results: Dict[int, Dict[str, Any]] = {}
async def _worker() -> None:
while True:
try:
job = queue.get_nowait()
except asyncio.QueueEmpty:
return
try:
async with limiter:
parse_coro = DirectoryParser._parse_file_with_parser(
job["classified_file"],
job["file_parser"],
preserve_structure=preserve_structure,
import_root=import_root,
split_content=split_content,
parse_options=job.get("parse_options"),
output_store=output_store,
)
sub_result = await asyncio.wait_for(parse_coro, timeout=job_timeout)
results[job["index"]] = {"result": sub_result, "error": None}
except asyncio.TimeoutError:
results[job["index"]] = {
"result": None,
"error": TimeoutError(
f"Understanding job timed out after {job_timeout:.1f}s"
),
}
except Exception as exc:
results[job["index"]] = {"result": None, "error": exc}
finally:
queue.task_done()
worker_count = min(max_concurrent, len(jobs))
workers = [asyncio.create_task(_worker()) for _ in range(worker_count)]
try:
await asyncio.gather(*workers)
finally:
for worker in workers:
if not worker.done():
worker.cancel()
await asyncio.gather(*workers, return_exceptions=True)
return results
@staticmethod
async def _parse_file_with_parser(
classified_file: "ClassifiedFile",
parser: Union[BaseParser, "ParserRouter"],
*,
preserve_structure: bool,
import_root: Optional[str],
split_content: bool,
parse_options: Optional[Dict[str, Any]] = None,
output_store: Any = None,
) -> ParseResult:
"""Run one parser without mutating the directory destination tree."""
options = dict(parse_options or {})
if output_store is not None:
options["parse_output_store"] = output_store
source = options.pop("_source", str(classified_file.path))
options.update(
enable_link_rewrite=preserve_structure,
link_rewrite_root=import_root,
allowed_media_dirs=[Path(import_root)] if import_root else None,
split_content=split_content,
flatten_single_output=bool(not split_content and preserve_structure),
)
return await parser.parse(source, **options)
@staticmethod
async def _merge_parser_result(
classified_file: "ClassifiedFile",
sub_result: ParseResult,
target_uri: str,
viking_fs: Any,
*,
target_writer: Optional[ParseArtifactWriter] = None,
preserve_structure: bool,
split_content: bool,
) -> None:
"""Merge one completed parser result in source-file order."""
no_content_error = "Parse failed: no content generated"
if sub_result.warnings:
no_content_error += "; " + "; ".join(sub_result.warnings)
if not sub_result.temp_dir_path:
raise ValueError(no_content_error)
if preserve_structure:
parent = str(PurePosixPath(classified_file.rel_path).parent)
dest = (
_normalize_output_target(f"{target_uri}/{parent}", target_writer)
if parent != "."
else target_uri
)
else:
dest = target_uri
source_ref = getattr(sub_result, "artifact_ref", None)
if target_writer is not None and source_ref is not None:
source_store = (
target_writer.store
if source_ref.backend == target_writer.store.backend
else store_for_artifact_ref(source_ref, viking_fs=viking_fs)
)
target_rel = target_writer.relative_path(dest)
try:
merged = await copy_artifact_tree(
source_store=source_store,
source_ref=source_ref,
source_rel="",
target=target_writer,
target_rel=target_rel,
allowed_hidden=_MERGE_SIDECAR_ALLOWLIST,
flatten_single_output=bool(not split_content and preserve_structure),
)
finally:
await source_store.cleanup(source_ref)
if not merged:
raise ValueError(no_content_error)
return
try:
merged = await DirectoryParser._merge_temp(
viking_fs,
sub_result.temp_dir_path,
dest,
flatten_single_output=bool(not split_content and preserve_structure),
)
except BaseException:
try:
await viking_fs.delete_temp(sub_result.temp_dir_path)
except Exception as exc:
logger.warning(
"[DirectoryParser] Failed to clean temporary artifact %s: %s",
sub_result.temp_dir_path,
exc,
)
raise
if not merged:
raise ValueError(no_content_error)
@staticmethod
async def _process_single_file(
classified_file: "ClassifiedFile",
parser: Optional[Union[BaseParser, "ParserRouter"]],
target_uri: str,
viking_fs: Any,
warnings: List[str],
target_writer: Optional[ParseArtifactWriter] = None,
output_store: Any = None,
preserve_structure: bool = True,
import_root: Optional[str] = None,
split_content: bool = True,
) -> Dict[str, Any]:
"""Process one file into the VikingFS directory temp.
- Files WITH a parser → ``parser.parse()`` → merge output into
*target_uri* at the correct relative location.
- Files WITHOUT a parser → read and write directly to VikingFS.
Args:
preserve_structure: When True, files keep their relative directory
hierarchy. When False, all files are placed directly under
*target_uri* (flat).
Returns:
Per-file status detail including the parser error when processing fails.
"""
rel_path = classified_file.rel_path
src_file = classified_file.path
if parser:
sub_result: Optional[ParseResult] = None
try:
sub_result = await DirectoryParser._parse_file_with_parser(
classified_file,
parser,
preserve_structure=preserve_structure,
import_root=import_root,
split_content=split_content,
output_store=output_store,
)
await DirectoryParser._merge_parser_result(
classified_file,
sub_result,
target_uri,
viking_fs,
target_writer=target_writer,
preserve_structure=preserve_structure,
split_content=split_content,
)
return {
"ok": True,
"meta": getattr(sub_result, "meta", {}) or {},
"error": None,
}
except Exception as exc:
warnings.append(f"Failed to parse {rel_path}: {exc}")
meta = getattr(sub_result, "meta", {}) if sub_result is not None else {}
return {
"ok": False,
"meta": meta if isinstance(meta, dict) else {},
"error": str(exc),
}
else:
try:
content = detect_and_convert_encoding(src_file.read_bytes(), src_file)
if preserve_structure:
dst_uri = _normalize_output_target(
f"{target_uri}/{rel_path}", target_writer
)
else:
dst_uri = _normalize_output_target(
f"{target_uri}/{PurePosixPath(rel_path).name}",
target_writer,
)
if target_writer is not None:
await target_writer.write_bytes(dst_uri, content)
else:
await viking_fs.write_file(dst_uri, content)
return {"ok": True, "meta": {}, "error": None}
except Exception as exc:
warnings.append(f"Failed to upload {rel_path}: {exc}")
return {"ok": False, "meta": {}, "error": str(exc)}
@staticmethod
async def _upload_file_directly(
classified_file: "ClassifiedFile",
target_uri: str,
viking_fs: Any,
warnings: List[str],
target_writer: Optional[ParseArtifactWriter] = None,
preserve_structure: bool = True,
) -> Dict[str, Any]:
"""Directly upload a file without using its parser.
Used for media files when directly_upload_media=True.
Args:
preserve_structure: When True, files keep their relative directory
hierarchy. When False, all files are placed directly under
*target_uri* (flat).
Returns:
Per-file status detail including the upload error when processing fails.
"""
rel_path = classified_file.rel_path
src_file = classified_file.path
try:
content = detect_and_convert_encoding(src_file.read_bytes(), src_file)
if preserve_structure:
dst_uri = _normalize_output_target(f"{target_uri}/{rel_path}", target_writer)
else:
dst_uri = _normalize_output_target(
f"{target_uri}/{PurePosixPath(rel_path).name}",
target_writer,
)
if target_writer is not None:
await target_writer.write_bytes(dst_uri, content)
else:
await viking_fs.write_file(dst_uri, content)
return {"ok": True, "meta": {}, "error": None}
except Exception as exc:
warnings.append(f"Failed to upload {rel_path}: {exc}")
return {"ok": False, "meta": {}, "error": str(exc)}
# ------------------------------------------------------------------
# VikingFS merge helpers
# ------------------------------------------------------------------
@staticmethod
def _is_dir_entry(entry: Dict[str, Any]) -> bool:
"""Check whether an AGFS ``ls`` entry represents a directory."""
return bool(entry.get("isDir", False)) or entry.get("type") == "directory"
@staticmethod
async def _merge_temp(
viking_fs: Any,
src_temp_uri: str,
dest_uri: str,
*,
flatten_single_output: bool = False,
) -> bool:
"""Move all content from a parser's temp directory into *dest_uri*.
After the move the source temp is deleted. Hidden files stay filtered,
except the sidecars in :data:`_MERGE_SIDECAR_ALLOWLIST` that downstream
steps depend on (e.g. ``.image_mappings.json`` for the post-commit
image rewrite). In no-split directory imports, a wrapper containing one
standalone file is promoted into ``dest_uri``; wrappers with additional
files, directories, sidecars, or destination-name conflicts are retained.
Returns False for a tree with no visible files, without creating any
destination directories. Sidecars alone do not count as content.
"""
entries = await viking_fs.ls(src_temp_uri, show_all_hidden=True, node_limit=LS_ALL_NODES)
async def has_content(parent_uri: str, children: List[Dict[str, Any]]) -> bool:
for entry in children:
name = entry.get("name", "")
if not name or name in {".", ".."}:
continue
if DirectoryParser._is_dir_entry(entry):
child_uri = entry.get("uri", f"{parent_uri.rstrip('/')}/{name}")
child_entries = await viking_fs.ls(
child_uri, show_all_hidden=True, node_limit=LS_ALL_NODES
)
if await has_content(child_uri, child_entries):
return True
elif not name.startswith("."):
return True
return False
if not await has_content(src_temp_uri, entries):
try:
await viking_fs.delete_temp(src_temp_uri)
except Exception:
pass
return False
merge_entries = [
entry
for entry in entries
if entry.get("name") not in ("", ".", "..")
and (
DirectoryParser._is_dir_entry(entry)
or not entry.get("name", "").startswith(".")
or entry.get("name") in _MERGE_SIDECAR_ALLOWLIST
)
]
if flatten_single_output and len(merge_entries) == 1:
wrapper = merge_entries[0]
if DirectoryParser._is_dir_entry(wrapper):
wrapper_uri = wrapper.get(
"uri",
f"{src_temp_uri.rstrip('/')}/{wrapper['name']}",
)
wrapper_entries = await viking_fs.ls(
wrapper_uri,
show_all_hidden=True,
node_limit=LS_ALL_NODES,
)
payloads = [
entry
for entry in wrapper_entries
if entry.get("name") not in ("", ".", "..")
and (
not entry.get("name", "").startswith(".")
or entry.get("name") in _MERGE_SIDECAR_ALLOWLIST
)
]
if len(payloads) == 1 and not DirectoryParser._is_dir_entry(payloads[0]):
payload = payloads[0]
await viking_fs.mkdir(dest_uri, exist_ok=True)
destination_entries = await viking_fs.ls(
dest_uri,
show_all_hidden=True,
node_limit=LS_ALL_NODES,
)
destination_names = {
entry.get("name")
for entry in destination_entries
if entry.get("name") not in ("", ".", "..")
}
destination_uri = normalize_storage_target_uri(
f"{dest_uri.rstrip('/')}/{payload['name']}"
)
destination_name = destination_uri.rsplit("/", 1)[-1]
if destination_name not in destination_names:
src = payload.get(
"uri",
f"{wrapper_uri.rstrip('/')}/{payload['name']}",
)
await viking_fs.move_file(src, destination_uri)
try:
await viking_fs.delete_temp(src_temp_uri)
except Exception:
pass
return True
for entry in entries:
name = entry.get("name", "")
if not name or name in (".", ".."):
continue
if (
not DirectoryParser._is_dir_entry(entry)
and name.startswith(".")
and name not in _MERGE_SIDECAR_ALLOWLIST
):
continue
src = entry.get("uri", f"{src_temp_uri.rstrip('/')}/{name}")
dst = normalize_storage_target_uri(f"{dest_uri.rstrip('/')}/{name}")
if DirectoryParser._is_dir_entry(entry):
await DirectoryParser._recursive_move(viking_fs, src, dst)
else:
await viking_fs.move_file(src, dst)
try:
await viking_fs.delete_temp(src_temp_uri)
except Exception:
pass
return True
@staticmethod
async def _is_git_repository(source_path: Path) -> bool:
"""Check if the directory contains a git repository (or has our .git_source_repo marker)."""
try:
git_dir = source_path / ".git"
marker_file = source_path / ".git_source_repo"
return (git_dir.exists() and git_dir.is_dir()) or marker_file.exists()
except (OSError, PermissionError):
return False
@staticmethod
async def _add_git_metadata(source_path: Path, kwargs: dict) -> None:
"""Add git metadata (branch, commit) from .git directory if available."""
try:
from openviking.parse.accessors.git_accessor import GitAccessor
git_dir = source_path / ".git"
if not git_dir.exists():
return # No .git directory, skip (we already have meta from accessor)
git_accessor = GitAccessor()
# Get branch
try:
branch = await git_accessor._run_git(
["git", "-C", str(source_path), "rev-parse", "--abbrev-ref", "HEAD"]
)
kwargs["repo_ref"] = branch
except Exception as e:
logger.debug(f"Failed to get git branch: {e}")
# Get commit
try:
commit = await git_accessor._run_git(
["git", "-C", str(source_path), "rev-parse", "HEAD"]
)
kwargs["repo_commit"] = commit
except Exception as e:
logger.debug(f"Failed to get git commit: {e}")
# repo_name and original_source are already set from accessor, no need to get from git
except Exception as e:
logger.debug(f"Failed to get git metadata: {e}")
@staticmethod
async def _recursive_move(
viking_fs: Any,
src_uri: str,
dst_uri: str,
) -> None:
"""Recursively move a VikingFS directory tree (hidden files filtered,
allowlisted sidecars carried)."""
await viking_fs.mkdir(dst_uri, exist_ok=True)
entries = await viking_fs.ls(src_uri, show_all_hidden=True, node_limit=LS_ALL_NODES)
for entry in entries:
name = entry.get("name", "")
if not name or name in (".", ".."):
continue
if (
not DirectoryParser._is_dir_entry(entry)
and name.startswith(".")
and name not in _MERGE_SIDECAR_ALLOWLIST
):
continue
s = f"{src_uri.rstrip('/')}/{name}"
d = normalize_storage_target_uri(f"{dst_uri.rstrip('/')}/{name}")
if DirectoryParser._is_dir_entry(entry):
await DirectoryParser._recursive_move(viking_fs, s, d)
else:
await viking_fs.move_file(s, d)