Skip to content

Self-evolving Context Database for AI Agents. Unify Agent Memory, Knowledge RAG and Skills.

13.9 KB
Raw Blame
# Copyright (c) 2026 Beijing Volcano Engine Technology Co., Ltd.
# SPDX-License-Identifier: AGPL-3.0
"""
Directory pre-scan validation module for OpenViking.

Implements phase-one of directory import (RFC #83): traverse directory tree,
classify files as processable / unsupported, validate format,
and report errors or warnings with optional strict mode.
"""

import fnmatch
import os
from dataclasses import dataclass, field
from pathlib import Path
from typing import Callable, List, Optional, Set, Union

from openviking.parse.gitignore import GitignoreMatcher
from openviking.parse.parsers.constants import IGNORE_DIRS
from openviking.parse.parsers.upload_utils import is_empty_file, is_text_file
from openviking.parse.registry import ParserRegistry, get_registry
from openviking_cli.exceptions import InvalidArgumentError, UnsupportedDirectoryFilesError
from openviking_cli.utils.logger import get_logger

logger = get_logger(__name__)

# File classification labels
CLASS_PROCESSABLE = "processable"
CLASS_UNSUPPORTED = "unsupported"


@dataclass
class ClassifiedFile:
    """A single file with its classification and relative path."""

    path: Path
    rel_path: str
    classification: str  # CLASS_PROCESSABLE | CLASS_UNSUPPORTED


@dataclass
class DirectoryScanResult:
    """Result of directory pre-scan: classified files and optional warnings."""

    root: Path
    processable: List[ClassifiedFile] = field(default_factory=list)
    unsupported: List[ClassifiedFile] = field(default_factory=list)
    skipped: List[str] = field(default_factory=list)  # reason -> count or paths for debugging
    warnings: List[str] = field(default_factory=list)

    def all_processable_files(self) -> List[ClassifiedFile]:
        """Return processable files in order (for phase-two routing)."""
        return self.processable


def _should_skip_file(file_path: Path) -> tuple[bool, str]:
    """
    Return (True, reason) if the file should be skipped (not counted as supported/unsupported).

    Skip: dot files, symlinks, empty or whitespace-only files.
    """
    if file_path.name.startswith("."):
        return True, "dot file"
    if file_path.is_symlink():
        return True, "symlink"
    try:
        if is_empty_file(file_path):
            return True, "empty file"
    except OSError:
        return True, "os error"
    return False, ""


def _should_skip_directory(
    dir_path: Path,
    root: Path,
    ignore_dirs: Optional[Set[str]] = None,
) -> tuple[bool, str]:
    """
    Return (True, reason) if the directory should be skipped (not counted as supported/unsupported).

    Skip: dot directories, symlinks, IGNORE_DIRS, and any dir in ignore_dirs.
    ignore_dirs: directory names, or relative paths (relative to root).
                 - Name only (no path sep): skip any dir with that name.
                 - Relative path (e.g. "parse/", "./storage/vectordb", "openviking/parse"): skip when dir's path matches.
    """
    if dir_path.name.startswith("."):
        return True, "dot directory"
    if dir_path.is_symlink():
        return True, "symlink"
    if dir_path.name in IGNORE_DIRS:
        return True, "IGNORE_DIRS"
    if not ignore_dirs:
        return False, ""

    try:
        dir_rel = _normalize_rel_path(str(dir_path.relative_to(root)))
    except ValueError:
        dir_rel = _normalize_rel_path(str(dir_path))

    for entry in ignore_dirs:
        if not entry or not str(entry).strip():
            continue
        raw = str(entry).strip().replace("\\", "/")
        if not raw:
            continue

        # Relative path (contains '/'): match by path relative to root
        if "/" in raw:
            prefix = raw.rstrip("/").lstrip("./")
            if prefix and (dir_rel == prefix or dir_rel.startswith(prefix + "/")):
                return True, "ignore_dirs"
            continue

        # Single segment: match by directory name
        if dir_path.name == raw:
            return True, "ignore_dirs"

    return False, ""


def _parse_patterns(value: Optional[str]) -> List[str]:
    """Parse comma-separated include/exclude string into list of stripped patterns."""
    if not value or not value.strip():
        return []
    return [p.strip() for p in value.split(",") if p.strip()]


def _normalize_rel_path(rel_path: str) -> str:
    """Use forward slashes for consistent matching across platforms."""
    return rel_path.replace("\\", "/")


def _matches_include(path_name: str, patterns: List[str]) -> bool:
    """True if file is included: no patterns means include all; else match path name against any pattern."""
    if not patterns:
        return True
    return any(fnmatch.fnmatch(path_name, p) for p in patterns)


def _matches_exclude(rel_path_norm: str, path_name: str, patterns: List[str]) -> bool:
    """
    True if file is excluded.
    - Pattern ending with '/' is a path prefix (e.g. 'drafts/' excludes paths under drafts/).
    - Otherwise match path name as glob (e.g. '*.tmp').
    """
    if not patterns:
        return False
    for p in patterns:
        if p.endswith("/"):
            prefix = p.rstrip("/").replace("\\", "/")
            if rel_path_norm == prefix or rel_path_norm.startswith(prefix + "/"):
                return True
        else:
            if fnmatch.fnmatch(path_name, p):
                return True
    return False


def _classify_file(
    file_path: Path,
    registry: ParserRegistry,
    additional_can_process: Optional[Callable[[Path], bool]] = None,
) -> str:
    """
    Classify a single file as CLASS_PROCESSABLE or CLASS_UNSUPPORTED.

    Processable: ParserRegistry has a parser, an additional parser backend can
    handle the file, or is_text_file (code/config/docs).
    """
    # Normal classification logic
    if registry.get_parser_for_file(file_path) is not None:
        return CLASS_PROCESSABLE
    if additional_can_process and additional_can_process(file_path):
        return CLASS_PROCESSABLE
    if is_text_file(file_path):
        return CLASS_PROCESSABLE
    return CLASS_UNSUPPORTED


def scan_directory(
    root: Union[str, Path],
    registry: Optional[ParserRegistry] = None,
    strict: bool = False,
    ignore_dirs: Optional[Set[str]] = None,
    include: Optional[str] = None,
    exclude: Optional[str] = None,
    additional_can_process: Optional[Callable[[Path], bool]] = None,
    max_files: Optional[int] = None,
    max_depth: Optional[int] = None,
) -> DirectoryScanResult:
    """
    Traverse directory tree and classify every file (phase-one validation).

    - Skips directories in IGNORE_DIRS (or ignore_dirs), gitignored paths,
      and skips dot files, symlinks, and empty files (they are not included in any list).
    - If include is set, only files whose name matches one of the glob patterns are considered
      (e.g. include="*.pdf,*.md"). If exclude is set, files matching any exclude pattern are
      skipped (e.g. exclude="drafts/" for path prefix, or "*.tmp" for name glob).
    - Classifies remaining files:
      - processable: ParserRegistry has a parser, an additional parser backend
        accepts it, or is_text_file (code/docs/config)
      - unsupported: everything else

    Args:
        root: Directory path to scan.
        registry: Parser registry for processable detection. Defaults to get_registry().
        strict: If True, raise UnsupportedDirectoryFilesError when any unsupported file exists.
                If False, append warnings and continue (unsupported list still populated).
        ignore_dirs: Directory names or relative paths (to root) to skip. E.g. "parse", "parse/", "./storage/".
        include: Comma-separated glob patterns for file names; only matching files are included
                 (e.g. "*.pdf,*.md"). If not set, all files (subject to exclude) are considered.
        exclude: Comma-separated patterns: trailing '/' = path prefix (e.g. "drafts/"),
                 else glob on file name (e.g. "*.tmp").
        additional_can_process: Optional capability check for parser backends
            outside ParserRegistry.
        max_files: Optional maximum number of selected files admitted by the scan.
        max_depth: Optional maximum nested directory depth below the root.

    Returns:
        DirectoryScanResult with processable, unsupported, warnings.

    Raises:
        InvalidArgumentError: When a configured directory safety limit is exceeded.
        UnsupportedDirectoryFilesError: When strict=True and there is at least one unsupported file.
        FileNotFoundError: When root does not exist.
        NotADirectoryError: When root is not a directory.
    """
    root = Path(root).resolve()
    if not root.exists():
        raise FileNotFoundError(f"Directory does not exist: {root}")
    if not root.is_dir():
        raise NotADirectoryError(f"Not a directory: {root}")

    limits = {
        "max_files": max_files,
        "max_depth": max_depth,
    }
    for name, value in limits.items():
        if value is None:
            continue
        if isinstance(value, bool) or not isinstance(value, int) or value <= 0:
            raise InvalidArgumentError(f"directory.{name} must be a positive integer")
    effective_registry = registry if registry is not None else get_registry()
    include_patterns = _parse_patterns(include)
    exclude_patterns = _parse_patterns(exclude)
    gitignore_matcher = GitignoreMatcher(root)

    # Normalize ignore_dirs:
    # - If caller passed a comma-separated string (common from CLI/HTTP),
    #   split it into a set of entries.
    # - If already a set/list-like, keep as is.
    ignore_dirs_set: Optional[Set[str]]
    if isinstance(ignore_dirs, str):
        entries = _parse_patterns(ignore_dirs)
        ignore_dirs_set = set(entries) if entries else None
    else:
        ignore_dirs_set = ignore_dirs

    result = DirectoryScanResult(root=root)
    selected_file_count = 0
    for dir_path_str, dir_names, file_names in os.walk(root, topdown=True):
        dir_path = Path(dir_path_str)
        try:
            relative_dir_path = dir_path.relative_to(root)
            directory_depth = len(relative_dir_path.parts)
        except ValueError:
            relative_dir_path = Path("<outside-root>")
            directory_depth = max_depth + 1 if max_depth is not None else 0
        if max_depth is not None and directory_depth > max_depth:
            display_path = _normalize_rel_path(str(relative_dir_path))
            raise InvalidArgumentError(
                f"Directory depth exceeds limit: depth={directory_depth}, "
                f"max_depth={max_depth}, path={display_path}"
            )
        dir_spec = gitignore_matcher.spec_for_dir(dir_path)

        # Prune subdirectories in-place so os.walk won't descend into them
        kept = []
        for d in dir_names:
            sub = dir_path / d
            skip, reason = _should_skip_directory(sub, root, ignore_dirs_set)
            if skip:
                skipped_path = str(sub.relative_to(root))
                skipped_path = _normalize_rel_path(skipped_path)
                result.skipped.append(f"{skipped_path} ({reason})")
                continue

            if gitignore_matcher.is_ignored_dir(sub, dir_spec):
                skipped_path = str(sub.relative_to(root))
                skipped_path = _normalize_rel_path(skipped_path)
                result.skipped.append(f"{skipped_path} (gitignore)")
                continue

            kept.append(d)

        dir_names[:] = kept

        for name in file_names:
            file_path = dir_path / name
            try:
                rel_path = str(file_path.relative_to(root))
            except ValueError:
                rel_path = str(file_path)
            rel_path_norm = _normalize_rel_path(rel_path)

            skip, reason = _should_skip_file(file_path)
            if skip:
                result.skipped.append(f"{rel_path} ({reason})")
                continue
            if gitignore_matcher.is_ignored_file(file_path, dir_spec):
                result.skipped.append(f"{rel_path} (gitignore)")
                continue

            if include_patterns and not _matches_include(name, include_patterns):
                result.skipped.append(f"{rel_path} (excluded by include filter)")
                continue
            if exclude_patterns and _matches_exclude(rel_path_norm, name, exclude_patterns):
                result.skipped.append(f"{rel_path} (excluded by exclude filter)")
                continue

            accepted_by_additional_parser = bool(
                additional_can_process and additional_can_process(file_path)
            )
            if max_files is not None:
                selected_file_count += 1
            if max_files is not None and selected_file_count > max_files:
                raise InvalidArgumentError(
                    f"Directory file count exceeds limit: count={selected_file_count}, "
                    f"max_files={max_files}, path={rel_path_norm}"
                )
            classification = (
                CLASS_PROCESSABLE
                if accepted_by_additional_parser
                else _classify_file(file_path, effective_registry)
            )
            classified = ClassifiedFile(
                path=file_path, rel_path=rel_path_norm, classification=classification
            )
            if classification == CLASS_PROCESSABLE:
                result.processable.append(classified)
            else:
                result.unsupported.append(classified)

    if result.unsupported:
        unsupported_paths = [f.rel_path for f in result.unsupported]
        msg = (
            f"Directory contains {len(result.unsupported)} unsupported file(s). "
            f"Unsupported: {unsupported_paths[:10]}{'...' if len(unsupported_paths) > 10 else ''}"
        )
        if strict:
            logger.error(msg)
            raise UnsupportedDirectoryFilesError(msg, unsupported_paths)
        else:
            logger.warning(msg)
            result.warnings.append(msg)

    result.processable.sort(key=lambda x: x.rel_path)
    result.unsupported.sort(key=lambda x: x.rel_path)
    result.skipped.sort()
    result.warnings.sort()
    return result