Skip to content

Self-evolving Context Database for AI Agents. Unify Agent Memory, Knowledge RAG and Skills.

23.6 KB
Raw Blame
# Copyright (c) 2026 Beijing Volcano Engine Technology Co., Ltd.
# SPDX-License-Identifier: AGPL-3.0
"""Unit tests for directory pre-scan validation module (RFC #83 T1)."""

from pathlib import Path

import pytest

from openviking.parse.directory_scan import (
    CLASS_PROCESSABLE,
    ClassifiedFile,
    DirectoryScanResult,
    scan_directory,
)
from openviking.parse.registry import ParserRegistry
from openviking_cli.exceptions import InvalidArgumentError, UnsupportedDirectoryFilesError

# ---------------------------------------------------------------------------
# Fixtures
# ---------------------------------------------------------------------------


@pytest.fixture
def tmp_tree(tmp_path: Path) -> Path:
    """Create a directory tree with mixed file types for scan tests."""
    # rich (parser exists): .md, .pdf, .html, .txt, .docx, .xlsx, .epub, .pptx, .zip
    (tmp_path / "readme.md").write_text("# README", encoding="utf-8")
    (tmp_path / "doc.html").write_text("<html></html>", encoding="utf-8")
    (tmp_path / "note.txt").write_text("plain text", encoding="utf-8")
    (tmp_path / "report.docx").write_bytes(b"PK\x03\x04")
    (tmp_path / "data.xlsx").write_bytes(b"PK\x03\x04")
    (tmp_path / "book.epub").write_bytes(b"PK\x03\x04")
    (tmp_path / "slides.pptx").write_bytes(b"PK\x03\x04")
    (tmp_path / "bundle.zip").write_bytes(b"PK\x03\x04")

    # text (code/config, no dedicated parser or text parser only): .py, .yaml
    (tmp_path / "main.py").write_text("print(1)", encoding="utf-8")
    (tmp_path / "engine.cc").write_text("int main() { return 0; }", encoding="utf-8")
    (tmp_path / "inline.inl").write_text("inline impl", encoding="utf-8")
    (tmp_path / "config.yaml").write_text("key: value", encoding="utf-8")

    # unsupported: unknown extension
    (tmp_path / "data.xyz").write_text("unknown", encoding="utf-8")
    (tmp_path / "archive.rar").write_bytes(b"RAR\x00")

    # skipped: dot file, empty, ignored dir
    (tmp_path / ".hidden").write_text("secret", encoding="utf-8")
    (tmp_path / "empty.txt").write_bytes(b"")
    (tmp_path / ".git").mkdir()
    (tmp_path / ".git" / "config").write_text("[core]", encoding="utf-8")
    (tmp_path / "node_modules").mkdir()
    (tmp_path / "node_modules" / "pkg").write_text("x", encoding="utf-8")

    # subdir with mixed
    sub = tmp_path / "src"
    sub.mkdir()
    (sub / "app.py").write_text("code", encoding="utf-8")
    (sub / "custom.bin").write_bytes(b"\x00\x01")

    return tmp_path


@pytest.fixture
def tmp_all_supported(tmp_path: Path) -> Path:
    """Directory where every file is rich or text (no unsupported)."""
    (tmp_path / "a.md").write_text("# A", encoding="utf-8")
    (tmp_path / "b.py").write_text("pass", encoding="utf-8")
    (tmp_path / "c.yaml").write_text("k: v", encoding="utf-8")
    return tmp_path


@pytest.fixture
def tmp_with_drafts(tmp_path: Path) -> Path:
    """Tree with drafts/ subdir and mixed extensions for include/exclude tests."""
    (tmp_path / "readme.md").write_text("# README", encoding="utf-8")
    (tmp_path / "doc.pdf").write_bytes(b"%PDF-1.0")
    (tmp_path / "main.py").write_text("pass", encoding="utf-8")
    (tmp_path / "drafts").mkdir()
    (tmp_path / "drafts" / "draft.pdf").write_bytes(b"%PDF")
    (tmp_path / "drafts" / "notes.md").write_text("notes", encoding="utf-8")
    (tmp_path / "drafts" / "skip.py").write_text("x", encoding="utf-8")
    return tmp_path


@pytest.fixture
def registry() -> ParserRegistry:
    """Default parser registry (includes markdown, pdf, html, text, etc.)."""
    return ParserRegistry()


# ---------------------------------------------------------------------------
# Traversal and classification
# ---------------------------------------------------------------------------


class TestScanDirectoryTraversal:
    """Test that scan_directory traverses the tree and respects IGNORE_DIRS."""

    def test_traverses_all_non_ignored_dirs(self, tmp_tree: Path, registry: ParserRegistry) -> None:
        result: DirectoryScanResult = scan_directory(tmp_tree, registry=registry, strict=False)
        rel_paths = {f.rel_path for f in result.all_processable_files()}
        assert "readme.md" in rel_paths
        assert "main.py" in rel_paths
        assert "src/app.py" in rel_paths
        # Ignored dirs must not appear
        assert not any(".git" in p for p in rel_paths)
        assert not any("node_modules" in p for p in rel_paths)

    def test_skips_dot_files_and_empty(self, tmp_tree: Path, registry: ParserRegistry) -> None:
        (tmp_tree / "application.properties").write_bytes(b"\n")
        (tmp_tree / "blank.txt").write_bytes(b" \t\r\n")
        result: DirectoryScanResult = scan_directory(tmp_tree, registry=registry, strict=False)
        all_rel = [f.rel_path for f in result.processable + result.unsupported]
        assert ".hidden" not in all_rel
        assert "empty.txt" not in all_rel
        assert "application.properties" not in all_rel
        assert "blank.txt" not in all_rel
        assert any("empty" in s or "dot" in s for s in result.skipped)


class TestScanDirectoryClassification:
    """Test processable / unsupported classification."""

    def test_processable_includes_extensionless_readme(
        self, tmp_path: Path, registry: ParserRegistry
    ) -> None:
        (tmp_path / "README").write_text("# Project", encoding="utf-8")

        result = scan_directory(tmp_path, registry=registry, strict=True)

        assert [file.rel_path for file in result.processable] == ["README"]
        assert result.unsupported == []

    def test_processable_includes_parser_files(
        self, tmp_tree: Path, registry: ParserRegistry
    ) -> None:
        result: DirectoryScanResult = scan_directory(tmp_tree, registry=registry, strict=False)
        processable_rel = [f.rel_path for f in result.processable]
        assert "readme.md" in processable_rel
        assert "doc.html" in processable_rel
        assert "note.txt" in processable_rel
        # AnyDoc Office/EPUB and Zip parsers
        assert "report.docx" in processable_rel
        assert "data.xlsx" in processable_rel
        assert "book.epub" in processable_rel
        assert "slides.pptx" in processable_rel
        assert "bundle.zip" in processable_rel

    def test_processable_includes_code_or_config(
        self, tmp_tree: Path, registry: ParserRegistry
    ) -> None:
        result: DirectoryScanResult = scan_directory(tmp_tree, registry=registry, strict=False)
        processable_rel = [f.rel_path for f in result.processable]
        assert "main.py" in processable_rel
        assert "engine.cc" in processable_rel
        assert "inline.inl" in processable_rel
        assert "config.yaml" in processable_rel
        assert "src/app.py" in processable_rel

    def test_processable_includes_skeleton_supported_code_extensions(
        self, tmp_path: Path, registry: ParserRegistry
    ) -> None:
        (tmp_path / "infra.tf").write_text('resource "local_file" "demo" {}\n', encoding="utf-8")
        (tmp_path / "module.rkt").write_text(
            "#lang racket\n(define (run x) (+ x 1))\n",
            encoding="utf-8",
        )
        (tmp_path / "request.http").write_text("GET https://example.com\n", encoding="utf-8")
        (tmp_path / "diagram.mmd").write_text("graph TD\n  A-->B\n", encoding="utf-8")

        result: DirectoryScanResult = scan_directory(tmp_path, registry=registry, strict=False)

        processable_rel = {f.rel_path for f in result.processable}
        unsupported_rel = {f.rel_path for f in result.unsupported}
        assert "infra.tf" in processable_rel
        assert "module.rkt" in processable_rel
        assert "request.http" in unsupported_rel
        assert "diagram.mmd" in unsupported_rel

    def test_unsupported_unknown_ext(self, tmp_tree: Path, registry: ParserRegistry) -> None:
        result: DirectoryScanResult = scan_directory(tmp_tree, registry=registry, strict=False)
        unsup_rel = [f.rel_path for f in result.unsupported]
        assert "data.xyz" in unsup_rel
        assert "archive.rar" in unsup_rel
        assert "src/custom.bin" in unsup_rel


# ---------------------------------------------------------------------------
# Strict vs non-strict (unsupported handling)
# ---------------------------------------------------------------------------


class TestStrictParameter:
    """Test strict=True raises; strict=False adds warnings."""

    def test_strict_raises_when_unsupported(self, tmp_tree: Path, registry: ParserRegistry) -> None:
        with pytest.raises(UnsupportedDirectoryFilesError) as exc_info:
            scan_directory(tmp_tree, registry=registry, strict=True)
        err = exc_info.value
        assert err.unsupported_files
        assert "data.xyz" in err.unsupported_files or any("xyz" in p for p in err.unsupported_files)

    def test_non_strict_returns_warnings(self, tmp_tree: Path, registry: ParserRegistry) -> None:
        result: DirectoryScanResult = scan_directory(tmp_tree, registry=registry, strict=False)
        assert result.unsupported
        assert result.warnings
        assert any("unsupported" in w.lower() for w in result.warnings)

    def test_strict_passes_when_no_unsupported(
        self, tmp_all_supported: Path, registry: ParserRegistry
    ) -> None:
        result: DirectoryScanResult = scan_directory(
            tmp_all_supported, registry=registry, strict=True
        )
        assert not result.unsupported
        assert not result.warnings


# ---------------------------------------------------------------------------
# Exception reporting
# ---------------------------------------------------------------------------


class TestExceptionReporting:
    """Test that UnsupportedDirectoryFilesError carries full unsupported list."""

    def test_error_contains_all_unsupported_paths(
        self, tmp_tree: Path, registry: ParserRegistry
    ) -> None:
        with pytest.raises(UnsupportedDirectoryFilesError) as exc_info:
            scan_directory(tmp_tree, registry=registry, strict=True)
        paths = exc_info.value.unsupported_files
        assert len(paths) >= 3  # data.xyz, archive.rar, src/custom.bin
        assert "data.xyz" in paths
        assert "archive.rar" in paths
        assert "src/custom.bin" in paths


# ---------------------------------------------------------------------------
# Edge cases and API
# ---------------------------------------------------------------------------


class TestScanDirectoryEdgeCases:
    """Edge cases: missing dir, not a dir, custom ignore_dirs."""

    def test_raises_on_nonexistent(self, registry: ParserRegistry) -> None:
        with pytest.raises(FileNotFoundError):
            scan_directory("/nonexistent/path/12345", registry=registry)

    def test_raises_on_file_not_dir(self, tmp_tree: Path, registry: ParserRegistry) -> None:
        with pytest.raises(NotADirectoryError):
            scan_directory(tmp_tree / "readme.md", registry=registry)

    def test_custom_ignore_dirs(self, tmp_tree: Path, registry: ParserRegistry) -> None:
        """Custom ignore_dirs supports directory names and relative paths."""

        # 1) Ignore by directory name ("src")
        result_name: DirectoryScanResult = scan_directory(
            tmp_tree,
            registry=registry,
            strict=False,
            ignore_dirs={".git", "node_modules", "src"},
        )
        all_rel_name = [f.rel_path for f in result_name.processable + result_name.unsupported]
        assert not any(p.startswith("src/") for p in all_rel_name)

        # 2) Ignore by relative path with trailing slash ("src/")
        result_rel_slash: DirectoryScanResult = scan_directory(
            tmp_tree,
            registry=registry,
            strict=False,
            ignore_dirs=["src/"],
        )
        all_rel_slash = [
            f.rel_path for f in result_rel_slash.processable + result_rel_slash.unsupported
        ]
        assert not any(p.startswith("src/") for p in all_rel_slash)

        # 3) Ignore by relative path with ./ prefix ("./src")
        result_rel_dot: DirectoryScanResult = scan_directory(
            tmp_tree,
            registry=registry,
            strict=False,
            ignore_dirs=["./src"],
        )
        all_rel_dot = [f.rel_path for f in result_rel_dot.processable + result_rel_dot.unsupported]
        assert not any(p.startswith("src/") for p in all_rel_dot)

    def test_result_all_processable(
        self, tmp_all_supported: Path, registry: ParserRegistry
    ) -> None:
        result: DirectoryScanResult = scan_directory(
            tmp_all_supported, registry=registry, strict=True
        )
        all_p = result.all_processable_files()
        assert len(all_p) == len(result.processable)
        for cf in all_p:
            assert cf.classification == CLASS_PROCESSABLE


class TestDirectorySafetyLimits:
    """Directory scans must fail before processing unbounded input trees."""

    def test_rejects_too_many_files(self, tmp_path: Path, registry: ParserRegistry) -> None:
        for index in range(3):
            (tmp_path / f"{index}.txt").write_text("x", encoding="utf-8")

        with pytest.raises(InvalidArgumentError, match="file count exceeds"):
            scan_directory(tmp_path, registry=registry, max_files=2)

    def test_additional_parser_marks_file_processable(
        self, tmp_path: Path, registry: ParserRegistry
    ) -> None:
        (tmp_path / "large.external").write_bytes(b"12345")

        result = scan_directory(
            tmp_path,
            registry=registry,
            strict=True,
            additional_can_process=lambda path: path.suffix == ".external",
        )

        assert [item.rel_path for item in result.processable] == ["large.external"]

    def test_rejects_excessive_depth(self, tmp_path: Path, registry: ParserRegistry) -> None:
        nested = tmp_path / "level-1" / "level-2"
        nested.mkdir(parents=True)
        (nested / "deep.txt").write_text("deep", encoding="utf-8")

        with pytest.raises(InvalidArgumentError, match="depth exceeds") as exc_info:
            scan_directory(tmp_path, registry=registry, max_depth=1)

        assert "path=level-1/level-2" in str(exc_info.value)
        assert str(tmp_path) not in str(exc_info.value)

    @pytest.mark.parametrize(
        "limit_name",
        [
            "max_files",
            "max_depth",
        ],
    )
    def test_rejects_non_positive_limit(
        self,
        tmp_path: Path,
        registry: ParserRegistry,
        limit_name: str,
    ) -> None:
        with pytest.raises(InvalidArgumentError, match=limit_name):
            scan_directory(tmp_path, registry=registry, **{limit_name: 0})


# ---------------------------------------------------------------------------
# Include / exclude filters
# ---------------------------------------------------------------------------


class TestIncludeExclude:
    """Test include and exclude parameters for user-defined file filtering."""

    def test_include_only_matching_files(
        self, tmp_with_drafts: Path, registry: ParserRegistry
    ) -> None:
        result: DirectoryScanResult = scan_directory(
            tmp_with_drafts,
            registry=registry,
            strict=False,
            include="*.pdf,*.md",
        )
        rel_paths = [f.rel_path for f in result.processable + result.unsupported]
        assert "readme.md" in rel_paths
        assert "doc.pdf" in rel_paths
        assert "drafts/draft.pdf" in rel_paths
        assert "drafts/notes.md" in rel_paths
        assert "main.py" not in rel_paths
        assert "drafts/skip.py" not in rel_paths
        skipped_reasons = " ".join(result.skipped)
        assert "excluded by include" in skipped_reasons

    def test_exclude_path_prefix(self, tmp_with_drafts: Path, registry: ParserRegistry) -> None:
        result: DirectoryScanResult = scan_directory(
            tmp_with_drafts,
            registry=registry,
            strict=False,
            exclude="drafts/",
        )
        rel_paths = [f.rel_path for f in result.processable + result.unsupported]
        assert "readme.md" in rel_paths
        assert "doc.pdf" in rel_paths
        assert "main.py" in rel_paths
        assert "drafts/draft.pdf" not in rel_paths
        assert "drafts/notes.md" not in rel_paths
        assert "drafts/skip.py" not in rel_paths
        skipped_reasons = " ".join(result.skipped)
        assert "excluded by exclude" in skipped_reasons

    def test_include_and_exclude_combined(
        self, tmp_with_drafts: Path, registry: ParserRegistry
    ) -> None:
        result: DirectoryScanResult = scan_directory(
            tmp_with_drafts,
            registry=registry,
            strict=False,
            include="*.pdf,*.md",
            exclude="drafts/",
        )
        rel_paths = [f.rel_path for f in result.processable + result.unsupported]
        assert "readme.md" in rel_paths
        assert "doc.pdf" in rel_paths
        assert "drafts/draft.pdf" not in rel_paths
        assert "drafts/notes.md" not in rel_paths
        assert "main.py" not in rel_paths

    def test_exclude_name_glob(self, tmp_tree: Path, registry: ParserRegistry) -> None:
        result: DirectoryScanResult = scan_directory(
            tmp_tree,
            registry=registry,
            strict=False,
            exclude="*.rar,*.xyz",
        )
        unsup_rel = [f.rel_path for f in result.unsupported]
        assert "data.xyz" not in unsup_rel
        assert "archive.rar" not in unsup_rel
        assert "src/custom.bin" in unsup_rel

    def test_no_include_means_all_files(
        self, tmp_with_drafts: Path, registry: ParserRegistry
    ) -> None:
        result: DirectoryScanResult = scan_directory(
            tmp_with_drafts, registry=registry, strict=False
        )
        rel_paths = [f.rel_path for f in result.processable + result.unsupported]
        assert "readme.md" in rel_paths
        assert "doc.pdf" in rel_paths
        assert "main.py" in rel_paths
        assert "drafts/draft.pdf" in rel_paths
        assert "drafts/notes.md" in rel_paths
        assert "drafts/skip.py" in rel_paths

    def test_empty_include_exclude_unchanged(
        self, tmp_tree: Path, registry: ParserRegistry
    ) -> None:
        r1 = scan_directory(tmp_tree, registry=registry, strict=False)
        r2 = scan_directory(tmp_tree, registry=registry, strict=False, include="", exclude="")
        paths1 = {f.rel_path for f in r1.processable + r1.unsupported}
        paths2 = {f.rel_path for f in r2.processable + r2.unsupported}
        assert paths1 == paths2

    def test_ignore_dirs_with_include_and_exclude(
        self, tmp_with_drafts: Path, registry: ParserRegistry
    ) -> None:
        """Combined ignore_dirs + include + exclude should work together."""

        result: DirectoryScanResult = scan_directory(
            tmp_with_drafts,
            registry=registry,
            strict=False,
            ignore_dirs={"drafts"},
            include="*.md,*.py",
            exclude="main.py",
        )

        rel_paths = [f.rel_path for f in result.processable + result.unsupported]

        # ignore_dirs: drafts/ 整个目录被跳过
        assert not any(p.startswith("drafts/") for p in rel_paths)
        # include: .md 仍然被保留
        assert "readme.md" in rel_paths
        # exclude: main.py 被排除
        assert "main.py" not in rel_paths

        skipped_reasons = " ".join(result.skipped)
        assert "ignore_dirs" in skipped_reasons
        assert "excluded by include" in skipped_reasons or "excluded by exclude" in skipped_reasons


class TestGitignoreHandling:
    """Test gitignore-aware exclusion in scan_directory."""

    def test_respects_root_gitignore(self, tmp_path: Path, registry: ParserRegistry) -> None:
        (tmp_path / ".gitignore").write_text("*.log\n", encoding="utf-8")
        (tmp_path / "keep.txt").write_text("ok", encoding="utf-8")
        (tmp_path / "skip.log").write_text("no", encoding="utf-8")

        result = scan_directory(tmp_path, registry=registry, strict=False)
        rel_paths = [f.rel_path for f in result.processable + result.unsupported]

        assert "keep.txt" in rel_paths
        assert "skip.log" not in rel_paths
        assert any("skip.log (gitignore)" in s for s in result.skipped)

    def test_respects_nested_gitignore(self, tmp_path: Path, registry: ParserRegistry) -> None:
        (tmp_path / ".gitignore").write_text("*.log\n", encoding="utf-8")

        (tmp_path / "src").mkdir()
        (tmp_path / "src" / "skip.log").write_text("no", encoding="utf-8")
        (tmp_path / "src" / "keep.tmp").write_text("ok", encoding="utf-8")

        (tmp_path / "src" / "sub").mkdir()
        (tmp_path / "src" / "sub" / ".gitignore").write_text("*.tmp\n", encoding="utf-8")
        (tmp_path / "src" / "sub" / "keep.txt").write_text("ok", encoding="utf-8")
        (tmp_path / "src" / "sub" / "skip.log").write_text("no", encoding="utf-8")
        (tmp_path / "src" / "sub" / "skip.tmp").write_text("no", encoding="utf-8")

        result = scan_directory(tmp_path, registry=registry, strict=False)
        rel_paths = [f.rel_path for f in result.processable + result.unsupported]

        # .tmp is accepted in src/
        assert "src/keep.tmp" in rel_paths
        assert "src/skip.log" not in rel_paths
        assert "src/sub/keep.txt" in rel_paths
        assert "src/sub/skip.tmp" not in rel_paths
        # .tmp is no longer accepted in src/sub/ due to nested gitignore
        assert "src/sub/skip.log" not in rel_paths
        assert any("src/skip.log (gitignore)" in s for s in result.skipped)
        assert any("src/sub/skip.tmp (gitignore)" in s for s in result.skipped)
        assert any("src/sub/skip.log (gitignore)" in s for s in result.skipped)

    def test_respects_negation(self, tmp_path: Path, registry: ParserRegistry) -> None:
        (tmp_path / ".gitignore").write_text("*.tmp\n!important.tmp\n", encoding="utf-8")
        (tmp_path / "a.tmp").write_text("no", encoding="utf-8")
        (tmp_path / "important.tmp").write_text("yes", encoding="utf-8")

        result = scan_directory(tmp_path, registry=registry, strict=False)
        rel_paths = [f.rel_path for f in result.processable + result.unsupported]

        assert "a.tmp" not in rel_paths
        assert "important.tmp" in rel_paths
        skipped_reasons = " ".join(result.skipped)
        assert "gitignore" in skipped_reasons

    def test_ignored_directory_is_pruned(self, tmp_path: Path, registry: ParserRegistry) -> None:
        (tmp_path / ".gitignore").write_text("generated/\n", encoding="utf-8")
        (tmp_path / "generated").mkdir()
        (tmp_path / "generated" / "skip.txt").write_text("no", encoding="utf-8")
        (tmp_path / "keep.txt").write_text("ok", encoding="utf-8")

        result = scan_directory(tmp_path, registry=registry, strict=False)
        rel_paths = [f.rel_path for f in result.processable + result.unsupported]

        assert "keep.txt" in rel_paths
        assert "generated/skip.txt" not in rel_paths
        assert any("generated (gitignore)" in s for s in result.skipped)


class TestClassifiedFileAndResult:
    """Test ClassifiedFile and DirectoryScanResult types."""

    def test_classified_file_has_rel_path_and_classification(self) -> None:
        p = Path("/tmp/foo/bar.txt")
        cf = ClassifiedFile(path=p, rel_path="bar.txt", classification=CLASS_PROCESSABLE)
        assert cf.rel_path == "bar.txt"
        assert cf.classification == CLASS_PROCESSABLE

    def test_scan_result_root_and_lists(
        self, tmp_all_supported: Path, registry: ParserRegistry
    ) -> None:
        result: DirectoryScanResult = scan_directory(
            tmp_all_supported, registry=registry, strict=True
        )
        assert result.root == tmp_all_supported.resolve()
        assert isinstance(result.processable, list)
        assert isinstance(result.unsupported, list)
        assert isinstance(result.warnings, list)
        assert isinstance(result.skipped, list)