Self-evolving Context Database for AI Agents. Unify Agent Memory, Knowledge RAG and Skills.
Python agent-contextagent-memoryagent-pluginsagentic-ragcontext-databasedsh-pluginknowledge-basememorymemory-managementself-evolvingvector-database
# Copyright (c) 2026 Beijing Volcano Engine Technology Co., Ltd.
# SPDX-License-Identifier: AGPL-3.0
"""Isolated unit tests for directory-import parser routing and path mapping.
This script verifies **two independent concerns** without invoking the full
``ResourceService`` pipeline:
1. **Parser selection** – given a set of file extensions, the ``ParserRegistry``
(and ``scan_directory``) correctly classifies each file and selects the
expected parser type (MarkdownParser, HTMLParser, PDFParser, TextParser,
or the text-fallback path for code / config files).
2. **Path mapping** – the ``_process_directory_file`` helper in
``ResourceService`` converts each file's relative path into the correct
Viking target URI so that the imported directory structure is preserved.
For example, ``a/b/c.md`` with base target ``viking://resources/mydir``
produces target ``viking://resources/mydir/a/b`` and the parser names
the document ``c``, yielding final URI ``viking://resources/mydir/a/b/c``.
"""
from pathlib import Path, PurePosixPath
from typing import Dict, List, Tuple
import pytest
from openviking.parse.directory_scan import (
DirectoryScanResult,
scan_directory,
)
from openviking.parse.parsers.anydoc import AnyDocParser
from openviking.parse.parsers.html import HTMLParser
from openviking.parse.parsers.markdown import MarkdownParser
from openviking.parse.parsers.pdf import PDFParser
from openviking.parse.parsers.text import TextParser
from openviking.parse.parsers.zip_parser import ZipParser
from openviking.parse.registry import ParserRegistry
# ═══════════════════════════════════════════════════════════════════════════
# Part 1 – Parser selection
# ═══════════════════════════════════════════════════════════════════════════
@pytest.fixture
def registry() -> ParserRegistry:
"""Default registry (no optional parsers like ImageParser)."""
return ParserRegistry()
# -- directory tree that covers every parser type ----------------------------
@pytest.fixture
def tmp_all_parsers(tmp_path: Path) -> Path:
"""Directory tree with files that exercise every built-in parser.
Layout::
tmp_path/
docs/
guide.md -> MarkdownParser
spec.markdown -> MarkdownParser
readme.mdown -> MarkdownParser
web/
index.html -> HTMLParser
page.htm -> HTMLParser
pdfs/
paper.pdf -> PDFParser (binary, requires real bytes)
text/
notes.txt -> TextParser
log.text -> TextParser
office/
legacy.doc -> AnyDocParser
report.docx -> AnyDocParser
macro.docm -> AnyDocParser
open.odt -> AnyDocParser
rich.rtf -> AnyDocParser
data.xlsx -> AnyDocParser
legacy.xls -> AnyDocParser
macro.xlsm -> AnyDocParser
binary.xlsb -> AnyDocParser
sheet.ods -> AnyDocParser
table.csv -> AnyDocParser
slides.pptx -> AnyDocParser
legacy.ppt -> AnyDocParser
show.pps -> AnyDocParser
books/
book.epub -> AnyDocParser
archives/
bundle.zip -> ZipParser
code/
app.py -> text-fallback (is_text_file)
engine.cc -> text-fallback
main.js -> text-fallback
style.css -> text-fallback
config/
settings.yaml -> text-fallback
data.json -> text-fallback
rules.toml -> text-fallback
unsupported/
image.bmp -> unsupported (binary, no parser)
archive.rar -> unsupported
"""
(tmp_path / "docs").mkdir()
(tmp_path / "docs" / "guide.md").write_text("# Guide", encoding="utf-8")
(tmp_path / "docs" / "spec.markdown").write_text("# Spec", encoding="utf-8")
(tmp_path / "docs" / "readme.mdown").write_text("# Readme", encoding="utf-8")
(tmp_path / "web").mkdir()
(tmp_path / "web" / "index.html").write_text("<html></html>", encoding="utf-8")
(tmp_path / "web" / "page.htm").write_text("<html></html>", encoding="utf-8")
(tmp_path / "pdfs").mkdir()
# Minimal PDF header so it's not empty
(tmp_path / "pdfs" / "paper.pdf").write_bytes(b"%PDF-1.4 minimal")
(tmp_path / "text").mkdir()
(tmp_path / "text" / "notes.txt").write_text("plain text", encoding="utf-8")
(tmp_path / "text" / "log.text").write_text("log entry", encoding="utf-8")
(tmp_path / "code").mkdir()
(tmp_path / "code" / "app.py").write_text("print(1)", encoding="utf-8")
(tmp_path / "code" / "engine.cc").write_text("int main() { return 0; }", encoding="utf-8")
(tmp_path / "code" / "main.js").write_text("console.log(1)", encoding="utf-8")
(tmp_path / "code" / "style.css").write_text("body{}", encoding="utf-8")
(tmp_path / "config").mkdir()
(tmp_path / "config" / "settings.yaml").write_text("k: v", encoding="utf-8")
(tmp_path / "config" / "data.json").write_text("{}", encoding="utf-8")
(tmp_path / "config" / "rules.toml").write_text("[section]", encoding="utf-8")
(tmp_path / "office").mkdir()
(tmp_path / "office" / "legacy.doc").write_bytes(b"\xd0\xcf\x11\xe0")
(tmp_path / "office" / "report.docx").write_bytes(b"PK\x03\x04")
(tmp_path / "office" / "macro.docm").write_bytes(b"PK\x03\x04")
(tmp_path / "office" / "open.odt").write_bytes(b"PK\x03\x04")
(tmp_path / "office" / "rich.rtf").write_text(r"{\rtf1 text}", encoding="utf-8")
(tmp_path / "office" / "data.xlsx").write_bytes(b"PK\x03\x04")
(tmp_path / "office" / "legacy.xls").write_bytes(b"\xd0\xcf\x11\xe0")
(tmp_path / "office" / "macro.xlsm").write_bytes(b"PK\x03\x04")
(tmp_path / "office" / "binary.xlsb").write_bytes(b"PK\x03\x04")
(tmp_path / "office" / "sheet.ods").write_bytes(b"PK\x03\x04")
(tmp_path / "office" / "table.csv").write_text("a,b\n1,2\n", encoding="utf-8")
(tmp_path / "office" / "slides.pptx").write_bytes(b"PK\x03\x04")
(tmp_path / "office" / "legacy.ppt").write_bytes(b"\xd0\xcf\x11\xe0")
(tmp_path / "office" / "show.pps").write_bytes(b"\xd0\xcf\x11\xe0")
(tmp_path / "books").mkdir()
(tmp_path / "books" / "book.epub").write_bytes(b"PK\x03\x04")
(tmp_path / "archives").mkdir()
(tmp_path / "archives" / "bundle.zip").write_bytes(b"PK\x03\x04")
(tmp_path / "unsupported").mkdir()
(tmp_path / "unsupported" / "image.bmp").write_bytes(b"BM\x00\x00")
(tmp_path / "unsupported" / "archive.rar").write_bytes(b"RAR\x00")
return tmp_path
class TestParserSelection:
"""Each file extension must be resolved to the correct parser class."""
# Extension -> expected parser class (or None = no dedicated parser, uses
# text-fallback through ParserRegistry.parse which falls through to TextParser)
DEDICATED_PARSER_MAP: Dict[str, type] = {
".md": MarkdownParser,
".markdown": MarkdownParser,
".mdown": MarkdownParser,
".html": HTMLParser,
".htm": HTMLParser,
".pdf": PDFParser,
".txt": TextParser,
".text": TextParser,
".doc": AnyDocParser,
".docx": AnyDocParser,
".docm": AnyDocParser,
".odt": AnyDocParser,
".rtf": AnyDocParser,
".xlsx": AnyDocParser,
".xls": AnyDocParser,
".xlsm": AnyDocParser,
".xlsb": AnyDocParser,
".ods": AnyDocParser,
".csv": AnyDocParser,
".epub": AnyDocParser,
".pptx": AnyDocParser,
".ppt": AnyDocParser,
".pps": AnyDocParser,
".zip": ZipParser,
}
# Extensions that are *processable* (via is_text_file) but have no
# dedicated parser in the registry – they fall back to TextParser at
# parse-time via ``ParserRegistry.parse``.
TEXT_FALLBACK_EXTENSIONS = {".py", ".cc", ".js", ".css", ".yaml", ".json", ".toml"}
def test_dedicated_parsers_resolve(self, registry: ParserRegistry) -> None:
"""get_parser_for_file returns the correct class for each extension."""
for ext, expected_cls in self.DEDICATED_PARSER_MAP.items():
dummy_path = Path(f"/tmp/file{ext}")
parser = registry.get_parser_for_file(dummy_path)
assert parser is not None, f"No parser returned for {ext}"
assert isinstance(parser, expected_cls), (
f"{ext}: expected {expected_cls.__name__}, got {type(parser).__name__}"
)
def test_text_fallback_returns_none_from_registry(self, registry: ParserRegistry) -> None:
"""Code / config extensions have no *dedicated* parser, so
``get_parser_for_file`` returns None. The registry's ``parse()``
falls back to TextParser internally."""
for ext in self.TEXT_FALLBACK_EXTENSIONS:
dummy_path = Path(f"/tmp/file{ext}")
parser = registry.get_parser_for_file(dummy_path)
assert parser is None, (
f"{ext}: expected None (text-fallback), got {type(parser).__name__}"
)
def test_scan_classifies_all_files_correctly(
self, tmp_all_parsers: Path, registry: ParserRegistry
) -> None:
"""scan_directory should mark dedicated-parser and text-fallback
files as processable, and truly unknown formats as unsupported."""
result: DirectoryScanResult = scan_directory(
tmp_all_parsers, registry=registry, strict=False
)
processable_exts = {Path(f.rel_path).suffix.lower() for f in result.processable}
unsupported_exts = {Path(f.rel_path).suffix.lower() for f in result.unsupported}
# All dedicated-parser extensions must be processable
for ext in self.DEDICATED_PARSER_MAP:
assert ext in processable_exts, f"{ext} should be processable"
# All text-fallback extensions must be processable
for ext in self.TEXT_FALLBACK_EXTENSIONS:
assert ext in processable_exts, f"{ext} should be processable (text-fallback)"
# .rar are unsupported
assert ".rar" in unsupported_exts
def test_each_processable_file_has_a_parser_or_is_text(
self, tmp_all_parsers: Path, registry: ParserRegistry
) -> None:
"""Every processable file must either have a dedicated parser or pass
``is_text_file``."""
from openviking.parse.parsers.upload_utils import is_text_file
result = scan_directory(tmp_all_parsers, registry=registry, strict=False)
for cf in result.processable:
has_parser = registry.get_parser_for_file(cf.path) is not None
is_text = is_text_file(cf.path)
assert has_parser or is_text, (
f"{cf.rel_path}: not a known parser type and not a text file"
)
# ═══════════════════════════════════════════════════════════════════════════
# Part 2 – Relative-path → Viking URI mapping
# ═══════════════════════════════════════════════════════════════════════════
# The mapping logic lives in ``ResourceService._process_directory_file``.
# Instead of pulling in the full service we replicate the *pure* path
# computation here so tests stay isolated and fast.
def _compute_file_target(rel_path: str, base_target: str) -> str:
"""Replicate the target-URI computation from _process_directory_file."""
parent_rel = str(PurePosixPath(rel_path).parent)
if parent_rel == ".":
return base_target
return f"{base_target}/{parent_rel}"
def _expected_final_uri(rel_path: str, base_target: str) -> str:
"""Expected final URI after the parser names the document by file stem.
The TreeBuilder computes: ``final_uri = base_uri.join(doc_name)``
where ``doc_name`` is typically the file stem.
"""
file_target = _compute_file_target(rel_path, base_target)
stem = Path(rel_path).stem
return f"{file_target}/{stem}"
class TestPathMapping:
"""Verify that relative file paths map to the correct Viking URIs."""
BASE = "viking://resources/mydir"
# (relative_path, expected_target_for_process_resource, expected_final_uri)
CASES: List[Tuple[str, str, str]] = [
# Root-level file
("top.md", "viking://resources/mydir", "viking://resources/mydir/top"),
("README.txt", "viking://resources/mydir", "viking://resources/mydir/README"),
# One level deep
(
"docs/guide.md",
"viking://resources/mydir/docs",
"viking://resources/mydir/docs/guide",
),
(
"src/app.py",
"viking://resources/mydir/src",
"viking://resources/mydir/src/app",
),
# Two levels deep
(
"a/b/c.md",
"viking://resources/mydir/a/b",
"viking://resources/mydir/a/b/c",
),
(
"a/b/d.txt",
"viking://resources/mydir/a/b",
"viking://resources/mydir/a/b/d",
),
# Three levels deep
(
"x/y/z/deep.md",
"viking://resources/mydir/x/y/z",
"viking://resources/mydir/x/y/z/deep",
),
]
@pytest.mark.parametrize("rel_path,expected_target,_", CASES)
def test_target_uri_computation(self, rel_path: str, expected_target: str, _: str) -> None:
"""_compute_file_target produces the correct parent-based target."""
assert _compute_file_target(rel_path, self.BASE) == expected_target
@pytest.mark.parametrize("rel_path,_,expected_uri", CASES)
def test_final_uri_matches_rel_path_structure(
self, rel_path: str, _: str, expected_uri: str
) -> None:
"""The final URI (target + file stem) preserves the directory tree."""
assert _expected_final_uri(rel_path, self.BASE) == expected_uri
class TestPathMappingFromScan:
"""End-to-end: scan a real directory, then verify every processable file's
relative path maps to the expected Viking URI."""
@pytest.fixture
def tmp_deep(self, tmp_path: Path) -> Path:
"""Create a three-level nested directory.
Structure::
tmp_path/
a/
b/
c.md
x.md
top.md
src/
main.py
"""
ab = tmp_path / "a" / "b"
ab.mkdir(parents=True)
(ab / "c.md").write_text("# C", encoding="utf-8")
(tmp_path / "a" / "x.md").write_text("# X", encoding="utf-8")
(tmp_path / "top.md").write_text("# Top", encoding="utf-8")
(tmp_path / "src").mkdir()
(tmp_path / "src" / "main.py").write_text("pass", encoding="utf-8")
return tmp_path
def test_scan_then_map_preserves_structure(self, tmp_deep: Path) -> None:
"""For every processable file, the computed final URI should embed
the same directory hierarchy as the original relative path."""
result = scan_directory(tmp_deep, strict=False)
base = f"viking://resources/{tmp_deep.name}"
for cf in result.processable:
rel = cf.rel_path.replace("\\", "/") # normalize for Windows
final_uri = _expected_final_uri(rel, base)
# The URI path (after viking://resources/) should equal
# <dir_name>/<rel_path_without_extension>
uri_path = final_uri[len("viking://resources/") :]
expected_path = f"{tmp_deep.name}/{str(PurePosixPath(rel).with_suffix(''))}"
assert uri_path == expected_path, (
f"Mapping mismatch for {rel}: got URI path '{uri_path}', expected '{expected_path}'"
)
def test_empty_directory_produces_no_mappings(self, tmp_path: Path) -> None:
"""An empty directory has no processable files → zero URI mappings."""
(tmp_path / ".gitkeep").write_text("", encoding="utf-8") # skipped: empty
result = scan_directory(tmp_path, strict=False)
assert len(result.processable) == 0