Skip to content

Self-evolving Context Database for AI Agents. Unify Agent Memory, Knowledge RAG and Skills.

tests(parsers): add unit tests for office extensions within add_resource directory (#273)

Eric Shaw6a9cd20a67c8
diff --git a/openviking/parse/registry.py b/openviking/parse/registry.py
index 5777edecb..77a90395a 100644
--- a/openviking/parse/registry.py
+++ b/openviking/parse/registry.py
@@ -65,8 +65,10 @@ class ParserRegistry:
self.register("powerpoint", PowerPointParser())
self.register("excel", ExcelParser())
self.register("epub", EPubParser())
- self.register("zip", ZipParser())
+ # CodeRepositoryParser also uses .zip; register it before ZipParser
+ # so that .zip resolves to ZipParser (file) rather than code repo.
self.register("code", CodeRepositoryParser())
+ self.register("zip", ZipParser())
self.register("directory", DirectoryParser())
self.register("image", ImageParser())
diff --git a/tests/parse/test_add_directory.py b/tests/parse/test_add_directory.py
index 81fd650bd..5c4e884b5 100644
--- a/tests/parse/test_add_directory.py
+++ b/tests/parse/test_add_directory.py
@@ -371,6 +371,212 @@ class TestParserDelegation:
assert result.meta["file_count"] == 1
assert len(fake_fs.files) > 0
+ @pytest.mark.asyncio
+ async def test_docx_file_goes_through_parser(self, tmp_path: Path, parser, fake_fs) -> None:
+ """Word (.docx) files should be processed by WordParser.parse()."""
+ (tmp_path / "report.docx").write_bytes(b"PK\x03\x04")
+
+ mock_temp = fake_fs.create_temp_uri()
+ doc_dir = f"{mock_temp}/report"
+ await fake_fs.mkdir(mock_temp)
+ await fake_fs.mkdir(doc_dir)
+ await fake_fs.write_file(f"{doc_dir}/report.md", "# Converted Word")
+
+ fake_result = create_parse_result(
+ root=ResourceNode(type=NodeType.ROOT),
+ source_path=str(tmp_path / "report.docx"),
+ source_format="docx",
+ parser_name="WordParser",
+ parse_time=0.1,
+ )
+ fake_result.temp_dir_path = mock_temp
+
+ with patch(
+ "openviking.parse.parsers.directory.DirectoryParser._assign_parser",
+ ) as mock_assign:
+ from openviking.parse.parsers.word import WordParser as _Word
+
+ mock_word = AsyncMock(spec=_Word)
+ mock_word.parse = AsyncMock(return_value=fake_result)
+
+ def assign_side_effect(cf, registry):
+ if cf.path.suffix == ".docx":
+ return mock_word
+ return registry.get_parser_for_file(cf.path)
+
+ mock_assign.side_effect = assign_side_effect
+ await parser.parse(str(tmp_path))
+
+ dir_name = tmp_path.name
+ found_md = any(
+ uri.endswith("report.md") and f"/{dir_name}/" in uri for uri in fake_fs.files
+ )
+ assert found_md, f"report.md not found. Files: {list(fake_fs.files.keys())}"
+
+ @pytest.mark.asyncio
+ async def test_xlsx_file_goes_through_parser(self, tmp_path: Path, parser, fake_fs) -> None:
+ """Excel (.xlsx) files should be processed by ExcelParser.parse()."""
+ (tmp_path / "data.xlsx").write_bytes(b"PK\x03\x04")
+
+ mock_temp = fake_fs.create_temp_uri()
+ doc_dir = f"{mock_temp}/data"
+ await fake_fs.mkdir(mock_temp)
+ await fake_fs.mkdir(doc_dir)
+ await fake_fs.write_file(f"{doc_dir}/data.md", "# Converted Excel")
+
+ fake_result = create_parse_result(
+ root=ResourceNode(type=NodeType.ROOT),
+ source_path=str(tmp_path / "data.xlsx"),
+ source_format="xlsx",
+ parser_name="ExcelParser",
+ parse_time=0.1,
+ )
+ fake_result.temp_dir_path = mock_temp
+
+ with patch(
+ "openviking.parse.parsers.directory.DirectoryParser._assign_parser",
+ ) as mock_assign:
+ from openviking.parse.parsers.excel import ExcelParser as _Excel
+
+ mock_excel = AsyncMock(spec=_Excel)
+ mock_excel.parse = AsyncMock(return_value=fake_result)
+
+ def assign_side_effect(cf, registry):
+ if cf.path.suffix in {".xlsx", ".xls", ".xlsm"}:
+ return mock_excel
+ return registry.get_parser_for_file(cf.path)
+
+ mock_assign.side_effect = assign_side_effect
+ await parser.parse(str(tmp_path))
+
+ dir_name = tmp_path.name
+ found_md = any(uri.endswith("data.md") and f"/{dir_name}/" in uri for uri in fake_fs.files)
+ assert found_md, f"data.md not found. Files: {list(fake_fs.files.keys())}"
+
+ @pytest.mark.asyncio
+ async def test_epub_file_goes_through_parser(self, tmp_path: Path, parser, fake_fs) -> None:
+ """EPub (.epub) files should be processed by EPubParser.parse()."""
+ (tmp_path / "book.epub").write_bytes(b"PK\x03\x04")
+
+ mock_temp = fake_fs.create_temp_uri()
+ doc_dir = f"{mock_temp}/book"
+ await fake_fs.mkdir(mock_temp)
+ await fake_fs.mkdir(doc_dir)
+ await fake_fs.write_file(f"{doc_dir}/book.md", "# Converted EPub")
+
+ fake_result = create_parse_result(
+ root=ResourceNode(type=NodeType.ROOT),
+ source_path=str(tmp_path / "book.epub"),
+ source_format="epub",
+ parser_name="EPubParser",
+ parse_time=0.1,
+ )
+ fake_result.temp_dir_path = mock_temp
+
+ with patch(
+ "openviking.parse.parsers.directory.DirectoryParser._assign_parser",
+ ) as mock_assign:
+ from openviking.parse.parsers.epub import EPubParser as _EPub
+
+ mock_epub = AsyncMock(spec=_EPub)
+ mock_epub.parse = AsyncMock(return_value=fake_result)
+
+ def assign_side_effect(cf, registry):
+ if cf.path.suffix == ".epub":
+ return mock_epub
+ return registry.get_parser_for_file(cf.path)
+
+ mock_assign.side_effect = assign_side_effect
+ await parser.parse(str(tmp_path))
+
+ dir_name = tmp_path.name
+ found_md = any(uri.endswith("book.md") and f"/{dir_name}/" in uri for uri in fake_fs.files)
+ assert found_md, f"book.md not found. Files: {list(fake_fs.files.keys())}"
+
+ @pytest.mark.asyncio
+ async def test_pptx_file_goes_through_parser(self, tmp_path: Path, parser, fake_fs) -> None:
+ """PowerPoint (.pptx) files should be processed by PowerPointParser.parse()."""
+ (tmp_path / "slides.pptx").write_bytes(b"PK\x03\x04")
+
+ mock_temp = fake_fs.create_temp_uri()
+ doc_dir = f"{mock_temp}/slides"
+ await fake_fs.mkdir(mock_temp)
+ await fake_fs.mkdir(doc_dir)
+ await fake_fs.write_file(f"{doc_dir}/slides.md", "# Converted PowerPoint")
+
+ fake_result = create_parse_result(
+ root=ResourceNode(type=NodeType.ROOT),
+ source_path=str(tmp_path / "slides.pptx"),
+ source_format="pptx",
+ parser_name="PowerPointParser",
+ parse_time=0.1,
+ )
+ fake_result.temp_dir_path = mock_temp
+
+ with patch(
+ "openviking.parse.parsers.directory.DirectoryParser._assign_parser",
+ ) as mock_assign:
+ from openviking.parse.parsers.powerpoint import PowerPointParser as _PPT
+
+ mock_ppt = AsyncMock(spec=_PPT)
+ mock_ppt.parse = AsyncMock(return_value=fake_result)
+
+ def assign_side_effect(cf, registry):
+ if cf.path.suffix == ".pptx":
+ return mock_ppt
+ return registry.get_parser_for_file(cf.path)
+
+ mock_assign.side_effect = assign_side_effect
+ await parser.parse(str(tmp_path))
+
+ dir_name = tmp_path.name
+ found_md = any(
+ uri.endswith("slides.md") and f"/{dir_name}/" in uri for uri in fake_fs.files
+ )
+ assert found_md, f"slides.md not found. Files: {list(fake_fs.files.keys())}"
+
+ @pytest.mark.asyncio
+ async def test_zip_file_goes_through_parser(self, tmp_path: Path, parser, fake_fs) -> None:
+ """ZIP (.zip) files should be processed by ZipParser.parse()."""
+ (tmp_path / "archive.zip").write_bytes(b"PK\x03\x04")
+
+ mock_temp = fake_fs.create_temp_uri()
+ doc_dir = f"{mock_temp}/archive"
+ await fake_fs.mkdir(mock_temp)
+ await fake_fs.mkdir(doc_dir)
+ await fake_fs.write_file(f"{doc_dir}/archive.md", "# Converted ZIP")
+
+ fake_result = create_parse_result(
+ root=ResourceNode(type=NodeType.ROOT),
+ source_path=str(tmp_path / "archive.zip"),
+ source_format="zip",
+ parser_name="ZipParser",
+ parse_time=0.1,
+ )
+ fake_result.temp_dir_path = mock_temp
+
+ with patch(
+ "openviking.parse.parsers.directory.DirectoryParser._assign_parser",
+ ) as mock_assign:
+ from openviking.parse.parsers.zip_parser import ZipParser as _Zip
+
+ mock_zip = AsyncMock(spec=_Zip)
+ mock_zip.parse = AsyncMock(return_value=fake_result)
+
+ def assign_side_effect(cf, registry):
+ if cf.path.suffix == ".zip":
+ return mock_zip
+ return registry.get_parser_for_file(cf.path)
+
+ mock_assign.side_effect = assign_side_effect
+ await parser.parse(str(tmp_path))
+
+ dir_name = tmp_path.name
+ found_md = any(
+ uri.endswith("archive.md") and f"/{dir_name}/" in uri for uri in fake_fs.files
+ )
+ assert found_md, f"archive.md not found. Files: {list(fake_fs.files.keys())}"
+
# ---------------------------------------------------------------------------
# Tests: PDF conversion via parser.parse()
diff --git a/tests/parse/test_directory_parser_routing.py b/tests/parse/test_directory_parser_routing.py
index 349fd2f9f..622f772d7 100644
--- a/tests/parse/test_directory_parser_routing.py
+++ b/tests/parse/test_directory_parser_routing.py
@@ -27,10 +27,15 @@ from openviking.parse.directory_scan import (
DirectoryScanResult,
scan_directory,
)
+from openviking.parse.parsers.epub import EPubParser
+from openviking.parse.parsers.excel import ExcelParser
from openviking.parse.parsers.html import HTMLParser
from openviking.parse.parsers.markdown import MarkdownParser
from openviking.parse.parsers.pdf import PDFParser
+from openviking.parse.parsers.powerpoint import PowerPointParser
from openviking.parse.parsers.text import TextParser
+from openviking.parse.parsers.word import WordParser
+from openviking.parse.parsers.zip_parser import ZipParser
from openviking.parse.registry import ParserRegistry
# ═══════════════════════════════════════════════════════════════════════════
@@ -66,6 +71,16 @@ def tmp_all_parsers(tmp_path: Path) -> Path:
text/
notes.txt -> TextParser
log.text -> TextParser
+ office/
+ report.docx -> WordParser
+ data.xlsx -> ExcelParser
+ legacy.xls -> ExcelParser
+ macro.xlsm -> ExcelParser
+ slides.pptx -> PowerPointParser
+ books/
+ book.epub -> EPubParser
+ archives/
+ bundle.zip -> ZipParser
code/
app.py -> text-fallback (is_text_file)
main.js -> text-fallback
@@ -105,6 +120,19 @@ def tmp_all_parsers(tmp_path: Path) -> Path:
(tmp_path / "config" / "data.json").write_text("{}", encoding="utf-8")
(tmp_path / "config" / "rules.toml").write_text("[section]", encoding="utf-8")
+ (tmp_path / "office").mkdir()
+ (tmp_path / "office" / "report.docx").write_bytes(b"PK\x03\x04")
+ (tmp_path / "office" / "data.xlsx").write_bytes(b"PK\x03\x04")
+ (tmp_path / "office" / "legacy.xls").write_bytes(b"\xd0\xcf\x11\xe0")
+ (tmp_path / "office" / "macro.xlsm").write_bytes(b"PK\x03\x04")
+ (tmp_path / "office" / "slides.pptx").write_bytes(b"PK\x03\x04")
+
+ (tmp_path / "books").mkdir()
+ (tmp_path / "books" / "book.epub").write_bytes(b"PK\x03\x04")
+
+ (tmp_path / "archives").mkdir()
+ (tmp_path / "archives" / "bundle.zip").write_bytes(b"PK\x03\x04")
+
(tmp_path / "unsupported").mkdir()
(tmp_path / "unsupported" / "image.bmp").write_bytes(b"BM\x00\x00")
(tmp_path / "unsupported" / "archive.rar").write_bytes(b"RAR\x00")
@@ -126,6 +154,13 @@ class TestParserSelection:
".pdf": PDFParser,
".txt": TextParser,
".text": TextParser,
+ ".docx": WordParser,
+ ".xlsx": ExcelParser,
+ ".xls": ExcelParser,
+ ".xlsm": ExcelParser,
+ ".epub": EPubParser,
+ ".pptx": PowerPointParser,
+ ".zip": ZipParser,
}
# Extensions that are *processable* (via is_text_file) but have no
@@ -203,6 +238,11 @@ class TestParserCanParse:
(HTMLParser, ["page.html", "site.htm"]),
(PDFParser, ["paper.pdf"]),
(TextParser, ["notes.txt", "log.text"]),
+ (WordParser, ["report.docx"]),
+ (ExcelParser, ["data.xlsx", "legacy.xls", "book.xlsm"]),
+ (EPubParser, ["book.epub"]),
+ (PowerPointParser, ["slides.pptx"]),
+ (ZipParser, ["archive.zip"]),
],
)
def test_can_parse_returns_true(self, parser_cls: type, filenames: List[str]) -> None:
@@ -219,6 +259,11 @@ class TestParserCanParse:
(HTMLParser, ["file.md", "file.pdf", "file.txt"]),
(PDFParser, ["file.md", "file.txt", "file.html"]),
(TextParser, ["file.md", "file.html", "file.pdf"]),
+ (WordParser, ["file.pdf", "file.xlsx", "file.txt"]),
+ (ExcelParser, ["file.docx", "file.pdf", "file.txt"]),
+ (EPubParser, ["file.pdf", "file.docx", "file.zip"]),
+ (PowerPointParser, ["file.pdf", "file.docx", "file.txt"]),
+ (ZipParser, ["file.rar", "file.pdf", "file.docx"]),
],
)
def test_can_parse_returns_false_for_wrong_extension(
diff --git a/tests/parse/test_directory_scan.py b/tests/parse/test_directory_scan.py
index 0a3166957..cd440dca0 100644
--- a/tests/parse/test_directory_scan.py
+++ b/tests/parse/test_directory_scan.py
@@ -23,10 +23,15 @@ from openviking_cli.exceptions import UnsupportedDirectoryFilesError
@pytest.fixture
def tmp_tree(tmp_path: Path) -> Path:
"""Create a directory tree with mixed file types for scan tests."""
- # rich (parser exists): .md, .pdf, .html, .txt
+ # rich (parser exists): .md, .pdf, .html, .txt, .docx, .xlsx, .epub, .pptx, .zip
(tmp_path / "readme.md").write_text("# README", encoding="utf-8")
(tmp_path / "doc.html").write_text("<html></html>", encoding="utf-8")
(tmp_path / "note.txt").write_text("plain text", encoding="utf-8")
+ (tmp_path / "report.docx").write_bytes(b"PK\x03\x04")
+ (tmp_path / "data.xlsx").write_bytes(b"PK\x03\x04")
+ (tmp_path / "book.epub").write_bytes(b"PK\x03\x04")
+ (tmp_path / "slides.pptx").write_bytes(b"PK\x03\x04")
+ (tmp_path / "bundle.zip").write_bytes(b"PK\x03\x04")
# text (code/config, no dedicated parser or text parser only): .py, .yaml
(tmp_path / "main.py").write_text("print(1)", encoding="utf-8")
@@ -118,6 +123,12 @@ class TestScanDirectoryClassification:
assert "readme.md" in processable_rel
assert "doc.html" in processable_rel
assert "note.txt" in processable_rel
+ # Word, Excel, EPub, PowerPoint, Zip parsers
+ assert "report.docx" in processable_rel
+ assert "data.xlsx" in processable_rel
+ assert "book.epub" in processable_rel
+ assert "slides.pptx" in processable_rel
+ assert "bundle.zip" in processable_rel
def test_processable_includes_code_or_config(
self, tmp_tree: Path, registry: ParserRegistry