Self-evolving Context Database for AI Agents. Unify Agent Memory, Knowledge RAG and Skills.
Python agent-contextagent-memoryagent-pluginsagentic-ragcontext-databasedsh-pluginknowledge-basememorymemory-managementself-evolvingvector-database
tests(parsers): add unit tests for office extensions within add_resource directory (#273)
Eric Shaw6a9cd20a67c8
diff --git a/openviking/parse/registry.py b/openviking/parse/registry.pyindex 5777edecb..77a90395a 100644--- a/openviking/parse/registry.py+++ b/openviking/parse/registry.py@@ -65,8 +65,10 @@ class ParserRegistry:self.register("powerpoint", PowerPointParser())self.register("excel", ExcelParser())self.register("epub", EPubParser())- self.register("zip", ZipParser())+ # CodeRepositoryParser also uses .zip; register it before ZipParser+ # so that .zip resolves to ZipParser (file) rather than code repo.self.register("code", CodeRepositoryParser())+ self.register("zip", ZipParser())self.register("directory", DirectoryParser())self.register("image", ImageParser())diff --git a/tests/parse/test_add_directory.py b/tests/parse/test_add_directory.pyindex 81fd650bd..5c4e884b5 100644--- a/tests/parse/test_add_directory.py+++ b/tests/parse/test_add_directory.py@@ -371,6 +371,212 @@ class TestParserDelegation:assert result.meta["file_count"] == 1assert len(fake_fs.files) > 0+ @pytest.mark.asyncio+ async def test_docx_file_goes_through_parser(self, tmp_path: Path, parser, fake_fs) -> None:+ """Word (.docx) files should be processed by WordParser.parse()."""+ (tmp_path / "report.docx").write_bytes(b"PK\x03\x04")++ mock_temp = fake_fs.create_temp_uri()+ doc_dir = f"{mock_temp}/report"+ await fake_fs.mkdir(mock_temp)+ await fake_fs.mkdir(doc_dir)+ await fake_fs.write_file(f"{doc_dir}/report.md", "# Converted Word")++ fake_result = create_parse_result(+ root=ResourceNode(type=NodeType.ROOT),+ source_path=str(tmp_path / "report.docx"),+ source_format="docx",+ parser_name="WordParser",+ parse_time=0.1,+ )+ fake_result.temp_dir_path = mock_temp++ with patch(+ "openviking.parse.parsers.directory.DirectoryParser._assign_parser",+ ) as mock_assign:+ from openviking.parse.parsers.word import WordParser as _Word++ mock_word = AsyncMock(spec=_Word)+ mock_word.parse = AsyncMock(return_value=fake_result)++ def assign_side_effect(cf, registry):+ if cf.path.suffix == ".docx":+ return mock_word+ return registry.get_parser_for_file(cf.path)++ mock_assign.side_effect = assign_side_effect+ await parser.parse(str(tmp_path))++ dir_name = tmp_path.name+ found_md = any(+ uri.endswith("report.md") and f"/{dir_name}/" in uri for uri in fake_fs.files+ )+ assert found_md, f"report.md not found. Files: {list(fake_fs.files.keys())}"++ @pytest.mark.asyncio+ async def test_xlsx_file_goes_through_parser(self, tmp_path: Path, parser, fake_fs) -> None:+ """Excel (.xlsx) files should be processed by ExcelParser.parse()."""+ (tmp_path / "data.xlsx").write_bytes(b"PK\x03\x04")++ mock_temp = fake_fs.create_temp_uri()+ doc_dir = f"{mock_temp}/data"+ await fake_fs.mkdir(mock_temp)+ await fake_fs.mkdir(doc_dir)+ await fake_fs.write_file(f"{doc_dir}/data.md", "# Converted Excel")++ fake_result = create_parse_result(+ root=ResourceNode(type=NodeType.ROOT),+ source_path=str(tmp_path / "data.xlsx"),+ source_format="xlsx",+ parser_name="ExcelParser",+ parse_time=0.1,+ )+ fake_result.temp_dir_path = mock_temp++ with patch(+ "openviking.parse.parsers.directory.DirectoryParser._assign_parser",+ ) as mock_assign:+ from openviking.parse.parsers.excel import ExcelParser as _Excel++ mock_excel = AsyncMock(spec=_Excel)+ mock_excel.parse = AsyncMock(return_value=fake_result)++ def assign_side_effect(cf, registry):+ if cf.path.suffix in {".xlsx", ".xls", ".xlsm"}:+ return mock_excel+ return registry.get_parser_for_file(cf.path)++ mock_assign.side_effect = assign_side_effect+ await parser.parse(str(tmp_path))++ dir_name = tmp_path.name+ found_md = any(uri.endswith("data.md") and f"/{dir_name}/" in uri for uri in fake_fs.files)+ assert found_md, f"data.md not found. Files: {list(fake_fs.files.keys())}"++ @pytest.mark.asyncio+ async def test_epub_file_goes_through_parser(self, tmp_path: Path, parser, fake_fs) -> None:+ """EPub (.epub) files should be processed by EPubParser.parse()."""+ (tmp_path / "book.epub").write_bytes(b"PK\x03\x04")++ mock_temp = fake_fs.create_temp_uri()+ doc_dir = f"{mock_temp}/book"+ await fake_fs.mkdir(mock_temp)+ await fake_fs.mkdir(doc_dir)+ await fake_fs.write_file(f"{doc_dir}/book.md", "# Converted EPub")++ fake_result = create_parse_result(+ root=ResourceNode(type=NodeType.ROOT),+ source_path=str(tmp_path / "book.epub"),+ source_format="epub",+ parser_name="EPubParser",+ parse_time=0.1,+ )+ fake_result.temp_dir_path = mock_temp++ with patch(+ "openviking.parse.parsers.directory.DirectoryParser._assign_parser",+ ) as mock_assign:+ from openviking.parse.parsers.epub import EPubParser as _EPub++ mock_epub = AsyncMock(spec=_EPub)+ mock_epub.parse = AsyncMock(return_value=fake_result)++ def assign_side_effect(cf, registry):+ if cf.path.suffix == ".epub":+ return mock_epub+ return registry.get_parser_for_file(cf.path)++ mock_assign.side_effect = assign_side_effect+ await parser.parse(str(tmp_path))++ dir_name = tmp_path.name+ found_md = any(uri.endswith("book.md") and f"/{dir_name}/" in uri for uri in fake_fs.files)+ assert found_md, f"book.md not found. Files: {list(fake_fs.files.keys())}"++ @pytest.mark.asyncio+ async def test_pptx_file_goes_through_parser(self, tmp_path: Path, parser, fake_fs) -> None:+ """PowerPoint (.pptx) files should be processed by PowerPointParser.parse()."""+ (tmp_path / "slides.pptx").write_bytes(b"PK\x03\x04")++ mock_temp = fake_fs.create_temp_uri()+ doc_dir = f"{mock_temp}/slides"+ await fake_fs.mkdir(mock_temp)+ await fake_fs.mkdir(doc_dir)+ await fake_fs.write_file(f"{doc_dir}/slides.md", "# Converted PowerPoint")++ fake_result = create_parse_result(+ root=ResourceNode(type=NodeType.ROOT),+ source_path=str(tmp_path / "slides.pptx"),+ source_format="pptx",+ parser_name="PowerPointParser",+ parse_time=0.1,+ )+ fake_result.temp_dir_path = mock_temp++ with patch(+ "openviking.parse.parsers.directory.DirectoryParser._assign_parser",+ ) as mock_assign:+ from openviking.parse.parsers.powerpoint import PowerPointParser as _PPT++ mock_ppt = AsyncMock(spec=_PPT)+ mock_ppt.parse = AsyncMock(return_value=fake_result)++ def assign_side_effect(cf, registry):+ if cf.path.suffix == ".pptx":+ return mock_ppt+ return registry.get_parser_for_file(cf.path)++ mock_assign.side_effect = assign_side_effect+ await parser.parse(str(tmp_path))++ dir_name = tmp_path.name+ found_md = any(+ uri.endswith("slides.md") and f"/{dir_name}/" in uri for uri in fake_fs.files+ )+ assert found_md, f"slides.md not found. Files: {list(fake_fs.files.keys())}"++ @pytest.mark.asyncio+ async def test_zip_file_goes_through_parser(self, tmp_path: Path, parser, fake_fs) -> None:+ """ZIP (.zip) files should be processed by ZipParser.parse()."""+ (tmp_path / "archive.zip").write_bytes(b"PK\x03\x04")++ mock_temp = fake_fs.create_temp_uri()+ doc_dir = f"{mock_temp}/archive"+ await fake_fs.mkdir(mock_temp)+ await fake_fs.mkdir(doc_dir)+ await fake_fs.write_file(f"{doc_dir}/archive.md", "# Converted ZIP")++ fake_result = create_parse_result(+ root=ResourceNode(type=NodeType.ROOT),+ source_path=str(tmp_path / "archive.zip"),+ source_format="zip",+ parser_name="ZipParser",+ parse_time=0.1,+ )+ fake_result.temp_dir_path = mock_temp++ with patch(+ "openviking.parse.parsers.directory.DirectoryParser._assign_parser",+ ) as mock_assign:+ from openviking.parse.parsers.zip_parser import ZipParser as _Zip++ mock_zip = AsyncMock(spec=_Zip)+ mock_zip.parse = AsyncMock(return_value=fake_result)++ def assign_side_effect(cf, registry):+ if cf.path.suffix == ".zip":+ return mock_zip+ return registry.get_parser_for_file(cf.path)++ mock_assign.side_effect = assign_side_effect+ await parser.parse(str(tmp_path))++ dir_name = tmp_path.name+ found_md = any(+ uri.endswith("archive.md") and f"/{dir_name}/" in uri for uri in fake_fs.files+ )+ assert found_md, f"archive.md not found. Files: {list(fake_fs.files.keys())}"+# ---------------------------------------------------------------------------# Tests: PDF conversion via parser.parse()diff --git a/tests/parse/test_directory_parser_routing.py b/tests/parse/test_directory_parser_routing.pyindex 349fd2f9f..622f772d7 100644--- a/tests/parse/test_directory_parser_routing.py+++ b/tests/parse/test_directory_parser_routing.py@@ -27,10 +27,15 @@ from openviking.parse.directory_scan import (DirectoryScanResult,scan_directory,)+from openviking.parse.parsers.epub import EPubParser+from openviking.parse.parsers.excel import ExcelParserfrom openviking.parse.parsers.html import HTMLParserfrom openviking.parse.parsers.markdown import MarkdownParserfrom openviking.parse.parsers.pdf import PDFParser+from openviking.parse.parsers.powerpoint import PowerPointParserfrom openviking.parse.parsers.text import TextParser+from openviking.parse.parsers.word import WordParser+from openviking.parse.parsers.zip_parser import ZipParserfrom openviking.parse.registry import ParserRegistry# ═══════════════════════════════════════════════════════════════════════════@@ -66,6 +71,16 @@ def tmp_all_parsers(tmp_path: Path) -> Path:text/notes.txt -> TextParserlog.text -> TextParser+ office/+ report.docx -> WordParser+ data.xlsx -> ExcelParser+ legacy.xls -> ExcelParser+ macro.xlsm -> ExcelParser+ slides.pptx -> PowerPointParser+ books/+ book.epub -> EPubParser+ archives/+ bundle.zip -> ZipParsercode/app.py -> text-fallback (is_text_file)main.js -> text-fallback@@ -105,6 +120,19 @@ def tmp_all_parsers(tmp_path: Path) -> Path:(tmp_path / "config" / "data.json").write_text("{}", encoding="utf-8")(tmp_path / "config" / "rules.toml").write_text("[section]", encoding="utf-8")+ (tmp_path / "office").mkdir()+ (tmp_path / "office" / "report.docx").write_bytes(b"PK\x03\x04")+ (tmp_path / "office" / "data.xlsx").write_bytes(b"PK\x03\x04")+ (tmp_path / "office" / "legacy.xls").write_bytes(b"\xd0\xcf\x11\xe0")+ (tmp_path / "office" / "macro.xlsm").write_bytes(b"PK\x03\x04")+ (tmp_path / "office" / "slides.pptx").write_bytes(b"PK\x03\x04")++ (tmp_path / "books").mkdir()+ (tmp_path / "books" / "book.epub").write_bytes(b"PK\x03\x04")++ (tmp_path / "archives").mkdir()+ (tmp_path / "archives" / "bundle.zip").write_bytes(b"PK\x03\x04")+(tmp_path / "unsupported").mkdir()(tmp_path / "unsupported" / "image.bmp").write_bytes(b"BM\x00\x00")(tmp_path / "unsupported" / "archive.rar").write_bytes(b"RAR\x00")@@ -126,6 +154,13 @@ class TestParserSelection:".pdf": PDFParser,".txt": TextParser,".text": TextParser,+ ".docx": WordParser,+ ".xlsx": ExcelParser,+ ".xls": ExcelParser,+ ".xlsm": ExcelParser,+ ".epub": EPubParser,+ ".pptx": PowerPointParser,+ ".zip": ZipParser,}# Extensions that are *processable* (via is_text_file) but have no@@ -203,6 +238,11 @@ class TestParserCanParse:(HTMLParser, ["page.html", "site.htm"]),(PDFParser, ["paper.pdf"]),(TextParser, ["notes.txt", "log.text"]),+ (WordParser, ["report.docx"]),+ (ExcelParser, ["data.xlsx", "legacy.xls", "book.xlsm"]),+ (EPubParser, ["book.epub"]),+ (PowerPointParser, ["slides.pptx"]),+ (ZipParser, ["archive.zip"]),],)def test_can_parse_returns_true(self, parser_cls: type, filenames: List[str]) -> None:@@ -219,6 +259,11 @@ class TestParserCanParse:(HTMLParser, ["file.md", "file.pdf", "file.txt"]),(PDFParser, ["file.md", "file.txt", "file.html"]),(TextParser, ["file.md", "file.html", "file.pdf"]),+ (WordParser, ["file.pdf", "file.xlsx", "file.txt"]),+ (ExcelParser, ["file.docx", "file.pdf", "file.txt"]),+ (EPubParser, ["file.pdf", "file.docx", "file.zip"]),+ (PowerPointParser, ["file.pdf", "file.docx", "file.txt"]),+ (ZipParser, ["file.rar", "file.pdf", "file.docx"]),],)def test_can_parse_returns_false_for_wrong_extension(diff --git a/tests/parse/test_directory_scan.py b/tests/parse/test_directory_scan.pyindex 0a3166957..cd440dca0 100644--- a/tests/parse/test_directory_scan.py+++ b/tests/parse/test_directory_scan.py@@ -23,10 +23,15 @@ from openviking_cli.exceptions import UnsupportedDirectoryFilesError@pytest.fixturedef tmp_tree(tmp_path: Path) -> Path:"""Create a directory tree with mixed file types for scan tests."""- # rich (parser exists): .md, .pdf, .html, .txt+ # rich (parser exists): .md, .pdf, .html, .txt, .docx, .xlsx, .epub, .pptx, .zip(tmp_path / "readme.md").write_text("# README", encoding="utf-8")(tmp_path / "doc.html").write_text("<html></html>", encoding="utf-8")(tmp_path / "note.txt").write_text("plain text", encoding="utf-8")+ (tmp_path / "report.docx").write_bytes(b"PK\x03\x04")+ (tmp_path / "data.xlsx").write_bytes(b"PK\x03\x04")+ (tmp_path / "book.epub").write_bytes(b"PK\x03\x04")+ (tmp_path / "slides.pptx").write_bytes(b"PK\x03\x04")+ (tmp_path / "bundle.zip").write_bytes(b"PK\x03\x04")# text (code/config, no dedicated parser or text parser only): .py, .yaml(tmp_path / "main.py").write_text("print(1)", encoding="utf-8")@@ -118,6 +123,12 @@ class TestScanDirectoryClassification:assert "readme.md" in processable_relassert "doc.html" in processable_relassert "note.txt" in processable_rel+ # Word, Excel, EPub, PowerPoint, Zip parsers+ assert "report.docx" in processable_rel+ assert "data.xlsx" in processable_rel+ assert "book.epub" in processable_rel+ assert "slides.pptx" in processable_rel+ assert "bundle.zip" in processable_reldef test_processable_includes_code_or_config(self, tmp_tree: Path, registry: ParserRegistry