Self-evolving Context Database for AI Agents. Unify Agent Memory, Knowledge RAG and Skills.
Python agent-contextagent-memoryagent-pluginsagentic-ragcontext-databasedsh-pluginknowledge-basememorymemory-managementself-evolvingvector-database
fix(parse): offload document conversions to thread (#2237)
Qin Haojied606b59fb811
diff --git a/openviking/parse/parsers/epub.py b/openviking/parse/parsers/epub.pyindex 09fce2848..6ecc76bbc 100644--- a/openviking/parse/parsers/epub.py+++ b/openviking/parse/parsers/epub.py@@ -7,6 +7,7 @@ Converts EPub e-books to Markdown then parses using MarkdownParser.Inspired by microsoft/markitdown approach."""+import asyncioimport htmlimport reimport zipfile@@ -111,7 +112,7 @@ class EPubParser(BaseParser):path = Path(source)if path.exists():- markdown_content = self._convert_to_markdown(path)+ markdown_content = await asyncio.to_thread(self._convert_to_markdown, path)result = await self._md_parser.parse_content(markdown_content, source_path=str(path), instruction=instruction, **kwargs)diff --git a/openviking/parse/parsers/excel.py b/openviking/parse/parsers/excel.pyindex 21e07151c..79d462d77 100644--- a/openviking/parse/parsers/excel.py+++ b/openviking/parse/parsers/excel.py@@ -7,6 +7,7 @@ Converts Excel spreadsheets to Markdown then parses using MarkdownParser.Inspired by microsoft/markitdown approach."""+import asynciofrom pathlib import Pathfrom typing import List, Optional, Union@@ -53,11 +54,13 @@ class ExcelParser(BaseParser):if path.exists():# Use xlrd for legacy .xls, openpyxl for .xlsx/.xlsmif path.suffix.lower() == ".xls":- markdown_content = self._convert_xls_to_markdown(path)+ markdown_content = await asyncio.to_thread(self._convert_xls_to_markdown, path)else:import openpyxl- markdown_content = self._convert_to_markdown(path, openpyxl)+ markdown_content = await asyncio.to_thread(+ self._convert_to_markdown, path, openpyxl+ )result = await self._md_parser.parse_content(markdown_content, source_path=str(path), instruction=instruction, **kwargs)diff --git a/openviking/parse/parsers/legacy_doc.py b/openviking/parse/parsers/legacy_doc.pyindex e30740e5d..882477e1a 100644--- a/openviking/parse/parsers/legacy_doc.py+++ b/openviking/parse/parsers/legacy_doc.py@@ -7,6 +7,7 @@ Extracts text from OLE2 compound binary .doc files using olefile,then delegates to MarkdownParser for tree structure creation."""+import asyncioimport structfrom pathlib import Pathfrom typing import List, Optional, Union@@ -49,7 +50,7 @@ class LegacyDocParser(BaseParser):path = Path(source)if path.exists():- text = self._extract_text(path)+ text = await asyncio.to_thread(self._extract_text, path)result = await self._md_parser.parse_content(text, source_path=str(path), instruction=instruction, **kwargs)diff --git a/openviking/parse/parsers/powerpoint.py b/openviking/parse/parsers/powerpoint.pyindex c4307dc43..9ef183996 100644--- a/openviking/parse/parsers/powerpoint.py+++ b/openviking/parse/parsers/powerpoint.py@@ -7,6 +7,7 @@ Converts PowerPoint presentations to Markdown then parses using MarkdownParser.Inspired by microsoft/markitdown approach."""+import asynciofrom pathlib import Pathfrom typing import List, Optional, Union@@ -53,7 +54,7 @@ class PowerPointParser(BaseParser):if path.exists():import pptx- markdown_content = self._convert_to_markdown(path, pptx)+ markdown_content = await asyncio.to_thread(self._convert_to_markdown, path, pptx)result = await self._md_parser.parse_content(markdown_content, source_path=str(path), instruction=instruction, **kwargs)diff --git a/openviking/parse/parsers/word.py b/openviking/parse/parsers/word.pyindex dc2a757c7..044e9277c 100644--- a/openviking/parse/parsers/word.py+++ b/openviking/parse/parsers/word.py@@ -7,6 +7,7 @@ Converts Word documents to Markdown then parses using MarkdownParser.Inspired by microsoft/markitdown approach."""+import asynciofrom pathlib import Pathfrom typing import List, Optional, Union@@ -46,7 +47,7 @@ class WordParser(BaseParser):if path.exists():import docx- markdown_content = self._convert_to_markdown(path, docx)+ markdown_content = await asyncio.to_thread(self._convert_to_markdown, path, docx)result = await self._md_parser.parse_content(markdown_content, source_path=str(path), instruction=instruction, **kwargs)diff --git a/tests/parse/test_document_parser_threading.py b/tests/parse/test_document_parser_threading.pynew file mode 100644index 000000000..9b4162e73--- /dev/null+++ b/tests/parse/test_document_parser_threading.py@@ -0,0 +1,183 @@+# Copyright (c) 2026 Beijing Volcano Engine Technology Co., Ltd.+# SPDX-License-Identifier: AGPL-3.0+"""Regression tests for offloading synchronous document conversions."""++import sys+from pathlib import Path+from types import SimpleNamespace+from typing import Any, Callable++import pytest++from openviking.parse.base import NodeType, ResourceNode, create_parse_result+from openviking.parse.parsers import epub, excel, legacy_doc, powerpoint, word+++def _stub_markdown_parse(parser) -> dict[str, Any]:+ seen: dict[str, Any] = {}++ async def parse_content(+ content: str,+ source_path: str | None = None,+ instruction: str = "",+ **kwargs,+ ):+ seen["content"] = content+ seen["source_path"] = source_path+ seen["instruction"] = instruction+ seen["kwargs"] = kwargs+ return create_parse_result(+ root=ResourceNode(type=NodeType.ROOT),+ source_path=source_path,+ source_format="markdown",+ parser_name="MarkdownParser",+ )++ parser._md_parser.parse_content = parse_content+ return seen+++def _patch_to_thread(monkeypatch, module) -> list[tuple[Callable[..., Any], tuple, dict]]:+ calls: list[tuple[Callable[..., Any], tuple, dict]] = []++ async def fake_to_thread(func, /, *args, **kwargs):+ calls.append((func, args, kwargs))+ return func(*args, **kwargs)++ monkeypatch.setattr(module.asyncio, "to_thread", fake_to_thread)+ return calls+++async def test_word_parser_offloads_docx_conversion(monkeypatch, tmp_path: Path):+ parser = word.WordParser()+ seen = _stub_markdown_parse(parser)+ calls = _patch_to_thread(monkeypatch, word)+ fake_docx = SimpleNamespace()+ monkeypatch.setitem(sys.modules, "docx", fake_docx)++ def convert(path: Path, docx_module) -> str:+ assert docx_module is fake_docx+ return "# converted docx"++ monkeypatch.setattr(parser, "_convert_to_markdown", convert)+ source = tmp_path / "sample.docx"+ source.write_bytes(b"placeholder")++ result = await parser.parse(source)++ assert calls == [(convert, (source, fake_docx), {})]+ assert seen["content"] == "# converted docx"+ assert result.source_format == "docx"+ assert result.parser_name == "WordParser"+++async def test_excel_parser_offloads_xlsx_conversion(monkeypatch, tmp_path: Path):+ parser = excel.ExcelParser()+ seen = _stub_markdown_parse(parser)+ calls = _patch_to_thread(monkeypatch, excel)+ fake_openpyxl = SimpleNamespace()+ monkeypatch.setitem(sys.modules, "openpyxl", fake_openpyxl)++ def convert(path: Path, openpyxl_module) -> str:+ assert openpyxl_module is fake_openpyxl+ return "# converted xlsx"++ monkeypatch.setattr(parser, "_convert_to_markdown", convert)+ source = tmp_path / "sample.xlsx"+ source.write_bytes(b"placeholder")++ result = await parser.parse(source)++ assert calls == [(convert, (source, fake_openpyxl), {})]+ assert seen["content"] == "# converted xlsx"+ assert result.source_format == "xlsx"+ assert result.parser_name == "ExcelParser"+++async def test_excel_parser_offloads_xls_conversion(monkeypatch, tmp_path: Path):+ parser = excel.ExcelParser()+ seen = _stub_markdown_parse(parser)+ calls = _patch_to_thread(monkeypatch, excel)++ def convert(path: Path) -> str:+ return "# converted xls"++ monkeypatch.setattr(parser, "_convert_xls_to_markdown", convert)+ source = tmp_path / "sample.xls"+ source.write_bytes(b"placeholder")++ result = await parser.parse(source)++ assert calls == [(convert, (source,), {})]+ assert seen["content"] == "# converted xls"+ assert result.source_format == "xls"+ assert result.parser_name == "ExcelParser"+++async def test_powerpoint_parser_offloads_pptx_conversion(monkeypatch, tmp_path: Path):+ parser = powerpoint.PowerPointParser()+ seen = _stub_markdown_parse(parser)+ calls = _patch_to_thread(monkeypatch, powerpoint)+ fake_pptx = SimpleNamespace()+ monkeypatch.setitem(sys.modules, "pptx", fake_pptx)++ def convert(path: Path, pptx_module) -> str:+ assert pptx_module is fake_pptx+ return "# converted pptx"++ monkeypatch.setattr(parser, "_convert_to_markdown", convert)+ source = tmp_path / "sample.pptx"+ source.write_bytes(b"placeholder")++ result = await parser.parse(source)++ assert calls == [(convert, (source, fake_pptx), {})]+ assert seen["content"] == "# converted pptx"+ assert result.source_format == "pptx"+ assert result.parser_name == "PowerPointParser"+++async def test_epub_parser_offloads_epub_conversion(monkeypatch, tmp_path: Path):+ parser = epub.EPubParser()+ seen = _stub_markdown_parse(parser)+ calls = _patch_to_thread(monkeypatch, epub)++ def convert(path: Path) -> str:+ return "# converted epub"++ monkeypatch.setattr(parser, "_convert_to_markdown", convert)+ source = tmp_path / "sample.epub"+ source.write_bytes(b"placeholder")++ result = await parser.parse(source)++ assert calls == [(convert, (source,), {})]+ assert seen["content"] == "# converted epub"+ assert result.source_format == "epub"+ assert result.parser_name == "EPubParser"+++async def test_legacy_doc_parser_offloads_doc_extraction(monkeypatch, tmp_path: Path):+ parser = legacy_doc.LegacyDocParser()+ seen = _stub_markdown_parse(parser)+ calls = _patch_to_thread(monkeypatch, legacy_doc)++ def extract(path: Path) -> str:+ return "# converted doc"++ monkeypatch.setattr(parser, "_extract_text", extract)+ source = tmp_path / "sample.doc"+ source.write_bytes(b"placeholder")++ result = await parser.parse(source)++ assert calls == [(extract, (source,), {})]+ assert seen["content"] == "# converted doc"+ assert result.source_format == "doc"+ assert result.parser_name == "LegacyDocParser"