Skip to content

Self-evolving Context Database for AI Agents. Unify Agent Memory, Knowledge RAG and Skills.

fix(parse): offload document conversions to thread (#2237)

Qin Haojied606b59fb811
diff --git a/openviking/parse/parsers/epub.py b/openviking/parse/parsers/epub.py
index 09fce2848..6ecc76bbc 100644
--- a/openviking/parse/parsers/epub.py
+++ b/openviking/parse/parsers/epub.py
@@ -7,6 +7,7 @@ Converts EPub e-books to Markdown then parses using MarkdownParser.
Inspired by microsoft/markitdown approach.
"""
+import asyncio
import html
import re
import zipfile
@@ -111,7 +112,7 @@ class EPubParser(BaseParser):
path = Path(source)
if path.exists():
- markdown_content = self._convert_to_markdown(path)
+ markdown_content = await asyncio.to_thread(self._convert_to_markdown, path)
result = await self._md_parser.parse_content(
markdown_content, source_path=str(path), instruction=instruction, **kwargs
)
diff --git a/openviking/parse/parsers/excel.py b/openviking/parse/parsers/excel.py
index 21e07151c..79d462d77 100644
--- a/openviking/parse/parsers/excel.py
+++ b/openviking/parse/parsers/excel.py
@@ -7,6 +7,7 @@ Converts Excel spreadsheets to Markdown then parses using MarkdownParser.
Inspired by microsoft/markitdown approach.
"""
+import asyncio
from pathlib import Path
from typing import List, Optional, Union
@@ -53,11 +54,13 @@ class ExcelParser(BaseParser):
if path.exists():
# Use xlrd for legacy .xls, openpyxl for .xlsx/.xlsm
if path.suffix.lower() == ".xls":
- markdown_content = self._convert_xls_to_markdown(path)
+ markdown_content = await asyncio.to_thread(self._convert_xls_to_markdown, path)
else:
import openpyxl
- markdown_content = self._convert_to_markdown(path, openpyxl)
+ markdown_content = await asyncio.to_thread(
+ self._convert_to_markdown, path, openpyxl
+ )
result = await self._md_parser.parse_content(
markdown_content, source_path=str(path), instruction=instruction, **kwargs
)
diff --git a/openviking/parse/parsers/legacy_doc.py b/openviking/parse/parsers/legacy_doc.py
index e30740e5d..882477e1a 100644
--- a/openviking/parse/parsers/legacy_doc.py
+++ b/openviking/parse/parsers/legacy_doc.py
@@ -7,6 +7,7 @@ Extracts text from OLE2 compound binary .doc files using olefile,
then delegates to MarkdownParser for tree structure creation.
"""
+import asyncio
import struct
from pathlib import Path
from typing import List, Optional, Union
@@ -49,7 +50,7 @@ class LegacyDocParser(BaseParser):
path = Path(source)
if path.exists():
- text = self._extract_text(path)
+ text = await asyncio.to_thread(self._extract_text, path)
result = await self._md_parser.parse_content(
text, source_path=str(path), instruction=instruction, **kwargs
)
diff --git a/openviking/parse/parsers/powerpoint.py b/openviking/parse/parsers/powerpoint.py
index c4307dc43..9ef183996 100644
--- a/openviking/parse/parsers/powerpoint.py
+++ b/openviking/parse/parsers/powerpoint.py
@@ -7,6 +7,7 @@ Converts PowerPoint presentations to Markdown then parses using MarkdownParser.
Inspired by microsoft/markitdown approach.
"""
+import asyncio
from pathlib import Path
from typing import List, Optional, Union
@@ -53,7 +54,7 @@ class PowerPointParser(BaseParser):
if path.exists():
import pptx
- markdown_content = self._convert_to_markdown(path, pptx)
+ markdown_content = await asyncio.to_thread(self._convert_to_markdown, path, pptx)
result = await self._md_parser.parse_content(
markdown_content, source_path=str(path), instruction=instruction, **kwargs
)
diff --git a/openviking/parse/parsers/word.py b/openviking/parse/parsers/word.py
index dc2a757c7..044e9277c 100644
--- a/openviking/parse/parsers/word.py
+++ b/openviking/parse/parsers/word.py
@@ -7,6 +7,7 @@ Converts Word documents to Markdown then parses using MarkdownParser.
Inspired by microsoft/markitdown approach.
"""
+import asyncio
from pathlib import Path
from typing import List, Optional, Union
@@ -46,7 +47,7 @@ class WordParser(BaseParser):
if path.exists():
import docx
- markdown_content = self._convert_to_markdown(path, docx)
+ markdown_content = await asyncio.to_thread(self._convert_to_markdown, path, docx)
result = await self._md_parser.parse_content(
markdown_content, source_path=str(path), instruction=instruction, **kwargs
)
diff --git a/tests/parse/test_document_parser_threading.py b/tests/parse/test_document_parser_threading.py
new file mode 100644
index 000000000..9b4162e73
--- /dev/null
+++ b/tests/parse/test_document_parser_threading.py
@@ -0,0 +1,183 @@
+# Copyright (c) 2026 Beijing Volcano Engine Technology Co., Ltd.
+# SPDX-License-Identifier: AGPL-3.0
+"""Regression tests for offloading synchronous document conversions."""
+
+import sys
+from pathlib import Path
+from types import SimpleNamespace
+from typing import Any, Callable
+
+import pytest
+
+from openviking.parse.base import NodeType, ResourceNode, create_parse_result
+from openviking.parse.parsers import epub, excel, legacy_doc, powerpoint, word
+
+
+def _stub_markdown_parse(parser) -> dict[str, Any]:
+ seen: dict[str, Any] = {}
+
+ async def parse_content(
+ content: str,
+ source_path: str | None = None,
+ instruction: str = "",
+ **kwargs,
+ ):
+ seen["content"] = content
+ seen["source_path"] = source_path
+ seen["instruction"] = instruction
+ seen["kwargs"] = kwargs
+ return create_parse_result(
+ root=ResourceNode(type=NodeType.ROOT),
+ source_path=source_path,
+ source_format="markdown",
+ parser_name="MarkdownParser",
+ )
+
+ parser._md_parser.parse_content = parse_content
+ return seen
+
+
+def _patch_to_thread(monkeypatch, module) -> list[tuple[Callable[..., Any], tuple, dict]]:
+ calls: list[tuple[Callable[..., Any], tuple, dict]] = []
+
+ async def fake_to_thread(func, /, *args, **kwargs):
+ calls.append((func, args, kwargs))
+ return func(*args, **kwargs)
+
+ monkeypatch.setattr(module.asyncio, "to_thread", fake_to_thread)
+ return calls
+
+
+async def test_word_parser_offloads_docx_conversion(monkeypatch, tmp_path: Path):
+ parser = word.WordParser()
+ seen = _stub_markdown_parse(parser)
+ calls = _patch_to_thread(monkeypatch, word)
+ fake_docx = SimpleNamespace()
+ monkeypatch.setitem(sys.modules, "docx", fake_docx)
+
+ def convert(path: Path, docx_module) -> str:
+ assert docx_module is fake_docx
+ return "# converted docx"
+
+ monkeypatch.setattr(parser, "_convert_to_markdown", convert)
+ source = tmp_path / "sample.docx"
+ source.write_bytes(b"placeholder")
+
+ result = await parser.parse(source)
+
+ assert calls == [(convert, (source, fake_docx), {})]
+ assert seen["content"] == "# converted docx"
+ assert result.source_format == "docx"
+ assert result.parser_name == "WordParser"
+
+
+async def test_excel_parser_offloads_xlsx_conversion(monkeypatch, tmp_path: Path):
+ parser = excel.ExcelParser()
+ seen = _stub_markdown_parse(parser)
+ calls = _patch_to_thread(monkeypatch, excel)
+ fake_openpyxl = SimpleNamespace()
+ monkeypatch.setitem(sys.modules, "openpyxl", fake_openpyxl)
+
+ def convert(path: Path, openpyxl_module) -> str:
+ assert openpyxl_module is fake_openpyxl
+ return "# converted xlsx"
+
+ monkeypatch.setattr(parser, "_convert_to_markdown", convert)
+ source = tmp_path / "sample.xlsx"
+ source.write_bytes(b"placeholder")
+
+ result = await parser.parse(source)
+
+ assert calls == [(convert, (source, fake_openpyxl), {})]
+ assert seen["content"] == "# converted xlsx"
+ assert result.source_format == "xlsx"
+ assert result.parser_name == "ExcelParser"
+
+
+async def test_excel_parser_offloads_xls_conversion(monkeypatch, tmp_path: Path):
+ parser = excel.ExcelParser()
+ seen = _stub_markdown_parse(parser)
+ calls = _patch_to_thread(monkeypatch, excel)
+
+ def convert(path: Path) -> str:
+ return "# converted xls"
+
+ monkeypatch.setattr(parser, "_convert_xls_to_markdown", convert)
+ source = tmp_path / "sample.xls"
+ source.write_bytes(b"placeholder")
+
+ result = await parser.parse(source)
+
+ assert calls == [(convert, (source,), {})]
+ assert seen["content"] == "# converted xls"
+ assert result.source_format == "xls"
+ assert result.parser_name == "ExcelParser"
+
+
+async def test_powerpoint_parser_offloads_pptx_conversion(monkeypatch, tmp_path: Path):
+ parser = powerpoint.PowerPointParser()
+ seen = _stub_markdown_parse(parser)
+ calls = _patch_to_thread(monkeypatch, powerpoint)
+ fake_pptx = SimpleNamespace()
+ monkeypatch.setitem(sys.modules, "pptx", fake_pptx)
+
+ def convert(path: Path, pptx_module) -> str:
+ assert pptx_module is fake_pptx
+ return "# converted pptx"
+
+ monkeypatch.setattr(parser, "_convert_to_markdown", convert)
+ source = tmp_path / "sample.pptx"
+ source.write_bytes(b"placeholder")
+
+ result = await parser.parse(source)
+
+ assert calls == [(convert, (source, fake_pptx), {})]
+ assert seen["content"] == "# converted pptx"
+ assert result.source_format == "pptx"
+ assert result.parser_name == "PowerPointParser"
+
+
+async def test_epub_parser_offloads_epub_conversion(monkeypatch, tmp_path: Path):
+ parser = epub.EPubParser()
+ seen = _stub_markdown_parse(parser)
+ calls = _patch_to_thread(monkeypatch, epub)
+
+ def convert(path: Path) -> str:
+ return "# converted epub"
+
+ monkeypatch.setattr(parser, "_convert_to_markdown", convert)
+ source = tmp_path / "sample.epub"
+ source.write_bytes(b"placeholder")
+
+ result = await parser.parse(source)
+
+ assert calls == [(convert, (source,), {})]
+ assert seen["content"] == "# converted epub"
+ assert result.source_format == "epub"
+ assert result.parser_name == "EPubParser"
+
+
+async def test_legacy_doc_parser_offloads_doc_extraction(monkeypatch, tmp_path: Path):
+ parser = legacy_doc.LegacyDocParser()
+ seen = _stub_markdown_parse(parser)
+ calls = _patch_to_thread(monkeypatch, legacy_doc)
+
+ def extract(path: Path) -> str:
+ return "# converted doc"
+
+ monkeypatch.setattr(parser, "_extract_text", extract)
+ source = tmp_path / "sample.doc"
+ source.write_bytes(b"placeholder")
+
+ result = await parser.parse(source)
+
+ assert calls == [(extract, (source,), {})]
+ assert seen["content"] == "# converted doc"
+ assert result.source_format == "doc"
+ assert result.parser_name == "LegacyDocParser"