Skip to content

Self-evolving Context Database for AI Agents. Unify Agent Memory, Knowledge RAG and Skills.

fix(parser): Fix parser config propagation for markdown splitting (#1480)

Jiahui Zhou29ec5ca0c8dd
diff --git a/openviking/parse/accessors/feishu_accessor.py b/openviking/parse/accessors/feishu_accessor.py
index a1cc6e0fa..e92f8214d 100644
--- a/openviking/parse/accessors/feishu_accessor.py
+++ b/openviking/parse/accessors/feishu_accessor.py
@@ -257,7 +257,9 @@ class FeishuAccessor(DataAccessor):
host = parsed.hostname or ""
path = parsed.path
is_feishu_domain = host.endswith(".feishu.cn") or host.endswith(".larksuite.com")
- has_doc_path = any(path == f"/{t}" or path.startswith(f"/{t}/") for t in ("docx", "wiki"))
+ has_doc_path = any(
+ path == f"/{t}" or path.startswith(f"/{t}/") for t in ("docx", "wiki", "sheets", "base")
+ )
return is_feishu_domain and has_doc_path
@staticmethod
diff --git a/openviking/parse/parsers/feishu.py b/openviking/parse/parsers/feishu.py
index 3e462997b..f477509b3 100644
--- a/openviking/parse/parsers/feishu.py
+++ b/openviking/parse/parsers/feishu.py
@@ -28,6 +28,7 @@ from openviking.parse.base import (
format_table_to_markdown,
)
from openviking.parse.parsers.base_parser import BaseParser
+from openviking_cli.utils.config.parser_config import FeishuConfig
from openviking_cli.utils.logger import get_logger
logger = get_logger(__name__)
@@ -173,9 +174,10 @@ class FeishuParser(BaseParser):
# Wiki obj_type normalization (API returns short names)
_WIKI_TYPE_MAP = {"doc": "docx", "sheet": "sheets", "bitable": "base"}
- def __init__(self):
+ def __init__(self, config: Optional[FeishuConfig] = None):
self._client = None
- self._config = None
+ self._config = config
+ self._markdown_parser = None
@property
def supported_extensions(self) -> List[str]:
@@ -191,6 +193,14 @@ class FeishuParser(BaseParser):
self._config = get_openviking_config().feishu
return self._config
+ def _get_markdown_parser(self):
+ """Lazy import and create MarkdownParser with Feishu parser config."""
+ if self._markdown_parser is None:
+ from openviking.parse.parsers.markdown import MarkdownParser
+
+ self._markdown_parser = MarkdownParser(config=self._get_config())
+ return self._markdown_parser
+
def _get_client(self):
"""Lazy-init lark-oapi client."""
if self._client is None:
@@ -256,15 +266,9 @@ class FeishuParser(BaseParser):
f"Unsupported Feishu document type: {doc_type}. "
f"Supported: {list(self._DOC_TYPE_HANDLERS.keys())}"
)
- markdown, doc_title = await asyncio.to_thread(getattr(self, handler_name), token)
-
- if title:
- doc_title = title
-
- # Delegate to MarkdownParser
- from openviking.parse.parsers.markdown import MarkdownParser
+ markdown, _doc_title = await asyncio.to_thread(getattr(self, handler_name), token)
- md_parser = MarkdownParser()
+ md_parser = self._get_markdown_parser()
result = await md_parser.parse_content(
markdown, source_path=url, instruction=instruction, **kwargs
)
diff --git a/openviking/parse/parsers/html.py b/openviking/parse/parsers/html.py
index f21c8d8d0..fd249db17 100644
--- a/openviking/parse/parsers/html.py
+++ b/openviking/parse/parsers/html.py
@@ -19,6 +19,7 @@ from openviking.parse.base import (
create_parse_result,
)
from openviking.parse.parsers.base_parser import BaseParser
+from openviking_cli.utils.config.parser_config import HTMLConfig
logger = __import__("openviking_cli.utils.logger").utils.logger.get_logger(__name__)
@@ -45,7 +46,12 @@ class HTMLParser(BaseParser):
in the new two-layer architecture. This parser only handles local files.
"""
- def __init__(self, timeout: float = 30.0, **kwargs):
+ def __init__(
+ self,
+ timeout: float = 30.0,
+ config: Optional[HTMLConfig] = None,
+ **kwargs,
+ ):
"""
Initialize HTML parser.
@@ -54,7 +60,8 @@ class HTMLParser(BaseParser):
URL downloading has been moved to HTTPAccessor.
**kwargs: Additional arguments (kept for backward compatibility)
"""
- pass
+ self.config = config or HTMLConfig()
+ self._markdown_parser = None
@staticmethod
def _extract_filename_from_url(url: str) -> str:
@@ -105,6 +112,14 @@ class HTMLParser(BaseParser):
)
return self._markdownify
+ def _get_markdown_parser(self):
+ """Lazy import and create MarkdownParser with the HTML parser config."""
+ if self._markdown_parser is None:
+ from openviking.parse.parsers.markdown import MarkdownParser
+
+ self._markdown_parser = MarkdownParser(config=self.config)
+ return self._markdown_parser
+
@property
def supported_extensions(self) -> List[str]:
"""List of supported file extensions."""
@@ -219,9 +234,7 @@ class HTMLParser(BaseParser):
markdown_content = self._html_to_markdown(content, base_url=source_path or "")
# Delegate to MarkdownParser
- from openviking.parse.parsers.markdown import MarkdownParser
-
- md_parser = MarkdownParser()
+ md_parser = self._get_markdown_parser()
result = await md_parser.parse_content(markdown_content, source_path=source_path, **kwargs)
# Update metadata
diff --git a/openviking/parse/parsers/pdf.py b/openviking/parse/parsers/pdf.py
index 0372b3e0d..123332c0f 100644
--- a/openviking/parse/parsers/pdf.py
+++ b/openviking/parse/parsers/pdf.py
@@ -79,7 +79,7 @@ class PDFParser(BaseParser):
if self._markdown_parser is None:
from openviking.parse.parsers.markdown import MarkdownParser
- self._markdown_parser = MarkdownParser()
+ self._markdown_parser = MarkdownParser(config=self.config)
return self._markdown_parser
@property
diff --git a/openviking/parse/registry.py b/openviking/parse/registry.py
index d4294a6f1..cd8318a1f 100644
--- a/openviking/parse/registry.py
+++ b/openviking/parse/registry.py
@@ -28,6 +28,7 @@ from openviking.parse.parsers.powerpoint import PowerPointParser
from openviking.parse.parsers.text import TextParser
from openviking.parse.parsers.word import WordParser
from openviking.parse.parsers.zip_parser import ZipParser
+from openviking_cli.utils.config.parser_config import ParserConfig
if TYPE_CHECKING:
from openviking.parse.custom import CustomParserProtocol
@@ -42,7 +43,11 @@ class ParserRegistry:
Automatically selects appropriate parser based on file extension.
"""
- def __init__(self, register_optional: bool = True):
+ def __init__(
+ self,
+ register_optional: bool = True,
+ parser_configs: Optional[Dict[str, ParserConfig]] = None,
+ ):
"""
Initialize registry with default parsers.
@@ -53,19 +58,20 @@ class ParserRegistry:
"""
self._parsers: Dict[str, BaseParser] = {}
self._extension_map: Dict[str, str] = {}
+ self._parser_configs = parser_configs or {}
# Register core parsers
- self.register("text", TextParser())
- self.register("markdown", MarkdownParser())
- self.register("pdf", PDFParser())
- self.register("html", HTMLParser()) # HTMLParser doesn't accept config yet
+ self.register("text", TextParser(config=self._parser_configs.get("text")))
+ self.register("markdown", MarkdownParser(config=self._parser_configs.get("markdown")))
+ self.register("pdf", PDFParser(config=self._parser_configs.get("pdf")))
+ self.register("html", HTMLParser(config=self._parser_configs.get("html")))
# Register markitdown-inspired parsers (built-in)
- self.register("word", WordParser())
- self.register("legacy_doc", LegacyDocParser())
- self.register("powerpoint", PowerPointParser())
- self.register("excel", ExcelParser())
- self.register("epub", EPubParser())
+ self.register("word", WordParser(config=self._parser_configs.get("word")))
+ self.register("legacy_doc", LegacyDocParser(config=self._parser_configs.get("legacy_doc")))
+ self.register("powerpoint", PowerPointParser(config=self._parser_configs.get("powerpoint")))
+ self.register("excel", ExcelParser(config=self._parser_configs.get("excel")))
+ self.register("epub", EPubParser(config=self._parser_configs.get("epub")))
self.register("zip", ZipParser())
self.register("directory", DirectoryParser())
@@ -261,7 +267,25 @@ def get_registry() -> ParserRegistry:
"""Get the default parser registry."""
global _default_registry
if _default_registry is None:
- _default_registry = ParserRegistry()
+ parser_configs = None
+ try:
+ from openviking_cli.utils.config import get_openviking_config
+
+ config = get_openviking_config()
+ parser_configs = {
+ "text": config.text,
+ "markdown": config.markdown,
+ "pdf": config.pdf,
+ "html": config.html,
+ "word": config.markdown,
+ "legacy_doc": config.markdown,
+ "powerpoint": config.markdown,
+ "excel": config.markdown,
+ "epub": config.markdown,
+ }
+ except Exception:
+ parser_configs = None
+ _default_registry = ParserRegistry(parser_configs=parser_configs)
return _default_registry
diff --git a/openviking/utils/media_processor.py b/openviking/utils/media_processor.py
index 105f5fb77..671818a5a 100644
--- a/openviking/utils/media_processor.py
+++ b/openviking/utils/media_processor.py
@@ -193,3 +193,13 @@ class UnifiedResourceProcessor:
def _is_url(self, source: str) -> bool:
"""Check if source is a URL."""
return is_remote_resource_source(source)
+
+ @staticmethod
+ def _is_feishu_url(source: str) -> bool:
+ """Backward-compatible Feishu URL detector used by legacy tests/callers."""
+ try:
+ from openviking.parse.accessors.feishu_accessor import FeishuAccessor
+
+ return FeishuAccessor._is_feishu_url(source)
+ except Exception:
+ return False
diff --git a/tests/parse/test_parser_config_wiring.py b/tests/parse/test_parser_config_wiring.py
new file mode 100644
index 000000000..8ec9caed6
--- /dev/null
+++ b/tests/parse/test_parser_config_wiring.py
@@ -0,0 +1,84 @@
+"""Tests for parser config propagation through registries and composed parsers."""
+
+from openviking.parse.parsers.feishu import FeishuParser
+from openviking.parse.parsers.html import HTMLParser
+from openviking.parse.parsers.pdf import PDFParser
+from openviking.parse.registry import ParserRegistry
+from openviking_cli.utils.config.parser_config import (
+ FeishuConfig,
+ HTMLConfig,
+ MarkdownConfig,
+ PDFConfig,
+ TextConfig,
+)
+
+
+def test_parser_registry_propagates_parser_configs_to_registered_parsers():
+ markdown_config = MarkdownConfig(max_section_size=4321, max_section_chars=9876)
+ text_config = TextConfig(max_section_size=3456, max_section_chars=8765)
+ pdf_config = PDFConfig(
+ strategy="local",
+ max_section_size=2345,
+ max_section_chars=7654,
+ )
+ html_config = HTMLConfig(max_section_size=1234, max_section_chars=6543)
+
+ registry = ParserRegistry(
+ parser_configs={
+ "markdown": markdown_config,
+ "text": text_config,
+ "pdf": pdf_config,
+ "html": html_config,
+ }
+ )
+
+ markdown_parser = registry.get_parser("markdown")
+ assert markdown_parser is not None
+ assert markdown_parser.config.max_section_size == 4321
+ assert markdown_parser.config.max_section_chars == 9876
+
+ text_parser = registry.get_parser("text")
+ assert text_parser is not None
+ assert text_parser._md_parser.config.max_section_size == 3456
+ assert text_parser._md_parser.config.max_section_chars == 8765
+
+ pdf_parser = registry.get_parser("pdf")
+ assert pdf_parser is not None
+ assert pdf_parser.config.max_section_size == 2345
+ assert pdf_parser.config.max_section_chars == 7654
+ assert pdf_parser._get_markdown_parser().config.max_section_size == 2345
+ assert pdf_parser._get_markdown_parser().config.max_section_chars == 7654
+
+ html_parser = registry.get_parser("html")
+ assert html_parser is not None
+ assert html_parser.config.max_section_size == 1234
+ assert html_parser.config.max_section_chars == 6543
+ assert html_parser._get_markdown_parser().config.max_section_size == 1234
+ assert html_parser._get_markdown_parser().config.max_section_chars == 6543
+
+
+def test_pdf_parser_passes_its_config_to_nested_markdown_parser():
+ parser = PDFParser(PDFConfig(strategy="local", max_section_size=2222, max_section_chars=5555))
+
+ markdown_parser = parser._get_markdown_parser()
+
+ assert markdown_parser.config.max_section_size == 2222
+ assert markdown_parser.config.max_section_chars == 5555
+
+
+def test_html_parser_passes_its_config_to_nested_markdown_parser():
+ parser = HTMLParser(config=HTMLConfig(max_section_size=2111, max_section_chars=5444))
+
+ markdown_parser = parser._get_markdown_parser()
+
+ assert markdown_parser.config.max_section_size == 2111
+ assert markdown_parser.config.max_section_chars == 5444
+
+
+def test_feishu_parser_passes_its_config_to_nested_markdown_parser():
+ parser = FeishuParser(config=FeishuConfig(max_section_size=1999, max_section_chars=5333))
+
+ markdown_parser = parser._get_markdown_parser()
+
+ assert markdown_parser.config.max_section_size == 1999
+ assert markdown_parser.config.max_section_chars == 5333