Self-evolving Context Database for AI Agents. Unify Agent Memory, Knowledge RAG and Skills.
Python agent-contextagent-memoryagent-pluginsagentic-ragcontext-databasedsh-pluginknowledge-basememorymemory-managementself-evolvingvector-database
fix(parser): Fix parser config propagation for markdown splitting (#1480)
Jiahui Zhou29ec5ca0c8dd
diff --git a/openviking/parse/accessors/feishu_accessor.py b/openviking/parse/accessors/feishu_accessor.pyindex a1cc6e0fa..e92f8214d 100644--- a/openviking/parse/accessors/feishu_accessor.py+++ b/openviking/parse/accessors/feishu_accessor.py@@ -257,7 +257,9 @@ class FeishuAccessor(DataAccessor):host = parsed.hostname or ""path = parsed.pathis_feishu_domain = host.endswith(".feishu.cn") or host.endswith(".larksuite.com")- has_doc_path = any(path == f"/{t}" or path.startswith(f"/{t}/") for t in ("docx", "wiki"))+ has_doc_path = any(+ path == f"/{t}" or path.startswith(f"/{t}/") for t in ("docx", "wiki", "sheets", "base")+ )return is_feishu_domain and has_doc_path@staticmethoddiff --git a/openviking/parse/parsers/feishu.py b/openviking/parse/parsers/feishu.pyindex 3e462997b..f477509b3 100644--- a/openviking/parse/parsers/feishu.py+++ b/openviking/parse/parsers/feishu.py@@ -28,6 +28,7 @@ from openviking.parse.base import (format_table_to_markdown,)from openviking.parse.parsers.base_parser import BaseParser+from openviking_cli.utils.config.parser_config import FeishuConfigfrom openviking_cli.utils.logger import get_loggerlogger = get_logger(__name__)@@ -173,9 +174,10 @@ class FeishuParser(BaseParser):# Wiki obj_type normalization (API returns short names)_WIKI_TYPE_MAP = {"doc": "docx", "sheet": "sheets", "bitable": "base"}- def __init__(self):+ def __init__(self, config: Optional[FeishuConfig] = None):self._client = None- self._config = None+ self._config = config+ self._markdown_parser = None@propertydef supported_extensions(self) -> List[str]:@@ -191,6 +193,14 @@ class FeishuParser(BaseParser):self._config = get_openviking_config().feishureturn self._config+ def _get_markdown_parser(self):+ """Lazy import and create MarkdownParser with Feishu parser config."""+ if self._markdown_parser is None:+ from openviking.parse.parsers.markdown import MarkdownParser++ self._markdown_parser = MarkdownParser(config=self._get_config())+ return self._markdown_parser+def _get_client(self):"""Lazy-init lark-oapi client."""if self._client is None:@@ -256,15 +266,9 @@ class FeishuParser(BaseParser):f"Unsupported Feishu document type: {doc_type}. "f"Supported: {list(self._DOC_TYPE_HANDLERS.keys())}")- markdown, doc_title = await asyncio.to_thread(getattr(self, handler_name), token)-- if title:- doc_title = title-- # Delegate to MarkdownParser- from openviking.parse.parsers.markdown import MarkdownParser+ markdown, _doc_title = await asyncio.to_thread(getattr(self, handler_name), token)- md_parser = MarkdownParser()+ md_parser = self._get_markdown_parser()result = await md_parser.parse_content(markdown, source_path=url, instruction=instruction, **kwargs)diff --git a/openviking/parse/parsers/html.py b/openviking/parse/parsers/html.pyindex f21c8d8d0..fd249db17 100644--- a/openviking/parse/parsers/html.py+++ b/openviking/parse/parsers/html.py@@ -19,6 +19,7 @@ from openviking.parse.base import (create_parse_result,)from openviking.parse.parsers.base_parser import BaseParser+from openviking_cli.utils.config.parser_config import HTMLConfiglogger = __import__("openviking_cli.utils.logger").utils.logger.get_logger(__name__)@@ -45,7 +46,12 @@ class HTMLParser(BaseParser):in the new two-layer architecture. This parser only handles local files."""- def __init__(self, timeout: float = 30.0, **kwargs):+ def __init__(+ self,+ timeout: float = 30.0,+ config: Optional[HTMLConfig] = None,+ **kwargs,+ ):"""Initialize HTML parser.@@ -54,7 +60,8 @@ class HTMLParser(BaseParser):URL downloading has been moved to HTTPAccessor.**kwargs: Additional arguments (kept for backward compatibility)"""- pass+ self.config = config or HTMLConfig()+ self._markdown_parser = None@staticmethoddef _extract_filename_from_url(url: str) -> str:@@ -105,6 +112,14 @@ class HTMLParser(BaseParser):)return self._markdownify+ def _get_markdown_parser(self):+ """Lazy import and create MarkdownParser with the HTML parser config."""+ if self._markdown_parser is None:+ from openviking.parse.parsers.markdown import MarkdownParser++ self._markdown_parser = MarkdownParser(config=self.config)+ return self._markdown_parser+@propertydef supported_extensions(self) -> List[str]:"""List of supported file extensions."""@@ -219,9 +234,7 @@ class HTMLParser(BaseParser):markdown_content = self._html_to_markdown(content, base_url=source_path or "")# Delegate to MarkdownParser- from openviking.parse.parsers.markdown import MarkdownParser-- md_parser = MarkdownParser()+ md_parser = self._get_markdown_parser()result = await md_parser.parse_content(markdown_content, source_path=source_path, **kwargs)# Update metadatadiff --git a/openviking/parse/parsers/pdf.py b/openviking/parse/parsers/pdf.pyindex 0372b3e0d..123332c0f 100644--- a/openviking/parse/parsers/pdf.py+++ b/openviking/parse/parsers/pdf.py@@ -79,7 +79,7 @@ class PDFParser(BaseParser):if self._markdown_parser is None:from openviking.parse.parsers.markdown import MarkdownParser- self._markdown_parser = MarkdownParser()+ self._markdown_parser = MarkdownParser(config=self.config)return self._markdown_parser@propertydiff --git a/openviking/parse/registry.py b/openviking/parse/registry.pyindex d4294a6f1..cd8318a1f 100644--- a/openviking/parse/registry.py+++ b/openviking/parse/registry.py@@ -28,6 +28,7 @@ from openviking.parse.parsers.powerpoint import PowerPointParserfrom openviking.parse.parsers.text import TextParserfrom openviking.parse.parsers.word import WordParserfrom openviking.parse.parsers.zip_parser import ZipParser+from openviking_cli.utils.config.parser_config import ParserConfigif TYPE_CHECKING:from openviking.parse.custom import CustomParserProtocol@@ -42,7 +43,11 @@ class ParserRegistry:Automatically selects appropriate parser based on file extension."""- def __init__(self, register_optional: bool = True):+ def __init__(+ self,+ register_optional: bool = True,+ parser_configs: Optional[Dict[str, ParserConfig]] = None,+ ):"""Initialize registry with default parsers.@@ -53,19 +58,20 @@ class ParserRegistry:"""self._parsers: Dict[str, BaseParser] = {}self._extension_map: Dict[str, str] = {}+ self._parser_configs = parser_configs or {}# Register core parsers- self.register("text", TextParser())- self.register("markdown", MarkdownParser())- self.register("pdf", PDFParser())- self.register("html", HTMLParser()) # HTMLParser doesn't accept config yet+ self.register("text", TextParser(config=self._parser_configs.get("text")))+ self.register("markdown", MarkdownParser(config=self._parser_configs.get("markdown")))+ self.register("pdf", PDFParser(config=self._parser_configs.get("pdf")))+ self.register("html", HTMLParser(config=self._parser_configs.get("html")))# Register markitdown-inspired parsers (built-in)- self.register("word", WordParser())- self.register("legacy_doc", LegacyDocParser())- self.register("powerpoint", PowerPointParser())- self.register("excel", ExcelParser())- self.register("epub", EPubParser())+ self.register("word", WordParser(config=self._parser_configs.get("word")))+ self.register("legacy_doc", LegacyDocParser(config=self._parser_configs.get("legacy_doc")))+ self.register("powerpoint", PowerPointParser(config=self._parser_configs.get("powerpoint")))+ self.register("excel", ExcelParser(config=self._parser_configs.get("excel")))+ self.register("epub", EPubParser(config=self._parser_configs.get("epub")))self.register("zip", ZipParser())self.register("directory", DirectoryParser())@@ -261,7 +267,25 @@ def get_registry() -> ParserRegistry:"""Get the default parser registry."""global _default_registryif _default_registry is None:- _default_registry = ParserRegistry()+ parser_configs = None+ try:+ from openviking_cli.utils.config import get_openviking_config++ config = get_openviking_config()+ parser_configs = {+ "text": config.text,+ "markdown": config.markdown,+ "pdf": config.pdf,+ "html": config.html,+ "word": config.markdown,+ "legacy_doc": config.markdown,+ "powerpoint": config.markdown,+ "excel": config.markdown,+ "epub": config.markdown,+ }+ except Exception:+ parser_configs = None+ _default_registry = ParserRegistry(parser_configs=parser_configs)return _default_registrydiff --git a/openviking/utils/media_processor.py b/openviking/utils/media_processor.pyindex 105f5fb77..671818a5a 100644--- a/openviking/utils/media_processor.py+++ b/openviking/utils/media_processor.py@@ -193,3 +193,13 @@ class UnifiedResourceProcessor:def _is_url(self, source: str) -> bool:"""Check if source is a URL."""return is_remote_resource_source(source)++ @staticmethod+ def _is_feishu_url(source: str) -> bool:+ """Backward-compatible Feishu URL detector used by legacy tests/callers."""+ try:+ from openviking.parse.accessors.feishu_accessor import FeishuAccessor++ return FeishuAccessor._is_feishu_url(source)+ except Exception:+ return Falsediff --git a/tests/parse/test_parser_config_wiring.py b/tests/parse/test_parser_config_wiring.pynew file mode 100644index 000000000..8ec9caed6--- /dev/null+++ b/tests/parse/test_parser_config_wiring.py@@ -0,0 +1,84 @@+"""Tests for parser config propagation through registries and composed parsers."""++from openviking.parse.parsers.feishu import FeishuParser+from openviking.parse.parsers.html import HTMLParser+from openviking.parse.parsers.pdf import PDFParser+from openviking.parse.registry import ParserRegistry+from openviking_cli.utils.config.parser_config import (+ FeishuConfig,+ HTMLConfig,+ MarkdownConfig,+ PDFConfig,+ TextConfig,+)+++def test_parser_registry_propagates_parser_configs_to_registered_parsers():+ markdown_config = MarkdownConfig(max_section_size=4321, max_section_chars=9876)+ text_config = TextConfig(max_section_size=3456, max_section_chars=8765)+ pdf_config = PDFConfig(+ strategy="local",+ max_section_size=2345,+ max_section_chars=7654,+ )+ html_config = HTMLConfig(max_section_size=1234, max_section_chars=6543)++ registry = ParserRegistry(+ parser_configs={+ "markdown": markdown_config,+ "text": text_config,+ "pdf": pdf_config,+ "html": html_config,+ }+ )++ markdown_parser = registry.get_parser("markdown")+ assert markdown_parser is not None+ assert markdown_parser.config.max_section_size == 4321+ assert markdown_parser.config.max_section_chars == 9876++ text_parser = registry.get_parser("text")+ assert text_parser is not None+ assert text_parser._md_parser.config.max_section_size == 3456+ assert text_parser._md_parser.config.max_section_chars == 8765++ pdf_parser = registry.get_parser("pdf")+ assert pdf_parser is not None+ assert pdf_parser.config.max_section_size == 2345+ assert pdf_parser.config.max_section_chars == 7654+ assert pdf_parser._get_markdown_parser().config.max_section_size == 2345+ assert pdf_parser._get_markdown_parser().config.max_section_chars == 7654++ html_parser = registry.get_parser("html")+ assert html_parser is not None+ assert html_parser.config.max_section_size == 1234+ assert html_parser.config.max_section_chars == 6543+ assert html_parser._get_markdown_parser().config.max_section_size == 1234+ assert html_parser._get_markdown_parser().config.max_section_chars == 6543+++def test_pdf_parser_passes_its_config_to_nested_markdown_parser():+ parser = PDFParser(PDFConfig(strategy="local", max_section_size=2222, max_section_chars=5555))++ markdown_parser = parser._get_markdown_parser()++ assert markdown_parser.config.max_section_size == 2222+ assert markdown_parser.config.max_section_chars == 5555+++def test_html_parser_passes_its_config_to_nested_markdown_parser():+ parser = HTMLParser(config=HTMLConfig(max_section_size=2111, max_section_chars=5444))++ markdown_parser = parser._get_markdown_parser()++ assert markdown_parser.config.max_section_size == 2111+ assert markdown_parser.config.max_section_chars == 5444+++def test_feishu_parser_passes_its_config_to_nested_markdown_parser():+ parser = FeishuParser(config=FeishuConfig(max_section_size=1999, max_section_chars=5333))++ markdown_parser = parser._get_markdown_parser()++ assert markdown_parser.config.max_section_size == 1999+ assert markdown_parser.config.max_section_chars == 5333