Self-evolving Context Database for AI Agents. Unify Agent Memory, Knowledge RAG and Skills.
Python agent-contextagent-memoryagent-pluginsagentic-ragcontext-databasedsh-pluginknowledge-basememorymemory-managementself-evolvingvector-database
# Copyright (c) 2026 Beijing Volcano Engine Technology Co., Ltd.
# SPDX-License-Identifier: AGPL-3.0
"""
Unified parser configuration management for OpenViking.
This module consolidates all parser configuration classes that were previously
scattered across different modules. All configurations inherit from ParserConfig
and can be loaded from ov.conf files.
"""
from dataclasses import dataclass
from pathlib import Path
from typing import Any, Dict, Optional, Union
from openviking_cli.utils.logger import get_logger
logger = get_logger(__name__)
@dataclass
class ParserConfig:
"""
Base configuration class for all parsers.
This serves as a foundation for parser-specific configurations,
providing common fields and utilities for all parsers.
Attributes:
enabled: Whether the parser is enabled
max_content_length: Maximum content length to process (characters)
encoding: Default file encoding
max_section_size: Legacy option, ignored by native Markdown splitting
section_size_flexibility: Allow overflow to maintain coherence (0.0-1.0)
max_section_chars: Target character limit per section. A single oversized
Markdown table row may remain intact to preserve table semantics.
"""
enabled: bool = True
max_content_length: int = 100000
encoding: str = "utf-8"
# Smart splitting configuration
max_section_size: int = 2048 # Legacy compatibility; splitting uses max_section_chars
section_size_flexibility: float = 0.3 # Allow 30% overflow to maintain coherence
max_section_chars: int = 2000 # Target character limit per section
@classmethod
def from_dict(cls, data: Dict[str, Any]) -> "ParserConfig":
"""
Create configuration from dictionary.
Args:
data: Configuration dictionary
Returns:
ParserConfig instance
Examples:
>>> config = ParserConfig.from_dict({"max_content_length": 50000})
"""
return cls(**{key: value for key, value in data.items() if key in cls.__dataclass_fields__})
@classmethod
def from_yaml(cls, yaml_path: Union[str, Path]) -> "ParserConfig":
"""
Load configuration from YAML file.
Args:
yaml_path: Path to YAML configuration file
Returns:
ParserConfig instance
Raises:
FileNotFoundError: If YAML file doesn't exist
ValueError: If YAML is invalid
Examples:
>>> config = ParserConfig.from_yaml("config.yaml")
"""
import yaml
yaml_path = Path(yaml_path)
if not yaml_path.exists():
raise FileNotFoundError(f"Configuration file not found: {yaml_path}")
with open(yaml_path, "r", encoding="utf-8") as f:
data = yaml.safe_load(f) or {}
return cls.from_dict(data)
def validate(self) -> None:
"""
Validate configuration.
Raises:
ValueError: If configuration is invalid
"""
if self.max_content_length <= 0:
raise ValueError("max_content_length must be positive")
if not self.encoding:
raise ValueError("encoding cannot be empty")
if self.max_section_size <= 0:
raise ValueError("max_section_size must be positive")
if not 0.0 <= self.section_size_flexibility <= 1.0:
raise ValueError("section_size_flexibility must be between 0.0 and 1.0")
if self.max_section_chars <= 0:
raise ValueError("max_section_chars must be positive")
def to_dict(self) -> Dict[str, Any]:
"""
Export configuration as dictionary.
Returns:
Configuration dictionary
Examples:
>>> config = ParserConfig()
>>> data = config.to_dict()
"""
from dataclasses import asdict
return asdict(self)
@dataclass
class PDFConfig(ParserConfig):
"""
Configuration for PDF parsing.
Supports three strategies:
- "local": Use pdfplumber for local PDF→Markdown conversion
- "mineru": Use MinerU API for remote PDF→Markdown conversion
- "auto": Try local first, fallback to MinerU if available
Attributes:
strategy: Parsing strategy ("local" | "mineru" | "auto")
mineru_endpoint: MinerU API endpoint URL
mineru_timeout: MinerU request timeout in seconds
mineru_bodys: Additional MinerU API multipart form fields
"""
strategy: str = "auto" # "local" | "mineru" | "auto"
# MinerU API configuration
mineru_endpoint: Optional[str] = None # API endpoint URL
mineru_timeout: float = 300.0 # Request timeout in seconds (5 minutes)
mineru_bodys: Optional[dict] = None # Additional API multipart form fields
# Heading detection configuration
heading_detection: str = "auto" # "bookmarks" | "font" | "auto" | "none"
font_heading_min_delta: float = 1.5 # Minimum font size delta from body text (pt)
max_heading_levels: int = 4 # Maximum heading levels for font analysis
# Image extraction configuration
image_resolution: int = 300 # Rendering DPI for extracted image regions
def validate(self) -> None:
"""
Validate configuration.
Raises:
ValueError: If configuration is invalid
"""
# Validate base class fields
super().validate()
# Validate PDF-specific fields
if self.strategy not in ("local", "mineru", "auto"):
raise ValueError(
f"Invalid strategy '{self.strategy}'. Must be 'local', 'mineru', or 'auto'"
)
if self.strategy == "mineru":
if not self.mineru_endpoint:
raise ValueError("mineru_endpoint is required when strategy='mineru'")
if self.mineru_timeout <= 0:
raise ValueError("mineru_timeout must be positive")
if self.heading_detection not in ("bookmarks", "font", "auto", "none"):
raise ValueError(f"Invalid heading_detection: {self.heading_detection}")
if self.font_heading_min_delta <= 0:
raise ValueError("font_heading_min_delta must be positive")
@dataclass
class CodeHostingConfig(ParserConfig):
"""
Base configuration for code hosting platform domains.
Attributes:
code_hosting_domains: List of allowed generic code hosting domains
github_domains: List of GitHub domains (github.com, www.github.com)
gitlab_domains: List of GitLab domains (gitlab.com, www.gitlab.com)
azure_devops_domains: List of Azure DevOps domains (dev.azure.com, ssh.dev.azure.com)
"""
# Code hosting platform configuration
code_hosting_domains: list = None
github_domains: list = None
gitlab_domains: list = None
azure_devops_domains: list = None
def __post_init__(self):
"""Initialize default values for mutable fields."""
if self.code_hosting_domains is None:
self.code_hosting_domains = [
"github.com",
"gitlab.com",
"gitcode.com",
"gitee.com",
"bitbucket.org",
"codeberg.org",
"gitea.com",
"atomgit.com",
"git.sr.ht",
]
if self.github_domains is None:
self.github_domains = ["github.com", "www.github.com"]
if self.gitlab_domains is None:
self.gitlab_domains = ["gitlab.com", "www.gitlab.com"]
if self.azure_devops_domains is None:
self.azure_devops_domains = [
"dev.azure.com",
"ssh.dev.azure.com",
"vs-ssh.visualstudio.com",
]
@dataclass
class CodeConfig(CodeHostingConfig):
"""
Configuration for code parsing.
Attributes:
extract_functions: Legacy compatibility field; ignored by the fixed skeleton route
extract_classes: Legacy compatibility field; ignored by the fixed skeleton route
extract_imports: Legacy compatibility field; ignored by the fixed skeleton route
include_comments: Legacy compatibility field; ignored by the fixed skeleton route
max_line_length: Legacy compatibility field; ignored by the fixed skeleton route
language_hint: Legacy compatibility field; ignored by the fixed skeleton route
max_token_limit: Legacy compatibility field; ignored by the fixed skeleton route
truncation_strategy: Legacy compatibility field; ignored by the fixed skeleton route
warn_on_truncation: Legacy compatibility field; ignored by the fixed skeleton route
github_raw_domain: Domain for GitHub raw content (raw.githubusercontent.com)
"""
extract_functions: bool = True
extract_classes: bool = True
extract_imports: bool = True
include_comments: bool = True
max_line_length: int = 1000
language_hint: Optional[str] = None
max_token_limit: int = 50000 # Maximum tokens to process per file
truncation_strategy: str = "head" # "head", "tail", or "balanced"
warn_on_truncation: bool = True
github_raw_domain: str = "raw.githubusercontent.com"
@classmethod
def from_dict(cls, data: Dict[str, Any]) -> "CodeConfig":
"""Create code configuration, accepting removed fields for upgrade compatibility."""
data = dict(data)
if "code_summary_mode" in data:
data.pop("code_summary_mode", None)
logger.warning(
"code.code_summary_mode is deprecated and ignored; "
"code summaries now always use the fixed skeleton route with LLM fallback"
)
return super().from_dict(data)
def validate(self) -> None:
"""
Validate configuration.
Raises:
ValueError: If configuration is invalid
"""
# Validate base class fields
super().validate()
# Validate code-specific fields
if self.max_line_length <= 0:
raise ValueError("max_line_length must be positive")
if self.max_token_limit <= 0:
raise ValueError("max_token_limit must be positive")
if self.truncation_strategy not in ("head", "tail", "balanced"):
raise ValueError(
f"Invalid truncation_strategy '{self.truncation_strategy}'. "
"Must be 'head', 'tail', or 'balanced'"
)
@dataclass
class ImageConfig(ParserConfig):
"""
Configuration for image parsing.
Attributes:
enable_ocr: Whether to perform OCR text extraction, not implemented
enable_vlm: Whether to use VLM for visual understanding
ocr_lang: Language for OCR (e.g., "chi_sim", "eng")
vlm_model: VLM model to use (e.g., "gpt-4-vision")
preview_max_dimension: Maximum dimension for preview resizing (resize if larger)
max_file_size_mb: Maximum file size before triggering large image processing
max_tile_dimension_px: Maximum dimension for individual tiles
tile_overlap_px: Number of pixels to overlap between tiles
large_image_threshold_dimension: Dimension threshold for large image detection
"""
enable_ocr: bool = False
enable_vlm: bool = True
ocr_lang: str = "eng"
vlm_model: Optional[str] = None
preview_max_dimension: int = 2048
# Large image processing settings
max_file_size_mb: float = 10.0 # 10 MB
max_tile_dimension_px: int = 2048 # 2048 pixels
tile_overlap_px: int = 2 # 2 pixels
large_image_threshold_dimension: int = 4096 # 4096 pixels
def validate(self) -> None:
"""
Validate configuration.
Raises:
ValueError: If configuration is invalid
"""
# Validate base class fields
super().validate()
# Validate image-specific fields
if self.preview_max_dimension <= 0:
raise ValueError("preview_max_dimension must be positive")
if self.max_file_size_mb <= 0:
raise ValueError("max_file_size_mb must be positive")
if self.max_tile_dimension_px <= 0:
raise ValueError("max_tile_dimension_px must be positive")
if self.tile_overlap_px < 0:
raise ValueError("tile_overlap_px must be non-negative")
if self.large_image_threshold_dimension <= 0:
raise ValueError("large_image_threshold_dimension must be positive")
@dataclass
class AudioConfig(ParserConfig):
"""
Configuration for audio parsing.
Attributes:
enable_transcription: Whether to transcribe speech to text
transcription_model: Model to use (e.g., "whisper-large-v3")
language: Audio language (None for auto-detection)
extract_metadata: Whether to extract audio metadata
"""
enable_transcription: bool = True
transcription_model: str = "whisper-large-v3"
language: Optional[str] = None
extract_metadata: bool = True
def validate(self) -> None:
"""
Validate configuration.
Raises:
ValueError: If configuration is invalid
"""
# Validate base class fields
super().validate()
# Validate audio-specific fields
if not self.transcription_model:
raise ValueError("transcription_model cannot be empty")
@dataclass
class VideoConfig(ParserConfig):
"""
Configuration for video parsing.
Attributes:
extract_frames: Whether to extract key frames
frame_interval: Seconds between frame extraction
enable_transcription: Whether to transcribe audio track
enable_vlm_description: Whether to use VLM for scene description
max_duration: Maximum video duration to process (seconds)
"""
extract_frames: bool = True
frame_interval: float = 10.0
enable_transcription: bool = True
enable_vlm_description: bool = False
max_duration: float = 3600.0 # 1 hour
def validate(self) -> None:
"""
Validate configuration.
Raises:
ValueError: If configuration is invalid
"""
# Validate base class fields
super().validate()
# Validate video-specific fields
if self.frame_interval <= 0:
raise ValueError("frame_interval must be positive")
if self.max_duration <= 0:
raise ValueError("max_duration must be positive")
@dataclass
class MarkdownConfig(ParserConfig):
"""
Configuration for Markdown parsing.
Attributes:
preserve_links: Whether to preserve hyperlinks in output
extract_frontmatter: Whether to REMOVE YAML frontmatter from the stored
document body. Frontmatter is parsed into the parse result metadata
regardless. Off by default: the parsed metadata is never persisted, so
removing the block would silently lose those fields.
include_metadata: Whether to include file metadata
max_heading_depth: Maximum heading depth to include in structure
"""
preserve_links: bool = True
extract_frontmatter: bool = False
include_metadata: bool = True
max_heading_depth: int = 3
def validate(self) -> None:
"""
Validate configuration.
Raises:
ValueError: If configuration is invalid
"""
# Validate base class fields
super().validate()
# Validate markdown-specific fields
if self.max_heading_depth < 1:
raise ValueError("max_heading_depth must be at least 1")
@dataclass
class AnydocConfig(ParserConfig):
"""Configuration for the shared anydoc Office converter."""
max_table_rows: int = 1000
def validate(self) -> None:
super().validate()
if self.max_table_rows < 0:
raise ValueError("max_table_rows must be non-negative")
@dataclass
class HTMLConfig(ParserConfig):
"""
Configuration for HTML parsing.
Attributes:
extract_text_only: Whether to extract only text content
preserve_structure: Whether to preserve HTML structure
clean_html: Whether to clean HTML tags and attributes
extract_metadata: Whether to extract metadata (title, description)
"""
extract_text_only: bool = False
preserve_structure: bool = True
clean_html: bool = True
extract_metadata: bool = True
def validate(self) -> None:
"""
Validate configuration.
Raises:
ValueError: If configuration is invalid
"""
# Validate base class fields
super().validate()
# No additional validation needed for HTML config
@dataclass
class TextConfig(ParserConfig):
"""
Configuration for plain text parsing.
Attributes:
detect_language: Whether to detect language automatically
split_by_paragraphs: Whether to split by paragraphs
max_paragraph_length: Maximum paragraph length before splitting
preserve_line_breaks: Whether to preserve original line breaks
"""
detect_language: bool = True
split_by_paragraphs: bool = True
max_paragraph_length: int = 1000
preserve_line_breaks: bool = False
def validate(self) -> None:
"""
Validate configuration.
Raises:
ValueError: If configuration is invalid
"""
# Validate base class fields
super().validate()
# Validate text-specific fields
if self.max_paragraph_length <= 0:
raise ValueError("max_paragraph_length must be positive")
@dataclass
class FeishuConfig(ParserConfig):
"""
Configuration for Feishu/Lark document parsing.
Attributes:
app_id: Feishu app ID (can also be set via FEISHU_APP_ID env var)
app_secret: Feishu app secret (can also be set via FEISHU_APP_SECRET env var)
domain: Feishu API domain
max_rows_per_sheet: Maximum rows per sheet for spreadsheets
max_records_per_table: Maximum records per table for bitable
download_images: Whether to download images from documents
request_timeout: HTTP request timeout in seconds
"""
app_id: str = ""
app_secret: str = ""
domain: str = "https://open.feishu.cn"
max_rows_per_sheet: int = 1000
max_records_per_table: int = 1000
download_images: bool = True
request_timeout: float = (
30.0 # TODO: not yet passed to lark-oapi client, reserved for future use
)
def __post_init__(self) -> None:
self.validate()
def validate(self) -> None:
"""
Validate configuration.
Raises:
ValueError: If configuration is invalid
"""
super().validate()
if not isinstance(self.domain, str) or not self.domain.strip():
raise ValueError("domain cannot be empty")
self.domain = self.domain.strip()
if (
isinstance(self.max_rows_per_sheet, bool)
or not isinstance(self.max_rows_per_sheet, int)
or self.max_rows_per_sheet <= 0
):
raise ValueError("max_rows_per_sheet must be positive")
if (
isinstance(self.max_records_per_table, bool)
or not isinstance(self.max_records_per_table, int)
or self.max_records_per_table <= 0
):
raise ValueError("max_records_per_table must be positive")
if (
isinstance(self.request_timeout, bool)
or not isinstance(self.request_timeout, (int, float))
or self.request_timeout <= 0
):
raise ValueError("request_timeout must be positive")
@dataclass
class DirectoryConfig(ParserConfig):
"""
Configuration for directory parsing.
Attributes:
preserve_structure: Whether to preserve nested directory structure when
adding directory resources. When True (default), files maintain their
relative path hierarchy. When False, all files are flattened to a
single level under the resource root.
max_files: Optional maximum number of selected files admitted by one
Understanding or Feishu directory import. None (default) means unlimited.
max_depth: Maximum nested directory depth below an Understanding directory
import root.
max_concurrent: Maximum concurrent Understanding jobs shared by all
directory imports in one service event loop.
"""
preserve_structure: bool = True
max_files: Optional[int] = None
max_depth: int = 10
max_concurrent: int = 4
def validate(self) -> None:
"""Validate directory resource and concurrency limits."""
super().validate()
for name in (
"max_files",
"max_depth",
"max_concurrent",
):
value = getattr(self, name)
if name == "max_files" and value is None:
continue
if isinstance(value, bool) or not isinstance(value, int) or value <= 0:
raise ValueError(f"{name} must be a positive integer")
def __post_init__(self) -> None:
self.validate()
@dataclass
class WebFeedConfig(ParserConfig):
"""
Configuration for whole-site ingestion via sitemap / RSS / Atom feeds.
Used by WebFeedAccessor (and its single-page detect-and-suggest helper).
Each setting can be overridden per call via add_resource ``args`` (e.g.
``args={"max_pages": 50}``).
Attributes:
max_pages: Hard cap on the number of pages mirrored per site.
max_concurrency: Max concurrent page fetches.
request_timeout: Per-request timeout in seconds.
politeness_delay: Delay (seconds) before each page fetch, to be polite.
same_host_only: Only ingest URLs on the same host as the feed.
respect_robots: Honor robots.txt Disallow rules (and discover sitemaps).
max_depth: Max recursion depth when following <sitemapindex> entries.
suggest_feed: When adding a single webpage, probe for a sitemap/RSS and
append a one-line hint suggesting whole-site ingestion (never auto-crawls).
suggest_timeout: Hard timeout (seconds) for that single-page probe.
"""
max_pages: int = 200
max_concurrency: int = 5
request_timeout: float = 30.0
politeness_delay: float = 0.2
same_host_only: bool = True
respect_robots: bool = True
max_depth: int = 2
suggest_feed: bool = True
suggest_timeout: float = 2.5
def validate(self) -> None:
"""Validate web feed configuration."""
super().validate()
if self.max_pages <= 0:
raise ValueError("max_pages must be positive")
if self.max_concurrency <= 0:
raise ValueError("max_concurrency must be positive")
if self.request_timeout <= 0:
raise ValueError("request_timeout must be positive")
if self.politeness_delay < 0:
raise ValueError("politeness_delay must be non-negative")
if self.max_depth <= 0:
raise ValueError("max_depth must be positive")
if self.suggest_timeout <= 0:
raise ValueError("suggest_timeout must be positive")
@dataclass
class SemanticConfig:
"""
Configuration for semantic processing (overview/abstract generation).
Controls prompt budget limits and output size constraints for the
SemanticProcessor pipeline.
"""
max_file_content_chars: int = 30000
"""Maximum characters of file content sent to LLM for summary generation."""
max_skeleton_chars: int = 12000
"""Maximum characters of code skeleton used for embedding (~3000 tokens)."""
max_overview_prompt_chars: int = 60000
"""Maximum characters allowed in the overview generation prompt.
If exceeded, file summaries are batched and merged."""
overview_batch_size: int = 50
"""Maximum number of file summaries per batch when splitting oversized prompts."""
overview_sample_limit: int = 32
"""Small-directory threshold for freshness scheduling; not a summary count cap."""
freshness_refresh_ratio: float = 0.10
"""Pending direct-child change ratio that refreshes a wide directory."""
abstract_max_chars: int = 256
"""Maximum characters for generated abstracts."""
overview_max_chars: int = 4000
"""Maximum characters for generated overviews."""
memory_chunk_chars: int = 2000
"""Legacy setting retained for configuration compatibility; memory vectors are not chunked."""
memory_chunk_overlap: int = 200
"""Legacy setting retained for configuration compatibility; unused by memory vectorization."""
def __post_init__(self):
if self.overview_sample_limit <= 0:
raise ValueError("overview_sample_limit must be positive")
if not 0 < self.freshness_refresh_ratio <= 1:
raise ValueError("freshness_refresh_ratio must be in the range (0, 1]")
if self.memory_chunk_chars <= 0:
raise ValueError("memory_chunk_chars must be positive")
if self.memory_chunk_overlap < 0:
raise ValueError("memory_chunk_overlap must be non-negative")
if self.memory_chunk_overlap >= self.memory_chunk_chars:
raise ValueError("memory_chunk_overlap must be smaller than memory_chunk_chars")
# Configuration registry for dynamic loading
PARSER_CONFIG_REGISTRY = {
"pdf": PDFConfig,
"code": CodeConfig,
"image": ImageConfig,
"audio": AudioConfig,
"video": VideoConfig,
"markdown": MarkdownConfig,
"anydoc": AnydocConfig,
"html": HTMLConfig,
"text": TextConfig,
"directory": DirectoryConfig,
"feishu": FeishuConfig,
"webfeed": WebFeedConfig,
}
def get_parser_config(
parser_type: str, config_data: Optional[Dict[str, Any]] = None
) -> ParserConfig:
"""
Get parser configuration for a specific parser type.
Args:
parser_type: Type of parser (e.g., "pdf", "code", "image")
config_data: Optional configuration data dictionary
Returns:
ParserConfig instance for the specified parser type
Raises:
ValueError: If parser_type is not supported
Examples:
>>> # Get default PDF configuration
>>> pdf_config = get_parser_config("pdf")
>>> # Get custom code configuration
>>> code_config = get_parser_config("code", {
... "github_raw_domain": "raw.githubusercontent.com"
... })
"""
if parser_type not in PARSER_CONFIG_REGISTRY:
supported = list(PARSER_CONFIG_REGISTRY.keys())
raise ValueError(f"Unsupported parser type: '{parser_type}'. Supported: {supported}")
config_class = PARSER_CONFIG_REGISTRY[parser_type]
# Always go through from_dict, even with no data: configs that track which
# keys a source provided need to see the empty mapping to record that none
# were set.
return config_class.from_dict(config_data or {})
def load_parser_configs_from_dict(config_dict: Dict[str, Any]) -> Dict[str, ParserConfig]:
"""
Load all parser configurations from a dictionary.
Args:
config_dict: Configuration dictionary with parser sections
Returns:
Dictionary mapping parser types to their configurations
Examples:
>>> configs = load_parser_configs_from_dict({
... "pdf": {"strategy": "auto"},
... "code": {"github_raw_domain": "raw.githubusercontent.com"}
... })
>>> pdf_config = configs["pdf"]
>>> code_config = configs["code"]
"""
configs = {}
for parser_type, config_class in PARSER_CONFIG_REGISTRY.items():
# from_dict on an empty mapping rather than a bare constructor, so
# configs that track provided keys record that a section was absent.
configs[parser_type] = config_class.from_dict(config_dict.get(parser_type) or {})
return configs