Skip to content

Self-evolving Context Database for AI Agents. Unify Agent Memory, Knowledge RAG and Skills.

69.2 KB
Raw Blame
# Copyright (c) 2026 Beijing Volcano Engine Technology Co., Ltd.
# SPDX-License-Identifier: AGPL-3.0
"""
Markdown parser for OpenViking (v5.0).

This parser implements the new simplified architecture:
- Parse structure and create directory structure directly in VikingFS
- No LLM calls during parsing (semantic generation moved to SemanticQueue)
- Support mixed directory structure (files + subdirectories)
- Small sections (< 512 characters) are merged with adjacent sections

The parser handles scenarios:
1. Small files (<= 2000 characters) → save as single file with original name
2. Large files with sections → split by sections with merge logic
3. Sections with subsections → section becomes directory
4. Small sections (< 512 characters) → merged with adjacent sections
5. Oversized sections without subsections → split by paragraphs
"""

import asyncio
import hashlib
import os
import re
import time
from dataclasses import dataclass, field
from pathlib import Path
from typing import TYPE_CHECKING, Any, Dict, List, Optional, Tuple, Union

import yaml

from openviking.parse.accessors.mime_types import IANA_MEDIA_TYPE_TO_EXTENSION
from openviking.parse.base import NodeType, ParseResult, ResourceNode, create_parse_result
from openviking.parse.image_validation import is_valid_image
from openviking.parse.output import (
    AgfsParseOutputStore,
    ParseArtifactRef,
    ParseArtifactWriter,
    create_parse_artifact_writer,
)
from openviking.parse.parsers.base_parser import BaseParser
from openviking.parse.parsers.code.ast.providers import supports_code_skeleton
from openviking.parse.parsers.constants import (
    CODE_EXTENSIONS,
    DOCUMENTATION_EXTENSIONS,
    IGNORE_EXTENSIONS,
)
from openviking_cli.utils.config.parser_config import ParserConfig
from openviking_cli.utils.logger import get_logger

# All known valid extensions - only these should be stripped when getting stem
KNOWN_EXTENSIONS: set[str] = set()
for extensions in IANA_MEDIA_TYPE_TO_EXTENSION.values():
    KNOWN_EXTENSIONS.update(extensions)
KNOWN_EXTENSIONS.update(CODE_EXTENSIONS)
KNOWN_EXTENSIONS.update(DOCUMENTATION_EXTENSIONS)
KNOWN_EXTENSIONS.update(IGNORE_EXTENSIONS)

# Markdown link/image in one pass: optional leading "!" marks images.
# Scope (per spec, YAGNI): only inline [..](..)/![..](..) are handled. Reference-style
# links, HTML, and links inside fenced/indented code blocks are NOT excluded — a
# code-block link to an existing relative .md could be rewritten. Acceptable for now.
# NOTE: `[^)]+` stops at the first ")", so a target whose filename literally
# contains ")" is captured truncated; it then fails the on-disk existence check
# and is left unchanged (conservative). Such filenames are unsupported.
_MD_LINK_RE = re.compile(r"(!?)\[([^\]]*)\]\(([^)]+)\)")
# Splits a link target into its path part and optional "#fragment"/"?query" suffix.
_MD_FRAGMENT_RE = re.compile(r"^([^#?]*)([#?].*)?$")
# Extensions whose targets become directories on ingest (see _rewrite_single_link).
_MD_DIR_EXTS = {".md", ".markdown", ".mdown", ".mkd"}


def _smart_stem(path_or_name: str | Path) -> str:
    """Get the stem of a filename, but only strip known valid extensions.

    For filenames like "2601.00014" where ".00014" is not a valid extension,
    returns the full name instead of just "2601".

    Args:
        path_or_name: Path object or string filename

    Returns:
        Stem with only known extensions stripped
    """
    path = Path(path_or_name)
    suffix = path.suffix.lower()

    if suffix in KNOWN_EXTENSIONS:
        return path.stem

    # If the suffix is not a known extension, treat the whole name as the stem
    return path.name


logger = get_logger(__name__)


def _env_bool(name: str, default: bool = False) -> bool:
    raw = os.environ.get(name)
    if raw is None:
        return default
    return raw.strip().lower() in {"1", "true", "yes", "on"}


def _gh_slug(text: str) -> str:
    """GitHub-style heading slug: lowercase, strip punctuation, spaces→'-', keep CJK."""
    s = text.strip().lower()
    s = re.sub(r"[^\w\s-]", "", s)
    # Map each whitespace character to its own hyphen (do NOT collapse runs).
    # GitHub's reference slugger does `.replace(/ /g, '-')` — one hyphen per
    # space, no collapsing. Removing punctuation can leave adjacent spaces
    # (e.g. "Foo & Bar" → "foo  bar"); collapsing them produces "foo-bar"
    # while GitHub generates "foo--bar", breaking intra-doc anchor links.
    s = re.sub(r"\s", "-", s)
    return s


@dataclass
class _LayoutOp:
    """One planned VikingFS operation. ``parse`` (``_compute_layout``) produces an
    ordered list of these without touching the store; ``write`` (``_apply_layout``)
    replays them. ``mkdir`` creates ``uri``; ``write`` stores ``content`` at ``uri``."""

    kind: str  # "mkdir" | "write"
    uri: str
    content: Optional[str] = None
    exist_ok: bool = False


@dataclass
class _Layout:
    """A markdown document parsed into a VikingFS write plan, with nothing written
    yet. ``ops`` holds raw (un-rewritten) section content; replay it via
    ``_apply_layout``. The link-rewrite probe reuses ``ops`` to read a target's split
    layout side-effect-free, so it no longer needs a fake filesystem."""

    temp_uri: str
    root_dir: str
    doc_title: str
    doc_name: str
    ops: List[_LayoutOp] = field(default_factory=list)
    meta: Dict[str, Any] = field(default_factory=dict)
    warnings: List[str] = field(default_factory=list)


@dataclass
class _RewriteContext:
    """Link-rewrite inputs and memoized results owned by one parse_content call."""

    source_path: str
    doc_name: str
    root_dir: str
    import_root: str
    base_dir: Optional[Path]
    allowed_media_dirs: Optional[List[Path]]
    split_content: bool
    adaptive_flatten_requested: bool
    flatten_single_output: bool
    split_cache: Dict[Tuple[str, bool, bool], Optional[Dict[str, str]]] = field(
        default_factory=dict
    )
    image_probe_cache: Dict[str, bool] = field(default_factory=dict)


if TYPE_CHECKING:
    pass


class MarkdownParser(BaseParser):
    """
    Markdown parser for OpenViking v5.0.

    Supports: .md, .markdown, .mdown, .mkd

    Features:
    - Direct directory structure creation in VikingFS
    - No LLM calls during parsing (moved to SemanticQueue)
    - Mixed directory structure support (files + subdirectories)
    - Smart content splitting for oversized sections
    - Size-based parsing decisions
    """

    # Configuration constants
    DEFAULT_MIN_SECTION_CHARS = 512  # Minimum chars to create a separate section
    TABLE_HEADER_REPEAT_MAX_CHARS = 1024
    MAX_MERGED_FILENAME_LENGTH = 32  # Maximum length for merged section filenames

    # Image validation constants
    IMAGE_MIN_SIDE = 14  # Minimum width/height in pixels (exclusive)
    IMAGE_MIN_PIXELS = 196  # Minimum width * height
    IMAGE_MAX_PIXELS = 36000000  # Maximum width * height
    IMAGE_MIN_ASPECT_RATIO = 1 / 150  # Minimum width/height ratio
    IMAGE_MAX_ASPECT_RATIO = 150  # Maximum width/height ratio
    IMAGE_MAX_FILE_BYTES = 10 * 1024 * 1024  # Local file path limit: 10 MB

    def __init__(
        self,
        extract_frontmatter: Optional[bool] = None,
        config: Optional[ParserConfig] = None,
    ):
        """
        Initialize the enhanced markdown parser.

        Args:
            extract_frontmatter: Whether to REMOVE YAML frontmatter from the stored
                document body. Frontmatter is parsed into the parse result metadata
                either way. Defaults to False (lossless ingestion); when None, uses
                config.
            config: Parser configuration (uses default if None)
        """
        self.config = config or ParserConfig()
        if extract_frontmatter is None:
            extract_frontmatter = getattr(self.config, "extract_frontmatter", False)
        self.extract_frontmatter = extract_frontmatter

        # Compile regex patterns for better performance
        self._heading_pattern = re.compile(r"^(#{1,6})\s+(.+)$", re.MULTILINE)
        self._code_block_pattern = re.compile(r"```(\w*)\n(.*?)```", re.DOTALL)
        self._inline_code_pattern = re.compile(r"`([^`]+)`")
        self._link_pattern = re.compile(r"\[([^\]]+)\]\(([^)]+)\)")
        self._image_pattern = re.compile(r"!\[([^\]]*)\]\(((?:[^()]|\([^()]*\))+)\)")
        self._list_pattern = re.compile(r"^(\s*)[-*+]\s+(.+)$", re.MULTILINE)
        self._numbered_list_pattern = re.compile(r"^(\s*)\d+\.\s+(.+)$", re.MULTILINE)
        self._frontmatter_pattern = re.compile(r"^---\n(.*?)\n---\n", re.DOTALL)
        self._html_comment_pattern = re.compile(r"<!--.*?-->", re.DOTALL)
        self._indented_code_pattern = re.compile(r"^(?:    |\t).+$", re.MULTILINE)

        # Cache for VikingFS instance
        self._viking_fs = None

    @property
    def supported_extensions(self) -> List[str]:
        """Return list of supported file extensions."""
        return [".md", ".markdown", ".mdown", ".mkd"]

    async def parse(self, source: Union[str, Path], instruction: str = "", **kwargs) -> ParseResult:
        """
        Parse from file path or content string.

        Args:
            source: File path or content string
            instruction: Processing instruction, guides LLM how to understand the resource
            **kwargs: Runtime options (e.g., base_dir for resolving relative paths)

        Returns:
            ParseResult with document tree (including temp_dir_path)
        """
        path = Path(source)

        if path.exists():
            content = self._read_file(path)
            # Pass base_dir for resolving relative image paths
            return await self.parse_content(
                content,
                source_path=str(path),
                instruction=instruction,
                base_dir=path.parent,
                **kwargs,
            )
        else:
            # Treat as raw content string
            return await self.parse_content(str(source), instruction=instruction, **kwargs)

    async def parse_content(
        self,
        content: str,
        source_path: Optional[str] = None,
        instruction: str = "",
        base_dir: Optional[Path] = None,
        allowed_media_dirs: Optional[List[Path]] = None,
        **kwargs,
    ) -> ParseResult:
        """
        Parse markdown content and create directory structure in VikingFS.

        New architecture (v5.0):
        - Directly create files and directories in temp VikingFS
        - No LLM calls during parsing (semantic generation moved to SemanticQueue)
        - Support mixed directory structure (files + subdirectories)

        Args:
            content: Markdown content string
            source_path: Optional source file path
            instruction: Processing instruction (unused in v5.0)
            base_dir: Base directory for relative paths
            allowed_media_dirs: Additional directories from which derived media
                (e.g. images extracted by the PDF/DOC parser) may be read. The
                caller is responsible for ensuring these belong to the current
                input resource's lifecycle.
            **kwargs: Additional runtime options, incl. resource_name/source_name and
                enable_link_rewrite/link_rewrite_root (the latter set by DirectoryParser)

        Returns:
            ParseResult with temp_dir_path (Viking URI)
        """
        start_time = time.time()
        output_store = kwargs.pop("parse_output_store", None)
        writer = await create_parse_artifact_writer(
            output_store, viking_fs=self._get_viking_fs() if output_store is None else None
        )

        try:
            split_content = bool(kwargs.get("split_content", True))
            adaptive_flatten_requested = bool(
                kwargs.get("flatten_single_output", False) and not split_content
            )
            flatten_single_output = adaptive_flatten_requested
            if flatten_single_output:
                flatten_single_output = not await self._has_ingestable_local_image(
                    content,
                    base_dir=base_dir,
                    allowed_media_dirs=allowed_media_dirs,
                )
            # Phase 1 — parse only: turn the markdown into an ordered VikingFS write
            # plan, touching nothing. The temp URI is allocated here (the one
            # FS-scoped step) and threaded in so layout planning stays side-effect free.
            temp_uri = writer.ref.root
            layout_kwargs = dict(kwargs)
            layout_kwargs["flatten_single_output"] = flatten_single_output
            layout = await self._compute_layout(
                content,
                temp_uri,
                source_path=source_path,
                instruction=instruction,
                **layout_kwargs,
            )

            import_root = kwargs.get("link_rewrite_root")
            rewrite_ctx = None
            if kwargs.get("enable_link_rewrite", False) and source_path and import_root:
                rewrite_ctx = _RewriteContext(
                    source_path=source_path,
                    doc_name=layout.doc_name,
                    root_dir=layout.root_dir,
                    import_root=import_root,
                    base_dir=base_dir,
                    allowed_media_dirs=allowed_media_dirs,
                    split_content=split_content,
                    adaptive_flatten_requested=adaptive_flatten_requested,
                    flatten_single_output=flatten_single_output,
                )

            # Phase 2 — write only: replay the plan against the real VikingFS,
            # rewriting links and ingesting local images.
            await self._apply_layout(
                layout,
                writer=writer,
                rewrite_ctx=rewrite_ctx,
                base_dir=base_dir,
                allowed_media_dirs=allowed_media_dirs,
            )

            parse_time = time.time() - start_time

            # Create dummy root node for compatibility
            root = ResourceNode(
                type=NodeType.ROOT,
                title=layout.doc_title,
                level=0,
                meta=layout.meta.get("frontmatter", {}),
            )

            result = create_parse_result(
                root=root,
                source_path=source_path,
                source_format="markdown",
                parser_name="MarkdownParser",
                parse_time=parse_time,
                meta=layout.meta,
                warnings=layout.warnings,
            )

            root_rel = writer.relative_path(layout.root_dir)
            result.artifact_ref = await writer.finalize(resource_rel=root_rel)
            result.temp_dir_path = result.artifact_ref.root

            return result

        except Exception as e:
            logger.error(f"[MarkdownParser] Parse failed: {e}", exc_info=True)
            await writer.cleanup()
            raise

    async def _compute_layout(
        self,
        content: str,
        temp_uri: str,
        source_path: Optional[str] = None,
        instruction: str = "",
        **kwargs,
    ) -> _Layout:
        """Phase 1 (parse only): plan ``content`` into an ordered VikingFS write plan
        plus metadata, WITHOUT touching VikingFS, rewriting links, or ingesting images.

        ``temp_uri`` is supplied by the caller (parse_content allocates a real one; the
        link-rewrite probe passes a throwaway), keeping this method pure so the probe
        can learn a target's split layout with zero side effects.
        """
        meta: Dict[str, Any] = {}
        warnings: List[str] = []

        # Frontmatter is ALWAYS parsed into ``meta`` (it drives doc_title below), but
        # it is only removed from the stored body when explicitly configured.
        # ``meta`` is transient — it is returned in the parse result and never written
        # to VikingFS — so stripping by default silently destroyed the frontmatter of
        # every ingested markdown file with no way to read those fields back.
        stripped_content, frontmatter = self._extract_frontmatter(content)
        if frontmatter:
            meta["frontmatter"] = frontmatter
        if self.extract_frontmatter:
            content = stripped_content

        explicit_name = kwargs.get("resource_name")
        if not explicit_name and kwargs.get("source_name"):
            explicit_name = _smart_stem(kwargs["source_name"])

        # Preserve the original uploaded filename when available instead of the temp
        # upload name (e.g. upload_<uuid>.txt).
        fallback_title = explicit_name or (_smart_stem(source_path) if source_path else "Document")
        frontmatter_title = meta.get("frontmatter", {}).get("title")
        doc_title = (
            frontmatter_title
            if isinstance(frontmatter_title, str) and frontmatter_title.strip()
            else fallback_title
        )
        doc_name = self._sanitize_for_path(doc_title)
        # Preserve code source filenames as the temp document directory.
        source_name = kwargs.get("source_name")
        root_name = source_name if source_name and supports_code_skeleton(source_name) else doc_name
        root_dir = (
            temp_uri
            if kwargs.get("flatten_single_output", False)
            else f"{temp_uri}/{self._sanitize_for_path(root_name)}"
        )

        # Find all headings
        headings = self._find_headings(content)

        # The temp dir is the first thing materialized on apply. A flattened
        # document uses the temp dir itself as its root, which _build_structure
        # creates below, so do not enqueue the same mkdir twice.
        ops: List[_LayoutOp] = []
        if root_dir != temp_uri:
            ops.append(_LayoutOp("mkdir", temp_uri))
        await self._build_structure(
            ops,
            content,
            headings,
            root_dir,
            source_path,
            doc_name,
            split_content=kwargs.get("split_content", True),
        )

        return _Layout(
            temp_uri=temp_uri,
            root_dir=root_dir,
            doc_title=doc_title,
            doc_name=doc_name,
            ops=ops,
            meta=meta,
            warnings=warnings,
        )

    async def _apply_layout(
        self,
        layout: _Layout,
        *,
        writer: Optional[ParseArtifactWriter] = None,
        rewrite_ctx: Optional[_RewriteContext] = None,
        base_dir: Optional[Path] = None,
        allowed_media_dirs: Optional[List[Path]] = None,
    ) -> None:
        """Phase 2 (write only): replay ``layout.ops`` against the real VikingFS —
        create dirs, write each section (rewriting relative links when enabled), then
        ingest the local images those sections reference."""
        started = time.perf_counter()
        if writer is None:
            writer = ParseArtifactWriter(
                AgfsParseOutputStore(viking_fs=self._get_viking_fs()),
                ParseArtifactRef(backend="agfs", root=layout.temp_uri),
            )
        mkdir_ops = [op for op in layout.ops if op.kind == "mkdir"]
        write_ops = [op for op in layout.ops if op.kind == "write"]
        write_chars = sum(len(op.content or "") for op in write_ops)

        rewrite_enabled = rewrite_ctx is not None

        mkdir_started = time.perf_counter()
        mkdir_count = 0
        for op in layout.ops:
            if op.kind == "mkdir":
                await writer.mkdir(op.uri)
                mkdir_count += 1
        mkdir_s = time.perf_counter() - mkdir_started

        write_started = time.perf_counter()
        written_sections: list[tuple[str, str]] = []
        for op in write_ops:
            written = await self._write_section(
                op.uri, op.content, writer=writer, rewrite_ctx=rewrite_ctx
            )
            written_sections.append((op.uri, written))
        write_s = time.perf_counter() - write_started

        # Ingest local image files, placing each image next to the markdown file
        # that references it. For generated tables with no image references, avoid
        # glob+read over every chunk file we just wrote.
        images_started = time.perf_counter()
        has_image_refs = self._layout_has_local_image_refs(write_ops)
        if has_image_refs:
            await self._ingest_local_images(
                layout.root_dir,
                base_dir,
                allowed_media_dirs,
                writer=writer,
                markdown_files=written_sections,
            )
        images_s = time.perf_counter() - images_started

        total_s = time.perf_counter() - started
        if _env_bool("OPENVIKING_MARKDOWN_APPLY_PROFILE", False):
            logger.info(
                f"[MarkdownApplyProfile] total={total_s:.3f}s mkdir={mkdir_s:.3f}s "
                f"write={write_s:.3f}s images={images_s:.3f}s ops={len(layout.ops)} "
                f"mkdir_ops={len(mkdir_ops)} mkdir_count={mkdir_count} "
                f"write_ops={len(write_ops)} write_chars={write_chars} "
                f"rewrite_enabled={rewrite_enabled} has_image_refs={has_image_refs} "
                f"root={layout.root_dir}"
            )

    # ========== Helper Methods ==========

    def _layout_has_local_image_refs(self, write_ops: List[_LayoutOp]) -> bool:
        """Return True when planned markdown content references local images.

        This mirrors the image patterns consumed by _ingest_local_images, but runs
        against in-memory layout content so pure-text imports avoid a VFS glob/read
        pass over every generated markdown chunk.
        """
        try:
            from openviking.parse.image_rewrite import HTML_IMG_PATTERN
        except ImportError:
            HTML_IMG_PATTERN = re.compile(
                r"""(<img\s[^>]*?src=["'])([^"']+)(["'][^>]*>)""", re.IGNORECASE
            )

        for op in write_ops:
            content = op.content or ""
            if "![" in content:
                for match in self._image_pattern.finditer(content):
                    if not self._is_remote_uri(match.group(2)):
                        return True
            if "<img" in content.lower():
                for match in HTML_IMG_PATTERN.finditer(content):
                    if not self._is_remote_uri(match.group(2)):
                        return True
        return False

    async def _has_ingestable_local_image(
        self,
        content: str,
        *,
        base_dir: Optional[Path],
        allowed_media_dirs: Optional[List[Path]],
    ) -> bool:
        """Return whether parsing will materialize at least one local image."""
        from openviking.parse.image_rewrite import HTML_IMG_PATTERN

        image_refs = [match.group(2) for match in self._image_pattern.finditer(content)]
        image_refs.extend(match.group(2) for match in HTML_IMG_PATTERN.finditer(content))
        seen: set[Path] = set()
        for image_ref in image_refs:
            if self._is_remote_uri(image_ref):
                continue
            resolved = self._resolve_image_path(image_ref, base_dir, allowed_media_dirs)
            if resolved is None or resolved in seen:
                continue
            seen.add(resolved)
            try:
                image_bytes = await asyncio.to_thread(resolved.read_bytes)
                if await asyncio.to_thread(self._is_valid_image, image_bytes, resolved):
                    return True
            except Exception:
                continue
        return False

    def _extract_frontmatter(self, content: str) -> Tuple[str, Optional[Dict[str, Any]]]:
        """
        Extract YAML frontmatter from content.

        Args:
            content: Markdown content

        Returns:
            Tuple of (content without frontmatter, frontmatter dict or None)
        """
        match = self._frontmatter_pattern.match(content)
        if not match:
            return content, None

        frontmatter_text = match.group(1)
        content_without_frontmatter = content[match.end() :]

        try:
            # Keep scalar values as strings, matching the previous parser and keeping
            # ParseResult metadata JSON-serializable, while honoring YAML nesting.
            frontmatter = yaml.load(frontmatter_text, Loader=yaml.BaseLoader)
        except yaml.YAMLError:
            logger.warning("Ignoring malformed YAML frontmatter")
            return content_without_frontmatter, None

        if not isinstance(frontmatter, dict):
            return content_without_frontmatter, None

        return content_without_frontmatter, frontmatter

    def _find_headings(self, content: str) -> List[Tuple[int, int, str, int]]:
        """
        Find all headings, excluding code blocks, HTML comments, and escaped characters.

        Args:
            content: Markdown content

        Returns:
            List of tuples (start_pos, end_pos, title, level)
        """
        # Collect all excluded ranges
        excluded_ranges = []

        # Triple backtick code blocks
        for match in self._code_block_pattern.finditer(content):
            excluded_ranges.append((match.start(), match.end()))

        # HTML comments <!-- ... -->
        for match in self._html_comment_pattern.finditer(content):
            excluded_ranges.append((match.start(), match.end()))

        # Four-space or tab indented code blocks
        for match in self._indented_code_pattern.finditer(content):
            excluded_ranges.append((match.start(), match.end()))

        # Find headings, skipping excluded ranges and escaped #
        headings = []
        for match in self._heading_pattern.finditer(content):
            pos = match.start()

            # Check if in excluded range
            in_excluded = any(start <= pos < end for start, end in excluded_ranges)
            if in_excluded:
                continue

            # Check if escaped \#
            if pos > 0 and content[pos - 1] == "\\":
                continue

            level = len(match.group(1))
            title = match.group(2).strip()
            headings.append((match.start(), match.end(), title, level))

        return headings

    def _smart_split_content(self, content: str, max_size: int) -> List[str]:
        """Split by character budget, preserving an indivisible Markdown table row."""
        max_chars = min(max_size, self.config.max_section_chars)
        paragraphs = content.split("\n\n")
        parts = []
        current = ""

        for para in paragraphs:
            if len(para) > max_chars:
                if current:
                    if len(current) < self.DEFAULT_MIN_SECTION_CHARS:
                        para = current + "\n\n" + para
                    else:
                        parts.append(current.strip())
                    current = ""
                table_parts = self._split_markdown_table_by_rows(para, max_chars=max_chars)
                if table_parts:
                    parts.extend(table_parts)
                else:
                    parts.extend(
                        para[i : i + max_chars].strip() for i in range(0, len(para), max_chars)
                    )
            elif current and len(current) + len(para) + 2 > max_chars:
                parts.append(current.strip())
                current = para
            else:
                current = current + "\n\n" + para if current else para

        if current.strip():
            tail = current.strip()
            if parts and len(parts[-1]) + 2 + len(tail) <= max_chars:
                parts[-1] = f"{parts[-1]}\n\n{tail}"
            else:
                parts.append(tail)

        return parts if parts else [content]

    @staticmethod
    def _is_markdown_table_row(line: str) -> bool:
        stripped = line.strip()
        return len(stripped) >= 2 and stripped.startswith("|") and stripped.endswith("|")

    @staticmethod
    def _is_markdown_table_separator(line: str) -> bool:
        stripped = line.strip()
        if not stripped.startswith("|") or not stripped.endswith("|"):
            return False
        cells = [cell.strip() for cell in stripped.strip("|").split("|")]
        return bool(cells) and all(re.fullmatch(r":?-{3,}:?", cell or "") for cell in cells)

    def _split_markdown_table_by_rows(
        self,
        content: str,
        *,
        max_chars: int,
    ) -> Optional[List[str]]:
        """Split a single oversized Markdown table without cutting rows.

        The first two table rows are treated as the header/separator and repeated
        in subsequent chunks when they are small enough to fit. Any prefix before
        the table, such as a section heading merged into the oversized paragraph,
        is kept on the first chunk only. A single row may exceed the requested
        limits because row integrity takes precedence at this source-storage stage.
        """
        lines = content.splitlines()
        table_start = -1
        for index in range(len(lines) - 1):
            if self._is_markdown_table_row(lines[index]) and self._is_markdown_table_separator(
                lines[index + 1]
            ):
                table_start = index
                break
        if table_start < 0:
            return None

        table_end = table_start + 2
        while table_end < len(lines) and self._is_markdown_table_row(lines[table_end]):
            table_end += 1

        if table_end - table_start < 3:
            return None
        if any(line.strip() for line in lines[table_end:]):
            return None

        prefix = "\n".join(lines[:table_start]).strip()
        header = lines[table_start : table_start + 2]
        rows = lines[table_start + 2 : table_end]
        header_text = "\n".join(header)
        repeat_header = (
            len(header_text) <= self.TABLE_HEADER_REPEAT_MAX_CHARS and len(header_text) < max_chars
        )

        parts: List[str] = []
        current_lines: List[str] = []

        def flush() -> None:
            nonlocal current_lines
            if current_lines:
                parts.append("\n".join(current_lines).strip())
                current_lines = []

        if prefix:
            parts.extend(prefix[i : i + max_chars] for i in range(0, len(prefix), max_chars))

        # Keep the header and separator together until the first data row is handled.
        current_lines.extend(header)
        for index, row in enumerate(rows):
            candidate = "\n".join([*current_lines, row])
            if current_lines and len(candidate) > max_chars:
                if index == 0 and len(row) > max_chars:
                    current_lines = [*header, row]
                    continue
                flush()
                if len(row) > max_chars or (
                    repeat_header and len(header_text) + 1 + len(row) <= max_chars
                ):
                    current_lines.extend(header)
            current_lines.append(row)

        flush()
        return parts or None

    async def _ingest_local_images(
        self,
        root_dir: str,
        base_dir: Optional[Path] = None,
        allowed_media_dirs: Optional[List[Path]] = None,
        *,
        writer: Optional[ParseArtifactWriter] = None,
        markdown_files: Optional[List[Tuple[str, str]]] = None,
    ) -> None:
        """
        Scan every processed markdown file under ``root_dir`` and copy the local
        images they reference into VikingFS, next to the markdown file itself.

        Images are processed file by file: for each markdown file, the local
        image references it contains are resolved and copied into the same
        directory as that markdown file. A single ``.image_mappings.json`` file,
        keyed by the markdown file's path relative to ``root_dir``, is written at
        ``root_dir`` for ``rewrite_image_uris`` to consume.

        Args:
            root_dir: Root directory URI in VikingFS containing the markdown files
            base_dir: Base directory for resolving relative paths
            allowed_media_dirs: Additional directories from which derived media
                may be read (passed through to ``_resolve_image_path``)
        """
        if writer is None:
            writer = ParseArtifactWriter(
                AgfsParseOutputStore(viking_fs=self._get_viking_fs()),
                ParseArtifactRef(backend="agfs", root=root_dir),
            )
        if markdown_files is None:
            raise ValueError("markdown_files are required for artifact image ingestion")

        root_prefix = root_dir.rstrip("/")

        # mapping: rel_md_path -> {original_path_str -> unique_filename}
        mappings: Dict[str, Dict[str, str]] = {}

        for md_uri, content in markdown_files:
            # Collect all image references in this markdown file: markdown
            # embeds (![...]) and HTML <img src="..."> tags alike.
            from openviking.parse.image_rewrite import HTML_IMG_PATTERN

            image_refs = [m.group(2) for m in self._image_pattern.finditer(content)]
            image_refs += [m.group(2) for m in HTML_IMG_PATTERN.finditer(content)]
            if not image_refs:
                continue

            # Resolve local image paths, skipping remote URIs and duplicates
            local_images = []
            origin_images_links = []
            seen_paths = set()

            for path_str in image_refs:
                # Skip remote URIs
                if self._is_remote_uri(path_str):
                    continue

                resolved_path = self._resolve_image_path(path_str, base_dir, allowed_media_dirs)
                if resolved_path is None:
                    logger.warning(f"[MarkdownParser] Image file not found: {path_str}")
                    continue

                # Skip duplicates within the same markdown file
                if resolved_path in seen_paths:
                    continue
                seen_paths.add(resolved_path)
                origin_images_links.append(path_str)
                local_images.append(resolved_path)

            if not local_images:
                continue

            # Target directory is the directory containing the markdown file
            md_dir = md_uri.rsplit("/", 1)[0]

            # Copy each local image next to the markdown file, deduplicating names
            used_names: set[str] = set()
            file_mappings: Dict[str, str] = {}  # original_path_str -> unique_filename
            for origin_link, resolved_path in zip(origin_images_links, local_images, strict=False):
                try:
                    if not await asyncio.to_thread(resolved_path.exists):
                        logger.warning(f"[MarkdownParser] Image file not found: {resolved_path}")
                        continue

                    # Read image bytes
                    image_bytes = await asyncio.to_thread(resolved_path.read_bytes)

                    # Validate pixel size and file size; skip non-compliant images
                    if not await asyncio.to_thread(
                        self._is_valid_image, image_bytes, resolved_path
                    ):
                        continue

                    # Get filename and deduplicate
                    filename = resolved_path.name
                    unique_filename = self._deduplicate_filename(filename, used_names)
                    used_names.add(unique_filename)

                    # Write next to the markdown file
                    viking_path = f"{md_dir}/{unique_filename}"
                    await writer.write_bytes(viking_path, image_bytes)

                    # Record mapping for post-commit rewrite
                    file_mappings[origin_link] = unique_filename

                except Exception as e:
                    logger.warning(f"[MarkdownParser] Failed to ingest image {resolved_path}: {e}")

            if file_mappings:
                rel_md_path = (
                    md_uri[len(root_prefix) + 1 :] if md_uri.startswith(root_prefix) else md_uri
                )
                mappings[rel_md_path] = file_mappings

        # Write a single mapping file at the root directory for rewrite_image_uris
        if mappings:
            import json

            from openviking.parse.image_rewrite import IMAGE_MAPPINGS_FILENAME

            await writer.write_text(
                f"{root_prefix}/{IMAGE_MAPPINGS_FILENAME}",
                json.dumps(mappings, ensure_ascii=False),
            )

    def _resolve_image_path(
        self,
        path_str: str,
        base_dir: Optional[Path],
        allowed_media_dirs: Optional[List[Path]] = None,
    ) -> Optional[Path]:
        """
        Resolve a local image reference to an existing filesystem path.

        Args:
            path_str: Raw image path from the markdown reference
            base_dir: Base directory for resolving relative paths
            allowed_media_dirs: Additional directories that belong to the current
                input resource's lifecycle (e.g. media extracted by the PDF/DOC
                parser).

        Returns:
            Resolved absolute Path if the file exists and stays within an
            allowed root, otherwise None
        """
        try:
            path = Path(self._unwrap_link_destination(path_str))

            # Reject absolute paths: they can point anywhere on the host
            if path.is_absolute():
                logger.warning(f"[MarkdownParser] Rejected absolute image path: {path_str}")
                return None

            # Build the list of allowed roots to confine resolution to.
            allowed_roots: list[Path] = []
            if base_dir:
                allowed_roots.append(base_dir)
            if allowed_media_dirs:
                allowed_roots.extend(allowed_media_dirs)

            if not allowed_roots:
                return None

            # Markdown semantics first: the reference is relative to the file's
            # own directory. Accept it when the resolved target stays inside ANY
            # allowed root (e.g. ../images/x.gif escaping base_dir but still
            # inside the import root a DirectoryParser passed down).
            if base_dir:
                candidate = (base_dir / path).resolve()
                if candidate.exists():
                    for root in allowed_roots:
                        try:
                            candidate.relative_to(root.resolve())
                            return candidate
                        except ValueError:
                            continue

            # Derived-media semantics: the reference may instead be relative to
            # one of the media roots themselves (e.g. images extracted by the
            # PDF/DOC parser into a media dir).
            for root in allowed_roots:
                candidate = (root / path).resolve()

                # Verify the resolved candidate stays under the allowed root,
                # rejecting traversal attempts such as ../../private.png.
                try:
                    candidate.relative_to(root.resolve())
                except ValueError:
                    logger.warning(
                        f"[MarkdownParser] Rejected image path outside base dir: {path_str}"
                    )
                    continue

                if candidate.exists():
                    return candidate

            return None
        except Exception:
            logger.warning(f"[MarkdownParser] Cannot resolve image path: {path_str}")
            return None

    def _is_valid_image(self, image_bytes: bytes, source_path: Path) -> bool:
        """
        Validate an image's pixel dimensions and file size.

        Requirements:
        - Width > 14px and height > 14px
        - Width * height within [196, 36000000]
        - Aspect ratio (width/height) within [1/150, 150]
        - File size (local path) <= 10 MB

        Args:
            image_bytes: Raw image bytes
            source_path: Original image path (for logging)

        Returns:
            True if the image satisfies all requirements, otherwise False
        """
        return is_valid_image(image_bytes, source_path)

    @staticmethod
    def _unwrap_link_destination(path: str) -> str:
        """Return the path represented by a Markdown ``<destination>``."""
        path = path.strip()
        if len(path) >= 2 and path.startswith("<") and path.endswith(">"):
            return path[1:-1]
        return path

    @classmethod
    def _is_remote_uri(cls, path: str) -> bool:
        """
        Check if a path is a remote URI.

        Args:
            path: Path to check

        Returns:
            True if path starts with http://, https://, viking://, data:, or ftp://
        """
        remote_prefixes = ("http://", "https://", "viking://", "data:", "ftp://")
        return cls._unwrap_link_destination(path).startswith(remote_prefixes)

    @staticmethod
    def _deduplicate_filename(filename: str, used_names: set[str]) -> str:
        """
        Generate a unique filename by appending _1, _2, etc. when collision occurs.

        Args:
            filename: Original filename
            used_names: Set of already used filenames

        Returns:
            Unique filename
        """
        if filename not in used_names:
            return filename

        path_obj = Path(filename)
        stem = path_obj.stem
        suffix = path_obj.suffix
        counter = 1

        while True:
            new_filename = f"{stem}_{counter}{suffix}"
            if new_filename not in used_names:
                return new_filename
            counter += 1

    def _sanitize_for_path(self, text: str, max_length: int = 50) -> str:
        safe = re.sub(
            r"[^\w\u0080-\u02af\u0400-\u052f\u0600-\u077f\u4e00-\u9fff\u3040-\u309f\u30a0-\u30ff\uac00-\ud7af\u3400-\u4dbf\U00020000-\U0002a6df\s.-]",
            "",
            text,
        )
        safe = re.sub(r"\s+", "_", safe)
        safe = safe.strip("._")
        if not safe:
            return "section"
        if len(safe) > max_length:
            hash_suffix = hashlib.sha256(text.encode()).hexdigest()[:8]
            return f"{safe[: max_length - 9]}_{hash_suffix}"
        return safe

    @staticmethod
    def _doc_landing(layout: Dict[str, str]) -> Tuple[str, bool]:
        """目标 .md 入库后的落点(相对其磁盘父目录),完全由真实 layout 推断,不对
        parser 是否目录化做任何假设:

        - 所有 section 收拢在同一公共目录下 → (该目录, True),链接指向目录;
        - 否则即单个裸文件(如未来小 .md 不再拆成目录)→ (该文件, False),指向文件。

        落点是文件还是目录、叫什么,全部由 in-memory parse 出来的 layout 结构决定;
        parse_content 怎么改,这里自动跟随。
        """
        keys = list(layout)
        first_segs = {k.split("/", 1)[0] for k in keys}
        if len(first_segs) == 1 and all("/" in k for k in keys):
            return first_segs.pop(), True
        return keys[0], False

    async def _rewrite_single_link(
        self,
        link: str,
        src_dir: str,
        import_root_abs: str,
        source_new_dir: str,
        rewrite_ctx: _RewriteContext,
    ) -> str:
        """Rewrite one link target; return original `link` if it must not change."""
        raw = link.strip()
        if (
            not raw
            or raw.startswith("#")
            or raw.startswith("/")
            or raw.startswith("mailto:")
            or "://" in raw
        ):
            return link

        m = _MD_FRAGMENT_RE.match(raw)
        path_part = m.group(1)
        suffix = m.group(2) or ""
        if not path_part:
            return link  # pure fragment/query

        target_disk = os.path.normpath(os.path.join(src_dir, path_part))
        if not os.path.exists(target_disk):
            return link
        try:
            if os.path.commonpath([import_root_abs, target_disk]) != import_root_abs:
                return link
        except ValueError:
            return link  # different drives, etc.

        def _to_rel(new_disk: str, keep_suffix: bool) -> str:
            r = os.path.relpath(new_disk, source_new_dir).replace(os.sep, "/")
            return (r + suffix) if keep_suffix else (r if r.endswith("/") else r + "/")

        ext = os.path.splitext(target_disk)[1].lower()
        if ext not in _MD_DIR_EXTS:
            # Non-.md target (image, sibling directory, ...): keep its path; only the
            # source's added depth shifts the relative prefix.
            return _to_rel(target_disk, keep_suffix=True)

        # .md target → on ingest MarkdownParser turns it into some layout (a directory
        # of section files, or — in future variants — a single file). Resolve against
        # that REAL layout from an in-memory parse with the SAME parser: the landing
        # (file vs directory), its name and the section files all come straight from
        # the parser, so NOTHING about ingest is assumed here. Whatever parse_content
        # does, running it in-memory tells us the final result.
        layout = await self._target_split_files(target_disk, rewrite_ctx)
        if not layout:
            return link  # target unparsable → leave the link untouched

        target_parent = os.path.dirname(target_disk)
        landing, landing_is_dir = self._doc_landing(layout)

        if suffix:
            # A) #anchor uniquely located in a real section → point at that section file.
            if suffix.startswith("#"):
                anchor = suffix[1:]
                hits = [
                    rel
                    for rel, text in layout.items()
                    if any(
                        _gh_slug(h) == anchor for h in re.findall(r"^#{1,6}\s+(.+)$", text, re.M)
                    )
                ]
                if len(hits) == 1:
                    return _to_rel(os.path.join(target_parent, hits[0]), keep_suffix=True)
            # B) Single-file document (a bare file, or a directory holding exactly one
            #    file) → the anchor/query lives in that one file; keep the suffix.
            if len(layout) == 1:
                return _to_rel(os.path.join(target_parent, next(iter(layout))), keep_suffix=True)

        # C) Single bare file with no suffix to place (e.g. a future small .md kept as
        #    a file) → point at the file itself (empty suffix ⇒ no trailing slash).
        if not landing_is_dir:
            return _to_rel(os.path.join(target_parent, landing), keep_suffix=True)
        # D) Directory landing → point at the directory and drop any suffix.
        return _to_rel(os.path.join(target_parent, landing), keep_suffix=False)

    async def _ingest_will_handle_image(self, link: str, rewrite_ctx: _RewriteContext) -> bool:
        """Whether ``_ingest_local_images`` will take this image reference: it must
        resolve within base_dir/allowed_media_dirs AND pass image validation — the
        exact conditions ingestion itself applies. Such embeds are left untouched so
        #2429 can copy them next to the section and ``rewrite_image_uris`` can turn
        them into viking:// URIs; everything else falls back to depth adjustment.
        Results are cached per parse_content call."""
        if link not in rewrite_ctx.image_probe_cache:
            handled = False
            resolved = self._resolve_image_path(
                link, rewrite_ctx.base_dir, rewrite_ctx.allowed_media_dirs
            )
            if resolved is not None:
                try:
                    image_bytes = await asyncio.to_thread(resolved.read_bytes)
                    handled = await asyncio.to_thread(self._is_valid_image, image_bytes, resolved)
                except Exception:
                    handled = False
            rewrite_ctx.image_probe_cache[link] = handled
        return rewrite_ctx.image_probe_cache[link]

    async def _rewrite_relative_links(
        self,
        content: str,
        *,
        section_subpath: str,
        rewrite_ctx: _RewriteContext,
    ) -> str:
        """Rewrite relative markdown links in `content` (disk-coordinate).

        Image embeds (``![...]``) that ingestion will take are left untouched —
        ``_ingest_local_images`` (PR #2429) copies them next to the section and
        ``rewrite_image_uris`` later rewrites them to viking:// URIs. Image embeds
        ingestion will NOT take (outside base_dir, missing, failing validation) get
        the same depth adjustment as document links so they stay valid after the
        document moves into its ingest directory.
        """
        from openviking.parse.image_rewrite import HTML_IMG_PATTERN

        src_dir = os.path.dirname(os.path.abspath(rewrite_ctx.source_path))
        import_root_abs = os.path.abspath(rewrite_ctx.import_root)
        if rewrite_ctx.flatten_single_output:
            source_new_dir = os.path.join(src_dir, section_subpath)
        else:
            source_new_dir = os.path.join(src_dir, rewrite_ctx.doc_name, section_subpath)

        out: List[str] = []
        last = 0
        for match in _MD_LINK_RE.finditer(content):
            out.append(content[last : match.start()])
            bang, text, link = match.group(1), match.group(2), match.group(3)
            if bang and await self._ingest_will_handle_image(link, rewrite_ctx):
                new_link = link
            else:
                new_link = await self._rewrite_single_link(
                    link, src_dir, import_root_abs, source_new_dir, rewrite_ctx
                )
            out.append(f"{bang}[{text}]({new_link})")
            last = match.end()
        out.append(content[last:])
        rewritten = "".join(out)

        # HTML <img src="..."> embeds get the same ownership split as ![...].
        pieces: List[str] = []
        last = 0
        for match in HTML_IMG_PATTERN.finditer(rewritten):
            pieces.append(rewritten[last : match.start()])
            src = match.group(2)
            if await self._ingest_will_handle_image(src, rewrite_ctx):
                new_src = src
            else:
                new_src = await self._rewrite_single_link(
                    src, src_dir, import_root_abs, source_new_dir, rewrite_ctx
                )
            pieces.append(f"{match.group(1)}{new_src}{match.group(3)}")
            last = match.end()
        pieces.append(rewritten[last:])
        return "".join(pieces)

    async def _target_split_files(
        self, target_path: str, rewrite_ctx: _RewriteContext
    ) -> Optional[Dict[str, str]]:
        """The target's real ingest layout {"<doc_dir>/<section...>": text} via a
        side-effect-free parse, cached per parse_content() call. None on parse failure."""
        split_content = rewrite_ctx.split_content
        flatten_single_output = rewrite_ctx.adaptive_flatten_requested
        key = (os.path.abspath(target_path), split_content, flatten_single_output)
        if key not in rewrite_ctx.split_cache:
            rewrite_ctx.split_cache[key] = await self._probe_split_layout(
                target_path,
                split_content=split_content,
                flatten_single_output=flatten_single_output,
                allowed_media_dirs=rewrite_ctx.allowed_media_dirs,
            )
        return rewrite_ctx.split_cache[key]

    async def _probe_split_layout(
        self,
        target_path: str,
        *,
        split_content: bool,
        flatten_single_output: bool = False,
        allowed_media_dirs: Optional[List[Path]] = None,
    ) -> Optional[Dict[str, str]]:
        """Plan the target's layout WITHOUT writing anything, returning
        {"<doc_dir>/<section...>": text} keyed by path relative to the temp root, i.e.
        INCLUDING the doc-root dir segment so callers can map each key straight onto
        the target's parent dir. Reuses the pure phase-1 planner (_compute_layout)
        with a throwaway temp URI, so no fake FS and no side effects are involved."""
        try:
            probe_root = "viking://temp/_probe"
            content = self._read_file(target_path)
            if flatten_single_output:
                flatten_single_output = not await self._has_ingestable_local_image(
                    content,
                    base_dir=Path(target_path).parent,
                    allowed_media_dirs=allowed_media_dirs,
                )
            layout = await self._compute_layout(
                content,
                probe_root,
                source_path=str(target_path),
                split_content=split_content,
                flatten_single_output=flatten_single_output,
            )
            root = layout.temp_uri.rstrip("/")
            out: Dict[str, str] = {}
            for op in layout.ops:
                if op.kind != "write" or not isinstance(op.content, str):
                    continue
                # rel = "<doc_dir>/<section...>" kept whole (temp_uri is the temp root,
                # WITHOUT the doc-root dir), so it maps directly onto the target's
                # parent directory on disk.
                uri = op.uri
                rel = uri[len(root) :].lstrip("/") if uri.startswith(root) else uri
                out[rel] = op.content
            return out
        except Exception as exc:
            logger.debug(f"[_probe_split_layout] failed for {target_path}: {exc}")
            return None

    def _section_subpath(self, uri: str, root_dir: str) -> str:
        """Return the section file's directory path relative to root_dir (POSIX)."""
        section_dir = uri.rsplit("/", 1)[0]
        root = root_dir.rstrip("/")
        if section_dir == root:
            return ""
        if section_dir.startswith(root + "/"):
            return section_dir[len(root) + 1 :]
        return ""

    async def _write_section(
        self,
        uri: str,
        content: str,
        *,
        writer: Optional[ParseArtifactWriter] = None,
        rewrite_ctx: Optional[_RewriteContext] = None,
    ) -> str:
        """Write a markdown section file, rewriting relative links when enabled."""
        if rewrite_ctx is not None:
            content = await self._rewrite_relative_links(
                content,
                section_subpath=self._section_subpath(uri, rewrite_ctx.root_dir),
                rewrite_ctx=rewrite_ctx,
            )
        if writer is None:
            writer = ParseArtifactWriter(
                AgfsParseOutputStore(viking_fs=self._get_viking_fs()),
                ParseArtifactRef(backend="agfs", root=uri.rsplit("/", 1)[0]),
            )
        await writer.write_text(uri, content)
        return content

    # ========== New Parsing Logic (v5.0) ==========

    async def _build_structure(
        self,
        ops: List[_LayoutOp],
        content: str,
        headings: List[Tuple[int, int, str, int]],
        root_dir: str,
        source_path: Optional[str] = None,
        doc_name: Optional[str] = None,
        split_content: bool = True,
    ) -> None:
        """
        Plan the document's directory/file layout into ``ops`` (no VikingFS writes).

        Logic:
        - Small files (< MAX_SECTION_SIZE): single file with original name
        - Large files: split by sections with merge logic for small sections
        - Sections with subsections: become directories
        - Direct content: treated as virtual section, participates in merge
        - Oversized sections without subsections: split by paragraphs

        Args:
            ops: Accumulator the planned mkdir/write operations are appended to
            content: Markdown content
            headings: List of (start, end, title, level) tuples
            root_dir: Root directory URI
            source_path: Source file path for naming
            split_content: Whether large Markdown may be split into multiple files
        """
        max_size = self.config.max_section_chars
        max_chars = self.config.max_section_chars
        min_size = self.DEFAULT_MIN_SECTION_CHARS

        # Create root directory
        ops.append(_LayoutOp("mkdir", root_dir))

        # Get document name
        doc_name = doc_name or self._sanitize_for_path(
            _smart_stem(source_path) if source_path else "content"
        )

        if not split_content:
            file_path = f"{root_dir}/{doc_name}.md"
            ops.append(_LayoutOp("write", file_path, content))
            logger.debug(f"[MarkdownParser] No-split document planned as: {file_path}")
            return

        # Small document: save as single file (character budget)
        if len(content) <= max_chars:
            file_path = f"{root_dir}/{doc_name}.md"
            ops.append(_LayoutOp("write", file_path, content))
            return

        # No headings: split by paragraphs
        if not headings:
            parts = self._smart_split_content(content, max_size)
            for part_idx, part in enumerate(parts, 1):
                ops.append(_LayoutOp("write", f"{root_dir}/{doc_name}_{part_idx}.md", part))
            return

        # Build virtual section list (content before the first top-level heading,
        # including any lower-level headings there, is the first virtual section)
        sections = []
        min_level = min(h[3] for h in headings)
        first_heading_start = next(h[0] for h in headings if h[3] == min_level)
        if first_heading_start > 0:
            pre_content = content[:first_heading_start].strip()
            if pre_content:
                pre_chars = len(pre_content)
                sections.append(
                    {
                        "name": doc_name,
                        "content": pre_content,
                        "chars": pre_chars,
                        "has_children": False,
                        "heading_idx": None,
                    }
                )

        # Add real sections (top-level only for this pass)
        i = 0
        while i < len(headings):
            if headings[i][3] == min_level:
                sections.append(
                    {
                        "heading_idx": i,
                    }
                )
            i += 1

        # Process sections with merge logic
        await self._process_sections_with_merge(
            ops, content, headings, root_dir, sections, doc_name, max_size, min_size
        )

    async def _process_sections_with_merge(
        self,
        ops: List[_LayoutOp],
        content: str,
        headings: List[Tuple[int, int, str, int]],
        parent_dir: str,
        sections: List[Dict[str, Any]],
        parent_name: str,
        max_size: int,
        min_size: int,
    ) -> None:
        """Plan sections into ``ops`` with small-section merge logic (no writes)."""
        # Expand section info
        expanded = [
            section
            if section.get("heading_idx") is None
            else self._get_section_info(content, headings, section["heading_idx"])
            for section in sections
        ]

        pending = []
        buffered_section = None

        async def flush_buffered() -> None:
            nonlocal buffered_section
            if buffered_section is not None:
                await self._save_section(
                    ops,
                    content,
                    headings,
                    parent_dir,
                    buffered_section,
                    max_size,
                    min_size,
                )
                buffered_section = None

        for sec in expanded:
            name, chars, content_text = sec["name"], sec["chars"], sec["content"]
            has_children = sec["has_children"]

            # Handle small sections
            if chars < min_size:
                if pending and sum(t for _, _, t in pending) + chars > max_size:
                    await flush_buffered()
                    await self._save_merged(ops, parent_dir, pending)
                    pending = []
                pending.append((name, content_text, chars))
                continue

            if pending:
                await flush_buffered()

                # Try merge with pending
                if self._can_merge(pending, chars, max_size, has_children):
                    pending.append((name, content_text, chars))
                    await self._save_merged(ops, parent_dir, pending)
                    pending = []
                    continue

                # Avoid flushing a single tiny section as a standalone low-value file.
                if self._should_merge_pending_into_next(pending):
                    sec = self._merge_pending_into_next_section(pending, sec)
                    pending = []
                else:
                    await self._save_merged(ops, parent_dir, pending)
                    pending = []
            else:
                await flush_buffered()

            buffered_section = sec

        if pending:
            # No next section exists. Fold a single tiny pending section back into
            # the previous saved candidate instead of emitting a standalone file.
            if buffered_section is not None and self._should_merge_pending_into_next(pending):
                buffered_section = self._merge_pending_into_previous_section(
                    buffered_section, pending
                )
                pending = []
            else:
                await flush_buffered()
                await self._save_merged(ops, parent_dir, pending)
                pending = []

        await flush_buffered()

    def _can_merge(self, pending: List, chars: int, max_size: int, has_children: bool) -> bool:
        """Check if section can merge with pending."""
        return sum(t for _, _, t in pending) + chars <= max_size and not has_children

    def _should_merge_pending_into_next(self, pending: List[Tuple[str, str, int]]) -> bool:
        """Prefer folding a single tiny pending section into the next section."""
        return len(pending) == 1 and pending[0][2] <= self.DEFAULT_MIN_SECTION_CHARS

    def _merge_pending_into_next_section(
        self, pending: List[Tuple[str, str, int]], section: Dict[str, Any]
    ) -> Dict[str, Any]:
        """Attach a tiny pending section to the following section."""
        _, pending_content, _ = pending[0]
        merged = dict(section)
        merged["content"] = f"{pending_content}\n\n{section['content']}".strip()
        merged["chars"] = len(merged["content"])

        if merged.get("has_children"):
            direct_content = section.get("direct_content", "").strip()
            merged["direct_content"] = (
                f"{pending_content}\n\n{direct_content}".strip()
                if direct_content
                else pending_content
            )

        return merged

    def _merge_pending_into_previous_section(
        self, section: Dict[str, Any], pending: List[Tuple[str, str, int]]
    ) -> Dict[str, Any]:
        """Attach a tiny trailing pending section back into the previous section."""
        _, pending_content, _ = pending[0]
        merged = dict(section)
        merged["content"] = f"{section['content']}\n\n{pending_content}".strip()
        merged["chars"] = len(merged["content"])

        if merged.get("has_children"):
            direct_content = section.get("direct_content", "").strip()
            merged["direct_content"] = (
                f"{direct_content}\n\n{pending_content}".strip()
                if direct_content
                else pending_content
            )

        return merged

    async def _save_section(
        self,
        ops: List[_LayoutOp],
        content: str,
        headings: List[Tuple[int, int, str, int]],
        parent_dir: str,
        section: Dict[str, Any],
        max_size: int,
        min_size: int,
    ) -> None:
        """Plan a single section (file or directory) into ``ops``."""
        name, content_text = section["name"], section["content"]
        has_children = section["has_children"]

        # Fits in one file (character budget)
        if len(content_text) <= max_size:
            ops.append(_LayoutOp("write", f"{parent_dir}/{name}.md", content_text))
            return

        # Create directory and handle children or split
        section_dir = f"{parent_dir}/{name}"
        ops.append(_LayoutOp("mkdir", section_dir, exist_ok=True))

        if has_children:
            await self._process_children(
                ops, content, headings, section_dir, section, name, max_size, min_size
            )
        else:
            await self._split_content(ops, section_dir, name, content_text, max_size)

    async def _process_children(
        self,
        ops: List[_LayoutOp],
        content: str,
        headings: List[Tuple[int, int, str, int]],
        section_dir: str,
        section: Dict[str, Any],
        name: str,
        max_size: int,
        min_size: int,
    ) -> None:
        """Build and plan child sections into ``ops``."""
        children = []
        if section.get("direct_content"):
            children.append(
                {
                    "name": name,
                    "content": section["direct_content"],
                    "chars": len(section["direct_content"]),
                    "has_children": False,
                    "heading_idx": None,
                }
            )
        for child_idx in section.get("child_indices", []):
            children.append({"heading_idx": child_idx})

        await self._process_sections_with_merge(
            ops, content, headings, section_dir, children, name, max_size, min_size
        )

    async def _split_content(
        self, ops: List[_LayoutOp], section_dir: str, name: str, content: str, max_size: int
    ) -> None:
        """Split content by paragraphs, planning each part as a write into ``ops``."""
        parts = self._smart_split_content(content, max_size)
        for i, part in enumerate(parts, 1):
            ops.append(_LayoutOp("write", f"{section_dir}/{name}_{i}.md", part))

    def _generate_merged_filename(self, sections: List[Tuple[str, str, int]]) -> str:
        """
        Smart merged filename generation, limited to MAX_MERGED_FILENAME_LENGTH characters.

        Strategy:
        - Single section: Use directly (truncated with hash if needed)
        - Multiple sections: {first_section}_{count}more (e.g., Intro_3more)
        - Total length strictly limited: MAX_MERGED_FILENAME_LENGTH characters
        - Hash suffix ensures uniqueness when truncation occurs
        """
        if not sections:
            return "merged"

        names = [n for n, _, _ in sections]
        count = len(names)
        max_len = self.MAX_MERGED_FILENAME_LENGTH

        # Build a content-aware hash from ALL section names AND indices to guarantee
        # uniqueness even when different merge groups share the same heading names.
        full_key = "_".join(f"{n}:{i}" for n, _, i in sections)
        hash_suffix = hashlib.sha256(full_key.encode()).hexdigest()[:8]

        if count == 1:
            base = names[0]
        else:
            suffix = f"_{count}more"
            max_first_len = max_len - len(suffix) - 9  # reserve space for _hash
            first_name = names[0][: max(max_first_len, 1)]
            base = f"{first_name}{suffix}"

        name = f"{base}_{hash_suffix}"

        if len(name) > max_len:
            name = f"{name[: max_len - 9]}_{hash_suffix}"

        name = name.strip("_")
        return name or "merged"

    async def _save_merged(
        self, ops: List[_LayoutOp], parent_dir: str, sections: List[Tuple[str, str, int]]
    ) -> None:
        """Plan merged sections as a single file with smart naming into ``ops``.

        If the joined content exceeds max_section_chars it is split further by
        _smart_split_content before writing. A single oversized Markdown table row
        remains intact to preserve the source table structure.
        """
        name = self._generate_merged_filename(sections)
        content = "\n\n".join(c for _, c, _ in sections)
        max_chars = self.config.max_section_chars
        if len(content) > max_chars:
            max_size = self.config.max_section_chars
            parts = self._smart_split_content(content, max_size)
            for i, part in enumerate(parts, 1):
                ops.append(_LayoutOp("write", f"{parent_dir}/{name}_{i}.md", part))
        else:
            ops.append(_LayoutOp("write", f"{parent_dir}/{name}.md", content))

    def _get_section_info(
        self,
        content: str,
        headings: List[Tuple[int, int, str, int]],
        idx: int,
    ) -> Dict[str, Any]:
        """
        Get section info including content, chars, children info.

        Args:
            content: Full markdown content
            headings: All headings list
            idx: Index of heading in list

        Returns:
            Dict with section info
        """
        start_pos, end_pos, title, level = headings[idx]
        section_name = self._sanitize_for_path(title)

        # Find section end (next same or higher level heading)
        section_end = len(content)
        next_same_level_idx = len(headings)
        for j in range(idx + 1, len(headings)):
            if headings[j][3] <= level:
                section_end = headings[j][0]
                next_same_level_idx = j
                break

        # Find direct content end (first child heading)
        direct_content_end = section_end
        first_child_idx = None
        child_indices = []
        for j in range(idx + 1, next_same_level_idx):
            if headings[j][3] == level + 1:
                if first_child_idx is None:
                    first_child_idx = j
                    direct_content_end = headings[j][0]
                child_indices.append(j)

        has_children = first_child_idx is not None

        # Build content
        heading_prefix = "#" * level
        section_start = end_pos  # After heading line
        full_content = f"{heading_prefix} {title}\n\n{content[section_start:section_end].strip()}"
        full_chars = len(full_content)

        direct_content = ""
        if has_children:
            direct_text = content[section_start:direct_content_end].strip()
            if direct_text:
                direct_content = f"{heading_prefix} {title}\n\n{direct_text}"

        return {
            "name": section_name,
            "content": full_content,
            "chars": full_chars,
            "has_children": has_children,
            "heading_idx": idx,
            "direct_content": direct_content,
            "child_indices": child_indices,
        }