Skip to content

Self-evolving Context Database for AI Agents. Unify Agent Memory, Knowledge RAG and Skills.

8.4 KB
Raw Blame
# Copyright (c) 2026 Beijing Volcano Engine Technology Co., Ltd.
# SPDX-License-Identifier: AGPL-3.0
"""
HTML Parser for OpenViking.

Parses local HTML files.

For URL downloading, use HTTPAccessor in the new two-layer architecture.
"""

import re
import time
from pathlib import Path
from typing import List, Optional, Union

from openviking.parse.base import (
    NodeType,
    ParseResult,
    ResourceNode,
    create_parse_result,
)
from openviking.parse.parsers.base_parser import BaseParser
from openviking_cli.utils.config.parser_config import HTMLConfig

logger = __import__("openviking_cli.utils.logger").utils.logger.get_logger(__name__)


class HTMLParser(BaseParser):
    """
    Parser for local HTML files.

    Features:
    - Parse local HTML files
    - Build hierarchy based on heading tags (h1-h6)
    - Filter out navigation, ads, and boilerplate
    - Extract tables and preserve structure

    NOTE: URL downloading functionality has been moved to HTTPAccessor
    in the new two-layer architecture. This parser only handles local files.
    """

    def __init__(
        self,
        timeout: float = 30.0,
        config: Optional[HTMLConfig] = None,
        **kwargs,
    ):
        """
        Initialize HTML parser.

        Args:
            timeout: [DEPRECATED] Kept for backward compatibility.
                URL downloading has been moved to HTTPAccessor.
            **kwargs: Additional arguments (kept for backward compatibility)
        """
        self.config = config or HTMLConfig()
        self._markdown_parser = None

    def _get_markdown_parser(self):
        """Lazy import and create MarkdownParser with the HTML parser config."""
        if self._markdown_parser is None:
            from openviking.parse.parsers.markdown import MarkdownParser

            self._markdown_parser = MarkdownParser(config=self.config)
        return self._markdown_parser

    @property
    def supported_extensions(self) -> List[str]:
        """List of supported file extensions."""
        return [".html", ".htm"]

    async def parse(self, source: Union[str, Path], instruction: str = "", **kwargs) -> ParseResult:
        """
        Parse a local HTML file.

        Args:
            source: HTML file path
            instruction: Processing instruction, guides LLM how to understand the resource
            **kwargs: Additional options

        Returns:
            ParseResult with document tree
        """
        start_time = time.time()
        path = Path(source)

        return await self._parse_local_file(path, start_time, **kwargs)

    async def _parse_local_file(self, path: Path, start_time: float, **kwargs) -> ParseResult:
        """Parse local HTML file."""
        if not path.exists():
            return create_parse_result(
                root=ResourceNode(type=NodeType.ROOT, content_path=None),
                source_path=str(path),
                source_format="html",
                parser_name="HTMLParser",
                parse_time=time.time() - start_time,
                warnings=[f"File not found: {path}"],
            )

        try:
            content = self._read_file(path)
            result = await self.parse_content(
                content, source_path=str(path), base_dir=path.parent, **kwargs
            )

            # Add timing info
            result.parse_time = time.time() - start_time
            result.parser_name = "HTMLParser"

            return result
        except Exception as e:
            return create_parse_result(
                root=ResourceNode(type=NodeType.ROOT, content_path=None),
                source_path=str(path),
                source_format="html",
                parser_name="HTMLParser",
                parse_time=time.time() - start_time,
                warnings=[f"Failed to read HTML: {e}"],
            )

    def _html_to_markdown(self, html: str, base_url: str = "") -> str:
        """Convert HTML to Markdown using trafilatura."""
        html = self._preprocess_html(html)
        content = self._extract_markdown(html, base_url or "")
        title = self._extract_title(html, base_url or "")
        content = self._clean_markdown(content)
        if not content:
            content = self._extract_noscript_notice(html)
        if title and title not in content:
            content = f"# {title}\n\n{content}" if content else f"# {title}"
        return content

    @staticmethod
    def _extract_noscript_notice(html: str) -> str:
        """Return the <noscript> text when the page has no extractable body.

        SPA shells (e.g. React apps) render their real content via JavaScript
        and only ship a static ``<noscript>You need to enable JavaScript...``
        placeholder. trafilatura ignores <noscript>, so such pages otherwise
        extract to an empty string. Surfacing the notice keeps at least a hint
        of why the page looks empty instead of writing a blank file.
        """
        try:
            from bs4 import BeautifulSoup

            soup = BeautifulSoup(html, "html.parser")
            texts = [
                node.get_text(" ", strip=True) for node in soup.find_all("noscript")
            ]
            return "\n\n".join(text for text in texts if text)
        except Exception:
            return ""

    @staticmethod
    def _extract_markdown(html: str, url: str) -> str:
        """Extract clean Markdown body from HTML."""
        try:
            import trafilatura

            return (
                trafilatura.extract(
                    html,
                    url=url,
                    output_format="markdown",
                    include_links=True,
                    include_tables=True,
                )
                or ""
            )
        except Exception as exc:
            logger.debug("[HTMLParser] trafilatura extraction failed: %s", exc)
            return ""

    @staticmethod
    def _extract_title(html: str, url: str) -> str:
        """Extract page title from HTML metadata."""
        try:
            import trafilatura

            metadata = trafilatura.extract_metadata(html, default_url=url)
            if metadata and metadata.title:
                return str(metadata.title).strip()
        except Exception as exc:
            logger.debug("[HTMLParser] trafilatura title extraction failed: %s", exc)
        return ""

    @staticmethod
    def _clean_markdown(markdown: str) -> str:
        """Remove noisy artifacts from extracted Markdown."""
        markdown = markdown or ""
        markdown = re.sub(r"!\[[^\]]*\]\(data:image/[^)]+\)", "", markdown)
        markdown = re.sub(
            r"<img[^>]*src\s*=\s*['\"]data:image/[^'\"]*['\"][^>]*/?>",
            "",
            markdown,
            flags=re.IGNORECASE,
        )
        markdown = re.sub(r"<(span|a)\s+(?:id|name)=['\"][^'\"]*['\"]\s*>\s*</\1>", "", markdown)
        markdown = re.sub(r"\n{3,}", "\n\n", markdown)
        return markdown.strip()

    def _preprocess_html(self, html: str) -> str:
        """Preprocess HTML to fix hidden content and lazy loading issues (e.g., WeChat public accounts)."""
        from bs4 import BeautifulSoup

        soup = BeautifulSoup(html, "html.parser")

        # WeChat public account: js_content is hidden by default, need to remove hidden style
        js_content = soup.find(id="js_content")
        if js_content:
            if js_content.get("style"):
                del js_content["style"]
            # Handle lazy loading images: data-src -> src
            for img in js_content.find_all("img"):
                if img.get("data-src") and not img.get("src"):
                    img["src"] = img["data-src"]
            return str(js_content)

        return html

    async def parse_content(
        self, content: str, source_path: Optional[str] = None, instruction: str = "", **kwargs
    ) -> ParseResult:
        """
        Parse HTML content.

        Converts HTML to Markdown and delegates to MarkdownParser.

        Args:
            content: HTML content string
            source_path: Optional source path for reference

        Returns:
            ParseResult with document tree
        """
        # Convert HTML to Markdown
        markdown_content = self._html_to_markdown(content, base_url=source_path or "")

        # Delegate to MarkdownParser
        md_parser = self._get_markdown_parser()
        result = await md_parser.parse_content(markdown_content, source_path=source_path, **kwargs)

        # Update metadata
        result.source_format = "html"
        result.parser_name = "HTMLParser"

        return result