Skip to content

OpenRAG is a comprehensive, single package Retrieval-Augmented Generation platform built on Langflow, Docling, and Opensearch.

Python
2.4 KB
Raw Blame
"""Inert HTML-to-Markdown conversion for public pages."""

from __future__ import annotations

import hashlib
import re
from dataclasses import dataclass
from html import unescape
from html.parser import HTMLParser


class _Parser(HTMLParser):
    def __init__(self) -> None:
        super().__init__(convert_charrefs=True)
        self.parts: list[str] = []
        self.skip = 0

    def handle_starttag(self, tag, attrs):
        tag = tag.lower()
        if tag in {"script", "style", "noscript", "template"}:
            self.skip += 1
        elif not self.skip and tag in {
            "p",
            "div",
            "section",
            "article",
            "li",
            "br",
            "h1",
            "h2",
            "h3",
            "h4",
            "h5",
            "h6",
        }:
            self.parts.append("\n\n" if tag != "li" else "\n- ")

    def handle_endtag(self, tag):
        if tag.lower() in {"script", "style", "noscript", "template"} and self.skip:
            self.skip -= 1

    def handle_data(self, data):
        if not self.skip:
            text = re.sub(r"\s+", " ", unescape(data)).strip()
            if text:
                self.parts.append(text + " ")


@dataclass(frozen=True)
class WebDocument:
    title: str
    markdown: str
    content_hash: str
    byte_size: int


def html_to_document(
    content: bytes, source_url: str, *, encoding: str | None = None
) -> WebDocument:
    try:
        raw = content.decode(encoding or "utf-8", errors="replace")
    except LookupError:
        raw = content.decode("utf-8", errors="replace")
    title_match = re.search(r"<title[^>]*>(.*?)</title\s*>", raw, re.I | re.S)
    title = (
        re.sub(r"\s+", " ", re.sub(r"<[^>]+>", " ", unescape(title_match.group(1)))).strip()
        if title_match
        else source_url
    )
    body = re.sub(r"<head[^>]*>.*?</head\s*>", "", raw, flags=re.I | re.S)
    parser = _Parser()
    parser.feed(body)
    text = re.sub(r"\n{3,}", "\n\n", "".join(parser.parts)).strip()
    if not text:
        raise ValueError("HTML document produced no extractable text")
    markdown = f"# {title}\n\n{text}\n".replace("\x00", "")
    normalized = re.sub(r"\s+", " ", markdown).strip()
    return WebDocument(
        title=title[:512],
        markdown=markdown,
        content_hash=hashlib.sha256(normalized.encode()).hexdigest(),
        byte_size=len(content),
    )