Skip to content

OpenRAG is a comprehensive, single package Retrieval-Augmented Generation platform built on Langflow, Docling, and Opensearch.

Python
2.2 KB
Raw Blame
"""Fetch a URL, strip HTML, and write the visible text to a temporary file.

Used by the default-docs ingestion flow when DEFAULT_DOCS_INGEST_SOURCE=url
and the OpenRAG (non-Langflow) ingestion path is active.
"""

import html
import re
import tempfile
from html.parser import HTMLParser

import httpx


class _VisibleTextHTMLParser(HTMLParser):
    """Extract visible text while skipping script/style content."""

    def __init__(self):
        super().__init__(convert_charrefs=True)
        self._ignored_depth = 0
        self._chunks: list[str] = []

    def handle_starttag(self, tag, attrs):
        if tag.lower() in {"script", "style"}:
            self._ignored_depth += 1

    def handle_endtag(self, tag):
        if tag.lower() in {"script", "style"} and self._ignored_depth > 0:
            self._ignored_depth -= 1

    def handle_data(self, data):
        if self._ignored_depth == 0 and data and not data.isspace():
            self._chunks.append(data)

    def get_text(self) -> str:
        return " ".join(self._chunks)


async def materialize_url_as_text_file(docs_url: str, crawl_depth: int) -> tuple[str, str]:
    """Fetch URL content and write a temporary text file for OpenRAG ingestion, returning path and title."""
    async with httpx.AsyncClient(timeout=30.0, follow_redirects=True) as client:
        response = await client.get(docs_url)
        response.raise_for_status()
        raw_html = response.text

    title_match = re.search(
        r"<title[^>]*>(.*?)</title\s*>",
        raw_html,
        flags=re.IGNORECASE | re.DOTALL,
    )
    title = html.unescape(title_match.group(1).strip()) if title_match else ""
    title = title or "OpenRAG"

    text_parser = _VisibleTextHTMLParser()
    text_parser.feed(raw_html)
    text_parser.close()
    normalized_text = re.sub(r"\s+", " ", text_parser.get_text()).strip()

    content = (
        f"{title}\n\nSource URL: {docs_url}\nCrawl depth: {crawl_depth}\n\n{normalized_text}\n"
    )

    temp_file = tempfile.NamedTemporaryFile(
        mode="w",
        suffix=".txt",
        prefix="openrag-url-default-",
        delete=False,
        encoding="utf-8",
    )
    with temp_file:
        temp_file.write(content)
    return temp_file.name, title