Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

5.7 KB
Raw Blame
"""
HTML -> markdown-ish plain text normalization for URL-backed knowledge sources.

Primary extractor is trafilatura — it's specifically designed to strip
boilerplate (nav, footer, cookie banners) from article-style pages while
preserving semantic structure. Falls back to BeautifulSoup for pages where
trafilatura returns nothing (error pages, SPA shells, very minimal HTML).

Output is markdown: blocks are separated by blank lines (which the chunker
in `logic.py` relies on as paragraph boundaries) and headings/lists/code
survive extraction, which improves both chunk boundaries and retrieval.

Only `logic.py` is allowed to import this module.
"""

from __future__ import annotations

import re

import structlog
from bs4 import BeautifulSoup

logger = structlog.get_logger(__name__)

_CHARSET_RE = re.compile(r"charset=[\"']?([\w.-]+)", re.IGNORECASE)

# Boilerplate containers stripped before text extraction in the bs4 fallback.
# script/style/etc carry no prose; nav/header/footer/aside/form/button are the
# classic nav-bar / cookie-banner / footer-link noise we must keep out of
# searchable content.
_FALLBACK_DROP_TAGS = (
    "script",
    "style",
    "template",
    "noscript",
    "iframe",
    "nav",
    "header",
    "footer",
    "aside",
    "form",
    "button",
)


def _decode(body: bytes, content_type: str | None) -> str:
    """
    Decode a response body for the bs4 fallback paths. Honors the charset
    declared in the Content-Type header when present and valid; otherwise
    utf-8 with errors='replace' as the floor — we'd rather get partial
    garbled text than drop the whole page. (trafilatura is fed raw bytes
    and does its own encoding detection.)
    """

    if content_type:
        match = _CHARSET_RE.search(content_type)
        if match:
            try:
                return body.decode(match.group(1))
            except (UnicodeDecodeError, LookupError):
                pass
    try:
        return body.decode("utf-8")
    except UnicodeDecodeError:
        return body.decode("utf-8", errors="replace")


def _bs4_fallback(html: str) -> str:
    """
    Last-resort extractor. Strips boilerplate containers (nav, header,
    footer, ...) plus script/style nodes, prefers <main>/<article> when the
    page has one, then collapses runs of whitespace. We never return raw
    attributes or inline event handlers — those are prompt-injection vectors
    when shown to the LLM.
    """

    soup = BeautifulSoup(html, "html.parser")
    for node in soup(_FALLBACK_DROP_TAGS):
        node.decompose()
    root = soup.find("main") or soup.find("article") or soup
    text = root.get_text(separator="\n")
    # Collapse whitespace runs without losing paragraph breaks.
    lines = [line.strip() for line in text.splitlines()]
    joined = "\n".join(line for line in lines if line)
    return joined


def parse_html(body: bytes, url: str, content_type: str | None = None) -> tuple[str, str]:
    """
    Extract (title, text) from an HTML response body.

    `content_type` is the raw Content-Type header, used for charset hints in
    the fallback paths.

    Returns empty strings if extraction produced nothing usable — callers
    should treat that as a soft failure and store an error on the source.
    """

    # Deferred import: trafilatura (and its dateparser/htmldate deps) is slow to import
    # and only needed when actually parsing a page, not at API/module load.
    import trafilatura
    from trafilatura.metadata import examine_meta, extract_meta_json, extract_title
    from trafilatura.utils import load_html

    # trafilatura.extract returns the main-content text only. load_html gets raw
    # bytes so its internal encoding detection runs (meta charset, BOM, ...)
    # instead of forcing utf-8. Markdown output keeps blank lines between
    # blocks — the chunker's paragraph boundary — and preserves headings.
    tree = load_html(body)

    title = ""
    if tree is not None:
        try:
            metadata = examine_meta(tree)
            try:
                metadata = extract_meta_json(tree, metadata)
            except Exception:
                logger.info("business_knowledge.html_parse.json_ld_failed", url=url)
            metadata.title = metadata.title or extract_title(tree)
            metadata.clean_and_trim()
            title = (metadata.title or "").strip()[:512]
        except Exception:  # noqa: BLE001 — trafilatura metadata parsing can crash on malformed HTML
            logger.info("business_knowledge.html_parse.metadata_failed", url=url)

    extracted = (
        trafilatura.extract(
            tree,
            url=url,
            include_comments=False,
            include_tables=True,
            favor_precision=True,
            output_format="markdown",
        )
        if tree is not None
        else None
    )

    html: str | None = None
    text = (extracted or "").strip()
    if not text:
        try:
            html = _decode(body, content_type)
            text = _bs4_fallback(html)
        except Exception:  # noqa: BLE001 — bs4 is a best-effort fallback
            logger.info("business_knowledge.html_parse.bs4_failed", url=url)
            text = ""

    if not title:
        # Extract a title from <title> as a secondary fallback — some pages
        # have no og:title / articleTitle but do have <title>.
        try:
            if html is None:
                html = _decode(body, content_type)
            soup = BeautifulSoup(html, "html.parser")
            if soup.title and soup.title.string:
                title = soup.title.string.strip()[:512]
        except Exception:  # noqa: BLE001
            pass

    # Postgres text fields reject NUL, and the bs4 fallback keeps them from bodies that are not HTML.
    return title.replace("\x00", ""), text.replace("\x00", "")