Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

4.8 KB
Raw Blame
import re
from collections import Counter
from html.parser import HTMLParser

from posthog.dataclasses import frozen

PARAGRAPH_TAGS = frozenset({"p", "h1", "h2", "h3", "h4", "h5", "h6", "ul", "ol", "table", "blockquote", "pre", "hr"})
LINE_TAGS = frozenset({"div", "li", "tr", "section", "article", "header", "footer", "dd", "dt"})
HIDDEN_TAGS = frozenset({"head", "title", "style", "script", "template"})
VOID_TAGS = frozenset(
    {
        "area",
        "base",
        "basefont",
        "bgsound",
        "br",
        "col",
        "command",
        "embed",
        "frame",
        "hr",
        "image",
        "img",
        "input",
        "isindex",
        "keygen",
        "link",
        "menuitem",
        "meta",
        "nextid",
        "param",
        "source",
        "spacer",
        "track",
        "wbr",
    }
)
LINK_SCHEMES = ("http://", "https://", "mailto:")
BLANK_LINES = re.compile(r"\n{3,}")
WHITESPACE = re.compile(r"\s+")
LINE_END = "\x00"
LINE_ENDS = re.compile(r"\n?\x00+\n?")


def text_from_html(html: str) -> str:
    parser = _TextParser()
    parser.feed(html.replace(LINE_END, ""))
    parser.close()
    text = LINE_ENDS.sub(_line_break, "".join(parser.parts))
    lines = "\n".join(" ".join(line.split()) for line in text.split("\n"))
    return BLANK_LINES.sub("\n\n", lines).strip()


def _line_break(match: re.Match[str]) -> str:
    return match.group(0).replace(LINE_END, "") or "\n"


@frozen
class _OpenLink:
    depth: int
    href: str
    label_start: int


class _TextParser(HTMLParser):
    def __init__(self) -> None:
        super().__init__(convert_charrefs=True)
        self.parts: list[str] = []
        self.open_tags: list[str] = []
        self.open_counts: Counter[str] = Counter()
        self.open_links: list[_OpenLink] = []
        self.hidden_from: int | None = None
        self.label_text: list[str] = []
        self.label_length = 0

    def handle_starttag(self, tag: str, attrs: list[tuple[str, str | None]]) -> None:
        if tag in VOID_TAGS:
            self.handle_startendtag(tag, attrs)
            return
        self._start(tag, attrs)
        self._push(tag)

    def handle_startendtag(self, tag: str, attrs: list[tuple[str, str | None]]) -> None:
        self._start(tag, attrs)
        self._push(tag)
        self._end_last()

    def handle_endtag(self, tag: str) -> None:
        if tag in VOID_TAGS or not self.open_counts[tag]:
            return
        while self.open_tags[-1] != tag:
            self._end_last()
        self._end_last()

    def handle_data(self, data: str) -> None:
        if self.hidden_from is not None:
            return
        self.parts.append(WHITESPACE.sub(" ", data))
        if stripped := data.strip():
            self.label_text.append(stripped)
            self.label_length += len(stripped)

    def close(self) -> None:
        super().close()
        while self.open_tags:
            self._end_last()

    def _start(self, tag: str, attrs: list[tuple[str, str | None]]) -> None:
        if self.hidden_from is not None:
            return
        if tag in HIDDEN_TAGS:
            self.hidden_from = len(self.open_tags)
            return
        if tag == "br":
            self.parts.append("\n")
        elif tag in PARAGRAPH_TAGS:
            self.parts.append("\n\n")
        elif tag in LINE_TAGS:
            self.parts.append("\n")
        href = dict(attrs).get("href") if tag == "a" else None
        if href is not None and href.startswith(LINK_SCHEMES):
            self.open_links.append(_OpenLink(depth=len(self.open_tags), href=href, label_start=self.label_length))

    def _push(self, tag: str) -> None:
        self.open_tags.append(tag)
        self.open_counts[tag] += 1

    def _end_last(self) -> None:
        tag = self.open_tags.pop()
        self.open_counts[tag] -= 1
        depth = len(self.open_tags)
        if self.hidden_from is not None:
            if depth == self.hidden_from:
                self.hidden_from = None
            return
        if self.open_links and self.open_links[-1].depth == depth:
            link = self.open_links.pop()
            if not self._label_is(link):
                self.parts.append(f" ({link.href})")
        elif tag in ("td", "th"):
            self.parts.append(" ")
        if tag in PARAGRAPH_TAGS:
            self.parts.append("\n\n")
        elif tag in LINE_TAGS:
            self.parts.append(LINE_END)

    def _label_is(self, link: _OpenLink) -> bool:
        if self.label_length - link.label_start != len(link.href):
            return False
        label: list[str] = []
        length = 0
        for text in reversed(self.label_text):
            if length >= len(link.href):
                break
            label.append(text)
            length += len(text)
        return "".join(reversed(label))[length - len(link.href) :] == link.href