Skip to content

~95% on SimpleQA (e.g. Qwen3.6-27B on a 3090). Supports all local and cloud LLMs (llama.cpp, Ollama, Google, ...). 10+ search engines - arXiv, PubMed, your private documents. Everything Local & Encrypted.

7.4 KB
Raw Blame
"""Tripwire for the unstructured HTML-output stored XSS (GHSA-v5mq-3xhg-98m9).

unstructured 0.18.32 — the newest release that installs on Python 3.14 — is
affected by GHSA-v5mq-3xhg-98m9 / CVE-2026-104250 (Medium): the v2 ontology
HTML emitter copies document markup into ``metadata.text_as_html`` and the
output of ``elements_to_html()`` without escaping, so a crafted document's
``<img onerror=...>`` runs wherever that HTML is shown. Every fixed release
so far (0.24.1 through 0.27.28, checked 2026-10-17) declares
``Requires-Python: <3.14``, as has every release since 0.20.2, so none can be
locked. The finding is ignored in osv-scanner.toml, .trivyignore and
.grype.yaml. The Trivy alert and both Grype alerts (lockfile and image,
#9736-#9738) were dismissed on 2026-10-18. Dismissal status persists
independently of scanner suppressions, so their entries in
.github/release-security-exceptions.json are dormant audit records. The
existing osv-scanner.toml ignoreUntil date (2026-10-31) requires review
through the completed release scan.

The compensating control this file pins: LDR never invokes the v2 ontology
HTML emitter and never reads element metadata or ``elements_to_html`` output.
Every unstructured-backed loader is a langchain-community
``Unstructured*Loader`` in its default ``"single"`` mode, which returns
``"\\n\\n".join(str(element))`` with only ``{"source": path}`` as metadata, and
``es_utils.index_file`` uses element text the same way. Single-mode loaders
still compute an escaped v1 ``text_as_html`` that langchain discards.

Element text is not markup-free: a document whose text literally reads
``<img onerror=...>`` keeps those characters. Escaping at render time
(autoescaped templates, ``textContent``, DOMPurify) is what keeps that text
inert, independent of this advisory.

Remove this tripwire together with the suppression entries (and the
GHSA-4mvj-m6j5-pmf7 tripwire) when the lock can move to unstructured
>= 0.24.1.
"""

from __future__ import annotations

import ast
from pathlib import Path

import pytest

from local_deep_research.document_loaders import loader_registry
from local_deep_research.document_loaders.loader_registry import (
    LOADER_REGISTRY,
    get_loader_for_path,
)

SRC = Path(loader_registry.__file__).resolve().parents[1]

# Names that only appear when code produces or reads unstructured's HTML
# rendering of a document.
FORBIDDEN_NEEDLES = (
    "text_as_html",
    "elements_to_html",
    "html_parser_version",
    "unstructured.documents.ontology",
    "unstructured.staging",
    "metadata.to_dict",
    "element.to_dict(",
    ".to_html(",
)

# Loader keywords that switch langchain's loaders away from plain joined text
# ("elements"/"paged" return per-element Documents carrying text_as_html).
FORBIDDEN_LOADER_KWARGS = ("mode", "html_parser_version")

# class="Document" is what routes a page to the vulnerable v2 ontology
# emitter when html_parser_version="v2" is passed; the loaders never pass it.
MALICIOUS_HTML = """<!doctype html>
<html><body class="Document">
<p>Visible paragraph <img src=x onerror="alert(1)"></p>
<a href="javascript:alert(2)">click</a>
<svg onload="alert(3)"><text>svg text</text></svg>
<table><tr><td>cell <img src=y onerror="alert(4)"></td></tr></table>
</body></html>
"""


def test_no_source_touches_unstructured_html_output() -> None:
    modules = sorted(SRC.rglob("*.py"))
    assert len(modules) > 100, "source tree not found"
    offenders = [
        f"{path.relative_to(SRC)}: {needle}"
        for path in modules
        for needle in FORBIDDEN_NEEDLES
        if needle in path.read_text(encoding="utf-8")
    ]
    assert offenders == [], (
        "Code now produces or reads unstructured's HTML output while the "
        "GHSA-v5mq-3xhg-98m9 stored XSS is only suppressed (no fixed release "
        "supports Python 3.14). Use plain element text, or re-evaluate the "
        "suppression:\n" + "\n".join(offenders)
    )


def _is_unstructured_entry_point(name: str) -> bool:
    return (
        name.startswith("Unstructured") and name.endswith("Loader")
    ) or name.startswith("partition")


def test_no_unstructured_loader_is_built_outside_single_mode() -> None:
    """Checks every call in src/, not only the registry.

    Calls are matched by name, including names bound through
    ``from ... import UnstructuredHTMLLoader as Alias``. A ``**kwargs`` splat
    into such a call is flagged too, because its keys cannot be read
    statically. A loader class stored in a variable or container and called
    later (as the registry does) is not seen here; the registry is covered by
    test_unstructured_loaders_stay_in_single_mode.
    """
    offenders = []
    for path in sorted(SRC.rglob("*.py")):
        tree = ast.parse(path.read_text(encoding="utf-8"), filename=str(path))
        aliases = {
            alias.asname: alias.name
            for node in ast.walk(tree)
            if isinstance(node, ast.ImportFrom)
            for alias in node.names
            if alias.asname and _is_unstructured_entry_point(alias.name)
        }
        for node in ast.walk(tree):
            if not isinstance(node, ast.Call):
                continue
            func = node.func
            name = (
                func.id
                if isinstance(func, ast.Name)
                else func.attr
                if isinstance(func, ast.Attribute)
                else ""
            )
            name = aliases.get(name, name)
            if not _is_unstructured_entry_point(name):
                continue
            keywords = {keyword.arg for keyword in node.keywords}
            bad = sorted(keywords & set(FORBIDDEN_LOADER_KWARGS))
            if None in keywords:
                bad.append("**")
            if bad:
                offenders.append(
                    f"{path.relative_to(SRC)}:{node.lineno}: "
                    f"{name}({', '.join(bad)}=...)"
                )
    assert offenders == [], (
        "unstructured is called in a mode that surfaces its HTML output "
        "(GHSA-v5mq-3xhg-98m9), or with a **kwargs splat that hides its "
        "keywords:\n" + "\n".join(offenders)
    )


def test_unstructured_loaders_stay_in_single_mode() -> None:
    unstructured_entries = {
        extension: entry
        for extension, entry in LOADER_REGISTRY.items()
        if entry["loader_class"].__name__.startswith("Unstructured")
    }
    assert ".html" in unstructured_entries, "HTML loader not found"
    offenders = [
        f"{extension}: {entry['loader_class'].__name__}({key}=...)"
        for extension, entry in unstructured_entries.items()
        for key in FORBIDDEN_LOADER_KWARGS
        if key in entry.get("loader_kwargs", {})
    ]
    assert offenders == [], (
        "An unstructured loader leaves the default single mode, which would "
        "surface per-element text_as_html (GHSA-v5mq-3xhg-98m9):\n"
        + "\n".join(offenders)
    )


def test_html_loader_returns_text_without_markup(tmp_path) -> None:
    pytest.importorskip("unstructured")
    path = tmp_path / "crafted.html"
    path.write_text(MALICIOUS_HTML, encoding="utf-8")
    loader = get_loader_for_path(path)
    assert loader is not None
    documents = loader.load()

    assert len(documents) == 1
    text = documents[0].page_content
    assert "Visible paragraph" in text
    assert "cell" in text
    for markup in ("<img", "onerror", "onload", "javascript:", "<svg"):
        assert markup not in text, f"{markup!r} survived into loader text"
    assert "text_as_html" not in documents[0].metadata