Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

32.3 KB
Raw Blame
# SPDX-License-Identifier: AGPL-3.0-only
# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0

"""Regression tests for web fetch decoding: binary bodies poisoning model context (#7084) and page charsets."""

from __future__ import annotations

import codecs
import random
import sys
import threading
import time
import tracemalloc
from email.message import Message
from pathlib import Path

import pytest

_BACKEND_DIR = str(Path(__file__).resolve().parent.parent)
if _BACKEND_DIR not in sys.path:
    sys.path.insert(0, _BACKEND_DIR)

from core.inference import tools


class _FakeResp:
    def __init__(self, body: bytes, content_type: str | None):
        self._body = body
        self._pos = 0
        self.headers = Message()
        if content_type is not None:
            self.headers["Content-Type"] = content_type

    def read(self, n: int | None = None) -> bytes:
        chunk = self._body[self._pos :] if n is None else self._body[self._pos : self._pos + n]
        self._pos += len(chunk)
        return chunk


class _FakeOpener:
    def __init__(self, resp):
        self._resp = resp

    def open(
        self,
        req,
        timeout = None,
    ):
        return self._resp


def _fetch_with(monkeypatch, body: bytes, content_type: str | None) -> str:
    monkeypatch.setattr(
        tools, "_validate_and_resolve_host", lambda host, port: (True, "", ["93.184.216.34"])
    )
    monkeypatch.setattr(
        tools.urllib.request,
        "build_opener",
        lambda *a, **k: _FakeOpener(_FakeResp(body, content_type)),
    )
    return tools._fetch_page_text("https://example.com/thing", timeout = 5)


def _pdf_bytes(*page_texts: str) -> bytes:
    pymupdf = pytest.importorskip("pymupdf")
    doc = pymupdf.open()
    for text in page_texts:
        page = doc.new_page()
        if text:
            page.insert_textbox(pymupdf.Rect(40, 40, 550, 750), text, fontsize = 11)
    data = doc.tobytes()
    doc.close()
    return data


@pytest.mark.parametrize(
    "content_type,expected",
    [
        ("text/html", True),
        ("text/plain; charset=utf-8", True),
        ("application/json", True),
        ("application/json; charset=utf-8", True),
        ("application/xml", True),
        ("application/xhtml+xml", True),
        ("application/ld+json", True),
        ("application/yaml", True),
        ("application/x-yaml", True),
        ("application/x-ndjson", True),
        ("application/ndjson", True),
        ("application/sql", True),
        ("application/x-www-form-urlencoded", True),
        ("application/pdf", False),
        ("image/png", False),
        ("image/svg+xml", False),
        ("application/octet-stream", True),
        ("application/zip", False),
        ("application/vnd.ms-excel", True),
        ("application/vnd.openxmlformats-officedocument.wordprocessingml.document", True),
        ("", True),
        (None, True),
    ],
)
def test_is_text_candidate_content_type(content_type, expected):
    assert tools._is_text_candidate_content_type(content_type) is expected


@pytest.mark.parametrize(
    "content_type",
    ["application/pdf", "application/octet-stream", "text/html", "text/plain", None],
)
def test_pdf_text_extracted(monkeypatch, content_type):
    out = _fetch_with(
        monkeypatch,
        _pdf_bytes("First page marker", "Second page marker"),
        content_type,
    )
    assert "## Page 1\n\nFirst page marker" in out
    assert "## Page 2" in out and "Second page marker" in out
    assert "binary content" not in out and "non-text content" not in out


@pytest.mark.parametrize("content_type", ["application/pdf", "text/plain"])
def test_malformed_pdf_returns_safe_placeholder(monkeypatch, content_type):
    out = _fetch_with(monkeypatch, b"%PDF-1.7\nnot a complete PDF", content_type)
    assert out == "(PDF content could not be read as text)"


def test_pdf_without_text_layer_reported(monkeypatch):
    out = _fetch_with(monkeypatch, _pdf_bytes(""), "application/pdf")
    assert out == "(PDF contains no extractable text)"


def test_encrypted_pdf_returns_safe_placeholder(monkeypatch):
    pymupdf = pytest.importorskip("pymupdf")
    doc = pymupdf.open()
    doc.new_page().insert_text((40, 40), "private text")
    data = doc.tobytes(
        encryption = pymupdf.PDF_ENCRYPT_AES_256,
        owner_pw = "owner",
        user_pw = "secret",
    )
    doc.close()
    out = _fetch_with(monkeypatch, data, "application/pdf")
    assert out == "(PDF content could not be read as text)"


def test_pdf_download_limit_enforced(monkeypatch):
    monkeypatch.setattr(tools, "_MAX_PDF_FETCH_BYTES", 256)
    out = _fetch_with(monkeypatch, _pdf_bytes("Readable but oversized"), "application/pdf")
    assert out == "(PDF content exceeds the download limit; not readable as text)"


def test_mislabeled_pdf_is_read_past_text_download_cap(monkeypatch):
    body = _pdf_bytes("Cross-reference data was fetched")
    monkeypatch.setattr(tools, "_MAX_FETCH_BYTES", 128)
    monkeypatch.setattr(tools, "_MAX_PDF_FETCH_BYTES", len(body) + 100)
    out = _fetch_with(monkeypatch, body, "text/plain")
    assert "Cross-reference data was fetched" in out


def test_html_labeled_pdf_with_body_bytes_is_read_to_eof(monkeypatch):
    pymupdf = pytest.importorskip("pymupdf")
    doc = pymupdf.open()
    payload = b"<body>" + random.Random(0).randbytes(1024 * 1024)
    doc.embfile_add("blob.bin", payload, ufilename = "blob.bin")
    page = doc.new_page()
    page.insert_text((40, 40), "PDF body-window marker")
    body = doc.tobytes(deflate = False, use_objstms = 0, compression_effort = 0)
    doc.close()

    out = _fetch_with(monkeypatch, body, "text/html")
    assert "PDF body-window marker" in out


def test_pdf_extraction_caps_pages_and_intermediate_text(monkeypatch):
    from core.rag.parsers import Page

    seen = {}

    def fake_parse(data, *, max_pages = None):
        seen["max_pages"] = max_pages
        pages = [Page(text = "x" * 1000, page_number = i, char_count = 1000) for i in range(1, 51)]
        return pages, 60

    monkeypatch.setattr("core.rag.parsers.parse_pdf_bytes", fake_parse)
    text = tools._extract_pdf_text(b"unused")
    assert seen["max_pages"] == tools._MAX_WEB_PDF_PAGES
    assert len(text) <= tools._MAX_PAGE_CHARS
    assert "text limited to 16,000 characters" in text
    assert "page processing capped at 50 pages" in text


def test_pdf_exactly_at_page_cap_not_marked_capped(monkeypatch):
    from core.rag.parsers import Page

    monkeypatch.setattr(
        "core.rag.parsers.parse_pdf_bytes",
        lambda data, *, max_pages = None: (
            [Page(text = "short", page_number = i, char_count = 5) for i in range(1, 51)],
            50,
        ),
    )
    text = tools._extract_pdf_text(b"unused")
    assert "page processing capped" not in text
    assert "## Page 50\n\nshort" in text


def test_pdf_page_cap_does_not_claim_later_pages_are_textless(monkeypatch):
    from core.rag.parsers import Page
    monkeypatch.setattr(
        "core.rag.parsers.parse_pdf_bytes",
        lambda data, *, max_pages = None: (
            [Page(text = "", page_number = i, char_count = 0) for i in range(1, 51)],
            60,
        ),
    )
    assert tools._extract_pdf_text(b"unused") == (
        "(PDF contains no extractable text in the first 50 pages)"
    )


def test_pdf_result_discarded_after_fetch_deadline(monkeypatch):
    clock = {"time": 1000.0}
    monkeypatch.setattr(tools.time, "monotonic", lambda: clock["time"])

    def slow_extract(data):
        clock["time"] += 10.0
        return "late PDF text"

    monkeypatch.setattr(tools, "_extract_pdf_text", slow_extract)
    out = _fetch_with(monkeypatch, _pdf_bytes("Readable text"), "application/pdf")
    assert out == "Failed to fetch URL: timed out."


def test_text_octet_stream_kept_after_sniffing(monkeypatch):
    body = b"level=info\nmessage=plain text artifact\n" * 100
    out = _fetch_with(monkeypatch, body, "application/octet-stream")
    assert "plain text artifact" in out
    assert "non-text content" not in out and "binary content" not in out


@pytest.mark.parametrize(
    "content_type",
    ["application/octet-stream", "application/x-custom-binary", "text/plain", None],
)
def test_binary_candidates_rejected_after_sniffing(monkeypatch, content_type):
    out = _fetch_with(monkeypatch, bytes(range(256)) * 20, content_type)
    assert "�" not in out
    assert "binary content" in out


@pytest.mark.parametrize("content_type", ["application/sql", "application/x-www-form-urlencoded"])
def test_unknown_application_text_kept_after_sniffing(monkeypatch, content_type):
    out = _fetch_with(monkeypatch, b"select readable_text from artifacts;\n" * 100, content_type)
    assert "readable_text" in out
    assert "non-text content" not in out and "binary content" not in out


def test_excel_labeled_csv_kept_after_sniffing(monkeypatch):
    body = b"name,value\nreadable,42\n" * 100
    out = _fetch_with(monkeypatch, body, "application/vnd.ms-excel")
    assert "readable" in out
    assert "binary content" not in out


@pytest.mark.parametrize(
    "bom,encoding",
    [
        (codecs.BOM_UTF16_LE, "utf-16-le"),
        (codecs.BOM_UTF16_BE, "utf-16-be"),
        (codecs.BOM_UTF32_LE, "utf-32-le"),
        (codecs.BOM_UTF32_BE, "utf-32-be"),
    ],
)
@pytest.mark.parametrize("content_type", ["text/plain", "application/vnd.ms-excel"])
def test_bom_unicode_text_without_charset_kept(monkeypatch, bom, encoding, content_type):
    body = bom + ("name,value\nreadable,42\n" * 100).encode(encoding)
    out = _fetch_with(monkeypatch, body, content_type)
    assert "readable" in out
    assert "binary content" not in out


def test_valid_utf8_binary_caught_by_control_chars(monkeypatch):
    body = bytes([0, 1, 2, 3, 4, 5, 6, 7]) * 400
    out = _fetch_with(monkeypatch, body, "text/plain")
    assert "binary content" in out


@pytest.mark.parametrize(
    "magic",
    [
        b"PK\x03\x04",
        b"\xd0\xcf\x11\xe0\xa1\xb1\x1a\xe1",
        b"\x1f\x8b",
        b"BZh",
        b"\xfd7zXZ\x00",
        b"\x28\xb5\x2f\xfd",
    ],
)
def test_text_labeled_binary_caught_by_magic(monkeypatch, magic):
    out = _fetch_with(monkeypatch, magic + b" printable text-heavy body" * 100, "text/plain")
    assert "binary content" in out


@pytest.mark.parametrize(
    "prefix",
    [
        codecs.BOM_UTF8,
        codecs.BOM_UTF16_LE,
        codecs.BOM_UTF16_BE,
        codecs.BOM_UTF32_LE,
        codecs.BOM_UTF32_BE,
        b" \r\n",
        b"\t\xef\xbb\xbf ",
    ],
)
def test_binary_magic_after_harmless_prefix(monkeypatch, prefix):
    body = prefix + b"\x1f\x8b" + b" printable text-heavy body" * 100
    out = _fetch_with(monkeypatch, body, "text/plain")
    assert "binary content" in out


@pytest.mark.parametrize(
    "content_type,magic",
    [
        ("application/vnd.ms-excel", b"\xd0\xcf\x11\xe0\xa1\xb1\x1a\xe1"),
        (
            "application/vnd.openxmlformats-officedocument.wordprocessingml.document",
            b"PK\x03\x04",
        ),
    ],
)
def test_office_labeled_binary_caught_by_magic(monkeypatch, content_type, magic):
    out = _fetch_with(monkeypatch, magic + b" printable text-heavy body" * 100, content_type)
    assert "binary content" in out


def test_latin1_text_without_charset_kept(monkeypatch):
    body = (
        "Muller lauft uber die Strasse: schoene, groesse. MARKERWORD ".replace("ue", "ü")
        + "äöüß éèà "
    ) * 30
    out = _fetch_with(monkeypatch, body.encode("cp1252"), "text/plain")
    assert "binary content" not in out
    assert "MARKERWORD" in out


@pytest.mark.parametrize("charset", ["iso-8859-1", "latin-1", "latin1"])
def test_declared_latin1_cp1252_punctuation_kept(monkeypatch, charset):
    body = ("“quoted” " * 100).encode("cp1252")
    out = _fetch_with(monkeypatch, body, f"text/plain; charset={charset}")
    assert "quoted" in out
    assert "binary content" not in out


@pytest.mark.parametrize("charset", ["unicode", "utf8mb4", "x-user-defined"])
def test_unknown_charset_falls_back_instead_of_failing_the_fetch(monkeypatch, charset):
    body = b"<html><body><p>MARKERWORD in a page a browser renders fine.</p></body></html>"
    out = _fetch_with(monkeypatch, body, f"text/html; charset={charset}")
    assert "MARKERWORD" in out
    assert "unknown encoding" not in out


def test_unknown_charset_still_reaches_the_bom_codec(monkeypatch):
    body = codecs.BOM_UTF16_LE + "MARKERWORD and more text".encode("utf-16-le")
    out = _fetch_with(monkeypatch, body, "text/plain; charset=utf16le")
    assert "MARKERWORD" in out


@pytest.mark.parametrize(
    "charset",
    ["base64", "hex", "zlib", "quopri", "rot13", "uu", "bz2", "undefined", "idna"],
)
def test_unusable_codec_charset_falls_back(monkeypatch, charset):
    # These pass codecs.lookup and fail at decode: guarding the lookup alone lost the page.
    body = b"<html><body><p>MARKERWORD in a page a browser renders fine.</p></body></html>"
    out = _fetch_with(monkeypatch, body, f"text/html; charset={charset}")
    assert "MARKERWORD" in out
    assert "not a text encoding" not in out


def test_malformed_charset_falls_back(monkeypatch):
    # A NUL in the label raises ValueError out of codecs.lookup, not LookupError.
    body = b"<html><body><p>MARKERWORD in a page a browser renders fine.</p></body></html>"
    out = _fetch_with(monkeypatch, body, 'text/html; charset="utf-8\x00"')
    assert "MARKERWORD" in out
    assert "Failed to fetch URL" not in out


@pytest.mark.parametrize("charset", ["utf-16", "utf-16-le", "utf-32"])
def test_declared_wide_codec_still_honoured(monkeypatch, charset):
    body = "MARKERWORD and more text".encode(charset)
    out = _fetch_with(monkeypatch, body, f"text/plain; charset={charset}")
    assert "MARKERWORD" in out


def test_high_byte_binary_not_rescued_as_cp1252(monkeypatch):
    body = bytes(range(0xA0, 0x100)) * 40
    out = _fetch_with(monkeypatch, body, "text/plain")
    assert "binary content" in out


def test_ansi_colored_text_log_kept(monkeypatch):
    # ESC is excluded from the binary set so ANSI logs remain readable.
    line = "".join(f"\x1b[32m+{i}\x1b[0m\n" for i in range(300)).encode()
    out = _fetch_with(monkeypatch, line, "text/plain")
    assert "binary content" not in out


def test_html_page_unaffected(monkeypatch):
    html = b"<html><body><h1>Hello</h1><p>Real text content here.</p></body></html>"
    out = _fetch_with(monkeypatch, html, "text/html; charset=utf-8")
    assert "Hello" in out
    assert "non-text content" not in out and "binary content" not in out


def test_article_after_a_large_inline_head_is_read(monkeypatch):
    style = b".c{color:red}\n" * (2 * 1024 * 1024 // 14)
    html = (
        b"<html><head><style>"
        + style
        + b"</style></head><body><article><h1>Brightline files for bankruptcy</h1>"
        + b"<p>The rail operator filed for Chapter 11 protection on Friday.</p></article></body></html>"
    )
    out = _fetch_with(monkeypatch, html, "text/html; charset=utf-8")
    assert "Brightline files for bankruptcy" in out
    assert "Chapter 11 protection" in out


def test_article_after_a_utf8_bom_and_large_inline_head_is_read(monkeypatch):
    html = (
        codecs.BOM_UTF8
        + b"<html><head><style>"
        + b"x" * (768 * 1024)
        + b"</style></head><body><article><h1>BOM article body</h1></article></body></html>"
    )
    out = _fetch_with(monkeypatch, html, "text/html; charset=utf-8")
    assert "BOM article body" in out


def test_body_text_inside_head_script_does_not_end_the_read(monkeypatch):
    html = (
        b'<html><head><script>const example = "<body>";</script><style>'
        + b"x" * (768 * 1024)
        + b"</style></head><body><article><h1>Actual article body</h1></article></body></html>"
    )
    out = _fetch_with(monkeypatch, html, "text/html; charset=utf-8")
    assert "Actual article body" in out


@pytest.mark.parametrize("reference", ["&#10;", "&#x20;", "&Tab;", "&NewLine;"])
def test_whitespace_reference_in_large_head_keeps_article(monkeypatch, reference):
    html = (
        "<html><head>"
        + reference
        + "<style>"
        + "x" * (768 * 1024)
        + "</style></head><body><h1>Whitespace article marker</h1></body></html>"
    ).encode()
    assert "Whitespace article marker" in _fetch_with(monkeypatch, html, "text/html")


@pytest.mark.parametrize(
    "codec", ["utf-16", "utf-32", "utf-16-le", "utf-16-be", "utf-32-le", "utf-32-be"]
)
def test_wide_encoded_large_head_keeps_article(monkeypatch, codec):
    html = (
        "<html><head><style>/*"
        + "🌍" * (200 * 1024)
        + "*/</style></head><body><h1>Wide article marker</h1></body></html>"
    ).encode(codec)
    content_type = "text/html" if codec in ("utf-16", "utf-32") else f"text/html; charset={codec}"
    assert "Wide article marker" in _fetch_with(monkeypatch, html, content_type)


def test_html_body_locator_does_not_store_every_newline():
    tracemalloc.start()
    try:
        locator = tools._HTMLBodyLocator()
        locator.feed_bytes(b"\n" * (256 * 1024))
        _, peak = tracemalloc.get_traced_memory()
    finally:
        tracemalloc.stop()

    assert locator.body_at is None
    assert peak < 4 * 1024 * 1024


def test_unterminated_html_token_has_linear_reparse_work(monkeypatch):
    payload = b"<" + b"x" * (4 * 1024 * 1024)
    resp = _FakeResp(payload, "text/html")
    scanned = 0
    feed = tools.HTMLParser.feed

    def counted_feed(parser, data):
        nonlocal scanned
        scanned += len(parser.rawdata) + len(data)
        return feed(parser, data)

    monkeypatch.setattr(tools.HTMLParser, "feed", counted_feed)
    error, body = tools._read_capped_body(
        resp,
        tools._MAX_HTML_FETCH_BYTES,
        timeout = 5,
        deadline = None,
        cancel_event = None,
        body_window = tools._MAX_FETCH_BYTES,
    )
    assert error is None
    assert body == payload
    assert scanned < 3 * len(payload)


@pytest.mark.parametrize(
    "opening,closing", [(b"<!--", b"-->"), (b'<link href="data:image/png;base64,', b'">')]
)
def test_large_head_token_keeps_article(monkeypatch, opening, closing):
    html = (
        b"<html><head>"
        + opening
        + b"x" * (768 * 1024)
        + closing
        + b"</head><body><h1>Long token article marker</h1></body></html>"
    )
    assert "Long token article marker" in _fetch_with(monkeypatch, html, "text/html")


_MENU = b'<li><a href="/api/item">Menu entry</a></li>\n'
_LINES = b"<p>line</p>" * (1024 * 1024)


def _menu_page(content: bytes) -> bytes:
    menu = _MENU * (700 * 1024 // len(_MENU))
    return (
        b"<html><head><title>t</title></head><body><nav><ul>"
        + menu
        + b"</ul></nav>"
        + content
        + b"</body></html>"
    )


def _read_html(resp, cancel_event = None):
    return tools._read_capped_body(
        resp,
        tools._MAX_HTML_FETCH_BYTES,
        timeout = 5,
        deadline = None,
        cancel_event = cancel_event,
        body_window = tools._MAX_FETCH_BYTES,
    )


def _first_window(body: bytes) -> bytes:
    return body[: body.index(b"</head>") + tools._MAX_FETCH_BYTES]


@pytest.mark.parametrize("tag", [b"main", b"article"])
def test_page_text_after_a_large_menu_is_read(monkeypatch, tag):
    html = _menu_page(b"<%b><h1>Dense layer</h1><p>BODY-MARK takes units.</p></%b>" % (tag, tag))
    assert "BODY-MARK takes units." in _fetch_with(monkeypatch, html, "text/html")


def test_page_text_after_a_large_menu_gets_a_full_window():
    body = _menu_page(b"<main><h1>Start</h1>" + _LINES + b"</main>")
    resp = _FakeResp(body, "text/html")
    error, read = _read_html(resp)
    window_end = body.index(b"<main>") + tools._MAX_FETCH_BYTES
    assert error is None
    assert window_end <= resp._pos < window_end + 2 * 65536
    assert read == body[: resp._pos]


def test_page_text_inside_the_window_reads_no_further():
    body = b"<html><head><title>t</title></head><body><main><h1>Start</h1>" + _LINES + b"</main>"
    resp = _FakeResp(body, "text/html")
    assert _read_html(resp) == (None, _first_window(body))
    assert resp._pos == len(_first_window(body))


def test_page_without_a_content_start_keeps_the_first_window():
    body = b"<html><head><title>t</title></head><body>" + _LINES + b"</body></html>"
    assert _read_html(_FakeResp(body, "text/html")) == (None, _first_window(body))


def test_failed_read_past_the_window_keeps_the_first_window():
    body = b"<html><head><title>t</title></head><body>" + _LINES + b"</body></html>"
    resp = _FakeResp(body, "text/html")
    read = resp.read

    def stalling_read(n = None):
        if resp._pos > 1024 * 1024:
            raise TimeoutError("timed out")
        return read(n)

    resp.read = stalling_read
    assert _read_html(resp) == (None, _first_window(body))


def test_failed_read_after_a_late_content_start_keeps_the_first_window():
    body = _menu_page(b"<main><h1>Start</h1>" + _LINES + b"</main>")
    resp = _FakeResp(body, "text/html")
    read = resp.read

    def stalling_read(n = None):
        if resp._pos > body.index(b"<main>") + 65536:
            raise TimeoutError("timed out")
        return read(n)

    resp.read = stalling_read
    assert _read_html(resp) == (None, _first_window(body))


def test_read_past_the_window_stops_when_cancelled():
    cancel = threading.Event()
    resp = _FakeResp(_menu_page(_LINES), "text/html")
    read = resp.read

    def cancelling_read(n = None):
        if resp._pos > 1024 * 1024:
            cancel.set()
        return read(n)

    resp.read = cancelling_read
    assert _read_html(resp, cancel) == ("Failed to fetch URL: cancelled.", b"")


@pytest.mark.parametrize(
    "content_type,body",
    [
        (
            "text/html",
            b"<html><head><title>t</title></head><body><article>"
            + b"<h1>Harbor ferry adds night service</h1><p>Boats run every thirty minutes.</p></article>"
            + b"<script>"
            + b"x" * (4 * 1024 * 1024)
            + b"</script></body></html>",
        ),
        (
            "text/html",
            b"<html><head><title>t</title></head><main>"
            + b"<h1>Harbor ferry adds night service</h1><p>Boats run every thirty minutes.</p></main>"
            + b"<script>"
            + b"x" * (4 * 1024 * 1024)
            + b"</script></html>",
        ),
        (
            "text/html",
            b"<html><head><title>t</title></head><body><p>Harbor ferry adds night service</p>"
            + b"<p>log line</p>" * (512 * 1024)
            + b"</body></html>",
        ),
        ("text/plain", b"Harbor ferry adds night service\n" + b"log line\n" * (512 * 1024)),
    ],
    ids = ["html", "html-without-body-tag", "html-without-content-start", "text"],
)
def test_large_page_on_a_slow_link_still_returns_its_start(monkeypatch, content_type, body):
    clock = {"time": 1000.0}
    monkeypatch.setattr(tools.time, "monotonic", lambda: clock["time"])
    resp = _FakeResp(body, content_type)
    read = resp.read

    def slow_read(n = None):
        chunk = read(n)
        clock["time"] += len(chunk) / (64 * 1024)
        return chunk

    resp.read = slow_read
    monkeypatch.setattr(
        tools, "_validate_and_resolve_host", lambda host, port: (True, "", ["93.184.216.34"])
    )
    monkeypatch.setattr(tools.urllib.request, "build_opener", lambda *a, **k: _FakeOpener(resp))
    out = tools._fetch_page_text("https://example.com/thing", timeout = 30)
    assert "Harbor ferry adds night service" in out


def test_content_type_sanitized_in_message(monkeypatch):
    out = _fetch_with(monkeypatch, b"PK\x03\x04" * 500, "application/zip\r\n data: injected")
    assert "\n" not in out and "\r" not in out
    assert "injected" not in out
    assert "application/zip" in out


@pytest.mark.parametrize(
    "n_bad,n_total,expect_binary",
    [
        (120, 1000, False),
        (130, 1000, True),
    ],
)
def test_binary_char_ratio_boundary(monkeypatch, n_bad, n_total, expect_binary):
    body = b"\x00" * n_bad + b"a" * (n_total - n_bad)
    out = _fetch_with(monkeypatch, body, "text/plain")
    assert ("binary content" in out) is expect_binary


def test_text_with_a_few_stray_replacement_chars_kept(monkeypatch):
    body = ("Real article text. " * 200).encode() + b"\xff\xfe\xff"
    out = _fetch_with(monkeypatch, body, "text/html")
    assert "Real article text." in out
    assert "binary content" not in out


_JAPANESE = "価格.com は日本最大級の購買支援サイトです。製品の価格比較とクチコミ。"


def _html_page(head: str, text: str) -> str:
    return (
        f"<!doctype html><html><head>{head}<title>t</title></head><body><p>{text}</p></body></html>"
    )


@pytest.mark.parametrize(
    "head",
    [
        '<meta charset="Shift_JIS">',
        "<meta charset=shift_jis>",
        '<meta charset="x-sjis">',
        '<meta charset="windows-31j">',
        '<meta http-equiv="Content-Type" content="text/html; charset=Shift_JIS">',
        '<meta content="text/html; charset=shift_jis" http-equiv="content-type">',
        '<META HTTP-EQUIV="Content-Type" CONTENT="text/html; charset=Shift_JIS">',
        "<meta http-equiv=content-type content=\"text/html; charset='shift_jis'\">",
    ],
)
def test_meta_charset_read_when_header_names_none(monkeypatch, head):
    body = _html_page(head, _JAPANESE * 40).encode("cp932")
    out = _fetch_with(monkeypatch, body, "text/html")
    assert _JAPANESE in out
    assert "�" not in out
    assert "binary content" not in out


@pytest.mark.parametrize("content_type", ["text/html", "application/xhtml+xml", "text/xml"])
def test_xml_prolog_encoding_read_when_header_names_none(monkeypatch, content_type):
    body = ('<?xml version="1.0" encoding="EUC-JP"?>\n' + _html_page("", _JAPANESE * 40)).encode(
        "euc_jp"
    )
    out = _fetch_with(monkeypatch, body, content_type)
    assert _JAPANESE in out
    assert "�" not in out


@pytest.mark.parametrize(
    "label,encoding,text",
    [
        ("gb2312", "gbk", "镕 GBK-only MARKERWORD"),
        ("latin1", "cp1252", "“quoted” MARKERWORD"),
        ("iso-8859-1", "cp1252", "“quoted” MARKERWORD"),
        ("windows-1251", "cp1251", "Привет MARKERWORD"),
        ("euc-jp", "euc_jp", _JAPANESE),
        ("euc-kr", "cp949", "작성자: 김똠, 댓글: 햏햏 뷁 MARKERWORD"),
        ("ks_c_5601-1987", "cp949", "작성자: 김똠, 댓글: 햏햏 뷁 MARKERWORD"),
        ("big5", "big5hkscs", "台灣裏面恒春 碁盤 MARKERWORD"),
        ("tis-620", "cp874", "ภาษาไทย “คำพูด” … MARKERWORD"),
        ("iso-8859-9", "cp1254", "Türkiye’nin “Ankara”dır… MARKERWORD"),
    ],
)
def test_meta_charset_labels_decode_with_whatwg_codecs(monkeypatch, label, encoding, text):
    body = _html_page(f'<meta charset="{label}">', text * 40).encode(encoding)
    out = _fetch_with(monkeypatch, body, "text/html")
    assert text in out
    assert "�" not in out


def test_header_charset_wins_over_contradicting_meta(monkeypatch):
    body = _html_page('<meta charset="Shift_JIS">', _JAPANESE * 40).encode("utf-8")
    out = _fetch_with(monkeypatch, body, "text/html; charset=utf-8")
    assert _JAPANESE in out
    assert "�" not in out


def test_bom_wins_over_contradicting_meta(monkeypatch):
    body = codecs.BOM_UTF8 + _html_page('<meta charset="Shift_JIS">', _JAPANESE * 40).encode()
    out = _fetch_with(monkeypatch, body, "text/html")
    assert _JAPANESE in out
    assert "�" not in out


@pytest.mark.parametrize(
    "label",
    [
        "x-mac-fantasy",
        "utf8mb4",
        "base64",
        "rot13",
        "idna",
        "undefined",
        "utf-7",
        "utf-32",
        "punycode",
    ],
)
def test_unknown_meta_charset_falls_back_to_utf8_without_raising(monkeypatch, label):
    body = _html_page(f'<meta charset="{label}">', "MARKERWORD über " * 40).encode("utf-8")
    out = _fetch_with(monkeypatch, body, "text/html")
    assert "MARKERWORD über" in out
    assert "Failed to fetch URL" not in out


@pytest.mark.parametrize("meta", ['<meta charset="utf-8">', '<meta charset="utf-16">'])
def test_meta_charset_keeps_cp1252_rescue_for_mislabeled_page(monkeypatch, meta):
    text = "¿Qué pasó? Canción, corazón, niño, España, José, señor. "
    body = _html_page(meta, text * 30).encode("cp1252")
    out = _fetch_with(monkeypatch, body, "text/html")
    assert text.strip() in out
    assert "binary content" not in out


def test_commented_out_meta_charset_is_ignored(monkeypatch):
    text = "Ünïcödé “smart” 日本語 MARKERWORD "
    head = '<!--[if IE]><meta charset="iso-8859-1"><![endif]--><meta charset="utf-8">'
    body = _html_page(head, text * 20).encode("utf-8")
    out = _fetch_with(monkeypatch, body, "text/html")
    assert text.strip() in out
    assert "Ã" not in out


@pytest.mark.parametrize("header_label", ["x-bogus-label", "base64"])
def test_meta_charset_used_when_header_charset_unusable(monkeypatch, header_label):
    body = _html_page('<meta charset="Shift_JIS">', _JAPANESE * 40).encode("cp932")
    out = _fetch_with(monkeypatch, body, f"text/html; charset={header_label}")
    assert _JAPANESE in out
    assert "�" not in out


def test_unusable_meta_charset_does_not_stop_the_scan(monkeypatch):
    head = '<meta charset="bogus"><meta charset="Shift_JIS">'
    body = _html_page(head, _JAPANESE * 40).encode("cp932")
    out = _fetch_with(monkeypatch, body, "text/html")
    assert _JAPANESE in out
    assert "�" not in out


@pytest.mark.parametrize(
    "head",
    [
        '<meta name="description" content="docs about charset=shift_jis">',
        '<meta http-equiv="refresh" content="0; url=/next?charset=shift_jis">',
        '<meta name="Content-Type" content="text/html; charset=shift_jis">',
        '<meta name="x" content="<meta charset=shift_jis>">',
        "<script data-x='<meta charset=\"shift_jis\">'></script>",
        '<meta charset="shift<!-- x -->_jis">',
        '<meta-info charset="shift_jis"><metadata charset="shift_jis">',
    ],
)
def test_charset_text_outside_a_declaration_is_ignored(monkeypatch, head):
    text = "Ünïcödé “smart” 日本語 MARKERWORD "
    body = _html_page(head, text * 20).encode("utf-8")
    out = _fetch_with(monkeypatch, body, "text/html")
    assert text.strip() in out
    assert "�" not in out


def test_meta_charset_wins_over_xml_prolog(monkeypatch):
    text = "Café crème brûlée — MARKERWORD "
    head = '<meta http-equiv="Content-Type" content="text/html; charset=utf-8">'
    body = ('<?xml version="1.0" encoding="iso-8859-1"?>\n' + _html_page(head, text * 20)).encode()
    out = _fetch_with(monkeypatch, body, "text/html")
    assert text.strip() in out
    assert "Ã" not in out


@pytest.mark.parametrize(
    "head",
    [
        b"<a" * 1024,
        b'<html><head><script data-x="' + b"<a" * 1024,
        b"<html><head><script>" + b"x<y;" * 512,
        b'<html><head><script data-x=\'<meta charset="shift_jis">' + b"x" * 2048,
    ],
)
def test_meta_prescan_stops_at_an_unterminated_tag(head):
    start = time.perf_counter()
    assert tools._sniff_meta_charset(head[: tools._META_CHARSET_SCAN_BYTES], "text/html") is None
    assert time.perf_counter() - start < 1


def test_whatwg_charset_table_only_names_text_codecs():
    for label, codec in tools._WHATWG_CHARSET_CODECS.items():
        assert label == label.lower()
        assert codecs.lookup(codec)._is_text_encoding, label
        assert b"a\xff".decode(codec, "replace")


@pytest.mark.parametrize(
    "content_type", ["text/plain", "application/json", "application/xml", "text/css"]
)
def test_meta_charset_in_non_html_body_is_ignored(monkeypatch, content_type):
    text = "日本語のテキスト。価格比較とクチコミ。MARKERWORD "
    body = ('<p>Example:</p> <meta charset="shift_jis"/> ' + text * 40).encode("utf-8")
    out = _fetch_with(monkeypatch, body, content_type)
    assert text.strip() in out
    assert "�" not in out


def test_meta_charset_read_when_content_type_missing(monkeypatch):
    body = _html_page('<meta charset="Shift_JIS">', _JAPANESE * 40).encode("cp932")
    out = _fetch_with(monkeypatch, body, None)
    assert _JAPANESE in out
    assert "�" not in out


def test_meta_charset_in_headerless_non_html_body_is_ignored(monkeypatch):
    text = "日本語のテキスト。価格比較とクチコミ。MARKERWORD "
    body = ('# Encoding\n\nUse `<meta charset="shift_jis">` on old pages.\n\n' + text * 40).encode()
    out = _fetch_with(monkeypatch, body, None)
    assert text.strip() in out
    assert "�" not in out


def test_xml_prolog_encoding_read_when_content_type_missing(monkeypatch):
    body = (
        '<?xml version="1.0" encoding="Shift_JIS"?>\n<doc><p>' + _JAPANESE * 40 + "</p></doc>"
    ).encode("cp932")
    out = _fetch_with(monkeypatch, body, None)
    assert _JAPANESE in out
    assert "�" not in out