Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
# SPDX-License-Identifier: AGPL-3.0-only
# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
"""Regression tests for web fetch decoding: binary bodies poisoning model context (#7084) and page charsets."""
from __future__ import annotations
import codecs
import random
import sys
import threading
import time
import tracemalloc
from email.message import Message
from pathlib import Path
import pytest
_BACKEND_DIR = str(Path(__file__).resolve().parent.parent)
if _BACKEND_DIR not in sys.path:
sys.path.insert(0, _BACKEND_DIR)
from core.inference import tools
class _FakeResp:
def __init__(self, body: bytes, content_type: str | None):
self._body = body
self._pos = 0
self.headers = Message()
if content_type is not None:
self.headers["Content-Type"] = content_type
def read(self, n: int | None = None) -> bytes:
chunk = self._body[self._pos :] if n is None else self._body[self._pos : self._pos + n]
self._pos += len(chunk)
return chunk
class _FakeOpener:
def __init__(self, resp):
self._resp = resp
def open(
self,
req,
timeout = None,
):
return self._resp
def _fetch_with(monkeypatch, body: bytes, content_type: str | None) -> str:
monkeypatch.setattr(
tools, "_validate_and_resolve_host", lambda host, port: (True, "", ["93.184.216.34"])
)
monkeypatch.setattr(
tools.urllib.request,
"build_opener",
lambda *a, **k: _FakeOpener(_FakeResp(body, content_type)),
)
return tools._fetch_page_text("https://example.com/thing", timeout = 5)
def _pdf_bytes(*page_texts: str) -> bytes:
pymupdf = pytest.importorskip("pymupdf")
doc = pymupdf.open()
for text in page_texts:
page = doc.new_page()
if text:
page.insert_textbox(pymupdf.Rect(40, 40, 550, 750), text, fontsize = 11)
data = doc.tobytes()
doc.close()
return data
@pytest.mark.parametrize(
"content_type,expected",
[
("text/html", True),
("text/plain; charset=utf-8", True),
("application/json", True),
("application/json; charset=utf-8", True),
("application/xml", True),
("application/xhtml+xml", True),
("application/ld+json", True),
("application/yaml", True),
("application/x-yaml", True),
("application/x-ndjson", True),
("application/ndjson", True),
("application/sql", True),
("application/x-www-form-urlencoded", True),
("application/pdf", False),
("image/png", False),
("image/svg+xml", False),
("application/octet-stream", True),
("application/zip", False),
("application/vnd.ms-excel", True),
("application/vnd.openxmlformats-officedocument.wordprocessingml.document", True),
("", True),
(None, True),
],
)
def test_is_text_candidate_content_type(content_type, expected):
assert tools._is_text_candidate_content_type(content_type) is expected
@pytest.mark.parametrize(
"content_type",
["application/pdf", "application/octet-stream", "text/html", "text/plain", None],
)
def test_pdf_text_extracted(monkeypatch, content_type):
out = _fetch_with(
monkeypatch,
_pdf_bytes("First page marker", "Second page marker"),
content_type,
)
assert "## Page 1\n\nFirst page marker" in out
assert "## Page 2" in out and "Second page marker" in out
assert "binary content" not in out and "non-text content" not in out
@pytest.mark.parametrize("content_type", ["application/pdf", "text/plain"])
def test_malformed_pdf_returns_safe_placeholder(monkeypatch, content_type):
out = _fetch_with(monkeypatch, b"%PDF-1.7\nnot a complete PDF", content_type)
assert out == "(PDF content could not be read as text)"
def test_pdf_without_text_layer_reported(monkeypatch):
out = _fetch_with(monkeypatch, _pdf_bytes(""), "application/pdf")
assert out == "(PDF contains no extractable text)"
def test_encrypted_pdf_returns_safe_placeholder(monkeypatch):
pymupdf = pytest.importorskip("pymupdf")
doc = pymupdf.open()
doc.new_page().insert_text((40, 40), "private text")
data = doc.tobytes(
encryption = pymupdf.PDF_ENCRYPT_AES_256,
owner_pw = "owner",
user_pw = "secret",
)
doc.close()
out = _fetch_with(monkeypatch, data, "application/pdf")
assert out == "(PDF content could not be read as text)"
def test_pdf_download_limit_enforced(monkeypatch):
monkeypatch.setattr(tools, "_MAX_PDF_FETCH_BYTES", 256)
out = _fetch_with(monkeypatch, _pdf_bytes("Readable but oversized"), "application/pdf")
assert out == "(PDF content exceeds the download limit; not readable as text)"
def test_mislabeled_pdf_is_read_past_text_download_cap(monkeypatch):
body = _pdf_bytes("Cross-reference data was fetched")
monkeypatch.setattr(tools, "_MAX_FETCH_BYTES", 128)
monkeypatch.setattr(tools, "_MAX_PDF_FETCH_BYTES", len(body) + 100)
out = _fetch_with(monkeypatch, body, "text/plain")
assert "Cross-reference data was fetched" in out
def test_html_labeled_pdf_with_body_bytes_is_read_to_eof(monkeypatch):
pymupdf = pytest.importorskip("pymupdf")
doc = pymupdf.open()
payload = b"<body>" + random.Random(0).randbytes(1024 * 1024)
doc.embfile_add("blob.bin", payload, ufilename = "blob.bin")
page = doc.new_page()
page.insert_text((40, 40), "PDF body-window marker")
body = doc.tobytes(deflate = False, use_objstms = 0, compression_effort = 0)
doc.close()
out = _fetch_with(monkeypatch, body, "text/html")
assert "PDF body-window marker" in out
def test_pdf_extraction_caps_pages_and_intermediate_text(monkeypatch):
from core.rag.parsers import Page
seen = {}
def fake_parse(data, *, max_pages = None):
seen["max_pages"] = max_pages
pages = [Page(text = "x" * 1000, page_number = i, char_count = 1000) for i in range(1, 51)]
return pages, 60
monkeypatch.setattr("core.rag.parsers.parse_pdf_bytes", fake_parse)
text = tools._extract_pdf_text(b"unused")
assert seen["max_pages"] == tools._MAX_WEB_PDF_PAGES
assert len(text) <= tools._MAX_PAGE_CHARS
assert "text limited to 16,000 characters" in text
assert "page processing capped at 50 pages" in text
def test_pdf_exactly_at_page_cap_not_marked_capped(monkeypatch):
from core.rag.parsers import Page
monkeypatch.setattr(
"core.rag.parsers.parse_pdf_bytes",
lambda data, *, max_pages = None: (
[Page(text = "short", page_number = i, char_count = 5) for i in range(1, 51)],
50,
),
)
text = tools._extract_pdf_text(b"unused")
assert "page processing capped" not in text
assert "## Page 50\n\nshort" in text
def test_pdf_page_cap_does_not_claim_later_pages_are_textless(monkeypatch):
from core.rag.parsers import Page
monkeypatch.setattr(
"core.rag.parsers.parse_pdf_bytes",
lambda data, *, max_pages = None: (
[Page(text = "", page_number = i, char_count = 0) for i in range(1, 51)],
60,
),
)
assert tools._extract_pdf_text(b"unused") == (
"(PDF contains no extractable text in the first 50 pages)"
)
def test_pdf_result_discarded_after_fetch_deadline(monkeypatch):
clock = {"time": 1000.0}
monkeypatch.setattr(tools.time, "monotonic", lambda: clock["time"])
def slow_extract(data):
clock["time"] += 10.0
return "late PDF text"
monkeypatch.setattr(tools, "_extract_pdf_text", slow_extract)
out = _fetch_with(monkeypatch, _pdf_bytes("Readable text"), "application/pdf")
assert out == "Failed to fetch URL: timed out."
def test_text_octet_stream_kept_after_sniffing(monkeypatch):
body = b"level=info\nmessage=plain text artifact\n" * 100
out = _fetch_with(monkeypatch, body, "application/octet-stream")
assert "plain text artifact" in out
assert "non-text content" not in out and "binary content" not in out
@pytest.mark.parametrize(
"content_type",
["application/octet-stream", "application/x-custom-binary", "text/plain", None],
)
def test_binary_candidates_rejected_after_sniffing(monkeypatch, content_type):
out = _fetch_with(monkeypatch, bytes(range(256)) * 20, content_type)
assert "�" not in out
assert "binary content" in out
@pytest.mark.parametrize("content_type", ["application/sql", "application/x-www-form-urlencoded"])
def test_unknown_application_text_kept_after_sniffing(monkeypatch, content_type):
out = _fetch_with(monkeypatch, b"select readable_text from artifacts;\n" * 100, content_type)
assert "readable_text" in out
assert "non-text content" not in out and "binary content" not in out
def test_excel_labeled_csv_kept_after_sniffing(monkeypatch):
body = b"name,value\nreadable,42\n" * 100
out = _fetch_with(monkeypatch, body, "application/vnd.ms-excel")
assert "readable" in out
assert "binary content" not in out
@pytest.mark.parametrize(
"bom,encoding",
[
(codecs.BOM_UTF16_LE, "utf-16-le"),
(codecs.BOM_UTF16_BE, "utf-16-be"),
(codecs.BOM_UTF32_LE, "utf-32-le"),
(codecs.BOM_UTF32_BE, "utf-32-be"),
],
)
@pytest.mark.parametrize("content_type", ["text/plain", "application/vnd.ms-excel"])
def test_bom_unicode_text_without_charset_kept(monkeypatch, bom, encoding, content_type):
body = bom + ("name,value\nreadable,42\n" * 100).encode(encoding)
out = _fetch_with(monkeypatch, body, content_type)
assert "readable" in out
assert "binary content" not in out
def test_valid_utf8_binary_caught_by_control_chars(monkeypatch):
body = bytes([0, 1, 2, 3, 4, 5, 6, 7]) * 400
out = _fetch_with(monkeypatch, body, "text/plain")
assert "binary content" in out
@pytest.mark.parametrize(
"magic",
[
b"PK\x03\x04",
b"\xd0\xcf\x11\xe0\xa1\xb1\x1a\xe1",
b"\x1f\x8b",
b"BZh",
b"\xfd7zXZ\x00",
b"\x28\xb5\x2f\xfd",
],
)
def test_text_labeled_binary_caught_by_magic(monkeypatch, magic):
out = _fetch_with(monkeypatch, magic + b" printable text-heavy body" * 100, "text/plain")
assert "binary content" in out
@pytest.mark.parametrize(
"prefix",
[
codecs.BOM_UTF8,
codecs.BOM_UTF16_LE,
codecs.BOM_UTF16_BE,
codecs.BOM_UTF32_LE,
codecs.BOM_UTF32_BE,
b" \r\n",
b"\t\xef\xbb\xbf ",
],
)
def test_binary_magic_after_harmless_prefix(monkeypatch, prefix):
body = prefix + b"\x1f\x8b" + b" printable text-heavy body" * 100
out = _fetch_with(monkeypatch, body, "text/plain")
assert "binary content" in out
@pytest.mark.parametrize(
"content_type,magic",
[
("application/vnd.ms-excel", b"\xd0\xcf\x11\xe0\xa1\xb1\x1a\xe1"),
(
"application/vnd.openxmlformats-officedocument.wordprocessingml.document",
b"PK\x03\x04",
),
],
)
def test_office_labeled_binary_caught_by_magic(monkeypatch, content_type, magic):
out = _fetch_with(monkeypatch, magic + b" printable text-heavy body" * 100, content_type)
assert "binary content" in out
def test_latin1_text_without_charset_kept(monkeypatch):
body = (
"Muller lauft uber die Strasse: schoene, groesse. MARKERWORD ".replace("ue", "ü")
+ "äöüß éèà "
) * 30
out = _fetch_with(monkeypatch, body.encode("cp1252"), "text/plain")
assert "binary content" not in out
assert "MARKERWORD" in out
@pytest.mark.parametrize("charset", ["iso-8859-1", "latin-1", "latin1"])
def test_declared_latin1_cp1252_punctuation_kept(monkeypatch, charset):
body = ("“quoted” " * 100).encode("cp1252")
out = _fetch_with(monkeypatch, body, f"text/plain; charset={charset}")
assert "quoted" in out
assert "binary content" not in out
@pytest.mark.parametrize("charset", ["unicode", "utf8mb4", "x-user-defined"])
def test_unknown_charset_falls_back_instead_of_failing_the_fetch(monkeypatch, charset):
body = b"<html><body><p>MARKERWORD in a page a browser renders fine.</p></body></html>"
out = _fetch_with(monkeypatch, body, f"text/html; charset={charset}")
assert "MARKERWORD" in out
assert "unknown encoding" not in out
def test_unknown_charset_still_reaches_the_bom_codec(monkeypatch):
body = codecs.BOM_UTF16_LE + "MARKERWORD and more text".encode("utf-16-le")
out = _fetch_with(monkeypatch, body, "text/plain; charset=utf16le")
assert "MARKERWORD" in out
@pytest.mark.parametrize(
"charset",
["base64", "hex", "zlib", "quopri", "rot13", "uu", "bz2", "undefined", "idna"],
)
def test_unusable_codec_charset_falls_back(monkeypatch, charset):
# These pass codecs.lookup and fail at decode: guarding the lookup alone lost the page.
body = b"<html><body><p>MARKERWORD in a page a browser renders fine.</p></body></html>"
out = _fetch_with(monkeypatch, body, f"text/html; charset={charset}")
assert "MARKERWORD" in out
assert "not a text encoding" not in out
def test_malformed_charset_falls_back(monkeypatch):
# A NUL in the label raises ValueError out of codecs.lookup, not LookupError.
body = b"<html><body><p>MARKERWORD in a page a browser renders fine.</p></body></html>"
out = _fetch_with(monkeypatch, body, 'text/html; charset="utf-8\x00"')
assert "MARKERWORD" in out
assert "Failed to fetch URL" not in out
@pytest.mark.parametrize("charset", ["utf-16", "utf-16-le", "utf-32"])
def test_declared_wide_codec_still_honoured(monkeypatch, charset):
body = "MARKERWORD and more text".encode(charset)
out = _fetch_with(monkeypatch, body, f"text/plain; charset={charset}")
assert "MARKERWORD" in out
def test_high_byte_binary_not_rescued_as_cp1252(monkeypatch):
body = bytes(range(0xA0, 0x100)) * 40
out = _fetch_with(monkeypatch, body, "text/plain")
assert "binary content" in out
def test_ansi_colored_text_log_kept(monkeypatch):
# ESC is excluded from the binary set so ANSI logs remain readable.
line = "".join(f"\x1b[32m+{i}\x1b[0m\n" for i in range(300)).encode()
out = _fetch_with(monkeypatch, line, "text/plain")
assert "binary content" not in out
def test_html_page_unaffected(monkeypatch):
html = b"<html><body><h1>Hello</h1><p>Real text content here.</p></body></html>"
out = _fetch_with(monkeypatch, html, "text/html; charset=utf-8")
assert "Hello" in out
assert "non-text content" not in out and "binary content" not in out
def test_article_after_a_large_inline_head_is_read(monkeypatch):
style = b".c{color:red}\n" * (2 * 1024 * 1024 // 14)
html = (
b"<html><head><style>"
+ style
+ b"</style></head><body><article><h1>Brightline files for bankruptcy</h1>"
+ b"<p>The rail operator filed for Chapter 11 protection on Friday.</p></article></body></html>"
)
out = _fetch_with(monkeypatch, html, "text/html; charset=utf-8")
assert "Brightline files for bankruptcy" in out
assert "Chapter 11 protection" in out
def test_article_after_a_utf8_bom_and_large_inline_head_is_read(monkeypatch):
html = (
codecs.BOM_UTF8
+ b"<html><head><style>"
+ b"x" * (768 * 1024)
+ b"</style></head><body><article><h1>BOM article body</h1></article></body></html>"
)
out = _fetch_with(monkeypatch, html, "text/html; charset=utf-8")
assert "BOM article body" in out
def test_body_text_inside_head_script_does_not_end_the_read(monkeypatch):
html = (
b'<html><head><script>const example = "<body>";</script><style>'
+ b"x" * (768 * 1024)
+ b"</style></head><body><article><h1>Actual article body</h1></article></body></html>"
)
out = _fetch_with(monkeypatch, html, "text/html; charset=utf-8")
assert "Actual article body" in out
@pytest.mark.parametrize("reference", [" ", " ", "	", "
"])
def test_whitespace_reference_in_large_head_keeps_article(monkeypatch, reference):
html = (
"<html><head>"
+ reference
+ "<style>"
+ "x" * (768 * 1024)
+ "</style></head><body><h1>Whitespace article marker</h1></body></html>"
).encode()
assert "Whitespace article marker" in _fetch_with(monkeypatch, html, "text/html")
@pytest.mark.parametrize(
"codec", ["utf-16", "utf-32", "utf-16-le", "utf-16-be", "utf-32-le", "utf-32-be"]
)
def test_wide_encoded_large_head_keeps_article(monkeypatch, codec):
html = (
"<html><head><style>/*"
+ "🌍" * (200 * 1024)
+ "*/</style></head><body><h1>Wide article marker</h1></body></html>"
).encode(codec)
content_type = "text/html" if codec in ("utf-16", "utf-32") else f"text/html; charset={codec}"
assert "Wide article marker" in _fetch_with(monkeypatch, html, content_type)
def test_html_body_locator_does_not_store_every_newline():
tracemalloc.start()
try:
locator = tools._HTMLBodyLocator()
locator.feed_bytes(b"\n" * (256 * 1024))
_, peak = tracemalloc.get_traced_memory()
finally:
tracemalloc.stop()
assert locator.body_at is None
assert peak < 4 * 1024 * 1024
def test_unterminated_html_token_has_linear_reparse_work(monkeypatch):
payload = b"<" + b"x" * (4 * 1024 * 1024)
resp = _FakeResp(payload, "text/html")
scanned = 0
feed = tools.HTMLParser.feed
def counted_feed(parser, data):
nonlocal scanned
scanned += len(parser.rawdata) + len(data)
return feed(parser, data)
monkeypatch.setattr(tools.HTMLParser, "feed", counted_feed)
error, body = tools._read_capped_body(
resp,
tools._MAX_HTML_FETCH_BYTES,
timeout = 5,
deadline = None,
cancel_event = None,
body_window = tools._MAX_FETCH_BYTES,
)
assert error is None
assert body == payload
assert scanned < 3 * len(payload)
@pytest.mark.parametrize(
"opening,closing", [(b"<!--", b"-->"), (b'<link href="data:image/png;base64,', b'">')]
)
def test_large_head_token_keeps_article(monkeypatch, opening, closing):
html = (
b"<html><head>"
+ opening
+ b"x" * (768 * 1024)
+ closing
+ b"</head><body><h1>Long token article marker</h1></body></html>"
)
assert "Long token article marker" in _fetch_with(monkeypatch, html, "text/html")
_MENU = b'<li><a href="/api/item">Menu entry</a></li>\n'
_LINES = b"<p>line</p>" * (1024 * 1024)
def _menu_page(content: bytes) -> bytes:
menu = _MENU * (700 * 1024 // len(_MENU))
return (
b"<html><head><title>t</title></head><body><nav><ul>"
+ menu
+ b"</ul></nav>"
+ content
+ b"</body></html>"
)
def _read_html(resp, cancel_event = None):
return tools._read_capped_body(
resp,
tools._MAX_HTML_FETCH_BYTES,
timeout = 5,
deadline = None,
cancel_event = cancel_event,
body_window = tools._MAX_FETCH_BYTES,
)
def _first_window(body: bytes) -> bytes:
return body[: body.index(b"</head>") + tools._MAX_FETCH_BYTES]
@pytest.mark.parametrize("tag", [b"main", b"article"])
def test_page_text_after_a_large_menu_is_read(monkeypatch, tag):
html = _menu_page(b"<%b><h1>Dense layer</h1><p>BODY-MARK takes units.</p></%b>" % (tag, tag))
assert "BODY-MARK takes units." in _fetch_with(monkeypatch, html, "text/html")
def test_page_text_after_a_large_menu_gets_a_full_window():
body = _menu_page(b"<main><h1>Start</h1>" + _LINES + b"</main>")
resp = _FakeResp(body, "text/html")
error, read = _read_html(resp)
window_end = body.index(b"<main>") + tools._MAX_FETCH_BYTES
assert error is None
assert window_end <= resp._pos < window_end + 2 * 65536
assert read == body[: resp._pos]
def test_page_text_inside_the_window_reads_no_further():
body = b"<html><head><title>t</title></head><body><main><h1>Start</h1>" + _LINES + b"</main>"
resp = _FakeResp(body, "text/html")
assert _read_html(resp) == (None, _first_window(body))
assert resp._pos == len(_first_window(body))
def test_page_without_a_content_start_keeps_the_first_window():
body = b"<html><head><title>t</title></head><body>" + _LINES + b"</body></html>"
assert _read_html(_FakeResp(body, "text/html")) == (None, _first_window(body))
def test_failed_read_past_the_window_keeps_the_first_window():
body = b"<html><head><title>t</title></head><body>" + _LINES + b"</body></html>"
resp = _FakeResp(body, "text/html")
read = resp.read
def stalling_read(n = None):
if resp._pos > 1024 * 1024:
raise TimeoutError("timed out")
return read(n)
resp.read = stalling_read
assert _read_html(resp) == (None, _first_window(body))
def test_failed_read_after_a_late_content_start_keeps_the_first_window():
body = _menu_page(b"<main><h1>Start</h1>" + _LINES + b"</main>")
resp = _FakeResp(body, "text/html")
read = resp.read
def stalling_read(n = None):
if resp._pos > body.index(b"<main>") + 65536:
raise TimeoutError("timed out")
return read(n)
resp.read = stalling_read
assert _read_html(resp) == (None, _first_window(body))
def test_read_past_the_window_stops_when_cancelled():
cancel = threading.Event()
resp = _FakeResp(_menu_page(_LINES), "text/html")
read = resp.read
def cancelling_read(n = None):
if resp._pos > 1024 * 1024:
cancel.set()
return read(n)
resp.read = cancelling_read
assert _read_html(resp, cancel) == ("Failed to fetch URL: cancelled.", b"")
@pytest.mark.parametrize(
"content_type,body",
[
(
"text/html",
b"<html><head><title>t</title></head><body><article>"
+ b"<h1>Harbor ferry adds night service</h1><p>Boats run every thirty minutes.</p></article>"
+ b"<script>"
+ b"x" * (4 * 1024 * 1024)
+ b"</script></body></html>",
),
(
"text/html",
b"<html><head><title>t</title></head><main>"
+ b"<h1>Harbor ferry adds night service</h1><p>Boats run every thirty minutes.</p></main>"
+ b"<script>"
+ b"x" * (4 * 1024 * 1024)
+ b"</script></html>",
),
(
"text/html",
b"<html><head><title>t</title></head><body><p>Harbor ferry adds night service</p>"
+ b"<p>log line</p>" * (512 * 1024)
+ b"</body></html>",
),
("text/plain", b"Harbor ferry adds night service\n" + b"log line\n" * (512 * 1024)),
],
ids = ["html", "html-without-body-tag", "html-without-content-start", "text"],
)
def test_large_page_on_a_slow_link_still_returns_its_start(monkeypatch, content_type, body):
clock = {"time": 1000.0}
monkeypatch.setattr(tools.time, "monotonic", lambda: clock["time"])
resp = _FakeResp(body, content_type)
read = resp.read
def slow_read(n = None):
chunk = read(n)
clock["time"] += len(chunk) / (64 * 1024)
return chunk
resp.read = slow_read
monkeypatch.setattr(
tools, "_validate_and_resolve_host", lambda host, port: (True, "", ["93.184.216.34"])
)
monkeypatch.setattr(tools.urllib.request, "build_opener", lambda *a, **k: _FakeOpener(resp))
out = tools._fetch_page_text("https://example.com/thing", timeout = 30)
assert "Harbor ferry adds night service" in out
def test_content_type_sanitized_in_message(monkeypatch):
out = _fetch_with(monkeypatch, b"PK\x03\x04" * 500, "application/zip\r\n data: injected")
assert "\n" not in out and "\r" not in out
assert "injected" not in out
assert "application/zip" in out
@pytest.mark.parametrize(
"n_bad,n_total,expect_binary",
[
(120, 1000, False),
(130, 1000, True),
],
)
def test_binary_char_ratio_boundary(monkeypatch, n_bad, n_total, expect_binary):
body = b"\x00" * n_bad + b"a" * (n_total - n_bad)
out = _fetch_with(monkeypatch, body, "text/plain")
assert ("binary content" in out) is expect_binary
def test_text_with_a_few_stray_replacement_chars_kept(monkeypatch):
body = ("Real article text. " * 200).encode() + b"\xff\xfe\xff"
out = _fetch_with(monkeypatch, body, "text/html")
assert "Real article text." in out
assert "binary content" not in out
_JAPANESE = "価格.com は日本最大級の購買支援サイトです。製品の価格比較とクチコミ。"
def _html_page(head: str, text: str) -> str:
return (
f"<!doctype html><html><head>{head}<title>t</title></head><body><p>{text}</p></body></html>"
)
@pytest.mark.parametrize(
"head",
[
'<meta charset="Shift_JIS">',
"<meta charset=shift_jis>",
'<meta charset="x-sjis">',
'<meta charset="windows-31j">',
'<meta http-equiv="Content-Type" content="text/html; charset=Shift_JIS">',
'<meta content="text/html; charset=shift_jis" http-equiv="content-type">',
'<META HTTP-EQUIV="Content-Type" CONTENT="text/html; charset=Shift_JIS">',
"<meta http-equiv=content-type content=\"text/html; charset='shift_jis'\">",
],
)
def test_meta_charset_read_when_header_names_none(monkeypatch, head):
body = _html_page(head, _JAPANESE * 40).encode("cp932")
out = _fetch_with(monkeypatch, body, "text/html")
assert _JAPANESE in out
assert "�" not in out
assert "binary content" not in out
@pytest.mark.parametrize("content_type", ["text/html", "application/xhtml+xml", "text/xml"])
def test_xml_prolog_encoding_read_when_header_names_none(monkeypatch, content_type):
body = ('<?xml version="1.0" encoding="EUC-JP"?>\n' + _html_page("", _JAPANESE * 40)).encode(
"euc_jp"
)
out = _fetch_with(monkeypatch, body, content_type)
assert _JAPANESE in out
assert "�" not in out
@pytest.mark.parametrize(
"label,encoding,text",
[
("gb2312", "gbk", "镕 GBK-only MARKERWORD"),
("latin1", "cp1252", "“quoted” MARKERWORD"),
("iso-8859-1", "cp1252", "“quoted” MARKERWORD"),
("windows-1251", "cp1251", "Привет MARKERWORD"),
("euc-jp", "euc_jp", _JAPANESE),
("euc-kr", "cp949", "작성자: 김똠, 댓글: 햏햏 뷁 MARKERWORD"),
("ks_c_5601-1987", "cp949", "작성자: 김똠, 댓글: 햏햏 뷁 MARKERWORD"),
("big5", "big5hkscs", "台灣裏面恒春 碁盤 MARKERWORD"),
("tis-620", "cp874", "ภาษาไทย “คำพูด” … MARKERWORD"),
("iso-8859-9", "cp1254", "Türkiye’nin “Ankara”dır… MARKERWORD"),
],
)
def test_meta_charset_labels_decode_with_whatwg_codecs(monkeypatch, label, encoding, text):
body = _html_page(f'<meta charset="{label}">', text * 40).encode(encoding)
out = _fetch_with(monkeypatch, body, "text/html")
assert text in out
assert "�" not in out
def test_header_charset_wins_over_contradicting_meta(monkeypatch):
body = _html_page('<meta charset="Shift_JIS">', _JAPANESE * 40).encode("utf-8")
out = _fetch_with(monkeypatch, body, "text/html; charset=utf-8")
assert _JAPANESE in out
assert "�" not in out
def test_bom_wins_over_contradicting_meta(monkeypatch):
body = codecs.BOM_UTF8 + _html_page('<meta charset="Shift_JIS">', _JAPANESE * 40).encode()
out = _fetch_with(monkeypatch, body, "text/html")
assert _JAPANESE in out
assert "�" not in out
@pytest.mark.parametrize(
"label",
[
"x-mac-fantasy",
"utf8mb4",
"base64",
"rot13",
"idna",
"undefined",
"utf-7",
"utf-32",
"punycode",
],
)
def test_unknown_meta_charset_falls_back_to_utf8_without_raising(monkeypatch, label):
body = _html_page(f'<meta charset="{label}">', "MARKERWORD über " * 40).encode("utf-8")
out = _fetch_with(monkeypatch, body, "text/html")
assert "MARKERWORD über" in out
assert "Failed to fetch URL" not in out
@pytest.mark.parametrize("meta", ['<meta charset="utf-8">', '<meta charset="utf-16">'])
def test_meta_charset_keeps_cp1252_rescue_for_mislabeled_page(monkeypatch, meta):
text = "¿Qué pasó? Canción, corazón, niño, España, José, señor. "
body = _html_page(meta, text * 30).encode("cp1252")
out = _fetch_with(monkeypatch, body, "text/html")
assert text.strip() in out
assert "binary content" not in out
def test_commented_out_meta_charset_is_ignored(monkeypatch):
text = "Ünïcödé “smart” 日本語 MARKERWORD "
head = '<!--[if IE]><meta charset="iso-8859-1"><![endif]--><meta charset="utf-8">'
body = _html_page(head, text * 20).encode("utf-8")
out = _fetch_with(monkeypatch, body, "text/html")
assert text.strip() in out
assert "Ã" not in out
@pytest.mark.parametrize("header_label", ["x-bogus-label", "base64"])
def test_meta_charset_used_when_header_charset_unusable(monkeypatch, header_label):
body = _html_page('<meta charset="Shift_JIS">', _JAPANESE * 40).encode("cp932")
out = _fetch_with(monkeypatch, body, f"text/html; charset={header_label}")
assert _JAPANESE in out
assert "�" not in out
def test_unusable_meta_charset_does_not_stop_the_scan(monkeypatch):
head = '<meta charset="bogus"><meta charset="Shift_JIS">'
body = _html_page(head, _JAPANESE * 40).encode("cp932")
out = _fetch_with(monkeypatch, body, "text/html")
assert _JAPANESE in out
assert "�" not in out
@pytest.mark.parametrize(
"head",
[
'<meta name="description" content="docs about charset=shift_jis">',
'<meta http-equiv="refresh" content="0; url=/next?charset=shift_jis">',
'<meta name="Content-Type" content="text/html; charset=shift_jis">',
'<meta name="x" content="<meta charset=shift_jis>">',
"<script data-x='<meta charset=\"shift_jis\">'></script>",
'<meta charset="shift<!-- x -->_jis">',
'<meta-info charset="shift_jis"><metadata charset="shift_jis">',
],
)
def test_charset_text_outside_a_declaration_is_ignored(monkeypatch, head):
text = "Ünïcödé “smart” 日本語 MARKERWORD "
body = _html_page(head, text * 20).encode("utf-8")
out = _fetch_with(monkeypatch, body, "text/html")
assert text.strip() in out
assert "�" not in out
def test_meta_charset_wins_over_xml_prolog(monkeypatch):
text = "Café crème brûlée — MARKERWORD "
head = '<meta http-equiv="Content-Type" content="text/html; charset=utf-8">'
body = ('<?xml version="1.0" encoding="iso-8859-1"?>\n' + _html_page(head, text * 20)).encode()
out = _fetch_with(monkeypatch, body, "text/html")
assert text.strip() in out
assert "Ã" not in out
@pytest.mark.parametrize(
"head",
[
b"<a" * 1024,
b'<html><head><script data-x="' + b"<a" * 1024,
b"<html><head><script>" + b"x<y;" * 512,
b'<html><head><script data-x=\'<meta charset="shift_jis">' + b"x" * 2048,
],
)
def test_meta_prescan_stops_at_an_unterminated_tag(head):
start = time.perf_counter()
assert tools._sniff_meta_charset(head[: tools._META_CHARSET_SCAN_BYTES], "text/html") is None
assert time.perf_counter() - start < 1
def test_whatwg_charset_table_only_names_text_codecs():
for label, codec in tools._WHATWG_CHARSET_CODECS.items():
assert label == label.lower()
assert codecs.lookup(codec)._is_text_encoding, label
assert b"a\xff".decode(codec, "replace")
@pytest.mark.parametrize(
"content_type", ["text/plain", "application/json", "application/xml", "text/css"]
)
def test_meta_charset_in_non_html_body_is_ignored(monkeypatch, content_type):
text = "日本語のテキスト。価格比較とクチコミ。MARKERWORD "
body = ('<p>Example:</p> <meta charset="shift_jis"/> ' + text * 40).encode("utf-8")
out = _fetch_with(monkeypatch, body, content_type)
assert text.strip() in out
assert "�" not in out
def test_meta_charset_read_when_content_type_missing(monkeypatch):
body = _html_page('<meta charset="Shift_JIS">', _JAPANESE * 40).encode("cp932")
out = _fetch_with(monkeypatch, body, None)
assert _JAPANESE in out
assert "�" not in out
def test_meta_charset_in_headerless_non_html_body_is_ignored(monkeypatch):
text = "日本語のテキスト。価格比較とクチコミ。MARKERWORD "
body = ('# Encoding\n\nUse `<meta charset="shift_jis">` on old pages.\n\n' + text * 40).encode()
out = _fetch_with(monkeypatch, body, None)
assert text.strip() in out
assert "�" not in out
def test_xml_prolog_encoding_read_when_content_type_missing(monkeypatch):
body = (
'<?xml version="1.0" encoding="Shift_JIS"?>\n<doc><p>' + _JAPANESE * 40 + "</p></doc>"
).encode("cp932")
out = _fetch_with(monkeypatch, body, None)
assert _JAPANESE in out
assert "�" not in out