Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
# SPDX-License-Identifier: AGPL-3.0-only
# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
"""The shared fetcher undoes gzip and deflate Content-Encoding. Servers send one unasked (python.org
does even for ``Accept-Encoding: identity``), and the browser panel, web_fetch and image search all
read through this fetcher."""
from __future__ import annotations
import gzip
import sys
import urllib.error
import zlib
from email.message import Message
from pathlib import Path
import pytest
_BACKEND_DIR = str(Path(__file__).resolve().parent.parent)
if _BACKEND_DIR not in sys.path:
sys.path.insert(0, _BACKEND_DIR)
from core.inference import tools
_PAGE = (
b"<!doctype html><html><head><title>About</title></head><body><h1>About Python</h1>"
+ b"x" * 200_000
+ b"</body></html>"
)
class _FakeResp:
def __init__(
self,
body: bytes,
headers: dict,
step: int = 4096,
):
self._body = body
self._pos = 0
self._step = step
self.headers = Message()
for name, value in headers.items():
self.headers[name] = value
def read1(self, n: int = -1) -> bytes:
chunk = self._body[self._pos : self._pos + min(n, self._step)]
self._pos += len(chunk)
return chunk
read = read1
def _serve(monkeypatch, responses):
"""`responses`: a list, one per request, of a _FakeResp or an exception to raise. Returns the sent headers."""
sent = []
queue = list(responses)
class _Opener:
def open(
self,
req,
timeout = None,
):
sent.append(dict(req.header_items()))
item = queue.pop(0)
if isinstance(item, Exception):
raise item
return item
monkeypatch.setattr(
tools, "_validate_and_resolve_host", lambda host, port: (True, "", ["93.184.216.34"])
)
monkeypatch.setattr(tools.urllib.request, "build_opener", lambda *a, **k: _Opener())
return sent
def _raw_deflate(data: bytes) -> bytes:
packer = zlib.compressobj(wbits = -15)
return packer.compress(data) + packer.flush()
@pytest.mark.parametrize(
"coding, body",
[
("gzip", gzip.compress(_PAGE)),
("x-gzip", gzip.compress(_PAGE)),
("deflate", zlib.compress(_PAGE)),
# Some servers send raw deflate for "deflate".
("deflate", _raw_deflate(_PAGE)),
# Concatenated members are one body (RFC 1952 2.2).
("gzip", gzip.compress(_PAGE[:1000]) + gzip.compress(_PAGE[1000:])),
# Padding after the last member is ignored, as browsers do.
("gzip", gzip.compress(_PAGE) + b"\0\0\0\0"),
("GZIP ", gzip.compress(_PAGE)),
],
ids = [
"gzip",
"x-gzip",
"zlib-deflate",
"raw-deflate",
"two-members",
"padding",
"case-and-space",
],
)
def test_a_coded_page_reaches_the_browser_panel_decoded(monkeypatch, coding, body):
_serve(
monkeypatch,
[_FakeResp(body, {"Content-Type": "text/html", "Content-Encoding": coding}, step = 1000)],
)
error, raw, content_type = tools._fetch_url_raw(
"https://www.python.org/about/", raw_bytes_max = 10_000_000
)
assert error is None
assert raw == _PAGE
assert content_type == "text/html"
def test_raw_deflate_output_zlib_held_back_at_the_end_is_not_truncation(monkeypatch):
# The last wire bytes fill the 64 KiB output buffer; zlib keeps the rest with no input left.
body = b"a" * 65_537
_serve(
monkeypatch,
[
_FakeResp(
_raw_deflate(body), {"Content-Type": "text/plain", "Content-Encoding": "deflate"}
)
],
)
error, raw, _ = tools._fetch_url_raw("https://example.com/a.txt", raw_bytes_max = 10_000_000)
assert error is None
assert raw == body
def test_web_fetch_reads_a_gzip_page_as_text(monkeypatch):
_serve(
monkeypatch,
[
_FakeResp(
gzip.compress(_PAGE),
{"Content-Type": "text/html; charset=utf-8", "Content-Encoding": "gzip"},
)
],
)
text = tools._fetch_page_text("https://www.python.org/about/", timeout = 5)
assert "About Python" in text
assert "binary content" not in text
def test_the_fetcher_asks_for_the_codings_it_can_undo(monkeypatch):
sent = _serve(
monkeypatch,
[
_FakeResp(_PAGE, {"Content-Type": "text/html"}),
_FakeResp(_PAGE, {"Content-Type": "text/html"}),
],
)
assert tools._fetch_url_raw("https://example.com/", raw_bytes_max = 10_000_000)[0] is None
assert sent[0]["Accept-encoding"] == "gzip, deflate"
# A caller that asks for something else still gets its way.
tools._fetch_url_raw(
"https://example.com/",
raw_bytes_max = 10_000_000,
extra_headers = {"Accept-Encoding": "identity"},
)
assert sent[1]["Accept-encoding"] == "identity"
def test_an_empty_coded_body_is_an_empty_page(monkeypatch):
_serve(monkeypatch, [_FakeResp(b"", {"Content-Type": "text/html", "Content-Encoding": "gzip"})])
error, raw, _ = tools._fetch_url_raw("https://example.com/", raw_bytes_max = 1000)
assert error is None and raw == b""
@pytest.mark.parametrize(
"coding, body, message",
[
("br", b"\x0b\x02\x80hello\x03", "unsupported Content-Encoding 'br'"),
("gzip", gzip.compress(_PAGE)[:-40], "truncated compressed response"),
("gzip", b"<html>not compressed</html>", "Failed to decode the compressed response"),
],
ids = ["brotli", "truncated", "not-gzip"],
)
def test_a_body_that_cannot_be_decoded_is_an_error_not_garbage(monkeypatch, coding, body, message):
_serve(
monkeypatch, [_FakeResp(body, {"Content-Type": "text/html", "Content-Encoding": coding})]
)
error, raw, _ = tools._fetch_url_raw("https://example.com/", raw_bytes_max = 10_000_000)
assert message in error
assert raw == ""
def test_a_compressed_bomb_stops_at_the_decoded_limit(monkeypatch):
bomb = gzip.compress(b"\0" * 64_000_000)
_serve(
monkeypatch,
[
_FakeResp(
bomb,
{"Content-Type": "application/octet-stream", "Content-Encoding": "gzip"},
step = 65536,
)
],
)
error, raw, _ = tools._fetch_url_raw("https://example.com/", raw_bytes_max = 1_000_000)
assert error == "(content exceeds the 1000000 byte limit)"
def test_a_compressed_body_that_never_decodes_stops_at_the_wire_limit(monkeypatch):
# A gzip header with a huge comment field: input keeps coming, output never does.
padded = b"\x1f\x8b\x08\x10\x00\x00\x00\x00\x00\xff" + b"a" * 3_000_000
_serve(
monkeypatch,
[_FakeResp(padded, {"Content-Type": "text/html", "Content-Encoding": "gzip"}, step = 65536)],
)
error, raw, _ = tools._fetch_url_raw("https://example.com/", raw_bytes_max = 1_000_000)
assert error.startswith("(compressed content exceeds the ")
assert raw == ""
def test_a_coded_error_page_is_decoded_too(monkeypatch):
page = gzip.compress(b"<html><body>Not Found</body></html>")
error_resp = _FakeResp(page, {"Content-Type": "text/html", "Content-Encoding": "gzip"})
http_error = urllib.error.HTTPError(
"https://example.com/x", 404, "Not Found", error_resp.headers, None
)
http_error.read1 = error_resp.read1
http_error.read = error_resp.read1
_serve(monkeypatch, [http_error])
error, raw, _ = tools._fetch_url_raw(
"https://example.com/x", raw_bytes_max = 1000, error_page = True
)
assert error.startswith("Failed to fetch URL: HTTP 404")
assert raw == b"<html><body>Not Found</body></html>"