Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
Studio: read large docs pages past their menus (#17543)
Nilayfe8b55aed82f
diff --git a/studio/backend/core/inference/tools.py b/studio/backend/core/inference/tools.pyindex 2af22d325da..9d360b5cdcb 100644--- a/studio/backend/core/inference/tools.py+++ b/studio/backend/core/inference/tools.py@@ -14921,6 +14921,8 @@ _HEX_PAIR_RE = re.compile(r"[0-9A-Fa-f]{2}")_MAX_FETCH_BYTES = 512 * 1024# news pages can inline about 2.5 MB in <head>, so reserve _MAX_FETCH_BYTES beyond its end._MAX_HTML_FETCH_BYTES = 8 * 1024 * 1024+# docs sites put over 1 MB of menus before the page text, so a window without one of these reads on.+_HTML_CONTENT_START_RE = re.compile(rb"<(?:main|article|h1)[\s>]", re.IGNORECASE)# keep % safe to avoid encoding existing escapes as %25._IRI_PATH_SAFE = "/%:@!$&'()*+,;="_IRI_QUERY_SAFE = "/%:@!$&'()*+,;=?"@@ -15908,11 +15910,17 @@ def _read_capped_body(read = lambda size: decoder.read(read_wire, size)chunks = []remaining = max_bytes+ got = 0body_at = None+ window_end = None+ fallback_end = Nonebody_locator = _HTMLBodyLocator(charset) if body_window is not None else Nonewhile remaining > 0:budget_error = _fetch_budget_exceeded(deadline, cancel_event)if budget_error is not None:+ if fallback_end is not None:+ window_end = fallback_end+ breaktry:resp.close()except Exception:@@ -15925,8 +15933,11 @@ def _read_capped_body(passtry:chunk = read(min(65536, remaining))- except (ValueError, zlib.error) as exc:- if decoder is None:+ except Exception as exc:+ if fallback_end is not None:+ window_end = fallback_end+ break+ if decoder is None or not isinstance(exc, (ValueError, zlib.error)):raiseif decoder.wire > decoder.wire_limit:return f"(compressed content exceeds the {decoder.wire_limit} byte limit)", b""@@ -15938,8 +15949,8 @@ def _read_capped_body(breakchunks.append(chunk)remaining -= len(chunk)+ got += len(chunk)if body_locator is not None and body_at is None:- got = max_bytes - remainingtry:body_locator.feed_bytes(chunk)except Exception:@@ -15947,14 +15958,29 @@ def _read_capped_body(if body_locator is not None and body_locator.body_at is not None:body_at = body_locator.body_atremaining = max(0, min(remaining, body_at + body_window - got))- budget_error = _fetch_budget_exceeded(deadline, cancel_event)+ elif window_end is not None and _HTML_CONTENT_START_RE.search(chunks[-2][-8:] + chunk):+ window_end = None+ remaining = min(remaining, body_window)+ if (+ not remaining+ and got < max_bytes+ and body_at is not None+ and body_locator._codec == "latin-1"+ and not _has_pdf_magic(so_far := b"".join(chunks))+ and not _HTML_CONTENT_START_RE.search(so_far, body_at)+ ):+ window_end = fallback_end = got+ remaining = max_bytes - got+ budget_error = _fetch_budget_exceeded(+ None if fallback_end is not None else deadline, cancel_event+ )if budget_error is not None:try:resp.close()except Exception:passreturn budget_error, b""- return None, b"".join(chunks)+ return None, b"".join(chunks)[:window_end]_DOTTED_HOST_RE = re.compile(r"[A-Za-z0-9-]+(\.[A-Za-z0-9-]+)+")diff --git a/studio/backend/tests/test_web_fetch_binary_guard.py b/studio/backend/tests/test_web_fetch_binary_guard.pyindex c82667b54dd..e008568059b 100644--- a/studio/backend/tests/test_web_fetch_binary_guard.py+++ b/studio/backend/tests/test_web_fetch_binary_guard.py@@ -8,6 +8,7 @@ from __future__ import annotationsimport codecsimport randomimport sys+import threadingimport timeimport tracemallocfrom email.message import Message@@ -524,6 +525,106 @@ def test_large_head_token_keeps_article(monkeypatch, opening, closing):assert "Long token article marker" in _fetch_with(monkeypatch, html, "text/html")+_MENU = b'<li><a href="/api/item">Menu entry</a></li>\n'+_LINES = b"<p>line</p>" * (1024 * 1024)+++def _menu_page(content: bytes) -> bytes:+ menu = _MENU * (700 * 1024 // len(_MENU))+ return (+ b"<html><head><title>t</title></head><body><nav><ul>"+ + menu+ + b"</ul></nav>"+ + content+ + b"</body></html>"+ )+++def _read_html(resp, cancel_event = None):+ return tools._read_capped_body(+ resp,+ tools._MAX_HTML_FETCH_BYTES,+ timeout = 5,+ deadline = None,+ cancel_event = cancel_event,+ body_window = tools._MAX_FETCH_BYTES,+ )+++def _first_window(body: bytes) -> bytes:+ return body[: body.index(b"</head>") + tools._MAX_FETCH_BYTES]++[email protected]("tag", [b"main", b"article"])+def test_page_text_after_a_large_menu_is_read(monkeypatch, tag):+ html = _menu_page(b"<%b><h1>Dense layer</h1><p>BODY-MARK takes units.</p></%b>" % (tag, tag))+ assert "BODY-MARK takes units." in _fetch_with(monkeypatch, html, "text/html")+++def test_page_text_after_a_large_menu_gets_a_full_window():+ body = _menu_page(b"<main><h1>Start</h1>" + _LINES + b"</main>")+ resp = _FakeResp(body, "text/html")+ error, read = _read_html(resp)+ window_end = body.index(b"<main>") + tools._MAX_FETCH_BYTES+ assert error is None+ assert window_end <= resp._pos < window_end + 2 * 65536+ assert read == body[: resp._pos]+++def test_page_text_inside_the_window_reads_no_further():+ body = b"<html><head><title>t</title></head><body><main><h1>Start</h1>" + _LINES + b"</main>"+ resp = _FakeResp(body, "text/html")+ assert _read_html(resp) == (None, _first_window(body))+ assert resp._pos == len(_first_window(body))+++def test_page_without_a_content_start_keeps_the_first_window():+ body = b"<html><head><title>t</title></head><body>" + _LINES + b"</body></html>"+ assert _read_html(_FakeResp(body, "text/html")) == (None, _first_window(body))+++def test_failed_read_past_the_window_keeps_the_first_window():+ body = b"<html><head><title>t</title></head><body>" + _LINES + b"</body></html>"+ resp = _FakeResp(body, "text/html")+ read = resp.read++ def stalling_read(n = None):+ if resp._pos > 1024 * 1024:+ raise TimeoutError("timed out")+ return read(n)++ resp.read = stalling_read+ assert _read_html(resp) == (None, _first_window(body))+++def test_failed_read_after_a_late_content_start_keeps_the_first_window():+ body = _menu_page(b"<main><h1>Start</h1>" + _LINES + b"</main>")+ resp = _FakeResp(body, "text/html")+ read = resp.read++ def stalling_read(n = None):+ if resp._pos > body.index(b"<main>") + 65536:+ raise TimeoutError("timed out")+ return read(n)++ resp.read = stalling_read+ assert _read_html(resp) == (None, _first_window(body))+++def test_read_past_the_window_stops_when_cancelled():+ cancel = threading.Event()+ resp = _FakeResp(_menu_page(_LINES), "text/html")+ read = resp.read++ def cancelling_read(n = None):+ if resp._pos > 1024 * 1024:+ cancel.set()+ return read(n)++ resp.read = cancelling_read+ assert _read_html(resp, cancel) == ("Failed to fetch URL: cancelled.", b"")++@pytest.mark.parametrize("content_type,body",[@@ -543,9 +644,15 @@ def test_large_head_token_keeps_article(monkeypatch, opening, closing):+ b"x" * (4 * 1024 * 1024)+ b"</script></html>",),+ (+ "text/html",+ b"<html><head><title>t</title></head><body><p>Harbor ferry adds night service</p>"+ + b"<p>log line</p>" * (512 * 1024)+ + b"</body></html>",+ ),("text/plain", b"Harbor ferry adds night service\n" + b"log line\n" * (512 * 1024)),],- ids = ["html", "html-without-body-tag", "text"],+ ids = ["html", "html-without-body-tag", "html-without-content-start", "text"],)def test_large_page_on_a_slow_link_still_returns_its_start(monkeypatch, content_type, body):clock = {"time": 1000.0}