Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

Studio: read large docs pages past their menus (#17543)

Nilayfe8b55aed82f
diff --git a/studio/backend/core/inference/tools.py b/studio/backend/core/inference/tools.py
index 2af22d325da..9d360b5cdcb 100644
--- a/studio/backend/core/inference/tools.py
+++ b/studio/backend/core/inference/tools.py
@@ -14921,6 +14921,8 @@ _HEX_PAIR_RE = re.compile(r"[0-9A-Fa-f]{2}")
_MAX_FETCH_BYTES = 512 * 1024
# news pages can inline about 2.5 MB in <head>, so reserve _MAX_FETCH_BYTES beyond its end.
_MAX_HTML_FETCH_BYTES = 8 * 1024 * 1024
+# docs sites put over 1 MB of menus before the page text, so a window without one of these reads on.
+_HTML_CONTENT_START_RE = re.compile(rb"<(?:main|article|h1)[\s>]", re.IGNORECASE)
# keep % safe to avoid encoding existing escapes as %25.
_IRI_PATH_SAFE = "/%:@!$&'()*+,;="
_IRI_QUERY_SAFE = "/%:@!$&'()*+,;=?"
@@ -15908,11 +15910,17 @@ def _read_capped_body(
read = lambda size: decoder.read(read_wire, size)
chunks = []
remaining = max_bytes
+ got = 0
body_at = None
+ window_end = None
+ fallback_end = None
body_locator = _HTMLBodyLocator(charset) if body_window is not None else None
while remaining > 0:
budget_error = _fetch_budget_exceeded(deadline, cancel_event)
if budget_error is not None:
+ if fallback_end is not None:
+ window_end = fallback_end
+ break
try:
resp.close()
except Exception:
@@ -15925,8 +15933,11 @@ def _read_capped_body(
pass
try:
chunk = read(min(65536, remaining))
- except (ValueError, zlib.error) as exc:
- if decoder is None:
+ except Exception as exc:
+ if fallback_end is not None:
+ window_end = fallback_end
+ break
+ if decoder is None or not isinstance(exc, (ValueError, zlib.error)):
raise
if decoder.wire > decoder.wire_limit:
return f"(compressed content exceeds the {decoder.wire_limit} byte limit)", b""
@@ -15938,8 +15949,8 @@ def _read_capped_body(
break
chunks.append(chunk)
remaining -= len(chunk)
+ got += len(chunk)
if body_locator is not None and body_at is None:
- got = max_bytes - remaining
try:
body_locator.feed_bytes(chunk)
except Exception:
@@ -15947,14 +15958,29 @@ def _read_capped_body(
if body_locator is not None and body_locator.body_at is not None:
body_at = body_locator.body_at
remaining = max(0, min(remaining, body_at + body_window - got))
- budget_error = _fetch_budget_exceeded(deadline, cancel_event)
+ elif window_end is not None and _HTML_CONTENT_START_RE.search(chunks[-2][-8:] + chunk):
+ window_end = None
+ remaining = min(remaining, body_window)
+ if (
+ not remaining
+ and got < max_bytes
+ and body_at is not None
+ and body_locator._codec == "latin-1"
+ and not _has_pdf_magic(so_far := b"".join(chunks))
+ and not _HTML_CONTENT_START_RE.search(so_far, body_at)
+ ):
+ window_end = fallback_end = got
+ remaining = max_bytes - got
+ budget_error = _fetch_budget_exceeded(
+ None if fallback_end is not None else deadline, cancel_event
+ )
if budget_error is not None:
try:
resp.close()
except Exception:
pass
return budget_error, b""
- return None, b"".join(chunks)
+ return None, b"".join(chunks)[:window_end]
_DOTTED_HOST_RE = re.compile(r"[A-Za-z0-9-]+(\.[A-Za-z0-9-]+)+")
diff --git a/studio/backend/tests/test_web_fetch_binary_guard.py b/studio/backend/tests/test_web_fetch_binary_guard.py
index c82667b54dd..e008568059b 100644
--- a/studio/backend/tests/test_web_fetch_binary_guard.py
+++ b/studio/backend/tests/test_web_fetch_binary_guard.py
@@ -8,6 +8,7 @@ from __future__ import annotations
import codecs
import random
import sys
+import threading
import time
import tracemalloc
from email.message import Message
@@ -524,6 +525,106 @@ def test_large_head_token_keeps_article(monkeypatch, opening, closing):
assert "Long token article marker" in _fetch_with(monkeypatch, html, "text/html")
+_MENU = b'<li><a href="/api/item">Menu entry</a></li>\n'
+_LINES = b"<p>line</p>" * (1024 * 1024)
+
+
+def _menu_page(content: bytes) -> bytes:
+ menu = _MENU * (700 * 1024 // len(_MENU))
+ return (
+ b"<html><head><title>t</title></head><body><nav><ul>"
+ + menu
+ + b"</ul></nav>"
+ + content
+ + b"</body></html>"
+ )
+
+
+def _read_html(resp, cancel_event = None):
+ return tools._read_capped_body(
+ resp,
+ tools._MAX_HTML_FETCH_BYTES,
+ timeout = 5,
+ deadline = None,
+ cancel_event = cancel_event,
+ body_window = tools._MAX_FETCH_BYTES,
+ )
+
+
+def _first_window(body: bytes) -> bytes:
+ return body[: body.index(b"</head>") + tools._MAX_FETCH_BYTES]
+
+
[email protected]("tag", [b"main", b"article"])
+def test_page_text_after_a_large_menu_is_read(monkeypatch, tag):
+ html = _menu_page(b"<%b><h1>Dense layer</h1><p>BODY-MARK takes units.</p></%b>" % (tag, tag))
+ assert "BODY-MARK takes units." in _fetch_with(monkeypatch, html, "text/html")
+
+
+def test_page_text_after_a_large_menu_gets_a_full_window():
+ body = _menu_page(b"<main><h1>Start</h1>" + _LINES + b"</main>")
+ resp = _FakeResp(body, "text/html")
+ error, read = _read_html(resp)
+ window_end = body.index(b"<main>") + tools._MAX_FETCH_BYTES
+ assert error is None
+ assert window_end <= resp._pos < window_end + 2 * 65536
+ assert read == body[: resp._pos]
+
+
+def test_page_text_inside_the_window_reads_no_further():
+ body = b"<html><head><title>t</title></head><body><main><h1>Start</h1>" + _LINES + b"</main>"
+ resp = _FakeResp(body, "text/html")
+ assert _read_html(resp) == (None, _first_window(body))
+ assert resp._pos == len(_first_window(body))
+
+
+def test_page_without_a_content_start_keeps_the_first_window():
+ body = b"<html><head><title>t</title></head><body>" + _LINES + b"</body></html>"
+ assert _read_html(_FakeResp(body, "text/html")) == (None, _first_window(body))
+
+
+def test_failed_read_past_the_window_keeps_the_first_window():
+ body = b"<html><head><title>t</title></head><body>" + _LINES + b"</body></html>"
+ resp = _FakeResp(body, "text/html")
+ read = resp.read
+
+ def stalling_read(n = None):
+ if resp._pos > 1024 * 1024:
+ raise TimeoutError("timed out")
+ return read(n)
+
+ resp.read = stalling_read
+ assert _read_html(resp) == (None, _first_window(body))
+
+
+def test_failed_read_after_a_late_content_start_keeps_the_first_window():
+ body = _menu_page(b"<main><h1>Start</h1>" + _LINES + b"</main>")
+ resp = _FakeResp(body, "text/html")
+ read = resp.read
+
+ def stalling_read(n = None):
+ if resp._pos > body.index(b"<main>") + 65536:
+ raise TimeoutError("timed out")
+ return read(n)
+
+ resp.read = stalling_read
+ assert _read_html(resp) == (None, _first_window(body))
+
+
+def test_read_past_the_window_stops_when_cancelled():
+ cancel = threading.Event()
+ resp = _FakeResp(_menu_page(_LINES), "text/html")
+ read = resp.read
+
+ def cancelling_read(n = None):
+ if resp._pos > 1024 * 1024:
+ cancel.set()
+ return read(n)
+
+ resp.read = cancelling_read
+ assert _read_html(resp, cancel) == ("Failed to fetch URL: cancelled.", b"")
+
+
@pytest.mark.parametrize(
"content_type,body",
[
@@ -543,9 +644,15 @@ def test_large_head_token_keeps_article(monkeypatch, opening, closing):
+ b"x" * (4 * 1024 * 1024)
+ b"</script></html>",
),
+ (
+ "text/html",
+ b"<html><head><title>t</title></head><body><p>Harbor ferry adds night service</p>"
+ + b"<p>log line</p>" * (512 * 1024)
+ + b"</body></html>",
+ ),
("text/plain", b"Harbor ferry adds night service\n" + b"log line\n" * (512 * 1024)),
],
- ids = ["html", "html-without-body-tag", "text"],
+ ids = ["html", "html-without-body-tag", "html-without-content-start", "text"],
)
def test_large_page_on_a_slow_link_still_returns_its_start(monkeypatch, content_type, body):
clock = {"time": 1000.0}