Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

Studio: read several web pages at once on local models (#16489)

Nilayba298cdab2b7
diff --git a/studio/backend/core/inference/llama_cpp.py b/studio/backend/core/inference/llama_cpp.py
index 35556178510..f7d24f82511 100644
--- a/studio/backend/core/inference/llama_cpp.py
+++ b/studio/backend/core/inference/llama_cpp.py
@@ -37387,8 +37387,11 @@ class LlamaCppBackend:
search_images_kwargs,
stream_tool_execution,
)
+ from core.inference.studio_tool_loop import _WEB_FETCH_CONCURRENCY, _web_fetch_host
from core.inference.tools import (
+ StartedUrlRead,
build_rag_autoinject,
+ compaction_receipt_field,
execute_tool,
has_text_only_provisional_card,
is_high_risk_tool_call,
@@ -37404,6 +37407,7 @@ class LlamaCppBackend:
needs_tool_confirmation,
normalize_sandbox_level,
normalize_tool_permissions,
+ require_tool_access,
requires_os_isolation,
runs_without_os_sandbox,
tool_call_may_prompt,
@@ -39476,6 +39480,67 @@ class LlamaCppBackend:
# a Mistral-style call carries its own id and would open a duplicate. Reuse
# the card's id for the first matching call (same tool name) to reconcile.
_text_provisional_id = _text_args_id if not has_structured_tc else ""
+ _started_reads: dict = {}
+
+ def _start_following_reads(_call_index, _host, _key):
+ # Network only: approvals, pricing, the fit and the record stay in call order.
+ _hosts = {_host, *(_started[1] for _started in _started_reads.values())}
+ _keys = {_key, *(_started[0].key for _started in _started_reads.values())}
+ _next = max(_started_reads, default = _call_index) + 1
+ while _next < len(tool_calls) and len(_hosts) < _WEB_FETCH_CONCURRENCY:
+ _function = tool_calls[_next].get("function") or {}
+ # By raw name first: preparing an MCP call can load tools.
+ if str(_function.get("name") or "").strip() != "web_search":
+ return
+ _spec = tool_controller.prepare_call(tool_calls[_next])
+ _next_host = _web_fetch_host(_spec.tool_name, _spec.arguments)
+ if (
+ not _spec.should_execute
+ or _next_host is None
+ or _next_host in _hosts
+ or _spec.key in _keys
+ or compaction_receipt_field(_spec.arguments) is not None
+ ):
+ return
+ _mode, _bypass = live_permissions.apply(permission_mode, bypass_permissions)
+ try:
+ require_tool_access(disable_sandbox = _bypass)
+ except Exception:
+ return
+ if needs_tool_confirmation(
+ confirm_tool_calls = bool(confirm_tool_calls),
+ bypass_permissions = _bypass,
+ permission_mode = _mode,
+ name = _spec.tool_name,
+ arguments = _spec.arguments,
+ is_high_risk = is_high_risk_tool_call,
+ never_needs = never_needs_approval,
+ sandbox_level = sandbox_level,
+ session_id = session_id,
+ ) or requires_os_isolation(
+ confirm_tool_calls = bool(confirm_tool_calls),
+ bypass_permissions = _bypass,
+ permission_mode = _mode,
+ name = _spec.tool_name,
+ arguments = _spec.arguments,
+ prompted = False,
+ is_high_risk = is_high_risk_tool_call,
+ sandbox_level = sandbox_level,
+ ):
+ return
+ _started_reads[_next] = (
+ _spec,
+ _next_host,
+ StartedUrlRead(
+ _spec.arguments["url"].strip(),
+ None if tool_call_timeout >= 9999 else tool_call_timeout,
+ cancel_event,
+ website_policy,
+ ),
+ )
+ _hosts.add(_next_host)
+ _keys.add(_spec.key)
+ _next += 1
for _call_index, tc in enumerate(tool_calls or []):
func = tc.get("function", {})
@@ -39864,12 +39929,32 @@ class LlamaCppBackend:
# Only a call the user answered: the executor lets it reach the host paths it names.
_host_access_approved = _decision not in (None, "deny")
+ _started_spec, _, _started_read = _started_reads.pop(
+ _call_index, (None, None, None)
+ )
+ if (
+ _started_spec is None
+ or _started_spec.key != decision.key
+ or needs_confirm
+ or strict_isolation
+ ):
+ _started_read = None
+ _read_host = _web_fetch_host(decision.tool_name, decision.arguments)
+ if (
+ _read_host is not None
+ and not forced_name
+ and mcp_image is None
+ and not (cancel_event is not None and cancel_event.is_set())
+ and accepts_kwarg(execute_tool, "started_read")
+ ):
+ _start_following_reads(_call_index, _read_host, decision.key)
def _invoke_tool(
_output_callback,
_decision = decision,
_approved = _host_access_approved,
_strict = strict_isolation,
+ _read = _started_read,
):
# Bound top_k to the room left: the result lands in the protected tool exchange, so an oversized top_k ends the turn in an unrecoverable context error.
kwargs = dict(
@@ -40109,6 +40194,8 @@ class LlamaCppBackend:
kwargs.update(search_images_kwargs(execute_tool, _decision.tool_name))
if image_share is not None:
kwargs["mcp_image"] = image_share["image"]
+ if _read is not None:
+ kwargs["started_read"] = _read
return execute_tool(
_decision.tool_name,
_decision.arguments,
diff --git a/studio/backend/core/inference/tools.py b/studio/backend/core/inference/tools.py
index f4338c1f8d6..7373879c677 100644
--- a/studio/backend/core/inference/tools.py
+++ b/studio/backend/core/inference/tools.py
@@ -13294,6 +13294,7 @@ def execute_tool(
tool_execution_mode: str = "auto",
host_access_approved: bool = False,
mcp_image = None,
+ started_read: "StartedUrlRead | None" = None,
) -> str:
"""Runs a tool by name; disable_sandbox skips python/terminal checks, blocklist and caps (secrets stay stripped); tool_execution_mode full is refused here."""
from state.tool_policy import require_tool_access
@@ -13544,6 +13545,8 @@ def execute_tool(
image_queries = arguments.get("image_queries") if isinstance(arguments, dict) else None
if not query and not url and not _clean_image_queries(image_queries):
return "No query provided."
+ if started_read is not None:
+ return _fit_result_to_room(started_read.result(), name)
return _fit_result_to_room(
_web_search(
query,
@@ -14649,6 +14652,10 @@ def _meta_refresh_target(body: bytes, page_url: str) -> str | None:
return None
+# PyMuPDF is not thread-safe.
+_WEB_PDF_LOCK = threading.Lock()
+
+
def _extract_pdf_text(data: bytes) -> str:
"""Extract page-delimited text with the same parser used by RAG ingestion."""
from ..rag.parsers import parse_pdf_bytes
@@ -15729,7 +15736,12 @@ def _fetch_url_raw(
if budget_error is not None:
return budget_error, "", content_type
try:
- pdf_text = _extract_pdf_text(raw_bytes)
+ queued_at = time.monotonic()
+ with _WEB_PDF_LOCK:
+ # Time queued behind another read's PDF is not spent on this read.
+ if deadline is not None:
+ deadline += time.monotonic() - queued_at
+ pdf_text = _extract_pdf_text(raw_bytes)
except Exception as exc:
logger.debug("web PDF text extraction failed (%s)", type(exc).__name__)
return "(PDF content could not be read as text)", "", content_type
@@ -16296,8 +16308,18 @@ def _fetch_page_text(
website_policy: dict | None = None,
) -> str:
"""Fetches a URL as readable text (HTML to Markdown); blocks private, loopback and link-local hosts"""
- if max_chars is None:
- max_chars = _page_char_budget()
+ page = _download_page(
+ url, timeout = timeout, cancel_event = cancel_event, website_policy = website_policy
+ )
+ return _render_page(page, max_chars)
+
+
+def _download_page(
+ url: str,
+ timeout: int = 30,
+ cancel_event = None,
+ website_policy: dict | None = None,
+):
# One deadline for the whole fetch so the HTML fallback does not get a fresh timeout.
deadline = None if timeout is None else time.monotonic() + timeout
from .web_access_policy import check_url_access
@@ -16323,17 +16345,7 @@ def _fetch_page_text(
)
# A 200 README body is authoritative even when HTML; fall back only on failure.
if err is None and body.strip():
- readme_body = body
- # Convert only real HTML; a Markdown README opening with a block tag is kept (_HTML_DOCUMENT_RE).
- if _looks_like_html_document(body):
- from ._html_to_md import html_to_markdown
- converted = html_to_markdown(body, main_content = True, max_span_chars = max_chars // 2)
- readme_body = converted if converted.strip() else body
- if readme_body.strip():
- return _truncate_page_text(
- f"README of {url} (fetched via the GitHub README API):\n\n" + readme_body,
- max_chars,
- )
+ return ("readme", url, body)
raw_file_url = _github_blob_raw_url(url)
if raw_file_url:
@@ -16345,7 +16357,7 @@ def _fetch_page_text(
**policy_kwargs,
)
if err is None and body.strip():
- return _truncate_page_text(body.strip(), max_chars)
+ return ("text", url, body.strip())
meta: dict = {}
err, body, content_type = _fetch_url_raw(
@@ -16365,14 +16377,34 @@ def _fetch_page_text(
return err
# Sniff the body when Content-Type is missing or wrong.
- is_html = "html" in content_type or _looks_like_html(body)
- if not is_html:
- # Converting plain text through the HTML renderer would collapse whitespace.
- return _truncate_page_text(body.strip(), max_chars)
+ if "html" in content_type or _looks_like_html(body):
+ return ("html", url, body)
+ # Converting plain text through the HTML renderer would collapse whitespace.
+ return ("text", url, body.strip())
- # Convert HTML to Markdown with the builtin converter (no external deps).
+
+# Reads the call's budget context vars: run inside execute_tool, never on a prefetch thread.
+def _render_page(page, max_chars: int | None = None) -> str:
+ if isinstance(page, str):
+ return page
+ if max_chars is None:
+ max_chars = _page_char_budget()
+ kind, url, body = page
+ if kind == "text":
+ return _truncate_page_text(body, max_chars)
from ._html_to_md import SiteLinks, html_to_markdown
+ if kind == "readme":
+ readme_body = body
+ # Convert only real HTML; a Markdown README opening with a block tag is kept (_HTML_DOCUMENT_RE).
+ if _looks_like_html_document(body):
+ converted = html_to_markdown(body, main_content = True, max_span_chars = max_chars // 2)
+ readme_body = converted if converted.strip() else body
+ return _truncate_page_text(
+ f"README of {url} (fetched via the GitHub README API):\n\n" + readme_body,
+ max_chars,
+ )
+
site_links = SiteLinks(url)
# generated span cells get half the window budget, as html_to_markdown's own cap does for 16K
text = html_to_markdown(
@@ -16689,6 +16721,43 @@ def _wrap_ddgs_client(cls, wrap, follow_redirects, signature, ddgs_exception) ->
cls._unsloth_http1_retry = True
+class StartedUrlRead:
+ def __init__(self, url: str, timeout, cancel_event, website_policy) -> None:
+ self._done = threading.Event()
+ self._value = None
+ self._error: BaseException | None = None
+ context = contextvars.copy_context()
+ threading.Thread(
+ target = context.run,
+ args = (self._run, url, timeout, cancel_event, website_policy),
+ name = "web-url-read",
+ daemon = True,
+ ).start()
+
+ def _run(self, url, timeout, cancel_event, website_policy) -> None:
+ try:
+ self._value = _download_page(
+ url,
+ timeout = _url_read_timeout(timeout),
+ cancel_event = cancel_event,
+ website_policy = website_policy,
+ )
+ except BaseException as exc: # noqa: BLE001 - re-raised to the caller by result()
+ self._error = exc
+ finally:
+ self._done.set()
+
+ def result(self) -> str:
+ self._done.wait()
+ if self._error is not None:
+ raise self._error
+ return _render_page(self._value)
+
+
+def _url_read_timeout(timeout):
+ return 60 if timeout is None else min(timeout, 60)
+
+
class _ImageSweep:
"""Image lookup started beside the text search; the caller still decides whether to register it."""
@@ -16868,10 +16937,9 @@ def _web_search(
"""search approved tiers, fetch a URL, or return registered ``[[img:<id>]]`` images alone."""
# Direct URL fetch mode.
if url and url.strip():
- fetch_timeout = 60 if timeout is None else min(timeout, 60)
return _fetch_page_text(
url.strip(),
- timeout = fetch_timeout,
+ timeout = _url_read_timeout(timeout),
cancel_event = cancel_event,
website_policy = website_policy,
)
diff --git a/studio/backend/tests/test_web_tools_overlap.py b/studio/backend/tests/test_web_tools_overlap.py
index 1cbc2ad5626..4ebc0b86acd 100644
--- a/studio/backend/tests/test_web_tools_overlap.py
+++ b/studio/backend/tests/test_web_tools_overlap.py
@@ -13,16 +13,22 @@ import asyncio
import http.server
import json
import re
+import sys
import threading
import time
+from pathlib import Path
import pytest
+from core.inference import llama_cpp
from core.inference import search_images
from core.inference import studio_tool_loop as loop_mod
from core.inference import tools
from core.inference.studio_tool_loop import ToolLoopPolicy, ToolLoopRun, stream_with_studio_tools
+sys.path.insert(0, str(Path(__file__).resolve().parent))
+from test_llama_cpp_tool_loop import _make_backend, _sse as local_sse # noqa: E402
+
WEB = {
"type": "function",
@@ -365,11 +371,24 @@ class _Handler(http.server.BaseHTTPRequestHandler):
elif host.startswith("private"):
self._redirect("http://169.254.169.254/latest/meta-data/")
else:
+ content_type = "text/html"
body = (
b"<html><body><main><h1>Page</h1><p>" + b"word " * 80 + b"</p></main></body></html>"
)
+ if host.startswith("pdf"):
+ content_type, body = "application/pdf", b"%PDF-1.4\n" + b"0" * 64
+ if host.startswith("links"):
+ links = b"".join(
+ b'<li><a href="/docs/' + b"section-%d/" % i * 12 + b'">part</a></li>'
+ for i in range(12)
+ )
+ body = (
+ b"<html><body><main><h1>Page</h1><ul>"
+ + links
+ + b"</ul><p>LAST_FACT_MARKER</p></main></body></html>"
+ )
self.send_response(200)
- self.send_header("Content-Type", "text/html")
+ self.send_header("Content-Type", content_type)
self.send_header("Content-Length", str(len(body)))
self.end_headers()
self.wfile.write(body)
@@ -401,6 +420,257 @@ def local_sites(monkeypatch):
server.server_close()
+class _SlowPages:
+ def __init__(self, monkeypatch):
+ self.lock = threading.Lock()
+ self.running = 0
+ self.peak = 0
+ self.fetched = []
+ real = tools._download_page
+
+ def fetch(url, *args, **kwargs):
+ with self.lock:
+ self.running += 1
+ self.peak = max(self.peak, self.running)
+ self.fetched.append(url)
+ cancel = kwargs.get("cancel_event")
+ deadline = time.monotonic() + 0.4
+ while time.monotonic() < deadline and not (cancel and cancel.is_set()):
+ time.sleep(0.01)
+ with self.lock:
+ self.running -= 1
+ if cancel and cancel.is_set():
+ return "Fetch cancelled."
+ return real(url, *args, **kwargs)
+
+ monkeypatch.setattr(tools, "_download_page", fetch)
+
+
+def _local_turn(
+ monkeypatch,
+ calls,
+ cancel_event = None,
+ wait_denies = False,
+ **loop_kwargs,
+):
+ if wait_denies:
+ monkeypatch.setattr(llama_cpp, "wait_tool_decision", lambda *a, **k: "deny", raising = False)
+
+ frames = [
+ local_sse(
+ {
+ "tool_calls": [
+ {
+ "index": index,
+ "id": call["id"],
+ "type": "function",
+ "function": call["function"],
+ }
+ ]
+ }
+ )
+ for index, call in enumerate(calls)
+ ]
+ finish = 'data: {"choices":[{"delta":{},"finish_reason":"tool_calls"}]}\n\n'
+ streams = [
+ [*frames, finish, "data: [DONE]\n\n"],
+ [local_sse({"content": "done"}), "data: [DONE]\n\n"],
+ ]
+ payloads = []
+ backend = _make_backend(monkeypatch, streams, payloads)
+ backend.count_chat_tokens = lambda *a, **k: 100
+ events = []
+ started = time.monotonic()
+ for event in backend.generate_chat_completion_with_tools(
+ messages = [{"role": "user", "content": "go"}],
+ tools = [WEB, PY],
+ max_tool_iterations = 2,
+ tool_call_timeout = 30,
+ cancel_event = cancel_event or threading.Event(),
+ **{"permission_mode": "off", **loop_kwargs},
+ ):
+ if isinstance(event, dict) and event.get("type") in ("tool_start", "tool_end"):
+ events.append({key: event.get(key) for key in ("type", "tool_call_id", "result")})
+ return events, payloads, time.monotonic() - started
+
+
+def _local_url(index, host, port):
+ return {
+ "id": f"c{index}",
+ "function": {
+ "name": "web_search",
+ "arguments": json.dumps({"url": f"http://{host}.sites.example:{port}/p"}),
+ },
+ }
+
+
+def _tool_replay(payload):
+ return [m for m in payload["messages"] if m.get("role") in ("assistant", "tool")]
+
+
+def test_local_loop_reads_run_together_and_replay_in_order(monkeypatch, local_sites):
+ calls = [_local_url(index, host, local_sites) for index, host in enumerate("abc")]
+ slow = _SlowPages(monkeypatch)
+
+ events, payloads, _ = _local_turn(monkeypatch, calls)
+
+ assert slow.peak == 3
+ assert [(e["type"], e["tool_call_id"]) for e in events] == [
+ (kind, f"c{index}") for index in range(3) for kind in ("tool_start", "tool_end")
+ ]
+ ends = [e["result"] for e in events if e["type"] == "tool_end"]
+ assert all("word word" in result for result in ends)
+ tool_messages = [m for m in _tool_replay(payloads[1]) if m["role"] == "tool"]
+ assert [m["tool_call_id"] for m in tool_messages] == ["c0", "c1", "c2"]
+
+
+def test_local_loop_fits_each_started_read_to_its_own_turn(monkeypatch, local_sites):
+ calls = [_local_url(index, host, local_sites) for index, host in enumerate("ab")]
+ budgets = []
+ real = tools.execute_tool
+
+ def execute(name, arguments, **kwargs):
+ budgets.append(kwargs.get("result_budget_tokens"))
+ kwargs["result_budget_tokens"] = 12
+ return real(name, arguments, **kwargs)
+
+ monkeypatch.setattr(tools, "execute_tool", execute)
+ _SlowPages(monkeypatch)
+
+ events, _, _ = _local_turn(monkeypatch, calls)
+
+ assert len(budgets) == 2 and all(budget is not None for budget in budgets)
+ ends = [e["result"] for e in events if e["type"] == "tool_end"]
+ assert len(ends) == 2 and all(len(result) < 400 for result in ends)
+
+
+def test_local_loop_renders_a_started_read_under_its_own_budget(monkeypatch, local_sites):
+ calls = [_local_url(0, "a", local_sites), _local_url(1, "links", local_sites)]
+ real = tools.execute_tool
+
+ def execute(name, arguments, **kwargs):
+ kwargs["result_budget_tokens"] = 200
+ return real(name, arguments, **kwargs)
+
+ monkeypatch.setattr(tools, "execute_tool", execute)
+ slow = _SlowPages(monkeypatch)
+ monkeypatch.setattr(loop_mod, "_WEB_FETCH_CONCURRENCY", 1, raising = False)
+ serial, _, _ = _local_turn(monkeypatch, calls)
+ assert slow.peak == 1
+ monkeypatch.setattr(loop_mod, "_WEB_FETCH_CONCURRENCY", 4, raising = False)
+ overlapped, _, _ = _local_turn(monkeypatch, calls)
+
+ assert slow.peak == 2
+ assert overlapped == serial
+ assert "LAST_FACT_MARKER" in [e["result"] for e in overlapped if e["type"] == "tool_end"][1]
+
+
+def test_local_loop_extracts_started_pdfs_one_at_a_time(monkeypatch, local_sites):
+ from core.rag import parsers
+
+ calls = [_local_url(0, "pdfa", local_sites), _local_url(1, "pdfb", local_sites)]
+ lock = threading.Lock()
+ state = {"running": 0, "peak": 0, "calls": 0}
+
+ def parse(data, max_pages = None):
+ with lock:
+ state["calls"] += 1
+ state["running"] += 1
+ state["peak"] = max(state["peak"], state["running"])
+ time.sleep(0.3)
+ with lock:
+ state["running"] -= 1
+ return [], 1
+
+ monkeypatch.setattr(parsers, "parse_pdf_bytes", parse)
+ slow = _SlowPages(monkeypatch)
+
+ _local_turn(monkeypatch, calls)
+
+ assert slow.peak == 2
+ assert state["calls"] == 2
+ assert state["peak"] == 1
+
+
+def test_local_loop_pdf_read_does_not_time_out_waiting_for_another(monkeypatch, local_sites):
+ from core.rag import parsers
+
+ calls = [_local_url(0, "pdfa", local_sites), _local_url(1, "pdfb", local_sites)]
+
+ def parse(data, max_pages = None):
+ time.sleep(1.0)
+ return [], 1
+
+ monkeypatch.setattr(parsers, "parse_pdf_bytes", parse)
+ # Under two parses, well over one.
+ monkeypatch.setattr(tools, "_url_read_timeout", lambda timeout: 1.8)
+ slow = _SlowPages(monkeypatch)
+
+ events, _, _ = _local_turn(monkeypatch, calls)
+
+ assert slow.peak == 2
+ ends = [e["result"] for e in events if e["type"] == "tool_end"]
+ assert ends == ["(PDF contains no extractable text)"] * 2
+
+
+def test_local_loop_reads_of_one_host_stay_serial(monkeypatch, local_sites):
+ calls = [
+ _local_url(0, "a", local_sites),
+ {
+ "id": "c1",
+ "function": {
+ "name": "web_search",
+ "arguments": json.dumps({"url": f"http://a.sites.example:{local_sites}/q"}),
+ },
+ },
+ ]
+ slow = _SlowPages(monkeypatch)
+
+ _local_turn(monkeypatch, calls)
+
+ assert slow.peak == 1
+ assert len(slow.fetched) == 2
+
+
+def test_local_loop_read_that_asks_first_is_not_started(monkeypatch, local_sites):
+ calls = [_local_url(index, host, local_sites) for index, host in enumerate("ab")]
+ slow = _SlowPages(monkeypatch)
+
+ events, _, _ = _local_turn(
+ monkeypatch, calls, permission_mode = "ask", confirm_tool_calls = True, wait_denies = True
+ )
+
+ assert slow.fetched == []
+ assert [e["tool_call_id"] for e in events if e["type"] == "tool_end"] == ["c0", "c1"]
+
+
+def test_local_loop_never_starts_a_read_after_another_tool(monkeypatch, local_sites):
+ calls = [
+ _local_url(0, "a", local_sites),
+ {"id": "c1", "function": {"name": "python", "arguments": json.dumps({"code": "1"})}},
+ _local_url(2, "b", local_sites),
+ ]
+ slow = _SlowPages(monkeypatch)
+
+ _local_turn(monkeypatch, calls)
+
+ assert slow.peak == 1
+
+
+def test_local_loop_cancel_stops_started_reads(monkeypatch, local_sites):
+ calls = [_local_url(index, host, local_sites) for index, host in enumerate("abc")]
+ slow = _SlowPages(monkeypatch)
+ cancel = threading.Event()
+ threading.Timer(0.1, cancel.set).start()
+
+ _local_turn(monkeypatch, calls, cancel_event = cancel)
+
+ deadline = time.monotonic() + 2
+ while slow.running and time.monotonic() < deadline:
+ time.sleep(0.01)
+ assert slow.running == 0
+
+
def test_guards_hold_on_every_hop_of_overlapped_reads(monkeypatch, local_sites):
port = local_sites
calls = [