Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
Studio: read several web pages at once on local models (#16489)
Nilayba298cdab2b7
diff --git a/studio/backend/core/inference/llama_cpp.py b/studio/backend/core/inference/llama_cpp.pyindex 35556178510..f7d24f82511 100644--- a/studio/backend/core/inference/llama_cpp.py+++ b/studio/backend/core/inference/llama_cpp.py@@ -37387,8 +37387,11 @@ class LlamaCppBackend:search_images_kwargs,stream_tool_execution,)+ from core.inference.studio_tool_loop import _WEB_FETCH_CONCURRENCY, _web_fetch_hostfrom core.inference.tools import (+ StartedUrlRead,build_rag_autoinject,+ compaction_receipt_field,execute_tool,has_text_only_provisional_card,is_high_risk_tool_call,@@ -37404,6 +37407,7 @@ class LlamaCppBackend:needs_tool_confirmation,normalize_sandbox_level,normalize_tool_permissions,+ require_tool_access,requires_os_isolation,runs_without_os_sandbox,tool_call_may_prompt,@@ -39476,6 +39480,67 @@ class LlamaCppBackend:# a Mistral-style call carries its own id and would open a duplicate. Reuse# the card's id for the first matching call (same tool name) to reconcile._text_provisional_id = _text_args_id if not has_structured_tc else ""+ _started_reads: dict = {}++ def _start_following_reads(_call_index, _host, _key):+ # Network only: approvals, pricing, the fit and the record stay in call order.+ _hosts = {_host, *(_started[1] for _started in _started_reads.values())}+ _keys = {_key, *(_started[0].key for _started in _started_reads.values())}+ _next = max(_started_reads, default = _call_index) + 1+ while _next < len(tool_calls) and len(_hosts) < _WEB_FETCH_CONCURRENCY:+ _function = tool_calls[_next].get("function") or {}+ # By raw name first: preparing an MCP call can load tools.+ if str(_function.get("name") or "").strip() != "web_search":+ return+ _spec = tool_controller.prepare_call(tool_calls[_next])+ _next_host = _web_fetch_host(_spec.tool_name, _spec.arguments)+ if (+ not _spec.should_execute+ or _next_host is None+ or _next_host in _hosts+ or _spec.key in _keys+ or compaction_receipt_field(_spec.arguments) is not None+ ):+ return+ _mode, _bypass = live_permissions.apply(permission_mode, bypass_permissions)+ try:+ require_tool_access(disable_sandbox = _bypass)+ except Exception:+ return+ if needs_tool_confirmation(+ confirm_tool_calls = bool(confirm_tool_calls),+ bypass_permissions = _bypass,+ permission_mode = _mode,+ name = _spec.tool_name,+ arguments = _spec.arguments,+ is_high_risk = is_high_risk_tool_call,+ never_needs = never_needs_approval,+ sandbox_level = sandbox_level,+ session_id = session_id,+ ) or requires_os_isolation(+ confirm_tool_calls = bool(confirm_tool_calls),+ bypass_permissions = _bypass,+ permission_mode = _mode,+ name = _spec.tool_name,+ arguments = _spec.arguments,+ prompted = False,+ is_high_risk = is_high_risk_tool_call,+ sandbox_level = sandbox_level,+ ):+ return+ _started_reads[_next] = (+ _spec,+ _next_host,+ StartedUrlRead(+ _spec.arguments["url"].strip(),+ None if tool_call_timeout >= 9999 else tool_call_timeout,+ cancel_event,+ website_policy,+ ),+ )+ _hosts.add(_next_host)+ _keys.add(_spec.key)+ _next += 1for _call_index, tc in enumerate(tool_calls or []):func = tc.get("function", {})@@ -39864,12 +39929,32 @@ class LlamaCppBackend:# Only a call the user answered: the executor lets it reach the host paths it names._host_access_approved = _decision not in (None, "deny")+ _started_spec, _, _started_read = _started_reads.pop(+ _call_index, (None, None, None)+ )+ if (+ _started_spec is None+ or _started_spec.key != decision.key+ or needs_confirm+ or strict_isolation+ ):+ _started_read = None+ _read_host = _web_fetch_host(decision.tool_name, decision.arguments)+ if (+ _read_host is not None+ and not forced_name+ and mcp_image is None+ and not (cancel_event is not None and cancel_event.is_set())+ and accepts_kwarg(execute_tool, "started_read")+ ):+ _start_following_reads(_call_index, _read_host, decision.key)def _invoke_tool(_output_callback,_decision = decision,_approved = _host_access_approved,_strict = strict_isolation,+ _read = _started_read,):# Bound top_k to the room left: the result lands in the protected tool exchange, so an oversized top_k ends the turn in an unrecoverable context error.kwargs = dict(@@ -40109,6 +40194,8 @@ class LlamaCppBackend:kwargs.update(search_images_kwargs(execute_tool, _decision.tool_name))if image_share is not None:kwargs["mcp_image"] = image_share["image"]+ if _read is not None:+ kwargs["started_read"] = _readreturn execute_tool(_decision.tool_name,_decision.arguments,diff --git a/studio/backend/core/inference/tools.py b/studio/backend/core/inference/tools.pyindex f4338c1f8d6..7373879c677 100644--- a/studio/backend/core/inference/tools.py+++ b/studio/backend/core/inference/tools.py@@ -13294,6 +13294,7 @@ def execute_tool(tool_execution_mode: str = "auto",host_access_approved: bool = False,mcp_image = None,+ started_read: "StartedUrlRead | None" = None,) -> str:"""Runs a tool by name; disable_sandbox skips python/terminal checks, blocklist and caps (secrets stay stripped); tool_execution_mode full is refused here."""from state.tool_policy import require_tool_access@@ -13544,6 +13545,8 @@ def execute_tool(image_queries = arguments.get("image_queries") if isinstance(arguments, dict) else Noneif not query and not url and not _clean_image_queries(image_queries):return "No query provided."+ if started_read is not None:+ return _fit_result_to_room(started_read.result(), name)return _fit_result_to_room(_web_search(query,@@ -14649,6 +14652,10 @@ def _meta_refresh_target(body: bytes, page_url: str) -> str | None:return None+# PyMuPDF is not thread-safe.+_WEB_PDF_LOCK = threading.Lock()++def _extract_pdf_text(data: bytes) -> str:"""Extract page-delimited text with the same parser used by RAG ingestion."""from ..rag.parsers import parse_pdf_bytes@@ -15729,7 +15736,12 @@ def _fetch_url_raw(if budget_error is not None:return budget_error, "", content_typetry:- pdf_text = _extract_pdf_text(raw_bytes)+ queued_at = time.monotonic()+ with _WEB_PDF_LOCK:+ # Time queued behind another read's PDF is not spent on this read.+ if deadline is not None:+ deadline += time.monotonic() - queued_at+ pdf_text = _extract_pdf_text(raw_bytes)except Exception as exc:logger.debug("web PDF text extraction failed (%s)", type(exc).__name__)return "(PDF content could not be read as text)", "", content_type@@ -16296,8 +16308,18 @@ def _fetch_page_text(website_policy: dict | None = None,) -> str:"""Fetches a URL as readable text (HTML to Markdown); blocks private, loopback and link-local hosts"""- if max_chars is None:- max_chars = _page_char_budget()+ page = _download_page(+ url, timeout = timeout, cancel_event = cancel_event, website_policy = website_policy+ )+ return _render_page(page, max_chars)+++def _download_page(+ url: str,+ timeout: int = 30,+ cancel_event = None,+ website_policy: dict | None = None,+):# One deadline for the whole fetch so the HTML fallback does not get a fresh timeout.deadline = None if timeout is None else time.monotonic() + timeoutfrom .web_access_policy import check_url_access@@ -16323,17 +16345,7 @@ def _fetch_page_text()# A 200 README body is authoritative even when HTML; fall back only on failure.if err is None and body.strip():- readme_body = body- # Convert only real HTML; a Markdown README opening with a block tag is kept (_HTML_DOCUMENT_RE).- if _looks_like_html_document(body):- from ._html_to_md import html_to_markdown- converted = html_to_markdown(body, main_content = True, max_span_chars = max_chars // 2)- readme_body = converted if converted.strip() else body- if readme_body.strip():- return _truncate_page_text(- f"README of {url} (fetched via the GitHub README API):\n\n" + readme_body,- max_chars,- )+ return ("readme", url, body)raw_file_url = _github_blob_raw_url(url)if raw_file_url:@@ -16345,7 +16357,7 @@ def _fetch_page_text(**policy_kwargs,)if err is None and body.strip():- return _truncate_page_text(body.strip(), max_chars)+ return ("text", url, body.strip())meta: dict = {}err, body, content_type = _fetch_url_raw(@@ -16365,14 +16377,34 @@ def _fetch_page_text(return err# Sniff the body when Content-Type is missing or wrong.- is_html = "html" in content_type or _looks_like_html(body)- if not is_html:- # Converting plain text through the HTML renderer would collapse whitespace.- return _truncate_page_text(body.strip(), max_chars)+ if "html" in content_type or _looks_like_html(body):+ return ("html", url, body)+ # Converting plain text through the HTML renderer would collapse whitespace.+ return ("text", url, body.strip())- # Convert HTML to Markdown with the builtin converter (no external deps).++# Reads the call's budget context vars: run inside execute_tool, never on a prefetch thread.+def _render_page(page, max_chars: int | None = None) -> str:+ if isinstance(page, str):+ return page+ if max_chars is None:+ max_chars = _page_char_budget()+ kind, url, body = page+ if kind == "text":+ return _truncate_page_text(body, max_chars)from ._html_to_md import SiteLinks, html_to_markdown+ if kind == "readme":+ readme_body = body+ # Convert only real HTML; a Markdown README opening with a block tag is kept (_HTML_DOCUMENT_RE).+ if _looks_like_html_document(body):+ converted = html_to_markdown(body, main_content = True, max_span_chars = max_chars // 2)+ readme_body = converted if converted.strip() else body+ return _truncate_page_text(+ f"README of {url} (fetched via the GitHub README API):\n\n" + readme_body,+ max_chars,+ )+site_links = SiteLinks(url)# generated span cells get half the window budget, as html_to_markdown's own cap does for 16Ktext = html_to_markdown(@@ -16689,6 +16721,43 @@ def _wrap_ddgs_client(cls, wrap, follow_redirects, signature, ddgs_exception) ->cls._unsloth_http1_retry = True+class StartedUrlRead:+ def __init__(self, url: str, timeout, cancel_event, website_policy) -> None:+ self._done = threading.Event()+ self._value = None+ self._error: BaseException | None = None+ context = contextvars.copy_context()+ threading.Thread(+ target = context.run,+ args = (self._run, url, timeout, cancel_event, website_policy),+ name = "web-url-read",+ daemon = True,+ ).start()++ def _run(self, url, timeout, cancel_event, website_policy) -> None:+ try:+ self._value = _download_page(+ url,+ timeout = _url_read_timeout(timeout),+ cancel_event = cancel_event,+ website_policy = website_policy,+ )+ except BaseException as exc: # noqa: BLE001 - re-raised to the caller by result()+ self._error = exc+ finally:+ self._done.set()++ def result(self) -> str:+ self._done.wait()+ if self._error is not None:+ raise self._error+ return _render_page(self._value)+++def _url_read_timeout(timeout):+ return 60 if timeout is None else min(timeout, 60)++class _ImageSweep:"""Image lookup started beside the text search; the caller still decides whether to register it."""@@ -16868,10 +16937,9 @@ def _web_search("""search approved tiers, fetch a URL, or return registered ``[[img:<id>]]`` images alone."""# Direct URL fetch mode.if url and url.strip():- fetch_timeout = 60 if timeout is None else min(timeout, 60)return _fetch_page_text(url.strip(),- timeout = fetch_timeout,+ timeout = _url_read_timeout(timeout),cancel_event = cancel_event,website_policy = website_policy,)diff --git a/studio/backend/tests/test_web_tools_overlap.py b/studio/backend/tests/test_web_tools_overlap.pyindex 1cbc2ad5626..4ebc0b86acd 100644--- a/studio/backend/tests/test_web_tools_overlap.py+++ b/studio/backend/tests/test_web_tools_overlap.py@@ -13,16 +13,22 @@ import asyncioimport http.serverimport jsonimport re+import sysimport threadingimport time+from pathlib import Pathimport pytest+from core.inference import llama_cppfrom core.inference import search_imagesfrom core.inference import studio_tool_loop as loop_modfrom core.inference import toolsfrom core.inference.studio_tool_loop import ToolLoopPolicy, ToolLoopRun, stream_with_studio_tools+sys.path.insert(0, str(Path(__file__).resolve().parent))+from test_llama_cpp_tool_loop import _make_backend, _sse as local_sse # noqa: E402+WEB = {"type": "function",@@ -365,11 +371,24 @@ class _Handler(http.server.BaseHTTPRequestHandler):elif host.startswith("private"):self._redirect("http://169.254.169.254/latest/meta-data/")else:+ content_type = "text/html"body = (b"<html><body><main><h1>Page</h1><p>" + b"word " * 80 + b"</p></main></body></html>")+ if host.startswith("pdf"):+ content_type, body = "application/pdf", b"%PDF-1.4\n" + b"0" * 64+ if host.startswith("links"):+ links = b"".join(+ b'<li><a href="/docs/' + b"section-%d/" % i * 12 + b'">part</a></li>'+ for i in range(12)+ )+ body = (+ b"<html><body><main><h1>Page</h1><ul>"+ + links+ + b"</ul><p>LAST_FACT_MARKER</p></main></body></html>"+ )self.send_response(200)- self.send_header("Content-Type", "text/html")+ self.send_header("Content-Type", content_type)self.send_header("Content-Length", str(len(body)))self.end_headers()self.wfile.write(body)@@ -401,6 +420,257 @@ def local_sites(monkeypatch):server.server_close()+class _SlowPages:+ def __init__(self, monkeypatch):+ self.lock = threading.Lock()+ self.running = 0+ self.peak = 0+ self.fetched = []+ real = tools._download_page++ def fetch(url, *args, **kwargs):+ with self.lock:+ self.running += 1+ self.peak = max(self.peak, self.running)+ self.fetched.append(url)+ cancel = kwargs.get("cancel_event")+ deadline = time.monotonic() + 0.4+ while time.monotonic() < deadline and not (cancel and cancel.is_set()):+ time.sleep(0.01)+ with self.lock:+ self.running -= 1+ if cancel and cancel.is_set():+ return "Fetch cancelled."+ return real(url, *args, **kwargs)++ monkeypatch.setattr(tools, "_download_page", fetch)+++def _local_turn(+ monkeypatch,+ calls,+ cancel_event = None,+ wait_denies = False,+ **loop_kwargs,+):+ if wait_denies:+ monkeypatch.setattr(llama_cpp, "wait_tool_decision", lambda *a, **k: "deny", raising = False)++ frames = [+ local_sse(+ {+ "tool_calls": [+ {+ "index": index,+ "id": call["id"],+ "type": "function",+ "function": call["function"],+ }+ ]+ }+ )+ for index, call in enumerate(calls)+ ]+ finish = 'data: {"choices":[{"delta":{},"finish_reason":"tool_calls"}]}\n\n'+ streams = [+ [*frames, finish, "data: [DONE]\n\n"],+ [local_sse({"content": "done"}), "data: [DONE]\n\n"],+ ]+ payloads = []+ backend = _make_backend(monkeypatch, streams, payloads)+ backend.count_chat_tokens = lambda *a, **k: 100+ events = []+ started = time.monotonic()+ for event in backend.generate_chat_completion_with_tools(+ messages = [{"role": "user", "content": "go"}],+ tools = [WEB, PY],+ max_tool_iterations = 2,+ tool_call_timeout = 30,+ cancel_event = cancel_event or threading.Event(),+ **{"permission_mode": "off", **loop_kwargs},+ ):+ if isinstance(event, dict) and event.get("type") in ("tool_start", "tool_end"):+ events.append({key: event.get(key) for key in ("type", "tool_call_id", "result")})+ return events, payloads, time.monotonic() - started+++def _local_url(index, host, port):+ return {+ "id": f"c{index}",+ "function": {+ "name": "web_search",+ "arguments": json.dumps({"url": f"http://{host}.sites.example:{port}/p"}),+ },+ }+++def _tool_replay(payload):+ return [m for m in payload["messages"] if m.get("role") in ("assistant", "tool")]+++def test_local_loop_reads_run_together_and_replay_in_order(monkeypatch, local_sites):+ calls = [_local_url(index, host, local_sites) for index, host in enumerate("abc")]+ slow = _SlowPages(monkeypatch)++ events, payloads, _ = _local_turn(monkeypatch, calls)++ assert slow.peak == 3+ assert [(e["type"], e["tool_call_id"]) for e in events] == [+ (kind, f"c{index}") for index in range(3) for kind in ("tool_start", "tool_end")+ ]+ ends = [e["result"] for e in events if e["type"] == "tool_end"]+ assert all("word word" in result for result in ends)+ tool_messages = [m for m in _tool_replay(payloads[1]) if m["role"] == "tool"]+ assert [m["tool_call_id"] for m in tool_messages] == ["c0", "c1", "c2"]+++def test_local_loop_fits_each_started_read_to_its_own_turn(monkeypatch, local_sites):+ calls = [_local_url(index, host, local_sites) for index, host in enumerate("ab")]+ budgets = []+ real = tools.execute_tool++ def execute(name, arguments, **kwargs):+ budgets.append(kwargs.get("result_budget_tokens"))+ kwargs["result_budget_tokens"] = 12+ return real(name, arguments, **kwargs)++ monkeypatch.setattr(tools, "execute_tool", execute)+ _SlowPages(monkeypatch)++ events, _, _ = _local_turn(monkeypatch, calls)++ assert len(budgets) == 2 and all(budget is not None for budget in budgets)+ ends = [e["result"] for e in events if e["type"] == "tool_end"]+ assert len(ends) == 2 and all(len(result) < 400 for result in ends)+++def test_local_loop_renders_a_started_read_under_its_own_budget(monkeypatch, local_sites):+ calls = [_local_url(0, "a", local_sites), _local_url(1, "links", local_sites)]+ real = tools.execute_tool++ def execute(name, arguments, **kwargs):+ kwargs["result_budget_tokens"] = 200+ return real(name, arguments, **kwargs)++ monkeypatch.setattr(tools, "execute_tool", execute)+ slow = _SlowPages(monkeypatch)+ monkeypatch.setattr(loop_mod, "_WEB_FETCH_CONCURRENCY", 1, raising = False)+ serial, _, _ = _local_turn(monkeypatch, calls)+ assert slow.peak == 1+ monkeypatch.setattr(loop_mod, "_WEB_FETCH_CONCURRENCY", 4, raising = False)+ overlapped, _, _ = _local_turn(monkeypatch, calls)++ assert slow.peak == 2+ assert overlapped == serial+ assert "LAST_FACT_MARKER" in [e["result"] for e in overlapped if e["type"] == "tool_end"][1]+++def test_local_loop_extracts_started_pdfs_one_at_a_time(monkeypatch, local_sites):+ from core.rag import parsers++ calls = [_local_url(0, "pdfa", local_sites), _local_url(1, "pdfb", local_sites)]+ lock = threading.Lock()+ state = {"running": 0, "peak": 0, "calls": 0}++ def parse(data, max_pages = None):+ with lock:+ state["calls"] += 1+ state["running"] += 1+ state["peak"] = max(state["peak"], state["running"])+ time.sleep(0.3)+ with lock:+ state["running"] -= 1+ return [], 1++ monkeypatch.setattr(parsers, "parse_pdf_bytes", parse)+ slow = _SlowPages(monkeypatch)++ _local_turn(monkeypatch, calls)++ assert slow.peak == 2+ assert state["calls"] == 2+ assert state["peak"] == 1+++def test_local_loop_pdf_read_does_not_time_out_waiting_for_another(monkeypatch, local_sites):+ from core.rag import parsers++ calls = [_local_url(0, "pdfa", local_sites), _local_url(1, "pdfb", local_sites)]++ def parse(data, max_pages = None):+ time.sleep(1.0)+ return [], 1++ monkeypatch.setattr(parsers, "parse_pdf_bytes", parse)+ # Under two parses, well over one.+ monkeypatch.setattr(tools, "_url_read_timeout", lambda timeout: 1.8)+ slow = _SlowPages(monkeypatch)++ events, _, _ = _local_turn(monkeypatch, calls)++ assert slow.peak == 2+ ends = [e["result"] for e in events if e["type"] == "tool_end"]+ assert ends == ["(PDF contains no extractable text)"] * 2+++def test_local_loop_reads_of_one_host_stay_serial(monkeypatch, local_sites):+ calls = [+ _local_url(0, "a", local_sites),+ {+ "id": "c1",+ "function": {+ "name": "web_search",+ "arguments": json.dumps({"url": f"http://a.sites.example:{local_sites}/q"}),+ },+ },+ ]+ slow = _SlowPages(monkeypatch)++ _local_turn(monkeypatch, calls)++ assert slow.peak == 1+ assert len(slow.fetched) == 2+++def test_local_loop_read_that_asks_first_is_not_started(monkeypatch, local_sites):+ calls = [_local_url(index, host, local_sites) for index, host in enumerate("ab")]+ slow = _SlowPages(monkeypatch)++ events, _, _ = _local_turn(+ monkeypatch, calls, permission_mode = "ask", confirm_tool_calls = True, wait_denies = True+ )++ assert slow.fetched == []+ assert [e["tool_call_id"] for e in events if e["type"] == "tool_end"] == ["c0", "c1"]+++def test_local_loop_never_starts_a_read_after_another_tool(monkeypatch, local_sites):+ calls = [+ _local_url(0, "a", local_sites),+ {"id": "c1", "function": {"name": "python", "arguments": json.dumps({"code": "1"})}},+ _local_url(2, "b", local_sites),+ ]+ slow = _SlowPages(monkeypatch)++ _local_turn(monkeypatch, calls)++ assert slow.peak == 1+++def test_local_loop_cancel_stops_started_reads(monkeypatch, local_sites):+ calls = [_local_url(index, host, local_sites) for index, host in enumerate("abc")]+ slow = _SlowPages(monkeypatch)+ cancel = threading.Event()+ threading.Timer(0.1, cancel.set).start()++ _local_turn(monkeypatch, calls, cancel_event = cancel)++ deadline = time.monotonic() + 2+ while slow.running and time.monotonic() < deadline:+ time.sleep(0.01)+ assert slow.running == 0++def test_guards_hold_on_every_hop_of_overlapped_reads(monkeypatch, local_sites):port = local_sitescalls = [