Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
Studio: overlap web image lookup with text search and independent URL reads in one tool batch (#16174)
Daniel Hancddf258f466f
diff --git a/studio/backend/core/inference/studio_tool_loop.py b/studio/backend/core/inference/studio_tool_loop.pyindex 17f96ecc8bf..607de727215 100644--- a/studio/backend/core/inference/studio_tool_loop.py+++ b/studio/backend/core/inference/studio_tool_loop.py@@ -37,10 +37,14 @@ self-hosted types to thirteen.from __future__ import annotationsimport asyncio+import contextvarsimport json+import queueimport threading+import urllib.parsefrom dataclasses import dataclass, field+from functools import partialfrom collections.abc import AsyncIterator, Callablefrom typing import Any, Protocol@@ -77,6 +81,7 @@ def _append_mcp_images_owned(from core.inference.tool_loop_controller import (+ UNPARSED_ARGUMENTS_KEY,ToolLoopController,_reject_json_constant,awaiting_approval_status,@@ -169,6 +174,9 @@ _USAGE_DETAIL_FIELDS = (_STEP_DONE = object()+# Max URL reads of one batch in flight, one per host.+_WEB_FETCH_CONCURRENCY = 4+def _truncate_for_model(text: str,@@ -1283,6 +1291,106 @@ def _advance_tool_stream(generator: Any, outcome: dict[str, Any]) -> Any:return _STEP_DONE+def _web_fetch_host(tool_name: str, arguments: Any) -> str | None:+ if tool_name != "web_search" or not isinstance(arguments, dict):+ return None+ if UNPARSED_ARGUMENTS_KEY in arguments:+ return None+ url = arguments.get("url")+ if not isinstance(url, str) or not url.strip():+ return None+ try:+ host = urllib.parse.urlsplit(tools_module._normalize_url_scheme(url.strip())).hostname+ except ValueError:+ return None+ return host or None+++class _StartedToolStream:+ """A URL read started before its turn; buffers its stream's events until the loop reads them."""++ def __init__(self, stream: Any, decision: Any, cancel_event: threading.Event) -> None:+ self.decision = decision+ self.host = _web_fetch_host(decision.tool_name, decision.arguments)+ self._stream = stream+ self._cancel_event = cancel_event+ self._events: queue.Queue = queue.Queue()+ self._stop = threading.Event()+ self._pumped = threading.Event()+ self._finished = False+ # Same account context as a step thread.+ context = contextvars.copy_context()+ self._thread = threading.Thread(+ target = context.run,+ args = (self._pump,),+ name = "tool-prefetch",+ daemon = True,+ )+ self._thread.start()++ def _pump(self) -> None:+ try:+ while not self._stop.is_set():+ try:+ event = next(self._stream)+ except StopIteration as stop:+ self._pumped.set()+ self._events.put((True, stop.value))+ return+ self._events.put((False, event))+ # Only after close(): the stream's early-close path joins its worker.+ self._stream.close()+ self._events.put((True, None))+ except BaseException as exc: # noqa: BLE001 - re-raised to the reader in order+ self._pumped.set()+ self._events.put((None, exc))++ def __iter__(self) -> "_StartedToolStream":+ return self++ def __next__(self) -> Any:+ if self._finished:+ raise StopIteration+ done, value = self._events.get()+ if done is False:+ return value+ self._finished = True+ if done is None:+ raise value+ raise StopIteration(value)++ def serves(self, decision: Any) -> bool:+ started = self.decision+ return (+ decision.tool_name == started.tool_name+ and decision.arguments == started.arguments+ and decision.key == started.key+ and decision.card_id == started.card_id+ )++ def wait(self) -> None:+ self._thread.join()++ def close(self) -> None:+ """Cancels the tool, as closing the generator mid-run does."""+ if self._finished or self._pumped.is_set():+ return+ if self._thread.is_alive():+ self._cancel_event.set()+ self._stop.set()+ self._thread.join(timeout = 5.0)+++async def _settle_started_stream(+ started: _StartedToolStream, cancel_event: threading.Event+) -> None:+ """An unused started read: stopped on cancel, else awaited, since stopping it cancels the turn."""+ if cancel_event.is_set():+ started.close()+ return+ await asyncio.to_thread(started.wait)++async def _drain_step_task(task: Any, cancel_event: threading.Event) -> None:"""Closing a generator mid-next() raises and skips its cleanup, so the worker thread is joined first."""if task is None:@@ -1315,6 +1423,37 @@ async def stream_with_studio_tools(policy: ToolLoopPolicy,cancel_event: threading.Event,mcp_image = None,+) -> AsyncIterator[str]:+ # A stream closed or failing mid-batch stops reads started ahead of their turn.+ started_fetches: dict[int, _StartedToolStream] = {}+ inner = _stream_with_studio_tools(+ transport,+ run = run,+ policy = policy,+ cancel_event = cancel_event,+ mcp_image = mcp_image,+ started_fetches = started_fetches,+ )+ try:+ async for line in inner:+ yield line+ finally:+ try:+ await inner.aclose()+ finally:+ for started in list(started_fetches.values()):+ started.close()+ started_fetches.clear()+++async def _stream_with_studio_tools(+ transport: ToolLoopTransport,+ *,+ run: ToolLoopRun,+ policy: ToolLoopPolicy,+ cancel_event: threading.Event,+ mcp_image,+ started_fetches: "dict[int, _StartedToolStream]",) -> AsyncIterator[str]:# Off the loop: restoring may parse an envelope.restored = await asyncio.to_thread(restore_replayed_tool_results, run.messages)@@ -1731,7 +1870,8 @@ async def stream_with_studio_tools(turn_mcp_images: list[list[dict[str, Any]]] = []turn_executed_real_tool = False- for call in calls:+ started_fetches.clear()+ for call_index, call in enumerate(calls):if cancel_event.is_set():break# Before the gate: an exhausted call is replayed too, and only the decision's replay is guaranteed to parse.@@ -1940,6 +2080,7 @@ async def stream_with_studio_tools(call = decision,approved = host_access_approved,strict = strict_isolation,+ share = image_share,) -> str:kwargs: dict[str, Any] = {"cancel_event": cancel_event,@@ -1979,17 +2120,95 @@ async def stream_with_studio_tools(if accepts_output_callback(execute_tool):kwargs["output_callback"] = output_callbackkwargs.update(search_images_kwargs(execute_tool, call.tool_name))- if image_share is not None:- kwargs["mcp_image"] = image_share["image"]+ if share is not None:+ kwargs["mcp_image"] = share["image"]return execute_tool(call.tool_name, call.arguments, **kwargs)- # Same wrapper as local loops: live stdout for the card and a heartbeat keeping long calls alive.- tool_stream = scoped_tool_stream(- _invoke,- tool_name = name,- tool_call_id = card_id,- cancel_event = cancel_event,- )+ tool_stream: Any = started_fetches.pop(call_index, None)+ if tool_stream is not None and not (+ tool_stream.serves(decision)+ and not needs_confirmation+ and not strict_isolation+ and image_share is None+ ):+ # Not expected; its answer is not used.+ await _settle_started_stream(tool_stream, cancel_event)+ tool_stream = None+ if tool_stream is None:+ # Same wrapper as local loops: live stdout for the card and a heartbeat keeping long calls alive.+ tool_stream = scoped_tool_stream(+ _invoke,+ tool_name = name,+ tool_call_id = card_id,+ cancel_event = cancel_event,+ )+ # Start following URL reads now only if each is exactly the call the loop will run: new host, no+ # approval, within budget, key not in flight. Results are still emitted in call order.+ current_host = _web_fetch_host(name, arguments)+ if current_host is not None and not cancel_event.is_set():+ in_flight_hosts = {current_host}+ in_flight_keys = {decision.key}+ for started in started_fetches.values():+ in_flight_hosts.add(started.host)+ in_flight_keys.add(started.decision.key)+ next_index = max(started_fetches, default = call_index) + 1+ while (+ next_index < len(calls)+ and len(in_flight_hosts) < _WEB_FETCH_CONCURRENCY+ and (unlimited or remaining - (next_index - call_index) >= 1)+ ):+ follower = calls[next_index]+ function = follower.get("function")+ raw_name = function.get("name") if isinstance(function, dict) else None+ # By raw name first: preparing an MCP call can load tools.+ if not isinstance(raw_name, str) or raw_name.strip() != "web_search":+ break+ spec = controller.prepare_call(follower)+ host = _web_fetch_host(spec.tool_name, spec.arguments)+ if (+ not spec.should_execute+ or host is None+ or host in in_flight_hosts+ or spec.key in in_flight_keys+ ):+ break+ follower_mode, follower_bypass = live_permissions.apply(+ permission_mode, bypass_permissions+ )+ if needs_tool_confirmation(+ confirm_tool_calls = confirm_tool_calls,+ bypass_permissions = follower_bypass,+ permission_mode = follower_mode,+ name = spec.tool_name,+ arguments = spec.arguments,+ is_high_risk = is_high_risk_tool_call,+ never_needs = never_needs_approval,+ sandbox_level = sandbox_level,+ session_id = session_id,+ ) or requires_os_isolation(+ confirm_tool_calls = confirm_tool_calls,+ bypass_permissions = follower_bypass,+ permission_mode = follower_mode,+ name = spec.tool_name,+ arguments = spec.arguments,+ prompted = False,+ is_high_risk = is_high_risk_tool_call,+ sandbox_level = sandbox_level,+ ):+ break+ started_fetches[next_index] = _StartedToolStream(+ scoped_tool_stream(+ partial(_invoke, call = spec, approved = False, strict = False, share = None),+ tool_name = spec.tool_name,+ tool_call_id = spec.card_id,+ cancel_event = cancel_event,+ ),+ spec,+ cancel_event,+ )+ in_flight_hosts.add(host)+ in_flight_keys.add(spec.key)+ next_index += 1outcome: dict[str, Any] = {}step_task: Any = Nonetry:@@ -2041,6 +2260,11 @@ async def stream_with_studio_tools(if _completion_images:turn_mcp_images.append(_completion_images)+ # Only a cancelled batch leaves these; never recorded.+ for started in started_fetches.values():+ await _settle_started_stream(started, cancel_event)+ started_fetches.clear()+yield _status_sse("")assistant_message: dict[str, Any] = {diff --git a/studio/backend/core/inference/tools.py b/studio/backend/core/inference/tools.pyindex 0125929bbc3..bcc41cf17f5 100644--- a/studio/backend/core/inference/tools.py+++ b/studio/backend/core/inference/tools.py@@ -26,6 +26,7 @@ import signalos.environ["UNSLOTH_IS_PRESENT"] = "1"import asyncio+import contextvarsimport queueimport randomimport re@@ -132,6 +133,9 @@ EMPTY_SEARCH_RESULTS = ()# ddgs signals an empty sweep by raising rather than returning []._DDGS_EMPTY_SWEEP = "No results found"+# While text engines fail, no early image lookup: its answer is discarded and adds to rate limiting.+_TEXT_SEARCH_FAILURE_HOLD_S = 120.0+_text_search_failed_at: "float | None" = None# Not "connection error": DNS failures and refused connections are not resets (#12638)._DDGS_RESET_MARKERS = ("connection reset",@@ -16653,7 +16657,74 @@ def _wrap_ddgs_client(cls, wrap, follow_redirects, signature, ddgs_exception) ->cls._unsloth_http1_retry = True-def _image_search_or_none(subjects: list, timeout, cancel_event, website_policy) -> "str | None":+class _ImageSweep:+ """Image lookup started beside the text search; the caller still decides whether to register it."""++ def __init__(self, lookup) -> None:+ from .search_images import cache_generation++ # Read before the sweep so a clear-all during it invalidates new entries.+ self.generation = cache_generation()+ self._done = threading.Event()+ self._value = None+ self._error: BaseException | None = None+ context = contextvars.copy_context()+ threading.Thread(+ target = context.run,+ args = (self._run, lookup),+ name = "web-search-images",+ daemon = True,+ ).start()++ def _run(self, lookup) -> None:+ try:+ self._value = lookup()+ except BaseException as exc: # noqa: BLE001 - re-raised to the caller by result()+ self._error = exc+ finally:+ self._done.set()++ def result(self):+ self._done.wait()+ if self._error is not None:+ raise self._error+ return self._value++ @classmethod+ def start(cls, subjects, query, wanted, timeout, website_policy) -> "_ImageSweep | None":+ """None when the serial path would not reach the network."""+ try:+ from ddgs import DDGS++ if not callable(getattr(DDGS, "images", None)):+ return None+ if subjects:+ cleaned = _clean_image_queries(subjects)+ if not cleaned:+ return None+ return cls(lambda: _image_lookups(cleaned, timeout, website_policy))+ from .search_images import MAX_IMAGES_PER_SEARCH++ images_fn = DDGS(timeout = timeout).images+ return cls(+ lambda: images_fn(+ query,+ max_results = max(wanted, MAX_IMAGES_PER_SEARCH * 2),+ safesearch = "moderate",+ )+ )+ except Exception: # noqa: BLE001 - the serial path stays available+ logger.debug("image lookup not started early", exc_info = True)+ return None+++def _image_search_or_none(+ subjects: list,+ timeout,+ cancel_event,+ website_policy,+ sweep = None,+) -> "str | None":"""Returns None on failure: an image error must not discard the text results already found."""try:return _image_search(@@ -16661,6 +16732,7 @@ def _image_search_or_none(subjects: list, timeout, cancel_event, website_policy)timeout = timeout,cancel_event = cancel_event,website_policy = website_policy,+ sweep = sweep,)except Exception as exc: # noqa: BLE001 - a garnish must not become the answerlogger.debug("image lookup failed (%s)", type(exc).__name__)@@ -16668,7 +16740,13 @@ def _image_search_or_none(subjects: list, timeout, cancel_event, website_policy)def _empty_result_with_requested_images(- empty_text: str, subjects: list, include_images: bool, timeout, cancel_event, website_policy+ empty_text: str,+ subjects: list,+ include_images: bool,+ timeout,+ cancel_event,+ website_policy,+ sweep = None,) -> str:"""Explicitly named image subjects still return pictures when the text search comes back empty."""if not subjects:@@ -16678,7 +16756,7 @@ def _empty_result_with_requested_images(return empty_text + "\n\n---\n\n" + IMAGE_SEARCH_DISABLEDif cancel_event is not None and cancel_event.is_set():return empty_text- found = _image_search_or_none(subjects, timeout, cancel_event, website_policy)+ found = _image_search_or_none(subjects, timeout, cancel_event, website_policy, sweep)if found is None:return empty_textreturn empty_text + "\n\n---\n\n" + found@@ -16735,6 +16813,16 @@ def _usable_search_results(results, website_policy):]+def _note_text_search_outcome(found: bool, error, timeout) -> None:+ """Holds back early image lookups after an engine failure; any answer, even empty, clears it."""+ global _text_search_failed_at+ if found:+ _text_search_failed_at = None+ elif error is not None:+ empty = _search_failure_message(error, timeout) == EMPTY_SEARCH_RESULTS[0]+ _text_search_failed_at = None if empty else time.monotonic()++def _web_search(query: str,max_results: int = 5,@@ -16771,6 +16859,7 @@ def _web_search(# DDGS.text() is blocking, so cancellation is checked before and after the call.if cancel_event is not None and cancel_event.is_set():return "Search cancelled."+ sweep = Nonetry:from .web_access_policy import check_url_access, scope_search_query@@ -16798,6 +16887,13 @@ def _web_search(raise RuntimeError("no approved search engine is available.")# reset after setup to keep the primary budget; the earlier deadline bounds fallback.deadline = time.monotonic() + timeout if timeout else None+ failed_at = _text_search_failed_at+ if include_images and (+ failed_at is None or time.monotonic() - failed_at > _TEXT_SEARCH_FAILURE_HOLD_S+ ):+ sweep = _ImageSweep.start(+ subjects, effective_query, wanted, timeout, website_policy+ )client = DDGS(timeout = timeout)# DDGS uses per-client timeouts; tiers share one budget and images reuse the client.for backend in engine_tiers:@@ -16819,6 +16915,7 @@ def _web_search(breakexcept Exception as exc:last_error = exc+ _note_text_search_outcome(bool(results) or rejected_results, last_error, timeout)if cancel_event is not None and cancel_event.is_set():return "Search cancelled."if not results:@@ -16859,6 +16956,7 @@ def _web_search(timeout,cancel_event,website_policy,+ sweep,)parts = []for r in results:@@ -16879,6 +16977,7 @@ def _web_search(timeout,cancel_event,website_policy,+ sweep,)text = "\n\n---\n\n".join(parts)if wikipedia_fallback:@@ -16894,7 +16993,7 @@ def _web_search()if include_images and subjects:# named subjects require one image each rather than a generic image batch.- found = _image_search_or_none(subjects, timeout, cancel_event, website_policy)+ found = _image_search_or_none(subjects, timeout, cancel_event, website_policy, sweep)if found is not None:text += "\n\n---\n\n" + foundelif include_images and not wikipedia_fallback:@@ -16904,6 +17003,7 @@ def _web_search(wanted,cancel_event,website_policy,+ sweep,)elif subjects:# replayed history must retain the disabled-search reminder.@@ -16920,6 +17020,7 @@ def _web_search(timeout,cancel_event,website_policy,+ sweep,)return failure@@ -16949,15 +17050,40 @@ def _clean_image_queries(queries) -> list[str]:return cleaned+def _image_lookups(cleaned: list, timeout, website_policy) -> list:+ from concurrent.futures import ThreadPoolExecutor++ from ddgs import DDGS++ from .web_access_policy import scope_search_query++ def lookup(subject: str) -> list:+ # A client per call: ddgs instances are not documented thread-safe.+ try:+ return list(+ DDGS(timeout = timeout).images(+ scope_search_query(subject, website_policy),+ max_results = IMAGE_SEARCH_PER_QUERY * 4,+ safesearch = "moderate",+ )+ or []+ )+ except Exception as exc: # noqa: BLE001 - one subject failing must not lose the rest+ logger.debug("image lookup skipped %r (%s)", subject, type(exc).__name__)+ return []++ with ThreadPoolExecutor(max_workers = len(cleaned)) as pool:+ return list(pool.map(lookup, cleaned))++def _image_search(queries,timeout: int = _EXEC_TIMEOUT,cancel_event = None,website_policy: dict | None = None,+ sweep: "_ImageSweep | None" = None,) -> str:- # One lookup per subject, concurrently; same registry and tokens as web_search.- from concurrent.futures import ThreadPoolExecutor-+ # Same registry and tokens as web_search; a sweep is these lookups already under way.from .search_images import cache_generation, images_envelope, register_imagescleaned = _clean_image_queries(queries)@@ -16965,11 +17091,11 @@ def _image_search(return "No subjects provided."if cancel_event is not None and cancel_event.is_set():return "Search cancelled."- expected_generation = cache_generation()+ expected_generation = cache_generation() if sweep is None else sweep.generationtry:from ddgs import DDGS- from .web_access_policy import scope_search_query+ from .web_access_policy import scope_search_query # noqa: F401 - fail here, as beforeexcept Exception as e:return _search_failure_message(e, timeout)_install_ddgs_http1_retry()@@ -16977,23 +17103,10 @@ def _image_search(if not callable(getattr(DDGS, "images", None)):return "Image search is unavailable in this install."- def lookup(subject: str) -> list:- # A client per call: ddgs instances are not documented thread-safe.- try:- return list(- DDGS(timeout = timeout).images(- scope_search_query(subject, website_policy),- max_results = IMAGE_SEARCH_PER_QUERY * 4,- safesearch = "moderate",- )- or []- )- except Exception as exc: # noqa: BLE001 - one subject failing must not lose the rest- logger.debug("image lookup skipped %r (%s)", subject, type(exc).__name__)- return []-- with ThreadPoolExecutor(max_workers = len(cleaned)) as pool:- raw_by_subject = list(pool.map(lookup, cleaned))+ if sweep is not None:+ raw_by_subject = sweep.result()+ else:+ raw_by_subject = _image_lookups(cleaned, timeout, website_policy)if cancel_event is not None and cancel_event.is_set():return "Search cancelled."@@ -17027,7 +17140,14 @@ def _image_search(return header + "\n\n" + "\n\n".join(sections) + images_envelope(entries_all)-def _web_search_images_suffix(client, query, wanted, cancel_event, website_policy) -> str:+def _web_search_images_suffix(+ client,+ query,+ wanted,+ cancel_event,+ website_policy,+ sweep = None,+) -> str:# "" when images are unavailable; never raises, the text results stand on their own.from .search_images import (MAX_IMAGES_PER_SEARCH,@@ -17041,11 +17161,14 @@ def _web_search_images_suffix(client, query, wanted, cancel_event, website_policif not callable(images_fn):return ""# Read before the sweep so a clear-all during it invalidates new entries.- expected_generation = cache_generation()+ expected_generation = cache_generation() if sweep is None else sweep.generationtry:- raw = images_fn(- query, max_results = max(wanted, MAX_IMAGES_PER_SEARCH * 2), safesearch = "moderate"- )+ if sweep is not None:+ raw = sweep.result()+ else:+ raw = images_fn(+ query, max_results = max(wanted, MAX_IMAGES_PER_SEARCH * 2), safesearch = "moderate"+ )except Exception as exc: # noqa: BLE001 - optional extra; the text results stand on their ownlogger.debug("web_search image lookup skipped (%s)", type(exc).__name__)return ""diff --git a/studio/backend/tests/test_control_markup_neutralize_7066.py b/studio/backend/tests/test_control_markup_neutralize_7066.pyindex dbea5760403..af4999f6107 100644--- a/studio/backend/tests/test_control_markup_neutralize_7066.py+++ b/studio/backend/tests/test_control_markup_neutralize_7066.py@@ -2814,7 +2814,7 @@ _CONTROLLER_MODULES = ("llama_cpp.py", "safetensors_agentic.py")# studio_tool_loop.py is a known gap; keyed by site count, so adding or removing a controller fails_CONTROLLER_SITES_OUT_OF_SCOPE = frozenset(- {("core/inference/studio_tool_loop.py", "stream_with_studio_tools", 1)}+ {("core/inference/studio_tool_loop.py", "_stream_with_studio_tools", 1)})_CONTROLLER_MODULE_PATHS = frozenset({f"core/inference/{module}" for module in _CONTROLLER_MODULES})diff --git a/studio/backend/tests/test_nudge_tool_calls_wiring.py b/studio/backend/tests/test_nudge_tool_calls_wiring.pyindex 35bfdf6565e..81214cc8d0e 100644--- a/studio/backend/tests/test_nudge_tool_calls_wiring.py+++ b/studio/backend/tests/test_nudge_tool_calls_wiring.py@@ -10,7 +10,8 @@ from core.inference.llama_cpp import LlamaCppBackendfrom core.inference.orchestrator import InferenceOrchestratorfrom core.inference.passthrough_healing import nudge_enabledfrom core.inference.safetensors_agentic import run_safetensors_tool_loop-from core.inference.studio_tool_loop import ToolLoopPolicy, stream_with_studio_tools+from core.inference import studio_tool_loop+from core.inference.studio_tool_loop import ToolLoopPolicy_CHAT_ADAPTER_SOURCE = (@@ -62,7 +63,8 @@ def test_delegating_backends_forward_the_flag_to_the_shared_loop():def test_gguf_and_external_loops_use_the_shared_nudge_normalizer():gguf_src = inspect.getsource(LlamaCppBackend.generate_chat_completion_with_tools)assert "_nudge_enabled(nudge_tool_calls)" in gguf_src- external_src = inspect.getsource(stream_with_studio_tools)+ # The public generator wraps the loop body, which is where the nudge is read.+ external_src = inspect.getsource(studio_tool_loop._stream_with_studio_tools)assert "nudge_enabled(policy.nudge_tool_calls)" in external_srcassert "nudge_tool_calls" in ToolLoopPolicy.__dataclass_fields__diff --git a/studio/backend/tests/test_search_images.py b/studio/backend/tests/test_search_images.pyindex 13e9aa50f8b..5f9d0c77489 100644--- a/studio/backend/tests/test_search_images.py+++ b/studio/backend/tests/test_search_images.py@@ -411,9 +411,16 @@ def test_a_genuine_search_failure_carries_no_pictures(monkeypatch):_SubjectDDGS.calls = []monkeypatch.setattr("ddgs.DDGS", Boom)+ registered = []+ monkeypatch.setattr(+ search_images,+ "register_images",+ lambda *a, **k: registered.append(a) or [],+ )result = tools._web_search("top dog breeds", include_images = True, image_queries = ["Pug"])assert result.startswith("Search failed:") and "[[img:" not in result- assert _SubjectDDGS.calls == []+ # The lookup may already be under way beside the text search; its answer is dropped unregistered.+ assert registered == []def test_clear_all_chats_beats_a_thumbnail_write_already_in_flight(monkeypatch, tmp_path):diff --git a/studio/backend/tests/test_web_tools_overlap.py b/studio/backend/tests/test_web_tools_overlap.pynew file mode 100644index 00000000000..1cbc2ad5626--- /dev/null+++ b/studio/backend/tests/test_web_tools_overlap.py@@ -0,0 +1,606 @@+# SPDX-License-Identifier: AGPL-3.0-only+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0++"""Web tool latency: image lookup beside the text search, and URL reads from one batch in flight at once.++Every test that checks an answer compares it with the serial path run in the same process, so the+overlap can only change when things happen, never what the model or the client sees.+"""++from __future__ import annotations++import asyncio+import http.server+import json+import re+import threading+import time++import pytest++from core.inference import search_images+from core.inference import studio_tool_loop as loop_mod+from core.inference import tools+from core.inference.studio_tool_loop import ToolLoopPolicy, ToolLoopRun, stream_with_studio_tools+++WEB = {+ "type": "function",+ "function": {+ "name": "web_search",+ "description": "",+ "parameters": {+ "type": "object",+ "properties": {"query": {"type": "string"}, "url": {"type": "string"}},+ },+ },+}+PY = {+ "type": "function",+ "function": {+ "name": "python",+ "description": "",+ "parameters": {"type": "object", "properties": {"code": {"type": "string"}}},+ },+}+++def _sse(delta = None, finish = None) -> str:+ choice: dict = {"index": 0, "delta": delta or {}}+ if finish is not None:+ choice["finish_reason"] = finish+ return "data: " + json.dumps({"choices": [choice]})+++def _batch(calls) -> list[str]:+ deltas = [+ {"index": i, "id": f"c{i}", "function": {"name": name, "arguments": json.dumps(args)}}+ for i, (name, args) in enumerate(calls)+ ]+ return [_sse({"tool_calls": deltas}), _sse(finish = "tool_calls"), "data: [DONE]"]+++class Transport:+ heals_text_tool_calls = True++ def __init__(self, calls):+ self.turns = [+ _batch(calls),+ [_sse({"content": "done"}), _sse(finish = "stop"), "data: [DONE]"],+ ]+ self.requests: list[list[dict]] = []++ def stream(self, *, messages, tools, tool_choice, cancel_event):+ self.requests.append([dict(m) for m in messages])+ lines = self.turns.pop(0) if self.turns else ["data: [DONE]"]++ async def gen():+ for line in lines:+ yield line++ return gen()+++class Recorder:+ """execute_tool stand-in: records each call's run interval and the peak number running at once."""++ def __init__(+ self,+ delay = 0.4,+ wait_for_cancel = False,+ ):+ self.delay = delay+ self.wait_for_cancel = wait_for_cancel+ self.lock = threading.Lock()+ self.running = 0+ self.peak = 0+ self.spans: dict[str, tuple[float, float]] = {}+ self.calls: list[str] = []+ self.saw_cancel: list[str] = []++ def __call__(self, name, arguments, **kwargs):+ label = arguments.get("url") or arguments.get("code") or arguments.get("query")+ with self.lock:+ self.calls.append(label)+ self.running += 1+ self.peak = max(self.peak, self.running)+ start = time.perf_counter()+ try:+ cancel = kwargs.get("cancel_event")+ if self.wait_for_cancel:+ if cancel.wait(5.0):+ self.saw_cancel.append(label)+ return "cancelled"+ else:+ time.sleep(self.delay if name == "web_search" else 0.05)+ return f"RESULT {name} {label}"+ finally:+ with self.lock:+ self.running -= 1+ self.spans[label] = (start, time.perf_counter())+++def recorder(monkeypatch):+ rec = Recorder()+ monkeypatch.setattr(loop_mod, "execute_tool", rec)+ monkeypatch.setattr(loop_mod, "build_rag_autoinject", lambda *a, **k: None)+ return rec+++def _run(+ transport,+ *,+ cancel_event = None,+ **policy,+):+ fields = {+ "tools": [WEB, PY],+ "max_calls": 25,+ "timeout": 60,+ "permission_mode": "off",+ "confirm_calls": False,+ "bypass_permissions": False,+ "rag_scope": None,+ }+ fields.update(policy)+ cancel_event = cancel_event or threading.Event()++ async def collect():+ out = []+ async for line in stream_with_studio_tools(+ transport,+ run = ToolLoopRun(+ messages = [{"role": "user", "content": "read"}], session_id = "s1", thread_id = "t1"+ ),+ policy = ToolLoopPolicy(**fields),+ cancel_event = cancel_event,+ ):+ out.append(line)+ return out++ t0 = time.perf_counter()+ lines = asyncio.run(asyncio.wait_for(collect(), timeout = 30))+ elapsed = time.perf_counter() - t0+ # Approval ids are minted fresh per call; keep-alives follow the clock.+ lines = [+ re.sub(r'"approval_id":"[^"]+"', '"approval_id":"<id>"', line)+ for line in lines+ if line != loop_mod._SSE_KEEPALIVE+ ]+ return lines, elapsed+++def _serial_and_overlapped(monkeypatch, recorder, calls, **policy):+ """The same batch run serially (concurrency 1, the old loop) and with overlap allowed."""+ monkeypatch.setattr(loop_mod, "_WEB_FETCH_CONCURRENCY", 1, raising = False)+ serial_transport = Transport(calls)+ serial_lines, serial_s = _run(serial_transport, **policy)+ serial_peak, serial_calls = recorder.peak, list(recorder.calls)+ recorder.peak, recorder.calls = 0, []+ monkeypatch.setattr(loop_mod, "_WEB_FETCH_CONCURRENCY", 4, raising = False)+ transport = Transport(calls)+ cancel = threading.Event()+ lines, overlapped_s = _run(transport, cancel_event = cancel, **policy)+ # A batch that finished cleanly never cancels the turn it belongs to.+ assert not cancel.is_set()+ assert lines == serial_lines+ assert transport.requests == serial_transport.requests+ # Same calls executed; start order among calls running together is not observable.+ assert sorted(recorder.calls) == sorted(serial_calls)+ return serial_peak, serial_s, overlapped_s, lines+++def _url(host, path = "p"):+ return ("web_search", {"url": f"https://{host}/{path}"})+++def _tool_ends(lines):+ out = []+ for line in lines:+ if line.startswith("data: {"):+ payload = json.loads(line[6:])+ if payload.get("type") == "tool_end":+ out.append((payload["tool_call_id"], payload["result"]))+ return out+++def test_independent_url_reads_run_together_and_replay_in_order(monkeypatch, recorder):+ calls = [_url("a.example"), _url("b.example"), _url("c.example")]+ serial_peak, serial_s, overlapped_s, lines = _serial_and_overlapped(+ monkeypatch, recorder, calls+ )+ assert serial_peak == 1+ assert recorder.peak == 3+ # Three 0.4 s reads: about 1.2 s one after another, about 0.4 s together.+ assert overlapped_s < serial_s - 0.5, (serial_s, overlapped_s)+ assert [cid for cid, _ in _tool_ends(lines)] == ["c0", "c1", "c2"]+++def test_reads_of_one_host_stay_serial(monkeypatch, recorder):+ calls = [_url("a.example", "1"), _url("a.example", "2"), _url("b.example")]+ _serial_and_overlapped(monkeypatch, recorder, calls)+ a1, a2 = recorder.spans["https://a.example/1"], recorder.spans["https://a.example/2"]+ assert a2[0] >= a1[1]+ # b.example may start beside a.example/2, never beside a.example/1.+ assert recorder.spans["https://b.example/p"][0] >= a1[1]+++def test_non_web_call_is_never_overlapped(monkeypatch, recorder):+ calls = [_url("a.example"), ("python", {"code": "print(1)"}), _url("b.example")]+ _serial_and_overlapped(monkeypatch, recorder, calls)+ a, py, b = (+ recorder.spans[k] for k in ("https://a.example/p", "print(1)", "https://b.example/p")+ )+ assert a[1] <= py[0] and py[1] <= b[0]+ assert recorder.peak == 1+++def test_web_search_queries_are_not_overlapped(monkeypatch, recorder):+ calls = [_url("a.example"), ("web_search", {"query": "x"}), _url("b.example")]+ _serial_and_overlapped(monkeypatch, recorder, calls)+ assert recorder.peak == 1+++def test_reads_that_ask_first_are_never_started_early(monkeypatch, recorder):+ # Auto mode asks before every URL read; each is answered Allow, after the previous one finished.+ answers: list[float] = []++ def allow(slot, approval_id, cancel_event):+ answers.append(time.perf_counter())+ return "allow"++ monkeypatch.setattr(loop_mod, "wait_tool_decision", allow)+ calls = [_url("a.example"), _url("b.example"), _url("c.example")]+ _serial_and_overlapped(monkeypatch, recorder, calls, permission_mode = "auto", confirm_calls = True)+ assert recorder.peak == 1+ starts = sorted(span[0] for span in recorder.spans.values())+ assert all(start >= answer for start, answer in zip(starts, answers[-3:]))+++def test_budget_is_reserved_before_reads_start(monkeypatch, recorder):+ calls = [_url("a.example"), _url("b.example"), _url("c.example")]+ _serial_and_overlapped(monkeypatch, recorder, calls, max_calls = 2)+ # The third read is answered by the budget card in both runs and never executes.+ assert sorted(recorder.calls) == ["https://a.example/p", "https://b.example/p"]+++def test_repeated_url_keeps_duplicate_handling(monkeypatch, recorder):+ calls = [_url("a.example"), _url("b.example"), _url("a.example")]+ _serial_and_overlapped(monkeypatch, recorder, calls)+ assert sorted(recorder.calls) == ["https://a.example/p", "https://b.example/p"]+++def test_cancel_stops_started_reads_promptly(monkeypatch):+ rec = Recorder(wait_for_cancel = True)+ monkeypatch.setattr(loop_mod, "execute_tool", rec)+ monkeypatch.setattr(loop_mod, "build_rag_autoinject", lambda *a, **k: None)+ cancel = threading.Event()+ threading.Timer(0.3, cancel.set).start()+ calls = [_url("a.example"), _url("b.example"), _url("c.example")]+ lines, elapsed = _run(Transport(calls), cancel_event = cancel)+ assert elapsed < 2.0+ # The started reads saw the cancel too, and only the call the serial loop had reached is recorded.+ assert sorted(rec.saw_cancel) == sorted(rec.calls)+ assert [cid for cid, _ in _tool_ends(lines)] == ["c0"]+ deadline = time.monotonic() + 2+ while rec.running and time.monotonic() < deadline:+ time.sleep(0.01)+ assert rec.running == 0+++def test_closing_the_stream_between_calls_stops_started_reads(monkeypatch):+ """Closed at a yield where no tool stream is open (the next card's tool_start), the reads started+ for later calls must stop as a closed running stream does, not run on against a gone turn."""+ lock = threading.Lock()+ running: list = []+ seen: list = []++ def execute(name, arguments, **kwargs):+ url = arguments["url"]+ with lock:+ seen.append(url)+ running.append(url)+ try:+ if "a.example" in url:+ time.sleep(0.2)+ else:+ kwargs["cancel_event"].wait(5.0)+ return f"RESULT {url}"+ finally:+ with lock:+ running.remove(url)++ monkeypatch.setattr(loop_mod, "execute_tool", execute)+ monkeypatch.setattr(loop_mod, "build_rag_autoinject", lambda *a, **k: None)+ cancel = threading.Event()+ calls = [_url("a.example"), _url("b.example"), _url("c.example")]++ async def close_at_second_card():+ gen = stream_with_studio_tools(+ Transport(calls),+ run = ToolLoopRun(+ messages = [{"role": "user", "content": "read"}], session_id = "s1", thread_id = "t1"+ ),+ policy = ToolLoopPolicy(+ tools = [WEB, PY],+ max_calls = 25,+ timeout = 60,+ permission_mode = "off",+ confirm_calls = False,+ bypass_permissions = False,+ rag_scope = None,+ ),+ cancel_event = cancel,+ )+ starts = 0+ async for line in gen:+ if '"tool_start"' in line:+ starts += 1+ if starts == 2:+ break+ await gen.aclose()++ asyncio.run(asyncio.wait_for(close_at_second_card(), timeout = 10))+ # The reads for c1 and c2 were started early, so the overlap was exercised.+ assert len(seen) == 3+ deadline = time.monotonic() + 2+ while running and time.monotonic() < deadline:+ time.sleep(0.01)+ assert running == []+ assert cancel.is_set()+++class _Handler(http.server.BaseHTTPRequestHandler):+ protocol_version = "HTTP/1.1"++ def log_message(self, *a):+ pass++ def do_GET(self):+ host = (self.headers.get("Host") or "").split(":")[0]+ if host.startswith("loop"):+ # Redirects to itself forever: the hop limit must end it.+ self._redirect(f"http://{self.headers['Host']}/again")+ elif host.startswith("private"):+ self._redirect("http://169.254.169.254/latest/meta-data/")+ else:+ body = (+ b"<html><body><main><h1>Page</h1><p>" + b"word " * 80 + b"</p></main></body></html>"+ )+ self.send_response(200)+ self.send_header("Content-Type", "text/html")+ self.send_header("Content-Length", str(len(body)))+ self.end_headers()+ self.wfile.write(body)++ def _redirect(self, location):+ self.send_response(302)+ self.send_header("Location", location)+ self.send_header("Content-Length", "0")+ self.end_headers()+++def local_sites(monkeypatch):+ server = http.server.ThreadingHTTPServer(("127.0.0.1", 0), _Handler)+ server.daemon_threads = True+ threading.Thread(target = server.serve_forever, daemon = True).start()+ real = tools._validate_and_resolve_host++ def resolve(host, port):+ # Only the test's own public-looking names reach loopback; everything else is checked for real.+ if host.endswith(".sites.example"):+ return True, "", ["127.0.0.1"]+ return real(host, port)++ monkeypatch.setattr(tools, "_validate_and_resolve_host", resolve)+ monkeypatch.setattr(loop_mod, "build_rag_autoinject", lambda *a, **k: None)+ yield server.server_address[1]+ server.shutdown()+ server.server_close()+++def test_guards_hold_on_every_hop_of_overlapped_reads(monkeypatch, local_sites):+ port = local_sites+ calls = [+ ("web_search", {"url": f"http://ok.sites.example:{port}/"}),+ ("web_search", {"url": f"http://private.sites.example:{port}/"}),+ ("web_search", {"url": f"http://loop.sites.example:{port}/"}),+ ]+ seen = []+ real_execute = loop_mod.execute_tool++ def counting(name, arguments, **kwargs):+ seen.append(arguments["url"])+ return real_execute(name, arguments, **kwargs)++ monkeypatch.setattr(loop_mod, "execute_tool", counting)+ monkeypatch.setattr(loop_mod, "_WEB_FETCH_CONCURRENCY", 1, raising = False)+ serial_lines, _ = _run(Transport(calls))+ monkeypatch.setattr(loop_mod, "_WEB_FETCH_CONCURRENCY", 4, raising = False)+ lines, _ = _run(Transport(calls))+ assert lines == serial_lines+ results = dict(_tool_ends(lines))+ assert "Page" in results["c0"]+ # The redirect hop to the metadata address is refused, not followed.+ assert results["c1"].startswith("Blocked")+ assert results["c2"] == "Failed to fetch URL: too many redirects."+ assert len(seen) == 6+++class _FakeDDGS:+ text_s = 0.3+ image_s = 0.3+ text_error: Exception | None = None+ image_queries: list = []++ def __init__(self, *a, **k):+ pass++ def text(self, query, **k):+ time.sleep(self.text_s)+ if self.text_error is not None:+ raise self.text_error+ return [+ {"title": f"T{i}", "href": f"https://site{i}.example.com/a", "body": f"snippet {i}"}+ for i in range(k.get("max_results") or 5)+ ]++ def images(self, query, **k):+ type(self).image_queries.append(query)+ time.sleep(self.image_s)+ return [+ {+ "title": f"{query} {i}",+ "image": f"https://img{i}.example.org/p.jpg",+ "thumbnail": f"https://img{i}.example.org/t.jpg",+ "url": f"https://src{i}.example.org/page",+ }+ for i in range(k.get("max_results") or 6)+ ]+++def fake_ddgs(monkeypatch, tmp_path):+ import ddgs++ monkeypatch.setattr(ddgs, "DDGS", _FakeDDGS)+ monkeypatch.setattr(_FakeDDGS, "text_error", None)+ monkeypatch.setattr(_FakeDDGS, "image_queries", [])+ monkeypatch.setattr(+ tools, "_ddgs_text", lambda client, q, n, backend: client.text(q, max_results = n)+ )+ monkeypatch.setattr(tools, "_resolve_engine_tiers", lambda engines: ["auto"])+ # A failure recorded by another test must not hold this one's lookups back.+ monkeypatch.setattr(tools, "_text_search_failed_at", None, raising = False)+ monkeypatch.setattr(search_images, "_cache_dir", lambda: tmp_path)+ # Registrations made here must not leak into other tests' registry.+ monkeypatch.setattr(search_images, "_registry", {})+ monkeypatch.setattr(search_images, "_inflight", {})+ monkeypatch.setattr(search_images, "_cleared_unservable", set())+ registered: list = []+ real_register = search_images.register_images++ def register(*a, **k):+ entries = real_register(*a, **k)+ registered.append(len(entries))+ return entries++ monkeypatch.setattr(search_images, "register_images", register)+ return registered+++def _norm(text: str) -> str:+ # Image ids are random per registration.+ return re.sub(r"\b[0-9a-f]{12}\b", "<id>", text)+++ "kwargs",+ [+ {"include_images": True},+ {"include_images": True, "image_queries": ["eiffel tower", "louvre"]},+ ],+ ids = ["generic", "named"],+)+def test_image_lookup_overlaps_text_search(monkeypatch, fake_ddgs, kwargs):+ # Untimed: the first search in a process pays one-off imports the serial arm below would skip.+ tools._web_search("paris", timeout = 20, **kwargs)+ t0 = time.perf_counter()+ overlapped = tools._web_search("paris", timeout = 20, **kwargs)+ overlapped_s = time.perf_counter() - t0+ # No sweep: the lookup runs after the text search, as it always did.+ if hasattr(tools, "_ImageSweep"):+ monkeypatch.setattr(tools._ImageSweep, "start", classmethod(lambda cls, *a: None))+ t0 = time.perf_counter()+ serial = tools._web_search("paris", timeout = 20, **kwargs)+ serial_s = time.perf_counter() - t0+ assert "[[img:" in overlapped or "img" in overlapped+ assert _norm(overlapped) == _norm(serial)+ # 0.3 s text + 0.3 s images one after the other; together about 0.3 s.+ assert overlapped_s < serial_s - 0.2, (serial_s, overlapped_s)+++def test_text_only_search_starts_no_image_lookup(fake_ddgs):+ tools._web_search("paris", timeout = 20, include_images = False)+ assert _FakeDDGS.image_queries == []++[email protected]("named", [False, True], ids = ["generic", "named"])+def test_discarded_image_lookup_registers_nothing(monkeypatch, fake_ddgs, named):+ # A genuine text failure with no fallback: the serial path never reached the images.+ monkeypatch.setattr(_FakeDDGS, "text_error", RuntimeError("engine exploded"))+ monkeypatch.setattr(+ tools, "_wikipedia_search", lambda *a, **k: (_ for _ in ()).throw(RuntimeError("down"))+ )+ extra = {"image_queries": ["eiffel tower"]} if named else {}+ result = tools._web_search("paris", timeout = 20, include_images = True, **extra)+ assert result.startswith("Search failed") and "img" not in result+ assert fake_ddgs == []+++def test_wikipedia_fallback_drops_generic_images_unregistered(monkeypatch, fake_ddgs):+ monkeypatch.setattr(_FakeDDGS, "text_error", RuntimeError("engine exploded"))+ monkeypatch.setattr(+ tools,+ "_wikipedia_search",+ lambda *a, **k: [+ {"title": "Paris", "href": "https://en.wikipedia.org/wiki/Paris", "body": "x"}+ ],+ )+ result = tools._web_search("paris", timeout = 20, include_images = True)+ assert "Wikipedia-only" in result and "[[img:" not in result+ assert fake_ddgs == []+++def test_cancel_during_text_search_discards_images(monkeypatch, fake_ddgs):+ cancel = threading.Event()+ threading.Timer(0.1, cancel.set).start()+ result = tools._web_search("paris", timeout = 20, include_images = True, cancel_event = cancel)+ assert result == "Search cancelled."+ assert fake_ddgs == []+++def test_failing_text_engines_hold_back_the_early_image_lookup(monkeypatch, fake_ddgs):+ """While text search keeps failing (a rate limited machine), only the first failure pays a discarded+ lookup; the next searches send none until the engines answer again."""+ monkeypatch.setattr(_FakeDDGS, "text_error", RuntimeError("202 Ratelimit"))+ monkeypatch.setattr(+ tools, "_wikipedia_search", lambda *a, **k: (_ for _ in ()).throw(RuntimeError("down"))+ )+ for named in (False, True, False):+ extra = {"image_queries": ["eiffel tower"]} if named else {}+ result = tools._web_search("paris", timeout = 20, include_images = True, **extra)+ assert result.startswith("Search failed") and "img" not in result+ time.sleep(0.5)+ assert len(_FakeDDGS.image_queries) == 1+ # An empty sweep is an answer, not a failure: it lifts the hold, and so do results.+ monkeypatch.setattr(_FakeDDGS, "text_error", None)+ found = tools._web_search("paris", timeout = 20, include_images = True)+ assert "img" in found+ assert tools._text_search_failed_at is None+ before = len(_FakeDDGS.image_queries)+ real_start = tools._ImageSweep.__dict__["start"].__func__+ early: list = []++ def start(cls, *a):+ sweep = real_start(cls, *a)+ early.append(sweep)+ return sweep++ monkeypatch.setattr(tools._ImageSweep, "start", classmethod(start))+ tools._web_search("paris", timeout = 20, include_images = True)+ # Overlapped again (wall time is not asserted: it moves with host load).+ assert len(early) == 1 and early[0] is not None+ assert len(_FakeDDGS.image_queries) == before + 1+ # The hold expires on its own too.+ monkeypatch.setattr(_FakeDDGS, "text_error", RuntimeError("202 Ratelimit"))+ tools._web_search("paris", timeout = 20, include_images = True)+ monkeypatch.setattr(+ tools, "_text_search_failed_at", time.monotonic() - tools._TEXT_SEARCH_FAILURE_HOLD_S - 1+ )+ before = len(_FakeDDGS.image_queries)+ tools._web_search("paris", timeout = 20, include_images = True)+ time.sleep(0.5)+ assert len(_FakeDDGS.image_queries) == before + 1