Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
fetch remote image urls before forwarding to llama-server (#11160)
oobabooga8b3c658c032e
diff --git a/studio/backend/core/inference/external_provider.py b/studio/backend/core/inference/external_provider.pyindex a6ba5c824c8..39e9fa9a08a 100644--- a/studio/backend/core/inference/external_provider.py+++ b/studio/backend/core/inference/external_provider.py@@ -9,6 +9,7 @@ import base64import ioimport json as _jsonimport mimetypes+import randomimport reimport threadingimport time@@ -871,61 +872,84 @@ def _client() -> httpx.AsyncClient:# Cap per-image fetch well below Gemini's ~20 MB total request budget._GEMINI_REMOTE_IMAGE_MAX_BYTES = 10 * 1024 * 1024_GEMINI_REMOTE_IMAGE_TIMEOUT_S = 15.0+# The socket timeout bounds one operation; a server dripping bytes needs a whole-fetch bound.+_REMOTE_IMAGE_FETCH_DEADLINE_S = 30.0-def _safe_fetch_image_for_gemini_sync(+def safe_fetch_remote_image_sync(url: str,fallback_mime: str,max_bytes: int = _GEMINI_REMOTE_IMAGE_MAX_BYTES,+ label: str = "Remote image fetch",+ deadline: Optional[float] = None,+ require_image_content_type: bool = True,) -> Optional[tuple[str, str]]:- """Synchronous IP-pinned HTTPS image fetch with SSRF guards. Uses the same pinned-IP + SNI- pattern as `tools._fetch_page_text` so DNS rebinding between validation and the connection- cannot redirect us to a private/metadata address. Follows up to 4 hops, re-validating each- redirect target. Returns (mime, base64) or None. `max_bytes` is clamped to the per-image cap- and also lets the caller pass the remaining per-request budget, so an over-budget URL is- rejected via Content-Length instead of being fully downloaded then discarded."""+ """Fetch an HTTPS image through a validated, pinned public IP.++ Redirects are revalidated and the response is capped by ``max_bytes``. ``deadline`` is a+ ``time.monotonic`` cutoff for the whole fetch. A caller that decodes the bytes itself can+ pass ``require_image_content_type=False`` to accept any declared type as ``fallback_mime``.+ All failures return ``None`` so callers do not expose details about the host network.+ """+ import http.clientimport urllib.errorimport urllib.request- from urllib.parse import urljoin, urlunparse+ from urllib.parse import quote, urljoin, urlunparse- # Refuse upfront if the per-request budget is already spent._byte_limit = min(max(0, int(max_bytes)), _GEMINI_REMOTE_IMAGE_MAX_BYTES)if _byte_limit <= 0:return None# Reuse tools.py's pinned-IP hardening: validate-once-then-pin.from .tools import (+ _IRI_PATH_SAFE,+ _IRI_QUERY_SAFE,_explicit_proxy_applies,+ _fetch_budget_exceeded,+ _fetch_hop_timeout,_NoRedirect,_pinned_netloc,+ _read_capped_body,+ _resolve_with_budget,_SNIHTTPSHandler,- _validate_and_resolve_host,+ _USER_AGENTS,)+ # Image hosts refuse a request with no User-Agent (Wikimedia answers 403), and until this+ # fetch moved here llama-server sent its own. Picked once, as _fetch_url_raw does.+ user_agent = random.choice(_USER_AGENTS)+ if deadline is None:+ deadline = time.monotonic() + _REMOTE_IMAGE_FETCH_DEADLINE_S+def _safe_parse_https(raw_url: str) -> Optional[tuple[Any, str, int]]:- """Validate https + hostname + port. Returns (parsed, host, port) or None. Handles- malformed-port and malformed-bracketed-IPv6 URLs that would else raise ValueError- mid-build."""+ """Return a parsed HTTPS URL, hostname and port, or ``None``."""try:parsed_url = urlparse(raw_url)host_value = parsed_url.hostnameport_value = parsed_url.port or 443except (ValueError, UnicodeError) as _err:logger.info(- "Gemini image fetch: refusing malformed url err=%s",+ f"{label}: refusing malformed url err=%s",type(_err).__name__,)return Nonescheme_value = (parsed_url.scheme or "").lower()if scheme_value != "https":logger.info(- "Gemini image fetch: refusing non-https scheme=%s",+ f"{label}: refusing non-https scheme=%s",scheme_value,)return Noneif not host_value:- logger.info("Gemini image fetch: refusing url with no hostname")+ logger.info(f"{label}: refusing url with no hostname")return None+ if not host_value.isascii():+ # http.client writes Host as ASCII, so an internationalized name travels as its A-label.+ try:+ host_value = host_value.encode("idna").decode("ascii")+ except UnicodeError:+ logger.info(f"{label}: refusing unencodable hostname")+ return Nonereturn parsed_url, host_value, port_valueparsed_info = _safe_parse_https(url)@@ -933,42 +957,59 @@ def _safe_fetch_image_for_gemini_sync(return Noneparsed, current_host, current_port = parsed_infocurrent_url = url- ok, reason, pinned_ips = _validate_and_resolve_host(current_host, current_port)+ ok, reason, pinned_ips = _resolve_with_budget(current_host, current_port, deadline, None)if not ok:logger.warning(- "Gemini image fetch: refusing host=%s reason=%s",+ f"{label}: refusing host=%s reason=%s",current_host,reason,)return Nonefor _hop in range(4):+ budget_error = _fetch_budget_exceeded(deadline, None)+ if budget_error is not None:+ logger.info(f"{label}: {budget_error} host=%s", current_host)+ return None# Pin to validated IP; SNI + cert still use the hostname via _SNIHTTPSHandler.cp_info = _safe_parse_https(current_url)if cp_info is None:return Nonecp, _cp_host, _cp_port = cp_info+ # http.client refuses a non-ASCII or spaced selector; encode it as _fetch_url_raw does.+ try:+ cp = cp._replace(+ path = quote(cp.path, safe = _IRI_PATH_SAFE),+ params = quote(cp.params, safe = _IRI_PATH_SAFE),+ query = quote(cp.query, safe = _IRI_QUERY_SAFE),+ )+ except UnicodeError:+ logger.info(f"{label}: refusing unencodable url host=%s", current_host)+ return Nonepinned_url = urlunparse(cp._replace(netloc = _pinned_netloc(pinned_ips[0], cp.port)))# Route on the hostname, as _fetch_url_raw does: no NO_PROXY entry matches the# pinned URL's IP. A proxied request reaches the origin through the proxy.- proxied = _explicit_proxy_applies("https", _pinned_netloc(current_host, cp.port))+ authority = _pinned_netloc(current_host, cp.port)+ proxied = _explicit_proxy_applies("https", authority)handlers = [_NoRedirect, _SNIHTTPSHandler(current_host, () if proxied else pinned_ips)]if not proxied:handlers.append(urllib.request.ProxyHandler({}))opener = urllib.request.build_opener(*handlers)req = urllib.request.Request(pinned_url,- headers = {"Host": current_host},+ headers = {"Host": authority, "User-Agent": user_agent},method = "GET",)try:- resp = opener.open(req, timeout = _GEMINI_REMOTE_IMAGE_TIMEOUT_S)+ resp = opener.open(+ req, timeout = _fetch_hop_timeout(_GEMINI_REMOTE_IMAGE_TIMEOUT_S, deadline)+ )except urllib.error.HTTPError as e:if e.code not in (301, 302, 303, 307, 308):logger.info(- "Gemini image fetch: status=%d host=%s",+ f"{label}: status=%d host=%s",e.code,current_host,)@@ -980,7 +1021,7 @@ def _safe_fetch_image_for_gemini_sync(current_url = urljoin(current_url, location)except (ValueError, UnicodeError) as _err:logger.info(- "Gemini image fetch: refusing malformed redirect err=%s",+ f"{label}: refusing malformed redirect err=%s",type(_err).__name__,)return None@@ -988,18 +1029,20 @@ def _safe_fetch_image_for_gemini_sync(if rp_info is None:return None_rp, current_host, current_port = rp_info- ok2, reason2, pinned_ips = _validate_and_resolve_host(current_host, current_port)+ ok2, reason2, pinned_ips = _resolve_with_budget(+ current_host, current_port, deadline, None+ )if not ok2:logger.warning(- "Gemini image fetch: refusing redirect host=%s reason=%s",+ f"{label}: refusing redirect host=%s reason=%s",current_host,reason2,)return Nonecontinue- except (urllib.error.URLError, OSError) as _err:+ except (urllib.error.URLError, OSError, http.client.HTTPException, ValueError) as _err:logger.warning(- "Gemini image fetch failed host=%s err=%s",+ f"{label} failed host=%s err=%s",current_host,type(_err).__name__,)@@ -1008,13 +1051,15 @@ def _safe_fetch_image_for_gemini_sync(with resp:status = getattr(resp, "status", None) or resp.getcode()if status != 200:- logger.info("Gemini image fetch: status=%s host=%s", status, current_host)+ logger.info(f"{label}: status=%s host=%s", status, current_host)return None_hdr_mime = (resp.headers.get("content-type") or "").split(";")[0].strip().lower()- # Declared non-image MIME is refused; missing MIME uses the caller's.+ # A declared non-image MIME is refused unless the caller decodes the bytes itself.+ if _hdr_mime and not _hdr_mime.startswith("image/") and not require_image_content_type:+ _hdr_mime = ""if _hdr_mime and not _hdr_mime.startswith("image/"):logger.info(- "Gemini image fetch: non-image content-type=%s host=%s",+ f"{label}: non-image content-type=%s host=%s",_hdr_mime,current_host,)@@ -1022,34 +1067,60 @@ def _safe_fetch_image_for_gemini_sync(_final_mime_pre = _hdr_mime if _hdr_mime else fallback_mimeif not isinstance(_final_mime_pre, str) or not _final_mime_pre.startswith("image/"):logger.info(- "Gemini image fetch: missing content-type and no image fallback host=%s",+ f"{label}: missing content-type and no image fallback host=%s",current_host,)return None_hdr_len = resp.headers.get("content-length")if _hdr_len and _hdr_len.isdigit() and int(_hdr_len) > _byte_limit:logger.info(- "Gemini image fetch: declared %s bytes exceeds cap=%s host=%s",+ f"{label}: declared %s bytes exceeds cap=%s host=%s",_hdr_len,_byte_limit,current_host,)return None# Read cap+1 to detect oversize without buffering unbounded data.- raw = resp.read(_byte_limit + 1)+ try:+ body_error, raw = _read_capped_body(+ resp, _byte_limit + 1, _GEMINI_REMOTE_IMAGE_TIMEOUT_S, deadline, None+ )+ except (OSError, http.client.HTTPException, ValueError) as _err:+ logger.warning(+ f"{label} failed host=%s err=%s",+ current_host,+ type(_err).__name__,+ )+ return None+ if body_error is not None:+ logger.info(f"{label}: {body_error} host=%s", current_host)+ return Noneif len(raw) > _byte_limit:logger.info(- "Gemini image fetch: streamed bytes exceed cap=%s host=%s",+ f"{label}: streamed bytes exceed cap=%s host=%s",_byte_limit,current_host,)return Nonereturn _final_mime_pre, base64.b64encode(raw).decode("ascii")- logger.info("Gemini image fetch: too many redirects host=%s", current_host)+ logger.info(f"{label}: too many redirects host=%s", current_host)return None+_GEMINI_IMAGE_FETCH_LABEL = "Gemini image fetch"+++def _safe_fetch_image_for_gemini_sync(+ url: str,+ fallback_mime: str,+ max_bytes: int = _GEMINI_REMOTE_IMAGE_MAX_BYTES,+) -> Optional[tuple[str, str]]:+ return safe_fetch_remote_image_sync(+ url, fallback_mime, max_bytes, label = _GEMINI_IMAGE_FETCH_LABEL+ )++async def _safe_fetch_image_for_gemini(url: str,fallback_mime: str,diff --git a/studio/backend/core/inference/tools.py b/studio/backend/core/inference/tools.pyindex 993efc48c1b..76490168760 100644--- a/studio/backend/core/inference/tools.py+++ b/studio/backend/core/inference/tools.py@@ -14011,6 +14011,9 @@ def _read_capped_body(resp, max_bytes, timeout, deadline, cancel_event):# Best-effort handle on the underlying socket so its timeout tightens as the deadline nears; absent on test# doubles, where the between-chunk budget check still bounds the read.sock = getattr(getattr(getattr(resp, "fp", None), "raw", None), "_sock", None)+ # A buffered read(n) keeps receiving until n bytes arrive, so a drip never reaches the+ # budget check; read1 returns after one receive.+ read = getattr(resp, "read1", None) or resp.readchunks = []remaining = max_byteswhile remaining > 0:@@ -14026,7 +14029,7 @@ def _read_capped_body(resp, max_bytes, timeout, deadline, cancel_event):sock.settimeout(_fetch_hop_timeout(timeout, deadline))except Exception:pass- chunk = resp.read(min(65536, remaining))+ chunk = read(min(65536, remaining))if not chunk:breakchunks.append(chunk)diff --git a/studio/backend/routes/inference.py b/studio/backend/routes/inference.pyindex eff38bb86f4..7b236f9c822 100644--- a/studio/backend/routes/inference.py+++ b/studio/backend/routes/inference.py@@ -8247,6 +8247,7 @@ async def _preflight_image_for_switch(" GGUF build of it, which accepts several."),)+ # Non-GGUF backends only receive decoded base64 images.if not target_is_gguf and image_preflight.get("remote") and image_preflight.get("b64") is None:raise HTTPException(status_code = 400,@@ -21509,7 +21510,9 @@ def _extract_content_parts(f"Ignoring image URL with no base64 payload: {url[:80]}...")else:- logger.warning(f"Remote image URLs not yet supported: {url[:80]}...")+ logger.warning(+ f"Dropping a remote image URL while flattening to text: {url[:80]}..."+ )# Latest message wins: send the image just attached, not the thread's first.if message_image_b64 is not None:latest_image_b64 = message_image_b64@@ -24345,6 +24348,14 @@ async def produce_openai_chat_completions("Image provided but current GGUF model does not support vision.",)+ # Fetch before the passthrough takes an admission lease, so a slow image host holds+ # no inference slot.+ if _messages_have_remote_image(payload.messages):+ try:+ await asyncio.to_thread(_inline_request_remote_images, payload)+ except HTTPException as exc:+ raise _reject(exc.status_code, exc.detail)+cancel_event = _chat_cancel_event(request)completion_id = f"chatcmpl-{uuid.uuid4().hex[:12]}"# `stream` defaults to False on ChatCompletionRequest (OpenAI spec@@ -32122,19 +32133,157 @@ def _image_bytes_to_png_b64(raw: bytes) -> str:return base64.b64encode(buf.getvalue()).decode("ascii")+# Match llama-server's per-image limit and add a per-request budget.+_REMOTE_IMAGE_MAX_BYTES = 10 * 1024 * 1024+_REMOTE_IMAGE_REQUEST_BUDGET_BYTES = 32 * 1024 * 1024+# Bound DNS, connection and redirect work independently of response size.+_REMOTE_IMAGE_MAX_COUNT = 8+# Wall-clock budget shared by every remote image fetch in one request.+_REMOTE_IMAGE_REQUEST_DEADLINE_S = 60.0++# Do not reveal why a caller-selected host could not be fetched.+_REMOTE_IMAGE_FETCH_REFUSAL = (+ "Could not fetch the remote image URL. Send the image as a base64 data URL instead."+)+# Token counting renders a fixed media marker and charges a flat per-image allowance.+_COUNT_IMAGE_PLACEHOLDER = (+ "data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAIAAACQd1Pe"+ "AAAADElEQVR4nGNgYGAAAAAEAAH2FzhVAAAAAElFTkSuQmCC"+)+++# A bare base64 payload holds neither a colon nor any of these, so a URL cannot pass as one.+_NOT_BASE64 = _re.compile(r"[^A-Za-z0-9+/=_\s-]")+# urlsplit drops these before it reads a scheme, so skip them the same way.+_URL_LEADING_BLANKS = "".join(map(chr, range(0x21)))+# Wider than any real scheme, so the scan stays bounded without missing one.+_MAX_IMAGE_SCHEME_CHARS = 64+++def _image_url_scheme(url: str) -> str:+ """Return a short URL scheme, or ``""`` for llama-server's bare-base64 form.++ Leading blanks are skipped and the window outruns any real scheme, so a scheme padded+ or stretched out of view cannot smuggle a URL through as a base64 payload.+ """+ head = url.lstrip(_URL_LEADING_BLANKS)[:_MAX_IMAGE_SCHEME_CHARS]+ if ":" not in head and not _NOT_BASE64.search(head):+ return ""+ return head.split(":", 1)[0][:_MAX_VIDEO_SCHEME_CHARS].lower()+++def _remote_image_scheme_rejection(scheme: str) -> Optional[tuple[int, str]]:+ """Allow only the HTTPS scheme supported by the pinned fetcher."""+ if scheme == "https":+ return None+ return (+ 400,+ f"Unsupported image URL scheme ('{scheme}:'). Send the image over https, "+ "or as a base64 data URL.",+ )+++def _placeholder_remote_images_for_count(openai_messages: list[dict]) -> None:+ """Replace remote images before ``/apply-template`` can fetch them."""+ for msg in openai_messages:+ content = msg.get("content")+ if not isinstance(content, list):+ continue+ for part in content:+ if not isinstance(part, dict) or part.get("type") != "image_url":+ continue+ image_url = part.get("image_url") or {}+ url = image_url.get("url", "")+ if url.startswith("data:"):+ continue+ scheme = _image_url_scheme(url)+ if not scheme:+ continue # Bare base64 payload, not a URL. Nothing to dial.+ rejection = _remote_image_scheme_rejection(scheme)+ if rejection is not None:+ raise HTTPException(status_code = rejection[0], detail = rejection[1])+ image_url["url"] = _COUNT_IMAGE_PLACEHOLDER+++def _inline_remote_image_url(+ url: str, scheme: str, budget_bytes: int, deadline: float+) -> tuple[str, int]:+ """Fetch an image safely and return its data URL and decoded size."""+ from core.inference.external_provider import safe_fetch_remote_image_sync++ rejection = _remote_image_scheme_rejection(scheme)+ if rejection is not None:+ raise HTTPException(status_code = rejection[0], detail = rejection[1])+ # llama-server never read the content type, and the bytes are decoded and re-encoded here.+ fetched = safe_fetch_remote_image_sync(+ url,+ "image/png",+ max_bytes = min(_REMOTE_IMAGE_MAX_BYTES, budget_bytes),+ label = "llama-server image fetch",+ deadline = deadline,+ require_image_content_type = False,+ )+ if fetched is None:+ raise HTTPException(status_code = 400, detail = _REMOTE_IMAGE_FETCH_REFUSAL)+ mime, b64 = fetched+ # Convert the encoded length back to the request budget's byte unit.+ return f"data:{mime};base64,{b64}", (len(b64) * 3) // 4+++class _RemoteImageFetches:+ """One request's remote image budget: bytes, URL count and a shared deadline."""++ def __init__(self):+ self.remaining_bytes = _REMOTE_IMAGE_REQUEST_BUDGET_BYTES+ self.remaining_fetches = _REMOTE_IMAGE_MAX_COUNT+ self.deadline: Optional[float] = None++ def inline(self, url: str) -> str:+ """Return a remote image as a data URL; llama-server's bare base64 form is returned as is."""+ scheme = _image_url_scheme(url)+ if not scheme:+ return url+ # A zero max_bytes value means no limit to some fetch callers.+ if self.remaining_bytes <= 0 or self.remaining_fetches <= 0:+ raise HTTPException(+ status_code = 400,+ detail = (+ f"Too many remote image URLs in one request (max {_REMOTE_IMAGE_MAX_COUNT})."+ if self.remaining_fetches <= 0+ else _REMOTE_IMAGE_FETCH_REFUSAL+ ),+ )+ self.remaining_fetches -= 1+ if self.deadline is None:+ self.deadline = time.monotonic() + _REMOTE_IMAGE_REQUEST_DEADLINE_S+ data_url, spent = _inline_remote_image_url(url, scheme, self.remaining_bytes, self.deadline)+ self.remaining_bytes -= spent+ return data_url+++def _inline_request_remote_images(payload) -> None:+ """Replace a chat request's remote image URLs with fetched data URLs, in place."""+ fetches = _RemoteImageFetches()+ for message in payload.messages:+ if not isinstance(message.content, list):+ continue+ for part in message.content:+ if isinstance(part, ImageContentPart) and not part.image_url.url.startswith("data:"):+ part.image_url.url = fetches.inline(part.image_url.url)++def _normalize_openai_image_parts_to_png(openai_messages: list[dict], on_image = None) -> bool:"""Re-encode every base64-data-URL ``image_url`` part to PNG, in place.- llama-server's stb_image only handles a few formats (JPEG/PNG/BMP/…), while- macOS and most browsers paste WebP and Claude Code sends WebP. Remote- (non-``data:``) URLs are forwarded as-is; llama-server will fetch (or fail)- per its own support matrix.+ Remote URLs are fetched here so llama-server receives bytes rather than a URL. All images+ are converted to PNG for llama-server's limited image decoder.``on_image`` runs once per image part before conversion, so a caller canapply its own guard. Returns ``True`` when any image part was seen. Raises- HTTPException(400) when an image cannot be decoded.+ HTTPException(400) when an image cannot be decoded or fetched."""has_image = False+ fetches = _RemoteImageFetches()for msg in openai_messages:content = msg.get("content")if not isinstance(content, list):@@ -32150,7 +32299,11 @@ def _normalize_openai_image_parts_to_png(openai_messages: list[dict], on_image =image_url = part.get("image_url") or {}url = image_url.get("url", "")if not url.startswith("data:"):- continue+ url = fetches.inline(url)+ if not url.startswith("data:"):+ # llama-server also accepts bare base64 payloads.+ continue+ image_url["url"] = urltry:_, b64data = url.split(",", 1)@@ -32824,6 +32977,8 @@ async def anthropic_count_tokens(# matches the prompt the real request would build (otherwise empty-assistant# sentinels / synthetic tool history inflate the count or hit the fallback).openai_messages = _sanitize_anthropic_openai_messages(openai_messages, llama_backend)+ # /apply-template fetches remote media even though token counting only needs its marker.+ _placeholder_remote_images_for_count(openai_messages)openai_tools = anthropic_tools_to_openai(payload.tools or []) or None# Only the client-tool passthrough is forwarded verbatim, so reproduce /messages' own# routing rather than "any tools": a Studio server-tool alias, or a template without@@ -35629,8 +35784,9 @@ def _openai_messages_for_passthrough(payload, normalize_images: bool = True) ->``role="tool"`` tool-result messages and assistant messages carryingstructured ``tool_calls``. Base64-data-URL images already in the list arere-encoded to PNG exactly as ``_openai_messages_for_gguf_chat`` does, so- turning tools on does not change which formats llama-server can decode;- remote URLs are forwarded as-is. The vision guard lives in the callers,+ turning tools on does not change which formats llama-server can decode, and+ a remote URL is fetched here rather than by llama-server, for the same+ reason and by the same helper. The vision guard lives in the callers,which reject a non-vision model before the body is built.``normalize_images=False`` is for callers that decode these bytes themselves: re-encodingdiff --git a/studio/backend/tests/test_anthropic_messages.py b/studio/backend/tests/test_anthropic_messages.pyindex 1133b8d56f6..cbc53e29311 100644--- a/studio/backend/tests/test_anthropic_messages.py+++ b/studio/backend/tests/test_anthropic_messages.py@@ -2604,7 +2604,14 @@ class TestNormalizeAnthropicOpenAIImages:assert new_url.startswith("data:image/png;base64,")assert new_url != original_url- def test_remote_url_left_unchanged(self):+ def test_remote_url_is_replaced_by_the_bytes_we_fetched(self, monkeypatch):+ import core.inference.external_provider as ep++ monkeypatch.setattr(+ ep,+ "safe_fetch_remote_image_sync",+ lambda *_a, **_k: ("image/png", _jpeg_data_url().partition(",")[2]),+ )msgs = [{"role": "user",@@ -2617,7 +2624,7 @@ class TestNormalizeAnthropicOpenAIImages:}]_normalize_anthropic_openai_images(msgs, is_vision = True)- assert msgs[0]["content"][0]["image_url"]["url"] == "https://x.example/y.png"+ assert msgs[0]["content"][0]["image_url"]["url"].startswith("data:image/png;base64,")def test_bad_base64_raises_400(self):msgs = [diff --git a/studio/backend/tests/test_gemini_provider.py b/studio/backend/tests/test_gemini_provider.pyindex 16561dd2da7..c86e455f53d 100644--- a/studio/backend/tests/test_gemini_provider.py+++ b/studio/backend/tests/test_gemini_provider.py@@ -31,6 +31,7 @@ and test_openai_image_generation.py.import asyncioimport base64+import ioimport jsonimport httpx@@ -2626,8 +2627,11 @@ def test_safe_fetch_image_pins_validated_ip_no_hostname_in_request(monkeypatch):def __exit__(self, *a):return False- def read(self, _n = None):- return b"PNG"+ # One body followed by EOF, as a real response reads.+ _body = io.BytesIO(b"PNG")++ def read(self, n = -1):+ return self._body.read(n)class _StubOpener:def open(@@ -3142,8 +3146,11 @@ def test_safe_fetch_image_missing_content_type_uses_fallback(monkeypatch):def __exit__(self, *a):return False- def read(self, _n = None):- return b"PNG"+ # One body followed by EOF, as a real response reads.+ _body = io.BytesIO(b"PNG")++ def read(self, n = -1):+ return self._body.read(n)class _StubOpener:def open(diff --git a/studio/backend/tests/test_openai_tool_passthrough.py b/studio/backend/tests/test_openai_tool_passthrough.pyindex ea1daa5518f..9b209e56a27 100644--- a/studio/backend/tests/test_openai_tool_passthrough.py+++ b/studio/backend/tests/test_openai_tool_passthrough.py@@ -10491,9 +10491,45 @@ class TestPassthroughImageNormalization:px = Image.open(BytesIO(out)).getpixel((0, 0))assert px == ((77, 77, 77) if mode == "L" else (10, 20, 30)), (mode, px)- def test_remote_url_is_forwarded_unchanged(self):+ def test_remote_url_is_fetched_here_and_never_forwarded(self, monkeypatch):+ from PIL import Image+ from io import BytesIO++ buf = BytesIO()+ Image.new("RGB", (2, 2), (4, 5, 6)).save(buf, format = "WEBP")+ webp_b64 = base64.b64encode(buf.getvalue()).decode("ascii")++ import core.inference.external_provider as ep++ monkeypatch.setattr(+ ep, "safe_fetch_remote_image_sync", lambda *a, **k: ("image/webp", webp_b64)+ )+messages = _openai_messages_for_passthrough(self._req("https://x.example/a.webp"))- assert messages[0]["content"][1]["image_url"]["url"] == "https://x.example/a.webp"++ url = messages[0]["content"][1]["image_url"]["url"]+ assert url.startswith("data:image/png;base64,")++ def test_remote_url_refused_when_the_fetch_is(self, monkeypatch):+ import core.inference.external_provider as ep++ monkeypatch.setattr(ep, "safe_fetch_remote_image_sync", lambda *a, **k: None)++ with pytest.raises(HTTPException) as exc:+ _openai_messages_for_passthrough(self._req("https://x.example/a.webp"))+ assert exc.value.status_code == 400++ def test_non_https_remote_url_is_refused_without_a_fetch(self, monkeypatch):+ import core.inference.external_provider as ep++ def _never(*_a, **_k):+ raise AssertionError("http:// must be refused before any request")++ monkeypatch.setattr(ep, "safe_fetch_remote_image_sync", _never)++ with pytest.raises(HTTPException) as exc:+ _openai_messages_for_passthrough(self._req("http://169.254.169.254/latest/meta-data/"))+ assert exc.value.status_code == 400def test_local_template_caller_leaves_a_payloadless_data_url_alone(self):# The safetensors/MLX client-tools path only gets here when the turn has nodiff --git a/studio/backend/tests/test_remote_image_url_fetch_behaviour.py b/studio/backend/tests/test_remote_image_url_fetch_behaviour.pynew file mode 100644index 00000000000..57ede0dcca2--- /dev/null+++ b/studio/backend/tests/test_remote_image_url_fetch_behaviour.py@@ -0,0 +1,768 @@+# SPDX-License-Identifier: AGPL-3.0-only+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0++"""Ensure remote image URLs are fetched safely or refused before llama-server sees them."""++from __future__ import annotations++import base64+import copy+import socket+import threading+import time+import urllib.error+from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer+from io import BytesIO+from types import SimpleNamespace++import pytest++pytest.importorskip("torch")++from fastapi import FastAPI # noqa: E402+from fastapi.testclient import TestClient # noqa: E402+from PIL import Image # noqa: E402++from auth.authentication import get_current_subject # noqa: E402+import core.inference.external_provider as external_provider # noqa: E402+import routes.inference as inference_route # noqa: E402++from .llama_backend_double import FakeLlamaCppBackend # noqa: E402++_METADATA = "http://169.254.169.254/latest/meta-data/"+++def _webp_b64() -> str:+ buf = BytesIO()+ Image.new("RGB", (2, 2), (7, 8, 9)).save(buf, format = "WEBP")+ return base64.b64encode(buf.getvalue()).decode("ascii")+++def _data_url() -> str:+ buf = BytesIO()+ Image.new("RGB", (2, 2), (1, 2, 3)).save(buf, format = "PNG")+ return "data:image/png;base64," + base64.b64encode(buf.getvalue()).decode("ascii")+++class _VisionGguf(FakeLlamaCppBackend):+ is_vision = True++ def __init__(self):+ self.dispatched: list[dict] = []+ self.counted: list[list[dict]] = []++ def generate_chat_completion(self, **kwargs):+ self.dispatched.append(kwargs)+ yield "ok"+ yield {"type": "metadata", "usage": {}, "timings": {}}++ generate_chat_completion_with_tools = generate_chat_completion++ def count_chat_tokens(self, messages, *_a, **_k):+ self.counted.append(copy.deepcopy(messages))+ return 100+++def _client(monkeypatch, backend):+ async def _no_switch(*_a, **_k):+ return None++ monkeypatch.setattr(inference_route, "get_llama_cpp_backend", lambda: backend)+ monkeypatch.setattr(inference_route, "_maybe_auto_switch_model", _no_switch)++ app = FastAPI()+ app.include_router(inference_route.router, prefix = "/v1")+ app.dependency_overrides[get_current_subject] = lambda: "tester"+ return TestClient(app, raise_server_exceptions = False)+++def _chat_body(*urls, model = "test/model.gguf"):+ parts = [{"type": "image_url", "image_url": {"url": u}} for u in urls]+ parts.append({"type": "text", "text": "what is this?"})+ return {+ "model": model,+ "stream": False,+ "messages": [{"role": "user", "content": parts}],+ }+++def _messages_body(url):+ return {+ "model": "test/model.gguf",+ "max_tokens": 16,+ "stream": False,+ "messages": [+ {+ "role": "user",+ "content": [+ {"type": "image", "source": {"type": "url", "url": url}},+ {"type": "text", "text": "what is this?"},+ ],+ }+ ],+ }+++def _image_urls(dispatched_messages) -> list[str]:+ return [+ part["image_url"]["url"]+ for msg in dispatched_messages+ for part in (msg.get("content") if isinstance(msg.get("content"), list) else [])+ if isinstance(part, dict) and part.get("type") == "image_url"+ ]+++def canary():+ """A loopback HTTP server that records every request it is asked to serve."""+ hits: list[str] = []++ class _Handler(BaseHTTPRequestHandler):+ def do_GET(self): # noqa: N802+ hits.append(self.path)+ body = base64.b64decode(_webp_b64())+ self.send_response(200)+ self.send_header("Content-Type", "image/webp")+ self.send_header("Content-Length", str(len(body)))+ self.end_headers()+ self.wfile.write(body)++ def log_message(self, *_a):+ pass++ server = ThreadingHTTPServer(("127.0.0.1", 0), _Handler)+ thread = threading.Thread(target = server.serve_forever, daemon = True)+ thread.start()+ try:+ yield SimpleNamespace(port = server.server_address[1], hits = hits)+ finally:+ server.shutdown()+ server.server_close()+ thread.join(timeout = 5)+++class TestRemoteUrlNeverReachesLlamaServer:+ def test_public_https_url_is_fetched_here_and_dispatched_as_bytes(self, monkeypatch):+ monkeypatch.setattr(+ external_provider,+ "safe_fetch_remote_image_sync",+ lambda *_a, **_k: ("image/webp", _webp_b64()),+ )+ backend = _VisionGguf()+ r = _client(monkeypatch, backend).post(+ "/v1/chat/completions", json = _chat_body("https://images.example/cat.webp")+ )++ assert r.status_code == 200, r.text+ urls = _image_urls(backend.dispatched[-1]["messages"])+ assert len(urls) == 1+ assert urls[0].startswith("data:image/png;base64,")++ def test_anthropic_url_source_is_fetched_here_too(self, monkeypatch):+ monkeypatch.setattr(+ external_provider,+ "safe_fetch_remote_image_sync",+ lambda *_a, **_k: ("image/webp", _webp_b64()),+ )+ backend = _VisionGguf()+ r = _client(monkeypatch, backend).post(+ "/v1/messages", json = _messages_body("https://images.example/cat.webp")+ )++ assert r.status_code == 200, r.text+ urls = _image_urls(backend.dispatched[-1]["messages"])+ assert urls and all(u.startswith("data:image/png;base64,") for u in urls)++ def test_responses_input_image_lands_on_the_same_guard(self, monkeypatch):+ backend = _VisionGguf()+ r = _client(monkeypatch, backend).post(+ "/v1/responses",+ json = {+ "model": "test/model.gguf",+ "stream": False,+ "input": [+ {+ "role": "user",+ "content": [+ {"type": "input_image", "image_url": _METADATA},+ {"type": "input_text", "text": "what is this?"},+ ],+ }+ ],+ },+ )++ assert r.status_code == 400, r.text+ assert backend.dispatched == []++ def test_the_tool_passthrough_fetches_before_taking_an_admission_lease(self, monkeypatch):+ from fastapi.responses import JSONResponse++ monkeypatch.setattr(+ external_provider,+ "safe_fetch_remote_image_sync",+ lambda *_a, **_k: ("image/webp", _webp_b64()),+ )+ monkeypatch.setattr(inference_route, "_takes_tool_passthrough", lambda *_a, **_k: True)+ admitted_urls = []++ async def _admitted(_backend, payload, *_a, **_k):+ admitted_urls.extend(+ part.image_url.url+ for message in payload.messages+ if isinstance(message.content, list)+ for part in message.content+ if getattr(part, "type", None) == "image_url"+ )+ return JSONResponse({"ok": True})++ monkeypatch.setattr(inference_route, "_openai_passthrough_non_streaming", _admitted)+ r = _client(monkeypatch, _VisionGguf()).post(+ "/v1/chat/completions", json = _chat_body("https://images.example/cat.webp")+ )++ assert r.status_code == 200, r.text+ assert len(admitted_urls) == 1+ assert admitted_urls[0].startswith("data:image/webp;base64,")++ def test_bare_base64_is_payload_and_still_passes_through(self, monkeypatch):+ def _never(*_a, **_k):+ raise AssertionError("payload must not be treated as a URL")++ monkeypatch.setattr(external_provider, "safe_fetch_remote_image_sync", _never)+ backend = _VisionGguf()+ r = _client(monkeypatch, backend).post("/v1/chat/completions", json = _chat_body(_webp_b64()))++ assert r.status_code == 200, r.text+ assert _image_urls(backend.dispatched[-1]["messages"]) == [_webp_b64()]++ def test_a_data_url_still_goes_through_untouched_by_the_fetch(self, monkeypatch):+ def _never(*_a, **_k):+ raise AssertionError("a data: url needs no fetch")++ monkeypatch.setattr(external_provider, "safe_fetch_remote_image_sync", _never)+ backend = _VisionGguf()+ r = _client(monkeypatch, backend).post("/v1/chat/completions", json = _chat_body(_data_url()))++ assert r.status_code == 200, r.text+ assert _image_urls(backend.dispatched[-1]["messages"])[0].startswith("data:image/png")+++class TestRefusalsAreRealRefusals:+ @pytest.mark.parametrize(+ "url_for",+ [+ pytest.param(lambda p: f"http://127.0.0.1:{p}/x.png", id = "http-loopback"),+ pytest.param(lambda p: f"https://127.0.0.1:{p}/x.png", id = "https-loopback"),+ pytest.param(lambda p: f"https://localhost:{p}/x.png", id = "https-resolves-loopback"),+ ],+ )+ def test_loopback_is_refused_and_never_contacted(self, monkeypatch, canary, url_for):+ backend = _VisionGguf()+ r = _client(monkeypatch, backend).post(+ "/v1/chat/completions", json = _chat_body(url_for(canary.port))+ )++ assert r.status_code == 400, r.text+ assert backend.dispatched == []+ assert canary.hits == [], "the guard let a request through to the loopback server"++ def test_the_metadata_endpoint_is_refused(self, monkeypatch):+ backend = _VisionGguf()+ r = _client(monkeypatch, backend).post("/v1/chat/completions", json = _chat_body(_METADATA))++ assert r.status_code == 400, r.text+ assert backend.dispatched == []++ def test_a_failed_fetch_refuses_rather_than_dropping_the_image(self, monkeypatch):+ monkeypatch.setattr(+ external_provider, "safe_fetch_remote_image_sync", lambda *_a, **_k: None+ )+ backend = _VisionGguf()+ r = _client(monkeypatch, backend).post(+ "/v1/chat/completions", json = _chat_body("https://images.example/gone.png")+ )++ assert r.status_code == 400, r.text+ assert backend.dispatched == []++ def test_one_bad_url_refuses_the_whole_request(self, monkeypatch, canary):+ monkeypatch.setattr(+ external_provider,+ "safe_fetch_remote_image_sync",+ lambda *_a, **_k: ("image/webp", _webp_b64()),+ )+ backend = _VisionGguf()+ r = _client(monkeypatch, backend).post(+ "/v1/chat/completions",+ json = _chat_body(+ "https://images.example/ok.webp", f"http://127.0.0.1:{canary.port}/x.png"+ ),+ )++ assert r.status_code == 400, r.text+ assert backend.dispatched == []+ assert canary.hits == []++ @pytest.mark.parametrize("url", ["file:///etc/passwd", "ftp://x.example/a.png"])+ def test_another_scheme_is_refused_by_name(self, monkeypatch, url):+ def _never(*_a, **_k):+ raise AssertionError("only https reaches the fetcher")++ monkeypatch.setattr(external_provider, "safe_fetch_remote_image_sync", _never)+ backend = _VisionGguf()+ r = _client(monkeypatch, backend).post("/v1/chat/completions", json = _chat_body(url))++ assert r.status_code == 400, r.text+ assert backend.dispatched == []++ @pytest.mark.parametrize(+ "url",+ [+ pytest.param("\n" * 13 + _METADATA, id = "blank-padded"),+ pytest.param("\x00" * 20 + "http://127.0.0.1/x.png", id = "nul-padded"),+ pytest.param("averyveryverylongscheme://x.example/a.png", id = "over-long-scheme"),+ ],+ )+ def test_a_scheme_pushed_out_of_the_window_is_still_a_scheme(self, monkeypatch, url):+ """urlsplit reads these as http; passing them off as base64 would forward a URL."""++ def _never(*_a, **_k):+ raise AssertionError("only https reaches the fetcher")++ monkeypatch.setattr(external_provider, "safe_fetch_remote_image_sync", _never)+ backend = _VisionGguf()+ r = _client(monkeypatch, backend).post("/v1/chat/completions", json = _chat_body(url))++ assert r.status_code == 400, r.text+ assert backend.dispatched == []++ @pytest.mark.parametrize(+ "payload",+ ["iVBORw0KGgoAAAANSUhEUgAAAAEAAAAB", "iVBORw0-KGgo_AAAANSUhEUg==", "iVBORw0KGgo\nAAAANS"],+ )+ def test_base64_shapes_are_still_read_as_payload(self, payload):+ assert inference_route._image_url_scheme(payload) == ""++ def test_the_vision_guard_still_runs_before_any_fetch(self, monkeypatch):+ def _never(*_a, **_k):+ raise AssertionError("a text-only model must refuse before fetching")++ monkeypatch.setattr(external_provider, "safe_fetch_remote_image_sync", _never)++ class _TextGguf(_VisionGguf):+ is_vision = False++ r = _client(monkeypatch, _TextGguf()).post(+ "/v1/chat/completions", json = _chat_body("https://images.example/cat.webp")+ )+ assert r.status_code == 400, r.text+++class TestBudget:+ def test_the_per_request_byte_budget_bounds_a_long_thread(self, monkeypatch):+ b64 = _webp_b64()+ each = (len(b64) * 3) // 4+ monkeypatch.setattr(inference_route, "_REMOTE_IMAGE_REQUEST_BUDGET_BYTES", 2 * each + 1)+ calls = []++ def _fetch(+ url,+ _mime,+ max_bytes = 0,+ **_k,+ ):+ calls.append((url, max_bytes))+ return ("image/webp", b64) if each <= max_bytes else None++ monkeypatch.setattr(external_provider, "safe_fetch_remote_image_sync", _fetch)+ backend = _VisionGguf()+ r = _client(monkeypatch, backend).post(+ "/v1/chat/completions",+ json = _chat_body(*[f"https://images.example/{i}.webp" for i in range(4)]),+ )++ assert r.status_code == 400, r.text+ assert [c[1] for c in calls] == [2 * each + 1, each + 1, 1], calls+ assert backend.dispatched == []++ def test_the_per_request_count_cap_bounds_the_round_trips(self, monkeypatch):+ calls = []++ def _fetch(url, *_a, **_k):+ calls.append(url)+ return "image/webp", _webp_b64()++ monkeypatch.setattr(external_provider, "safe_fetch_remote_image_sync", _fetch)+ backend = _VisionGguf()+ over = inference_route._REMOTE_IMAGE_MAX_COUNT + 1+ r = _client(monkeypatch, backend).post(+ "/v1/chat/completions",+ json = _chat_body(*[f"https://images.example/{i}.webp" for i in range(over)]),+ )++ assert r.status_code == 400, r.text+ assert "Too many remote image URLs" in r.text+ assert len(calls) == inference_route._REMOTE_IMAGE_MAX_COUNT, calls+ assert backend.dispatched == []++ def test_every_fetch_in_a_request_shares_one_deadline(self, monkeypatch):+ deadlines = []++ def _fetch(+ url,+ *_a,+ deadline = None,+ **_k,+ ):+ deadlines.append(deadline)+ return "image/webp", _webp_b64()++ monkeypatch.setattr(external_provider, "safe_fetch_remote_image_sync", _fetch)+ backend = _VisionGguf()+ r = _client(monkeypatch, backend).post(+ "/v1/chat/completions",+ json = _chat_body("https://images.example/a.webp", "https://images.example/b.webp"),+ )++ assert r.status_code == 200, r.text+ assert len(deadlines) == 2 and deadlines[0] is not None, deadlines+ assert deadlines[0] == deadlines[1]+++def _resolve_publicly(monkeypatch):+ real = socket.getaddrinfo++ def _fake(host, *args, **kwargs):+ if host in ("images.example", "xn--bcher-kva.example"):+ return [(socket.AF_INET, socket.SOCK_STREAM, 0, "", ("93.184.216.34", 0))]+ if host == "2606:4700::1111":+ return [(socket.AF_INET6, socket.SOCK_STREAM, 0, "", (host, 0, 0, 0))]+ return real(host, *args, **kwargs)++ monkeypatch.setattr(socket, "getaddrinfo", _fake)+++class TestFetcher:+ def test_a_server_dripping_bytes_is_cut_off_at_the_deadline(self, monkeypatch):+ _resolve_publicly(monkeypatch)++ class _DripResponse:+ status = 200+ headers = {"content-type": "image/png"}++ def __enter__(self):+ return self++ def __exit__(self, *_a):+ return False++ def close(self):+ pass++ def read(self, _n = -1):+ time.sleep(0.02)+ return b"x"++ class _Opener:+ def open(+ self,+ _req,+ timeout = None,+ ):+ return _DripResponse()++ monkeypatch.setattr("urllib.request.build_opener", lambda *_a, **_k: _Opener())+ started = time.monotonic()+ result = external_provider.safe_fetch_remote_image_sync(+ "https://images.example/slow.png",+ "image/png",+ deadline = started + 0.3,+ )++ assert result is None+ assert time.monotonic() - started < 5++ def test_a_body_read_timeout_is_a_refusal_not_an_error(self, monkeypatch):+ _resolve_publicly(monkeypatch)++ class _StalledResponse:+ status = 200+ headers = {"content-type": "image/png"}++ def __enter__(self):+ return self++ def __exit__(self, *_a):+ return False++ def read(self, _n = -1):+ raise TimeoutError("timed out")++ class _Opener:+ def open(+ self,+ _req,+ timeout = None,+ ):+ return _StalledResponse()++ monkeypatch.setattr("urllib.request.build_opener", lambda *_a, **_k: _Opener())+ assert (+ external_provider.safe_fetch_remote_image_sync(+ "https://images.example/stalled.png", "image/png"+ )+ is None+ )++ def test_a_non_ascii_or_spaced_path_is_percent_encoded(self, monkeypatch):+ _resolve_publicly(monkeypatch)+ sent = []++ class _Opener:+ def open(+ self,+ req,+ timeout = None,+ ):+ sent.append(req.full_url)+ raise urllib.error.URLError("stop after the request is built")++ monkeypatch.setattr("urllib.request.build_opener", lambda *_a, **_k: _Opener())+ external_provider.safe_fetch_remote_image_sync(+ "https://images.example/caf\u00e9 1.png?q=\u00e9", "image/png"+ )++ assert sent == ["https://93.184.216.34/caf%C3%A9%201.png?q=%C3%A9"]++ @pytest.mark.parametrize("require,expected", [(True, None), (False, "image/png")])+ def test_a_generic_content_type_is_left_to_callers_that_decode(+ self, monkeypatch, require, expected+ ):+ _resolve_publicly(monkeypatch)+ body = base64.b64decode(_webp_b64())++ class _Response:+ status = 200+ headers = {"content-type": "application/octet-stream"}+ _body = BytesIO(body)++ def __enter__(self):+ return self++ def __exit__(self, *_a):+ return False++ def read(self, n = -1):+ return self._body.read(n)++ class _Opener:+ def open(+ self,+ _req,+ timeout = None,+ ):+ return _Response()++ monkeypatch.setattr("urllib.request.build_opener", lambda *_a, **_k: _Opener())+ result = external_provider.safe_fetch_remote_image_sync(+ "https://images.example/object",+ "image/png",+ require_image_content_type = require,+ )++ assert (result and result[0]) == expected++ def test_the_llama_server_path_decodes_an_octet_stream_image(self, monkeypatch):+ seen = {}++ def _fetch(url, fallback_mime, **kwargs):+ seen.update(kwargs)+ return (+ None+ if kwargs.get("require_image_content_type", True)+ else (fallback_mime, _webp_b64())+ )++ monkeypatch.setattr(external_provider, "safe_fetch_remote_image_sync", _fetch)+ backend = _VisionGguf()+ r = _client(monkeypatch, backend).post(+ "/v1/chat/completions", json = _chat_body("https://images.example/object")+ )++ assert r.status_code == 200, r.text+ assert seen["require_image_content_type"] is False+ assert _image_urls(backend.dispatched[-1]["messages"])[0].startswith(+ "data:image/png;base64,"+ )++ def test_an_unencodable_path_is_a_400(self, monkeypatch):+ def _never(*_a, **_k):+ raise AssertionError("an unencodable URL must not be dialled")++ _resolve_publicly(monkeypatch)+ monkeypatch.setattr("urllib.request.build_opener", _never)+ backend = _VisionGguf()+ r = _client(monkeypatch, backend).post(+ "/v1/chat/completions",+ content = (+ '{"model": "test/model.gguf", "stream": false, "messages": [{"role": "user",'+ ' "content": [{"type": "image_url", "image_url": {"url":'+ ' "https://images.example/\\ud800.png"}}, {"type": "text", "text": "x"}]}]}'+ ),+ headers = {"Content-Type": "application/json"},+ )++ assert r.status_code == 400, r.text+ assert backend.dispatched == []++ def test_a_request_the_client_cannot_build_is_a_400(self, monkeypatch):+ import http.client++ _resolve_publicly(monkeypatch)++ class _Opener:+ def open(+ self,+ _req,+ timeout = None,+ ):+ raise http.client.InvalidURL("URL can't contain control characters")++ monkeypatch.setattr("urllib.request.build_opener", lambda *_a, **_k: _Opener())+ backend = _VisionGguf()+ r = _client(monkeypatch, backend).post(+ "/v1/chat/completions", json = _chat_body("https://images.example/a.png")+ )++ assert r.status_code == 400, r.text+ assert backend.dispatched == []++ @pytest.mark.parametrize(+ "url,authority",+ [+ ("https://images.example:8443/a.png", "images.example:8443"),+ ("https://[2606:4700::1111]/a.png", "[2606:4700::1111]"),+ ("https://b\u00fccher.example/a.png", "xn--bcher-kva.example"),+ ],+ )+ def test_the_host_header_keeps_the_url_authority(self, monkeypatch, url, authority):+ _resolve_publicly(monkeypatch)+ sent = []++ class _Opener:+ def open(+ self,+ req,+ timeout = None,+ ):+ sent.append(req.get_header("Host"))+ raise urllib.error.URLError("stop after the request is built")++ monkeypatch.setattr("urllib.request.build_opener", lambda *_a, **_k: _Opener())+ assert external_provider.safe_fetch_remote_image_sync(url, "image/png") is None+ assert sent == [authority]++ def test_the_request_carries_a_user_agent(self, monkeypatch):+ """Wikimedia and friends answer 403 without one, and llama-server used to send its own."""+ _resolve_publicly(monkeypatch)+ sent = []++ class _Opener:+ def open(+ self,+ req,+ timeout = None,+ ):+ sent.append(req.get_header("User-agent"))+ raise urllib.error.URLError("stop after the request is built")++ monkeypatch.setattr("urllib.request.build_opener", lambda *_a, **_k: _Opener())+ external_provider.safe_fetch_remote_image_sync("https://images.example/a.png", "image/png")++ from core.inference.tools import _USER_AGENTS++ assert sent and sent[0], "the fetch went out with no User-Agent"+ assert sent[0] in _USER_AGENTS+++class TestCountingIsNotAFetchSurface:+ def test_openai_count_tokens_still_refuses_an_image(self, monkeypatch):+ def _never(*_a, **_k):+ raise AssertionError("counting must not spend a request")++ monkeypatch.setattr(external_provider, "safe_fetch_remote_image_sync", _never)+ backend = _VisionGguf()+ backend.supports_tools = True+ r = _client(monkeypatch, backend).post(+ "/v1/chat/count_tokens", json = _chat_body("https://images.example/cat.webp")+ )++ assert r.status_code == 503, r.text+ assert "images" in r.text++ def test_anthropic_count_tokens_prices_a_remote_image_without_dialling_it(+ self, monkeypatch, canary+ ):+ def _never(*_a, **_k):+ raise AssertionError("counting must not spend a request")++ monkeypatch.setattr(external_provider, "safe_fetch_remote_image_sync", _never)+ backend = _VisionGguf()+ r = _client(monkeypatch, backend).post(+ "/v1/messages/count_tokens",+ json = _messages_body(f"https://127.0.0.1:{canary.port}/counted.png"),+ )++ assert r.status_code == 200, r.text+ assert canary.hits == []+ counted_urls = _image_urls(backend.counted[-1])+ assert counted_urls and all(u.startswith("data:image/png;base64,") for u in counted_urls)++ def test_the_counted_placeholder_prices_the_same_as_a_real_image(self, monkeypatch):+ backend = _VisionGguf()+ client = _client(monkeypatch, backend)+ remote = client.post(+ "/v1/messages/count_tokens",+ json = _messages_body("https://images.example/cat.png"),+ )+ inline = client.post(+ "/v1/messages/count_tokens",+ json = {+ "model": "test/model.gguf",+ "max_tokens": 16,+ "messages": [+ {+ "role": "user",+ "content": [+ {+ "type": "image",+ "source": {+ "type": "base64",+ "media_type": "image/png",+ "data": _data_url().partition(",")[2],+ },+ },+ {"type": "text", "text": "what is this?"},+ ],+ }+ ],+ },+ )++ assert remote.status_code == 200, remote.text+ assert inline.status_code == 200, inline.text+ assert remote.json()["input_tokens"] == inline.json()["input_tokens"]++ def test_anthropic_count_tokens_refuses_the_schemes_generation_refuses(+ self, monkeypatch, canary+ ):+ backend = _VisionGguf()+ r = _client(monkeypatch, backend).post(+ "/v1/messages/count_tokens",+ json = _messages_body(f"http://127.0.0.1:{canary.port}/counted.png"),+ )++ assert r.status_code == 400, r.text+ assert backend.counted == []+ assert canary.hits == []diff --git a/studio/backend/tests/test_web_fetch_extraction.py b/studio/backend/tests/test_web_fetch_extraction.pyindex 16e6c45d8ac..38ef6c7492d 100644--- a/studio/backend/tests/test_web_fetch_extraction.py+++ b/studio/backend/tests/test_web_fetch_extraction.py@@ -1612,6 +1612,47 @@ def test_fetch_url_raw_deadline_aborts_slow_body(monkeypatch):assert body == ""+def test_read_capped_body_deadline_bounds_a_real_socket_drip():+ # A buffered read(n) keeps receiving until n bytes arrive, so only a real socket shows+ # whether a server sending one byte at a time can outlast the deadline.+ import socket+ import threading+ import urllib.request++ from core.inference.tools import _read_capped_body++ server = socket.create_server(("127.0.0.1", 0))+ stop = threading.Event()++ def _drip():+ conn, _ = server.accept()+ with conn:+ conn.recv(4096)+ conn.sendall(b"HTTP/1.1 200 OK\r\nContent-Length: 1000000\r\n\r\n")+ try:+ while not stop.is_set():+ conn.sendall(b"x")+ time.sleep(0.05)+ except OSError:+ pass++ threading.Thread(target = _drip, daemon = True).start()+ opener = urllib.request.build_opener(urllib.request.ProxyHandler({}))+ resp = opener.open(f"http://127.0.0.1:{server.getsockname()[1]}/", timeout = 15)+ started = time.monotonic()+ try:+ err, _body = _read_capped_body(resp, 1_000_000, 15, started + 0.5, None)+ except TimeoutError:+ err = "timed out"+ finally:+ stop.set()+ resp.close()+ server.close()++ assert err is not None+ assert time.monotonic() - started < 5++def test_resolve_with_budget_aborts_on_slow_resolver(monkeypatch):# getaddrinfo has no deadline of its own; a resolver slower than the budget must# abort on time instead of blocking the whole fetch.