Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

fetch remote image urls before forwarding to llama-server (#11160)

oobabooga8b3c658c032e
diff --git a/studio/backend/core/inference/external_provider.py b/studio/backend/core/inference/external_provider.py
index a6ba5c824c8..39e9fa9a08a 100644
--- a/studio/backend/core/inference/external_provider.py
+++ b/studio/backend/core/inference/external_provider.py
@@ -9,6 +9,7 @@ import base64
import io
import json as _json
import mimetypes
+import random
import re
import threading
import time
@@ -871,61 +872,84 @@ def _client() -> httpx.AsyncClient:
# Cap per-image fetch well below Gemini's ~20 MB total request budget.
_GEMINI_REMOTE_IMAGE_MAX_BYTES = 10 * 1024 * 1024
_GEMINI_REMOTE_IMAGE_TIMEOUT_S = 15.0
+# The socket timeout bounds one operation; a server dripping bytes needs a whole-fetch bound.
+_REMOTE_IMAGE_FETCH_DEADLINE_S = 30.0
-def _safe_fetch_image_for_gemini_sync(
+def safe_fetch_remote_image_sync(
url: str,
fallback_mime: str,
max_bytes: int = _GEMINI_REMOTE_IMAGE_MAX_BYTES,
+ label: str = "Remote image fetch",
+ deadline: Optional[float] = None,
+ require_image_content_type: bool = True,
) -> Optional[tuple[str, str]]:
- """Synchronous IP-pinned HTTPS image fetch with SSRF guards. Uses the same pinned-IP + SNI
- pattern as `tools._fetch_page_text` so DNS rebinding between validation and the connection
- cannot redirect us to a private/metadata address. Follows up to 4 hops, re-validating each
- redirect target. Returns (mime, base64) or None. `max_bytes` is clamped to the per-image cap
- and also lets the caller pass the remaining per-request budget, so an over-budget URL is
- rejected via Content-Length instead of being fully downloaded then discarded."""
+ """Fetch an HTTPS image through a validated, pinned public IP.
+
+ Redirects are revalidated and the response is capped by ``max_bytes``. ``deadline`` is a
+ ``time.monotonic`` cutoff for the whole fetch. A caller that decodes the bytes itself can
+ pass ``require_image_content_type=False`` to accept any declared type as ``fallback_mime``.
+ All failures return ``None`` so callers do not expose details about the host network.
+ """
+ import http.client
import urllib.error
import urllib.request
- from urllib.parse import urljoin, urlunparse
+ from urllib.parse import quote, urljoin, urlunparse
- # Refuse upfront if the per-request budget is already spent.
_byte_limit = min(max(0, int(max_bytes)), _GEMINI_REMOTE_IMAGE_MAX_BYTES)
if _byte_limit <= 0:
return None
# Reuse tools.py's pinned-IP hardening: validate-once-then-pin.
from .tools import (
+ _IRI_PATH_SAFE,
+ _IRI_QUERY_SAFE,
_explicit_proxy_applies,
+ _fetch_budget_exceeded,
+ _fetch_hop_timeout,
_NoRedirect,
_pinned_netloc,
+ _read_capped_body,
+ _resolve_with_budget,
_SNIHTTPSHandler,
- _validate_and_resolve_host,
+ _USER_AGENTS,
)
+ # Image hosts refuse a request with no User-Agent (Wikimedia answers 403), and until this
+ # fetch moved here llama-server sent its own. Picked once, as _fetch_url_raw does.
+ user_agent = random.choice(_USER_AGENTS)
+ if deadline is None:
+ deadline = time.monotonic() + _REMOTE_IMAGE_FETCH_DEADLINE_S
+
def _safe_parse_https(raw_url: str) -> Optional[tuple[Any, str, int]]:
- """Validate https + hostname + port. Returns (parsed, host, port) or None. Handles
- malformed-port and malformed-bracketed-IPv6 URLs that would else raise ValueError
- mid-build."""
+ """Return a parsed HTTPS URL, hostname and port, or ``None``."""
try:
parsed_url = urlparse(raw_url)
host_value = parsed_url.hostname
port_value = parsed_url.port or 443
except (ValueError, UnicodeError) as _err:
logger.info(
- "Gemini image fetch: refusing malformed url err=%s",
+ f"{label}: refusing malformed url err=%s",
type(_err).__name__,
)
return None
scheme_value = (parsed_url.scheme or "").lower()
if scheme_value != "https":
logger.info(
- "Gemini image fetch: refusing non-https scheme=%s",
+ f"{label}: refusing non-https scheme=%s",
scheme_value,
)
return None
if not host_value:
- logger.info("Gemini image fetch: refusing url with no hostname")
+ logger.info(f"{label}: refusing url with no hostname")
return None
+ if not host_value.isascii():
+ # http.client writes Host as ASCII, so an internationalized name travels as its A-label.
+ try:
+ host_value = host_value.encode("idna").decode("ascii")
+ except UnicodeError:
+ logger.info(f"{label}: refusing unencodable hostname")
+ return None
return parsed_url, host_value, port_value
parsed_info = _safe_parse_https(url)
@@ -933,42 +957,59 @@ def _safe_fetch_image_for_gemini_sync(
return None
parsed, current_host, current_port = parsed_info
current_url = url
- ok, reason, pinned_ips = _validate_and_resolve_host(current_host, current_port)
+ ok, reason, pinned_ips = _resolve_with_budget(current_host, current_port, deadline, None)
if not ok:
logger.warning(
- "Gemini image fetch: refusing host=%s reason=%s",
+ f"{label}: refusing host=%s reason=%s",
current_host,
reason,
)
return None
for _hop in range(4):
+ budget_error = _fetch_budget_exceeded(deadline, None)
+ if budget_error is not None:
+ logger.info(f"{label}: {budget_error} host=%s", current_host)
+ return None
# Pin to validated IP; SNI + cert still use the hostname via _SNIHTTPSHandler.
cp_info = _safe_parse_https(current_url)
if cp_info is None:
return None
cp, _cp_host, _cp_port = cp_info
+ # http.client refuses a non-ASCII or spaced selector; encode it as _fetch_url_raw does.
+ try:
+ cp = cp._replace(
+ path = quote(cp.path, safe = _IRI_PATH_SAFE),
+ params = quote(cp.params, safe = _IRI_PATH_SAFE),
+ query = quote(cp.query, safe = _IRI_QUERY_SAFE),
+ )
+ except UnicodeError:
+ logger.info(f"{label}: refusing unencodable url host=%s", current_host)
+ return None
pinned_url = urlunparse(cp._replace(netloc = _pinned_netloc(pinned_ips[0], cp.port)))
# Route on the hostname, as _fetch_url_raw does: no NO_PROXY entry matches the
# pinned URL's IP. A proxied request reaches the origin through the proxy.
- proxied = _explicit_proxy_applies("https", _pinned_netloc(current_host, cp.port))
+ authority = _pinned_netloc(current_host, cp.port)
+ proxied = _explicit_proxy_applies("https", authority)
handlers = [_NoRedirect, _SNIHTTPSHandler(current_host, () if proxied else pinned_ips)]
if not proxied:
handlers.append(urllib.request.ProxyHandler({}))
opener = urllib.request.build_opener(*handlers)
req = urllib.request.Request(
pinned_url,
- headers = {"Host": current_host},
+ headers = {"Host": authority, "User-Agent": user_agent},
method = "GET",
)
try:
- resp = opener.open(req, timeout = _GEMINI_REMOTE_IMAGE_TIMEOUT_S)
+ resp = opener.open(
+ req, timeout = _fetch_hop_timeout(_GEMINI_REMOTE_IMAGE_TIMEOUT_S, deadline)
+ )
except urllib.error.HTTPError as e:
if e.code not in (301, 302, 303, 307, 308):
logger.info(
- "Gemini image fetch: status=%d host=%s",
+ f"{label}: status=%d host=%s",
e.code,
current_host,
)
@@ -980,7 +1021,7 @@ def _safe_fetch_image_for_gemini_sync(
current_url = urljoin(current_url, location)
except (ValueError, UnicodeError) as _err:
logger.info(
- "Gemini image fetch: refusing malformed redirect err=%s",
+ f"{label}: refusing malformed redirect err=%s",
type(_err).__name__,
)
return None
@@ -988,18 +1029,20 @@ def _safe_fetch_image_for_gemini_sync(
if rp_info is None:
return None
_rp, current_host, current_port = rp_info
- ok2, reason2, pinned_ips = _validate_and_resolve_host(current_host, current_port)
+ ok2, reason2, pinned_ips = _resolve_with_budget(
+ current_host, current_port, deadline, None
+ )
if not ok2:
logger.warning(
- "Gemini image fetch: refusing redirect host=%s reason=%s",
+ f"{label}: refusing redirect host=%s reason=%s",
current_host,
reason2,
)
return None
continue
- except (urllib.error.URLError, OSError) as _err:
+ except (urllib.error.URLError, OSError, http.client.HTTPException, ValueError) as _err:
logger.warning(
- "Gemini image fetch failed host=%s err=%s",
+ f"{label} failed host=%s err=%s",
current_host,
type(_err).__name__,
)
@@ -1008,13 +1051,15 @@ def _safe_fetch_image_for_gemini_sync(
with resp:
status = getattr(resp, "status", None) or resp.getcode()
if status != 200:
- logger.info("Gemini image fetch: status=%s host=%s", status, current_host)
+ logger.info(f"{label}: status=%s host=%s", status, current_host)
return None
_hdr_mime = (resp.headers.get("content-type") or "").split(";")[0].strip().lower()
- # Declared non-image MIME is refused; missing MIME uses the caller's.
+ # A declared non-image MIME is refused unless the caller decodes the bytes itself.
+ if _hdr_mime and not _hdr_mime.startswith("image/") and not require_image_content_type:
+ _hdr_mime = ""
if _hdr_mime and not _hdr_mime.startswith("image/"):
logger.info(
- "Gemini image fetch: non-image content-type=%s host=%s",
+ f"{label}: non-image content-type=%s host=%s",
_hdr_mime,
current_host,
)
@@ -1022,34 +1067,60 @@ def _safe_fetch_image_for_gemini_sync(
_final_mime_pre = _hdr_mime if _hdr_mime else fallback_mime
if not isinstance(_final_mime_pre, str) or not _final_mime_pre.startswith("image/"):
logger.info(
- "Gemini image fetch: missing content-type and no image fallback host=%s",
+ f"{label}: missing content-type and no image fallback host=%s",
current_host,
)
return None
_hdr_len = resp.headers.get("content-length")
if _hdr_len and _hdr_len.isdigit() and int(_hdr_len) > _byte_limit:
logger.info(
- "Gemini image fetch: declared %s bytes exceeds cap=%s host=%s",
+ f"{label}: declared %s bytes exceeds cap=%s host=%s",
_hdr_len,
_byte_limit,
current_host,
)
return None
# Read cap+1 to detect oversize without buffering unbounded data.
- raw = resp.read(_byte_limit + 1)
+ try:
+ body_error, raw = _read_capped_body(
+ resp, _byte_limit + 1, _GEMINI_REMOTE_IMAGE_TIMEOUT_S, deadline, None
+ )
+ except (OSError, http.client.HTTPException, ValueError) as _err:
+ logger.warning(
+ f"{label} failed host=%s err=%s",
+ current_host,
+ type(_err).__name__,
+ )
+ return None
+ if body_error is not None:
+ logger.info(f"{label}: {body_error} host=%s", current_host)
+ return None
if len(raw) > _byte_limit:
logger.info(
- "Gemini image fetch: streamed bytes exceed cap=%s host=%s",
+ f"{label}: streamed bytes exceed cap=%s host=%s",
_byte_limit,
current_host,
)
return None
return _final_mime_pre, base64.b64encode(raw).decode("ascii")
- logger.info("Gemini image fetch: too many redirects host=%s", current_host)
+ logger.info(f"{label}: too many redirects host=%s", current_host)
return None
+_GEMINI_IMAGE_FETCH_LABEL = "Gemini image fetch"
+
+
+def _safe_fetch_image_for_gemini_sync(
+ url: str,
+ fallback_mime: str,
+ max_bytes: int = _GEMINI_REMOTE_IMAGE_MAX_BYTES,
+) -> Optional[tuple[str, str]]:
+ return safe_fetch_remote_image_sync(
+ url, fallback_mime, max_bytes, label = _GEMINI_IMAGE_FETCH_LABEL
+ )
+
+
async def _safe_fetch_image_for_gemini(
url: str,
fallback_mime: str,
diff --git a/studio/backend/core/inference/tools.py b/studio/backend/core/inference/tools.py
index 993efc48c1b..76490168760 100644
--- a/studio/backend/core/inference/tools.py
+++ b/studio/backend/core/inference/tools.py
@@ -14011,6 +14011,9 @@ def _read_capped_body(resp, max_bytes, timeout, deadline, cancel_event):
# Best-effort handle on the underlying socket so its timeout tightens as the deadline nears; absent on test
# doubles, where the between-chunk budget check still bounds the read.
sock = getattr(getattr(getattr(resp, "fp", None), "raw", None), "_sock", None)
+ # A buffered read(n) keeps receiving until n bytes arrive, so a drip never reaches the
+ # budget check; read1 returns after one receive.
+ read = getattr(resp, "read1", None) or resp.read
chunks = []
remaining = max_bytes
while remaining > 0:
@@ -14026,7 +14029,7 @@ def _read_capped_body(resp, max_bytes, timeout, deadline, cancel_event):
sock.settimeout(_fetch_hop_timeout(timeout, deadline))
except Exception:
pass
- chunk = resp.read(min(65536, remaining))
+ chunk = read(min(65536, remaining))
if not chunk:
break
chunks.append(chunk)
diff --git a/studio/backend/routes/inference.py b/studio/backend/routes/inference.py
index eff38bb86f4..7b236f9c822 100644
--- a/studio/backend/routes/inference.py
+++ b/studio/backend/routes/inference.py
@@ -8247,6 +8247,7 @@ async def _preflight_image_for_switch(
" GGUF build of it, which accepts several."
),
)
+ # Non-GGUF backends only receive decoded base64 images.
if not target_is_gguf and image_preflight.get("remote") and image_preflight.get("b64") is None:
raise HTTPException(
status_code = 400,
@@ -21509,7 +21510,9 @@ def _extract_content_parts(
f"Ignoring image URL with no base64 payload: {url[:80]}..."
)
else:
- logger.warning(f"Remote image URLs not yet supported: {url[:80]}...")
+ logger.warning(
+ f"Dropping a remote image URL while flattening to text: {url[:80]}..."
+ )
# Latest message wins: send the image just attached, not the thread's first.
if message_image_b64 is not None:
latest_image_b64 = message_image_b64
@@ -24345,6 +24348,14 @@ async def produce_openai_chat_completions(
"Image provided but current GGUF model does not support vision.",
)
+ # Fetch before the passthrough takes an admission lease, so a slow image host holds
+ # no inference slot.
+ if _messages_have_remote_image(payload.messages):
+ try:
+ await asyncio.to_thread(_inline_request_remote_images, payload)
+ except HTTPException as exc:
+ raise _reject(exc.status_code, exc.detail)
+
cancel_event = _chat_cancel_event(request)
completion_id = f"chatcmpl-{uuid.uuid4().hex[:12]}"
# `stream` defaults to False on ChatCompletionRequest (OpenAI spec
@@ -32122,19 +32133,157 @@ def _image_bytes_to_png_b64(raw: bytes) -> str:
return base64.b64encode(buf.getvalue()).decode("ascii")
+# Match llama-server's per-image limit and add a per-request budget.
+_REMOTE_IMAGE_MAX_BYTES = 10 * 1024 * 1024
+_REMOTE_IMAGE_REQUEST_BUDGET_BYTES = 32 * 1024 * 1024
+# Bound DNS, connection and redirect work independently of response size.
+_REMOTE_IMAGE_MAX_COUNT = 8
+# Wall-clock budget shared by every remote image fetch in one request.
+_REMOTE_IMAGE_REQUEST_DEADLINE_S = 60.0
+
+# Do not reveal why a caller-selected host could not be fetched.
+_REMOTE_IMAGE_FETCH_REFUSAL = (
+ "Could not fetch the remote image URL. Send the image as a base64 data URL instead."
+)
+# Token counting renders a fixed media marker and charges a flat per-image allowance.
+_COUNT_IMAGE_PLACEHOLDER = (
+ "data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAIAAACQd1Pe"
+ "AAAADElEQVR4nGNgYGAAAAAEAAH2FzhVAAAAAElFTkSuQmCC"
+)
+
+
+# A bare base64 payload holds neither a colon nor any of these, so a URL cannot pass as one.
+_NOT_BASE64 = _re.compile(r"[^A-Za-z0-9+/=_\s-]")
+# urlsplit drops these before it reads a scheme, so skip them the same way.
+_URL_LEADING_BLANKS = "".join(map(chr, range(0x21)))
+# Wider than any real scheme, so the scan stays bounded without missing one.
+_MAX_IMAGE_SCHEME_CHARS = 64
+
+
+def _image_url_scheme(url: str) -> str:
+ """Return a short URL scheme, or ``""`` for llama-server's bare-base64 form.
+
+ Leading blanks are skipped and the window outruns any real scheme, so a scheme padded
+ or stretched out of view cannot smuggle a URL through as a base64 payload.
+ """
+ head = url.lstrip(_URL_LEADING_BLANKS)[:_MAX_IMAGE_SCHEME_CHARS]
+ if ":" not in head and not _NOT_BASE64.search(head):
+ return ""
+ return head.split(":", 1)[0][:_MAX_VIDEO_SCHEME_CHARS].lower()
+
+
+def _remote_image_scheme_rejection(scheme: str) -> Optional[tuple[int, str]]:
+ """Allow only the HTTPS scheme supported by the pinned fetcher."""
+ if scheme == "https":
+ return None
+ return (
+ 400,
+ f"Unsupported image URL scheme ('{scheme}:'). Send the image over https, "
+ "or as a base64 data URL.",
+ )
+
+
+def _placeholder_remote_images_for_count(openai_messages: list[dict]) -> None:
+ """Replace remote images before ``/apply-template`` can fetch them."""
+ for msg in openai_messages:
+ content = msg.get("content")
+ if not isinstance(content, list):
+ continue
+ for part in content:
+ if not isinstance(part, dict) or part.get("type") != "image_url":
+ continue
+ image_url = part.get("image_url") or {}
+ url = image_url.get("url", "")
+ if url.startswith("data:"):
+ continue
+ scheme = _image_url_scheme(url)
+ if not scheme:
+ continue # Bare base64 payload, not a URL. Nothing to dial.
+ rejection = _remote_image_scheme_rejection(scheme)
+ if rejection is not None:
+ raise HTTPException(status_code = rejection[0], detail = rejection[1])
+ image_url["url"] = _COUNT_IMAGE_PLACEHOLDER
+
+
+def _inline_remote_image_url(
+ url: str, scheme: str, budget_bytes: int, deadline: float
+) -> tuple[str, int]:
+ """Fetch an image safely and return its data URL and decoded size."""
+ from core.inference.external_provider import safe_fetch_remote_image_sync
+
+ rejection = _remote_image_scheme_rejection(scheme)
+ if rejection is not None:
+ raise HTTPException(status_code = rejection[0], detail = rejection[1])
+ # llama-server never read the content type, and the bytes are decoded and re-encoded here.
+ fetched = safe_fetch_remote_image_sync(
+ url,
+ "image/png",
+ max_bytes = min(_REMOTE_IMAGE_MAX_BYTES, budget_bytes),
+ label = "llama-server image fetch",
+ deadline = deadline,
+ require_image_content_type = False,
+ )
+ if fetched is None:
+ raise HTTPException(status_code = 400, detail = _REMOTE_IMAGE_FETCH_REFUSAL)
+ mime, b64 = fetched
+ # Convert the encoded length back to the request budget's byte unit.
+ return f"data:{mime};base64,{b64}", (len(b64) * 3) // 4
+
+
+class _RemoteImageFetches:
+ """One request's remote image budget: bytes, URL count and a shared deadline."""
+
+ def __init__(self):
+ self.remaining_bytes = _REMOTE_IMAGE_REQUEST_BUDGET_BYTES
+ self.remaining_fetches = _REMOTE_IMAGE_MAX_COUNT
+ self.deadline: Optional[float] = None
+
+ def inline(self, url: str) -> str:
+ """Return a remote image as a data URL; llama-server's bare base64 form is returned as is."""
+ scheme = _image_url_scheme(url)
+ if not scheme:
+ return url
+ # A zero max_bytes value means no limit to some fetch callers.
+ if self.remaining_bytes <= 0 or self.remaining_fetches <= 0:
+ raise HTTPException(
+ status_code = 400,
+ detail = (
+ f"Too many remote image URLs in one request (max {_REMOTE_IMAGE_MAX_COUNT})."
+ if self.remaining_fetches <= 0
+ else _REMOTE_IMAGE_FETCH_REFUSAL
+ ),
+ )
+ self.remaining_fetches -= 1
+ if self.deadline is None:
+ self.deadline = time.monotonic() + _REMOTE_IMAGE_REQUEST_DEADLINE_S
+ data_url, spent = _inline_remote_image_url(url, scheme, self.remaining_bytes, self.deadline)
+ self.remaining_bytes -= spent
+ return data_url
+
+
+def _inline_request_remote_images(payload) -> None:
+ """Replace a chat request's remote image URLs with fetched data URLs, in place."""
+ fetches = _RemoteImageFetches()
+ for message in payload.messages:
+ if not isinstance(message.content, list):
+ continue
+ for part in message.content:
+ if isinstance(part, ImageContentPart) and not part.image_url.url.startswith("data:"):
+ part.image_url.url = fetches.inline(part.image_url.url)
+
+
def _normalize_openai_image_parts_to_png(openai_messages: list[dict], on_image = None) -> bool:
"""Re-encode every base64-data-URL ``image_url`` part to PNG, in place.
- llama-server's stb_image only handles a few formats (JPEG/PNG/BMP/…), while
- macOS and most browsers paste WebP and Claude Code sends WebP. Remote
- (non-``data:``) URLs are forwarded as-is; llama-server will fetch (or fail)
- per its own support matrix.
+ Remote URLs are fetched here so llama-server receives bytes rather than a URL. All images
+ are converted to PNG for llama-server's limited image decoder.
``on_image`` runs once per image part before conversion, so a caller can
apply its own guard. Returns ``True`` when any image part was seen. Raises
- HTTPException(400) when an image cannot be decoded.
+ HTTPException(400) when an image cannot be decoded or fetched.
"""
has_image = False
+ fetches = _RemoteImageFetches()
for msg in openai_messages:
content = msg.get("content")
if not isinstance(content, list):
@@ -32150,7 +32299,11 @@ def _normalize_openai_image_parts_to_png(openai_messages: list[dict], on_image =
image_url = part.get("image_url") or {}
url = image_url.get("url", "")
if not url.startswith("data:"):
- continue
+ url = fetches.inline(url)
+ if not url.startswith("data:"):
+ # llama-server also accepts bare base64 payloads.
+ continue
+ image_url["url"] = url
try:
_, b64data = url.split(",", 1)
@@ -32824,6 +32977,8 @@ async def anthropic_count_tokens(
# matches the prompt the real request would build (otherwise empty-assistant
# sentinels / synthetic tool history inflate the count or hit the fallback).
openai_messages = _sanitize_anthropic_openai_messages(openai_messages, llama_backend)
+ # /apply-template fetches remote media even though token counting only needs its marker.
+ _placeholder_remote_images_for_count(openai_messages)
openai_tools = anthropic_tools_to_openai(payload.tools or []) or None
# Only the client-tool passthrough is forwarded verbatim, so reproduce /messages' own
# routing rather than "any tools": a Studio server-tool alias, or a template without
@@ -35629,8 +35784,9 @@ def _openai_messages_for_passthrough(payload, normalize_images: bool = True) ->
``role="tool"`` tool-result messages and assistant messages carrying
structured ``tool_calls``. Base64-data-URL images already in the list are
re-encoded to PNG exactly as ``_openai_messages_for_gguf_chat`` does, so
- turning tools on does not change which formats llama-server can decode;
- remote URLs are forwarded as-is. The vision guard lives in the callers,
+ turning tools on does not change which formats llama-server can decode, and
+ a remote URL is fetched here rather than by llama-server, for the same
+ reason and by the same helper. The vision guard lives in the callers,
which reject a non-vision model before the body is built.
``normalize_images=False`` is for callers that decode these bytes themselves: re-encoding
diff --git a/studio/backend/tests/test_anthropic_messages.py b/studio/backend/tests/test_anthropic_messages.py
index 1133b8d56f6..cbc53e29311 100644
--- a/studio/backend/tests/test_anthropic_messages.py
+++ b/studio/backend/tests/test_anthropic_messages.py
@@ -2604,7 +2604,14 @@ class TestNormalizeAnthropicOpenAIImages:
assert new_url.startswith("data:image/png;base64,")
assert new_url != original_url
- def test_remote_url_left_unchanged(self):
+ def test_remote_url_is_replaced_by_the_bytes_we_fetched(self, monkeypatch):
+ import core.inference.external_provider as ep
+
+ monkeypatch.setattr(
+ ep,
+ "safe_fetch_remote_image_sync",
+ lambda *_a, **_k: ("image/png", _jpeg_data_url().partition(",")[2]),
+ )
msgs = [
{
"role": "user",
@@ -2617,7 +2624,7 @@ class TestNormalizeAnthropicOpenAIImages:
}
]
_normalize_anthropic_openai_images(msgs, is_vision = True)
- assert msgs[0]["content"][0]["image_url"]["url"] == "https://x.example/y.png"
+ assert msgs[0]["content"][0]["image_url"]["url"].startswith("data:image/png;base64,")
def test_bad_base64_raises_400(self):
msgs = [
diff --git a/studio/backend/tests/test_gemini_provider.py b/studio/backend/tests/test_gemini_provider.py
index 16561dd2da7..c86e455f53d 100644
--- a/studio/backend/tests/test_gemini_provider.py
+++ b/studio/backend/tests/test_gemini_provider.py
@@ -31,6 +31,7 @@ and test_openai_image_generation.py.
import asyncio
import base64
+import io
import json
import httpx
@@ -2626,8 +2627,11 @@ def test_safe_fetch_image_pins_validated_ip_no_hostname_in_request(monkeypatch):
def __exit__(self, *a):
return False
- def read(self, _n = None):
- return b"PNG"
+ # One body followed by EOF, as a real response reads.
+ _body = io.BytesIO(b"PNG")
+
+ def read(self, n = -1):
+ return self._body.read(n)
class _StubOpener:
def open(
@@ -3142,8 +3146,11 @@ def test_safe_fetch_image_missing_content_type_uses_fallback(monkeypatch):
def __exit__(self, *a):
return False
- def read(self, _n = None):
- return b"PNG"
+ # One body followed by EOF, as a real response reads.
+ _body = io.BytesIO(b"PNG")
+
+ def read(self, n = -1):
+ return self._body.read(n)
class _StubOpener:
def open(
diff --git a/studio/backend/tests/test_openai_tool_passthrough.py b/studio/backend/tests/test_openai_tool_passthrough.py
index ea1daa5518f..9b209e56a27 100644
--- a/studio/backend/tests/test_openai_tool_passthrough.py
+++ b/studio/backend/tests/test_openai_tool_passthrough.py
@@ -10491,9 +10491,45 @@ class TestPassthroughImageNormalization:
px = Image.open(BytesIO(out)).getpixel((0, 0))
assert px == ((77, 77, 77) if mode == "L" else (10, 20, 30)), (mode, px)
- def test_remote_url_is_forwarded_unchanged(self):
+ def test_remote_url_is_fetched_here_and_never_forwarded(self, monkeypatch):
+ from PIL import Image
+ from io import BytesIO
+
+ buf = BytesIO()
+ Image.new("RGB", (2, 2), (4, 5, 6)).save(buf, format = "WEBP")
+ webp_b64 = base64.b64encode(buf.getvalue()).decode("ascii")
+
+ import core.inference.external_provider as ep
+
+ monkeypatch.setattr(
+ ep, "safe_fetch_remote_image_sync", lambda *a, **k: ("image/webp", webp_b64)
+ )
+
messages = _openai_messages_for_passthrough(self._req("https://x.example/a.webp"))
- assert messages[0]["content"][1]["image_url"]["url"] == "https://x.example/a.webp"
+
+ url = messages[0]["content"][1]["image_url"]["url"]
+ assert url.startswith("data:image/png;base64,")
+
+ def test_remote_url_refused_when_the_fetch_is(self, monkeypatch):
+ import core.inference.external_provider as ep
+
+ monkeypatch.setattr(ep, "safe_fetch_remote_image_sync", lambda *a, **k: None)
+
+ with pytest.raises(HTTPException) as exc:
+ _openai_messages_for_passthrough(self._req("https://x.example/a.webp"))
+ assert exc.value.status_code == 400
+
+ def test_non_https_remote_url_is_refused_without_a_fetch(self, monkeypatch):
+ import core.inference.external_provider as ep
+
+ def _never(*_a, **_k):
+ raise AssertionError("http:// must be refused before any request")
+
+ monkeypatch.setattr(ep, "safe_fetch_remote_image_sync", _never)
+
+ with pytest.raises(HTTPException) as exc:
+ _openai_messages_for_passthrough(self._req("http://169.254.169.254/latest/meta-data/"))
+ assert exc.value.status_code == 400
def test_local_template_caller_leaves_a_payloadless_data_url_alone(self):
# The safetensors/MLX client-tools path only gets here when the turn has no
diff --git a/studio/backend/tests/test_remote_image_url_fetch_behaviour.py b/studio/backend/tests/test_remote_image_url_fetch_behaviour.py
new file mode 100644
index 00000000000..57ede0dcca2
--- /dev/null
+++ b/studio/backend/tests/test_remote_image_url_fetch_behaviour.py
@@ -0,0 +1,768 @@
+# SPDX-License-Identifier: AGPL-3.0-only
+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
+
+"""Ensure remote image URLs are fetched safely or refused before llama-server sees them."""
+
+from __future__ import annotations
+
+import base64
+import copy
+import socket
+import threading
+import time
+import urllib.error
+from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer
+from io import BytesIO
+from types import SimpleNamespace
+
+import pytest
+
+pytest.importorskip("torch")
+
+from fastapi import FastAPI # noqa: E402
+from fastapi.testclient import TestClient # noqa: E402
+from PIL import Image # noqa: E402
+
+from auth.authentication import get_current_subject # noqa: E402
+import core.inference.external_provider as external_provider # noqa: E402
+import routes.inference as inference_route # noqa: E402
+
+from .llama_backend_double import FakeLlamaCppBackend # noqa: E402
+
+_METADATA = "http://169.254.169.254/latest/meta-data/"
+
+
+def _webp_b64() -> str:
+ buf = BytesIO()
+ Image.new("RGB", (2, 2), (7, 8, 9)).save(buf, format = "WEBP")
+ return base64.b64encode(buf.getvalue()).decode("ascii")
+
+
+def _data_url() -> str:
+ buf = BytesIO()
+ Image.new("RGB", (2, 2), (1, 2, 3)).save(buf, format = "PNG")
+ return "data:image/png;base64," + base64.b64encode(buf.getvalue()).decode("ascii")
+
+
+class _VisionGguf(FakeLlamaCppBackend):
+ is_vision = True
+
+ def __init__(self):
+ self.dispatched: list[dict] = []
+ self.counted: list[list[dict]] = []
+
+ def generate_chat_completion(self, **kwargs):
+ self.dispatched.append(kwargs)
+ yield "ok"
+ yield {"type": "metadata", "usage": {}, "timings": {}}
+
+ generate_chat_completion_with_tools = generate_chat_completion
+
+ def count_chat_tokens(self, messages, *_a, **_k):
+ self.counted.append(copy.deepcopy(messages))
+ return 100
+
+
+def _client(monkeypatch, backend):
+ async def _no_switch(*_a, **_k):
+ return None
+
+ monkeypatch.setattr(inference_route, "get_llama_cpp_backend", lambda: backend)
+ monkeypatch.setattr(inference_route, "_maybe_auto_switch_model", _no_switch)
+
+ app = FastAPI()
+ app.include_router(inference_route.router, prefix = "/v1")
+ app.dependency_overrides[get_current_subject] = lambda: "tester"
+ return TestClient(app, raise_server_exceptions = False)
+
+
+def _chat_body(*urls, model = "test/model.gguf"):
+ parts = [{"type": "image_url", "image_url": {"url": u}} for u in urls]
+ parts.append({"type": "text", "text": "what is this?"})
+ return {
+ "model": model,
+ "stream": False,
+ "messages": [{"role": "user", "content": parts}],
+ }
+
+
+def _messages_body(url):
+ return {
+ "model": "test/model.gguf",
+ "max_tokens": 16,
+ "stream": False,
+ "messages": [
+ {
+ "role": "user",
+ "content": [
+ {"type": "image", "source": {"type": "url", "url": url}},
+ {"type": "text", "text": "what is this?"},
+ ],
+ }
+ ],
+ }
+
+
+def _image_urls(dispatched_messages) -> list[str]:
+ return [
+ part["image_url"]["url"]
+ for msg in dispatched_messages
+ for part in (msg.get("content") if isinstance(msg.get("content"), list) else [])
+ if isinstance(part, dict) and part.get("type") == "image_url"
+ ]
+
+
+def canary():
+ """A loopback HTTP server that records every request it is asked to serve."""
+ hits: list[str] = []
+
+ class _Handler(BaseHTTPRequestHandler):
+ def do_GET(self): # noqa: N802
+ hits.append(self.path)
+ body = base64.b64decode(_webp_b64())
+ self.send_response(200)
+ self.send_header("Content-Type", "image/webp")
+ self.send_header("Content-Length", str(len(body)))
+ self.end_headers()
+ self.wfile.write(body)
+
+ def log_message(self, *_a):
+ pass
+
+ server = ThreadingHTTPServer(("127.0.0.1", 0), _Handler)
+ thread = threading.Thread(target = server.serve_forever, daemon = True)
+ thread.start()
+ try:
+ yield SimpleNamespace(port = server.server_address[1], hits = hits)
+ finally:
+ server.shutdown()
+ server.server_close()
+ thread.join(timeout = 5)
+
+
+class TestRemoteUrlNeverReachesLlamaServer:
+ def test_public_https_url_is_fetched_here_and_dispatched_as_bytes(self, monkeypatch):
+ monkeypatch.setattr(
+ external_provider,
+ "safe_fetch_remote_image_sync",
+ lambda *_a, **_k: ("image/webp", _webp_b64()),
+ )
+ backend = _VisionGguf()
+ r = _client(monkeypatch, backend).post(
+ "/v1/chat/completions", json = _chat_body("https://images.example/cat.webp")
+ )
+
+ assert r.status_code == 200, r.text
+ urls = _image_urls(backend.dispatched[-1]["messages"])
+ assert len(urls) == 1
+ assert urls[0].startswith("data:image/png;base64,")
+
+ def test_anthropic_url_source_is_fetched_here_too(self, monkeypatch):
+ monkeypatch.setattr(
+ external_provider,
+ "safe_fetch_remote_image_sync",
+ lambda *_a, **_k: ("image/webp", _webp_b64()),
+ )
+ backend = _VisionGguf()
+ r = _client(monkeypatch, backend).post(
+ "/v1/messages", json = _messages_body("https://images.example/cat.webp")
+ )
+
+ assert r.status_code == 200, r.text
+ urls = _image_urls(backend.dispatched[-1]["messages"])
+ assert urls and all(u.startswith("data:image/png;base64,") for u in urls)
+
+ def test_responses_input_image_lands_on_the_same_guard(self, monkeypatch):
+ backend = _VisionGguf()
+ r = _client(monkeypatch, backend).post(
+ "/v1/responses",
+ json = {
+ "model": "test/model.gguf",
+ "stream": False,
+ "input": [
+ {
+ "role": "user",
+ "content": [
+ {"type": "input_image", "image_url": _METADATA},
+ {"type": "input_text", "text": "what is this?"},
+ ],
+ }
+ ],
+ },
+ )
+
+ assert r.status_code == 400, r.text
+ assert backend.dispatched == []
+
+ def test_the_tool_passthrough_fetches_before_taking_an_admission_lease(self, monkeypatch):
+ from fastapi.responses import JSONResponse
+
+ monkeypatch.setattr(
+ external_provider,
+ "safe_fetch_remote_image_sync",
+ lambda *_a, **_k: ("image/webp", _webp_b64()),
+ )
+ monkeypatch.setattr(inference_route, "_takes_tool_passthrough", lambda *_a, **_k: True)
+ admitted_urls = []
+
+ async def _admitted(_backend, payload, *_a, **_k):
+ admitted_urls.extend(
+ part.image_url.url
+ for message in payload.messages
+ if isinstance(message.content, list)
+ for part in message.content
+ if getattr(part, "type", None) == "image_url"
+ )
+ return JSONResponse({"ok": True})
+
+ monkeypatch.setattr(inference_route, "_openai_passthrough_non_streaming", _admitted)
+ r = _client(monkeypatch, _VisionGguf()).post(
+ "/v1/chat/completions", json = _chat_body("https://images.example/cat.webp")
+ )
+
+ assert r.status_code == 200, r.text
+ assert len(admitted_urls) == 1
+ assert admitted_urls[0].startswith("data:image/webp;base64,")
+
+ def test_bare_base64_is_payload_and_still_passes_through(self, monkeypatch):
+ def _never(*_a, **_k):
+ raise AssertionError("payload must not be treated as a URL")
+
+ monkeypatch.setattr(external_provider, "safe_fetch_remote_image_sync", _never)
+ backend = _VisionGguf()
+ r = _client(monkeypatch, backend).post("/v1/chat/completions", json = _chat_body(_webp_b64()))
+
+ assert r.status_code == 200, r.text
+ assert _image_urls(backend.dispatched[-1]["messages"]) == [_webp_b64()]
+
+ def test_a_data_url_still_goes_through_untouched_by_the_fetch(self, monkeypatch):
+ def _never(*_a, **_k):
+ raise AssertionError("a data: url needs no fetch")
+
+ monkeypatch.setattr(external_provider, "safe_fetch_remote_image_sync", _never)
+ backend = _VisionGguf()
+ r = _client(monkeypatch, backend).post("/v1/chat/completions", json = _chat_body(_data_url()))
+
+ assert r.status_code == 200, r.text
+ assert _image_urls(backend.dispatched[-1]["messages"])[0].startswith("data:image/png")
+
+
+class TestRefusalsAreRealRefusals:
+ @pytest.mark.parametrize(
+ "url_for",
+ [
+ pytest.param(lambda p: f"http://127.0.0.1:{p}/x.png", id = "http-loopback"),
+ pytest.param(lambda p: f"https://127.0.0.1:{p}/x.png", id = "https-loopback"),
+ pytest.param(lambda p: f"https://localhost:{p}/x.png", id = "https-resolves-loopback"),
+ ],
+ )
+ def test_loopback_is_refused_and_never_contacted(self, monkeypatch, canary, url_for):
+ backend = _VisionGguf()
+ r = _client(monkeypatch, backend).post(
+ "/v1/chat/completions", json = _chat_body(url_for(canary.port))
+ )
+
+ assert r.status_code == 400, r.text
+ assert backend.dispatched == []
+ assert canary.hits == [], "the guard let a request through to the loopback server"
+
+ def test_the_metadata_endpoint_is_refused(self, monkeypatch):
+ backend = _VisionGguf()
+ r = _client(monkeypatch, backend).post("/v1/chat/completions", json = _chat_body(_METADATA))
+
+ assert r.status_code == 400, r.text
+ assert backend.dispatched == []
+
+ def test_a_failed_fetch_refuses_rather_than_dropping_the_image(self, monkeypatch):
+ monkeypatch.setattr(
+ external_provider, "safe_fetch_remote_image_sync", lambda *_a, **_k: None
+ )
+ backend = _VisionGguf()
+ r = _client(monkeypatch, backend).post(
+ "/v1/chat/completions", json = _chat_body("https://images.example/gone.png")
+ )
+
+ assert r.status_code == 400, r.text
+ assert backend.dispatched == []
+
+ def test_one_bad_url_refuses_the_whole_request(self, monkeypatch, canary):
+ monkeypatch.setattr(
+ external_provider,
+ "safe_fetch_remote_image_sync",
+ lambda *_a, **_k: ("image/webp", _webp_b64()),
+ )
+ backend = _VisionGguf()
+ r = _client(monkeypatch, backend).post(
+ "/v1/chat/completions",
+ json = _chat_body(
+ "https://images.example/ok.webp", f"http://127.0.0.1:{canary.port}/x.png"
+ ),
+ )
+
+ assert r.status_code == 400, r.text
+ assert backend.dispatched == []
+ assert canary.hits == []
+
+ @pytest.mark.parametrize("url", ["file:///etc/passwd", "ftp://x.example/a.png"])
+ def test_another_scheme_is_refused_by_name(self, monkeypatch, url):
+ def _never(*_a, **_k):
+ raise AssertionError("only https reaches the fetcher")
+
+ monkeypatch.setattr(external_provider, "safe_fetch_remote_image_sync", _never)
+ backend = _VisionGguf()
+ r = _client(monkeypatch, backend).post("/v1/chat/completions", json = _chat_body(url))
+
+ assert r.status_code == 400, r.text
+ assert backend.dispatched == []
+
+ @pytest.mark.parametrize(
+ "url",
+ [
+ pytest.param("\n" * 13 + _METADATA, id = "blank-padded"),
+ pytest.param("\x00" * 20 + "http://127.0.0.1/x.png", id = "nul-padded"),
+ pytest.param("averyveryverylongscheme://x.example/a.png", id = "over-long-scheme"),
+ ],
+ )
+ def test_a_scheme_pushed_out_of_the_window_is_still_a_scheme(self, monkeypatch, url):
+ """urlsplit reads these as http; passing them off as base64 would forward a URL."""
+
+ def _never(*_a, **_k):
+ raise AssertionError("only https reaches the fetcher")
+
+ monkeypatch.setattr(external_provider, "safe_fetch_remote_image_sync", _never)
+ backend = _VisionGguf()
+ r = _client(monkeypatch, backend).post("/v1/chat/completions", json = _chat_body(url))
+
+ assert r.status_code == 400, r.text
+ assert backend.dispatched == []
+
+ @pytest.mark.parametrize(
+ "payload",
+ ["iVBORw0KGgoAAAANSUhEUgAAAAEAAAAB", "iVBORw0-KGgo_AAAANSUhEUg==", "iVBORw0KGgo\nAAAANS"],
+ )
+ def test_base64_shapes_are_still_read_as_payload(self, payload):
+ assert inference_route._image_url_scheme(payload) == ""
+
+ def test_the_vision_guard_still_runs_before_any_fetch(self, monkeypatch):
+ def _never(*_a, **_k):
+ raise AssertionError("a text-only model must refuse before fetching")
+
+ monkeypatch.setattr(external_provider, "safe_fetch_remote_image_sync", _never)
+
+ class _TextGguf(_VisionGguf):
+ is_vision = False
+
+ r = _client(monkeypatch, _TextGguf()).post(
+ "/v1/chat/completions", json = _chat_body("https://images.example/cat.webp")
+ )
+ assert r.status_code == 400, r.text
+
+
+class TestBudget:
+ def test_the_per_request_byte_budget_bounds_a_long_thread(self, monkeypatch):
+ b64 = _webp_b64()
+ each = (len(b64) * 3) // 4
+ monkeypatch.setattr(inference_route, "_REMOTE_IMAGE_REQUEST_BUDGET_BYTES", 2 * each + 1)
+ calls = []
+
+ def _fetch(
+ url,
+ _mime,
+ max_bytes = 0,
+ **_k,
+ ):
+ calls.append((url, max_bytes))
+ return ("image/webp", b64) if each <= max_bytes else None
+
+ monkeypatch.setattr(external_provider, "safe_fetch_remote_image_sync", _fetch)
+ backend = _VisionGguf()
+ r = _client(monkeypatch, backend).post(
+ "/v1/chat/completions",
+ json = _chat_body(*[f"https://images.example/{i}.webp" for i in range(4)]),
+ )
+
+ assert r.status_code == 400, r.text
+ assert [c[1] for c in calls] == [2 * each + 1, each + 1, 1], calls
+ assert backend.dispatched == []
+
+ def test_the_per_request_count_cap_bounds_the_round_trips(self, monkeypatch):
+ calls = []
+
+ def _fetch(url, *_a, **_k):
+ calls.append(url)
+ return "image/webp", _webp_b64()
+
+ monkeypatch.setattr(external_provider, "safe_fetch_remote_image_sync", _fetch)
+ backend = _VisionGguf()
+ over = inference_route._REMOTE_IMAGE_MAX_COUNT + 1
+ r = _client(monkeypatch, backend).post(
+ "/v1/chat/completions",
+ json = _chat_body(*[f"https://images.example/{i}.webp" for i in range(over)]),
+ )
+
+ assert r.status_code == 400, r.text
+ assert "Too many remote image URLs" in r.text
+ assert len(calls) == inference_route._REMOTE_IMAGE_MAX_COUNT, calls
+ assert backend.dispatched == []
+
+ def test_every_fetch_in_a_request_shares_one_deadline(self, monkeypatch):
+ deadlines = []
+
+ def _fetch(
+ url,
+ *_a,
+ deadline = None,
+ **_k,
+ ):
+ deadlines.append(deadline)
+ return "image/webp", _webp_b64()
+
+ monkeypatch.setattr(external_provider, "safe_fetch_remote_image_sync", _fetch)
+ backend = _VisionGguf()
+ r = _client(monkeypatch, backend).post(
+ "/v1/chat/completions",
+ json = _chat_body("https://images.example/a.webp", "https://images.example/b.webp"),
+ )
+
+ assert r.status_code == 200, r.text
+ assert len(deadlines) == 2 and deadlines[0] is not None, deadlines
+ assert deadlines[0] == deadlines[1]
+
+
+def _resolve_publicly(monkeypatch):
+ real = socket.getaddrinfo
+
+ def _fake(host, *args, **kwargs):
+ if host in ("images.example", "xn--bcher-kva.example"):
+ return [(socket.AF_INET, socket.SOCK_STREAM, 0, "", ("93.184.216.34", 0))]
+ if host == "2606:4700::1111":
+ return [(socket.AF_INET6, socket.SOCK_STREAM, 0, "", (host, 0, 0, 0))]
+ return real(host, *args, **kwargs)
+
+ monkeypatch.setattr(socket, "getaddrinfo", _fake)
+
+
+class TestFetcher:
+ def test_a_server_dripping_bytes_is_cut_off_at_the_deadline(self, monkeypatch):
+ _resolve_publicly(monkeypatch)
+
+ class _DripResponse:
+ status = 200
+ headers = {"content-type": "image/png"}
+
+ def __enter__(self):
+ return self
+
+ def __exit__(self, *_a):
+ return False
+
+ def close(self):
+ pass
+
+ def read(self, _n = -1):
+ time.sleep(0.02)
+ return b"x"
+
+ class _Opener:
+ def open(
+ self,
+ _req,
+ timeout = None,
+ ):
+ return _DripResponse()
+
+ monkeypatch.setattr("urllib.request.build_opener", lambda *_a, **_k: _Opener())
+ started = time.monotonic()
+ result = external_provider.safe_fetch_remote_image_sync(
+ "https://images.example/slow.png",
+ "image/png",
+ deadline = started + 0.3,
+ )
+
+ assert result is None
+ assert time.monotonic() - started < 5
+
+ def test_a_body_read_timeout_is_a_refusal_not_an_error(self, monkeypatch):
+ _resolve_publicly(monkeypatch)
+
+ class _StalledResponse:
+ status = 200
+ headers = {"content-type": "image/png"}
+
+ def __enter__(self):
+ return self
+
+ def __exit__(self, *_a):
+ return False
+
+ def read(self, _n = -1):
+ raise TimeoutError("timed out")
+
+ class _Opener:
+ def open(
+ self,
+ _req,
+ timeout = None,
+ ):
+ return _StalledResponse()
+
+ monkeypatch.setattr("urllib.request.build_opener", lambda *_a, **_k: _Opener())
+ assert (
+ external_provider.safe_fetch_remote_image_sync(
+ "https://images.example/stalled.png", "image/png"
+ )
+ is None
+ )
+
+ def test_a_non_ascii_or_spaced_path_is_percent_encoded(self, monkeypatch):
+ _resolve_publicly(monkeypatch)
+ sent = []
+
+ class _Opener:
+ def open(
+ self,
+ req,
+ timeout = None,
+ ):
+ sent.append(req.full_url)
+ raise urllib.error.URLError("stop after the request is built")
+
+ monkeypatch.setattr("urllib.request.build_opener", lambda *_a, **_k: _Opener())
+ external_provider.safe_fetch_remote_image_sync(
+ "https://images.example/caf\u00e9 1.png?q=\u00e9", "image/png"
+ )
+
+ assert sent == ["https://93.184.216.34/caf%C3%A9%201.png?q=%C3%A9"]
+
+ @pytest.mark.parametrize("require,expected", [(True, None), (False, "image/png")])
+ def test_a_generic_content_type_is_left_to_callers_that_decode(
+ self, monkeypatch, require, expected
+ ):
+ _resolve_publicly(monkeypatch)
+ body = base64.b64decode(_webp_b64())
+
+ class _Response:
+ status = 200
+ headers = {"content-type": "application/octet-stream"}
+ _body = BytesIO(body)
+
+ def __enter__(self):
+ return self
+
+ def __exit__(self, *_a):
+ return False
+
+ def read(self, n = -1):
+ return self._body.read(n)
+
+ class _Opener:
+ def open(
+ self,
+ _req,
+ timeout = None,
+ ):
+ return _Response()
+
+ monkeypatch.setattr("urllib.request.build_opener", lambda *_a, **_k: _Opener())
+ result = external_provider.safe_fetch_remote_image_sync(
+ "https://images.example/object",
+ "image/png",
+ require_image_content_type = require,
+ )
+
+ assert (result and result[0]) == expected
+
+ def test_the_llama_server_path_decodes_an_octet_stream_image(self, monkeypatch):
+ seen = {}
+
+ def _fetch(url, fallback_mime, **kwargs):
+ seen.update(kwargs)
+ return (
+ None
+ if kwargs.get("require_image_content_type", True)
+ else (fallback_mime, _webp_b64())
+ )
+
+ monkeypatch.setattr(external_provider, "safe_fetch_remote_image_sync", _fetch)
+ backend = _VisionGguf()
+ r = _client(monkeypatch, backend).post(
+ "/v1/chat/completions", json = _chat_body("https://images.example/object")
+ )
+
+ assert r.status_code == 200, r.text
+ assert seen["require_image_content_type"] is False
+ assert _image_urls(backend.dispatched[-1]["messages"])[0].startswith(
+ "data:image/png;base64,"
+ )
+
+ def test_an_unencodable_path_is_a_400(self, monkeypatch):
+ def _never(*_a, **_k):
+ raise AssertionError("an unencodable URL must not be dialled")
+
+ _resolve_publicly(monkeypatch)
+ monkeypatch.setattr("urllib.request.build_opener", _never)
+ backend = _VisionGguf()
+ r = _client(monkeypatch, backend).post(
+ "/v1/chat/completions",
+ content = (
+ '{"model": "test/model.gguf", "stream": false, "messages": [{"role": "user",'
+ ' "content": [{"type": "image_url", "image_url": {"url":'
+ ' "https://images.example/\\ud800.png"}}, {"type": "text", "text": "x"}]}]}'
+ ),
+ headers = {"Content-Type": "application/json"},
+ )
+
+ assert r.status_code == 400, r.text
+ assert backend.dispatched == []
+
+ def test_a_request_the_client_cannot_build_is_a_400(self, monkeypatch):
+ import http.client
+
+ _resolve_publicly(monkeypatch)
+
+ class _Opener:
+ def open(
+ self,
+ _req,
+ timeout = None,
+ ):
+ raise http.client.InvalidURL("URL can't contain control characters")
+
+ monkeypatch.setattr("urllib.request.build_opener", lambda *_a, **_k: _Opener())
+ backend = _VisionGguf()
+ r = _client(monkeypatch, backend).post(
+ "/v1/chat/completions", json = _chat_body("https://images.example/a.png")
+ )
+
+ assert r.status_code == 400, r.text
+ assert backend.dispatched == []
+
+ @pytest.mark.parametrize(
+ "url,authority",
+ [
+ ("https://images.example:8443/a.png", "images.example:8443"),
+ ("https://[2606:4700::1111]/a.png", "[2606:4700::1111]"),
+ ("https://b\u00fccher.example/a.png", "xn--bcher-kva.example"),
+ ],
+ )
+ def test_the_host_header_keeps_the_url_authority(self, monkeypatch, url, authority):
+ _resolve_publicly(monkeypatch)
+ sent = []
+
+ class _Opener:
+ def open(
+ self,
+ req,
+ timeout = None,
+ ):
+ sent.append(req.get_header("Host"))
+ raise urllib.error.URLError("stop after the request is built")
+
+ monkeypatch.setattr("urllib.request.build_opener", lambda *_a, **_k: _Opener())
+ assert external_provider.safe_fetch_remote_image_sync(url, "image/png") is None
+ assert sent == [authority]
+
+ def test_the_request_carries_a_user_agent(self, monkeypatch):
+ """Wikimedia and friends answer 403 without one, and llama-server used to send its own."""
+ _resolve_publicly(monkeypatch)
+ sent = []
+
+ class _Opener:
+ def open(
+ self,
+ req,
+ timeout = None,
+ ):
+ sent.append(req.get_header("User-agent"))
+ raise urllib.error.URLError("stop after the request is built")
+
+ monkeypatch.setattr("urllib.request.build_opener", lambda *_a, **_k: _Opener())
+ external_provider.safe_fetch_remote_image_sync("https://images.example/a.png", "image/png")
+
+ from core.inference.tools import _USER_AGENTS
+
+ assert sent and sent[0], "the fetch went out with no User-Agent"
+ assert sent[0] in _USER_AGENTS
+
+
+class TestCountingIsNotAFetchSurface:
+ def test_openai_count_tokens_still_refuses_an_image(self, monkeypatch):
+ def _never(*_a, **_k):
+ raise AssertionError("counting must not spend a request")
+
+ monkeypatch.setattr(external_provider, "safe_fetch_remote_image_sync", _never)
+ backend = _VisionGguf()
+ backend.supports_tools = True
+ r = _client(monkeypatch, backend).post(
+ "/v1/chat/count_tokens", json = _chat_body("https://images.example/cat.webp")
+ )
+
+ assert r.status_code == 503, r.text
+ assert "images" in r.text
+
+ def test_anthropic_count_tokens_prices_a_remote_image_without_dialling_it(
+ self, monkeypatch, canary
+ ):
+ def _never(*_a, **_k):
+ raise AssertionError("counting must not spend a request")
+
+ monkeypatch.setattr(external_provider, "safe_fetch_remote_image_sync", _never)
+ backend = _VisionGguf()
+ r = _client(monkeypatch, backend).post(
+ "/v1/messages/count_tokens",
+ json = _messages_body(f"https://127.0.0.1:{canary.port}/counted.png"),
+ )
+
+ assert r.status_code == 200, r.text
+ assert canary.hits == []
+ counted_urls = _image_urls(backend.counted[-1])
+ assert counted_urls and all(u.startswith("data:image/png;base64,") for u in counted_urls)
+
+ def test_the_counted_placeholder_prices_the_same_as_a_real_image(self, monkeypatch):
+ backend = _VisionGguf()
+ client = _client(monkeypatch, backend)
+ remote = client.post(
+ "/v1/messages/count_tokens",
+ json = _messages_body("https://images.example/cat.png"),
+ )
+ inline = client.post(
+ "/v1/messages/count_tokens",
+ json = {
+ "model": "test/model.gguf",
+ "max_tokens": 16,
+ "messages": [
+ {
+ "role": "user",
+ "content": [
+ {
+ "type": "image",
+ "source": {
+ "type": "base64",
+ "media_type": "image/png",
+ "data": _data_url().partition(",")[2],
+ },
+ },
+ {"type": "text", "text": "what is this?"},
+ ],
+ }
+ ],
+ },
+ )
+
+ assert remote.status_code == 200, remote.text
+ assert inline.status_code == 200, inline.text
+ assert remote.json()["input_tokens"] == inline.json()["input_tokens"]
+
+ def test_anthropic_count_tokens_refuses_the_schemes_generation_refuses(
+ self, monkeypatch, canary
+ ):
+ backend = _VisionGguf()
+ r = _client(monkeypatch, backend).post(
+ "/v1/messages/count_tokens",
+ json = _messages_body(f"http://127.0.0.1:{canary.port}/counted.png"),
+ )
+
+ assert r.status_code == 400, r.text
+ assert backend.counted == []
+ assert canary.hits == []
diff --git a/studio/backend/tests/test_web_fetch_extraction.py b/studio/backend/tests/test_web_fetch_extraction.py
index 16e6c45d8ac..38ef6c7492d 100644
--- a/studio/backend/tests/test_web_fetch_extraction.py
+++ b/studio/backend/tests/test_web_fetch_extraction.py
@@ -1612,6 +1612,47 @@ def test_fetch_url_raw_deadline_aborts_slow_body(monkeypatch):
assert body == ""
+def test_read_capped_body_deadline_bounds_a_real_socket_drip():
+ # A buffered read(n) keeps receiving until n bytes arrive, so only a real socket shows
+ # whether a server sending one byte at a time can outlast the deadline.
+ import socket
+ import threading
+ import urllib.request
+
+ from core.inference.tools import _read_capped_body
+
+ server = socket.create_server(("127.0.0.1", 0))
+ stop = threading.Event()
+
+ def _drip():
+ conn, _ = server.accept()
+ with conn:
+ conn.recv(4096)
+ conn.sendall(b"HTTP/1.1 200 OK\r\nContent-Length: 1000000\r\n\r\n")
+ try:
+ while not stop.is_set():
+ conn.sendall(b"x")
+ time.sleep(0.05)
+ except OSError:
+ pass
+
+ threading.Thread(target = _drip, daemon = True).start()
+ opener = urllib.request.build_opener(urllib.request.ProxyHandler({}))
+ resp = opener.open(f"http://127.0.0.1:{server.getsockname()[1]}/", timeout = 15)
+ started = time.monotonic()
+ try:
+ err, _body = _read_capped_body(resp, 1_000_000, 15, started + 0.5, None)
+ except TimeoutError:
+ err = "timed out"
+ finally:
+ stop.set()
+ resp.close()
+ server.close()
+
+ assert err is not None
+ assert time.monotonic() - started < 5
+
+
def test_resolve_with_budget_aborts_on_slow_resolver(monkeypatch):
# getaddrinfo has no deadline of its own; a resolver slower than the budget must
# abort on time instead of blocking the whole fetch.