Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

Studio: send Gemini the image just attached in a chat full of images (#13636)

Nilayb6dc1903cb1c
diff --git a/studio/backend/core/inference/external_provider.py b/studio/backend/core/inference/external_provider.py
index ee6ca439400..6d34efb5751 100644
--- a/studio/backend/core/inference/external_provider.py
+++ b/studio/backend/core/inference/external_provider.py
@@ -4143,7 +4143,143 @@ class ExternalProviderClient:
_GEMINI_REMOTE_IMAGE_MAX_TOTAL_BYTES = 14 * 1024 * 1024
_remote_image_count = 0
_remote_image_total_bytes = 0
- for msg in messages:
+ _gemini_image_parts: dict[tuple[int, int], dict[str, Any]] = {}
+ for msg_idx in range(len(messages) - 1, -1, -1):
+ msg = messages[msg_idx]
+ content = msg.get("content")
+ if msg.get("role") in ("system", "tool") or not isinstance(content, list):
+ continue
+ for part_idx, part in enumerate(content):
+ if not isinstance(part, dict) or part.get("type") != "image_url":
+ continue
+ url = part.get("image_url", {}).get("url", "")
+ if url.startswith("data:"):
+ header, _, b64data = url.partition(",")
+ media_type = (
+ header.split(";")[0].replace("data:", "").strip().lower() or "image/jpeg"
+ )
+ # Reject non-image data URLs (e.g. data:text/html); they would 400 the request as
+ # inlineData. Mirrors the remote-fetch path's Content-Type check.
+ if not media_type.startswith("image/"):
+ logger.info(
+ "Gemini inlineData: refusing non-image data URL media_type=%s",
+ media_type,
+ )
+ elif b64data:
+ # data: URLs share the same caps as fetched URLs so inline payloads do not bypass
+ # them.
+ _data_approx_bytes = (len(b64data) * 3) // 4
+ if _remote_image_count >= _GEMINI_REMOTE_IMAGE_MAX_COUNT:
+ logger.info(
+ "Gemini inlineData: per-request count cap %d reached, dropping image",
+ _GEMINI_REMOTE_IMAGE_MAX_COUNT,
+ )
+ elif (
+ _remote_image_total_bytes + _data_approx_bytes
+ > _GEMINI_REMOTE_IMAGE_MAX_TOTAL_BYTES
+ ):
+ logger.info(
+ "Gemini inlineData: per-request byte cap reached, dropping image",
+ )
+ else:
+ _remote_image_count += 1
+ _remote_image_total_bytes += _data_approx_bytes
+ _gemini_image_parts[(msg_idx, part_idx)] = {
+ "inlineData": {
+ "mimeType": media_type,
+ "data": b64data,
+ }
+ }
+ elif url:
+ # fileData.fileUri only accepts Files-API URIs and YouTube; everything else is downloaded
+ # and inlined. Parse host explicitly so e.g. https://evil.com/youtube.com/x is not
+ # mis-detected.
+ try:
+ _parsed_image_url = urlparse(url)
+ except (ValueError, UnicodeError):
+ _parsed_image_url = None
+ if _parsed_image_url is None:
+ _img_scheme = ""
+ _img_host = ""
+ _img_path = ""
+ else:
+ _img_scheme = (_parsed_image_url.scheme or "").lower()
+ _img_host = (_parsed_image_url.hostname or "").lower()
+ _img_path = _parsed_image_url.path or ""
+ _is_native_uri = (
+ _img_scheme == "https"
+ and _img_host == "generativelanguage.googleapis.com"
+ and _img_path.startswith("/v1beta/files/")
+ )
+ _is_youtube = _img_scheme == "https" and (
+ _img_host == "youtu.be"
+ or _img_host == "youtube.com"
+ or _img_host.endswith(".youtube.com")
+ )
+ _guessed, _ = mimetypes.guess_type(_img_path)
+ _media_type = (
+ _guessed
+ if isinstance(_guessed, str) and _guessed.startswith("image/")
+ else "image/jpeg"
+ )
+ if _is_youtube:
+ # YouTube URIs must use video/mp4; the default image/jpeg yields a 400.
+ _gemini_image_parts[(msg_idx, part_idx)] = {
+ "fileData": {
+ "fileUri": url,
+ "mimeType": "video/mp4",
+ }
+ }
+ elif _is_native_uri:
+ _gemini_image_parts[(msg_idx, part_idx)] = {
+ "fileData": {
+ "fileUri": url,
+ "mimeType": _media_type,
+ }
+ }
+ elif _remote_image_count >= _GEMINI_REMOTE_IMAGE_MAX_COUNT:
+ logger.info(
+ "Gemini image fetch: per-request count cap %d reached, dropping image",
+ _GEMINI_REMOTE_IMAGE_MAX_COUNT,
+ )
+ else:
+ # Refuse pre-fetch when the per-request byte budget is spent; pass the remainder so
+ # over-budget URLs reject on Content-Length.
+ _remaining_bytes = (
+ _GEMINI_REMOTE_IMAGE_MAX_TOTAL_BYTES - _remote_image_total_bytes
+ )
+ if _remaining_bytes <= 0:
+ logger.info(
+ "Gemini image fetch: per-request byte cap already reached, dropping image",
+ )
+ else:
+ # Count attempts before awaiting so slow URLs do not each burn the timeout.
+ _remote_image_count += 1
+ _fetched = await _safe_fetch_image_for_gemini(
+ url,
+ _media_type,
+ max_bytes = _remaining_bytes,
+ )
+ if _fetched is not None:
+ _final_mime, _b64 = _fetched
+ # base64 expands ~4/3 — recover bytes from len(_b64).
+ _approx_bytes = (len(_b64) * 3) // 4
+ if (
+ _remote_image_total_bytes + _approx_bytes
+ > _GEMINI_REMOTE_IMAGE_MAX_TOTAL_BYTES
+ ):
+ logger.info(
+ "Gemini image fetch: per-request byte cap reached, dropping image",
+ )
+ else:
+ _remote_image_total_bytes += _approx_bytes
+ _gemini_image_parts[(msg_idx, part_idx)] = {
+ "inlineData": {
+ "mimeType": _final_mime,
+ "data": _b64,
+ }
+ }
+ for msg_idx, msg in enumerate(messages):
role = msg.get("role")
content = msg.get("content", "")
if role == "system":
@@ -4166,7 +4302,7 @@ class ExternalProviderClient:
if content:
parts.append({"text": content})
elif isinstance(content, list):
- for part in content:
+ for part_idx, part in enumerate(content):
if not isinstance(part, dict):
continue
ptype = part.get("type")
@@ -4175,142 +4311,9 @@ class ExternalProviderClient:
if text:
parts.append({"text": text})
elif ptype == "image_url":
- url = part.get("image_url", {}).get("url", "")
- if url.startswith("data:"):
- header, _, b64data = url.partition(",")
- media_type = (
- header.split(";")[0].replace("data:", "").strip().lower()
- or "image/jpeg"
- )
- # Reject non-image data URLs (e.g. data:text/html); they would 400 the request as
- # inlineData. Mirrors the remote-fetch path's Content-Type check.
- if not media_type.startswith("image/"):
- logger.info(
- "Gemini inlineData: refusing non-image data URL media_type=%s",
- media_type,
- )
- elif b64data:
- # data: URLs share the same caps as fetched URLs so inline payloads do not bypass
- # them.
- _data_approx_bytes = (len(b64data) * 3) // 4
- if _remote_image_count >= _GEMINI_REMOTE_IMAGE_MAX_COUNT:
- logger.info(
- "Gemini inlineData: per-request count cap %d reached, dropping image",
- _GEMINI_REMOTE_IMAGE_MAX_COUNT,
- )
- elif (
- _remote_image_total_bytes + _data_approx_bytes
- > _GEMINI_REMOTE_IMAGE_MAX_TOTAL_BYTES
- ):
- logger.info(
- "Gemini inlineData: per-request byte cap reached, dropping image",
- )
- else:
- _remote_image_count += 1
- _remote_image_total_bytes += _data_approx_bytes
- parts.append(
- {
- "inlineData": {
- "mimeType": media_type,
- "data": b64data,
- }
- }
- )
- elif url:
- # fileData.fileUri only accepts Files-API URIs and YouTube; everything else is downloaded
- # and inlined. Parse host explicitly so e.g. https://evil.com/youtube.com/x is not
- # mis-detected.
- try:
- _parsed_image_url = urlparse(url)
- except (ValueError, UnicodeError):
- _parsed_image_url = None
- if _parsed_image_url is None:
- _img_scheme = ""
- _img_host = ""
- _img_path = ""
- else:
- _img_scheme = (_parsed_image_url.scheme or "").lower()
- _img_host = (_parsed_image_url.hostname or "").lower()
- _img_path = _parsed_image_url.path or ""
- _is_native_uri = (
- _img_scheme == "https"
- and _img_host == "generativelanguage.googleapis.com"
- and _img_path.startswith("/v1beta/files/")
- )
- _is_youtube = _img_scheme == "https" and (
- _img_host == "youtu.be"
- or _img_host == "youtube.com"
- or _img_host.endswith(".youtube.com")
- )
- _guessed, _ = mimetypes.guess_type(_img_path)
- _media_type = (
- _guessed
- if isinstance(_guessed, str) and _guessed.startswith("image/")
- else "image/jpeg"
- )
- if _is_youtube:
- # YouTube URIs must use video/mp4; the default image/jpeg yields a 400.
- parts.append(
- {
- "fileData": {
- "fileUri": url,
- "mimeType": "video/mp4",
- }
- }
- )
- elif _is_native_uri:
- parts.append(
- {
- "fileData": {
- "fileUri": url,
- "mimeType": _media_type,
- }
- }
- )
- elif _remote_image_count >= _GEMINI_REMOTE_IMAGE_MAX_COUNT:
- logger.info(
- "Gemini image fetch: per-request count cap %d reached, dropping image",
- _GEMINI_REMOTE_IMAGE_MAX_COUNT,
- )
- else:
- # Refuse pre-fetch when the per-request byte budget is spent; pass the remainder so
- # over-budget URLs reject on Content-Length.
- _remaining_bytes = (
- _GEMINI_REMOTE_IMAGE_MAX_TOTAL_BYTES - _remote_image_total_bytes
- )
- if _remaining_bytes <= 0:
- logger.info(
- "Gemini image fetch: per-request byte cap already reached, dropping image",
- )
- else:
- # Count attempts before awaiting so slow URLs do not each burn the timeout.
- _remote_image_count += 1
- _fetched = await _safe_fetch_image_for_gemini(
- url,
- _media_type,
- max_bytes = _remaining_bytes,
- )
- if _fetched is not None:
- _final_mime, _b64 = _fetched
- # base64 expands ~4/3 — recover bytes from len(_b64).
- _approx_bytes = (len(_b64) * 3) // 4
- if (
- _remote_image_total_bytes + _approx_bytes
- > _GEMINI_REMOTE_IMAGE_MAX_TOTAL_BYTES
- ):
- logger.info(
- "Gemini image fetch: per-request byte cap reached, dropping image",
- )
- else:
- _remote_image_total_bytes += _approx_bytes
- parts.append(
- {
- "inlineData": {
- "mimeType": _final_mime,
- "data": _b64,
- }
- }
- )
+ _image_part = _gemini_image_parts.get((msg_idx, part_idx))
+ if _image_part is not None:
+ parts.append(_image_part)
# Gemini history must keep every native thought/answer boundary intact so a signature is never moved
# onto adjacent unsigned text. The frontend carries those exact parts because OpenAI reasoning_content
# is not otherwise replayed to native Gemini.
diff --git a/studio/backend/tests/test_gemini_provider.py b/studio/backend/tests/test_gemini_provider.py
index 425d1366086..caf66150f38 100644
--- a/studio/backend/tests/test_gemini_provider.py
+++ b/studio/backend/tests/test_gemini_provider.py
@@ -1207,6 +1207,43 @@ def test_vision_data_url_translates_to_inline_data(monkeypatch):
assert inline_parts[0]["inlineData"] == {"mimeType": "image/jpeg", "data": fake}
+def _image_turns(images: list[bytes]) -> list[dict]:
+ messages: list[dict] = []
+ for idx, image in enumerate(images):
+ data_url = f"data:image/jpeg;base64,{base64.b64encode(image).decode()}"
+ messages.append(
+ {
+ "role": "user",
+ "content": [
+ {"type": "text", "text": f"photo {idx + 1}"},
+ {"type": "image_url", "image_url": {"url": data_url}},
+ ],
+ }
+ )
+ messages.append({"role": "assistant", "content": f"reply {idx + 1}"})
+ return messages[:-1]
+
+
+def _turns_with_image(captured) -> list[bool]:
+ return [
+ any("inlineData" in p for p in c["parts"])
+ for c in captured["body"]["contents"]
+ if c["role"] == "user"
+ ]
+
+
+def test_newest_image_survives_the_per_request_image_count_cap(monkeypatch):
+ messages = _image_turns([f"JPG{idx}".encode() for idx in range(10)])
+ captured = _capture_body(monkeypatch, messages = messages)
+ assert _turns_with_image(captured) == [False, False] + [True] * 8
+
+
+def test_newest_image_survives_the_per_request_image_byte_cap(monkeypatch):
+ messages = _image_turns([bytes([idx]) * (5 * 1024 * 1024) for idx in range(3)])
+ captured = _capture_body(monkeypatch, messages = messages)
+ assert _turns_with_image(captured) == [False, True, True]
+
+
# ── finish reason mapping ────────────────────────────────────────────