Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
Studio: send Gemini the image just attached in a chat full of images (#13636)
Nilayb6dc1903cb1c
diff --git a/studio/backend/core/inference/external_provider.py b/studio/backend/core/inference/external_provider.pyindex ee6ca439400..6d34efb5751 100644--- a/studio/backend/core/inference/external_provider.py+++ b/studio/backend/core/inference/external_provider.py@@ -4143,7 +4143,143 @@ class ExternalProviderClient:_GEMINI_REMOTE_IMAGE_MAX_TOTAL_BYTES = 14 * 1024 * 1024_remote_image_count = 0_remote_image_total_bytes = 0- for msg in messages:+ _gemini_image_parts: dict[tuple[int, int], dict[str, Any]] = {}+ for msg_idx in range(len(messages) - 1, -1, -1):+ msg = messages[msg_idx]+ content = msg.get("content")+ if msg.get("role") in ("system", "tool") or not isinstance(content, list):+ continue+ for part_idx, part in enumerate(content):+ if not isinstance(part, dict) or part.get("type") != "image_url":+ continue+ url = part.get("image_url", {}).get("url", "")+ if url.startswith("data:"):+ header, _, b64data = url.partition(",")+ media_type = (+ header.split(";")[0].replace("data:", "").strip().lower() or "image/jpeg"+ )+ # Reject non-image data URLs (e.g. data:text/html); they would 400 the request as+ # inlineData. Mirrors the remote-fetch path's Content-Type check.+ if not media_type.startswith("image/"):+ logger.info(+ "Gemini inlineData: refusing non-image data URL media_type=%s",+ media_type,+ )+ elif b64data:+ # data: URLs share the same caps as fetched URLs so inline payloads do not bypass+ # them.+ _data_approx_bytes = (len(b64data) * 3) // 4+ if _remote_image_count >= _GEMINI_REMOTE_IMAGE_MAX_COUNT:+ logger.info(+ "Gemini inlineData: per-request count cap %d reached, dropping image",+ _GEMINI_REMOTE_IMAGE_MAX_COUNT,+ )+ elif (+ _remote_image_total_bytes + _data_approx_bytes+ > _GEMINI_REMOTE_IMAGE_MAX_TOTAL_BYTES+ ):+ logger.info(+ "Gemini inlineData: per-request byte cap reached, dropping image",+ )+ else:+ _remote_image_count += 1+ _remote_image_total_bytes += _data_approx_bytes+ _gemini_image_parts[(msg_idx, part_idx)] = {+ "inlineData": {+ "mimeType": media_type,+ "data": b64data,+ }+ }+ elif url:+ # fileData.fileUri only accepts Files-API URIs and YouTube; everything else is downloaded+ # and inlined. Parse host explicitly so e.g. https://evil.com/youtube.com/x is not+ # mis-detected.+ try:+ _parsed_image_url = urlparse(url)+ except (ValueError, UnicodeError):+ _parsed_image_url = None+ if _parsed_image_url is None:+ _img_scheme = ""+ _img_host = ""+ _img_path = ""+ else:+ _img_scheme = (_parsed_image_url.scheme or "").lower()+ _img_host = (_parsed_image_url.hostname or "").lower()+ _img_path = _parsed_image_url.path or ""+ _is_native_uri = (+ _img_scheme == "https"+ and _img_host == "generativelanguage.googleapis.com"+ and _img_path.startswith("/v1beta/files/")+ )+ _is_youtube = _img_scheme == "https" and (+ _img_host == "youtu.be"+ or _img_host == "youtube.com"+ or _img_host.endswith(".youtube.com")+ )+ _guessed, _ = mimetypes.guess_type(_img_path)+ _media_type = (+ _guessed+ if isinstance(_guessed, str) and _guessed.startswith("image/")+ else "image/jpeg"+ )+ if _is_youtube:+ # YouTube URIs must use video/mp4; the default image/jpeg yields a 400.+ _gemini_image_parts[(msg_idx, part_idx)] = {+ "fileData": {+ "fileUri": url,+ "mimeType": "video/mp4",+ }+ }+ elif _is_native_uri:+ _gemini_image_parts[(msg_idx, part_idx)] = {+ "fileData": {+ "fileUri": url,+ "mimeType": _media_type,+ }+ }+ elif _remote_image_count >= _GEMINI_REMOTE_IMAGE_MAX_COUNT:+ logger.info(+ "Gemini image fetch: per-request count cap %d reached, dropping image",+ _GEMINI_REMOTE_IMAGE_MAX_COUNT,+ )+ else:+ # Refuse pre-fetch when the per-request byte budget is spent; pass the remainder so+ # over-budget URLs reject on Content-Length.+ _remaining_bytes = (+ _GEMINI_REMOTE_IMAGE_MAX_TOTAL_BYTES - _remote_image_total_bytes+ )+ if _remaining_bytes <= 0:+ logger.info(+ "Gemini image fetch: per-request byte cap already reached, dropping image",+ )+ else:+ # Count attempts before awaiting so slow URLs do not each burn the timeout.+ _remote_image_count += 1+ _fetched = await _safe_fetch_image_for_gemini(+ url,+ _media_type,+ max_bytes = _remaining_bytes,+ )+ if _fetched is not None:+ _final_mime, _b64 = _fetched+ # base64 expands ~4/3 — recover bytes from len(_b64).+ _approx_bytes = (len(_b64) * 3) // 4+ if (+ _remote_image_total_bytes + _approx_bytes+ > _GEMINI_REMOTE_IMAGE_MAX_TOTAL_BYTES+ ):+ logger.info(+ "Gemini image fetch: per-request byte cap reached, dropping image",+ )+ else:+ _remote_image_total_bytes += _approx_bytes+ _gemini_image_parts[(msg_idx, part_idx)] = {+ "inlineData": {+ "mimeType": _final_mime,+ "data": _b64,+ }+ }+ for msg_idx, msg in enumerate(messages):role = msg.get("role")content = msg.get("content", "")if role == "system":@@ -4166,7 +4302,7 @@ class ExternalProviderClient:if content:parts.append({"text": content})elif isinstance(content, list):- for part in content:+ for part_idx, part in enumerate(content):if not isinstance(part, dict):continueptype = part.get("type")@@ -4175,142 +4311,9 @@ class ExternalProviderClient:if text:parts.append({"text": text})elif ptype == "image_url":- url = part.get("image_url", {}).get("url", "")- if url.startswith("data:"):- header, _, b64data = url.partition(",")- media_type = (- header.split(";")[0].replace("data:", "").strip().lower()- or "image/jpeg"- )- # Reject non-image data URLs (e.g. data:text/html); they would 400 the request as- # inlineData. Mirrors the remote-fetch path's Content-Type check.- if not media_type.startswith("image/"):- logger.info(- "Gemini inlineData: refusing non-image data URL media_type=%s",- media_type,- )- elif b64data:- # data: URLs share the same caps as fetched URLs so inline payloads do not bypass- # them.- _data_approx_bytes = (len(b64data) * 3) // 4- if _remote_image_count >= _GEMINI_REMOTE_IMAGE_MAX_COUNT:- logger.info(- "Gemini inlineData: per-request count cap %d reached, dropping image",- _GEMINI_REMOTE_IMAGE_MAX_COUNT,- )- elif (- _remote_image_total_bytes + _data_approx_bytes- > _GEMINI_REMOTE_IMAGE_MAX_TOTAL_BYTES- ):- logger.info(- "Gemini inlineData: per-request byte cap reached, dropping image",- )- else:- _remote_image_count += 1- _remote_image_total_bytes += _data_approx_bytes- parts.append(- {- "inlineData": {- "mimeType": media_type,- "data": b64data,- }- }- )- elif url:- # fileData.fileUri only accepts Files-API URIs and YouTube; everything else is downloaded- # and inlined. Parse host explicitly so e.g. https://evil.com/youtube.com/x is not- # mis-detected.- try:- _parsed_image_url = urlparse(url)- except (ValueError, UnicodeError):- _parsed_image_url = None- if _parsed_image_url is None:- _img_scheme = ""- _img_host = ""- _img_path = ""- else:- _img_scheme = (_parsed_image_url.scheme or "").lower()- _img_host = (_parsed_image_url.hostname or "").lower()- _img_path = _parsed_image_url.path or ""- _is_native_uri = (- _img_scheme == "https"- and _img_host == "generativelanguage.googleapis.com"- and _img_path.startswith("/v1beta/files/")- )- _is_youtube = _img_scheme == "https" and (- _img_host == "youtu.be"- or _img_host == "youtube.com"- or _img_host.endswith(".youtube.com")- )- _guessed, _ = mimetypes.guess_type(_img_path)- _media_type = (- _guessed- if isinstance(_guessed, str) and _guessed.startswith("image/")- else "image/jpeg"- )- if _is_youtube:- # YouTube URIs must use video/mp4; the default image/jpeg yields a 400.- parts.append(- {- "fileData": {- "fileUri": url,- "mimeType": "video/mp4",- }- }- )- elif _is_native_uri:- parts.append(- {- "fileData": {- "fileUri": url,- "mimeType": _media_type,- }- }- )- elif _remote_image_count >= _GEMINI_REMOTE_IMAGE_MAX_COUNT:- logger.info(- "Gemini image fetch: per-request count cap %d reached, dropping image",- _GEMINI_REMOTE_IMAGE_MAX_COUNT,- )- else:- # Refuse pre-fetch when the per-request byte budget is spent; pass the remainder so- # over-budget URLs reject on Content-Length.- _remaining_bytes = (- _GEMINI_REMOTE_IMAGE_MAX_TOTAL_BYTES - _remote_image_total_bytes- )- if _remaining_bytes <= 0:- logger.info(- "Gemini image fetch: per-request byte cap already reached, dropping image",- )- else:- # Count attempts before awaiting so slow URLs do not each burn the timeout.- _remote_image_count += 1- _fetched = await _safe_fetch_image_for_gemini(- url,- _media_type,- max_bytes = _remaining_bytes,- )- if _fetched is not None:- _final_mime, _b64 = _fetched- # base64 expands ~4/3 — recover bytes from len(_b64).- _approx_bytes = (len(_b64) * 3) // 4- if (- _remote_image_total_bytes + _approx_bytes- > _GEMINI_REMOTE_IMAGE_MAX_TOTAL_BYTES- ):- logger.info(- "Gemini image fetch: per-request byte cap reached, dropping image",- )- else:- _remote_image_total_bytes += _approx_bytes- parts.append(- {- "inlineData": {- "mimeType": _final_mime,- "data": _b64,- }- }- )+ _image_part = _gemini_image_parts.get((msg_idx, part_idx))+ if _image_part is not None:+ parts.append(_image_part)# Gemini history must keep every native thought/answer boundary intact so a signature is never moved# onto adjacent unsigned text. The frontend carries those exact parts because OpenAI reasoning_content# is not otherwise replayed to native Gemini.diff --git a/studio/backend/tests/test_gemini_provider.py b/studio/backend/tests/test_gemini_provider.pyindex 425d1366086..caf66150f38 100644--- a/studio/backend/tests/test_gemini_provider.py+++ b/studio/backend/tests/test_gemini_provider.py@@ -1207,6 +1207,43 @@ def test_vision_data_url_translates_to_inline_data(monkeypatch):assert inline_parts[0]["inlineData"] == {"mimeType": "image/jpeg", "data": fake}+def _image_turns(images: list[bytes]) -> list[dict]:+ messages: list[dict] = []+ for idx, image in enumerate(images):+ data_url = f"data:image/jpeg;base64,{base64.b64encode(image).decode()}"+ messages.append(+ {+ "role": "user",+ "content": [+ {"type": "text", "text": f"photo {idx + 1}"},+ {"type": "image_url", "image_url": {"url": data_url}},+ ],+ }+ )+ messages.append({"role": "assistant", "content": f"reply {idx + 1}"})+ return messages[:-1]+++def _turns_with_image(captured) -> list[bool]:+ return [+ any("inlineData" in p for p in c["parts"])+ for c in captured["body"]["contents"]+ if c["role"] == "user"+ ]+++def test_newest_image_survives_the_per_request_image_count_cap(monkeypatch):+ messages = _image_turns([f"JPG{idx}".encode() for idx in range(10)])+ captured = _capture_body(monkeypatch, messages = messages)+ assert _turns_with_image(captured) == [False, False] + [True] * 8+++def test_newest_image_survives_the_per_request_image_byte_cap(monkeypatch):+ messages = _image_turns([bytes([idx]) * (5 * 1024 * 1024) for idx in range(3)])+ captured = _capture_body(monkeypatch, messages = messages)+ assert _turns_with_image(captured) == [False, True, True]++# ── finish reason mapping ────────────────────────────────────────────