Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

Studio: keep image follow-ups running after the tab closes (#16491)

Nilay70f11ee6e938
diff --git a/studio/backend/core/inference/chat_generation_runs.py b/studio/backend/core/inference/chat_generation_runs.py
index f84c968f1cc..fb7c9136bd1 100644
--- a/studio/backend/core/inference/chat_generation_runs.py
+++ b/studio/backend/core/inference/chat_generation_runs.py
@@ -7,12 +7,13 @@ from __future__ import annotations
import asyncio
import contextlib
+import hashlib
import json
import math
import os
import threading
import time
-from typing import Any, AsyncIterator
+from typing import Any, AsyncIterator, Iterator
from starlette.requests import Request
@@ -423,9 +424,35 @@ def _renew_interval_seconds() -> float:
return min(30.0, max(0.25, lease / 4.0))
+def _image_urls(request_payload: dict[str, Any]) -> Iterator[dict[str, Any]]:
+ for message in request_payload.get("messages", []):
+ content = message.get("content")
+ if isinstance(content, list):
+ for part in content:
+ if part.get("type") == "image_url" and isinstance(part.get("image_url"), dict):
+ yield part["image_url"]
+
+
+def detach_embedded_images(request_payload: dict[str, Any]) -> dict[str, str]:
+ images: dict[str, str] = {}
+ for image_url in _image_urls(request_payload):
+ url = image_url.get("url")
+ if isinstance(url, str) and url.startswith("data:"):
+ digest = hashlib.sha256(url.encode("utf-8")).hexdigest()
+ images[digest] = url
+ image_url["url"] = digest
+ return images
+
+
+def _attach_embedded_images(request_payload: dict[str, Any], images: dict[str, str]) -> None:
+ for image_url in _image_urls(request_payload):
+ image_url["url"] = images.get(image_url.get("url"), image_url.get("url"))
+
+
class ChatGenerationSupervisor:
def __init__(self, app: Any) -> None:
self.app = app
+ self._detached_images: dict[str, dict[str, str]] = {}
self._tasks: dict[str, asyncio.Task] = {}
self._cancel_events: dict[str, threading.Event] = {}
self._active_registrations: dict[str, active_generations.ActiveGeneration] = {}
@@ -480,6 +507,7 @@ class ChatGenerationSupervisor:
*,
thread_id: str | None = None,
model: str | None = None,
+ images: dict[str, str] | None = None,
) -> None:
if (
self._stopping
@@ -487,6 +515,8 @@ class ChatGenerationSupervisor:
or not self._ensure_reservation(run_id, thread_id = thread_id, model = model)
):
return
+ if images:
+ self._detached_images[run_id] = images
cancel_event = self._cancel_events[run_id]
activity = self._activities[run_id]
try:
@@ -501,6 +531,7 @@ class ChatGenerationSupervisor:
task.add_done_callback(lambda completed, rid = run_id: self._task_done(rid, completed))
def _cleanup_registration(self, run_id: str) -> None:
+ self._detached_images.pop(run_id, None)
self._cancel_events.pop(run_id, None)
activity = self._activities.pop(run_id, None)
if activity is not None:
@@ -705,6 +736,7 @@ class ChatGenerationSupervisor:
request_payload = dict(run["requestPayload"])
timezone_headers = request_payload.pop(db.TIMEZONE_HEADERS_FIELD, None)
via_api_key = request_payload.pop(db.API_MONITOR_ORIGIN_FIELD, None)
+ _attach_embedded_images(request_payload, self._detached_images.pop(run_id, {}))
payload = ChatCompletionRequest.model_validate(request_payload)
# Prep runs before the first-token budget; one later touch cannot cover prep longer than the lease
response = await produce_openai_chat_completions(
diff --git a/studio/backend/routes/chat_generation_runs.py b/studio/backend/routes/chat_generation_runs.py
index 958d800bf45..24bb69f709a 100644
--- a/studio/backend/routes/chat_generation_runs.py
+++ b/studio/backend/routes/chat_generation_runs.py
@@ -23,6 +23,7 @@ from auth.authentication import get_current_subject
from auth import policy
from state import active_generations, run_subscribers, unattended_runs
from utils.account_context import current_account, current_account_id, run_as
+from core.inference.chat_generation_runs import detach_embedded_images
from core.inference.llama_keepwarm import inference_lifecycle_gate
from models.inference import ChatCompletionRequest
from storage import chat_generation_runs_db as db
@@ -204,12 +205,15 @@ def _sanitize_request(payload: CreateChatGenerationRun) -> dict[str, Any]:
status_code = 400,
detail = "Durable chat runs are available only for local inference",
)
- # A media turn persists verbatim, so refuse base64 blobs in fields or in earlier
- # messages[].content (image follow-ups no longer set top-level image_base64).
+ # This turn's media stays legacy; images of earlier turns are detached before the request persists.
+ latest_user = max(
+ (index for index, message in enumerate(request.messages) if message.role == "user"),
+ default = 0,
+ )
if (
any(raw.get(field) not in (None, "") for field in _MEDIA_FIELDS)
or _messages_have_input_audio(request.messages)
- or _messages_have_embedded_image(request.messages)
+ or _messages_have_embedded_image(request.messages[latest_user:])
or _request_has_video(request)
):
raise HTTPException(
@@ -355,6 +359,7 @@ async def create_chat_generation_run(
current_subject: str = Depends(get_current_subject),
):
sanitized = _sanitize_request(payload)
+ images = detach_embedded_images(sanitized)
from routes.inference import _request_used_api_key
sanitized[db.API_MONITOR_ORIGIN_FIELD] = _request_used_api_key(request)
@@ -387,6 +392,7 @@ async def create_chat_generation_run(
run["id"],
thread_id = run["threadId"],
model = run["requestPayload"].get("model"),
+ images = images,
)
if payload.stopWhenUnattended and run["status"] not in db.TERMINAL_STATUSES:
_watch_unattended(request, run["id"])
diff --git a/studio/backend/routes/inference.py b/studio/backend/routes/inference.py
index a1a2792f5df..7d531432038 100644
--- a/studio/backend/routes/inference.py
+++ b/studio/backend/routes/inference.py
@@ -27197,8 +27197,7 @@ def _messages_have_input_audio(messages) -> bool:
def _messages_have_embedded_image(messages) -> bool:
- """True only for inline base64 image data URLs; durable runs would persist them into every later
- turn."""
+ """True only for inline base64 image data URLs."""
for msg in messages:
content = getattr(msg, "content", None)
if not isinstance(content, list):
diff --git a/studio/backend/tests/test_chat_generation_runs.py b/studio/backend/tests/test_chat_generation_runs.py
index dcf029bce44..8f09c571e01 100644
--- a/studio/backend/tests/test_chat_generation_runs.py
+++ b/studio/backend/tests/test_chat_generation_runs.py
@@ -921,14 +921,48 @@ _INLINE_PNG_PART = {
}
-def test_a_text_only_follow_up_carrying_an_earlier_inline_image_stays_legacy():
- """The case turn-scoping opened. No top-level image_base64 is set on a text-only follow-up, so
- the field-shaped guard admitted it, and a durable run persists its request verbatim: the
- thread's earlier screenshot would be written into request_json again on every follow-up."""
- messages = [
+def _image_thread(*follow_up):
+ return [
{"role": "user", "content": [{"type": "text", "text": "what is this?"}, _INLINE_PNG_PART]},
{"role": "assistant", "content": "A red square."},
- {"role": "user", "content": [{"type": "text", "text": "and what colour was it again?"}]},
+ *follow_up,
+ ]
+
+
+def test_a_text_follow_up_to_an_inline_image_is_admitted_without_storing_the_blob():
+ from core.inference.chat_generation_runs import detach_embedded_images
+
+ messages = _image_thread(
+ {"role": "user", "content": [{"type": "text", "text": "and what colour was it again?"}]}
+ )
+ sanitized = _sanitize_request(_model(messages = messages))
+ images = detach_embedded_images(sanitized)
+ url = _INLINE_PNG_PART["image_url"]["url"]
+ assert url not in json.dumps(sanitized)
+ assert list(images.values()) == [url]
+
+
+def test_a_different_image_is_a_different_stored_request():
+ from core.inference.chat_generation_runs import detach_embedded_images
+
+ follow_up = {"role": "user", "content": "again"}
+ first = _sanitize_request(_model(messages = _image_thread(follow_up)))
+ other = _image_thread(follow_up)
+ other[0]["content"][1] = {
+ "type": "image_url",
+ "image_url": {"url": "data:image/png;base64,AAAA"},
+ }
+ second = _sanitize_request(_model(messages = other))
+ detach_embedded_images(first)
+ detach_embedded_images(second)
+ assert first != second
+
+
+def test_an_image_on_the_latest_turn_stays_legacy():
+ messages = [
+ {"role": "user", "content": "hello"},
+ {"role": "assistant", "content": "hi"},
+ {"role": "user", "content": [{"type": "text", "text": "what is this?"}, _INLINE_PNG_PART]},
]
with pytest.raises(Exception, match = "Media chat runs use the legacy streaming path"):
_sanitize_request(_model(messages = messages))
diff --git a/studio/backend/tests/test_chat_generation_supervisor.py b/studio/backend/tests/test_chat_generation_supervisor.py
index 39f241ccddb..b963c3a78e9 100644
--- a/studio/backend/tests/test_chat_generation_supervisor.py
+++ b/studio/backend/tests/test_chat_generation_supervisor.py
@@ -233,7 +233,7 @@ async def test_create_route_schedules_producer_on_request_loop(monkeypatch):
monkeypatch.setattr(run_routes.db, "create_run", create_run)
response = await run_routes.create_chat_generation_run(_create_payload(), request, "alice")
assert response["created"] is True
- assert started == [("run-1", {"thread_id": "thread-1", "model": "local"})]
+ assert started == [("run-1", {"thread_id": "thread-1", "model": "local", "images": {}})]
@pytest.mark.asyncio
@@ -366,6 +366,43 @@ async def test_producer_dates_the_prompt_in_the_browser_timezone(monkeypatch):
assert observed == [(["generation_run_id"], date(2026, 9, 25))]
+async def test_a_follow_up_reaches_the_model_with_an_image_the_run_never_stored(monkeypatch):
+ studio_db.upsert_chat_thread(
+ {"id": "thread-1", "title": "Chat", "modelType": "base", "modelId": "local", "createdAt": 1}
+ )
+ studio_db.upsert_chat_message(
+ {"id": "user-1", "threadId": "thread-1", "role": "user", "content": [], "createdAt": 2}
+ )
+ url = "data:image/png;base64,iVBORw0KGgo="
+ image_turn = [
+ {"type": "text", "text": "what?"},
+ {"type": "image_url", "image_url": {"url": url}},
+ ]
+ payload = _create_payload("and the colour?")
+ payload.requestPayload["messages"] = [
+ {"role": "user", "content": image_turn},
+ {"role": "assistant", "content": "A square."},
+ *payload.requestPayload["messages"],
+ ]
+ observed = []
+
+ async def body():
+ yield "data: [DONE]\n\n"
+
+ async def fake(payload, _request, _subject, *, cancel_on_disconnect):
+ observed.append(payload.messages[0].content[1].image_url.url)
+ return SimpleNamespace(status_code = 200, body_iterator = body())
+
+ monkeypatch.setattr(inference, "produce_openai_chat_completions", fake)
+ supervisor = ChatGenerationSupervisor(SimpleNamespace(state = SimpleNamespace()))
+ await run_routes.create_chat_generation_run(payload, _route_request(supervisor), "alice")
+ await supervisor._tasks["run-1"]
+ assert observed == [url]
+ assert url not in json.dumps(runs_db.get_run("run-1")["requestPayload"])
+ assert supervisor._detached_images == {}
+
+
@pytest.mark.asyncio
async def test_a_create_retried_across_a_dst_change_returns_the_committed_run():
studio_db.upsert_chat_thread(