Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

Studio: MLX chats with a picture keep going when the context fills (#17424)

Nilay31d7eb47ea44
diff --git a/studio/backend/core/inference/mlx_inference.py b/studio/backend/core/inference/mlx_inference.py
index 2ef224d898a..0e066518e04 100644
--- a/studio/backend/core/inference/mlx_inference.py
+++ b/studio/backend/core/inference/mlx_inference.py
@@ -6949,24 +6949,30 @@ class MLXInferenceBackend:
enable_thinking = None,
reasoning_effort = None,
preserve_thinking = None,
+ images = None,
) -> int:
"""Prompt tokens this model would receive for these messages. Renders and tokenizes exactly
as generation does."""
if self._model is None:
raise RuntimeError("No model loaded")
- full_messages = self._with_system_prompt(messages, system_prompt)
+ images = list(images or ())
+ full_messages = self._conversation_to_render(messages, system_prompt, images or None)
+ if len(images) > 1 and not self._multi_image_marker:
+ # Generation serves the newest picture alone.
+ full_messages = list(full_messages)
+ trim_image_turns(full_messages, images, limit = 1, keep = (len(images) - 1,))
if self._is_vlm:
- # images=None: an image anywhere makes the structured-item check raise.
+ # One image per marker, else the structured-item check raises.
prompt, _target, _markers = self._render_vlm_prompt(
full_messages,
- None,
+ images or None,
tools = tools,
enable_thinking = enable_thinking,
reasoning_effort = reasoning_effort,
preserve_thinking = preserve_thinking,
)
- return self._count_prompt_tokens(prompt)
+ return self._count_prompt_tokens(prompt, images)
render_result = self._render_text_prompt(
full_messages,
@@ -6975,6 +6981,17 @@ class MLXInferenceBackend:
reasoning_effort = reasoning_effort,
preserve_thinking = preserve_thinking,
)
+ if images and self._reads_vision_input():
+ from unsloth_zoo.mlx.loader import _mlx_lm_image_inputs
+
+ # The ids generation embeds, without running the vision tower.
+ expanded = _mlx_lm_image_inputs(
+ self._tokenizer,
+ self._tokenizer._unsloth_image_processor,
+ images,
+ render_result.prompt,
+ )
+ return len(expanded["input_ids"][0])
return self._count_prompt_tokens(render_result.prompt)
def generate_chat_response(
diff --git a/studio/backend/core/inference/orchestrator.py b/studio/backend/core/inference/orchestrator.py
index 188eb2175f9..3e28b484033 100644
--- a/studio/backend/core/inference/orchestrator.py
+++ b/studio/backend/core/inference/orchestrator.py
@@ -3045,6 +3045,7 @@ class InferenceOrchestrator:
reasoning_effort: Optional[str] = None,
preserve_thinking: Optional[bool] = None,
timeout: float = 30.0,
+ images: Optional[list] = None,
) -> tuple[int, Optional[str]]:
"""Reads through an addressed mailbox: in compare mode a dispatcher owns the response queue
instead."""
@@ -3079,6 +3080,7 @@ class InferenceOrchestrator:
"enable_thinking": enable_thinking,
"reasoning_effort": reasoning_effort,
"preserve_thinking": preserve_thinking,
+ "images_base64": _encoded_images(images, self._pil_to_base64) or None,
}
)
resp = self._await_reply(
@@ -3119,6 +3121,7 @@ class InferenceOrchestrator:
live_branch: Optional[list] = None,
force_reset: bool = False,
keeps_boundary: bool = False,
+ images: Optional[list] = None,
) -> dict:
"""Fit one MLX or managed-engine prompt into the served window under the policy GGUF uses.
@@ -3155,6 +3158,7 @@ class InferenceOrchestrator:
retrieval_budget,
truncate_oldest_messages,
)
+ from core.inference.mcp_images import image_marker_parts
from core.inference.llama_cpp import (
_archive_and_recall,
_boundary_metadata,
@@ -3171,6 +3175,16 @@ class InferenceOrchestrator:
if messages_without_unpriced_media(conversation) is not conversation:
return unchanged
+ # One picture per marker, in order; only the MLX count prices them.
+ images = _encoded_images(images, self._pil_to_base64)
+ markers = image_marker_parts(conversation) if images else []
+ if len(markers) != len(images) or (images and not model_info.get("is_mlx")):
+ return unchanged
+
+ def _marked(fitted):
+ kept = {id(part) for part in image_marker_parts(fitted)}
+ return [index for index, marker in enumerate(markers) if id(marker) in kept]
+
if cancel_event is not None and cancel_event.is_set():
return unchanged
# The count prices a new reply, not a resumed reply or thought.
@@ -3213,6 +3227,9 @@ class InferenceOrchestrator:
raise RuntimeError("Context compaction cancelled")
if estimated:
return _utf8_half_tokens(messages_without_unpriced_media(fitted))
+ if images:
+ pictures = [images[index] for index in _marked(fitted)]
+ return self.count_chat_tokens(fitted, "", images = pictures, **count_kwargs)[0]
return self.count_chat_tokens(fitted, "", **count_kwargs)[0]
can_reset = _can_reset_epoch(
@@ -3315,6 +3332,7 @@ class InferenceOrchestrator:
"recalled": bool(recall["recalled"]),
"anchored": list(recall["anchored"]),
"boundary_applied": True,
+ "dropped_images": sorted(set(range(len(markers))) - set(_marked(fitted))),
}
except Exception as exc:
logger.warning("Could not preflight the context window: %s", exc)
@@ -3801,9 +3819,6 @@ class InferenceOrchestrator:
def _fit_iteration(conversation: list, active_tools: list, live_branch: list) -> dict:
nonlocal _sticky_boundary_applied, _conversation_recall_done, _loop_cut
- if loop_images:
- # The count cannot price pictures.
- return {}
# Pin the newest tool result and its user turn for this fit only, or results fill the window.
pinned = _rolling_anchor_ids
for index in range(len(conversation) - 1, -1, -1):
@@ -3833,6 +3848,7 @@ class InferenceOrchestrator:
request_branch = _request_branch,
live_branch = live_branch,
keeps_boundary = _loop_cut,
+ images = loop_images,
)
_loop_cut = _loop_cut or any(
event.get("dropped_messages") for event in result.get("events") or ()
diff --git a/studio/backend/core/inference/safetensors_agentic.py b/studio/backend/core/inference/safetensors_agentic.py
index 7ae30f28d3f..a8329922631 100644
--- a/studio/backend/core/inference/safetensors_agentic.py
+++ b/studio/backend/core/inference/safetensors_agentic.py
@@ -760,6 +760,9 @@ def run_safetensors_tool_loop(
conversation, active_tools, _extend_live_branch(conversation)
)
conversation = list(fit_result.get("messages") or conversation)
+ for gone in reversed(fit_result.get("dropped_images") or ()):
+ del images_sink[gone]
+ caller_images = tuple(i - (i > gone) for i in caller_images if i != gone)
yield from fit_result.get("events") or ()
# Cumulative snapshots: keep scans incremental with overlap for split literals.
diff --git a/studio/backend/core/inference/worker.py b/studio/backend/core/inference/worker.py
index e54d7055f8b..b46373fb9ef 100644
--- a/studio/backend/core/inference/worker.py
+++ b/studio/backend/core/inference/worker.py
@@ -1164,18 +1164,17 @@ def _dispatch_generate(backend, cmd: dict, resp_queue: Any, cancel_event) -> Non
_handle_generate(backend, cmd, resp_queue, cancel_event)
-def _generation_kwargs(backend, cmd: dict, cancel_event) -> dict:
+def _decoded_images(backend, encoded) -> list:
resize = _resize_image
if _active_model_is_deepseek_ocr(backend):
from core.inference.model_ids import DEEPSEEK_OCR_MAX_IMAGE_SIDE
resize = functools.partial(_resize_image, max_size = DEEPSEEK_OCR_MAX_IMAGE_SIDE)
- image = None
- image_b64 = cmd.get("image_base64")
- if image_b64:
- image = resize(_decode_image(image_b64))
- images = [
- resize(_decode_image(encoded)) for encoded in cmd.get("images_base64") or () if encoded
- ]
+ return [resize(_decode_image(one)) for one in encoded or () if one]
+
+
+def _generation_kwargs(backend, cmd: dict, cancel_event) -> dict:
+ image = next(iter(_decoded_images(backend, [cmd.get("image_base64")])), None)
+ images = _decoded_images(backend, cmd.get("images_base64"))
gen_kwargs = {
"messages": cmd["messages"],
@@ -1375,6 +1374,7 @@ def _handle_count_tokens(backend, cmd: dict, resp_queue: Any) -> None:
enable_thinking = cmd.get("enable_thinking"),
reasoning_effort = cmd.get("reasoning_effort"),
preserve_thinking = cmd.get("preserve_thinking"),
+ images = _decoded_images(backend, cmd.get("images_base64")),
)
except Exception as exc:
_send_response(
diff --git a/studio/backend/routes/inference.py b/studio/backend/routes/inference.py
index c44cc87b89f..02514522aae 100644
--- a/studio/backend/routes/inference.py
+++ b/studio/backend/routes/inference.py
@@ -35545,10 +35545,8 @@ async def produce_openai_chat_completions(
if "kwargs" in _sf_fit_cache:
return _sf_fit_cache["kwargs"], ()
fitted_kwargs, events = gen_kwargs, ()
- # The count cannot price pictures or video, so those prompts are left alone.
- if _sf_fit_overflow and all(
- gen_kwargs.get(key) is None for key in ("image", "images", "videos")
- ):
+ # The count cannot price video, or an attachment beside replayed pictures.
+ if _sf_fit_overflow and all(gen_kwargs.get(key) is None for key in ("images", "videos")):
fitted = backend.compact_chat_context(
gen_kwargs.get("messages") or [],
system_prompt = gen_kwargs.get("system_prompt") or "",
@@ -35565,12 +35563,15 @@ async def produce_openai_chat_completions(
preserve_thinking = gen_kwargs.get("preserve_thinking"),
continue_final_message = bool(gen_kwargs.get("continue_final_message", False)),
recall_reachable = _sf_recall_reachable,
+ images = [image] if image is not None else None,
)
fitted_kwargs = {
**gen_kwargs,
"messages": fitted["messages"],
"system_prompt": fitted["system_prompt"],
}
+ if fitted.get("dropped_images"):
+ fitted_kwargs["image"] = None
events = fitted.get("events") or ()
if _sf_call_prefix:
# After fitting: compaction leaves a prompt ending in a continuation untouched.
@@ -36019,14 +36020,19 @@ async def produce_openai_chat_completions(
# Mark the owning turn before the correction is appended,
# or the reverse scan attaches the picture to it (#10092).
# The fitted prompt: the retry extends it and is not refitted.
- _nudge_base = _sf_fit_request()[0]["messages"]
- if _sf_renders_image or _video_clips:
+ _nudge_fitted = _sf_fit_request()[0]
+ _nudge_base = _nudge_fitted["messages"]
+ # A picture the fit dropped with its turn is not marked again.
+ _nudge_image = (
+ _sf_renders_image and _nudge_fitted.get("image") is image
+ )
+ if _nudge_image or _video_clips:
from core.inference.chat_template_helpers import (
messages_with_attached_image as _nudge_attach,
)
_nudge_base = _nudge_attach(
_nudge_base,
- image = _sf_renders_image,
+ image = _nudge_image,
video = len(_video_clips),
)
retry_messages = [
diff --git a/studio/backend/tests/test_mlx_context_compaction.py b/studio/backend/tests/test_mlx_context_compaction.py
index b15d0f11281..4e22af44532 100644
--- a/studio/backend/tests/test_mlx_context_compaction.py
+++ b/studio/backend/tests/test_mlx_context_compaction.py
@@ -10,6 +10,7 @@ import pytest
import core.inference.tools as tools_mod
from core.inference import llama_cpp
from core.inference.context_window import retrieval_budget
+from core.inference.mcp_images import count_image_parts
from core.inference.orchestrator import InferenceOrchestrator
INSTRUCTION = "Standing instruction for the rest of this task: always answer in a markdown table."
@@ -125,6 +126,73 @@ def test_a_prompt_the_fit_must_not_touch_is_returned_unchanged(case):
assert result["events"] == [] and "boundary_applied" not in result
+PICTURE = "cGl4ZWxz"
+
+
+def _pictured(text):
+ return {"role": "user", "content": [{"type": "image"}, {"type": "text", "text": text}]}
+
+
+def _count_pictures(
+ messages,
+ *_args,
+ images = (),
+ **_kwargs,
+):
+ assert len(images) == count_image_parts(messages, "image")
+ return _count(messages)[0] + 300 * len(images), "mlx-test"
+
+
[email protected]("owner, dropped", [(2, [0]), (-1, [])], ids = ["old_turn", "newest_turn"])
+def test_a_picture_is_priced_into_the_fit_and_leaves_with_its_turn(archive_calls, owner, dropped):
+ backend, messages = _backend(), _thread()
+ backend.count_chat_tokens = _count_pictures
+ messages[owner] = _pictured(messages[owner]["content"])
+
+ result = _fit(backend, messages, thread_id = "thread-1", images = [PICTURE])
+
+ [truncation] = result["events"]
+ kept = count_image_parts(result["messages"], "image")
+ assert truncation["fits"] and truncation["dropped_messages"] > 0
+ assert result["dropped_images"] == dropped and kept == 1 - len(dropped)
+ assert truncation["prompt_tokens_after"] == _count(result["messages"])[0] + 300 * kept
+
+
+def test_a_picture_without_its_marker_leaves_the_prompt_unfitted():
+ messages = _thread()
+
+ result = _fit(_backend(), messages, images = [PICTURE])
+
+ assert result["messages"] == messages and result["events"] == []
+
+
+def test_the_tool_loop_stops_sending_a_picture_once_its_turn_is_dropped(monkeypatch, archive_calls):
+ backend, sent = _backend(), []
+ backend.models["mlx-test"]["is_vision"] = True
+ backend.count_chat_tokens = _count_pictures
+
+ def _generate(**call):
+ sent.append((count_image_parts(call["messages"], "image"), call["images"]))
+ yield CALL % len(sent) if len(sent) < 3 else "done"
+
+ monkeypatch.setattr(backend, "generate_chat_response", _generate)
+ monkeypatch.setattr(tools_mod, "execute_tool", lambda *_a, **_k: "r" * 2400)
+ events = list(
+ backend.generate_chat_completion_with_tools(
+ messages = [_pictured("o" * 400), ANSWERED, QUESTION],
+ tools = [WEB],
+ images = [PICTURE],
+ caller_image_indexes = (0,),
+ max_tokens = 200,
+ context_overflow = "truncate_oldest",
+ thread_id = "thread-1",
+ )
+ )
+
+ assert any(event.get("dropped_messages") for event in events)
+ assert sent[0] == (1, [PICTURE]) and sent[-1] == (0, None)
+
+
def test_a_request_resets_only_where_search_can_follow_and_names_only_an_offered_tool(
archive_calls, monkeypatch
):
diff --git a/studio/backend/tests/test_mlx_inference_backend.py b/studio/backend/tests/test_mlx_inference_backend.py
index afd4cfdc793..ce9c4643767 100644
--- a/studio/backend/tests/test_mlx_inference_backend.py
+++ b/studio/backend/tests/test_mlx_inference_backend.py
@@ -7929,6 +7929,54 @@ def test_the_context_budget_counts_the_request_in_tokens_the_processor_would_pro
backend._check_context_budget(prompt, 16, images = [object(), object()])
+def test_a_count_prices_the_pictures_it_is_given_as_the_processor_expands_them(monkeypatch):
+ _stub_prepare_inputs(monkeypatch)
+ backend, rendered = _vision_backend(), []
+ turn = [{"role": "user", "content": [{"type": "image"}, {"type": "text", "text": "what"}]}]
+
+ def _render(messages, images, **_kwargs):
+ rendered.append(images)
+ return "placeholder describe this picture", None, None
+
+ backend._render_vlm_prompt = _render
+ picture = object()
+
+ assert backend.count_chat_tokens(turn, images = [picture]) == 524
+ assert backend.count_chat_tokens(turn[:0]) == 4
+ assert rendered == [[picture], None]
+
+
+def test_a_count_prices_one_picture_where_the_template_marks_one(monkeypatch):
+ _stub_prepare_inputs(monkeypatch)
+ backend, rendered = _vision_backend(), []
+ turn = {"role": "user", "content": [{"type": "image"}, {"type": "text", "text": "what"}]}
+
+ def _render(messages, images, **_kwargs):
+ rendered.append((sum(len(m["content"]) == 2 for m in messages), images))
+ return "placeholder describe this picture", None, None
+
+ backend._render_vlm_prompt = _render
+ old, new = object(), object()
+
+ assert backend.count_chat_tokens([turn, turn], images = [old, new]) == 524
+ assert rendered == [(1, [new])]
+
+
+def test_a_count_through_mlx_lm_prices_the_ids_generation_embeds(monkeypatch):
+ backend = _vision_backend()
+ backend._is_vlm, backend._reads_vision = False, True
+ backend._tokenizer._unsloth_image_processor = "processor"
+ backend._render_text_prompt = lambda *_a, **_k: SimpleNamespace(prompt = "P")
+ loader = types.ModuleType("unsloth_zoo.mlx.loader")
+ loader._mlx_lm_image_inputs = lambda _tok, processor, images, prompt: {
+ "input_ids": [[0] * (len(prompt) + 300 * len(images) * (processor == "processor"))]
+ }
+ monkeypatch.setitem(sys.modules, "unsloth_zoo.mlx.loader", loader)
+ turn = [{"role": "user", "content": [{"type": "image"}, {"type": "text", "text": "what"}]}]
+
+ assert backend.count_chat_tokens(turn, images = [object()]) == 301
+
+
def test_the_vlm_entry_point_hands_the_limit_a_clip_the_processor_can_read(monkeypatch):
"""Every clip is counted from a readable file, not the route's base64."""
import base64
diff --git a/studio/backend/tests/test_sf_client_tools_passthrough.py b/studio/backend/tests/test_sf_client_tools_passthrough.py
index 7ec9b79b535..7399b72bfd2 100644
--- a/studio/backend/tests/test_sf_client_tools_passthrough.py
+++ b/studio/backend/tests/test_sf_client_tools_passthrough.py
@@ -13,6 +13,7 @@ import pytest
from models.inference import ChatCompletionRequest, ChatMessage
from routes.inference import openai_chat_completions
from core.inference.api_monitor import ApiMonitor
+from core.inference.mcp_images import count_image_parts
# #12382: with the date setting on and no system prompt, the chat's first message opens with
@@ -2106,7 +2107,13 @@ class _TrimmingBackend(_FittedToolLoopBackend):
def compact_chat_context(self, messages, **kwargs):
self.fits.append(kwargs)
event = {"type": "context_truncated", "dropped_messages": len(messages) - 1, "fits": True}
- return {"messages": messages[-1:], "system_prompt": "fitted", "events": [event]}
+ orphaned = bool(kwargs.get("images")) and not count_image_parts(messages[-1:], "image")
+ return {
+ "messages": messages[-1:],
+ "system_prompt": "fitted",
+ "events": [event],
+ "dropped_images": [0] if orphaned else [],
+ }
def _texts(messages):
@@ -2140,12 +2147,19 @@ def test_every_choice_of_an_mlx_request_starts_from_one_fit(monkeypatch, slots):
assert body["context_truncated"]["dropped_messages"] == 2
-def test_an_mlx_prompt_carrying_a_picture_is_sent_unfitted(monkeypatch):
[email protected]("owner", [0, 2], ids = ["old_turn", "newest_turn"])
+def test_an_mlx_prompt_carrying_a_picture_is_fitted_and_loses_it_with_its_turn(monkeypatch, owner):
backend = _TrimmingBackend(_fixed("done"))
backend.models["sf-model"].update(is_mlx = True, is_vision = True)
- _call(_overflowing(image_base64 = _PNG_1x1), monkeypatch, backend)
+ backend.models["sf-model"]["chat_template_info"]["renders_image"] = True
+ payload = _overflowing()
+ payload.messages[owner] = _image_message(payload.messages[owner].content)
+ _call(payload, monkeypatch, backend)
- assert backend.fits == [] and len(backend.calls[0]["messages"]) == 3
+ [fit], [call] = backend.fits, backend.calls
+ assert len(fit["images"]) == 1 and len(call["messages"]) == 1
+ assert (call["image"] is None) is (owner == 0)
+ assert count_image_parts(call["messages"], "image") == (owner == 2)
def test_a_nudge_retry_extends_the_fitted_prompt_and_is_not_refitted(monkeypatch):