Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
Studio: MLX chats with a picture keep going when the context fills (#17424)
Nilay31d7eb47ea44
diff --git a/studio/backend/core/inference/mlx_inference.py b/studio/backend/core/inference/mlx_inference.pyindex 2ef224d898a..0e066518e04 100644--- a/studio/backend/core/inference/mlx_inference.py+++ b/studio/backend/core/inference/mlx_inference.py@@ -6949,24 +6949,30 @@ class MLXInferenceBackend:enable_thinking = None,reasoning_effort = None,preserve_thinking = None,+ images = None,) -> int:"""Prompt tokens this model would receive for these messages. Renders and tokenizes exactlyas generation does."""if self._model is None:raise RuntimeError("No model loaded")- full_messages = self._with_system_prompt(messages, system_prompt)+ images = list(images or ())+ full_messages = self._conversation_to_render(messages, system_prompt, images or None)+ if len(images) > 1 and not self._multi_image_marker:+ # Generation serves the newest picture alone.+ full_messages = list(full_messages)+ trim_image_turns(full_messages, images, limit = 1, keep = (len(images) - 1,))if self._is_vlm:- # images=None: an image anywhere makes the structured-item check raise.+ # One image per marker, else the structured-item check raises.prompt, _target, _markers = self._render_vlm_prompt(full_messages,- None,+ images or None,tools = tools,enable_thinking = enable_thinking,reasoning_effort = reasoning_effort,preserve_thinking = preserve_thinking,)- return self._count_prompt_tokens(prompt)+ return self._count_prompt_tokens(prompt, images)render_result = self._render_text_prompt(full_messages,@@ -6975,6 +6981,17 @@ class MLXInferenceBackend:reasoning_effort = reasoning_effort,preserve_thinking = preserve_thinking,)+ if images and self._reads_vision_input():+ from unsloth_zoo.mlx.loader import _mlx_lm_image_inputs++ # The ids generation embeds, without running the vision tower.+ expanded = _mlx_lm_image_inputs(+ self._tokenizer,+ self._tokenizer._unsloth_image_processor,+ images,+ render_result.prompt,+ )+ return len(expanded["input_ids"][0])return self._count_prompt_tokens(render_result.prompt)def generate_chat_response(diff --git a/studio/backend/core/inference/orchestrator.py b/studio/backend/core/inference/orchestrator.pyindex 188eb2175f9..3e28b484033 100644--- a/studio/backend/core/inference/orchestrator.py+++ b/studio/backend/core/inference/orchestrator.py@@ -3045,6 +3045,7 @@ class InferenceOrchestrator:reasoning_effort: Optional[str] = None,preserve_thinking: Optional[bool] = None,timeout: float = 30.0,+ images: Optional[list] = None,) -> tuple[int, Optional[str]]:"""Reads through an addressed mailbox: in compare mode a dispatcher owns the response queueinstead."""@@ -3079,6 +3080,7 @@ class InferenceOrchestrator:"enable_thinking": enable_thinking,"reasoning_effort": reasoning_effort,"preserve_thinking": preserve_thinking,+ "images_base64": _encoded_images(images, self._pil_to_base64) or None,})resp = self._await_reply(@@ -3119,6 +3121,7 @@ class InferenceOrchestrator:live_branch: Optional[list] = None,force_reset: bool = False,keeps_boundary: bool = False,+ images: Optional[list] = None,) -> dict:"""Fit one MLX or managed-engine prompt into the served window under the policy GGUF uses.@@ -3155,6 +3158,7 @@ class InferenceOrchestrator:retrieval_budget,truncate_oldest_messages,)+ from core.inference.mcp_images import image_marker_partsfrom core.inference.llama_cpp import (_archive_and_recall,_boundary_metadata,@@ -3171,6 +3175,16 @@ class InferenceOrchestrator:if messages_without_unpriced_media(conversation) is not conversation:return unchanged+ # One picture per marker, in order; only the MLX count prices them.+ images = _encoded_images(images, self._pil_to_base64)+ markers = image_marker_parts(conversation) if images else []+ if len(markers) != len(images) or (images and not model_info.get("is_mlx")):+ return unchanged++ def _marked(fitted):+ kept = {id(part) for part in image_marker_parts(fitted)}+ return [index for index, marker in enumerate(markers) if id(marker) in kept]+if cancel_event is not None and cancel_event.is_set():return unchanged# The count prices a new reply, not a resumed reply or thought.@@ -3213,6 +3227,9 @@ class InferenceOrchestrator:raise RuntimeError("Context compaction cancelled")if estimated:return _utf8_half_tokens(messages_without_unpriced_media(fitted))+ if images:+ pictures = [images[index] for index in _marked(fitted)]+ return self.count_chat_tokens(fitted, "", images = pictures, **count_kwargs)[0]return self.count_chat_tokens(fitted, "", **count_kwargs)[0]can_reset = _can_reset_epoch(@@ -3315,6 +3332,7 @@ class InferenceOrchestrator:"recalled": bool(recall["recalled"]),"anchored": list(recall["anchored"]),"boundary_applied": True,+ "dropped_images": sorted(set(range(len(markers))) - set(_marked(fitted))),}except Exception as exc:logger.warning("Could not preflight the context window: %s", exc)@@ -3801,9 +3819,6 @@ class InferenceOrchestrator:def _fit_iteration(conversation: list, active_tools: list, live_branch: list) -> dict:nonlocal _sticky_boundary_applied, _conversation_recall_done, _loop_cut- if loop_images:- # The count cannot price pictures.- return {}# Pin the newest tool result and its user turn for this fit only, or results fill the window.pinned = _rolling_anchor_idsfor index in range(len(conversation) - 1, -1, -1):@@ -3833,6 +3848,7 @@ class InferenceOrchestrator:request_branch = _request_branch,live_branch = live_branch,keeps_boundary = _loop_cut,+ images = loop_images,)_loop_cut = _loop_cut or any(event.get("dropped_messages") for event in result.get("events") or ()diff --git a/studio/backend/core/inference/safetensors_agentic.py b/studio/backend/core/inference/safetensors_agentic.pyindex 7ae30f28d3f..a8329922631 100644--- a/studio/backend/core/inference/safetensors_agentic.py+++ b/studio/backend/core/inference/safetensors_agentic.py@@ -760,6 +760,9 @@ def run_safetensors_tool_loop(conversation, active_tools, _extend_live_branch(conversation))conversation = list(fit_result.get("messages") or conversation)+ for gone in reversed(fit_result.get("dropped_images") or ()):+ del images_sink[gone]+ caller_images = tuple(i - (i > gone) for i in caller_images if i != gone)yield from fit_result.get("events") or ()# Cumulative snapshots: keep scans incremental with overlap for split literals.diff --git a/studio/backend/core/inference/worker.py b/studio/backend/core/inference/worker.pyindex e54d7055f8b..b46373fb9ef 100644--- a/studio/backend/core/inference/worker.py+++ b/studio/backend/core/inference/worker.py@@ -1164,18 +1164,17 @@ def _dispatch_generate(backend, cmd: dict, resp_queue: Any, cancel_event) -> Non_handle_generate(backend, cmd, resp_queue, cancel_event)-def _generation_kwargs(backend, cmd: dict, cancel_event) -> dict:+def _decoded_images(backend, encoded) -> list:resize = _resize_imageif _active_model_is_deepseek_ocr(backend):from core.inference.model_ids import DEEPSEEK_OCR_MAX_IMAGE_SIDEresize = functools.partial(_resize_image, max_size = DEEPSEEK_OCR_MAX_IMAGE_SIDE)- image = None- image_b64 = cmd.get("image_base64")- if image_b64:- image = resize(_decode_image(image_b64))- images = [- resize(_decode_image(encoded)) for encoded in cmd.get("images_base64") or () if encoded- ]+ return [resize(_decode_image(one)) for one in encoded or () if one]+++def _generation_kwargs(backend, cmd: dict, cancel_event) -> dict:+ image = next(iter(_decoded_images(backend, [cmd.get("image_base64")])), None)+ images = _decoded_images(backend, cmd.get("images_base64"))gen_kwargs = {"messages": cmd["messages"],@@ -1375,6 +1374,7 @@ def _handle_count_tokens(backend, cmd: dict, resp_queue: Any) -> None:enable_thinking = cmd.get("enable_thinking"),reasoning_effort = cmd.get("reasoning_effort"),preserve_thinking = cmd.get("preserve_thinking"),+ images = _decoded_images(backend, cmd.get("images_base64")),)except Exception as exc:_send_response(diff --git a/studio/backend/routes/inference.py b/studio/backend/routes/inference.pyindex c44cc87b89f..02514522aae 100644--- a/studio/backend/routes/inference.py+++ b/studio/backend/routes/inference.py@@ -35545,10 +35545,8 @@ async def produce_openai_chat_completions(if "kwargs" in _sf_fit_cache:return _sf_fit_cache["kwargs"], ()fitted_kwargs, events = gen_kwargs, ()- # The count cannot price pictures or video, so those prompts are left alone.- if _sf_fit_overflow and all(- gen_kwargs.get(key) is None for key in ("image", "images", "videos")- ):+ # The count cannot price video, or an attachment beside replayed pictures.+ if _sf_fit_overflow and all(gen_kwargs.get(key) is None for key in ("images", "videos")):fitted = backend.compact_chat_context(gen_kwargs.get("messages") or [],system_prompt = gen_kwargs.get("system_prompt") or "",@@ -35565,12 +35563,15 @@ async def produce_openai_chat_completions(preserve_thinking = gen_kwargs.get("preserve_thinking"),continue_final_message = bool(gen_kwargs.get("continue_final_message", False)),recall_reachable = _sf_recall_reachable,+ images = [image] if image is not None else None,)fitted_kwargs = {**gen_kwargs,"messages": fitted["messages"],"system_prompt": fitted["system_prompt"],}+ if fitted.get("dropped_images"):+ fitted_kwargs["image"] = Noneevents = fitted.get("events") or ()if _sf_call_prefix:# After fitting: compaction leaves a prompt ending in a continuation untouched.@@ -36019,14 +36020,19 @@ async def produce_openai_chat_completions(# Mark the owning turn before the correction is appended,# or the reverse scan attaches the picture to it (#10092).# The fitted prompt: the retry extends it and is not refitted.- _nudge_base = _sf_fit_request()[0]["messages"]- if _sf_renders_image or _video_clips:+ _nudge_fitted = _sf_fit_request()[0]+ _nudge_base = _nudge_fitted["messages"]+ # A picture the fit dropped with its turn is not marked again.+ _nudge_image = (+ _sf_renders_image and _nudge_fitted.get("image") is image+ )+ if _nudge_image or _video_clips:from core.inference.chat_template_helpers import (messages_with_attached_image as _nudge_attach,)_nudge_base = _nudge_attach(_nudge_base,- image = _sf_renders_image,+ image = _nudge_image,video = len(_video_clips),)retry_messages = [diff --git a/studio/backend/tests/test_mlx_context_compaction.py b/studio/backend/tests/test_mlx_context_compaction.pyindex b15d0f11281..4e22af44532 100644--- a/studio/backend/tests/test_mlx_context_compaction.py+++ b/studio/backend/tests/test_mlx_context_compaction.py@@ -10,6 +10,7 @@ import pytestimport core.inference.tools as tools_modfrom core.inference import llama_cppfrom core.inference.context_window import retrieval_budget+from core.inference.mcp_images import count_image_partsfrom core.inference.orchestrator import InferenceOrchestratorINSTRUCTION = "Standing instruction for the rest of this task: always answer in a markdown table."@@ -125,6 +126,73 @@ def test_a_prompt_the_fit_must_not_touch_is_returned_unchanged(case):assert result["events"] == [] and "boundary_applied" not in result+PICTURE = "cGl4ZWxz"+++def _pictured(text):+ return {"role": "user", "content": [{"type": "image"}, {"type": "text", "text": text}]}+++def _count_pictures(+ messages,+ *_args,+ images = (),+ **_kwargs,+):+ assert len(images) == count_image_parts(messages, "image")+ return _count(messages)[0] + 300 * len(images), "mlx-test"++[email protected]("owner, dropped", [(2, [0]), (-1, [])], ids = ["old_turn", "newest_turn"])+def test_a_picture_is_priced_into_the_fit_and_leaves_with_its_turn(archive_calls, owner, dropped):+ backend, messages = _backend(), _thread()+ backend.count_chat_tokens = _count_pictures+ messages[owner] = _pictured(messages[owner]["content"])++ result = _fit(backend, messages, thread_id = "thread-1", images = [PICTURE])++ [truncation] = result["events"]+ kept = count_image_parts(result["messages"], "image")+ assert truncation["fits"] and truncation["dropped_messages"] > 0+ assert result["dropped_images"] == dropped and kept == 1 - len(dropped)+ assert truncation["prompt_tokens_after"] == _count(result["messages"])[0] + 300 * kept+++def test_a_picture_without_its_marker_leaves_the_prompt_unfitted():+ messages = _thread()++ result = _fit(_backend(), messages, images = [PICTURE])++ assert result["messages"] == messages and result["events"] == []+++def test_the_tool_loop_stops_sending_a_picture_once_its_turn_is_dropped(monkeypatch, archive_calls):+ backend, sent = _backend(), []+ backend.models["mlx-test"]["is_vision"] = True+ backend.count_chat_tokens = _count_pictures++ def _generate(**call):+ sent.append((count_image_parts(call["messages"], "image"), call["images"]))+ yield CALL % len(sent) if len(sent) < 3 else "done"++ monkeypatch.setattr(backend, "generate_chat_response", _generate)+ monkeypatch.setattr(tools_mod, "execute_tool", lambda *_a, **_k: "r" * 2400)+ events = list(+ backend.generate_chat_completion_with_tools(+ messages = [_pictured("o" * 400), ANSWERED, QUESTION],+ tools = [WEB],+ images = [PICTURE],+ caller_image_indexes = (0,),+ max_tokens = 200,+ context_overflow = "truncate_oldest",+ thread_id = "thread-1",+ )+ )++ assert any(event.get("dropped_messages") for event in events)+ assert sent[0] == (1, [PICTURE]) and sent[-1] == (0, None)++def test_a_request_resets_only_where_search_can_follow_and_names_only_an_offered_tool(archive_calls, monkeypatch):diff --git a/studio/backend/tests/test_mlx_inference_backend.py b/studio/backend/tests/test_mlx_inference_backend.pyindex afd4cfdc793..ce9c4643767 100644--- a/studio/backend/tests/test_mlx_inference_backend.py+++ b/studio/backend/tests/test_mlx_inference_backend.py@@ -7929,6 +7929,54 @@ def test_the_context_budget_counts_the_request_in_tokens_the_processor_would_probackend._check_context_budget(prompt, 16, images = [object(), object()])+def test_a_count_prices_the_pictures_it_is_given_as_the_processor_expands_them(monkeypatch):+ _stub_prepare_inputs(monkeypatch)+ backend, rendered = _vision_backend(), []+ turn = [{"role": "user", "content": [{"type": "image"}, {"type": "text", "text": "what"}]}]++ def _render(messages, images, **_kwargs):+ rendered.append(images)+ return "placeholder describe this picture", None, None++ backend._render_vlm_prompt = _render+ picture = object()++ assert backend.count_chat_tokens(turn, images = [picture]) == 524+ assert backend.count_chat_tokens(turn[:0]) == 4+ assert rendered == [[picture], None]+++def test_a_count_prices_one_picture_where_the_template_marks_one(monkeypatch):+ _stub_prepare_inputs(monkeypatch)+ backend, rendered = _vision_backend(), []+ turn = {"role": "user", "content": [{"type": "image"}, {"type": "text", "text": "what"}]}++ def _render(messages, images, **_kwargs):+ rendered.append((sum(len(m["content"]) == 2 for m in messages), images))+ return "placeholder describe this picture", None, None++ backend._render_vlm_prompt = _render+ old, new = object(), object()++ assert backend.count_chat_tokens([turn, turn], images = [old, new]) == 524+ assert rendered == [(1, [new])]+++def test_a_count_through_mlx_lm_prices_the_ids_generation_embeds(monkeypatch):+ backend = _vision_backend()+ backend._is_vlm, backend._reads_vision = False, True+ backend._tokenizer._unsloth_image_processor = "processor"+ backend._render_text_prompt = lambda *_a, **_k: SimpleNamespace(prompt = "P")+ loader = types.ModuleType("unsloth_zoo.mlx.loader")+ loader._mlx_lm_image_inputs = lambda _tok, processor, images, prompt: {+ "input_ids": [[0] * (len(prompt) + 300 * len(images) * (processor == "processor"))]+ }+ monkeypatch.setitem(sys.modules, "unsloth_zoo.mlx.loader", loader)+ turn = [{"role": "user", "content": [{"type": "image"}, {"type": "text", "text": "what"}]}]++ assert backend.count_chat_tokens(turn, images = [object()]) == 301++def test_the_vlm_entry_point_hands_the_limit_a_clip_the_processor_can_read(monkeypatch):"""Every clip is counted from a readable file, not the route's base64."""import base64diff --git a/studio/backend/tests/test_sf_client_tools_passthrough.py b/studio/backend/tests/test_sf_client_tools_passthrough.pyindex 7ec9b79b535..7399b72bfd2 100644--- a/studio/backend/tests/test_sf_client_tools_passthrough.py+++ b/studio/backend/tests/test_sf_client_tools_passthrough.py@@ -13,6 +13,7 @@ import pytestfrom models.inference import ChatCompletionRequest, ChatMessagefrom routes.inference import openai_chat_completionsfrom core.inference.api_monitor import ApiMonitor+from core.inference.mcp_images import count_image_parts# #12382: with the date setting on and no system prompt, the chat's first message opens with@@ -2106,7 +2107,13 @@ class _TrimmingBackend(_FittedToolLoopBackend):def compact_chat_context(self, messages, **kwargs):self.fits.append(kwargs)event = {"type": "context_truncated", "dropped_messages": len(messages) - 1, "fits": True}- return {"messages": messages[-1:], "system_prompt": "fitted", "events": [event]}+ orphaned = bool(kwargs.get("images")) and not count_image_parts(messages[-1:], "image")+ return {+ "messages": messages[-1:],+ "system_prompt": "fitted",+ "events": [event],+ "dropped_images": [0] if orphaned else [],+ }def _texts(messages):@@ -2140,12 +2147,19 @@ def test_every_choice_of_an_mlx_request_starts_from_one_fit(monkeypatch, slots):assert body["context_truncated"]["dropped_messages"] == 2-def test_an_mlx_prompt_carrying_a_picture_is_sent_unfitted(monkeypatch):[email protected]("owner", [0, 2], ids = ["old_turn", "newest_turn"])+def test_an_mlx_prompt_carrying_a_picture_is_fitted_and_loses_it_with_its_turn(monkeypatch, owner):backend = _TrimmingBackend(_fixed("done"))backend.models["sf-model"].update(is_mlx = True, is_vision = True)- _call(_overflowing(image_base64 = _PNG_1x1), monkeypatch, backend)+ backend.models["sf-model"]["chat_template_info"]["renders_image"] = True+ payload = _overflowing()+ payload.messages[owner] = _image_message(payload.messages[owner].content)+ _call(payload, monkeypatch, backend)- assert backend.fits == [] and len(backend.calls[0]["messages"]) == 3+ [fit], [call] = backend.fits, backend.calls+ assert len(fit["images"]) == 1 and len(call["messages"]) == 1+ assert (call["image"] is None) is (owner == 0)+ assert count_image_parts(call["messages"], "image") == (owner == 2)def test_a_nudge_retry_extends_the_fitted_prompt_and_is_not_refitted(monkeypatch):