Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
studio: send mcp tool images to vision models (#10088)
Nilay992849427e93
diff --git a/.github/workflows/studio-backend-ci.yml b/.github/workflows/studio-backend-ci.ymlindex 0dfed4699bc..2321d6d36e0 100644--- a/.github/workflows/studio-backend-ci.yml+++ b/.github/workflows/studio-backend-ci.yml@@ -339,6 +339,7 @@ jobs:# fails that guard instead of flaking here.## test_media_auto_switch has a 0.4s budget that xdist scheduling can exceed.+ # test_pr5624_regressions exceeded its 1s R1 parser bound in two contended runs.## --timeout: a per-test wall-clock cap, so a test that never returns fails with its# own name in the summary instead of taking the job's 45 minutes down with it and@@ -387,6 +388,7 @@ jobs:--ignore=tests/test_anthropic_messages.py \--ignore=tests/test_profile_stats.py \--ignore=tests/test_media_auto_switch.py \+ --ignore=tests/test_pr5624_regressions.py \-k 'not llama_cpp_load_progress_live and not TestGpuAutoSelection and not TestPreSpawnGpuResolution and not TestPerGpuFitGuardAllCounts and not TestTransformersIntrospection and not test_returns_cuda_when_cuda_available and not test_calls_cuda_cache_when_cuda'- name: Backend tests that cannot share a worker@@ -412,7 +414,8 @@ jobs:tests/test_scan_loras_off_event_loop.py \tests/test_anthropic_messages.py \tests/test_profile_stats.py \- tests/test_media_auto_switch.py+ tests/test_media_auto_switch.py \+ tests/test_pr5624_regressions.py- name: Pre-3.12 branches, on the newest interpreter that takes themif: matrix.scope == 'floor-spot-check'diff --git a/studio/backend/core/inference/chat_template_helpers.py b/studio/backend/core/inference/chat_template_helpers.pyindex 5dd45247d1d..4dc24338af5 100644--- a/studio/backend/core/inference/chat_template_helpers.py+++ b/studio/backend/core/inference/chat_template_helpers.py@@ -3078,6 +3078,10 @@ def render_prompt_with_boundary(the kwarg get a manual splice, taking the partial from *messages* (which the caller alreadyswept) rather than a separate copy: a raw partial could close the turn or open another roleinstead of resuming (#7066)."""+ from core.inference.mcp_images import prepare_image_turn_boundaries++ for template in _selected_chat_template_strings(processor, tools):+ messages = prepare_image_turn_boundaries(messages, template)extra = {"tools": tools} if tools else {}partial = trailing_assistant_text(messages) if continue_final_message else Noneif not partial:@@ -3128,6 +3132,10 @@ def apply_chat_template_for_generation(inside the trailing assistant turn, so the model resumes the partial instead of restartingit."""# Shared choke point for the transformers and MLX backends (#7066).+ from core.inference.mcp_images import prepare_image_turn_boundaries++ for template in _selected_chat_template_strings(tokenizer, tools):+ messages = prepare_image_turn_boundaries(messages, template)messages, tools, _markup = neutralize_for_render(tokenizer, messages, tools)reasoning_kwargs: dict = {}if enable_thinking is not None:diff --git a/studio/backend/core/inference/inference.py b/studio/backend/core/inference/inference.pyindex 3f3824f967b..c9b95fe987e 100644--- a/studio/backend/core/inference/inference.py+++ b/studio/backend/core/inference/inference.py@@ -471,6 +471,34 @@ def _prompt_already_has_bos(tokenizer, prompt):return bool(len(ids)) and ids[0] == bos_token_id+def _without_image_parts(messages) -> list[dict]:+ """The same conversation with every image placeholder dropped, collapsing a+ turn back to its text when that is all it held."""+ out = []+ for message in messages:+ content = message.get("content")+ if not isinstance(content, list):+ out.append(message)+ continue+ kept = [+ part+ for part in content+ if not (+ isinstance(part, dict) and part.get("type") in ("image", "image_url", "input_image")+ )+ ]+ if len(kept) == len(content):+ out.append(message)+ continue+ texts = [+ part.get("text", "")+ for part in kept+ if isinstance(part, dict) and part.get("type") == "text"+ ]+ out.append({**message, "content": "\n".join(texts) if len(kept) == len(texts) else kept})+ return out++class InferenceBackend:"""Unified inference backend supporting text, vision, and LoRA models"""@@ -1219,6 +1247,8 @@ class InferenceBackend:messages: list,system_prompt: str,image = None,+ images: Optional[list] = None,+ image_ordinal: Optional[int] = None,temperature: float = 0.7,top_p: float = 0.9,top_k: int = 40,@@ -1234,8 +1264,6 @@ class InferenceBackend:presence_penalty: float = 0.0,tool_protocol_active: Optional[bool] = None,stop: Optional[list] = None,- *,- images = None,) -> Generator[str, None, None]:"""Generate response for text or vision models (lock held by background thread).@@ -1249,6 +1277,7 @@ class InferenceBackend:system_prompt = system_prompt,image = image,images = images,+ image_ordinal = image_ordinal,temperature = temperature,top_p = top_p,top_k = top_k,@@ -1271,6 +1300,8 @@ class InferenceBackend:messages: list,system_prompt: str = "",image = None,+ images: Optional[list] = None,+ image_ordinal: Optional[int] = None,temperature: float = 0.7,top_p: float = 0.9,top_k: int = 40,@@ -1287,8 +1318,6 @@ class InferenceBackend:presence_penalty: float = 0.0,tool_protocol_active: Optional[bool] = None,stop: Optional[list] = None,- *,- images = None,) -> Generator[str, None, None]:"""Inner generation logic.@@ -1302,22 +1331,13 @@ class InferenceBackend:if not self.active_model_name:raise RuntimeError("No active model")- # Every generation entry point funnels through here, so the list collapses here.- if images:- if len(images) > 1:- raise ValueError(- "The transformers backend takes one image per request; "- f"{len(images)} were supplied."- )- image = images[0]-model_info = self.models[self.active_model_name]is_vision = model_info.get("is_vision", False)tokenizer = model_info.get("tokenizer") or model_info.get("processor")tokenizer = getattr(tokenizer, "tokenizer", tokenizer)top_k = self._normalize_top_k(top_k)- if is_vision and image:+ if is_vision and (image or images):# Verify the stored processor can handle images; FastVisionModel may# return a raw tokenizer instead of a ProcessorMixin (e.g. Gemma-3).from transformers import ProcessorMixin@@ -1342,11 +1362,13 @@ class InferenceBackend:presence_penalty = presence_penalty,continue_final_message = continue_final_message,tools = tools,- tool_protocol_active = tool_protocol_active,- stop = stop,+ images = images,+ image_ordinal = image_ordinal,enable_thinking = enable_thinking,reasoning_effort = reasoning_effort,preserve_thinking = preserve_thinking,+ tool_protocol_active = tool_protocol_active,+ stop = stop,)returnelse:@@ -1355,6 +1377,10 @@ class InferenceBackend:f"({type(processor).__name__}) has no image_processor — "f"falling back to text-only generation (image will be ignored).")+ # Really text-only: the promoted history still carries {"type": "image"}+ # placeholders, and a text template either rejects the list content or+ # renders image tokens with no pixels behind them.+ messages = _without_image_parts(messages)# Text path: messages are already in ChatML format from eval.py.@@ -1497,11 +1523,13 @@ class InferenceBackend:presence_penalty: float = 0.0,continue_final_message: bool = False,tools: Optional[list] = None,- tool_protocol_active: Optional[bool] = None,- stop: Optional[list] = None,+ images: Optional[list] = None,+ image_ordinal: Optional[int] = None,enable_thinking: Optional[bool] = None,reasoning_effort: Optional[str] = None,preserve_thinking: Optional[bool] = None,+ tool_protocol_active: Optional[bool] = None,+ stop: Optional[list] = None,) -> Generator[str, None, None]:"""Handle vision model generation with true token-by-token streaming."""# Reset so a failed or uncountable run cannot surface stale stats.@@ -1524,14 +1552,22 @@ class InferenceBackend:trailing_assistant_text,vlm_prompt_issue,)+ from core.inference.mcp_images import (+ image_marker_parts,+ pixels_in_marker_order,+ top_up_image_markers,+ )+ # History first, the attachment last: the pixels bind to the markers in+ # order, and the attachment's marker sits on the newest user turn.+ attached = list(images or []) + ([image] if image is not None else [])user_message = last_user_text(messages)continue_partial = trailing_assistant_text(messages) if continue_final_message else Noneif not user_message:- user_message = "Describe this image." if image else "Hello"+ user_message = "Describe this image." if attached else "Hello"- if image:+ if attached:has_tool_history = messages_have_tool_history(messages)# Client-tools route signature: tool_choice="none" and a forced unknown name# also arrive tools=None, and the catalog alone missed them (#10092).@@ -1546,6 +1582,16 @@ class InferenceBackend:fallback_user_text = user_message,structured_content = True,)+ # The helper leaves existing markers alone, which is right for a retry but+ # not for replayed MCP pictures: those markers are not the attachment's.+ _prior_markers = image_marker_parts(vision_messages)+ vision_messages = top_up_image_markers(+ vision_messages, len(attached), ordinal = image_ordinal+ )+ if image is not None:+ attached = pixels_in_marker_order(+ vision_messages, _prior_markers, list(images or []), image+ )if bool(tools) or has_tool_history or folded_system:# The conversation the LAST render used, not the no-tools probe's (#10092).rendered_with: dict = {"messages": vision_messages}@@ -1644,7 +1690,7 @@ class InferenceBackend:else:raiseinputs = processor(- image,+ attached[0] if len(attached) == 1 else attached,input_text,add_special_tokens = False,return_tensors = "pt",@@ -1686,7 +1732,11 @@ class InferenceBackend:# this request's response protocol. Passing *tools* matches the# render: a named template selects "tool_use", not "default".reasoning_channel_markers = detect_reasoning_channel_markers(processor, tools = tools)- if image+ # Every attached image, not just a bare attachment: a replay-only+ # turn renders through this processor too, and resolving it to no+ # markers suppresses the fallback detection and lets native+ # reasoning output through as visible answer text.+ if attachedelse None,reasoning_channel_markers_resolved = True,prompt = prompt_text,diff --git a/studio/backend/core/inference/llama_cpp.py b/studio/backend/core/inference/llama_cpp.pyindex f8056933562..592fdf37805 100644--- a/studio/backend/core/inference/llama_cpp.py+++ b/studio/backend/core/inference/llama_cpp.py@@ -455,6 +455,12 @@ from core.inference.tool_call_parser import ()from core.inference.passthrough_healing import nudge_enabled as _nudge_enabledfrom core.inference.repetition_guard import is_repetition_dominated+from core.inference.mcp_images import (+ DETACHED_IMAGE_TURN_TEXT as MCP_DETACHED_IMAGE_TURN_TEXT,+ IMAGE_TURN_TEXT as MCP_IMAGE_TURN_TEXT,+ append_image_turn as append_mcp_image_turn,+ mentions_images as mcp_images_mentioned_in,+)from core.inference.tool_loop_controller import (_WORKSPACE_TOOLS,ToolLoopController,@@ -33352,6 +33358,17 @@ class LlamaCppBackend:httpx.Client + auth headers for the stream's lifetime; raisesRuntimeError on a non-200. Shared scaffold for the streaming consumers,which differ only in how they parse the SSE body."""+ from core.inference.mcp_images import prepare_image_turn_boundaries++ if "messages" in payload:+ payload = {+ **payload,+ "messages": prepare_image_turn_boundaries(+ payload["messages"],+ getattr(self, "_chat_template_override", None)+ or getattr(self, "_chat_template", None),+ ),+ }stream_timeout = httpx.Timeout(connect = 10, read = 0.5, write = 10, pool = 10)with httpx.Client(timeout = stream_timeout,@@ -34264,6 +34281,7 @@ class LlamaCppBackend:self,messages: list[dict],tools: list[dict],+ replayed_image_parts: "tuple" = (),temperature: float = 0.6,top_p: float = 0.95,top_k: int = 20,@@ -34805,6 +34823,13 @@ class LlamaCppBackend:_continuation_credits = 0_MAX_CONTINUATION_CREDITS = _MAX_LENGTH_CONTINUATIONS * max(1, max_tool_iterations)iteration = -1+ # Outside the loop: the cap is across the whole run, and a per-iteration list+ # would only ever see the current batch, leaving every earlier iteration's+ # images in the conversation untrimmed.+ # Seeded with what promotion already put in the conversation, not empty. The cap+ # is across the CONVERSATION, so a resumed chat whose history already carries the+ # allowance would otherwise get a second one for this run and send both.+ loop_mcp_image_parts: list = list(replayed_image_parts)while True:iteration += 1# Here rather than at each append: six sites grow the conversation and all of@@ -36468,6 +36493,11 @@ class LlamaCppBackend:# Which tools those no-ops were about, so the flush below can tell# whether the trailing result belongs to the same tool.deferred_noop_tools: set = set()+ # Per result, so a parallel batch is not squeezed into one+ # result's worth of images.+ batch_mcp_images: list = []+ # Where this batch's results start, for the image turn\'s wording.+ batch_conversation_start = len(conversation)# The text-path provisional card uses the parser's default id ("call_0");# a Mistral-style call carries its own id and would open a duplicate. Reuse@@ -37293,6 +37323,15 @@ class LlamaCppBackend:_forced_choice_resolved = Trueyield completion.tool_end_event()conversation.append(completion.tool_message())+ # Parsed only for a result that carries them and a model that will+ # be shown them; the marker test first, so a text result never asks.+ _completion_images = (+ completion.mcp_images()+ if mcp_images_mentioned_in(completion.result or "") and self.is_vision+ else []+ )+ if _completion_images:+ batch_mcp_images.append(_completion_images)if _compact_after_execution and decision.tool_call_id:# The promise the gate made when it let this run. Applied here# rather than on the next pass because the next pass may not@@ -37439,6 +37478,26 @@ class LlamaCppBackend:conversation.append({"role": "user", "content": _limit_text})_limit_notice_in_user_turn = True+ if batch_mcp_images and self.is_vision:+ # One block after the whole batch. With a single result "the tool+ # call above" is exact; with several it names whichever ran last,+ # which may have returned no picture at all, so the block says so+ # instead -- the external loop's rule.+ _batch_results = sum(+ 1+ for m in conversation[batch_conversation_start:]+ if isinstance(m, dict) and m.get("role") == "tool"+ )+ append_mcp_image_turn(+ conversation,+ batch_mcp_images,+ per_result = True,+ owned = loop_mcp_image_parts,+ lead = MCP_DETACHED_IMAGE_TURN_TEXT+ if _batch_results != 1+ else MCP_IMAGE_TURN_TEXT,+ )+# Close provisional cards not resolved by execution/no-op handling.for _pid, _pname in provisional_started_tool_calls.items():if _pid not in resolved_provisional_tool_call_ids:@@ -38448,7 +38507,13 @@ class LlamaCppBackend:return len(tokens)# 1. Try /apply-template to render the real chat prompt.- template_messages = list(messages) if messages else []+ from core.inference.mcp_images import prepare_image_turn_boundaries++ template_messages = prepare_image_turn_boundaries(+ list(messages) if messages else [],+ getattr(self, "_chat_template_override", None)+ or getattr(self, "_chat_template", None),+ )if system_text:template_messages = [{"role": "system", "content": system_text}diff --git a/studio/backend/core/inference/mcp_client.py b/studio/backend/core/inference/mcp_client.pyindex 2c1f7b76df6..bda78139472 100644--- a/studio/backend/core/inference/mcp_client.py+++ b/studio/backend/core/inference/mcp_client.py@@ -35,6 +35,8 @@ from utils.account_context import (is_owner_context,)+from core.inference import mcp_images+logger = get_logger(__name__)MCP_TOOL_PREFIX = "mcp__"@@ -1518,7 +1520,7 @@ def invalidate_tool_cache(server_id: Optional[str] = None) -> None:_probe_cooloff_until.pop(_account_key(server_id), None)-MCP_IMAGES_SENTINEL = "__MCP_IMAGES__:"+MCP_IMAGES_SENTINEL = mcp_images.SENTINELMAX_IMAGE_PAYLOAD_CHARS = 12_000_000@@ -1739,7 +1741,7 @@ def _flatten_result(result: Any) -> str:notes = []if images:n = len(images)- notes.append(f"{n} image{'s' if n > 1 else ''} attached; displayed to the user")+ notes.append(f"{n} image{'s' if n > 1 else ''} returned")if omitted:notes.append(f"{omitted} image{'s' if omitted > 1 else ''} omitted (too large)")notes.extend(unshown)diff --git a/studio/backend/core/inference/mcp_images.py b/studio/backend/core/inference/mcp_images.pynew file mode 100644index 00000000000..5d072d960c2--- /dev/null+++ b/studio/backend/core/inference/mcp_images.py@@ -0,0 +1,1244 @@+# SPDX-License-Identifier: AGPL-3.0-only+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0++from __future__ import annotations++import base64+import binascii+import io+import json+import re+from typing import Any, Sequence++from loggers import get_logger++logger = get_logger(__name__)++SENTINEL = "__MCP_IMAGES__:"+# Stamped by mcp_client on every tool it registers; the provenance the envelope+# is trusted on, on the replay side as well as the live one.+MCP_TOOL_PREFIX = "mcp__"+IMAGE_TURN_TEXT = "Images returned by the tool call above:"+# The same pictures where the turn cannot sit beside the result that produced them.+# The local client-tool passthrough flattens every content part to text before the+# markers are rebuilt, so they come back as one block rather than at the positions+# they were taken from, and "above" would name whatever turn happens to precede it.+DETACHED_IMAGE_TURN_TEXT = "Images returned by earlier tool calls in this conversation:"++MAX_MODEL_IMAGES = 4+MAX_TOTAL_MODEL_IMAGES = 8+# Safetensors and MLX require one image per message, including promoted tool+# batches. GGUF uses image_url parts and is exempt from this marker-path cap.+LOCAL_MAX_IMAGES_PER_TURN = 1+# Candidates carried past the cap when choosing what to decode, because which+# entries a decoder accepts is not known until it has tried. Mirrors+# DECODE_FAILURE_ALLOWANCE in studio/frontend/src/features/chat/api/mcp-images.ts.+DECODE_FAILURE_ALLOWANCE = 4+# The entry's other field. A token subtype has no length bound, and an entry's+# metadata must not be where megabytes hide from the byte budgets on both sides.+# Mirrors MAX_MCP_IMAGE_MIME_CHARS in studio/frontend/src/features/chat/api/mcp-images.ts.+MAX_MCP_IMAGE_MIME_CHARS = 256+MAX_IMAGE_EDGE = 1024+# A PNG stays small while its raster does not: 12 MB of encoded payload can hold+# tens of gigapixels. Bounded off the header, before a pixel is allocated.+MAX_IMAGE_PIXELS = 40_000_000+++def split_images(result: str) -> tuple[str, list[dict]]:+ """Validated, so tool text that merely mentions the marker is not truncated."""+ head, sep, payload = result.rpartition("\n" + SENTINEL)+ if not sep:+ return result, []+ try:+ images = json.loads(payload)+ except (ValueError, RecursionError):+ return result, []+ if not isinstance(images, list) or not images:+ return result, []+ if not all(_is_image(image) for image in images):+ return result, []+ return head.rstrip(), images+++def _is_image(image: Any) -> bool:+ return (+ isinstance(image, dict)+ and isinstance(image.get("data"), str)+ and isinstance(image.get("mimeType"), str)+ )+++# Deny known non-image markup. An image-format allowlist could miss formats+# Pillow supports and under-reserve KV for images promotion actually sends;+# false positives here only over-reserve.+_UNDECODABLE_PREFIXES = (+ b"<svg",+ b"<?xml",+ b"<!DOCTYPE",+ b"<html",+ b"{",+ b"[",+)+++def _mime_is_bounded(image: Any) -> bool:+ """Metadata is not where megabytes may hide from the byte budgets on either side;+ an entry whose mimeType runs past the bound is not a picture this path sends."""+ mime = image.get("mimeType") if isinstance(image, dict) else None+ return not (isinstance(mime, str) and len(mime) > MAX_MCP_IMAGE_MIME_CHARS)+++def probably_decodable(image: Any) -> bool:+ """Whether this entry could become a picture.++ Header sniff only: admission runs per request and must not decode rasters to+ price them. Answers True unless the payload is plainly not an image, so a+ format Pillow can open is never charged nothing.+ """+ data = image.get("data") if isinstance(image, dict) else None+ if not isinstance(data, str) or not data:+ return False+ if not _mime_is_bounded(image):+ return False+ try:+ head = base64.b64decode(data[:32], validate = False)+ except (binascii.Error, ValueError, TypeError):+ return False+ if not head:+ return False+ return not head.lstrip()[:16].startswith(_UNDECODABLE_PREFIXES)+++def count_probably_decodable(images: Sequence[dict]) -> int:+ return sum(1 for image in images if probably_decodable(image))+++def text_before_envelope(result: str) -> str:+ """The text with a trailing envelope cut off, WITHOUT parsing it -- for display+ only. split_images json-loads the whole array to decide whether the suffix is a+ real envelope; a monitor row built on the event loop cannot afford that on 12 MB,+ and showing a little less of a malformed suffix costs nothing."""+ index = result.rfind("\n" + SENTINEL)+ return result if index == -1 else result[:index]+++def has_images(result: str) -> bool:+ return bool(split_images(result)[1])+++def mentions_images(result: str) -> bool:+ """Substring only, for deciding WHERE to run: has_images json-parses the whole+ array, and doing that on the event loop to decide whether to leave the event+ loop parsed a permitted 12 MB envelope right there. A false positive here costs+ a thread hop; the worker validates for real."""+ return ("\n" + SENTINEL) in result+++def _decoded_urls(+ images: Sequence[dict],+ limit: int = MAX_MODEL_IMAGES,+ attempts: "list | None" = None,+ cache: "dict | None" = None,+) -> list[str]:+ """Up to *limit* data URLs, counting only what decoded.++ Slicing first would spend the quota on formats Pillow cannot read -- an SVG+ _flatten_result accepted, say -- and drop the real PNGs behind them. So the+ ATTEMPTS are bounded instead: the payload-byte budget caps the size of a result,+ never the number of entries in it, and a server answering with thousands of tiny+ unreadable ones would otherwise hold an inference worker open on Pillow for a+ turn that can show four pictures at most. Same allowance the replay side spends,+ for the same reason -- enough slack to look past a run of rejects.++ *attempts* is a one-element budget shared across a parallel batch. Per result it+ resets, and a 25-call turn of malformed results then buys 25 x 8 decodes of+ attacker-chosen rasters against a cap of eight pictures.+ """+ urls = []+ own = [limit + DECODE_FAILURE_ALLOWANCE] if attempts is None else attempts+ for image in images:+ if len(urls) >= limit or own[0] <= 0:+ break+ own[0] -= 1+ if not _mime_is_bounded(image):+ continue+ data = image.get("data", "")+ # *cache* is one request's decodes, keyed by the payload: a route that+ # promotes the same history twice must not pay Pillow twice for it.+ if cache is not None and data in cache:+ url = cache[data]+ else:+ url = _png_data_url(data)+ if cache is not None:+ cache[data] = url+ if url:+ urls.append(url)+ return urls+++def _decoded_urls_per_result(results: Sequence[Sequence[dict]]) -> list[str]:+ """MAX_MODEL_IMAGES from EACH result, then the conversation cap over the whole.++ A parallel batch arrives as several results concatenated. Applying the+ per-result quota to the concatenation gives the first result the whole+ allowance and delivers none of the second, though the conversation cap has+ room for both.++ Filled from the NEWEST result back when the batch cannot fit. Everything else+ that bounds these pictures keeps the newest -- the conversation trim, and the+ frontend's replay bound -- so filling from the front would show this turn+ results 1 and 2 and the next turn results 2 and 3, off the same history.+ """+ chosen: list[list[str]] = []+ room = MAX_TOTAL_MODEL_IMAGES+ # One budget for the whole batch, not one per result: room only falls on a+ # SUCCESSFUL decode, so results that fail late in Pillow never close the loop+ # and a parallel turn could pay for the allowance again on every call it made.+ attempts = [MAX_TOTAL_MODEL_IMAGES + DECODE_FAILURE_ALLOWANCE]+ for images in reversed(results):+ if room <= 0 or attempts[0] <= 0:+ break+ urls = _decoded_urls(images, min(MAX_MODEL_IMAGES, room), attempts = attempts)+ room -= len(urls)+ chosen.append(urls)+ # Back into document order: the parts are positional and a batch's own results+ # must still read in the order the model made the calls.+ return [url for urls in reversed(chosen) for url in urls]+++def eligible_replay_images(+ messages: Sequence[dict],+ *,+ local: bool = False,+ budget: int = MAX_TOTAL_MODEL_IMAGES,+) -> dict:+ """Which envelope entries can still be in the prompt once the cap has run.++ The decoders are the expensive part -- a permitted raster is 40 megapixels and+ a few hundred bytes of base64 can ask for one -- and until now every envelope in+ a replayed history was decoded before the eight-image trim threw nearly all of+ it away. A caller that posts its own history therefore chose how much Pillow+ work one bounded request did. This picks the survivors first, from the message+ list alone, so the count of decodes is a property of the cap rather than of the+ history.++ Keyed by position in *messages*; the value is how many leading entries of that+ result may be decoded. Mirrors the frontend's boundMcpImageEnvelopes, spare+ allowance included: this side cannot know which entries decode either, so a+ result whose images all fail must not strand the valid ones behind it.++ *local* budgets the way the marker paths spend: a tool batch -- consecutive+ results -- lands as one turn carrying one picture, so the batch is charged once+ and its further candidates are decode fallbacks, not a charge. Budgeting four per+ result here spent the conversation's allowance on pictures the local path never+ sends and dropped older batches that still had room.+ """+ eligible: dict = {}+ # Keep fallback candidates because header-only selection cannot know which+ # images decode. With no image budget, failures cannot free room for spares.+ spare = DECODE_FAILURE_ALLOWANCE if budget > 0 else 0+ per_result = LOCAL_MAX_IMAGES_PER_TURN if local else MAX_MODEL_IMAGES+ # Use promotion's provenance rules so unnamed non-MCP results cannot consume+ # the replay allowance ahead of genuine MCP results.+ call_names = resolve_tool_names(messages)++ def _is_tool(position: int) -> bool:+ message = messages[position]+ return isinstance(message, dict) and message.get("role") == "tool"++ def _mcp_images_at(position: int) -> "list | None":+ content = messages[position].get("content")+ if not isinstance(content, str):+ return None+ name = messages[position].get("name") or call_names.get(position)+ if isinstance(name, str) and name and not name.startswith(MCP_TOOL_PREFIX):+ return None+ _text, images = split_images(content)+ return images or None++ index = len(messages) - 1+ while index >= 0:+ if not _is_tool(index):+ index -= 1+ continue+ start = index+ if local:+ while start > 0 and _is_tool(start - 1):+ start -= 1+ # Newest first within the batch, the order the local decoder tries them.+ batch = [position for position in range(index, start - 1, -1)]+ index = start - 1+ room = min(budget, per_result)+ allowance = room + spare+ taken = 0+ for position in batch:+ images = _mcp_images_at(position)+ if images is None:+ continue+ take = min(len(images), allowance)+ if take <= 0:+ eligible[position] = 0+ continue+ eligible[position] = take+ allowance -= take+ taken += take+ charged = min(taken, room)+ budget -= charged+ spare -= taken - charged+ return eligible+++def content_parts(images: Sequence[dict]) -> list[dict]:+ return [{"type": "image_url", "image_url": {"url": url}} for url in _decoded_urls(images)]+++def content_parts_per_result(results: Sequence[Sequence[dict]]) -> list[dict]:+ """content_parts for a batch, keeping each result's own quota."""+ return [+ {"type": "image_url", "image_url": {"url": url}}+ for url in _decoded_urls_per_result(results)+ ]+++def png_payloads_per_result(+ results: Sequence[Sequence[dict]], cache: "dict | None" = None+) -> list[str]:+ """For the local marker paths: at most LOCAL_MAX_IMAGES_PER_TURN pictures, taken+ from the NEWEST result that decodes, since a batch lands as one turn and a+ non-GGUF message takes one image."""+ # One attempt budget across the batch, as _decoded_urls_per_result keeps: reset+ # per result, a 25-call turn of malformed results bought 25 allowances of Pillow+ # decodes for a path that keeps a single picture.+ attempts = [LOCAL_MAX_IMAGES_PER_TURN + DECODE_FAILURE_ALLOWANCE]+ for images in reversed(list(results)):+ if attempts[0] <= 0:+ break+ urls = _decoded_urls(images, LOCAL_MAX_IMAGES_PER_TURN, attempts = attempts, cache = cache)+ if urls:+ return [url.split(",", 1)[1] for url in urls]+ return []+++def flattened_rgb(image):+ """RGB with any transparency composited onto white, not simply dropped.++ ``convert("RGB")`` keeps whatever colour sits UNDER the alpha, and a tool that+ never painted a background leaves that black -- so a transparent screenshot's+ dark text or line art converts to black on black and the model is handed a+ blank rectangle. Only images that actually carry alpha take the composite.+ """+ from PIL import Image++ # Match routes/inference.py's _image_bytes_to_png_b64: scale declared I;16+ # values to 8-bit before RGB conversion clips them. I;16B/I;16L must pass+ # through "I" because they reject point().+ if image.mode.startswith("I;16"):+ if image.mode != "I;16":+ image = image.convert("I")+ image = image.point(lambda v: v * (1.0 / 257), mode = "L")+ has_alpha = image.mode in ("RGBA", "LA", "PA") or (+ image.mode == "P" and "transparency" in image.info+ )+ if not has_alpha:+ return image.convert("RGB")+ rgba = image.convert("RGBA")+ canvas = Image.new("RGB", rgba.size, (255, 255, 255))+ canvas.paste(rgba, mask = rgba.getchannel("A"))+ return canvas+++def _png_data_url(data: str) -> str | None:+ # PNG regardless of what the server sent: llama-server's stb_image reads only+ # a few formats, and MCP servers commonly answer with WebP.+ try:+ raw = base64.b64decode(data, validate = True)+ except (binascii.Error, ValueError, TypeError):+ logger.debug("MCP image payload is not base64")+ return None+ try:+ from PIL import Image++ # open() parses the header only, so the size is known before the raster+ # exists. load() below is what allocates it.+ image = Image.open(io.BytesIO(raw))+ width, height = image.size+ if width * height > MAX_IMAGE_PIXELS:+ logger.debug("MCP image is %dx%d, past the pixel budget", width, height)+ return None+ # JPEG decodes straight to a smaller raster; a no-op for every other format.+ image.draft("RGB", (MAX_IMAGE_EDGE, MAX_IMAGE_EDGE))+ image.load()+ # A camera JPEG carries its display orientation in EXIF; re-encoded as PNG+ # without applying it, the model was shown the picture sideways.+ from PIL import ImageOps++ image = ImageOps.exif_transpose(image) or image+ if max(image.size) > MAX_IMAGE_EDGE:+ image.thumbnail((MAX_IMAGE_EDGE, MAX_IMAGE_EDGE), Image.Resampling.LANCZOS)+ buffer = io.BytesIO()+ flattened_rgb(image).save(buffer, format = "PNG")+ except Exception:+ logger.debug("MCP image could not be decoded", exc_info = True)+ return None+ return "data:image/png;base64," + base64.b64encode(buffer.getvalue()).decode("ascii")+++def png_payloads(images: Sequence[dict]) -> list[str]:+ """Normalized PNG base64, for backends that take images as objects rather+ than as data URLs inside the prompt."""+ return [url.split(",", 1)[1] for url in _decoded_urls(images)]+++def _turn_text(+ shown: int,+ total: int,+ lead: str = IMAGE_TURN_TEXT,+) -> str:+ # The tool result's own note counts every image it returned, so a turn that+ # carries fewer has to say so rather than let the model wait for the rest.+ if total > shown:+ # Report counts: batches keep newest results and trims remove oldest parts,+ # so a positional label could identify images the model never saw.+ return f"{lead} ({shown} of {total})"+ return lead+++def placeholder_turn(+ count: int,+ total: "int | None" = None,+ lead: str = IMAGE_TURN_TEXT,+) -> dict:+ """The user turn a local processor renders: ``{"type": "image"}`` markers the+ template turns into image tokens, with the pixels passed alongside."""+ return {+ "role": "user",+ "content": [+ *({"type": "image"} for _ in range(count)),+ {+ "type": "text",+ "text": _turn_text(count, count if total is None else total, lead),+ },+ ],+ }+++def _relabelled(+ kept: list,+ part_type: str,+ original: int,+ owned: "set | None" = None,+) -> list:+ """The turn's note rewritten for what actually survived a partial trim.++ The label exists to tell the model which of the returned images it was really+ shown, so a turn left holding two while still saying it carries four defeats it.+ *owned* names the promoted parts; the caller's own attachments in the same turn+ are not the tool's and are not counted into its note.+ """+ remaining = sum(+ 1+ for part in kept+ if isinstance(part, dict)+ and part.get("type") == part_type+ and (owned is None or id(part) in owned)+ )+ out = []+ for part in kept:+ if (+ isinstance(part, dict)+ and part.get("type") == "text"+ and _is_image_turn_note(part.get("text"))+ ):+ lead = (+ DETACHED_IMAGE_TURN_TEXT+ if str(part.get("text", "")).startswith(DETACHED_IMAGE_TURN_TEXT)+ else IMAGE_TURN_TEXT+ )+ total = _note_total(part.get("text"), original)+ out.append({**part, "text": _turn_text(remaining, total, lead)})+ continue+ out.append(part)+ return out+++def _note_total(text, original: int) -> int:+ """How many the tool returned, for a turn that no longer carries them all.++ The note's own "of N" wins, so a second trim does not re-baseline to whatever is+ left. A turn that arrived COMPLETE carries no such suffix, and falling back to+ the post-trim count there told the model nothing had been dropped -- the one+ thing the note exists to say. Fall back to what the turn held before this trim.+ """+ match = re.search(r"\((?:first )?\d+ of (\d+)\)\s*$", str(text or ""))+ if match:+ return int(match.group(1))+ return original+++def _is_image_turn_note(text) -> bool:+ """Either lead the placeholder turn can carry, so a turn whose last picture went+ is recognised as having nothing left to say on both paths."""+ value = str(text or "")+ return value.startswith(IMAGE_TURN_TEXT) or value.startswith(DETACHED_IMAGE_TURN_TEXT)+++def _image_parts(conversation: Sequence[dict], part_type: str):+ for message in conversation:+ content = message.get("content")+ if isinstance(content, list):+ yield from (+ part for part in content if isinstance(part, dict) and part.get("type") == part_type+ )+++def _all_image_url_parts(conversation: Sequence[dict]) -> list:+ return list(_image_parts(conversation, "image_url"))+++def count_image_parts(conversation: Sequence[dict], part_type: str) -> int:+ return sum(1 for _ in _image_parts(conversation, part_type))+++def _drop_oldest_image_parts(+ conversation: list,+ excess: int,+ part_type: str,+ only: "list | None" = None,+) -> None:+ """Drop the *excess* oldest image parts, and any turn they emptied.++ With *only*, a list of the exact part objects promotion created, nothing else+ is touched: a caller that attaches more than the cap keeps every one of its own+ images, which admission has already charged for.+ """+ owned = {id(part) for part in only} if only is not None else None+ ordinals = set()+ for ordinal, part in enumerate(_image_parts(conversation, part_type)):+ if len(ordinals) >= excess:+ break+ if owned is None or id(part) in owned:+ ordinals.add(ordinal)+ _drop_image_parts_at(conversation, ordinals, part_type, owned = owned)+++def _drop_image_parts_at(+ conversation: list,+ ordinals: set,+ part_type: str,+ *,+ owned: "set | None" = None,+) -> None:+ """Drop the image parts at the given positions in document order.++ Positional rather than oldest-first: a protected pixel can sit anywhere in the+ sink, because the attachment's marker belongs to the turn that supplied it and+ that turn can precede a tool's pictures. Dropping by count would take the wrong+ marker and hand every later pixel to the one before it.+ """+ if not ordinals:+ return+ seen = 0+ drained = []+ for index, message in enumerate(conversation):+ content = message.get("content")+ if not isinstance(content, list):+ continue+ original = sum(+ 1+ for part in content+ if isinstance(part, dict)+ and part.get("type") == part_type+ and (owned is None or id(part) in owned)+ )+ kept = []+ for part in content:+ if isinstance(part, dict) and part.get("type") == part_type:+ ordinal = seen+ seen += 1+ if ordinal in ordinals:+ continue+ kept.append(part)+ if len(kept) == len(content):+ continue+ if not kept or (+ len(kept) == 1+ and kept[0].get("type") == "text"+ and _is_image_turn_note(kept[0].get("text"))+ ):+ drained.append(index)+ else:+ conversation[index] = {+ **message,+ "content": _relabelled(kept, part_type, original, owned = owned),+ }+ for index in reversed(drained):+ del conversation[index]+++def trim_image_turns(+ conversation: list,+ payloads: list,+ limit: int = MAX_TOTAL_MODEL_IMAGES,+ keep: "Sequence[int] | None" = None,+) -> tuple:+ """Keep the newest *limit* pictures: a loop that keeps calling an image tool+ otherwise re-sends every one it has seen. Markers and their own pixels go+ together, or the processor counts image tokens it was given none for.++ *keep* holds indexes into *payloads* that must survive -- the caller's own+ attachment, which this cap has no business evicting: it is about what a tool+ loop RE-SENDS, and dropping the picture the question was asked about answers a+ different question. Bare markers carry no identity the way promoted+ ``image_url`` parts do, so the position in the sink is what names them.++ They still COUNT against *limit*. Exempting them would let a resumed chat's+ replayed pictures carry a second full allowance and put twice the cap in the+ prompt; the route reserves the attachment's slot by trimming replay to+ ``limit - 1`` before interleaving it, which is where that reservation belongs.++ Returns *keep* rebased onto the payload list this call leaves behind.+ """+ protected = sorted(index for index in (keep or ()) if 0 <= index < len(payloads))+ excess = len(payloads) - limit+ if excess <= 0:+ return tuple(protected)+ drop = [index for index in range(len(payloads)) if index not in set(protected)][:excess]+ if not drop:+ return tuple(protected)+ _drop_image_parts_at(conversation, set(drop), "image")+ for index in reversed(drop):+ del payloads[index]+ # Rebase protected positions after deletion; callers reuse them on later+ # trims, where stale indices could leave the attachment unprotected.+ dropped = set(drop)+ return tuple(index - sum(1 for gone in dropped if gone < index) for index in protected)+++def trim_image_url_turns(+ conversation: list,+ limit: int = MAX_TOTAL_MODEL_IMAGES,+ only: "list | None" = None,+) -> None:+ """The same cap where the pixels ride in the prompt as data URLs.++ GGUF and the external providers carry no separate payload list, so the parts+ themselves are the budget: without this a screenshot loop resends every+ picture it has ever taken on every later turn.++ *only* scopes the cap to the parts promotion created. This cap is about what+ REPLAY re-sends; a caller's own attachments are not counted against it and are+ never deleted by it.+ """+ if only is not None:+ # Prune before counting: the rolling-context fitter may have evicted whole+ # turns, and stale parts would cause surviving images to be over-trimmed.+ present = {id(part) for part in _all_image_url_parts(conversation)}+ only[:] = [part for part in only if id(part) in present]+ counted = len(only) if only is not None else count_image_parts(conversation, "image_url")+ excess = counted - limit+ if excess <= 0:+ return+ _drop_oldest_image_parts(conversation, excess, "image_url", only = only)+ if only is not None:+ # Drop what the trim removed, or the next call counts parts that are no+ # longer in the conversation and cuts far more than the cap asks for.+ still_present = {id(part) for part in _all_image_url_parts(conversation)}+ only[:] = [part for part in only if id(part) in still_present]+++def _merge_into_trailing_user_turn(conversation: list, parts: list[dict]) -> bool:+ """Fold *parts* into a trailing ``role=user`` turn, if there is one.++ A deferred no-op nudge lands as exactly such a turn, and appending after it+ would put two user messages in a row -- which a strict VLM template rejects.+ """+ last = conversation[-1] if conversation else None+ if not isinstance(last, dict) or last.get("role") != "user":+ return False+ content = last.get("content")+ own = list(content) if isinstance(content, list) else [{"type": "text", "text": content or ""}]+ conversation[-1] = {**last, "content": [*own, *parts]}+ return True+++def append_image_turn(+ conversation: list,+ images: Sequence,+ *,+ limit: "int | None" = MAX_TOTAL_MODEL_IMAGES,+ per_result: bool = False,+ owned: "list | None" = None,+ reserve_caller_images: bool = False,+ returned: "int | None" = None,+ lead: str = IMAGE_TURN_TEXT,+) -> None:+ """A user turn, not the ``role=tool`` result they came with: tool messages take+ no image parts, and local templates render tool content as a string.++ With *per_result*, *images* is a list of per-result lists and each keeps its own+ MAX_MODEL_IMAGES quota; flattened first, a parallel batch would spend the whole+ allowance on its first call.+ """+ parts = content_parts_per_result(images) if per_result else content_parts(images)+ if not parts:+ return+ # What the TOOL returned. The caller passes it when the candidates it hands over+ # have already been sliced by admission, since counting those would describe the+ # admission pass rather than the result the note sits beside.+ total = (+ returned+ if returned is not None+ else (sum(len(result) for result in images) if per_result else len(images))+ )+ if owned is not None:+ # Everything this loop has appended, across turns. The cap counts what the+ # tools returned and never a caller's own attachments, which admission has+ # already charged for and which are not this cap's business.+ owned.extend(parts)+ # The note rides along on the merge too: merged bare into a deferred no-op nudge,+ # the pictures read as attachments to that nudge rather than as the tool's output,+ # and the next turn misattributes them.+ note = {"type": "text", "text": _turn_text(len(parts), total, lead)}+ if not _merge_into_trailing_user_turn(conversation, [*parts, note]):+ conversation.append(+ {+ "role": "user",+ "content": [+ {"type": "text", "text": _turn_text(len(parts), total, lead)},+ *parts,+ ],+ }+ )+ if limit is not None:+ if reserve_caller_images:+ # Remote providers cap all images in document order, so reserve attachment+ # slots or their cap may drop the newest tool result. Replay does the same.+ # This is opt-in: local loops use a context window instead of a provider cap.+ limit = max(0, limit - (len(_all_image_url_parts(conversation)) - len(owned or ())))+ trim_image_url_turns(conversation, limit, only = owned)+++def insert_placeholder_turn(+ conversation: list,+ index: int,+ count: int,+ total: "int | None" = None,+ lead: str = IMAGE_TURN_TEXT,+) -> None:+ """A marker-only user turn placed *before* ``index``.++ The pixels arrive history-first with the current attachment last, and a+ positional VLM processor binds them to markers in document order. So the+ replayed markers have to sit ahead of the turn that owns the attachment,+ not after it.+ """+ if count > 0:+ conversation.insert(index, placeholder_turn(count, total, lead))+++def append_placeholder_turn(+ conversation: list,+ count: int,+ total: "int | None" = None,+ lead: str = IMAGE_TURN_TEXT,+) -> None:+ """The marker-only form of the above, for backends taking pixels alongside.++ The note rides along on the merge as well. Merged bare into a deferred no-op+ nudge, the markers read as pictures attached to that nudge rather than as the+ tool's output, and the next turn misattributes them; merged into a real question+ they read as ones the user sent.+ """+ markers = [{"type": "image"} for _ in range(count)]+ if not markers:+ return+ note = {"type": "text", "text": _turn_text(count, count if total is None else total, lead)}+ if not _merge_into_trailing_user_turn(conversation, [*markers, note]):+ conversation.append(placeholder_turn(count, total, lead))+++def _without_replay_parts(content: list) -> list:+ """The turn without a replay's marker AND its note. A non-GGUF message takes one+ image, so the attachment displaces a replay marker merged into its turn; the+ note that came with it would then say the caller's own picture was the tool's."""+ return [+ part+ for part in content+ if not (+ isinstance(part, dict)+ and (+ part.get("type") == "image"+ or (part.get("type") == "text" and _is_image_turn_note(part.get("text")))+ )+ )+ ]+++def top_up_image_markers(+ messages: Sequence[dict],+ total: int,+ *,+ ordinal: "int | None" = None,+) -> list[dict]:+ """Give the conversation exactly *total* image markers, adding any shortfall+ to the newest user turn.++ ``messages_with_attached_image`` leaves a conversation that already carries+ markers alone, which is right for a nudge retry but wrong for replayed MCP+ pictures: those markers belong to earlier turns, not to the attachment. The+ top-up goes last, matching where the attachment sits in the pixel list.+ """+ out = list(messages)+ have = sum(+ 1+ for message in out+ if isinstance(message, dict) and isinstance(message.get("content"), list)+ for part in message["content"]+ if isinstance(part, dict) and part.get("type") in ("image", "image_url", "input_image")+ )+ missing = total - have+ if missing <= 0:+ return out+ markers = [{"type": "image"} for _ in range(missing)]+ if ordinal is not None:+ # The turn that supplied the attachment, which need not be the newest: a+ # later text-only question would otherwise be shown as carrying it.+ seen = 0+ for index, message in enumerate(out):+ if not isinstance(message, dict) or message.get("role") != "user":+ continue+ if is_synthetic_image_turn(message):+ continue+ if seen == ordinal:+ out[index] = _with_attachment_markers(message, markers, after_text = True)+ return out+ seen += 1+ # The legacy top-level image has no ordinal; use the newest real user turn.+ # Displace any replay marker there to preserve the non-GGUF one-image limit.+ candidates = [+ index+ for index in range(len(out) - 1, -1, -1)+ if isinstance(out[index], dict) and out[index].get("role") == "user"+ ]+ real = [index for index in candidates if not is_synthetic_image_turn(out[index])]+ for index in real or candidates:+ out[index] = _with_attachment_markers(out[index], markers, after_text = True)+ break+ return out+++def image_marker_parts(conversation: Sequence[dict]) -> list:+ """Every ``{"type": "image"}`` marker part, in document order."""+ return list(_image_parts(conversation, "image"))+++def pixels_in_marker_order(+ conversation: Sequence[dict],+ prior_markers: Sequence[dict],+ prior_payloads: Sequence,+ new_payload,+ placed_at: "list | None" = None,+) -> list:+ """The pixel list ordered the way its markers actually appear.++ A positional VLM binds the Nth pixel to the Nth marker, so "history first,+ attachment last" is only right when the attachment's turn is last. It is not+ when the attachment came with an earlier question and a tool returned pictures+ after it, so the order is read off the conversation rather than assumed.+ """+ # Each history payload rides with ITS marker, by identity. Popping from the front+ # let a displaced marker's payload slide onto the marker after it, which is the+ # off-by-one the whole positional scheme exists to prevent.+ by_marker = {id(part): payload for part, payload in zip(prior_markers, prior_payloads)}+ ordered = []+ placed_new = False+ for part in image_marker_parts(conversation):+ # A marker that predates the top-up belongs to history when history has a+ # pixel for it. A pre-existing marker with none is the attachment's own --+ # the client may have marked it before the request ever reached this path.+ if id(part) in by_marker:+ ordered.append(by_marker[id(part)])+ elif not placed_new:+ # Where the attachment landed, so the loop's cap can leave it alone: it+ # is not always last, and the sink's positions are all that name it.+ if placed_at is not None:+ placed_at.append(len(ordered))+ ordered.append(new_payload)+ placed_new = True+ return ordered+++def is_synthetic_image_turn(message) -> bool:+ """Whether promotion inserted this user turn, rather than the caller sending it.++ An ordinal computed against the ORIGINAL history counts only real user turns,+ so resolving it against the promoted list has to skip the turns promotion added+ or the attachment's marker lands on a historical picture's turn.+ """+ if not isinstance(message, dict) or message.get("role") != "user":+ return False+ content = message.get("content")+ if not isinstance(content, list):+ return False+ # Both leads: a detached block is promotion's turn just as much, and counting it+ # as a real user turn put the attachment's marker on it instead of the question.+ # ALL of its text, though: a replay merged into the user's question carries the+ # note beside the question's own text, and that turn is still the user's.+ texts = [+ part.get("text")+ for part in content+ if isinstance(part, dict) and part.get("type") == "text"+ ]+ return bool(texts) and all(_is_image_turn_note(text) for text in texts)+++def prepare_image_turn_boundaries(messages: list, template: str | None) -> list:+ """Balance inserted image turns for Ministral's tool-skipping alternation check."""+ if not isinstance(template, str) or (+ "conversation roles must alternate user and assistant roles except for tool calls and results."+ not in template+ ):+ return messages+ out = []+ expects_user = True+ changed = False+ for message in messages:+ role = message.get("role")+ if role == "user":+ if not expects_user and is_synthetic_image_turn(message):+ out.append({"role": "assistant", "content": "The tool returned image content."})+ changed = True+ expects_user = False+ elif role == "assistant" and not message.get("tool_calls"):+ expects_user = True+ out.append(message)+ return out if changed else messages+++def _with_attachment_markers(+ message: dict,+ markers: list[dict],+ *,+ after_text: bool = False,+) -> dict:+ """Replace a replay's marker and attribution with the caller's attachment."""+ content = message.get("content")+ own = (+ _without_replay_parts(content)+ if isinstance(content, list)+ else [{"type": "text", "text": content or ""}]+ )+ parts = [*own, *markers] if after_text and isinstance(content, list) else [*markers, *own]+ return {**message, "content": parts}+++def mark_last_user_turn(+ messages: Sequence[dict],+ count: int,+ *,+ ordinal: "int | None" = None,+) -> list[dict]:+ """Mark the user turn carrying ``count`` images, where an attachment belongs.++ *ordinal* names that turn counted among user turns. The newest user turn is+ only the right guess when the attachment came with the newest question: the+ extractor takes the newest user image from anywhere in the thread, so a+ text-only latest turn would otherwise be told it supplied an older picture.++ A replay marker already merged into that turn is displaced: a non-GGUF message+ takes one image, and the attachment is the one the question is about. The+ displaced payload drops in pixels_in_marker_order rather than sliding on.+ """+ out = list(messages)+ markers = [{"type": "image"} for _ in range(count)]+ if ordinal is not None:+ seen = 0+ for index, message in enumerate(out):+ # Promotion inserts its own user turns ahead of this, and the ordinal was+ # counted before they existed.+ if message.get("role") != "user" or is_synthetic_image_turn(message):+ continue+ if seen == ordinal:+ out[index] = _with_attachment_markers(message, markers)+ return out+ seen += 1+ for index in range(len(out) - 1, -1, -1):+ if out[index].get("role") == "user":+ out[index] = _with_attachment_markers(out[index], markers)+ break+ return out+++def promote_history(+ messages: Sequence[dict],+ *,+ vision: bool,+ promoted_out: "list | None" = None,+ reserve_for_caller: bool = False,+) -> list[dict]:+ """Rebuild image turns from replayed envelopes. The envelope leaves the tool+ text either way: a text-only model must not be shown its base64.++ *promoted_out* collects the exact image parts this call created, so a tool loop+ resuming the conversation can seed its own cap with them instead of starting+ from zero and letting the history's images through uncounted.+ """+ out, _payloads, promoted = _promote(+ messages, vision, local = False, reserve_for_caller = reserve_for_caller+ )+ if promoted_out is not None:+ promoted_out.extend(promoted)+ return out+++def promote_history_local(+ messages: Sequence[dict],+ *,+ vision: bool,+ decode_cache: "dict | None" = None,+ caller_images: Sequence = (),+) -> tuple[list[dict], list[str]]:+ """The same, for backends that take the pixels beside the prompt: the turns+ carry markers and the payloads come back with them. ``caller_images`` supplies+ the pixels for existing markers, which retain their positions and survive trimming."""+ out, payloads, _promoted = _promote(+ messages, vision, local = True, decode_cache = decode_cache, caller_images = caller_images+ )+ return out, payloads+++def _field(message, key):+ return message.get(key) if isinstance(message, dict) else getattr(message, key, None)+++def resolve_tool_names(messages: Sequence) -> dict:+ """Which tool each role="tool" message answers, keyed by its POSITION.++ A tool message carries no ``name`` in plain OpenAI (the field is optional) or in+ anything translated from Anthropic, and an absent name is read as legacy MCP+ history that may be trusted. Correlating the call is what lets the mcp__ gate+ run on those wire formats at all.++ Each result is paired with the nearest UNMATCHED preceding call bearing its id,+ not with a conversation-wide last-wins lookup: the backend itself restarts ids+ like ``call_0`` every response, so one dict let a later non-MCP ``call_0``+ rename an earlier MCP result -- or the reverse.+ """+ open_calls: dict = {}+ names: dict = {}+ for index, message in enumerate(messages or ()):+ for call in _field(message, "tool_calls") or ():+ if not isinstance(call, dict):+ continue+ function = call.get("function")+ call_id = call.get("id")+ if isinstance(function, dict) and isinstance(call_id, str):+ name = function.get("name")+ if isinstance(name, str) and name:+ open_calls.setdefault(call_id, []).append(name)+ if _field(message, "role") == "tool":+ call_id = _field(message, "tool_call_id")+ stack = open_calls.get(call_id) if isinstance(call_id, str) else None+ if stack:+ names[index] = stack.pop()+ return names+++def _returned_count(images: Sequence[dict]) -> int:+ """How many the tool returned. An upstream bound may already have shortened the+ array; it leaves the original length on the first entry when it does."""+ first = images[0] if images else None+ stated = first.get("returned") if isinstance(first, dict) else None+ if isinstance(stated, int) and stated >= len(images):+ return stated+ return len(images)+++def _promote(+ messages,+ vision: bool,+ *,+ local: bool,+ reserve_for_caller: bool = False,+ decode_cache: "dict | None" = None,+ caller_images: Sequence = (),+) -> tuple[list[dict], list[str], list[dict]]:+ out: list[dict] = []+ caller_payloads = (+ dict(zip(map(id, image_marker_parts(messages)), caller_images)) if caller_images else {}+ )+ # Resolved once for the whole conversation, so the provenance gate below works on+ # every wire format rather than only the ones that happen to send ``name``.+ call_names = resolve_tool_names(messages)+ # Set while an image result waits for its turn and a LATER, image-free result of+ # the same batch is appended after it: "the tool call above" would then name the+ # wrong call, so the turn takes the wording that claims no adjacency.+ interrupted = [False]+ # Resolved before a single decode runs: the trim at the bottom keeps the newest+ # eight, and decoding a whole replayed history to throw nearly all of it away is+ # work a caller's own message list gets to choose the size of.+ # The caller's room is reserved HERE, ahead of the decodes, not only by the trim+ # at the bottom: with eight attachments every replay candidate was decoded and+ # re-encoded -- a permitted raster is 40 megapixels -- to be dropped whole.+ # Before promotion every image_url part in the list is the caller's own.+ _reserved = (+ len(caller_payloads)+ if local+ else (len(_all_image_url_parts(messages)) if vision and reserve_for_caller else 0)+ )+ eligible = (+ eligible_replay_images(+ messages, local = local, budget = max(0, MAX_TOTAL_MODEL_IMAGES - _reserved)+ )+ if vision+ else {}+ )+ # One entry per tool result, not flattened: two parallel calls each returning+ # four images would otherwise share a single result's quota and replay only the+ # first call's four.+ pending: list[list[dict]] = []+ # One per entry in *pending*: how many that result really returned.+ returned_totals: list[int] = []+ payloads: list[str] = []+ # The exact part objects promotion creates, so the cap below can leave a+ # caller's own attachments alone.+ promoted: list[dict] = []++ def flush(into: "dict | None" = None) -> "dict | None":+ if not pending or not vision:+ pending.clear()+ returned_totals.clear()+ return into+ returned = sum(returned_totals) or sum(len(result) for result in pending)+ # Detached wording for a batch of several results too, not only an interrupted+ # one: two parallel calls both returning pictures share the turn, and "the tool+ # call above" would hand every picture to whichever ran last. The loops apply+ # the same rule to a live batch.+ lead = DETACHED_IMAGE_TURN_TEXT if interrupted[0] or len(pending) > 1 else IMAGE_TURN_TEXT+ interrupted[0] = False+ if local:+ # the caller's attachment takes the single image slot on its turn.+ if into is not None and any(+ id(part) in caller_payloads for part in image_marker_parts([into])+ ):+ pending.clear()+ returned_totals.clear()+ return into+ encoded = png_payloads_per_result(pending, cache = decode_cache)+ pending.clear()+ returned_totals.clear()+ if not encoded:+ return into+ payloads.extend(encoded)+ markers = [{"type": "image"} for _ in encoded]+ if into is None:+ out.append(placeholder_turn(len(encoded), returned, lead))+ return None+ # The note rides along on the merge: merged bare into the question, the+ # markers read as pictures the user attached, and with a result of another+ # tool between, as that tool's output.+ note = {"type": "text", "text": _turn_text(len(encoded), returned, lead)}+ return _with_parts(into, [*markers, note])+ results = list(pending)+ pending.clear()+ returned_totals.clear()+ if into is None:+ before = {id(part) for part in _all_image_url_parts(out)}+ append_image_turn(+ out, results, per_result = True, limit = None, returned = returned, lead = lead+ )+ promoted.extend(part for part in _all_image_url_parts(out) if id(part) not in before)+ return None+ parts = content_parts_per_result(results)+ promoted.extend(parts)+ if not parts:+ return into+ note = {"type": "text", "text": _turn_text(len(parts), returned, lead)}+ return _with_parts(into, [*parts, note])++ for position, message in enumerate(messages):+ content = message.get("content")+ if message.get("role") == "tool" and isinstance(content, str):+ text, images = split_images(content)+ # The suffix always comes off -- it is megabytes of base64 and the model+ # must never read it as text. Provenance decides only whether it becomes+ # IMAGE input: a named non-MCP tool that happens to end in a valid+ # envelope is not one an MCP server served.+ name = message.get("name") or call_names.get(position)+ if isinstance(name, str) and name and not name.startswith(MCP_TOOL_PREFIX):+ # A non-MCP result sitting between the images and their turn makes+ # "the tool call above" name web_search or read_file.+ if pending:+ interrupted[0] = True+ out.append(+ {**message, "content": text or "[image returned]"} if images else message+ )+ continue+ if images:+ # Only the entries the cap can still admit. The suffix comes off the+ # text either way, above; this decides how many are decoded.+ admitted = images[: eligible.get(position, len(images))]+ if admitted:+ pending.append(admitted)+ # What the TOOL returned, which the slice above has already lost.+ # Summing the admitted candidates instead made a 100-image result+ # read "(4 of 8)" beside a tool result saying 100 -- the note+ # describing the admission pass rather than the tool. The frontend+ # bounds the envelope before it ever gets here and records the+ # count it started from on the first entry; honour that too.+ # Counted whether or not anything of it was admitted: a result the+ # allowance left nothing of is still part of what the batch returned,+ # and leaving it out read "(1 of 6)" for three results of three.+ returned_totals.append(_returned_count(images))+ elif pending:+ interrupted[0] = True+ out.append({**message, "content": text or "[image returned]"} if images else message)+ continue+ if pending and vision and message.get("role") == "user":+ # Merged, not inserted ahead of it: two user turns in a row is what+ # a strict template rejects.+ out.append(flush(message))+ continue+ flush()+ out.append(message)+ flush()+ # A replay carries every image turn the conversation ever had; the cap has to+ # hold here too or the whole history is re-sent on every later turn.+ if local:+ protected = ()+ if caller_payloads:+ markers = image_marker_parts(out)+ replay_payloads = dict(+ zip((id(part) for part in markers if id(part) not in caller_payloads), payloads)+ )+ by_marker = {**replay_payloads, **caller_payloads}+ payloads = [by_marker[id(part)] for part in markers]+ protected = tuple(+ index for index, part in enumerate(markers) if id(part) in caller_payloads+ )+ trim_image_turns(out, payloads, keep = protected)+ else:+ # The cap says attachments are never counted against it, which is right for+ # what THIS cap protects, and llama-server is bounded by its context window+ # rather than a fixed image count: a GGUF replay keeps the full allowance+ # beside the caller's picture, as its live loop did.+ #+ # A provider is different. It applies its own per-request cap in document+ # order, and promotion prepends the replay to the user turn: on Gemini (8+ # images, later ones dropped silently) eight replayed screenshots evicted the+ # picture the current question was about, and the model answered it from+ # stale tool output. So the external caller reserves the attachment's room,+ # the way the local route reserves its slot before interleaving it.+ _caller_parts = len(_all_image_url_parts(out)) - len(promoted) if reserve_for_caller else 0+ trim_image_url_turns(+ out,+ limit = max(0, MAX_TOTAL_MODEL_IMAGES - _caller_parts),+ only = promoted,+ )+ return out, payloads, promoted+++def _with_parts(message: dict, parts: list[dict]) -> dict:+ content = message.get("content")+ own = list(content) if isinstance(content, list) else [{"type": "text", "text": content or ""}]+ return {**message, "content": [*parts, *own]}diff --git a/studio/backend/core/inference/mlx_inference.py b/studio/backend/core/inference/mlx_inference.pyindex bae62f65fe8..0762d3a80b6 100644--- a/studio/backend/core/inference/mlx_inference.py+++ b/studio/backend/core/inference/mlx_inference.py@@ -44,6 +44,12 @@ from core.inference.chat_template_helpers import (trailing_assistant_text,vlm_prompt_issue as _vlm_prompt_issue,)+from core.inference.mcp_images import (+ image_marker_parts,+ pixels_in_marker_order,+ top_up_image_markers,+ trim_image_turns,+)from utils.models.model_config import is_audio_input_typefrom loggers import get_logger@@ -3217,6 +3223,8 @@ class MLXInferenceBackend:messages,system_prompt = "",image = None,+ images = None,+ image_ordinal = None,temperature = 0.7,top_p = 0.9,top_k = 40,@@ -3240,8 +3248,6 @@ class MLXInferenceBackend:# cannot tell that the wrappers below still have to survive decoding.tool_protocol_active = None,video = None,- *,- images = None,) -> Generator[str, None, None]:if self._model is None:raise RuntimeError("No model loaded")@@ -3254,19 +3260,17 @@ class MLXInferenceBackend:# Reset so a failed run cannot surface stale stats.self.last_generation_stats = None- images = list(images) if images else ([image] if image is not None else [])- if len(images) > 1 and not self._multi_image_marker:- raise ValueError(- f"'{self.active_model_name}' takes one image per request: its chat template does "- f"not mark {len(images)} images separately."- )+ full_messages = self._with_system_prompt(messages, system_prompt)+ # History first, the attachment last: MLX binds pixels to markers in order.+ attached = list(images or []) + ([image] if image is not None else [])# Shared with the transformers vision path so both render the same turns (#10092).- if self._is_vlm and (images or video is not None):+ if self._is_vlm and (attached or video is not None):# Processor templates want part lists, the tokenizer fallback wants strings.from core.inference.chat_template_helpers import (chat_render_target as _chat_render_target,)+_renders_via_processor = (self._processor is not Noneand _chat_render_target(self._processor) is self._processor@@ -3275,16 +3279,41 @@ class MLXInferenceBackend:messages,system_prompt = system_prompt,structured_content = _renders_via_processor,- image = len(images or ()),+ image = len(attached),video = video is not None,)- else:- full_messages = self._with_system_prompt(messages, system_prompt)+ # That helper leaves a conversation that already carries markers alone,+ # which is right for a retry but not for replayed MCP pictures: those+ # markers are not the attachment's, and the render would be one short+ # for two pixels.+ _prior_markers = image_marker_parts(full_messages)+ full_messages = top_up_image_markers(+ full_messages, len(attached), ordinal = image_ordinal+ )+ if image is not None:+ # Read off the conversation: the attachment's turn can precede a+ # replayed picture, and pixels bind to markers in document order.+ attached = pixels_in_marker_order(+ full_messages, _prior_markers, list(images or []), image+ )+ if len(attached) > 1 and not self._multi_image_marker:+ # The template marks one image, so replayed pictures cannot each have their+ # own. Serve the caller's attachment, or else the newest picture, with its marker.+ keep = next(+ (i for i, one in enumerate(attached) if one is image), len(attached) - 1+ )+ logger.info(+ "Serving 1 of %d images: '%s' marks one image per request.",+ len(attached),+ self.active_model_name,+ )+ full_messages = list(full_messages)+ trim_image_turns(full_messages, attached, limit = 1, keep = (keep,))if self._is_vlm:stream = self._generate_vlm(full_messages,- images,+ attached,temperature,top_p,top_k,diff --git a/studio/backend/core/inference/openai_codex_tool_loop.py b/studio/backend/core/inference/openai_codex_tool_loop.pyindex 03e39d2f848..8b0b51ecd9f 100644--- a/studio/backend/core/inference/openai_codex_tool_loop.py+++ b/studio/backend/core/inference/openai_codex_tool_loop.py@@ -36,6 +36,8 @@ class CodexRunContext:response_format: dict[str, Any] | None = Nonetool_choice: Any = Nonecontinue_final_message: bool = False+ supports_vision: bool = False+ promoted_image_parts: tuple = ()@dataclass(frozen = True)@@ -106,6 +108,8 @@ def stream_codex_with_studio_tools(model = run.model,tool_choice = run.tool_choice,continue_final_message = run.continue_final_message,+ supports_vision = run.supports_vision,+ promoted_image_parts = run.promoted_image_parts,),policy = ToolLoopPolicy(tools = policy.tools,diff --git a/studio/backend/core/inference/orchestrator.py b/studio/backend/core/inference/orchestrator.pyindex fb819812572..08725f71972 100644--- a/studio/backend/core/inference/orchestrator.py+++ b/studio/backend/core/inference/orchestrator.py@@ -329,11 +329,9 @@ def _summed_tool_loop_stats(total, turn):return summed-def _request_images(image, images):- """One request's images in render order; ``image`` is the older single-image spelling."""- if images:- return list(images)- return [image] if image is not None else []+def _encoded_images(images, to_base64) -> list:+ """Replayed MCP pictures arrive already PNG-encoded; a caller's decoded list does not."""+ return [one if isinstance(one, str) else to_base64(one) for one in images or ()]def _mirrored_model_entry(model_info: dict, model_name: str) -> dict:@@ -1222,8 +1220,10 @@ class InferenceOrchestrator:def _build_generate_cmd(self,request_id: str,- images_b64: list,+ image_b64: Optional[str],*,+ images_b64: Optional[list] = None,+ image_ordinal: Optional[int] = None,messages: list = None,system_prompt: str = "",temperature: float = 0.7,@@ -1252,7 +1252,9 @@ class InferenceOrchestrator:"request_id": request_id,"messages": messages or [],"system_prompt": system_prompt,- "images_base64": images_b64,+ "image_base64": image_b64,+ "images_base64": images_b64 or None,+ "image_ordinal": image_ordinal,"temperature": temperature,"top_p": top_p,"top_k": top_k,@@ -1454,6 +1456,8 @@ class InferenceOrchestrator:messages: list = None,system_prompt: str = "",image = None,+ images: Optional[list] = None,+ image_ordinal: Optional[int] = None,temperature: float = 0.7,top_p: float = 0.9,top_k: int = 40,@@ -1475,8 +1479,6 @@ class InferenceOrchestrator:logit_bias: Optional[dict] = None,stop: Optional[list] = None,video: Optional[str] = None,- *,- images = None,) -> Generator[str, None, None]:"""Dispatched generation, sending the command without holding _gen_lock. Uses a per-requestmailbox for tokens so two compare-mode requests can be queued at once. The subprocess@@ -1512,11 +1514,14 @@ class InferenceOrchestrator:request_id = str(uuid.uuid4())- images_b64 = [self._pil_to_base64(one) for one in _request_images(image, images)]+ image_b64 = self._pil_to_base64(image) if image is not None else None+ images_b64 = _encoded_images(images, self._pil_to_base64)cmd = self._build_generate_cmd(request_id,- images_b64,+ image_b64,+ images_b64 = images_b64,+ image_ordinal = image_ordinal,messages = messages,system_prompt = system_prompt,temperature = temperature,@@ -2296,6 +2301,8 @@ class InferenceOrchestrator:messages: list,system_prompt: str = "",image = None,+ images: Optional[list] = None,+ image_ordinal: Optional[int] = None,temperature: float = 0.7,top_p: float = 0.9,top_k: int = 40,@@ -2316,8 +2323,6 @@ class InferenceOrchestrator:logit_bias: Optional[dict] = None,stop: Optional[list] = None,video: Optional[str] = None,- *,- images = None,) -> Generator[str, None, None]:"""Generate response, streaming tokens from subprocess. ``tools`` / ``enable_thinking`` /``reasoning_effort`` / ``preserve_thinking`` are forwarded so the template can render@@ -2330,6 +2335,7 @@ class InferenceOrchestrator:system_prompt = system_prompt,image = image,images = images,+ image_ordinal = image_ordinal,temperature = temperature,top_p = top_p,top_k = top_k,@@ -2358,6 +2364,7 @@ class InferenceOrchestrator:messages: list,tools: list,system_prompt: str = "",+ images: Optional[list] = None,temperature: float = 0.7,top_p: float = 0.9,top_k: int = 40,@@ -2387,6 +2394,7 @@ class InferenceOrchestrator:stop: Optional[list] = None,reasoning_prefilled: bool = False,seed: Optional[int] = None,+ caller_image_indexes: "tuple[int, ...]" = (),**_unused,):"""Run the safetensors agentic tool loop in the parent process, calling the worker for each@@ -2397,6 +2405,13 @@ class InferenceOrchestrator:# None lets the backend size an unset limit once it has counted the prompt.max_new_tokens = max_tokens if max_tokens and max_tokens > 0 else None+ # Only a model that reads images gets a sink; the loop leaves MCP pictures+ # out of the prompt without one.+ loop_images: Optional[list] = (+ list(images or [])+ if self.models.get(self.active_model_name, {}).get("is_vision")+ else None+ )# The worker's usage for the LATEST turn only. Hoisted out of the turn so the loop can size a conversation# search against a real prompt count, and cleared on the way in rather than on each way out, so a turn that@@ -2417,6 +2432,7 @@ class InferenceOrchestrator:messages = conv,system_prompt = "",image = None,+ images = list(loop_images) if loop_images else None,temperature = temperature,top_p = top_p,top_k = top_k,@@ -2516,6 +2532,11 @@ class InferenceOrchestrator:context_length = _model_info.get("context_length"),max_tokens = max_new_tokens,generation_stats_holder = turn_stats,+ images_sink = loop_images,+ # Which sink entries are the caller's own attachment, so the loop's cap+ # never evicts it. Empty when the model reads no images, since there is+ # then no sink to protect anything in.+ caller_image_indexes = tuple(caller_image_indexes) if loop_images else (),)def generate_with_adapter_control(@@ -2552,6 +2573,8 @@ class InferenceOrchestrator:messages: list = None,system_prompt: str = "",image = None,+ images: Optional[list] = None,+ image_ordinal: Optional[int] = None,temperature: float = 0.7,top_p: float = 0.9,top_k: int = 40,@@ -2573,8 +2596,6 @@ class InferenceOrchestrator:logit_bias: Optional[dict] = None,stop: Optional[list] = None,video: Optional[str] = None,- *,- images = None,) -> Generator[str, None, None]:"""Inner generation logic: sends the command to the subprocess and yields tokens. Serializedby _gen_lock (one generation at a time) so concurrent readers don't consume each other's@@ -2599,10 +2620,13 @@ class InferenceOrchestrator:if cancel_event is not None and cancel_event.is_set():returnrequest_id = str(uuid.uuid4())- images_b64 = [self._pil_to_base64(one) for one in _request_images(image, images)]+ image_b64 = self._pil_to_base64(image) if image is not None else None+ images_b64 = _encoded_images(images, self._pil_to_base64)cmd = self._build_generate_cmd(request_id,- images_b64,+ image_b64,+ images_b64 = images_b64,+ image_ordinal = image_ordinal,messages = messages,system_prompt = system_prompt,temperature = temperature,diff --git a/studio/backend/core/inference/safetensors_agentic.py b/studio/backend/core/inference/safetensors_agentic.pyindex 24ba0ebcd3e..c288054bcca 100644--- a/studio/backend/core/inference/safetensors_agentic.py+++ b/studio/backend/core/inference/safetensors_agentic.py@@ -53,6 +53,13 @@ from core.tool_healing import (_think_spans_outside_tool_markup,strip_outside_think,)+from core.inference.mcp_images import (+ DETACHED_IMAGE_TURN_TEXT as MCP_DETACHED_IMAGE_TURN_TEXT,+ IMAGE_TURN_TEXT as MCP_IMAGE_TURN_TEXT,+ append_placeholder_turn,+ png_payloads_per_result,+ trim_image_turns,+)from core.inference.tool_loop_controller import (_WORKSPACE_TOOLS,ToolLoopController,@@ -600,6 +607,8 @@ def run_safetensors_tool_loop(context_length: Optional[int] = None,max_tokens: Optional[int] = None,generation_stats_holder: Optional[dict] = None,+ images_sink: Optional[list] = None,+ caller_image_indexes: "tuple[int, ...]" = (),) -> Generator[dict, None, None]:"""Drive an agentic tool loop on top of a cumulative-text generator.@@ -622,6 +631,12 @@ def run_safetensors_tool_loop(* ``{"type": "tool_end", "tool_name", "tool_call_id", "result"}``"""conversation = list(messages)+ # Where the caller's own attachment sits in the seeded sink. The cap is about what+ # the loop RE-SENDS, so that entry is never the one it drops -- but it is not the+ # whole seed: a resumed chat seeds replayed pictures too, and exempting those would+ # let the prompt carry a second full allowance. The route reserves the attachment's+ # slot by trimming replay to limit - 1 before interleaving it.+ caller_images = tuple(caller_image_indexes)# The branch this request is on, before the loop appends anything. A GGUF-compacted# thread keeps its archive across a switch to safetensors, so search_conversation is# advertised here too and needs the same filtering: the stored rows are the whole@@ -1354,6 +1369,10 @@ def run_safetensors_tool_loop(# Collect no-op nudges and flush them after the batch, so a no-op doesn't# abort it and drop the parallel calls that follow.deferred_noop_msgs: list = []+ # Per result; see append_image_turn's per_result note.+ batch_mcp_images: list = []+ # Where this batch's results start, for the image turn\'s wording.+ batch_conversation_start = len(conversation)for _call_index, tc in enumerate(tool_calls or []):func = tc.get("function", {}) or {}@@ -1364,6 +1383,12 @@ def run_safetensors_tool_loop(and tc.get("id", "") == provisional_render_html_id)decision = tool_controller.prepare_call(tc, provisional = provisional_match)+ # The frontend keeps a round's tool cards together by this id+ # (codexLocalToolRoundId) and otherwise flushes each completed pair on its+ # own. This loop shows a parallel batch as ONE picture, and replay groups+ # consecutive results the same way -- so without the id a batch persisted+ # as separate pairs replayed as several pictures on the next request.+ decision.provenance["round_id"] = iterationif not decision.should_execute:if content_text and not assistant_appended:@@ -1608,6 +1633,10 @@ def run_safetensors_tool_loop(_turn_executed_real_tool = Trueyield completion.tool_end_event()conversation.append(completion.tool_message())+ # Parsed only when there is a sink for them.+ _completion_images = completion.mcp_images() if images_sink is not None else []+ if _completion_images:+ batch_mcp_images.append(_completion_images)# The turn that ends the loop offers no tools again, so do not ask for a retry there.over_cap_final = bool(over_cap) and (@@ -1630,6 +1659,33 @@ def run_safetensors_tool_loop())append_deferred_nudges(conversation, deferred_noop_msgs)+ if batch_mcp_images and images_sink is not None:+ # A sink only when the loaded model reads images; the pixels ride+ # beside the prompt, so the turn carries markers rather than data.+ encoded = png_payloads_per_result(batch_mcp_images)+ if encoded:+ images_sink.extend(encoded)+ # Merged into the deferred nudge above when there was one: two+ # user turns in a row is what a strict VLM template rejects.+ # Same wording rule as the other loops: several results, and "the+ # tool call above" names whichever ran last.+ _batch_results = sum(+ 1+ for m in conversation[batch_conversation_start:]+ if isinstance(m, dict) and m.get("role") == "tool"+ )+ append_placeholder_turn(+ conversation,+ len(encoded),+ sum(len(r) for r in batch_mcp_images),+ lead = MCP_DETACHED_IMAGE_TURN_TEXT+ if _batch_results != 1+ else MCP_IMAGE_TURN_TEXT,+ )+ # Rebased on the way out: this trim deletes entries before the+ # attachment, and reusing the original index on the next batch+ # would protect the wrong payload and delete the attachment.+ caller_images = trim_image_turns(conversation, images_sink, keep = caller_images)yield {"type": "status", "text": ""}diff --git a/studio/backend/core/inference/studio_tool_loop.py b/studio/backend/core/inference/studio_tool_loop.pyindex 80d99859061..3be1beca53a 100644--- a/studio/backend/core/inference/studio_tool_loop.py+++ b/studio/backend/core/inference/studio_tool_loop.py@@ -55,6 +55,32 @@ from core.inference.tool_call_parser import (reprompt_to_act_message,strip_tool_markup,)+from core.inference.mcp_images import append_image_turn as append_mcp_image_turn+++def _append_mcp_images_owned(+ conversation,+ results,+ owned,+ lead = None,+):+ """append_image_turn with the loop's own part list, for asyncio.to_thread.++ Reserving here and not on the GGUF loop: this one talks to a remote provider that+ applies its own per-request image cap in document order, so an attachment beside a+ full allowance of tool results silently loses the newest result -- the one the+ model just asked for. llama-server is local and answers to the context window.+ """+ append_mcp_image_turn(+ conversation,+ results,+ per_result = True,+ owned = owned,+ reserve_caller_images = True,+ **({"lead": lead} if lead else {}),+ )++from core.inference.tool_loop_controller import (ToolLoopController,awaiting_approval_status,@@ -339,6 +365,11 @@ class ToolLoopRun:model: str | None = Nonetool_choice: Any = Nonecontinue_final_message: bool = False+ supports_vision: bool = False+ # Image parts an earlier promotion already put in `messages`. The loop's cap+ # counts these too: starting from zero lets a resumed image-heavy chat carry a+ # full history through and then add a whole batch on top.+ promoted_image_parts: tuple = ()@dataclass(frozen = True)@@ -1180,6 +1211,10 @@ async def stream_with_studio_tools() -> AsyncIterator[str]:"""Stream a provider, execute requested Unsloth tools, continue to a final answer."""conversation = [dict(message) for message in run.messages]+ # The image parts this run appends, so its cap never counts a caller's own+ # attachments. Run-scoped, not turn-scoped: the cap is across the whole loop,+ # and seeded with what promotion already put in the conversation.+ loop_mcp_image_parts: list = list(run.promoted_image_parts)# Kept before the loop appends anything: this is the branch the request is on.request_branch = list(run.messages)remaining = policy.max_calls@@ -1545,6 +1580,9 @@ async def stream_with_studio_tools(assistant_tool_calls: list[dict[str, Any]] = []tool_messages: list[dict[str, Any]] = []noop_messages: list[dict[str, Any]] = []+ # One entry per tool result, not flattened: the per-result image quota+ # would otherwise be spent entirely on the first call of a parallel batch.+ turn_mcp_images: list[list[dict[str, Any]]] = []turn_executed_real_tool = Falsefor call in calls:@@ -1790,6 +1828,13 @@ async def stream_with_studio_tools(last_reprompt_text = ""yield _sse(completion.tool_end_event())tool_messages.append(completion.tool_message())+ # Only for a target that reads them, and off the loop: the envelope is+ # a 12 MB json-load, and a text-only run discards the result anyway.+ _completion_images = (+ await asyncio.to_thread(completion.mcp_images) if run.supports_vision else []+ )+ if _completion_images:+ turn_mcp_images.append(_completion_images)yield _status_sse("")@@ -1828,6 +1873,20 @@ async def stream_with_studio_tools(conversation,"\n\n".join(dict.fromkeys(message["content"] for message in noop_messages)),)+ if turn_mcp_images and run.supports_vision:+ # One block after the whole batch. With a single result "the tool call+ # above" is exact; with several it names whichever ran last, which may+ # have returned no picture at all, so the block says so instead.+ from core.inference.mcp_images import DETACHED_IMAGE_TURN_TEXT+ _lead = DETACHED_IMAGE_TURN_TEXT if len(tool_messages) != 1 else None+ # Off the event loop: each image is decoded and re-encoded.+ await asyncio.to_thread(+ _append_mcp_images_owned,+ conversation,+ turn_mcp_images,+ loop_mcp_image_parts,+ _lead,+ )if turn_executed_real_tool:max_reprompts = _MAX_POST_TOOL_REPROMPTSdiff --git a/studio/backend/core/inference/tool_loop_controller.py b/studio/backend/core/inference/tool_loop_controller.pyindex eba04f01e48..4a8a5d49452 100644--- a/studio/backend/core/inference/tool_loop_controller.py+++ b/studio/backend/core/inference/tool_loop_controller.py@@ -21,6 +21,11 @@ from typing import Any, Collection, Literal, Mapping, Sequencefrom urllib.parse import urlparsefrom core.inference.llama_tool_schema import unrelaxed+from core.inference.mcp_images import split_images as split_mcp_images++# Stamped by mcp_client on every tool it registers; the provenance the envelope+# is trusted on.+MCP_TOOL_PREFIX = "mcp__"from core.inference.tool_call_parser import TOOL_ERROR_NUDGE, TOOL_ERROR_PREFIXES@@ -332,6 +337,17 @@ class ToolCallCompletion:message["tool_call_id"] = self.decision.tool_call_idreturn message+ def mcp_images(self) -> list[dict]:+ """Images this call returned, and only for a call an MCP server served.++ The envelope is a plain suffix, so any tool whose output happens to end in+ one -- terminal output, a fetched page -- would otherwise have its bytes+ decoded and attached as model image input.+ """+ if not self.executed or not self.decision.tool_name.startswith(MCP_TOOL_PREFIX):+ return []+ return split_mcp_images(self.result)[1]+@dataclass(frozen = True)class _ToolCallRecord:@@ -817,29 +833,6 @@ def is_tool_error(result: str) -> bool:return isinstance(result, str) and result.lstrip().startswith(TOOL_ERROR_PREFIXES)-def _strip_mcp_image_suffix(result: str) -> str:- """Drop a trailing __MCP_IMAGES__ envelope only when it is the valid JSON- image array appended by _flatten_result, so legit tool text that merely- mentions the marker is not truncated."""- head, sep, payload = result.rpartition("\n__MCP_IMAGES__:")- if not sep:- return result- try:- images = json.loads(payload)- except (ValueError, RecursionError):- return result- if not isinstance(images, list) or not images:- return result- if not all(- isinstance(img, dict)- and isinstance(img.get("data"), str)- and isinstance(img.get("mimeType"), str)- for img in images- ):- return result- return head.rstrip()--def _strip_files_sentinel(result: str) -> str:"""Drop a trailing ``__FILES__`` envelope, and only that.@@ -990,7 +983,10 @@ def strip_result_for_model(if tool_name is None or tool_name == "web_search":from .search_images import strip_images_suffixresult = strip_images_suffix(result)- result = _strip_mcp_image_suffix(result)+ # Always, whoever produced it: these bytes run to megabytes and the model must+ # never be shown them as text. Provenance decides whether they become IMAGE+ # input, which is a separate question answered in mcp_images._promote.+ result = split_mcp_images(result)[0]if tool_name is None or tool_name in _SANDBOX_TOOLS:result = _strip_files_sentinel(result)if tool_name is None or tool_name in _IMAGE_SENTINEL_TOOLS:diff --git a/studio/backend/core/inference/worker.py b/studio/backend/core/inference/worker.pyindex 877afed0ae8..644f892d0a5 100644--- a/studio/backend/core/inference/worker.py+++ b/studio/backend/core/inference/worker.py@@ -676,6 +676,10 @@ def _handle_generate(backend, cmd: dict, resp_queue: Any, cancel_event) -> None:request_id = cmd.get("request_id", "")try:+ image = None+ image_b64 = cmd.get("image_base64")+ if image_b64:+ image = _resize_image(_decode_image(image_b64))images = [_resize_image(_decode_image(encoded))for encoded in cmd.get("images_base64") or ()@@ -685,6 +689,7 @@ def _handle_generate(backend, cmd: dict, resp_queue: Any, cancel_event) -> None:gen_kwargs = {"messages": cmd["messages"],"system_prompt": cmd.get("system_prompt", ""),+ "image": image,"images": images,"temperature": cmd.get("temperature", 0.7),"top_p": cmd.get("top_p", 0.9),@@ -707,6 +712,9 @@ def _handle_generate(backend, cmd: dict, resp_queue: Any, cancel_event) -> None:if opt_key in cmd:gen_kwargs[opt_key] = cmd[opt_key]+ if cmd.get("image_ordinal") is not None and _backend_declares(backend, "image_ordinal"):+ gen_kwargs["image_ordinal"] = cmd["image_ordinal"]+# Not every backend declares these (transformers declares only ``stop``)# and none takes **kwargs, so forwarding unconditionally would turn a# backend's documented "ignores them" behavior into a TypeError.diff --git a/studio/backend/routes/inference.py b/studio/backend/routes/inference.pyindex 7b236f9c822..3af42c342d0 100644--- a/studio/backend/routes/inference.py+++ b/studio/backend/routes/inference.py@@ -2015,7 +2015,9 @@ def _openai_llama_admission_compact_image_part(part: dict) -> dict:return {"type": "image_url", "image_url": compact_image_url}-def _openai_llama_admission_messages_for_estimate(messages) -> tuple[list[dict], int]:+def _openai_llama_admission_messages_for_estimate(+ messages, *, vision: bool = True+) -> tuple[list[dict], int]:"""Remove image bytes before estimating the textual part of a prompt.mtmd turns an image into model-specific embedding tokens, so its base64 transport@@ -2024,7 +2026,15 @@ def _openai_llama_admission_messages_for_estimate(messages) -> tuple[list[dict],"""estimate_messages = []image_parts = 0- for message in messages:+ # Counted apart from the parts already on the request: promote_history caps how+ # many an envelope becomes, but a caller that really attaches more images than+ # that must still be charged for every one (#9842).+ envelope_image_parts = 0+ # Same provenance resolution generation uses: an unnamed result correlated to a+ # non-MCP call is never promoted, so charging its envelope reserves up to eight+ # embeddings the request never sends.+ _resolved = _mcp_resolve_tool_names(messages)+ for _index, message in enumerate(messages):# exclude_none like every other dump here, including the generation paths this# predicts: the five unset optionals on ChatMessage otherwise serialise as# `"name": null` and get priced as prompt text (34 tokens for a two-key message@@ -2034,6 +2044,36 @@ def _openai_llama_admission_messages_for_estimate(messages) -> tuple[list[dict],)estimate_message = dict(message_dict)content = message_dict.get("content")+ if message_dict.get("role") == "tool" and isinstance(content, str):+ # An MCP envelope is still base64 text here, but the chat path promotes+ # it into projector images before generation. Priced as text it would+ # reserve megabytes of prompt for bytes never sent; ignored entirely it+ # charges zero KV for images that are. Strip it and charge the images.+ head, envelope_images = split_mcp_images(content)+ if envelope_images:+ # Generation strips the suffix for everyone, so it is never priced+ # as prompt text.+ estimate_message["content"] = head+ # Charged only where generation will really send pixels: a text-only+ # model has them stripped, and a named non-MCP result is not promoted+ # at all, so charging either reserves KV for images never sent.+ _correlated = _resolved.get(_index)+ _non_mcp = _names_a_non_mcp_tool(message_dict) or (+ isinstance(_correlated, str)+ and bool(_correlated)+ and not _correlated.startswith("mcp__")+ )+ if vision and not _non_mcp:+ # Only entries that could become a picture: _flatten_result+ # accepts formats Pillow cannot read, promotion drops them, and+ # four such entries would reserve most of a small window for+ # images llama-server is never sent.+ envelope_image_parts += min(+ _mcp_count_probably_decodable(envelope_images),+ _MCP_MAX_MODEL_IMAGES,+ )+ estimate_messages.append(estimate_message)+ continueif isinstance(content, list):estimate_content = []for part in content:@@ -2077,7 +2117,10 @@ def _openai_llama_admission_messages_for_estimate(messages) -> tuple[list[dict],estimate_content.append(_openai_llama_admission_compact_image_part(part))estimate_message["content"] = estimate_contentestimate_messages.append(estimate_message)- return estimate_messages, image_parts+ # Match GGUF promotion: cap replay images separately from caller attachments.+ # One attachment plus eight replay images needs nine KV charges; reserving+ # attachment slots within this cap would undercharge concurrent requests.+ return estimate_messages, image_parts + min(envelope_image_parts, _MCP_MAX_TOTAL_MODEL_IMAGES)def _conversation_video_clips(messages) -> list[str]:@@ -2181,6 +2224,8 @@ def _openai_llama_admission_tokens(image_tokens: int = _OPENAI_LLAMA_ADMISSION_IMAGE_TOKENS,injected_tools = None,context_window: Optional[int] = None,+ vision: bool = True,+ messages_override = None,) -> Optional[int]:"""KV a request will occupy: what is sent, plus what it may generate.@@ -2194,11 +2239,16 @@ def _openai_llama_admission_tokens("""if not budget:return None- messages = getattr(payload, "messages", None)+ # Use the translated OpenAI messages for Anthropic: its raw user/tool_result+ # blocks bypass envelope handling, mispricing base64 as text and omitting+ # image KV charges, which can consume the budget and serialize requests.+ messages = (+ messages_override if messages_override is not None else getattr(payload, "messages", None)+ )if isinstance(messages, list) and messages:try:estimate_messages, message_image_parts = _openai_llama_admission_messages_for_estimate(- messages+ messages, vision = vision)prompt_tokens = estimate_messages_tokens_dense(estimate_messages)prompt_tokens += _openai_llama_admission_extra_prompt_tokens(payload)@@ -2266,6 +2316,36 @@ def _openai_llama_admission_tokens(return max(1, min(budget, prompt_tokens + output_tokens))+def _openai_llama_admission_estimate(+ *,+ request: Optional[Request],+ llama_backend,+ payload,+ tool_loop: bool = False,+ injected_tools = None,+ messages_override = None,+):+ """The token estimate a reservation is sized by. Split from the reservation so a+ caller on the event loop can compute it in a worker: it strips every replayed+ envelope, which json-loads a permitted 12 MB array per result."""+ capacity = _openai_llama_admission_capacity(request, llama_backend)+ budget = _openai_llama_admission_budget(llama_backend)+ return _openai_llama_admission_tokens(+ payload,+ budget = budget,+ capacity = capacity,+ tool_loop = tool_loop,+ image_tokens = _openai_llama_admission_image_tokens(llama_backend),+ injected_tools = injected_tools,+ context_window = _openai_llama_admission_context_window(llama_backend),+ vision = bool(getattr(llama_backend, "is_vision", False)),+ messages_override = messages_override,+ )+++_TOKENS_UNSET = object()++def _openai_llama_admission_reserve(*,request: Optional[Request],@@ -2273,28 +2353,70 @@ def _openai_llama_admission_reserve(payload = None,tool_loop: bool = False,injected_tools = None,+ messages_override = None,+ tokens = _TOKENS_UNSET,) -> tuple[LlamaAdmissionReservation, LlamaAdmissionConfig]:config = llama_admission_config_from_env()capacity = _openai_llama_admission_capacity(request, llama_backend)key = str(getattr(llama_backend, "base_url", "llama-server"))budget = _openai_llama_admission_budget(llama_backend)+ if tokens is _TOKENS_UNSET:+ tokens = (+ _openai_llama_admission_estimate(+ request = request,+ llama_backend = llama_backend,+ payload = payload,+ tool_loop = tool_loop,+ injected_tools = injected_tools,+ messages_override = messages_override,+ )+ if payload is not None+ else None+ )+ # The reservation itself stays on the loop: reserve() binds a waiter to the+ # running loop, so only the estimate above may be computed elsewhere.reservation = get_llama_admission_queue(key).reserve(capacity = capacity,config = config,budget = budget,- tokens = _openai_llama_admission_tokens(- payload,- budget = budget,- capacity = capacity,+ tokens = tokens,+ )+ return reservation, config+++async def _openai_llama_admission_reserve_async(+ *,+ request: Optional[Request],+ llama_backend,+ payload = None,+ tool_loop: bool = False,+ injected_tools = None,+ messages_override = None,+) -> tuple[LlamaAdmissionReservation, LlamaAdmissionConfig]:+ """The same reservation, with the estimate off the loop when a replayed envelope+ would make it parse one."""+ tokens = _TOKENS_UNSET+ if payload is not None and _messages_mention_mcp_images(+ messages_override if messages_override is not None else payload.messages+ ):+ tokens = await asyncio.to_thread(+ _openai_llama_admission_estimate,+ request = request,+ llama_backend = llama_backend,+ payload = payload,tool_loop = tool_loop,- image_tokens = _openai_llama_admission_image_tokens(llama_backend),injected_tools = injected_tools,- context_window = _openai_llama_admission_context_window(llama_backend),+ messages_override = messages_override,)- if payload is not None- else None,+ return _openai_llama_admission_reserve(+ request = request,+ llama_backend = llama_backend,+ payload = payload,+ tool_loop = tool_loop,+ injected_tools = injected_tools,+ messages_override = messages_override,+ tokens = tokens,)- return reservation, configdef _openai_llama_admission_recost(@@ -2344,7 +2466,8 @@ def _openai_llama_admission_recost(# and since the callback fires at the top of round zero, before any growth, it# would hand back room llama-server is already using.estimate_messages, message_image_parts = _openai_llama_admission_messages_for_estimate(- conversation+ conversation,+ vision = bool(getattr(llama_backend, "is_vision", False)),)prompt_tokens = estimate_messages_tokens_dense(estimate_messages)# Re-sent every round, so it belongs in every re-costing, not just the opening one.@@ -3425,6 +3548,23 @@ from state.tool_approvals import resolve_tool_decisionfrom core.inference.model_ids import display_model_name, model_id_matches, public_model_idfrom core.inference.api_monitor import api_monitorfrom core.inference.llama_http import nonstreaming_client+from core.inference.mcp_images import (+ MAX_MODEL_IMAGES as _MCP_MAX_MODEL_IMAGES,+ count_probably_decodable as _mcp_count_probably_decodable,+ image_marker_parts as mcp_image_marker_parts,+ flattened_rgb as _mcp_flattened_rgb,+ resolve_tool_names as _mcp_resolve_tool_names,+ pixels_in_marker_order as mcp_pixels_in_marker_order,+ trim_image_turns as trim_mcp_image_turns,+ MAX_TOTAL_MODEL_IMAGES as _MCP_MAX_TOTAL_MODEL_IMAGES,+ has_images as mcp_images_sentinel_in,+ mentions_images as mcp_images_mentioned_in,+ text_before_envelope as mcp_text_before_envelope,+ mark_last_user_turn as mark_mcp_image_turn,+ promote_history as promote_mcp_history_images,+ promote_history_local as promote_mcp_history_images_local,+ split_images as split_mcp_images,+)from core.inference.tool_call_parser import (_strip_function_xml_calls,_strip_gemma_wrapperless_calls,@@ -6028,7 +6168,11 @@ def _monitor_content_text(content) -> str:if content is None:return ""if isinstance(content, str):- return content+ # A replayed envelope is megabytes of base64 in a plain string. Generation+ # and the search index both strip it; the monitor copied it into the prompt+ # entry, exposing image bytes in inspection and truncating away the later+ # conversation behind them. Cut, not parsed: this runs on the event loop.+ return mcp_text_before_envelope(content)if isinstance(content, list):parts: list[str] = []for part in content:@@ -8285,12 +8429,145 @@ def _messages_have_remote_image(messages) -> bool:)-def _request_has_image(payload) -> bool:+def _request_has_attached_image(payload) -> bool:+ """An image the request itself carries, which the target model has to be able+ to read. History is not counted: a replayed MCP envelope is stripped for a+ text-only model, so it must not veto a switch to one."""if getattr(payload, "image_base64", None):return Truereturn _messages_have_image(payload.messages)+def _names_a_non_mcp_tool(message) -> bool:+ """Whether the message names a tool that is definitely not an MCP one.++ An absent name is not evidence against it: older stored turns carry none, and+ the envelope only ever came from an MCP server.+ """+ name = message.get("name") if isinstance(message, dict) else getattr(message, "name", None)+ return isinstance(name, str) and bool(name) and not name.startswith("mcp__")+++# Mirrors NON_VISION_PROVIDER_TYPES in studio/frontend/src/features/chat/+# external-providers.ts. The backend registry marks these providers+# supports_vision=True for the family, but their main chat endpoint is text-only,+# and appending a picture there fails upstream mid-run.+_TEXT_ONLY_PROVIDER_TYPES = frozenset({"cohere", "deepseek", "mistral"})+++# Mixed catalogs require model-level vision capability; family flags do not+# identify which models accept images. Qwen has unrestricted text defaults;+# its vision models use separate qwen-vl-* or Qwen2.5-VL IDs. Kimi is excluded+# because its allowlist permits only multimodal kimi-k2.5/k2.6.+# Single-endpoint types (custom, vllm, ollama, llamacpp, lmstudio) retain the+# registry default: no model catalog exists to check the configured endpoint.+_MIXED_CATALOG_PROVIDER_TYPES = frozenset({"huggingface", "openrouter", "qwen"})+++def _external_takes_mcp_images(+ provider_type,+ supports_vision: bool,+ model: "str | None" = None,+ info: "dict | None" = None,+) -> bool:+ """Whether an MCP picture may be appended to this external loop.++ Deliberately stricter than the provider-wide flag: an image the endpoint+ cannot take does not degrade, it fails the run as soon as a tool returns one.+ A model-level answer is used wherever the registry has one; where the catalog+ mixes modalities and says nothing about this model, the answer is no.+ """+ if provider_type in _TEXT_ONLY_PROVIDER_TYPES:+ return False+ if not supports_vision:+ return False+ capabilities = ((info or {}).get("model_capabilities") or {}) if info else {}+ if model and model in capabilities:+ return bool(capabilities[model].get("vision"))+ if provider_type in _MIXED_CATALOG_PROVIDER_TYPES:+ # Vision for the family, unknown for this model: an image would be sent to+ # something that may not take one, and the failure lands mid-run.+ return False+ return True+++def _messages_mention_mcp_images(messages) -> bool:+ """Dispatch only -- whether the work belongs off the loop. Substring, never a+ parse: the exact check json-loads the whole array, and doing that on the loop+ to decide whether to leave it parsed a 12 MB envelope right there."""+ for message in messages or ():+ role = message.get("role") if isinstance(message, dict) else getattr(message, "role", None)+ if role != "tool":+ continue+ content = (+ message.get("content")+ if isinstance(message, dict)+ else getattr(message, "content", None)+ )+ if isinstance(content, str) and mcp_images_mentioned_in(content):+ return True+ return False+++def _messages_have_promotable_mcp_images(messages) -> bool:+ """The exact envelope check, minus results generation would never promote.++ For the refusals: a named non-MCP result has its suffix stripped and sends no+ pixel, so refusing a count on it turned away a countable prompt. Callers stamp+ names from the calls first (_named_anthropic_tool_results), so an unnamed result+ here really is one nothing can attribute, and legacy history keeps its trust.+ """+ for message in messages or ():+ if not isinstance(message, dict) or message.get("role") != "tool":+ continue+ content = message.get("content")+ if not isinstance(content, str) or not mcp_images_sentinel_in(content):+ continue+ if _names_a_non_mcp_tool(message):+ continue+ return True+ return False+++def _request_has_replayed_mcp_images(payload) -> bool:+ return _messages_mention_mcp_images(payload.messages)+++def _request_has_promotable_mcp_images(payload, *, exact: bool = True) -> bool:+ """Envelopes generation would really promote.++ ``_promote`` preserves a named non-MCP result verbatim, so treating its+ trailing suffix as images here refuses a countable prompt and can buy a+ model-catalog fetch nothing needs.++ ``exact=False`` is the substring form, for a decision made on the event loop:+ the exact check json-loads the whole array, and a permitted envelope is 12 MB.+ A false positive there costs a catalog fetch; the worker validates for real.+ """+ present = mcp_images_sentinel_in if exact else mcp_images_mentioned_in+ # The same positional correlation generation applies: an unnamed result whose+ # call was a non-MCP tool is never promoted, and reading it as promotable here+ # refused a countable prompt and bought a catalog fetch for nothing.+ names = _mcp_resolve_tool_names(payload.messages)+ for index, message in enumerate(payload.messages):+ if getattr(message, "role", None) != "tool":+ continue+ content = getattr(message, "content", None)+ if not isinstance(content, str) or not present(content):+ continue+ name = getattr(message, "name", None) or names.get(index)+ if isinstance(name, str) and name and not name.startswith("mcp__"):+ continue+ return True+ return False+++def _request_has_image(payload) -> bool:+ # A replayed envelope is decoded and re-encoded like any other image, so the+ # work belongs off the event loop even when no picture is attached.+ return _request_has_attached_image(payload) or _request_has_replayed_mcp_images(payload)++def _anthropic_image_blocks(payload, *, tool_results: bool = True):for msg in getattr(payload, "messages", None) or ():content = msg.get("content") if isinstance(msg, dict) else msg.content@@ -19015,7 +19292,7 @@ async def generate_audio()# Extract text from the last user message- _, chat_messages, _ = _extract_content_parts(payload.messages)+ _, chat_messages, _ = await _extract_content_parts_async(payload.messages)if not chat_messages:raise HTTPException(status_code = 400, detail = "No messages provided.")last_user_msg = next((m for m in reversed(chat_messages) if m["role"] == "user"), None)@@ -21317,8 +21594,7 @@ def _video_scheme_rejection(clip: str) -> Optional[tuple[int, str]]:return Nonereturn (400,- f"Unsupported video URL scheme ('{head.split(':', 1)[0]}:'). "- "Send the clip as a data URI.",+ f"Unsupported video URL scheme ('{head.split(':', 1)[0]}:'). Send the clip as a data URI.",)@@ -21445,7 +21721,10 @@ def _inject_audio_part(messages: list[dict], audio_b64: str, audio_format: str)def _extract_content_parts(- messages: list, structured: bool = False+ messages: list,+ structured: bool = False,+ *,+ keep_tool_images: bool = False,) -> tuple[str, list[dict], "Union[Optional[str], list[str]]"]:"""Parse OpenAI-format messages into components the inference backend expects.@@ -21466,7 +21745,8 @@ def _extract_content_parts(latest_user_image_b64: Optional[str] = Noneserved_images: list[str] = []- for msg in messages:+ _resolved_tool_names = _mcp_resolve_tool_names(messages)+ for _msg_index, msg in enumerate(messages):# ── System / developer messages → extract as system_prompt ────────if msg.role in ("system", "developer"):if isinstance(msg.content, str):@@ -21526,9 +21806,26 @@ def _extract_content_parts(if combined_text is None:continue+ if msg.role == "tool" and not keep_tool_images:+ # The exact split, always: an MCP tool's own text can contain the marker+ # line without an envelope behind it (the envelope is appended only when+ # there were images), and a cut at the marker truncated such a result.+ # The json-load this costs on a permitted 12 MB envelope is why the+ # awaited callers go through _extract_content_parts_async.+ combined_text = split_mcp_images(combined_text)[0]chat_message = {"role": msg.role, "content": combined_text}+ # Carried through: promote_history reads it to decide whether an envelope+ # came from an MCP server, and dropping it here made an unnamed tool+ # message that bypasses the check entirely. Resolved from the call when the+ # result itself is unnamed: this rebuild drops tool_call_id and the calls,+ # so the correlation has to happen here or the local path cannot run the+ # provenance gate at all.if msg.name:chat_message["name"] = msg.name+ if msg.role == "tool":+ _tool_name = getattr(msg, "name", None) or _resolved_tool_names.get(_msg_index)+ if isinstance(_tool_name, str) and _tool_name:+ chat_message["name"] = _tool_nameif msg.role == "assistant" and msg.reasoning_content:chat_message["reasoning_content"] = msg.reasoning_contentchat_messages.append(chat_message)@@ -21542,6 +21839,15 @@ def _extract_content_parts()+async def _extract_content_parts_async(messages: list, **kwargs):+ """The same, off the shared loop when a tool result carries the envelope marker:+ stripping it is the exact split, a json-load of a permitted 12 MB array, and the+ awaited audio, count and safetensors callers must not do that on the loop."""+ if _messages_mention_mcp_images(messages):+ return await asyncio.to_thread(_extract_content_parts, messages, **kwargs)+ return _extract_content_parts(messages, **kwargs)++def _user_ordinal_supplying_the_image(messages: list) -> Optional[int]:"""Which user turn, counted among user turns, the selected image came from.@@ -21566,15 +21872,27 @@ def _user_ordinal_supplying_the_image(messages: list) -> Optional[int]:return ordinal-def _mark_image_owner_turn(messages: list, ordinal: int) -> list:+def _mark_image_owner_turn(+ messages: list,+ ordinal: int,+ marked_at: "list | None" = None,+) -> list:"""Give the *ordinal*-th user turn a structured image part ahead of its text: the- renderers attach it to the newest turn unless one already carries it."""+ renderers attach it to the newest turn unless one already carries it.++ *marked_at* collects that turn's index. A replayed MCP picture has to be placed+ ahead of it -- the pixels go history-first with the attachment last, and a+ positional processor binds them in document order -- so the caller needs to know+ which turn owns the attachment, not just that one does.+ """marked = list(messages)seen = 0for index, message in enumerate(marked):if not isinstance(message, dict) or message.get("role") != "user":continueif seen == ordinal:+ if marked_at is not None:+ marked_at.append(index)body = message.get("content")if isinstance(body, str):marked[index] = {@@ -21868,6 +22186,8 @@ def _build_external_messages(supports_vision: bool,provider_type: Optional[str] = None,base_url: Optional[str] = None,+ promoted_out: "Optional[list]" = None,+ promote_mcp_images: "Optional[bool]" = None,) -> list[dict]:"""Convert ChatMessage list to OpenAI-compatible dicts for external providers.@@ -21875,6 +22195,10 @@ def _build_external_messages(Behaviour per content-part type:- `text`: always preserved.- `image_url`: preserved on vision providers; stripped on non-vision.+ `promote_mcp_images` governs only whether a replayed MCP envelope becomes+ image input; it defaults to `supports_vision` and must not be passed as+ that flag, or a stricter MCP answer would also strip what the caller+ attached.- `input_document`: preserved ONLY when the provider's stream helper hasexplicit translation logic (Anthropic + OpenAI today, see``_INPUT_DOCUMENT_PROVIDERS``). Stripped for every other provider so the@@ -22186,7 +22510,69 @@ def _build_external_messages(entry["tool_calls"] = _replay_tool_call_ids(entry["tool_calls"], replay_ids)if isinstance(entry.get("tool_call_id"), str):entry["tool_call_id"] = replay_ids.get(entry["tool_call_id"], entry["tool_call_id"])- return result+ promote = supports_vision if promote_mcp_images is None else promote_mcp_images+ # A provider applies its own per-request cap in document order, so the replay+ # leaves the caller's own pictures room; llama-server is bounded by its context+ # window instead, and the GGUF callers keep the full allowance.+ return promote_mcp_history_images(+ result, vision = promote, promoted_out = promoted_out, reserve_for_caller = True+ )+++async def _promote_mcp_history_images_async(+ messages,+ *,+ vision: bool,+ promoted_out = None,+):+ """Promotion off the shared loop when there is really an envelope to rebuild.+ Same hop, and the same reason, as the local and external replay paths take.+ On marker presence alone, not on vision: a text-only target still has the+ envelope stripped, and stripping json-loads the whole 12 MB array."""+ if _messages_mention_mcp_images(messages):+ return await asyncio.to_thread(+ promote_mcp_history_images, messages, vision = vision, promoted_out = promoted_out+ )+ return promote_mcp_history_images(messages, vision = vision, promoted_out = promoted_out)+++async def _promote_local_mcp_images_async(+ messages,+ *,+ vision: bool,+ decode_cache = None,+ caller_images = (),+):+ """Rebuilding a replayed envelope decodes and re-encodes every picture in it.+ A permitted image runs to 40 megapixels, so that belongs off the shared loop --+ the same hop the GGUF and external replay paths already take. Taken on marker+ presence alone: a text-only target still has the envelope stripped, which parses+ the same 12 MB array."""+ if _messages_mention_mcp_images(messages):+ return await asyncio.to_thread(+ promote_mcp_history_images_local,+ messages,+ vision = vision,+ decode_cache = decode_cache,+ caller_images = caller_images,+ )+ return promote_mcp_history_images_local(+ messages, vision = vision, decode_cache = decode_cache, caller_images = caller_images+ )+++async def _build_external_messages_async(messages, supports_vision, **kwargs) -> list[dict]:+ """Also collects what promotion created when the caller passes promoted_out."""+ """Promoting a replayed envelope decodes and re-encodes every picture in it,+ which is PIL work on the shared loop. Hop to a thread only when there is an+ envelope to promote, so an ordinary text turn keeps its direct call."""+ # On marker presence alone, not on whether promotion is on: a text-only target+ # still has the envelope stripped, and stripping parses the same 12 MB array.+ if _messages_mention_mcp_images(messages):+ return await asyncio.to_thread(+ _build_external_messages, messages, supports_vision, **kwargs+ )+ return _build_external_messages(messages, supports_vision, **kwargs)async def _proxy_to_external_provider(@@ -22427,11 +22813,25 @@ async def _proxy_to_external_provider(if model in capabilitieselse offered_subscription_model(payload.provider_id, model))- if image_requested and model not in capabilities and listed_model is None:+ # An MCP tool can hand this loop a picture on a later turn, and the vision+ # flag is fixed here for the whole run: a text-only opening turn would+ # otherwise pin it to False and discard that picture on a capable model.+ _may_receive_image = (+ image_requested+ # A tool can hand this loop a picture on a later turn...+ or bool(getattr(payload, "mcp_enabled", False))+ # ...and a conversation that already carries one needs the capability+ # resolved even when MCP has since been switched off, or the replay is+ # stripped from a model that could have read it. Substring form: this+ # runs on the loop, and the exact check parses the whole envelope.+ or _request_has_promotable_mcp_images(payload, exact = False)+ )+ if _may_receive_image and model not in capabilities and listed_model is None:# Admitted straight off the saved row, so no catalog read happened this# process and nothing here knows the modalities. Defaulting to text-only- # would refuse an image for a model the picker offered as capable. Only an- # actual image is worth the fetch; a text turn should not pay for one.+ # would refuse an image for a model the picker offered as capable. Only a+ # request that can carry one is worth the fetch; a plain text turn with no+ # tools should not pay for it.try:await ensure_subscription_models(payload.provider_id)except (CodexAuthError, CodexReauthorizationError) as exc:@@ -22473,11 +22873,13 @@ async def _proxy_to_external_provider(mark_subscription_catalog_stale(payload.provider_id)if model not in _allowed_codex_models():raise HTTPException(status_code = 400, detail = "Choose a curated Codex model.")- chat_messages = _build_external_messages(+ _codex_promoted_parts: list = []+ chat_messages = await _build_external_messages_async(payload.messages,model_supports_vision,provider_type = provider_type,base_url = base_url,+ promoted_out = _codex_promoted_parts,)tool_payloads = [tool.model_dump(exclude_none = True) if hasattr(tool, "model_dump") else tool@@ -22561,6 +22963,8 @@ async def _proxy_to_external_provider(response_format = _extract_response_format(payload),tool_choice = payload.tool_choice,continue_final_message = _continue_final_message(payload),+ supports_vision = model_supports_vision,+ promoted_image_parts = tuple(_codex_promoted_parts),)policy = (CodexToolPolicy(@@ -22790,11 +23194,21 @@ async def _proxy_to_external_provider(_pinfo = _get_provider_info(provider_type) or {}_supports_vision = _pinfo.get("supports_vision", False)- chat_messages = _build_external_messages(+ _external_promoted_parts: list = []+ chat_messages = await _build_external_messages_async(payload.messages,+ # The provider-wide flag, so an image the caller attached is preserved+ # exactly as it was before MCP promotion existed. The stricter+ # model-level answer gates promotion alone: openrouter and huggingface+ # are vision-capable for the family and name no model, so folding the+ # two together stripped a picture sent to a chosen vision model._supports_vision,provider_type = provider_type,base_url = base_url,+ promoted_out = _external_promoted_parts,+ promote_mcp_images = _external_takes_mcp_images(+ provider_type, _supports_vision, model, _pinfo+ ),)monitor_id = Noneif not getattr(request.state, "skip_api_monitor", False):@@ -22927,6 +23341,10 @@ async def _proxy_to_external_provider(model = model,tool_choice = payload.tool_choice,continue_final_message = _continue_final_message(payload),+ supports_vision = _external_takes_mcp_images(+ provider_type, _supports_vision, model, _pinfo+ ),+ promoted_image_parts = tuple(_external_promoted_parts),),policy = ToolLoopPolicy(tools = external_studio_tools,@@ -23564,7 +23982,7 @@ async def produce_openai_chat_completions(_preprepared_audio = None_image_preflight = Noneif _should_validate_before_switch():- _pre_parsed = _extract_content_parts(payload.messages)+ _pre_parsed = _extract_content_parts(payload.messages, keep_tool_images = True)if not _pre_parsed[1]:raise HTTPException(status_code = 400, detail = "At least one non-system message is required."@@ -23670,7 +24088,7 @@ async def produce_openai_chat_completions(# audio-only request asks for the projector alone, since an audio model's# projector carries no vision tower. A safetensors or MLX checkpoint# declares audio input separately, so it is tracked apart as well.- _needs_image = bool(_pre_parsed[2]) or _request_has_image(payload)+ _needs_image = bool(_pre_parsed[2]) or _request_has_attached_image(payload)# Video rides that projector too. Its own /props gate can only run after# the load, so this at least keeps a text-only target from evicting a# working model to serve a clip it could never take.@@ -23959,7 +24377,9 @@ async def produce_openai_chat_completions(if _predecoded_audio is not Noneelse _decode_audio_base64(payload.audio_base64))- system_prompt, chat_messages, _ = _extract_content_parts(payload.messages)+ system_prompt, chat_messages, _ = await _extract_content_parts_async(+ payload.messages+ )system_prompt = _apply_current_date_prompt(system_prompt, request)except _DecodedAudioTooLongError as e:# A limit the caller can act on, not a server fault.@@ -24391,7 +24811,9 @@ async def produce_openai_chat_completions(if _pre_parsed is not None:system_prompt, chat_messages, extracted_image_b64 = _pre_parsedelse:- system_prompt, chat_messages, extracted_image_b64 = _extract_content_parts(payload.messages)+ system_prompt, chat_messages, extracted_image_b64 = _extract_content_parts(+ payload.messages, keep_tool_images = True+ )# applied once so both backends inherit it, with or without tools, and never state it twice.system_prompt = _apply_current_date_prompt(system_prompt, request)@@ -24446,9 +24868,15 @@ async def produce_openai_chat_completions(video_b64, _ = _video_b64_rejection(payload.video_base64)video_b64 = await _shrink_clip_for_llama(video_b64, llama_backend)+ # The parts promotion created from replayed envelopes. The loop's cap counts+ # what MCP tools have returned across the whole conversation, so a resumed chat+ # has to hand it these: seeded empty it counted only the current run and let a+ # replayed eight sit beside a fresh eight.+ _gguf_replayed_image_parts: list = []gguf_messages, _ = await _openai_messages_for_gguf_chat_async(payload,llama_backend.is_vision,+ _gguf_replayed_image_parts,)gguf_messages = _set_or_prepend_system_message(gguf_messages, system_prompt)image_b64 = None@@ -24692,6 +25120,7 @@ async def produce_openai_chat_completions(def gguf_generate_with_tools():return llama_backend.generate_chat_completion_with_tools(messages = gguf_messages,+ replayed_image_parts = tuple(_gguf_replayed_image_parts),tools = tools_to_use,temperature = payload.temperature,top_p = payload.top_p,@@ -24740,7 +25169,7 @@ async def produce_openai_chat_completions(_tool_admission_mode = "chat_tool_stream" if payload.stream else "chat_tool_nonstream"try:- reservation, admission_config = _openai_llama_admission_reserve(+ reservation, admission_config = await _openai_llama_admission_reserve_async(request = request,llama_backend = llama_backend,payload = payload,@@ -25569,7 +25998,7 @@ async def produce_openai_chat_completions(_tracker = _TrackedCancel.for_payload(cancel_event, payload, *_cancel_keys)_tracker.__enter__()try:- reservation, admission_config = _openai_llama_admission_reserve(+ reservation, admission_config = await _openai_llama_admission_reserve_async(request = request,llama_backend = llama_backend,payload = payload,@@ -25913,7 +26342,7 @@ async def produce_openai_chat_completions()else:try:- reservation, admission_config = _openai_llama_admission_reserve(+ reservation, admission_config = await _openai_llama_admission_reserve_async(request = request,llama_backend = llama_backend,payload = payload,@@ -26198,7 +26627,9 @@ async def produce_openai_chat_completions(served_images: list[str] = []images: list = []if _serves_several_images(backend):- _, _msgs, _payloads = _extract_content_parts(payload.messages, structured = True)+ _, _msgs, _payloads = _extract_content_parts(+ payload.messages, structured = True, keep_tool_images = True+ )_legacy_distinct = _legacy_image_is_distinct(payload)# A legacy image the messages do not carry joins the newest user turn, as GGUF splices it.if _legacy_distinct:@@ -26274,6 +26705,18 @@ async def produce_openai_chat_completions(# Classify capability flags from the loaded template._sf_model_info = backend.models.get(backend.active_model_name, {})+ # Strips for a text-only model, rebuilds the picture as a marker turn for one+ # that reads images; either way no envelope reaches the template.+ # One request's decodes, shared with the client-tool rebuild below, which promotes+ # the same history again from the raw payload: a retained picture can be a 40+ # megapixel raster, and it is decoded once, not once per promotion.+ _sf_mcp_decode_cache: dict = {}+ chat_messages, sf_mcp_images = await _promote_local_mcp_images_async(+ chat_messages,+ vision = bool(_sf_model_info.get("is_vision")),+ decode_cache = _sf_mcp_decode_cache,+ caller_images = served_images,+ )_sf_tpl = (_sf_model_info.get("chat_template_info") or {}).get("template")# Resolve the tool policy BEFORE the protocol is classified: the template# branch chosen here must be the one generation renders. Reading the raw@@ -26397,6 +26840,32 @@ async def produce_openai_chat_completions(_sf_template_tools)+ # A replayed MCP picture sends generation down the processor path just as an+ # attachment does, so protocol classification has to read the same template it+ # will render with -- otherwise tool schemas and reasoning markers are derived+ # from the tokenizer template while the prompt comes from the processor one.+ _sf_has_any_image = image is not None or bool(sf_mcp_images) or bool(images)+ # Resolved BEFORE the capability gate below, which classifies from this body (#10092).+ _sf_image_tpl = (+ (_sf_model_info.get("chat_template_info") or {}).get("processor_template")+ # A clip renders through the processor as an image does.+ if _sf_has_any_image or _video_clip is not None+ else None+ )+ # Differs from processor_template: a template-less processor still places the image.+ _sf_renders_image = _sf_has_any_image and bool(+ (_sf_model_info.get("chat_template_info") or {}).get("renders_image")+ )+ if _sf_image_tpl is not None:+ # The WHOLE protocol, not just tool support: a processor body can carry a reasoning+ # channel the tokenizer never declares, whose <think> markup then leaked as visible+ # content. prefer_tool_use is off: a processor never selects "tool_use" (#10092).+ _sf_features, _sf_parse_think, _sf_reasoning_prefilled = _sf_response_protocol(+ _sf_template_tools,+ template = _sf_image_tpl,+ prefer_tool_use = False,+ )+# A continued turn renders no generation prompt, so nothing is prefilled and the# resumed text is the visible answer. Only the first turn continues; later tool-loop# turns render a fresh generation prompt and prefill as usual.@@ -26437,7 +26906,22 @@ async def produce_openai_chat_completions(_sf_use_tools = ((_sf_tools_on or _sf_mcp_allowed)and _sf_features.get("supports_tools", False)- and image is None+ # An attachment used to withdraw the tools: the loop had no way to carry+ # a picture. It has one now, so only a model that cannot read images does.+ #+ # Except when the caller sent its own catalog. #10092 routes image-plus-tools+ # to the client passthrough so those schemas are the ones rendered; claiming+ # the request here would answer it with Unsloth's built-ins instead, which is+ # the client's tools silently gone. Studio's own chat names tools through+ # enabled_tools rather than sending schemas, so the picture still reaches the+ # loop there -- which is the request this whole path exists for.+ #+ # Gated on _sf_has_any_image, not on the attachment: a resumed chat whose+ # only pictures are replayed MCP ones carries image input just the same, and+ # reading `image is None` there let the loop take the request and swap the+ # caller's schemas out on exactly the path this PR adds.+ and (not _sf_has_any_image or (bool(_sf_model_info.get("is_vision")) and not payload.tools))+ # A caller's several images and a video clip stay out of the loop, as on main.and not imagesand _video_clip is Noneand not _sf_is_gptoss@@ -26533,6 +27017,46 @@ async def produce_openai_chat_completions(else:_sf_chat_messages.append(_msg)+ # The attachment rides in last, matching where its marker sits: the MCP+ # pictures came from earlier turns.+ _sf_loop_images = list(sf_mcp_images)+ # Where the attachment lands among the replayed pixels: its marker sits on the+ # turn that supplied it, which can precede a tool's pictures, so the position+ # is read off the conversation rather than assumed to be last.+ _sf_attachment_at: list[int] = []+ if image is not None:+ # The turn that supplied it, not simply the newest one: the extractor+ # takes the newest user image from anywhere in the thread, so a+ # text-only latest question would otherwise be told it carried an older+ # picture -- and with replayed images in between, the marker order stops+ # matching the pixel order too.+ # Trimmed BEFORE the attachment's marker exists, which is what the+ # ordering here is for. The trim drops by marker ORDINAL against a payload+ # list that holds replay only, so with the attachment's turn ahead of the+ # replayed pictures it deleted ordinal 0 -- the attachment's own marker --+ # while charging replay payload 0, and every later pixel shifted onto the+ # marker before it.+ trim_mcp_image_turns(+ _sf_chat_messages, sf_mcp_images, limit = _MCP_MAX_TOTAL_MODEL_IMAGES - 1+ )+ # Snapshot after the trim, so "history" names the markers that survived it.+ _sf_prior_markers = mcp_image_marker_parts(_sf_chat_messages)+ # Read off the conversation, not assumed: the attachment's marker sits+ # on the turn that supplied it, which can precede a tool's pictures, and+ # a positional VLM binds the Nth pixel to the Nth marker.+ _sf_chat_messages = mark_mcp_image_turn(+ _sf_chat_messages,+ 1,+ ordinal = _user_ordinal_supplying_the_image(payload.messages),+ )+ _sf_loop_images = mcp_pixels_in_marker_order(+ _sf_chat_messages,+ _sf_prior_markers,+ sf_mcp_images,+ _pil_to_png_b64(image),+ placed_at = _sf_attachment_at,+ )+# Request-scoped usage/timings receptacle (filled at gen_done)._sf_stats_holder: dict = {}@@ -26541,6 +27065,10 @@ async def produce_openai_chat_completions(messages = _sf_chat_messages,tools = _sf_tools_to_use,system_prompt = _sf_system_prompt or "",+ images = _sf_loop_images or None,+ # The loop's cap trims the replay it grows; the caller's own picture+ # is not the loop's to evict, so it is named by position.+ caller_image_indexes = tuple(_sf_attachment_at),temperature = payload.temperature,top_p = payload.top_p,top_k = payload.top_k,@@ -26935,12 +27463,22 @@ async def produce_openai_chat_completions(finally:_sf_tracker.__exit__(None, None, None)+ # The attachment is passed beside the replayed list and both reach the backend+ # as one `attached` sequence, so the cap has to cover the pair. Reserve its slot+ # by trimming the replay, oldest first, with the markers that go with them.+ if image is not None and sf_mcp_images:+ trim_mcp_image_turns(chat_messages, sf_mcp_images, limit = _MCP_MAX_TOTAL_MODEL_IMAGES - 1)+# Shared generation kwargsgen_kwargs = dict(messages = chat_messages,system_prompt = system_prompt,image = image,- images = images,+ images = sf_mcp_images or images or None,+ # Which user turn actually supplied the attachment: the extractor takes the+ # newest user image from anywhere in the thread, so the backends cannot+ # assume it belongs to the latest question.+ image_ordinal = _user_ordinal_supplying_the_image(payload.messages),temperature = payload.temperature,top_p = payload.top_p,top_k = payload.top_k,@@ -26971,27 +27509,10 @@ async def produce_openai_chat_completions(# tools into the template, generate one turn, heal text-form calls (#6801).# supports_tools=False falls through to plain relay (GGUF gate parity)._sf_has_tool_msgs = any(m.role == "tool" or m.tool_calls for m in payload.messages)- # Resolved BEFORE the capability gate below, which classifies from this body (#10092).- _sf_has_image = image is not None or bool(images)- # A clip renders through the processor as an image does.- _sf_image_tpl = (- (_sf_model_info.get("chat_template_info") or {}).get("processor_template")- if _sf_has_image or _video_clip is not None- else None- )- # Differs from processor_template: a template-less processor still places the image.- _sf_renders_image = _sf_has_image and bool(- (_sf_model_info.get("chat_template_info") or {}).get("renders_image")- )- if _sf_image_tpl is not None:- # The WHOLE protocol, not just tool support: a processor body can carry a reasoning- # channel the tokenizer never declares, whose <think> markup then leaked as visible- # content. prefer_tool_use is off: a processor never selects "tool_use" (#10092).- _sf_features, _sf_parse_think, _sf_reasoning_prefilled = _sf_response_protocol(- _sf_template_tools,- template = _sf_image_tpl,- prefer_tool_use = False,- )+ # Classified against the processor body above, before the server loop's own gate:+ # that gate reads the same flag, and deciding it here left the loop enabled on a+ # tokenizer body that renders tools while generation used a processor body that+ # does not, so the model was never shown the schemas the loop was driving._sf_supports_tools = _sf_features.get("supports_tools", False)# Gate on _sf_use_tools (did the server-side path claim the request?), not# raw mcp_enabled: an empty MCP registry must not silently drop client tools.@@ -26999,8 +27520,9 @@ async def produce_openai_chat_completions(# Read the resolved value, not a fresh _effective_enable_tools: the gate# above withdraws the launcher default for exactly these requests, and# recomputing here would hide that and drop the client catalog.- # An image or a clip rules out the server loop, so the passthrough takes the request (#10092).- (not _sf_tools_on or ((_sf_has_image or _video_clip is not None) and not _sf_use_tools))+ # Once an image or a clip rules out the server loop the passthrough takes the+ # request (#10092). _sf_has_any_image counts replayed MCP pictures too.+ (not _sf_tools_on or ((_sf_has_any_image or _video_clip is not None) and not _sf_use_tools))and not _sf_use_toolsand not _sf_is_gptossand _sf_supports_tools@@ -27064,29 +27586,53 @@ async def produce_openai_chat_completions(if served_images:# One pass over the conversation this renders, so markers and payloads stay in step._sf_rebuilt, _sf_payloads = _conversation_with_image_markers(- _openai_messages_for_passthrough(payload, normalize_images = False)+ _openai_messages_for_passthrough(+ payload, normalize_images = False, promote_mcp_images = False+ )+ )+ _sf_rebuilt, _sf_rebuilt_images = await _promote_local_mcp_images_async(+ _sf_rebuilt,+ vision = _sf_renders_image,+ decode_cache = _sf_mcp_decode_cache,+ caller_images = _sf_payloads,)# The same pictures the extraction already decoded, unless the rebuild dropped one.gen_kwargs["images"] = await _decode_request_images(- backend, _sf_payloads, dict(zip(served_images, images)), reject = _reject+ backend, _sf_rebuilt_images, dict(zip(served_images, images)), reject = _reject)gen_kwargs["messages"] = _set_or_prepend_system_message(_structured_tool_history_for_local_template(_sf_rebuilt), system_prompt)else:- gen_kwargs["messages"] = _set_or_prepend_system_message(+ #+ # Promote after flattening, which drops image parts. Preserve one marker per+ # batch at its original position or merge it into the following user turn;+ # a detached block can violate non-GGUF image limits and turn alternation.+ _sf_rebuilt, _sf_rebuilt_images = await _promote_local_mcp_images_async(_structured_tool_history_for_local_template(_flatten_content_parts_for_local_template(- # Not a llama-server body: the flatten below drops image parts.- _openai_messages_for_passthrough(payload, normalize_images = False)+ _openai_messages_for_passthrough(+ payload, normalize_images = False, promote_mcp_images = False+ ))),- system_prompt,- )+ vision = _sf_renders_image,+ decode_cache = _sf_mcp_decode_cache,+ )+ if image is not None and _sf_rebuilt_images:+ # The attachment rides beside the replay and both reach the backend as+ # one sequence, so its slot is reserved here as the server-tool path does.+ trim_mcp_image_turns(+ _sf_rebuilt, _sf_rebuilt_images, limit = _MCP_MAX_TOTAL_MODEL_IMAGES - 1+ )+ gen_kwargs["messages"] = _set_or_prepend_system_message(_sf_rebuilt, system_prompt)+ gen_kwargs["images"] = _sf_rebuilt_images or None# Mark the turn that owns the image so the newest-user-turn scan does not move an# older picture onto a later question. Gated on _sf_renders_image, not on an image:- # a text-template render must not be handed part lists (#10092). Kept markers need none.- if _sf_renders_image and not served_images:+ # a text-template render must not be handed part lists (#10092). Not when pictures+ # were replayed or kept in place: the backends then place the attachment's marker+ # themselves, by ordinal, and a marker set here would bind a replayed picture.+ if _sf_renders_image and not _sf_rebuilt_images and not served_images:_sf_image_ordinal = _user_ordinal_supplying_the_image(payload.messages)if _sf_image_ordinal is not None:gen_kwargs["messages"] = _mark_image_owner_turn(@@ -27113,8 +27659,15 @@ async def produce_openai_chat_completions(] or Noneelse:gen_kwargs["tools"] = payload.tools- elif _sf_renders_image:+ elif _sf_renders_image and not sf_mcp_images:# The plain route too: later turns then share the prefix that holds the image.+ #+ # Only when nothing was replayed. Both backends snapshot the conversation's+ # existing markers as "history" before topping up, so a marker added HERE is+ # counted as a replayed picture's and the pixels bind to the wrong turns: with+ # the attachment on an earlier turn than a tool's picture, the model is shown+ # the screenshot where its own diagram belongs. The top-up places the marker+ # at the same ordinal anyway, so the only thing lost is the ambiguity._sf_image_ordinal = _user_ordinal_supplying_the_image(payload.messages)if _sf_image_ordinal is not None:gen_kwargs["messages"] = _mark_image_owner_turn(@@ -30818,7 +31371,7 @@ async def _responses_stream(cancel_event = threading.Event()_tracker = _TrackedCancel.for_payload(cancel_event, payload, resp_id)try:- reservation, admission_config = _openai_llama_admission_reserve(+ reservation, admission_config = await _openai_llama_admission_reserve_async(request = request,llama_backend = llama_backend,# chat_req, not payload: a ResponsesRequest carries `input` and@@ -32119,6 +32672,16 @@ def _scaled_from_16_bit(image):return image.point(lambda v: v * (1.0 / 257), mode = "L")+def _pil_to_png_b64(img) -> str:+ buf = io.BytesIO()+ # Composited, not merely converted: convert("RGB") keeps whatever colour sits+ # under the alpha, so a transparent attachment whose background was never painted+ # reaches the worker as black -- and its dark text with it. The ordinary IPC path+ # carries the PNG's alpha through, so only this serialisation flattened it.+ _mcp_flattened_rgb(img).save(buf, format = "PNG")+ return base64.b64encode(buf.getvalue()).decode("ascii")++def _image_bytes_to_png_b64(raw: bytes) -> str:"""Decode raw image bytes and re-encode to a base64-ascii PNG string.@@ -32407,6 +32970,25 @@ def _resident_context_satisfies(model_info: dict, max_seq_length: Any) -> bool:return _positive_int_or_none(max_seq_length) == _positive_int_or_none(recorded)+async def _resident_model_reads_images() -> bool:+ """Whether whatever is resident would be handed pixels for a promoted envelope.++ Both backends are asked: llama.cpp answers for a loaded GGUF, and the inference+ backend for a resident MLX or safetensors model, which serves its own count and+ would otherwise never reach a llama-only guard.+ """+ if bool(getattr(get_llama_cpp_backend(), "is_vision", False)):+ return True+ # Off the loop for the same reason the MLX count is: building the singleton runs+ # the torch import.+ backend = await asyncio.to_thread(get_inference_backend)+ active = getattr(backend, "active_model_name", None)+ if not active:+ return False+ entry = (getattr(backend, "models", {}) or {}).get(active) or {}+ return bool(entry.get("is_vision"))++async def _mlx_count_chat_tokens(payload, request = None) -> Optional[JSONResponse]:"""Count with the resident MLX model's tokenizer, or None if MLX is not serving one.@@ -32434,7 +33016,7 @@ async def _mlx_count_chat_tokens(payload, request = None) -> Optional[JSONRespon# The completion's own helper: rebuilding it here is how a count prices a prompt# nobody sends.- system_prompt, messages, _image = _extract_content_parts(payload.messages)+ system_prompt, messages, _image = await _extract_content_parts_async(payload.messages)# The completion applies this once for both non-GGUF backends before it branches.# Only with a request: the helper's requestless mode injects the date unconditionally.if request is not None:@@ -32539,12 +33121,17 @@ async def _mlx_count_chat_tokens(payload, request = None) -> Optional[JSONRespon] or Noneelse:_tools_to_use = payload.tools or None- messages = _set_or_prepend_system_message(+ messages = await _promote_mcp_history_images_async(_structured_tool_history_for_local_template(- _flatten_content_parts_for_local_template(_openai_messages_for_passthrough(payload))+ _flatten_content_parts_for_local_template(+ _openai_messages_for_passthrough(+ payload, normalize_images = False, promote_mcp_images = False+ )+ )),- system_prompt,+ vision = False,)+ messages = _set_or_prepend_system_message(messages, system_prompt)system_prompt = ""elif _tools_to_use:# A PENDING turn is the shape this loop answers from exactly these messages, splicing@@ -32725,7 +33312,7 @@ async def chat_count_tokens(_tool["type"] = "function"# /apply-template swaps each image for a short media marker, so refuse rather than undercount.- if _request_has_image(payload):+ if _request_has_attached_image(payload):raise HTTPException(status_code = 503,detail = "Cannot count tokens for messages containing images.",@@ -32743,6 +33330,23 @@ async def chat_count_tokens(detail = "Cannot count tokens for messages containing video.",)+ # A replay envelope is countable on a text-only model, where generation strips it+ # and sends no pixels. On a model that reads images the completion promotes it, so+ # counting would underreport the prompt by every image-embedding token -- the same+ # reason a direct attachment is refused above. Checked ahead of the MLX dispatch,+ # which returns its own count and would otherwise never reach the guard.+ # The exact check parses the envelope, so it runs off the loop, and only once the+ # substring form says there is one to parse.+ if (+ _request_has_promotable_mcp_images(payload, exact = False)+ and await asyncio.to_thread(_request_has_promotable_mcp_images, payload)+ and await _resident_model_reads_images()+ ):+ raise HTTPException(+ status_code = 503,+ detail = "Cannot count tokens for messages containing images.",+ )+llama_backend = get_llama_cpp_backend()if not llama_backend.is_loaded:# The refusals above are about the request and so are shared; the ones below are@@ -32762,12 +33366,22 @@ async def chat_count_tokens(_count_messages = payload.messagesif _folds_studio_tool_history(payload, llama_backend):_count_messages = _folded_studio_tool_messages(_count_messages)- openai_messages = _strip_provider_synthetic_tool_history(- _drop_empty_assistant_sentinels([m.model_dump(exclude_none = True) for m in _count_messages])+ # Awaited: stripping parses the whole envelope, and this recount runs in the+ # background on every turn, so it must not do that on the shared loop.+ openai_messages = await _promote_mcp_history_images_async(+ _strip_provider_synthetic_tool_history(+ _drop_empty_assistant_sentinels(+ [m.model_dump(exclude_none = True) for m in _count_messages]+ )+ ),+ # Never as pixels: /apply-template prices a media marker, not the image.+ # This drops the base64 the completion path would have promoted, which is+ # the one part a count must not price as text.+ vision = False,)if not _takes_passthrough:openai_messages = _coalesce_consecutive_user_turns(openai_messages)- _system_prompt, _, _ = _extract_content_parts(payload.messages)+ _system_prompt, _, _ = await _extract_content_parts_async(payload.messages)# the verbatim passthrough carries no date line, so counting one here would overcount it.if not _takes_passthrough:_system_prompt = _apply_current_date_prompt(_system_prompt, request)@@ -32976,6 +33590,36 @@ async def anthropic_count_tokens(# Apply the same sanitization /messages does before generation, so the count# matches the prompt the real request would build (otherwise empty-assistant# sentinels / synthetic tool history inflate the count or hit the fallback).+ # Named and judged BEFORE the sanitizer, in the same order /v1/messages uses: it+ # folds every role="tool" into a user message on a template without tool-role+ # support, and both the refusal and _promote only look at tool messages.+ openai_messages = _named_anthropic_tool_results(openai_messages)++ # Like /chat/count_tokens, refuse promoted images: /apply-template counts+ # media markers without the projector tokens generation uses. Check the+ # translated, named results so Anthropic blocks are recognized and non-MCP+ # envelopes remain countable. Parse off the event loop after a substring check.+ if (+ llama_backend.is_vision+ and _messages_mention_mcp_images(openai_messages)+ and await asyncio.to_thread(_messages_have_promotable_mcp_images, openai_messages)+ ):+ raise HTTPException(+ status_code = 400,+ detail = "Cannot count tokens for messages containing images.",+ )++ # Priced as the images the request really sends, not as the base64 the envelope+ # carries: rendered verbatim a replayed screenshot counts thousands of tokens the+ # completion never sends, and the two endpoints must agree on the same prompt.+ # Awaited, like the completion path: promotion is Pillow work on a permitted+ # 40-megapixel raster, and running it inline stalls every other request on the loop.+ openai_messages = await _promote_mcp_history_images_async(+ openai_messages, vision = llama_backend.is_vision+ )++ # Apply the same sanitization /messages does before generation, so the count+ # matches the prompt the real request would build.openai_messages = _sanitize_anthropic_openai_messages(openai_messages, llama_backend)# /apply-template fetches remote media even though token counting only needs its marker._placeholder_remote_images_for_count(openai_messages)@@ -33289,11 +33933,26 @@ async def anthropic_messages(# builders apply the same strip; without it an Anthropic /v1/messages caller# replaying a prior provider-side tool_use forwards fake builtin tool# history to a backend with no matching function declarations.+ # BEFORE the sanitizer. On a template without tool-role support it folds every+ # role="tool" into a user message, and _promote only looks at tool messages -- so+ # the envelope survived as JSON in the prompt and the model read megabytes of+ # base64 while being shown no picture at all.+ openai_messages = _named_anthropic_tool_results(openai_messages)+ _anthropic_replayed_image_parts: list = []+ openai_messages = await _promote_mcp_history_images_async(+ openai_messages,+ vision = llama_backend.is_vision,+ promoted_out = _anthropic_replayed_image_parts,+ )+openai_messages = _sanitize_anthropic_openai_messages(openai_messages, llama_backend)# Enforce vision guard + re-encode embedded images to PNG so the Anthropic# endpoint matches /v1/chat/completions.- if _anthropic_has_image:+ # Promoted parts count too: _anthropic_has_image was read off the original+ # blocks, so a replay-only request took the synchronous branch and re-decoded up+ # to eight promoted PNGs on the shared loop.+ if _anthropic_has_image or _anthropic_replayed_image_parts:_has_image = await asyncio.to_thread(_normalize_anthropic_openai_images,openai_messages,@@ -33342,9 +34001,9 @@ async def anthropic_messages(# 1. enable_tools=true → server-side execution of built-in tools (Unsloth shorthand)# 2. tools=[...] only → client-side pass-through (standard Anthropic behavior)# 3. neither → plain chat- # The server-side agentic loop doesn't support multimodal input -- matches- # the `not image_b64` gate in /v1/chat/completions. requested_studio_tools and- # the mixed-mode rejection were computed before the switch above.+ # Match /v1/chat/completions: server tools reject caller attachments but+ # accept replayed images. Check original blocks because promotion also sets+ # _has_image. Tool selection and mixed-mode rejection preceded the switch.openai_client_tools = [toolfor tool in anthropic_tools_to_openai(payload.tools or [])@@ -33356,7 +34015,9 @@ async def anthropic_messages(# enable_tools=false). Explicit False always wins. Same predicate as the# permission gate above: deciding "did this request select server tools"# twice is what let the gate reject requests the router then served.- server_tools = _selects_server_tools and llama_backend.supports_tools and not _has_image+ server_tools = (+ _selects_server_tools and llama_backend.supports_tools and not _anthropic_has_image+ )# One short-circuiting chain: a backend whose supports_tools raises must not turn a plain# no-tools turn into a 500.client_tools = (@@ -33560,13 +34221,17 @@ async def anthropic_messages(async def _admitted_anthropic(coro, *, tool_loop: bool = False):try:- reservation, admission_config = _openai_llama_admission_reserve(+ reservation, admission_config = await _openai_llama_admission_reserve_async(request = request,llama_backend = llama_backend,payload = payload,tool_loop = tool_loop,# Only the tool branch resolves a catalogue; the plain branch sends none.injected_tools = openai_tools if tool_loop else None,+ # The list generation really runs on: already translated out of the+ # Anthropic block shape and already promoted, so a replayed screenshot+ # is charged as the image it becomes rather than as its base64.+ messages_override = openai_messages,)if tool_loop:_anthropic_admission_hold["reservation"] = reservation@@ -33815,6 +34480,7 @@ async def anthropic_messages(return llama_backend.generate_chat_completion_with_tools(reasoning_provenance = _think_prov,messages = openai_messages,+ replayed_image_parts = tuple(_anthropic_replayed_image_parts),tools = openai_tools,temperature = temperature,top_p = top_p,@@ -35586,6 +36252,24 @@ def _template_supports_tools(backend) -> bool:return True+def _named_anthropic_tool_results(messages: list[dict]) -> list[dict]:+ """Stamp each role="tool" message with the tool it answers.++ anthropic_messages_to_openai renders a tool_result block as tool_call_id plus+ content and no ``name``, and _promote reads an absent name as legacy MCP history+ it may trust. Delegates to the positional resolver rather than a conversation-wide+ id map: the backend restarts ids like call_0 every response, and a last-wins map+ renamed every earlier result with that id after the newest call.+ """+ names = _mcp_resolve_tool_names(messages)+ return [+ {**message, "name": names[index]}+ if message.get("role") == "tool" and not message.get("name") and index in names+ else message+ for index, message in enumerate(messages)+ ]++def _sanitize_anthropic_openai_messages(messages: list[dict], backend) -> list[dict]:"""Post-conversion chain shared by /v1/messages and its token counter, so adiverging list can never make input_tokens describe a different prompt.@@ -35775,7 +36459,12 @@ def _splice_image_into_last_user(messages: list[dict], image_part: dict) -> Nonemessages.append({"role": "user", "content": [image_part]})-def _openai_messages_for_passthrough(payload, normalize_images: bool = True) -> list[dict]:+def _openai_messages_for_passthrough(+ payload,+ vision: bool = False,+ normalize_images: bool = True,+ promote_mcp_images: bool = True,+) -> list[dict]:"""Build OpenAI-format message dicts for the /v1/chat/completionspassthrough path.@@ -35805,6 +36494,10 @@ def _openai_messages_for_passthrough(payload, normalize_images: bool = True) ->messages = _strip_provider_synthetic_tool_history(_drop_empty_assistant_sentinels([m.model_dump(exclude_none = True) for m in payload.messages]))+ # Left on for the local-template rebuild, which promotes them itself once its+ # flatten has run: promoted here, the flatten would drop the parts again.+ if promote_mcp_images:+ messages = promote_mcp_history_images(messages, vision = vision)if normalize_images:_normalize_openai_image_parts_to_png(messages)@@ -35881,7 +36574,11 @@ def _structured_tool_history_for_local_template(messages: list[dict]) -> list[direturn out-def _openai_messages_for_gguf_chat(payload, is_vision: bool) -> tuple[list[dict], bool]:+def _openai_messages_for_gguf_chat(+ payload,+ is_vision: bool,+ promoted_out: "Optional[list]" = None,+) -> tuple[list[dict], bool]:"""Build llama-server messages for the standard GGUF chat path.llama-server accepts OpenAI multimodal content parts directly. Preserve all@@ -35911,13 +36608,22 @@ def _openai_messages_for_gguf_chat(payload, is_vision: bool) -> tuple[list[dict]}_splice_image_into_last_user(messages, image_part)has_image = _normalize_anthropic_openai_images(messages, is_vision)- return messages, has_image+ return (+ promote_mcp_history_images(messages, vision = is_vision, promoted_out = promoted_out),+ has_image,+ )-async def _openai_messages_for_gguf_chat_async(payload, is_vision: bool) -> tuple[list[dict], bool]:+async def _openai_messages_for_gguf_chat_async(+ payload,+ is_vision: bool,+ promoted_out: "Optional[list]" = None,+) -> tuple[list[dict], bool]:if _request_has_image(payload):- return await asyncio.to_thread(_openai_messages_for_gguf_chat, payload, is_vision)- return _openai_messages_for_gguf_chat(payload, is_vision)+ return await asyncio.to_thread(+ _openai_messages_for_gguf_chat, payload, is_vision, promoted_out+ )+ return _openai_messages_for_gguf_chat(payload, is_vision, promoted_out)def _extract_response_format(payload):@@ -35955,7 +36661,9 @@ def _build_openai_passthrough_body(extensions (``enable_tools``, ``enabled_tools``, ``session_id``, ...) neverleak to the backend."""- messages = _openai_messages_for_passthrough(payload)+ messages = _openai_messages_for_passthrough(+ payload, vision = bool(getattr(llama_backend, "is_vision", False))+ )system_prompt, _, _ = _extract_content_parts(payload.messages)messages = _set_or_prepend_system_message(messages, system_prompt)# Markup is broken in _build_passthrough_payload, shared with both /v1/messages (#7066).@@ -36036,7 +36744,7 @@ async def _openai_passthrough_stream(_tracker = _TrackedCancel.for_payload(cancel_event, payload, *_cancel_keys)_tracker.__enter__()try:- reservation, admission_config = _openai_llama_admission_reserve(+ reservation, admission_config = await _openai_llama_admission_reserve_async(request = request,llama_backend = llama_backend,payload = payload,@@ -37097,7 +37805,7 @@ async def _openai_passthrough_non_streaming():"""Non-streaming pass-through guarded by local llama-server admission."""try:- reservation, admission_config = _openai_llama_admission_reserve(+ reservation, admission_config = await _openai_llama_admission_reserve_async(request = request,llama_backend = llama_backend,payload = payload,diff --git a/studio/backend/tests/test_anthropic_messages.py b/studio/backend/tests/test_anthropic_messages.pyindex cbc53e29311..a4f08335bfe 100644--- a/studio/backend/tests/test_anthropic_messages.py+++ b/studio/backend/tests/test_anthropic_messages.py@@ -5490,3 +5490,357 @@ def test_x_unsloth_effort_still_outranks_thinking_when_sent_explicitly():)assert args["enable_thinking"] is Trueassert args["reasoning_effort"] == "high"+++def test_the_anthropic_paths_promote_a_replayed_mcp_envelope():+ """A client replaying an Anthropic history sends the envelope back inside the+ tool_result. Without promotion the model reads megabytes of base64 as text and is+ shown no picture -- the very defect this feature exists to remove, on the endpoint+ Claude Code actually uses."""+ import inspect++ from routes import inference++ generate = inspect.getsource(inference.anthropic_messages)+ assert (+ "_promote_mcp_history_images_async(" in generate+ ), "the /v1/messages path never promotes the replayed envelope"+ assert (+ "replayed_image_parts = tuple(_anthropic_replayed_image_parts)" in generate+ ), "the loop's conversation cap has to start from what promotion put back"++ count = inspect.getsource(inference.anthropic_count_tokens)+ assert "await _promote_mcp_history_images_async(" in count, (+ "the count endpoint prices the base64 the completion never sends, and must "+ "not do the Pillow work inline on the event loop"+ )+ assert "messages_override = openai_messages" in generate, (+ "admission has to reserve against the translated list, or an Anthropic "+ "tool_result block prices its envelope as dense text"+ )+++def test_an_anthropic_replay_hands_the_model_the_picture_not_the_base64():+ import asyncio+ import base64+ import io+ import json++ from PIL import Image++ from core.inference import mcp_images+ from routes.inference import _promote_mcp_history_images_async++ buffer = io.BytesIO()+ Image.new("RGB", (12, 12), (24, 90, 219)).save(buffer, format = "PNG")+ payload = base64.b64encode(buffer.getvalue()).decode()+ envelope = json.dumps([{"data": payload, "mimeType": "image/png"}])++ # The shape anthropic_messages_to_openai produces from a replayed tool_result.+ messages = [+ {"role": "user", "content": "take a screenshot"},+ {+ "role": "assistant",+ "content": "",+ "tool_calls": [+ {+ "id": "toolu_0",+ "type": "function",+ "function": {"name": "mcp__shot__capture", "arguments": "{}"},+ }+ ],+ },+ {+ "role": "tool",+ "tool_call_id": "toolu_0",+ "name": "mcp__shot__capture",+ "content": "[1 image returned]\n" + mcp_images.SENTINEL + envelope,+ },+ {"role": "user", "content": "what colour was it"},+ ]++ promoted: list = []+ out = asyncio.run(+ _promote_mcp_history_images_async(messages, vision = True, promoted_out = promoted)+ )++ text = "".join(+ message["content"] if isinstance(message.get("content"), str) else "" for message in out+ ) + "".join(+ part.get("text", "")+ for message in out+ if isinstance(message.get("content"), list)+ for part in message["content"]+ if part.get("type") == "text"+ )+ assert mcp_images.SENTINEL not in text+ assert payload not in text, "the base64 reached the model as prompt text"+ assert len(promoted) == 1+ assert (+ sum(+ 1+ for message in out+ if isinstance(message.get("content"), list)+ for part in message["content"]+ if part.get("type") == "image_url"+ )+ == 1+ )+++def test_a_text_only_anthropic_model_is_not_shown_the_envelope_either():+ import asyncio+ import json++ from core.inference import mcp_images+ from routes.inference import _promote_mcp_history_images_async++ envelope = json.dumps([{"data": "QUJD", "mimeType": "image/png"}])+ messages = [+ {+ "role": "tool",+ "tool_call_id": "toolu_0",+ "name": "mcp__shot__capture",+ "content": "[1 image returned]\n" + mcp_images.SENTINEL + envelope,+ }+ ]++ out = asyncio.run(_promote_mcp_history_images_async(messages, vision = False))++ assert mcp_images.SENTINEL not in json.dumps(out)+ assert not any(isinstance(message.get("content"), list) for message in out)+++def test_the_anthropic_count_refuses_a_promoted_image_rather_than_undercount():+ """count_chat_tokens renders /apply-template, which swaps each image for a short+ media marker. Counting a promoted envelope there reports none of the projector+ tokens /v1/messages really spends, and an undercount is what a client sizes its+ context against -- so the OpenAI counter refuses this shape and so must this one."""+ import inspect++ from routes import inference++ count = inspect.getsource(inference.anthropic_count_tokens)+ refusal = count.index(+ "asyncio.to_thread(_messages_have_promotable_mcp_images, openai_messages)"+ )+ promotion = count.index("_promote_mcp_history_images_async(")+ assert refusal < promotion, (+ "the refusal has to come BEFORE promotion, or the envelope is already "+ "image parts by the time it is checked"+ )+ assert "Cannot count tokens for messages containing images." in count+ assert (+ "llama_backend.is_vision\n"+ " and _messages_mention_mcp_images(openai_messages)\n"+ " and await asyncio.to_thread(_messages_have_promotable_mcp_images, openai_messages)"+ ) in count, (+ "a text-only model has the envelope stripped and sends no pixels, so it "+ "must still be counted rather than refused"+ )+++def test_the_anthropic_envelope_is_promoted_before_tool_roles_are_folded_away():+ """A template without tool-role support has the sanitizer fold every role="tool"+ into a user message. _promote only looks at tool messages, so promoting after it+ left the envelope as JSON in the prompt: megabytes of base64 read as text, and no+ picture shown at all."""+ import inspect++ from routes import inference++ for name, source in (+ ("generation", inspect.getsource(inference.anthropic_messages)),+ ("count", inspect.getsource(inference.anthropic_count_tokens)),+ ):+ promote = source.index("_promote_mcp_history_images_async(")+ sanitize = source.index("_sanitize_anthropic_openai_messages(openai_messages")+ assert (+ promote < sanitize+ ), f"{name}: the fold runs first and the envelope never reaches promotion"+ named = source.index("_named_anthropic_tool_results(openai_messages)")+ assert named < promote, f"{name}: provenance has to be restored first"+++def test_an_anthropic_client_tool_is_not_trusted_as_an_mcp_image_source():+ """anthropic_messages_to_openai renders a tool_result with tool_call_id and no+ name, and _promote reads an absent name as legacy MCP history it may trust. An+ ordinary client tool whose output merely ends in a valid suffix was therefore+ promoted as image input on the strength of nothing."""+ import base64+ import io+ import json++ from PIL import Image++ from core.inference import mcp_images+ from routes.inference import _named_anthropic_tool_results++ buffer = io.BytesIO()+ Image.new("RGB", (8, 8), (10, 120, 200)).save(buffer, format = "PNG")+ envelope = json.dumps(+ [{"data": base64.b64encode(buffer.getvalue()).decode(), "mimeType": "image/png"}]+ )+ translated = [+ {+ "role": "assistant",+ "content": "",+ "tool_calls": [+ {+ "id": "toolu_0",+ "type": "function",+ "function": {"name": "read_file", "arguments": "{}"},+ }+ ],+ },+ {+ "role": "tool",+ "tool_call_id": "toolu_0",+ "content": "here you go\n" + mcp_images.SENTINEL + envelope,+ },+ ]++ named = _named_anthropic_tool_results(translated)+ assert named[1]["name"] == "read_file", "the result was not correlated to its call"++ out = mcp_images.promote_history(named, vision = True)+ assert not any(+ isinstance(m.get("content"), list) for m in out+ ), "a non-mcp__ tool was promoted as trusted image input"+ # The suffix still comes off the text for everyone, MCP or not.+ assert mcp_images.SENTINEL not in json.dumps(out)+++def test_a_real_mcp_tool_still_promotes_through_the_anthropic_naming():+ import base64+ import io+ import json++ from PIL import Image++ from core.inference import mcp_images+ from routes.inference import _named_anthropic_tool_results++ buffer = io.BytesIO()+ Image.new("RGB", (8, 8), (10, 120, 200)).save(buffer, format = "PNG")+ envelope = json.dumps(+ [{"data": base64.b64encode(buffer.getvalue()).decode(), "mimeType": "image/png"}]+ )+ translated = [+ {+ "role": "assistant",+ "content": "",+ "tool_calls": [+ {+ "id": "toolu_0",+ "type": "function",+ "function": {"name": "mcp__shot__capture", "arguments": "{}"},+ }+ ],+ },+ {+ "role": "tool",+ "tool_call_id": "toolu_0",+ "content": "[1 image returned]\n" + mcp_images.SENTINEL + envelope,+ },+ ]++ out = mcp_images.promote_history(_named_anthropic_tool_results(translated), vision = True)++ assert (+ sum(+ 1+ for message in out+ if isinstance(message.get("content"), list)+ for part in message["content"]+ if part.get("type") == "image_url"+ )+ == 1+ )+++def test_repeated_anthropic_call_ids_are_paired_positionally():+ """A conversation-wide id map renamed every earlier result with that id after the+ newest call, suppressing an earlier MCP picture or trusting an earlier non-MCP one."""+ from routes.inference import _named_anthropic_tool_results++ def _call(name):+ return {+ "role": "assistant",+ "content": "",+ "tool_calls": [+ {"id": "call_0", "type": "function", "function": {"name": name, "arguments": "{}"}}+ ],+ }++ named = _named_anthropic_tool_results(+ [+ _call("mcp__shot__capture"),+ {"role": "tool", "tool_call_id": "call_0", "content": "first"},+ _call("read_file"),+ {"role": "tool", "tool_call_id": "call_0", "content": "second"},+ ]+ )++ assert [m.get("name") for m in named if m["role"] == "tool"] == [+ "mcp__shot__capture",+ "read_file",+ ]+++def test_promoted_parts_take_the_anthropic_normalizer_off_the_loop():+ """_anthropic_has_image was read off the original blocks, so a replay-only request+ took the synchronous branch and re-decoded up to eight promoted PNGs on the loop."""+ import inspect++ from routes import inference++ src = inspect.getsource(inference.anthropic_messages)+ assert "if _anthropic_has_image or _anthropic_replayed_image_parts:" in src+++def test_the_anthropic_count_refusal_is_name_aware():+ """The names are stamped from the calls right before the check, so a client tool+ whose output merely ends in a valid envelope -- never promoted, suffix stripped --+ must not turn a countable prompt into a 400."""+ import inspect+ import json++ from core.inference import mcp_images+ from routes import inference+ from routes.inference import _messages_have_promotable_mcp_images, _named_anthropic_tool_results++ src = inspect.getsource(inference.anthropic_count_tokens)+ assert "asyncio.to_thread(_messages_have_promotable_mcp_images, openai_messages)" in src+ assert "_messages_have_mcp_image_envelope(openai_messages)" not in src++ envelope = json.dumps([{"data": "QUJD", "mimeType": "image/png"}])++ def _translated(tool):+ return _named_anthropic_tool_results(+ [+ {+ "role": "assistant",+ "content": "",+ "tool_calls": [+ {+ "id": "toolu_0",+ "type": "function",+ "function": {"name": tool, "arguments": "{}"},+ }+ ],+ },+ {+ "role": "tool",+ "tool_call_id": "toolu_0",+ "content": "out\n" + mcp_images.SENTINEL + envelope,+ },+ ]+ )++ assert not _messages_have_promotable_mcp_images(_translated("read_file"))+ assert _messages_have_promotable_mcp_images(_translated("mcp__shot__capture"))+ # Unnamed and uncorrelated is still legacy history, and still refused.+ assert _messages_have_promotable_mcp_images(+ [{"role": "tool", "tool_call_id": "x", "content": "out\n" + mcp_images.SENTINEL + envelope}]+ )diff --git a/studio/backend/tests/test_chat_template_tool_arguments.py b/studio/backend/tests/test_chat_template_tool_arguments.pyindex 8a927ea93ce..276f8b48ce5 100644--- a/studio/backend/tests/test_chat_template_tool_arguments.py+++ b/studio/backend/tests/test_chat_template_tool_arguments.py@@ -309,3 +309,75 @@ def test_lenient_template_never_sees_a_split_conversation():apply_chat_template_for_generation(_Lenient(), _parallel_conv())assert seen["n"] == 4 # unsplit+++_MINISTRAL_IMAGE_TEMPLATE = """+{% set ns = namespace(index=0) %}+{% for message in messages %}+{% if message.role == 'user' or (message.role == 'assistant' and not message.tool_calls) %}+{% if (message.role == 'user') != (ns.index % 2 == 0) %}+{{ raise_exception('After the optional system message, conversation roles must alternate user and assistant roles except for tool calls and results.') }}+{% endif %}+{% set ns.index = ns.index + 1 %}+{% endif %}+{% for part in message.content if message.content is not string %}+{% if part.type == 'image' %}[IMG]{% endif %}+{% endfor %}+{% endfor %}+"""+++class _MinistralImageTokenizer:+ chat_template = _MINISTRAL_IMAGE_TEMPLATE++ def apply_chat_template(self, messages, **kwargs):+ from transformers.utils.chat_template_utils import _compile_jinja_template+ self.messages = messages+ return _compile_jinja_template(self.chat_template).render(messages = messages)++[email protected]("processor", [False, True])+def test_tool_image_turn_renders_with_ministral_alternation(processor):+ from core.inference.chat_template_helpers import render_prompt_with_boundary+ from core.inference.mcp_images import placeholder_turn++ conversation = _conv({})+ conversation.append(placeholder_turn(1))+ conversation.append({"role": "assistant", "content": "The image is red."})+ conversation.append({"role": "user", "content": "Another screenshot, please."})+ conversation.extend(_conv({})[1:])+ conversation.append(placeholder_turn(1))+ tokenizer = _MinistralImageTokenizer()+ render = render_prompt_with_boundary if processor else apply_chat_template_for_generation+ assert render(tokenizer, conversation).count("[IMG]") == 2+ assert [m["role"] for m in tokenizer.messages] == [+ "user",+ "assistant",+ "tool",+ "assistant",+ "user",+ "assistant",+ "user",+ "assistant",+ "tool",+ "assistant",+ "user",+ ]+ assert len(conversation) == 9+++def test_image_boundary_repair_preserves_markers_and_working_templates():+ from core.inference.mcp_images import placeholder_turn, prepare_image_turn_boundaries++ conversation = _conv({}) + [placeholder_turn(1)]+ assert prepare_image_turn_boundaries(conversation, None) is conversation+ assert prepare_image_turn_boundaries(conversation, "{{ messages }}") is conversation+ fixed = prepare_image_turn_boundaries(conversation, _MINISTRAL_IMAGE_TEMPLATE)+ assert fixed[-1] is conversation[-1]+ assert prepare_image_turn_boundaries(fixed, _MINISTRAL_IMAGE_TEMPLATE) is fixed+++def test_image_boundary_repair_does_not_hide_invalid_caller_roles():+ conversation = _conv({}) + [{"role": "user", "content": "A second user turn."}]+ with pytest.raises(Exception, match = "conversation roles must alternate"):+ apply_chat_template_for_generation(_MinistralImageTokenizer(), conversation)diff --git a/studio/backend/tests/test_external_tool_call_id_replay.py b/studio/backend/tests/test_external_tool_call_id_replay.pyindex 86eb892d5ab..249af5e9cab 100644--- a/studio/backend/tests/test_external_tool_call_id_replay.py+++ b/studio/backend/tests/test_external_tool_call_id_replay.py@@ -162,3 +162,111 @@ def test_replay_is_idempotent_for_every_provider():once, _ = _replayed_ids(MINTED, provider_type = provider)twice, paired = _replayed_ids(once, provider_type = provider)assert twice == once == paired, (provider, once, twice)+++def _mcp_image_history():+ import base64+ import io+ import json++ from PIL import Image++ from core.inference import mcp_images++ buffer = io.BytesIO()+ Image.new("RGB", (6, 6), (10, 120, 200)).save(buffer, format = "PNG")+ envelope = json.dumps(+ [{"data": base64.b64encode(buffer.getvalue()).decode(), "mimeType": "image/png"}]+ )+ return [+ ChatMessage.model_validate(+ {+ "role": "assistant",+ "content": "",+ "tool_calls": [+ {+ "id": ORIGINAL,+ "type": "function",+ "function": {"name": "mcp__fs__read_media_file", "arguments": "{}"},+ }+ ],+ }+ ),+ ChatMessage.model_validate(+ {+ "role": "tool",+ "tool_call_id": ORIGINAL,+ "content": "[1 image returned]\n" + mcp_images.SENTINEL + envelope,+ }+ ),+ ]+++def test_replayed_mcp_images_become_an_image_turn_on_a_vision_provider():+ out = _build_external_messages(+ _mcp_image_history(), supports_vision = True, provider_type = "openai"+ )++ assert [message["role"] for message in out] == ["assistant", "tool", "user"]+ assert out[1]["content"] == "[1 image returned]"+ assert out[2]["content"][1]["image_url"]["url"].startswith("data:image/png;base64,")+++def test_replayed_mcp_images_are_stripped_on_a_text_only_provider():+ out = _build_external_messages(+ _mcp_image_history(), supports_vision = False, provider_type = "openai"+ )++ assert [message["role"] for message in out] == ["assistant", "tool"]+ assert out[1]["content"] == "[1 image returned]"+++def test_replay_promotion_is_moved_off_the_event_loop(monkeypatch):+ """Promoting an envelope decodes and re-encodes every picture in it. That is+ PIL work, and _proxy_to_external_provider calls the builder from the loop."""+ import asyncio++ import routes.inference as inference_route++ hops: list = []+ real_to_thread = asyncio.to_thread++ async def counting_to_thread(fn, *a, **kw):+ hops.append(getattr(fn, "__name__", str(fn)))+ return await real_to_thread(fn, *a, **kw)++ monkeypatch.setattr(inference_route.asyncio, "to_thread", counting_to_thread)++ out = asyncio.run(+ inference_route._build_external_messages_async(+ _mcp_image_history(), True, provider_type = "openai"+ )+ )++ assert hops == ["_build_external_messages"]+ assert out[-1]["content"][1]["image_url"]["url"].startswith("data:image/png;base64,")+++def test_a_plain_conversation_keeps_its_direct_call(monkeypatch):+ """No envelope, no PIL work: the thread hop would be pure overhead."""+ import asyncio++ import routes.inference as inference_route++ hops: list = []++ async def counting_to_thread(fn, *a, **kw):+ hops.append(getattr(fn, "__name__", str(fn)))+ return await asyncio.to_thread(fn, *a, **kw)++ monkeypatch.setattr(inference_route.asyncio, "to_thread", counting_to_thread)++ asyncio.run(+ inference_route._build_external_messages_async(+ [ChatMessage.model_validate({"role": "user", "content": "hello"})],+ True,+ provider_type = "openai",+ )+ )++ assert hops == []diff --git a/studio/backend/tests/test_llama_cpp_tool_loop.py b/studio/backend/tests/test_llama_cpp_tool_loop.pyindex 32e93c68f73..44f96462701 100644--- a/studio/backend/tests/test_llama_cpp_tool_loop.py+++ b/studio/backend/tests/test_llama_cpp_tool_loop.py@@ -115,6 +115,9 @@ def _run_tool_loop()+_MCP_PNG_B64 = "iVBORw0KGgoAAAANSUhEUgAAAAYAAAAGCAIAAABvrngfAAAAFElEQVR4nGM8ISfHgAqY0PhUFgIAdYgBEED+8ToAAAAASUVORK5CYII="++def _sse(delta: dict) -> str:return "data: " + json.dumps({"choices": [{"index": 0, "delta": delta}]}) + "\n"@@ -6488,6 +6491,58 @@ def test_the_synthesized_final_pass_is_recosted_before_it_is_sent(monkeypatch):)+def _mcp_image_result() -> str:+ from core.inference import mcp_images+ image = {"data": _MCP_PNG_B64, "mimeType": "image/png"}+ return "[1 image returned]\n" + mcp_images.SENTINEL + json.dumps([image])+++def _vision_backend(monkeypatch, streams, payloads, *, vision: bool):+ backend = _make_backend(monkeypatch, streams, payloads)+ backend._is_vision = vision+ backend._mmproj_accepts_image = vision+ return backend+++def _run_mcp_image_turn(monkeypatch, *, vision: bool) -> list[dict]:+ streams = [+ _structured_tool_call("mcp__fs__read_media_file", {"path": "cat.png"}, "call_mcp"),+ [_sse({"content": "A tabby cat."}), _done()],+ ]+ payloads: list[dict] = []+ backend = _vision_backend(monkeypatch, streams, payloads, vision = vision)+ monkeypatch.setattr(+ "core.inference.tools.execute_tool",+ lambda name, arguments, **_kwargs: _mcp_image_result(),+ )++ list(+ backend.generate_chat_completion_with_tools(+ messages = [{"role": "user", "content": "describe the image"}],+ tools = [{"type": "function", "function": {"name": "mcp__fs__read_media_file"}}],+ max_tool_iterations = 2,+ )+ )+ return payloads[1]["messages"]+++def test_mcp_images_reach_a_vision_model_as_their_own_user_turn(monkeypatch):+ messages = _run_mcp_image_turn(monkeypatch, vision = True)++ tool_message = next(m for m in messages if m["role"] == "tool")+ assert "__MCP_IMAGES__" not in tool_message["content"]+ assert messages[-1]["role"] == "user"+ assert messages[-1]["content"][1]["type"] == "image_url"+ assert messages[-1]["content"][1]["image_url"]["url"].startswith("data:image/png;base64,")+++def test_mcp_images_are_not_sent_to_a_text_only_model(monkeypatch):+ messages = _run_mcp_image_turn(monkeypatch, vision = False)++ assert [m["role"] for m in messages] == ["user", "assistant", "tool"]+ assert "__MCP_IMAGES__" not in messages[-1]["content"]++@pytest.mark.parametrize("wanted", [True, False])def test_the_decode_slot_is_asked_for_and_read_only_when_a_caller_wants_it(monkeypatch, wanted):"""Verbose alone would attach the whole prompt, so it travels with the narrowing."""diff --git a/studio/backend/tests/test_mcp_flatten_result.py b/studio/backend/tests/test_mcp_flatten_result.pyindex 3f368442c03..82343862169 100644--- a/studio/backend/tests/test_mcp_flatten_result.py+++ b/studio/backend/tests/test_mcp_flatten_result.py@@ -79,28 +79,28 @@ def test_text_only_result_unchanged():def test_image_only_result_keeps_image_and_notes_model():flat = _flatten_result(_result(_image()))body, payload = flat.split("\n" + MCP_IMAGES_SENTINEL, 1)- assert body == "[1 image attached; displayed to the user]"+ assert body == "[1 image returned]"assert json.loads(payload) == [{"data": PNG_B64, "mimeType": "image/png"}]def test_text_plus_image_keeps_both():flat = _flatten_result(_result(_text("Took a screenshot"), _image()))body, payload = flat.split("\n" + MCP_IMAGES_SENTINEL, 1)- assert body == "Took a screenshot\n[1 image attached; displayed to the user]"+ assert body == "Took a screenshot\n[1 image returned]"assert json.loads(payload)[0]["mimeType"] == "image/png"def test_multiple_images_pluralized():flat = _flatten_result(_result(_image(), _image(mime = "image/jpeg")))body, payload = flat.split("\n" + MCP_IMAGES_SENTINEL, 1)- assert "[2 images attached; displayed to the user]" in body+ assert "[2 images returned]" in bodyassert [img["mimeType"] for img in json.loads(payload)] == ["image/png", "image/jpeg"]def test_strip_result_for_model_drops_image_payload():flat = _flatten_result(_result(_text("Took a screenshot"), _image()))stripped = strip_result_for_model(flat)- assert stripped == "Took a screenshot\n[1 image attached; displayed to the user]"+ assert stripped == "Took a screenshot\n[1 image returned]"assert PNG_B64 not in stripped@@ -120,12 +120,11 @@ def test_strip_preserves_non_image_json_after_marker():def test_strip_removes_only_valid_terminal_envelope():text = ("Earlier mention: __MCP_IMAGES__: is documented here"- "\n[1 image attached; displayed to the user]"+ "\n[1 image returned]"'\n__MCP_IMAGES__:[{"data": "AAAA", "mimeType": "image/png"}]')assert strip_result_for_model(text) == (- "Earlier mention: __MCP_IMAGES__: is documented here"- "\n[1 image attached; displayed to the user]"+ "Earlier mention: __MCP_IMAGES__: is documented here\n[1 image returned]")@@ -143,7 +142,7 @@ def test_error_result_keeps_error_prefix_and_images():def test_image_only_error_no_longer_reports_no_content():flat = _flatten_result(_result(_image(), is_error = True))- assert flat.startswith("Error: [1 image attached")+ assert flat.startswith("Error: [1 image returned")assert "tool returned no content" not in flat@@ -158,7 +157,7 @@ def test_oversized_budget_shared_across_images():big = "A" * (MAX_IMAGE_PAYLOAD_CHARS - 10)flat = _flatten_result(_result(_image(data = big), _image()))body, payload = flat.split("\n" + MCP_IMAGES_SENTINEL, 1)- assert "1 image attached" in body+ assert "1 image returned" in bodyassert "1 image omitted (too large)" in bodyimages = json.loads(payload)assert len(images) == 1 and images[0]["data"] == big@@ -187,7 +186,7 @@ def test_image_mirrored_in_structured_content_is_not_dumped():structured = {"content": [{"type": "image", "data": PNG_B64, "mimeType": "image/png"}]}flat = _flatten_result(_result(_image(), structured = structured))body, payload = flat.split("\n" + MCP_IMAGES_SENTINEL, 1)- assert body == "[1 image attached; displayed to the user]"+ assert body == "[1 image returned]"assert json.loads(payload) == [{"data": PNG_B64, "mimeType": "image/png"}]assert PNG_B64 not in strip_result_for_model(flat)huge = "A" * (MAX_IMAGE_PAYLOAD_CHARS + 1)@@ -200,16 +199,14 @@ def test_image_mirrored_in_structured_content_is_not_dumped():def test_image_and_audio_share_one_note():flat = _flatten_result(_result(_image(), _audio()))body, payload = flat.split("\n" + MCP_IMAGES_SENTINEL, 1)- assert body == (- "[1 image attached; displayed to the user; audio attachment (audio/wav) not shown to the model]"- )+ assert body == ("[1 image returned; audio attachment (audio/wav) not shown to the model]")assert json.loads(payload) == [{"data": PNG_B64, "mimeType": "image/png"}]def test_independent_structured_content_is_kept_beside_attachments():flat = _flatten_result(_result(_image(), structured = {"rows": 3, "max": 41.5}))body, payload = flat.split("\n" + MCP_IMAGES_SENTINEL, 1)- assert body == "{'rows': 3, 'max': 41.5}\n[1 image attached; displayed to the user]"+ assert body == "{'rows': 3, 'max': 41.5}\n[1 image returned]"assert json.loads(payload) == [{"data": PNG_B64, "mimeType": "image/png"}]assert _flatten_result(_result(_audio(), structured = {"duration_s": 1.0})) == ("{'duration_s': 1.0}\n[audio attachment (audio/wav) not shown to the model]"@@ -374,14 +371,14 @@ def test_stdio_session_call_also_passes_raise_on_error_false(monkeypatch):def test_embedded_resource_image_is_rendered():flat = _flatten_result(_result(_blob_resource()))body, payload = flat.split("\n" + MCP_IMAGES_SENTINEL, 1)- assert body == "[1 image attached; displayed to the user]"+ assert body == "[1 image returned]"assert json.loads(payload) == [{"data": PNG_B64, "mimeType": "image/png"}]def test_embedded_resource_image_shares_budget_with_image_content():flat = _flatten_result(_result(_text("rendered"), _image(), _blob_resource(mime = "image/webp")))body, payload = flat.split("\n" + MCP_IMAGES_SENTINEL, 1)- assert body == "rendered\n[2 images attached; displayed to the user]"+ assert body == "rendered\n[2 images returned]"assert [img["mimeType"] for img in json.loads(payload)] == ["image/png", "image/webp"]@@ -441,7 +438,7 @@ def test_fastmcp_file_format_png_is_rendered():# fastmcp File(data=..., format="png") labels the blob application/pngflat = _flatten_result(_result(_blob_resource(mime = "application/png")))body, payload = flat.split("\n" + MCP_IMAGES_SENTINEL, 1)- assert body == "[1 image attached; displayed to the user]"+ assert body == "[1 image returned]"assert json.loads(payload) == [{"data": PNG_B64, "mimeType": "image/png"}]diff --git a/studio/backend/tests/test_mcp_images.py b/studio/backend/tests/test_mcp_images.pynew file mode 100644index 00000000000..b341a283f61--- /dev/null+++ b/studio/backend/tests/test_mcp_images.py@@ -0,0 +1,2827 @@+# SPDX-License-Identifier: AGPL-3.0-only+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0++from __future__ import annotations++import base64+import importlib+import importlib.machinery+import io+import json+import json as _json+import sys+import types+from pathlib import Path+from unittest.mock import MagicMock++import pytest++_STUBBED = {}++_BACKEND_DIR = str(Path(__file__).resolve().parent.parent)+if _BACKEND_DIR not in sys.path:+ sys.path.insert(0, _BACKEND_DIR)+++# core.inference.inference imports unsloth and trl at module scope; studio-backend-ci.yml+# does not install them, so stub them here rather than rely on an earlier test file having done so.+def _stub_if_missing(+ name,+ attrs = (),+ named_spec = False,+):+ if name in sys.modules:+ return+ try:+ importlib.import_module(name)+ return+ except Exception: # noqa: BLE001+ pass+ module = types.ModuleType(name)+ module.__spec__ = importlib.machinery.ModuleSpec(name, None) if named_spec else None+ module.__version__ = "0.0.0"+ module.__getattr__ = lambda _attr: MagicMock()+ for attr in attrs:+ setattr(module, attr, MagicMock())+ sys.modules[name] = module+ _STUBBED[name] = module+ parent, _, child = name.rpartition(".")+ if parent and parent in sys.modules:+ setattr(sys.modules[parent], child, module)++[email protected](scope = "module", autouse = True)+def _optional_inference_dependencies():+ try:+ for _torchao in (+ "torchao",+ "torchao.prototype",+ "torchao.prototype.safetensors",+ "torchao.prototype.safetensors.safetensors_support",+ "torchao.prototype.safetensors.safetensors_utils",+ "torchao.quantization",+ "torchao.dtypes",+ "torchao.float8",+ "torchao.utils",+ ):+ _stub_if_missing(_torchao, named_spec = True)++ _stub_if_missing(+ "unsloth", ("FastLanguageModel", "FastVisionModel", "is_bfloat16_supported")+ )+ _stub_if_missing("unsloth.chat_templates", ("get_chat_template",))+ _stub_if_missing("unsloth_zoo")+ _stub_if_missing("trl", ("SFTTrainer", "SFTConfig"))+ yield+ finally:+ for name, module in reversed(list(_STUBBED.items())):+ if sys.modules.get(name) is module:+ sys.modules.pop(name)+ parent, _, child = name.rpartition(".")+ if parent in sys.modules and vars(sys.modules[parent]).get(child) is module:+ delattr(sys.modules[parent], child)+ _STUBBED.clear()+++from PIL import Image++from core.inference import mcp_images+from core.inference.mcp_images import (+ IMAGE_TURN_TEXT,+ MAX_IMAGE_EDGE,+ MAX_MODEL_IMAGES,+ append_image_turn,+ content_parts,+ promote_history,+ split_images,+)+++def _png(size = (8, 8), fmt = "PNG") -> str:+ from PIL import Image++ buffer = io.BytesIO()+ Image.new("RGB", size, (10, 120, 200)).save(buffer, format = fmt)+ return base64.b64encode(buffer.getvalue()).decode("ascii")+++def _image(data = None, mime = "image/png") -> dict:+ return {"data": data if data is not None else _png(), "mimeType": mime}+++def _envelope(text: str, *images: dict) -> str:+ return text + "\n" + mcp_images.SENTINEL + json.dumps(list(images))+++def _decode(part: dict):+ from PIL import Image++ url = part["image_url"]["url"]+ assert url.startswith("data:image/png;base64,")+ return Image.open(io.BytesIO(base64.b64decode(url.split(",", 1)[1])))+++def test_split_returns_text_and_images():+ text, images = split_images(_envelope("a screenshot", _image()))++ assert text == "a screenshot"+ assert len(images) == 1+++def test_split_leaves_a_result_that_only_mentions_the_marker():+ result = "docs say the marker is\n__MCP_IMAGES__: and nothing follows"++ assert split_images(result) == (result, [])+++def test_split_leaves_an_envelope_that_is_not_an_image_array():+ result = 'log\n__MCP_IMAGES__:["not", "image", "dicts"]'++ assert split_images(result) == (result, [])+++def test_content_parts_reencode_to_png():+ parts = content_parts([_image(data = _png(fmt = "WEBP"), mime = "image/webp")])++ assert len(parts) == 1+ assert _decode(parts[0]).format == "PNG"+++def test_content_parts_downscale_a_large_image():+ parts = content_parts([_image(data = _png(size = (MAX_IMAGE_EDGE * 2, MAX_IMAGE_EDGE)))])++ assert max(_decode(parts[0]).size) == MAX_IMAGE_EDGE+++def test_content_parts_cap_how_many_reach_the_model():+ parts = content_parts([_image() for _ in range(MAX_MODEL_IMAGES + 3)])++ assert len(parts) == MAX_MODEL_IMAGES+++def test_content_parts_drop_an_undecodable_payload():+ assert content_parts([_image(data = "not base64 at all")]) == []+ assert content_parts([_image(data = base64.b64encode(b"nope").decode())]) == []+++def test_image_turn_is_its_own_user_message():+ conversation = [{"role": "tool", "name": "mcp__fs__read", "content": "[1 image returned]"}]++ append_image_turn(conversation, [_image()])++ assert conversation[-1]["role"] == "user"+ assert conversation[-1]["content"][0] == {"type": "text", "text": IMAGE_TURN_TEXT}+ assert conversation[-1]["content"][1]["type"] == "image_url"+ assert conversation[1] is conversation[-1]+++def test_image_turn_merges_into_a_trailing_user_turn():+ conversation = [{"role": "user", "content": "a nudge"}]++ append_image_turn(conversation, [_image()])++ assert len(conversation) == 1+ assert conversation[0]["content"][0] == {"type": "text", "text": "a nudge"}+ assert conversation[0]["content"][1]["type"] == "image_url"+++def test_image_turn_is_skipped_when_nothing_decodes():+ conversation = [{"role": "tool", "content": "[1 image returned]"}]++ append_image_turn(conversation, [_image(data = "///")])++ assert len(conversation) == 1+++def test_history_promotes_a_replayed_envelope():+ messages = promote_history(+ [+ {"role": "user", "content": "what is in the file"},+ {"role": "assistant", "content": "", "tool_calls": [{"id": "call_0"}]},+ {+ "role": "tool",+ "tool_call_id": "call_0",+ "content": _envelope("[1 image returned]", _image()),+ },+ {"role": "assistant", "content": "a blue square"},+ ],+ vision = True,+ )++ assert messages[2]["content"] == "[1 image returned]"+ assert messages[3]["role"] == "user"+ assert messages[3]["content"][1]["type"] == "image_url"+ assert messages[4]["content"] == "a blue square"+++def test_history_flushes_after_the_whole_batch_of_tool_results():+ messages = promote_history(+ [+ {"role": "assistant", "content": "", "tool_calls": [{"id": "a"}, {"id": "b"}]},+ {"role": "tool", "tool_call_id": "a", "content": _envelope("first", _image())},+ {"role": "tool", "tool_call_id": "b", "content": _envelope("second", _image())},+ ],+ vision = True,+ )++ assert [message["role"] for message in messages] == ["assistant", "tool", "tool", "user"]+ assert len(messages[3]["content"]) == 3+++def test_history_strips_the_envelope_for_a_model_that_cannot_see_it():+ messages = promote_history(+ [{"role": "tool", "content": _envelope("[1 image returned]", _image())}],+ vision = False,+ )++ assert messages == [{"role": "tool", "content": "[1 image returned]"}]+++def test_history_keeps_an_image_only_result_from_emptying_its_tool_message():+ messages = promote_history(+ [{"role": "tool", "content": _envelope("", _image())}],+ vision = False,+ )++ assert messages[0]["content"] == "[image returned]"+++def test_history_leaves_other_messages_untouched():+ original = [+ {"role": "user", "content": "hello"},+ {"role": "tool", "content": "plain result"},+ ]++ assert promote_history(original, vision = True) == original+++def test_history_merges_into_a_following_user_turn():+ messages = promote_history(+ [+ {"role": "tool", "content": _envelope("[1 image returned]", _image())},+ {"role": "user", "content": "what colour was it"},+ ],+ vision = True,+ )++ assert [message["role"] for message in messages] == ["tool", "user"]+ assert messages[1]["content"][0]["type"] == "image_url"+ # The note rides along on the merge, ahead of the question's own text.+ assert messages[1]["content"][1]["text"].startswith(mcp_images.IMAGE_TURN_TEXT)+ assert messages[1]["content"][2] == {"type": "text", "text": "what colour was it"}+++def test_local_history_carries_markers_and_payloads():+ messages, payloads = mcp_images.promote_history_local(+ [+ {"role": "user", "content": "what is in the file"},+ {"role": "tool", "content": _envelope("[1 image returned]", _image())},+ {"role": "assistant", "content": "a blue square"},+ ],+ vision = True,+ )++ assert messages[2]["role"] == "user"+ assert messages[2]["content"][0] == {"type": "image"}+ assert len(payloads) == 1+ assert base64.b64decode(payloads[0])[:8] == b"\x89PNG\r\n\x1a\n"+++def test_local_history_merges_markers_into_a_following_user_turn():+ messages, payloads = mcp_images.promote_history_local(+ [+ {"role": "tool", "content": _envelope("[1 image returned]", _image())},+ {"role": "user", "content": "what colour was it"},+ ],+ vision = True,+ )++ assert [message["role"] for message in messages] == ["tool", "user"]+ assert messages[1]["content"][0] == {"type": "image"}+ assert len(payloads) == 1+++def test_local_history_strips_without_vision():+ messages, payloads = mcp_images.promote_history_local(+ [{"role": "tool", "content": _envelope("[1 image returned]", _image())}],+ vision = False,+ )++ assert messages == [{"role": "tool", "content": "[1 image returned]"}]+ assert payloads == []+++def test_placeholder_turn_marks_one_image_per_payload():+ turn = mcp_images.placeholder_turn(2)++ assert [part["type"] for part in turn["content"]] == ["image", "image", "text"]+++def test_an_oversized_raster_is_rejected_off_the_header():+ from PIL import Image++ buffer = io.BytesIO()+ # Uniform colour, so a 9000x9000 raster still encodes to a few kilobytes and+ # clears every payload-size gate ahead of the decode.+ Image.new("RGB", (9000, 9000), (0, 0, 0)).save(buffer, format = "PNG")+ encoded = base64.b64encode(buffer.getvalue()).decode("ascii")+ from core.inference.mcp_client import MAX_IMAGE_PAYLOAD_CHARS++ assert len(encoded) < MAX_IMAGE_PAYLOAD_CHARS++ assert content_parts([{"data": encoded, "mimeType": "image/png"}]) == []+++def test_an_image_inside_the_pixel_budget_still_downscales():+ parts = content_parts([_image(data = _png(size = (2000, 1000)))])++ assert max(_decode(parts[0]).size) == MAX_IMAGE_EDGE+++def test_local_history_caps_the_images_a_replay_resends():+ turns = []+ for _ in range(mcp_images.MAX_TOTAL_MODEL_IMAGES + 3):+ turns.append({"role": "tool", "content": _envelope("[1 image returned]", _image())})+ turns.append({"role": "assistant", "content": "noted"})++ messages, payloads = mcp_images.promote_history_local(turns, vision = True)+ markers = sum(+ 1+ for message in messages+ if isinstance(message.get("content"), list)+ for part in message["content"]+ if part.get("type") == "image"+ )++ assert len(payloads) == mcp_images.MAX_TOTAL_MODEL_IMAGES+ # A marker with no pixels behind it makes the processor count image tokens it+ # was given nothing for, so the two have to come down together.+ assert markers == len(payloads)+++def _svg_image() -> dict:+ # _flatten_result accepts image/svg+xml, and Pillow cannot decode it.+ svg = b'<svg xmlns="http://www.w3.org/2000/svg"><rect width="8" height="8"/></svg>'+ return {"data": base64.b64encode(svg).decode(), "mimeType": "image/svg+xml"}+++def test_undecodable_images_do_not_spend_the_quota():+ images = [_svg_image() for _ in range(MAX_MODEL_IMAGES)] + [_image(), _image()]++ parts = content_parts(images)++ assert len(parts) == 2+ assert len(mcp_images.png_payloads(images)) == 2+++def test_the_data_url_history_is_capped_too():+ turns = []+ for _ in range(mcp_images.MAX_TOTAL_MODEL_IMAGES + 3):+ turns.append({"role": "tool", "content": _envelope("[1 image returned]", _image())})+ turns.append({"role": "assistant", "content": "noted"})++ messages = promote_history(turns, vision = True)+ parts = mcp_images.count_image_parts(messages, "image_url")++ assert parts == mcp_images.MAX_TOTAL_MODEL_IMAGES+++def test_a_live_image_turn_merges_into_a_trailing_nudge():+ """append_deferred_nudges leaves a role=user turn; a second one in a row is+ what a strict VLM template rejects."""+ conversation = [+ {"role": "user", "content": "take a screenshot"},+ {"role": "assistant", "content": "", "tool_calls": [{"id": "c"}]},+ {"role": "tool", "tool_call_id": "c", "content": "[1 image returned]"},+ {"role": "user", "content": "One earlier request was not executed."},+ ]++ append_image_turn(conversation, [_image()])++ assert [m["role"] for m in conversation] == ["user", "assistant", "tool", "user"]+ assert conversation[-1]["content"][0] == {+ "type": "text",+ "text": "One earlier request was not executed.",+ }+ assert conversation[-1]["content"][1]["type"] == "image_url"+++def test_a_marker_turn_merges_into_a_trailing_nudge():+ conversation = [+ {"role": "tool", "tool_call_id": "c", "content": "[1 image returned]"},+ {"role": "user", "content": "One earlier request was not executed."},+ ]++ mcp_images.append_placeholder_turn(conversation, 2, 2)++ assert [m["role"] for m in conversation] == ["tool", "user"]+ # The note rides along: merged bare, the markers read as pictures attached to the+ # nudge rather than as the tool's output.+ assert [p["type"] for p in conversation[-1]["content"]] == ["text", "image", "image", "text"]+ assert conversation[-1]["content"][-1]["text"].startswith(mcp_images.IMAGE_TURN_TEXT)+++def test_a_marker_turn_still_opens_its_own_turn_after_a_tool_result():+ conversation = [{"role": "tool", "tool_call_id": "c", "content": "[1 image returned]"}]++ mcp_images.append_placeholder_turn(conversation, 1, 1)++ assert [m["role"] for m in conversation] == ["tool", "user"]+ assert conversation[-1]["content"][0] == {"type": "image"}+++def test_a_non_mcp_tool_never_has_its_output_read_as_images():+ """The envelope is a plain text suffix, so terminal output or a fetched page+ can end in a syntactically valid one. Only an mcp__ call is trusted."""+ from core.inference.tool_loop_controller import ToolCallCompletion, ToolCallDecision++ payload = _envelope("$ cat notes.txt", _image())++ def completion(tool_name):+ decision = ToolCallDecision(+ action = "execute",+ tool_name = tool_name,+ arguments = {},+ tool_call_id = "call_0",+ card_call_id = "",+ key = tool_name,+ provenance = {},+ noop_result = "",+ )+ return ToolCallCompletion(decision = decision, result = payload, executed = True)++ assert completion("mcp__fs__read_media_file").mcp_images()+ assert completion("bash").mcp_images() == []+ assert completion("web_fetch").mcp_images() == []+++def test_image_parts_are_dropped_for_a_text_only_fallback():+ from core.inference.inference import _without_image_parts++ messages = [+ {"role": "user", "content": "what is in the file"},+ {"role": "tool", "content": "[1 image returned]"},+ {+ "role": "user",+ "content": [{"type": "image"}, {"type": "text", "text": IMAGE_TURN_TEXT}],+ },+ {"role": "user", "content": [{"type": "image"}]},+ ]++ out = _without_image_parts(messages)++ # And the fallback branch really calls it, not just defines it.+ import inspect++ from core.inference import inference as inference_module++ body = inspect.getsource(inference_module.InferenceBackend._generate_chat_response_inner)+ assert "messages = _without_image_parts(messages)" in body++ assert out[0] == messages[0]+ assert out[1] == messages[1]+ # Collapsed back to a plain string, which is what a text template takes.+ assert out[2]["content"] == IMAGE_TURN_TEXT+ assert out[3]["content"] == ""+++def test_a_replay_only_turn_takes_the_vision_render():+ """The reasoning-channel markers follow the vision render, and #10092 made that+ unconditional on the vision path -- so what this PR has to keep true is that a+ replay-only turn reaches that path at all, with no attachment to trigger it."""+ import inspect++ from core.inference import inference as inference_module++ inner = inspect.getsource(inference_module.InferenceBackend._generate_chat_response_inner)+ assert (+ "if is_vision and (image or images):" in inner+ ), "a conversation whose only pictures are replayed still has to render as vision"+ vision = inspect.getsource(inference_module.InferenceBackend._generate_vision_response)+ assert "if attached:" in vision, "and the branch inside keys off every attached image"+++def test_a_named_non_mcp_tool_result_is_not_promoted_on_replay():+ """The live loop checks the call's tool name; the replay path reads the+ message's own `name`. Terminal output ending in a valid envelope must not+ become image input on the next request either."""+ history = [+ {+ "role": "tool",+ "tool_call_id": "call_0",+ "name": "bash",+ "content": _envelope("$ cat notes.txt", _image()),+ }+ ]++ out = promote_history(history, vision = True)++ # The suffix still comes off -- it is base64 and the model must not read it as+ # text -- but it never becomes image input.+ assert out[0]["content"] == "$ cat notes.txt"+ assert len(out) == 1, "a bash result was promoted into image input"+++def test_an_mcp_named_result_is_still_promoted_on_replay():+ history = [+ {+ "role": "tool",+ "tool_call_id": "call_0",+ "name": "mcp__fs__read_media_file",+ "content": _envelope("[1 image returned]", _image()),+ }+ ]++ out = promote_history(history, vision = True)++ assert out[0]["content"] == "[1 image returned]"+ assert out[1]["content"][1]["type"] == "image_url"+++def test_an_unnamed_tool_result_keeps_working():+ """Older stored turns carry no name; the envelope only ever came from an MCP+ server, so an absent name is not evidence against it."""+ history = [{"role": "tool", "content": _envelope("[1 image returned]", _image())}]++ out = promote_history(history, vision = True)++ assert len(out) == 2+ assert out[1]["content"][1]["type"] == "image_url"+++def test_stripping_is_unconditional_and_only_promotion_is_gated():+ """Two different rules, and conflating them breaks one or the other.++ The suffix runs to megabytes of base64, so it comes off every model-facing+ text path whoever produced it -- that is a context-window property, and+ test_tool_result_fits_window depends on it. Provenance answers a separate+ question: whether those bytes are trusted enough to become IMAGE input.+ """+ from core.inference.mcp_images import promote_history+ from core.inference.tool_loop_controller import (+ ToolCallCompletion,+ ToolCallDecision,+ strip_result_for_model,+ )++ payload = _envelope("$ cat notes.txt", _image())++ def completion(tool_name):+ decision = ToolCallDecision(+ action = "execute",+ tool_name = tool_name,+ arguments = {},+ tool_call_id = "call_0",+ card_call_id = "",+ key = tool_name,+ provenance = {},+ noop_result = "",+ )+ return ToolCallCompletion(decision = decision, result = payload, executed = True)++ for name in ("mcp__fs__read", "bash", "web_search", "mcp"):+ # 1. never as text, on the live path or the replay+ assert strip_result_for_model(payload, name) == "$ cat notes.txt", name+ replayed = promote_history(+ [{"role": "tool", "name": name, "content": payload}], vision = True+ )+ assert replayed[0]["content"] == "$ cat notes.txt", name++ # 2. as image input only from a tool an MCP server served+ assert completion("mcp__fs__read").mcp_images()+ assert completion("bash").mcp_images() == []+ promoted = promote_history(+ [{"role": "tool", "name": "mcp__fs__read", "content": payload}], vision = True+ )+ assert any(isinstance(m.get("content"), list) for m in promoted)+ not_promoted = promote_history(+ [{"role": "tool", "name": "bash", "content": payload}], vision = True+ )+ assert not any(isinstance(m.get("content"), list) for m in not_promoted)+++def test_a_parallel_batch_keeps_each_result_its_own_quota():+ """Two calls returning four images each deliver eight, not four: flattening+ first hands the whole per-result allowance to the first call."""+ from core.inference.mcp_images import content_parts_per_result++ one = [_image() for _ in range(MAX_MODEL_IMAGES)]+ two = [_image() for _ in range(MAX_MODEL_IMAGES)]++ assert len(content_parts(one + two)) == MAX_MODEL_IMAGES+ assert len(content_parts_per_result([one, two])) == 2 * MAX_MODEL_IMAGES+++def test_a_parallel_batch_still_stops_at_the_conversation_cap():+ from core.inference.mcp_images import content_parts_per_result+ results = [[_image() for _ in range(MAX_MODEL_IMAGES)] for _ in range(4)]++ assert len(content_parts_per_result(results)) == mcp_images.MAX_TOTAL_MODEL_IMAGES+++def test_an_emptied_image_turn_is_dropped_not_left_blank():+ """content: [] is an empty user message, which strict provider APIs and chat+ templates reject rather than serving the newer images."""+ conversation = [+ {"role": "user", "content": [{"type": "image"}]},+ {"role": "user", "content": [{"type": "image"}]},+ ]+ payloads = ["a", "b"]++ mcp_images.trim_image_turns(conversation, payloads, limit = 1)++ assert len(conversation) == 1+ assert all(message.get("content") != [] for message in conversation)+++def test_replayed_markers_sit_ahead_of_the_attachment_marker():+ """Pixels go history-first with the attachment last, and a positional VLM+ processor binds them in document order."""+ messages = [+ {"role": "user", "content": [{"type": "image"}, {"type": "text", "text": "this one"}]},+ ]++ mcp_images.insert_placeholder_turn(messages, 0, 2, 2)++ replayed = [+ index+ for index, message in enumerate(messages)+ for part in message["content"]+ if part.get("type") == "image"+ ]+ # Both replayed markers come before the attachment's turn.+ assert replayed[:2] == [0, 0]+ assert replayed[-1] == 1+++def test_the_attachment_is_marked_on_the_turn_that_supplied_it():+ """_extract_content_parts takes the newest user image from anywhere in the+ thread, so marking the newest TURN tells a text-only latest question that it+ carried an older picture."""+ conversation = [+ {"role": "user", "content": "here is the screenshot"},+ {"role": "assistant", "content": "noted"},+ {"role": "user", "content": "now a text-only follow-up"},+ ]++ marked = mcp_images.mark_last_user_turn(conversation, 1, ordinal = 0)++ assert isinstance(marked[0]["content"], list)+ assert marked[2]["content"] == "now a text-only follow-up"+++def test_without_an_ordinal_the_newest_user_turn_still_takes_it():+ conversation = [+ {"role": "user", "content": "first"},+ {"role": "user", "content": "second"},+ ]++ marked = mcp_images.mark_last_user_turn(conversation, 1)++ assert marked[0]["content"] == "first"+ assert isinstance(marked[1]["content"], list)+++def test_the_provenance_name_survives_local_extraction():+ """promote_history reads message['name'] to decide whether an envelope came+ from an MCP server; an extraction that drops it bypasses the check."""+ import inspect++ import routes.inference as inference_route++ body = inspect.getsource(inference_route._extract_content_parts)+ assert (+ 'chat_message["name"] = _tool_name' in body+ ), "the tool message's name has to reach promote_history"+++def test_a_parallel_batch_survives_the_replay_too():+ """The live loops kept result boundaries; the replay still flattened them, so+ a conversation resumed after two parallel calls lost the second call's images."""+ history = [+ {+ "role": "tool",+ "name": "mcp__a__shot",+ "content": _envelope("[4 images returned]", *[_image() for _ in range(4)]),+ },+ {+ "role": "tool",+ "name": "mcp__b__shot",+ "content": _envelope("[4 images returned]", *[_image() for _ in range(4)]),+ },+ ]++ promoted = promote_history(history, vision = True)++ assert mcp_images.count_image_parts(promoted, "image_url") == 2 * MAX_MODEL_IMAGES+++def test_the_local_replay_keeps_markers_and_payloads_in_step_across_results():+ history = [+ {+ "role": "tool",+ "name": f"mcp__{server}__shot",+ "content": _envelope("[4 images returned]", *[_image() for _ in range(4)]),+ }+ for server in ("a", "b")+ ]++ messages, payloads = mcp_images.promote_history_local(history, vision = True)++ # Both results flush as one batch, and a local placeholder turn is one message,+ # which on a non-GGUF model takes one picture. Parity is the invariant.+ assert len(payloads) == mcp_images.LOCAL_MAX_IMAGES_PER_TURN+ assert mcp_images.count_image_parts(messages, "image") == len(payloads)+++def test_the_ordinal_skips_the_turns_promotion_inserted():+ """The ordinal is counted on the ORIGINAL history, but applied to the promoted+ one, which has synthetic image turns in it. Counting those would put the+ attachment's marker on a historical picture's turn."""+ promoted = [+ {"role": "user", "content": "here is the shot"},+ mcp_images.placeholder_turn(1, 1),+ {"role": "assistant", "content": "ok"},+ {"role": "user", "content": "and this new one"},+ ]++ out = mcp_images.mark_last_user_turn(promoted, 1, ordinal = 1)++ # The second REAL user turn, not the second user turn in the promoted list.+ assert isinstance(out[3]["content"], list)+ assert out[3]["content"][-1] == {"type": "text", "text": "and this new one"}+++def test_a_synthetic_image_turn_is_recognised():+ assert mcp_images.is_synthetic_image_turn(mcp_images.placeholder_turn(2, 2))+ assert not mcp_images.is_synthetic_image_turn({"role": "user", "content": "hello"})+ assert not mcp_images.is_synthetic_image_turn(+ {"role": "user", "content": [{"type": "text", "text": "hello"}]}+ )+++def test_undecodable_entries_are_not_counted_as_images():+ """_flatten_result accepts formats Pillow cannot read, and promotion drops+ them, so charging their KV reserves cache for images never sent."""+ svg = {+ "data": base64.b64encode(b'<svg xmlns="http://www.w3.org/2000/svg"/>').decode(),+ "mimeType": "image/svg+xml",+ }++ assert mcp_images.count_probably_decodable([svg] * 4) == 0+ assert mcp_images.count_probably_decodable([_image()] * 3) == 3+ assert mcp_images.count_probably_decodable([svg, _image(), svg, _image()]) == 2+ # And the sniff never claims something it cannot open.+ assert not mcp_images.probably_decodable({"data": "not base64 at all!!", "mimeType": "x"})+ assert not mcp_images.probably_decodable({})+++def test_the_replay_cap_never_deletes_a_callers_own_attachments():+ """promote_history runs on every request now, so an unscoped cap silently+ dropped the oldest attachments of a caller that simply sent many images."""+ attachments = [+ {"type": "image_url", "image_url": {"url": f"data:image/png;base64,{_png()}"}}+ for _ in range(mcp_images.MAX_TOTAL_MODEL_IMAGES + 4)+ ]+ conversation = [+ {"role": "user", "content": [{"type": "text", "text": "compare these"}, *attachments]}+ ]++ out = promote_history(conversation, vision = True)++ assert mcp_images.count_image_parts(out, "image_url") == len(attachments)+++def test_the_replay_cap_still_bounds_what_promotion_adds():+ turns = []+ for _ in range(mcp_images.MAX_TOTAL_MODEL_IMAGES + 3):+ turns.append({"role": "tool", "name": "mcp__a__s", "content": _envelope("[1]", _image())})+ turns.append({"role": "assistant", "content": "noted"})++ out = promote_history(turns, vision = True)++ assert mcp_images.count_image_parts(out, "image_url") == mcp_images.MAX_TOTAL_MODEL_IMAGES+++def test_the_sniff_charges_every_format_promotion_can_decode():+ """A deny list, not an allow list: a format Pillow gains that an allow list+ missed would be promoted and reserved nothing for, which overcommits the KV+ cache. Guessing the other way only over-reserves."""+ for fmt in ("PNG", "JPEG", "GIF", "BMP", "TIFF", "WEBP", "ICO"):+ buffer = io.BytesIO()+ Image.new("RGB", (8, 8), (1, 2, 3)).save(buffer, format = fmt)+ encoded = base64.b64encode(buffer.getvalue()).decode()+ assert mcp_images.probably_decodable({"data": encoded}), fmt++ svg = base64.b64encode(b'<svg xmlns="http://www.w3.org/2000/svg"/>').decode()+ assert not mcp_images.probably_decodable({"data": svg})+ assert not mcp_images.probably_decodable({"data": base64.b64encode(b'{"a": 1}').decode()})+++def test_the_live_cap_also_leaves_callers_attachments_alone():+ """The replay cap was scoped first; the live loops kept the unscoped one, so a+ tool loop that starts from caller attachments still deleted them."""+ attachments = [+ {"type": "image_url", "image_url": {"url": f"data:image/png;base64,{_png()}"}}+ for _ in range(6)+ ]+ conversation = [+ {"role": "user", "content": [{"type": "text", "text": "compare"}, *attachments]}+ ]+ owned: list = []+ for _ in range(3):+ conversation.append({"role": "tool", "content": "[4 images returned]"})+ append_image_turn(+ conversation, [[_image() for _ in range(4)]], per_result = True, owned = owned+ )++ survived = sum(+ 1+ for part in attachments+ for message in conversation+ if isinstance(message.get("content"), list)+ for other in message["content"]+ if other is part+ )+ assert survived == len(attachments), "a caller attachment was deleted by the loop's cap"+ assert len(owned) == mcp_images.MAX_TOTAL_MODEL_IMAGES+++def test_pixels_follow_the_markers_when_the_attachment_came_first():+ """ "History first, attachment last" only holds when the attachment is on the+ newest turn. It is not when an earlier question carried it and a tool returned+ pictures afterwards, and a positional VLM binds the Nth pixel to the Nth marker."""+ conversation = [+ {"role": "user", "content": "look at this"},+ {"role": "tool", "content": "[1 image returned]"},+ mcp_images.placeholder_turn(1, 1),+ ]+ prior = mcp_images.image_marker_parts(conversation)+ conversation = mcp_images.mark_last_user_turn(conversation, 1, ordinal = 0)++ ordered = mcp_images.pixels_in_marker_order(conversation, prior, ["MCP"], "ATTACH")++ assert ordered == ["ATTACH", "MCP"]+++def test_pixels_stay_history_first_when_the_attachment_is_newest():+ conversation = [+ {"role": "tool", "content": "[1 image returned]"},+ mcp_images.placeholder_turn(1, 1),+ {"role": "user", "content": "and this one"},+ ]+ prior = mcp_images.image_marker_parts(conversation)+ conversation = mcp_images.mark_last_user_turn(conversation, 1, ordinal = 1)++ ordered = mcp_images.pixels_in_marker_order(conversation, prior, ["MCP"], "ATTACH")++ assert ordered == ["MCP", "ATTACH"]+++def test_the_vision_streamer_resolves_markers_for_a_replay_only_turn():+ """The streamer marks its markers resolved either way, so gating them on the+ singular attachment resolves a replay-only turn to none AND suppresses the+ fallback detection -- native reasoning output then reaches the user as answer+ text. Route classification keys off every image; generation has to as well."""+ import inspect++ from core.inference import inference as inference_module++ body = inspect.getsource(inference_module.InferenceBackend._generate_vision_response)+ call = body.index("detect_reasoning_channel_markers(processor")+ window = body[call : call + 400]+ assert "if attached" in window, window[:200]+ assert "if image\n" not in window+++def test_the_gguf_loop_caps_images_across_iterations_not_per_batch():+ """A per-iteration ownership list only ever sees the current batch, so three+ sequential four-image results leave twelve images in the conversation against+ a cap of eight."""+ import inspect++ from core.inference import llama_cpp++ body = inspect.getsource(llama_cpp.LlamaCppBackend.generate_chat_completion_with_tools)+ declared = body.index("loop_mcp_image_parts: list = list(replayed_image_parts)")+ loop_start = body.index("while True:")+ assert declared < loop_start, (+ "the ownership list has to outlive the iteration, or the cap only ever "+ "counts the newest batch"+ )+++def test_promotion_reports_the_parts_it_created():+ """A resumed chat already carries promoted images; a loop starting its cap at+ zero lets the whole history through and then adds a batch on top."""+ history = [+ {"role": "tool", "name": "mcp__a__s", "content": _envelope("[1]", _image())},+ {"role": "assistant", "content": "noted"},+ ]+ promoted: list = []++ out = promote_history(history, vision = True, promoted_out = promoted)++ assert len(promoted) == 1+ live = [+ part+ for message in out+ if isinstance(message.get("content"), list)+ for part in message["content"]+ if part.get("type") == "image_url"+ ]+ assert all(any(part is other for other in live) for part in promoted)+++def test_the_plain_vision_path_marks_the_turn_that_supplied_the_attachment():+ """messages_with_attached_image returns early when markers exist, so the top-up+ placed the attachment's marker on the newest turn -- showing a later text-only+ question as carrying an older image."""+ conversation = [+ {"role": "user", "content": "here it is"},+ {"role": "assistant", "content": "ok"},+ mcp_images.placeholder_turn(1, 1),+ {"role": "user", "content": "a later text question"},+ ]+ prior = mcp_images.image_marker_parts(conversation)++ out = mcp_images.top_up_image_markers(conversation, 2, ordinal = 0)++ assert isinstance(out[0]["content"], list)+ assert out[3]["content"] == "a later text question"+ assert mcp_images.pixels_in_marker_order(out, prior, ["MCP"], "ATTACH") == ["ATTACH", "MCP"]+++def test_the_attachment_is_not_the_one_the_cap_drops():+ """Trimming the combined list removes whatever is first, and the attachment is+ first whenever its turn precedes the replayed pictures."""+ conversation = [mcp_images.placeholder_turn(1, 1) for _ in range(8)]+ payloads = [f"mcp{i}" for i in range(8)]++ # what the route now does: trim the replay to leave room, THEN interleave+ mcp_images.trim_image_turns(conversation, payloads, limit = mcp_images.MAX_TOTAL_MODEL_IMAGES - 1)++ assert len(payloads) == mcp_images.MAX_TOTAL_MODEL_IMAGES - 1+ assert payloads[-1] == "mcp7", "the newest replayed image survived"+ assert "mcp0" not in payloads, "the oldest replayed image went, not the attachment"+++def test_a_client_marked_attachment_still_gets_its_pixel():+ """A marker that predates the top-up is history's only while history has+ pixels left. When the client marked the attachment itself, the top-up adds+ nothing and that pre-existing marker is the attachment's."""+ conversation = [{"role": "user", "content": [{"type": "image"}, {"type": "text", "text": "x"}]}]+ prior = mcp_images.image_marker_parts(conversation)++ ordered = mcp_images.pixels_in_marker_order(conversation, prior, [], "ATTACH")++ assert ordered == ["ATTACH"], "the attachment's pixel was dropped"+++def test_history_still_takes_its_own_markers_first():+ conversation = [+ mcp_images.placeholder_turn(2, 2),+ {"role": "user", "content": [{"type": "image"}, {"type": "text", "text": "and this"}]},+ ]+ prior = mcp_images.image_marker_parts(conversation)++ ordered = mcp_images.pixels_in_marker_order(conversation, prior, ["A", "B"], "ATTACH")++ assert ordered == ["A", "B", "ATTACH"]+++def test_the_gguf_loop_starts_its_cap_from_the_replayed_parts():+ """The list is seeded from the conversation, not from zero: a resumed chat whose+ history already carries the allowance would otherwise get a second one for this+ run and send both."""+ import inspect++ from core.inference import llama_cpp++ signature = inspect.signature(llama_cpp.LlamaCppBackend.generate_chat_completion_with_tools)+ assert "replayed_image_parts" in signature.parameters++ body = inspect.getsource(llama_cpp.LlamaCppBackend.generate_chat_completion_with_tools)+ assert "loop_mcp_image_parts: list = list(replayed_image_parts)" in body+++def test_the_gguf_route_hands_the_loop_what_promotion_created():+ import inspect++ from routes import inference++ builder = inspect.signature(inference._openai_messages_for_gguf_chat)+ assert "promoted_out" in builder.parameters++ route = inspect.getsource(inference.produce_openai_chat_completions)+ assert "_gguf_replayed_image_parts: list = []" in route+ assert "replayed_image_parts = tuple(_gguf_replayed_image_parts)" in route+++def test_a_replayed_history_is_not_decoded_past_what_the_cap_can_keep():+ """A permitted raster is 40 megapixels and a few hundred bytes of base64 can ask+ for one, so decoding every envelope in a caller-supplied history before the trim+ let the request choose how much Pillow work it cost."""+ decoded: list = []+ real = _png()++ def _counting_url(data):+ decoded.append(data)+ return "data:image/png;base64," + real++ history: list = []+ for index in range(60):+ history.append(+ {+ "role": "tool",+ "name": f"mcp__s__shot{index}",+ "content": _envelope("[4]", *[_image() for _ in range(4)]),+ }+ )+ history.append({"role": "assistant", "content": f"turn {index}"})+ history.append({"role": "user", "content": "again"})++ original = mcp_images._png_data_url+ mcp_images._png_data_url = _counting_url+ try:+ out = promote_history(history, vision = True)+ finally:+ mcp_images._png_data_url = original++ live = sum(+ 1+ for message in out+ if isinstance(message.get("content"), list)+ for part in message["content"]+ if part.get("type") == "image_url"+ )+ assert live == mcp_images.MAX_TOTAL_MODEL_IMAGES, live+ assert len(decoded) <= (+ mcp_images.MAX_TOTAL_MODEL_IMAGES + mcp_images.DECODE_FAILURE_ALLOWANCE+ ), f"{len(decoded)} decodes for a history of 240 images"+++def test_the_eligible_pass_keeps_candidates_behind_an_undecodable_result():+ """It cannot decode either, so a newest result of formats Pillow rejects must not+ starve the valid pictures behind it."""+ history = [+ {+ "role": "tool",+ "name": "mcp__s__a",+ "content": _envelope("[4]", *[_image() for _ in range(4)]),+ },+ {+ "role": "tool",+ "name": "mcp__s__b",+ "content": _envelope("[4]", *[_image() for _ in range(4)]),+ },+ {+ "role": "tool",+ "name": "mcp__s__c",+ "content": _envelope("[4]", *[_image() for _ in range(4)]),+ },+ ]++ eligible = mcp_images.eligible_replay_images(history)++ assert eligible[2] == 4 and eligible[1] == 4, eligible+ assert (+ eligible[0] == mcp_images.DECODE_FAILURE_ALLOWANCE+ ), "the spare allowance keeps the oldest result as candidates"+ assert sum(eligible.values()) <= (+ mcp_images.MAX_TOTAL_MODEL_IMAGES + mcp_images.DECODE_FAILURE_ALLOWANCE+ )+++def test_an_oversized_parallel_batch_keeps_its_newest_results():+ """Everything else that bounds these keeps the newest, so filling the batch from+ the front showed the model results 1-2 on this turn and 2-3 on the next."""+ # Distinct sizes so each result's decoded URLs are distinguishable.+ results = [+ [{"data": _png(size = (4 + index, 4 + index)), "mimeType": "image/png"} for _ in range(4)]+ for index in range(3)+ ]+ per_result = [set(mcp_images._decoded_urls(images, 4)) for images in results]++ urls = mcp_images._decoded_urls_per_result(results)++ assert len(urls) == mcp_images.MAX_TOTAL_MODEL_IMAGES+ kept = set(urls)+ assert per_result[2] <= kept, "the newest result must survive"+ assert per_result[1] <= kept, "so must the one before it"+ assert not (per_result[0] & kept), "the OLDEST result is the one that goes"+ # Within the batch the surviving results still read in call order.+ assert set(urls[:4]) == per_result[1] and set(urls[4:]) == per_result[2]+++def test_a_partially_trimmed_turn_stops_claiming_the_images_it_lost():+ """The note exists to say which of the returned pictures the model was really+ shown, so a turn left holding two while still reading "first 4 of 8" defeats it."""+ conversation = [+ {+ "role": "user",+ "content": [+ {"type": "image"},+ {"type": "image"},+ {"type": "image"},+ {"type": "image"},+ {"type": "text", "text": mcp_images._turn_text(4, 8)},+ ],+ },+ mcp_images.placeholder_turn(4, 4),+ mcp_images.placeholder_turn(2, 2),+ ]+ payloads = [f"p{i}" for i in range(10)]++ mcp_images.trim_image_turns(conversation, payloads)++ partial = conversation[0]+ remaining = sum(1 for part in partial["content"] if part.get("type") == "image")+ note = next(part["text"] for part in partial["content"] if part.get("type") == "text")+ assert remaining == 2, remaining+ assert f"({remaining} of 8)" in note, note+ assert "first" not in note, "the survivors are the newest images, not the first ones"+++def test_the_owned_list_drops_evicted_parts_before_it_counts():+ """A rolling-context fitter evicts whole turns mid-loop without telling this list.+ Counting parts that already left over-trims the ones still in it."""+ owned: list = []+ conversation: list = []+ mcp_images.append_image_turn(+ conversation,+ [[{"data": _png(), "mimeType": "image/png"} for _ in range(4)]],+ per_result = True,+ owned = owned,+ )+ mcp_images.append_image_turn(+ conversation,+ [[{"data": _png(), "mimeType": "image/png"} for _ in range(4)]],+ per_result = True,+ owned = owned,+ )+ assert len(owned) == 8++ # What truncation does: the turns go, the ownership list is not told.+ conversation.clear()++ mcp_images.append_image_turn(+ conversation,+ [[{"data": _png(), "mimeType": "image/png"} for _ in range(4)]],+ per_result = True,+ owned = owned,+ )++ live = sum(+ 1+ for message in conversation+ if isinstance(message.get("content"), list)+ for part in message["content"]+ if part.get("type") == "image_url"+ )+ assert live == 4, f"the fresh result was over-trimmed to {live}"+++def test_a_live_result_of_unreadable_entries_bounds_its_decode_attempts():+ """The payload-byte budget caps how BIG a result is, never how many entries it+ holds, so a server answering with thousands of tiny unreadable ones could hold an+ inference worker open on Pillow for a turn that can show four pictures at most."""+ attempts: list = []++ def _counting_url(data):+ attempts.append(data)+ return None++ junk = [{"data": f"not-an-image-{i}", "mimeType": "image/png"} for i in range(5000)]++ original = mcp_images._png_data_url+ mcp_images._png_data_url = _counting_url+ try:+ urls = mcp_images._decoded_urls(junk)+ finally:+ mcp_images._png_data_url = original++ assert urls == []+ assert len(attempts) <= (+ mcp_images.MAX_MODEL_IMAGES + mcp_images.DECODE_FAILURE_ALLOWANCE+ ), f"{len(attempts)} Pillow opens for one tool result"+++def test_the_allowance_still_looks_past_a_run_of_rejects():+ """The bound must not defeat what it is bounding: the whole reason entries are not+ sliced first is that valid PNGs sit behind formats Pillow rejects."""+ seen: list = []++ def _url(data):+ seen.append(data)+ return None if data.startswith("bad") else "data:image/png;base64," + _png()++ images = [{"data": f"bad{i}", "mimeType": "image/svg+xml"} for i in range(3)]+ images += [{"data": f"good{i}", "mimeType": "image/png"} for i in range(4)]++ original = mcp_images._png_data_url+ mcp_images._png_data_url = _url+ try:+ urls = mcp_images._decoded_urls(images)+ finally:+ mcp_images._png_data_url = original++ assert (+ len(urls) == mcp_images.MAX_MODEL_IMAGES+ ), f"the bound cut into the real pictures: {len(urls)}"+++def test_the_note_never_claims_the_survivors_are_the_first_ones():+ """An oversized parallel batch keeps its NEWEST results, so a note reading+ 'first 8 of 12' names images the model was never shown."""+ results = [+ [{"data": _png(size = (4 + index, 4 + index)), "mimeType": "image/png"} for _ in range(4)]+ for index in range(3)+ ]+ conversation: list = []++ mcp_images.append_image_turn(conversation, results, per_result = True, limit = None)++ note = next(+ part["text"]+ for message in conversation+ if isinstance(message.get("content"), list)+ for part in message["content"]+ if part.get("type") == "text"+ )+ assert "(8 of 12)" in note, note+ assert "first" not in note, note+++def test_a_parallel_batch_shares_one_decode_attempt_budget():+ """room falls only on a SUCCESSFUL decode, so results that fail late in Pillow+ never close the loop. Per result the allowance reset, and a 25-call turn of+ malformed results bought 25 x 8 decodes of attacker-chosen rasters against a+ conversation cap of eight pictures."""+ attempts: list = []+ original = mcp_images._png_data_url+ mcp_images._png_data_url = lambda data: (attempts.append(data), None)[1]+ try:+ results = [+ [{"data": f"junk{r}-{i}", "mimeType": "image/png"} for i in range(8)] for r in range(25)+ ]+ assert mcp_images._decoded_urls_per_result(results) == []+ finally:+ mcp_images._png_data_url = original++ assert len(attempts) <= (+ mcp_images.MAX_TOTAL_MODEL_IMAGES + mcp_images.DECODE_FAILURE_ALLOWANCE+ ), f"{len(attempts)} Pillow opens for one parallel turn"+++def test_the_shared_budget_still_reaches_a_good_result_behind_bad_ones():+ """The bound must not defeat what it protects: the newest result failing must+ still leave enough attempts to find the real pictures behind it."""++ def _url(data):+ return None if data.startswith("bad") else "data:image/png;base64," + _png()++ original = mcp_images._png_data_url+ mcp_images._png_data_url = _url+ try:+ results = [+ [{"data": f"good{i}", "mimeType": "image/png"} for i in range(4)],+ [{"data": f"bad{i}", "mimeType": "image/svg+xml"} for i in range(4)],+ ]+ urls = mcp_images._decoded_urls_per_result(results)+ finally:+ mcp_images._png_data_url = original++ assert len(urls) == 4, f"the older result's real pictures were lost: {len(urls)}"+++def test_a_complete_turn_keeps_its_total_when_the_cap_first_trims_it():+ """A turn that arrived complete carries no "(n of m)" suffix. Falling back to the+ post-trim count there told the model nothing had been dropped -- the one thing+ the note exists to say."""+ conversation = [+ mcp_images.placeholder_turn(4, 4),+ mcp_images.placeholder_turn(4, 4),+ mcp_images.placeholder_turn(2, 2),+ ]+ payloads = [f"p{i}" for i in range(10)]++ mcp_images.trim_image_turns(conversation, payloads)++ kept = sum(1 for part in conversation[0]["content"] if part.get("type") == "image")+ note = next(part["text"] for part in conversation[0]["content"] if part.get("type") == "text")+ assert kept == 2+ assert f"({kept} of 4)" in note, note+++def test_a_transparent_screenshot_is_composited_rather_than_flattened_to_black():+ """convert("RGB") keeps whatever colour sits UNDER the alpha, and a tool that+ never painted a background leaves that black -- so a transparent screenshot's+ dark text converted to black on black and the model saw a blank rectangle."""+ import base64 as _b64+ import io as _io++ from PIL import Image++ def _roundtrip(image):+ buffer = _io.BytesIO()+ image.save(buffer, format = "PNG")+ url = mcp_images._png_data_url(_b64.b64encode(buffer.getvalue()).decode())+ assert url, "the image did not decode at all"+ return Image.open(_io.BytesIO(_b64.b64decode(url.split(",", 1)[1])))++ rgba = Image.new("RGBA", (8, 8), (0, 0, 0, 0))+ rgba.putpixel((4, 4), (0, 0, 0, 255))+ palette = Image.new("P", (8, 8), 0)+ palette.putpalette([0, 0, 0] * 256)+ palette.info["transparency"] = 0+ palette.putpixel((4, 4), 1)+ grey = Image.new("LA", (8, 8), (0, 0))+ grey.putpixel((4, 4), (0, 255))++ for name, image in (("RGBA", rgba), ("P", palette), ("LA", grey)):+ out = _roundtrip(image)+ assert out.getpixel((0, 0)) == (255, 255, 255), f"{name}: background not composited"+ assert out.getpixel((4, 4)) == (0, 0, 0), f"{name}: the drawn pixel was lost"+++def test_replay_leaves_room_for_the_pictures_the_caller_attached():+ """Providers apply their own per-request cap in document order, and promotion+ PREPENDS the replay to the user turn. On Gemini (8 images, later ones dropped+ silently) eight replayed screenshots evicted the picture the current question+ was about."""+ attachment = {+ "type": "image_url",+ "image_url": {"url": "data:image/png;base64," + _png()},+ }+ history = []+ for index in range(2):+ history.append(+ {+ "role": "assistant",+ "content": "",+ "tool_calls": [+ {+ "id": f"c{index}",+ "type": "function",+ "function": {"name": "mcp__s__shot", "arguments": "{}"},+ }+ ],+ }+ )+ history.append(+ {+ "role": "tool",+ "tool_call_id": f"c{index}",+ "name": "mcp__s__shot",+ "content": _envelope("[4]", *[_image() for _ in range(4)]),+ }+ )+ history.append(+ {+ "role": "user",+ "content": [{"type": "text", "text": "and how does this compare?"}, attachment],+ }+ )++ out = promote_history(history, vision = True, reserve_for_caller = True)+ urls = [+ part["image_url"]["url"]+ for message in out+ if isinstance(message.get("content"), list)+ for part in message["content"]+ if part.get("type") == "image_url"+ ]++ assert len(urls) <= mcp_images.MAX_TOTAL_MODEL_IMAGES, len(urls)+ assert attachment["image_url"]["url"] in urls, "the caller's own picture was trimmed"+ # And it survives a provider that keeps only the first MAX_TOTAL_MODEL_IMAGES.+ assert attachment["image_url"]["url"] in urls[: mcp_images.MAX_TOTAL_MODEL_IMAGES]+++def test_the_replay_trim_runs_before_the_attachment_marker_exists():+ """The trim drops by marker ORDINAL against a payload list holding replay only.+ With the attachment's turn AHEAD of the replayed pictures, running it after the+ mark deleted ordinal 0 -- the attachment's own marker -- while charging replay+ payload 0, and every later pixel shifted onto the marker before it."""++ def _scene():+ return [+ {"role": "user", "content": "here is my diagram"},+ mcp_images.placeholder_turn(4, 4),+ mcp_images.placeholder_turn(4, 4),+ {"role": "user", "content": "which is bluer?"},+ ]++ replay = [f"MCP{i}" for i in range(8)]++ # The order the route now uses: trim, snapshot, then mark.+ conversation, payloads = _scene(), list(replay)+ mcp_images.trim_image_turns(conversation, payloads, limit = mcp_images.MAX_TOTAL_MODEL_IMAGES - 1)+ prior = mcp_images.image_marker_parts(conversation)+ conversation = mcp_images.mark_last_user_turn(conversation, 1, ordinal = 0)+ ordered = mcp_images.pixels_in_marker_order(conversation, prior, payloads, "ATTACHMENT")++ owner = conversation[0]["content"]+ assert isinstance(owner, list) and any(+ part.get("type") == "image" for part in owner+ ), "the attachment's own marker was trimmed away"+ assert ordered[0] == "ATTACHMENT", ordered+ assert len(mcp_images.image_marker_parts(conversation)) == len(ordered)+++def test_the_route_trims_replay_before_marking_the_attachment():+ import inspect++ from routes import inference++ body = inspect.getsource(inference.produce_openai_chat_completions)+ trim = body.index("trim_mcp_image_turns(\n _sf_chat_messages")+ mark = body.index("_sf_chat_messages = mark_mcp_image_turn(")+ prior = body.index("_sf_prior_markers = mcp_image_marker_parts(_sf_chat_messages)")+ assert trim < prior < mark, (+ "trim, then snapshot history, then mark -- any other order binds the "+ "attachment to a replayed picture's marker"+ )+++def test_a_live_result_leaves_room_for_the_pictures_the_caller_attached():+ """owned holds only what the loop appended, so the cap allowed the attachment+ PLUS a full eight tool images. A provider keeping the first eight in document+ order then drops the newest result -- the one the model just asked for."""+ conversation = [+ {+ "role": "user",+ "content": [+ {"type": "text", "text": "compare with this"},+ {"type": "image_url", "image_url": {"url": "data:image/png;base64," + _png()}},+ ],+ }+ ]+ owned: list = []+ for _ in range(3):+ mcp_images.append_image_turn(+ conversation,+ [[{"data": _png(), "mimeType": "image/png"} for _ in range(4)]],+ per_result = True,+ owned = owned,+ # What the EXTERNAL loop passes. The GGUF loop does not: it answers to a+ # context window rather than to a provider's per-request image count.+ reserve_caller_images = True,+ )++ total = sum(+ 1+ for message in conversation+ if isinstance(message.get("content"), list)+ for part in message["content"]+ if part.get("type") == "image_url"+ )+ assert total <= mcp_images.MAX_TOTAL_MODEL_IMAGES, (+ f"{total} images in the conversation against a cap of "+ f"{mcp_images.MAX_TOTAL_MODEL_IMAGES}"+ )+ # And the caller's own picture is not what made room.+ assert any(+ part.get("type") == "image_url" for part in conversation[0]["content"]+ ), "the attachment was trimmed to fit the tool's results"+++def test_the_local_loop_keeps_its_own_allowance_beside_an_attachment():+ """The reservation is the EXTERNAL loop's, asked for rather than assumed: a+ remote provider counts images per request, llama-server answers to a context+ window. Reserving here would let six attachments squeeze the tool results the+ model asked for down to two."""+ attachments = [+ {"type": "image_url", "image_url": {"url": f"data:image/png;base64,{_png()}"}}+ for _ in range(6)+ ]+ conversation = [+ {"role": "user", "content": [{"type": "text", "text": "compare"}, *attachments]}+ ]+ owned: list = []+ for _ in range(3):+ mcp_images.append_image_turn(+ conversation,+ [[{"data": _png(), "mimeType": "image/png"} for _ in range(4)]],+ per_result = True,+ owned = owned,+ )++ assert len(owned) == mcp_images.MAX_TOTAL_MODEL_IMAGES+ survived = sum(+ 1+ for part in attachments+ for message in conversation+ if isinstance(message.get("content"), list)+ for other in message["content"]+ if other is part+ )+ assert survived == len(attachments), "the loop's cap is still not the caller's"+++def test_an_unnamed_tool_result_is_judged_by_the_call_that_made_it():+ """role="tool" carries no name in plain OpenAI (the field is optional) or in+ anything translated from Anthropic, and an absent name is read as legacy MCP+ history that may be trusted -- so any client tool whose output merely ends in a+ valid envelope was promoted as image input."""+ for tool, promotes in (("read_file", False), ("mcp__shot__capture", True)):+ history = [+ {+ "role": "assistant",+ "content": "",+ "tool_calls": [+ {+ "id": "call_0",+ "type": "function",+ "function": {"name": tool, "arguments": "{}"},+ }+ ],+ },+ # No "name" on the result, as the wire format allows.+ {"role": "tool", "tool_call_id": "call_0", "content": _envelope("[1]", _image())},+ ]++ out = promote_history(history, vision = True)+ parts = sum(+ 1+ for message in out+ if isinstance(message.get("content"), list)+ for part in message["content"]+ if part.get("type") == "image_url"+ )+ assert bool(parts) is promotes, f"{tool}: promoted={bool(parts)}"+ # Either way the suffix comes off the text; only IMAGE input is gated.+ assert mcp_images.SENTINEL not in _json.dumps(out)+++def test_the_note_reports_what_the_tool_returned_not_what_admission_allowed():+ """The admission pass slices the candidates before promotion sees them, so+ summing those made a 100-image result read as a handful -- the note describing+ the admission pass rather than the tool, next to a result saying 100."""+ history = [+ {+ "role": "tool",+ "name": "mcp__s__shot",+ "content": _envelope("[100 images returned]", *[_image() for _ in range(100)]),+ }+ ]++ out = promote_history(history, vision = True)++ note = next(+ part["text"]+ for message in out+ if isinstance(message.get("content"), list)+ for part in message["content"]+ if part.get("type") == "text"+ )+ assert "of 100)" in note, note+++def test_the_caller_attachment_is_composited_on_the_way_to_the_worker():+ """The server-tool path serialises the attachment to base64 for the worker. Plain+ convert("RGB") kept whatever colour sat under the alpha, so a transparent+ attachment whose background was never painted arrived black -- with its dark text+ gone. The ordinary IPC path carries the PNG's alpha through untouched."""+ import base64 as _b64+ import io as _io++ from PIL import Image++ from routes.inference import _pil_to_png_b64++ image = Image.new("RGBA", (8, 8), (0, 0, 0, 0))+ image.putpixel((4, 4), (0, 0, 0, 255))++ out = Image.open(_io.BytesIO(_b64.b64decode(_pil_to_png_b64(image))))++ assert out.getpixel((0, 0)) == (255, 255, 255), "transparency flattened to black"+ assert out.getpixel((4, 4)) == (0, 0, 0), "the drawn pixel was lost"+++def _call(call_id, name):+ return {+ "role": "assistant",+ "content": "",+ "tool_calls": [+ {"id": call_id, "type": "function", "function": {"name": name, "arguments": "{}"}}+ ],+ }+++def test_repeated_call_ids_pair_each_result_with_its_own_call():+ """The backend restarts ids like call_0 every response. A conversation-wide+ last-wins lookup let a later non-MCP call_0 rename an earlier MCP result -- and+ the reverse -- so an earlier picture was suppressed or a later envelope trusted."""+ history = [+ _call("call_0", "mcp__shot__capture"),+ {"role": "tool", "tool_call_id": "call_0", "content": _envelope("[1]", _image())},+ {"role": "assistant", "content": "a blue square"},+ _call("call_0", "read_file"),+ {"role": "tool", "tool_call_id": "call_0", "content": _envelope("[1]", _image())},+ ]++ names = mcp_images.resolve_tool_names(history)+ assert names == {1: "mcp__shot__capture", 4: "read_file"}, names++ out = promote_history(history, vision = True)+ promoted = sum(+ 1+ for message in out+ if isinstance(message.get("content"), list)+ for part in message["content"]+ if part.get("type") == "image_url"+ )+ assert promoted == 1, "exactly the MCP result's picture, not the client tool's"+++def test_a_batch_with_a_later_text_result_does_not_say_the_tool_call_above():+ """pending stays open across a later image-free result of the same batch, so the+ image turn lands after it and 'the tool call above' named the wrong call."""+ history = [+ {"role": "tool", "name": "mcp__s__shot", "content": _envelope("[1]", _image())},+ {"role": "tool", "name": "mcp__s__list", "content": "three files"},+ # An assistant follows, so the turn is INSERTED after the batch rather than+ # merged into a user message -- the case that carries a note at all.+ {"role": "assistant", "content": "done"},+ ]++ out = promote_history(history, vision = True)++ note = next(+ part["text"]+ for message in out+ if isinstance(message.get("content"), list)+ for part in message["content"]+ if part.get("type") == "text" and part["text"].startswith("Images returned by")+ )+ assert note.startswith(mcp_images.DETACHED_IMAGE_TURN_TEXT), note+++def test_the_note_honours_a_returned_count_the_frontend_left_on_the_envelope():+ """The frontend bounds the envelope before it is ever seen here; it records the+ length it started from on the first entry so the note can still say so."""+ images = [_image() for _ in range(2)]+ images[0]["returned"] = 100+ history = [{"role": "tool", "name": "mcp__s__shot", "content": _envelope("[100]", *images)}]++ out = promote_history(history, vision = True)++ note = next(+ part["text"]+ for message in out+ if isinstance(message.get("content"), list)+ for part in message["content"]+ if part.get("type") == "text"+ )+ assert "of 100)" in note, note+++def test_the_local_rebuild_names_an_unnamed_result_from_its_call():+ """_extract_content_parts drops tool_call_id and the calls, so the local path+ could not run the provenance gate on an unnamed result at all."""+ from models.inference import ChatMessage+ from routes.inference import _extract_content_parts++ messages = [+ ChatMessage(**_call("call_0", "read_file")),+ ChatMessage(role = "tool", tool_call_id = "call_0", content = _envelope("[1]", _image())),+ ]++ _, chat_messages, _ = _extract_content_parts(messages, keep_tool_images = True)++ tool = next(message for message in chat_messages if message["role"] == "tool")+ assert tool.get("name") == "read_file", tool+++def test_admission_does_not_charge_an_envelope_a_non_mcp_call_produced():+ from routes.inference import _openai_llama_admission_messages_for_estimate++ history = [+ _call("call_0", "read_file"),+ {+ "role": "tool",+ "tool_call_id": "call_0",+ "content": _envelope("[4]", *[_image() for _ in range(4)]),+ },+ ]++ _, image_parts = _openai_llama_admission_messages_for_estimate(history)++ assert image_parts == 0, f"reserved {image_parts} embeddings generation never sends"+++def test_the_monitor_prompt_never_retains_envelope_bytes():+ from routes.inference import _monitor_prompt_from_messages++ payload = _png()+ text = _monitor_prompt_from_messages(+ [+ {+ "role": "tool",+ "content": _envelope(+ "[1 image returned]", {"data": payload, "mimeType": "image/png"}+ ),+ }+ ]+ )++ assert payload not in text+ assert mcp_images.SENTINEL not in text+ assert "[1 image returned]" in text+++def test_eligibility_is_not_spent_on_results_a_non_mcp_call_produced():+ """The preselection read only the explicit name, so two newer unnamed results+ correlated to non-MCP calls took the allowance and a genuine older MCP result was+ left an eligibility of zero -- nothing useful replayed with capacity to spare."""+ history = [+ _call("call_0", "mcp__shot__capture"),+ {+ "role": "tool",+ "tool_call_id": "call_0",+ "content": _envelope("[4]", *[_image() for _ in range(4)]),+ },+ _call("call_1", "read_file"),+ {+ "role": "tool",+ "tool_call_id": "call_1",+ "content": _envelope("[4]", *[_image() for _ in range(4)]),+ },+ _call("call_2", "read_file"),+ {+ "role": "tool",+ "tool_call_id": "call_2",+ "content": _envelope("[4]", *[_image() for _ in range(4)]),+ },+ ]++ eligible = mcp_images.eligible_replay_images(history)++ assert 3 not in eligible and 5 not in eligible, eligible+ assert eligible.get(1) == 4, eligible+++def test_a_detached_image_turn_is_synthetic_too():+ """Counting the detached block as a real user turn put the attachment's marker on+ it instead of on the question that supplied the picture."""+ detached = mcp_images.placeholder_turn(1, 1, mcp_images.DETACHED_IMAGE_TURN_TEXT)+ assert mcp_images.is_synthetic_image_turn(detached)++ conversation = [+ {"role": "user", "content": "first question"},+ detached,+ {"role": "user", "content": "the question with the picture"},+ ]+ topped = mcp_images.top_up_image_markers(conversation, 2, ordinal = 1)++ assert isinstance(topped[2]["content"], list), topped+ assert any(part.get("type") == "image" for part in topped[2]["content"])+ assert isinstance(topped[0]["content"], str), "the first question gained no marker"+++def test_a_local_placeholder_turn_carries_one_picture():+ """The route refuses a caller message with more than one image on every non-GGUF+ target -- "This model takes one image per message" -- and no processor is known+ to take several. Promotion was building exactly that shape, so a processor with+ the limit failed mid-generation after the tool had already run."""+ results = [[_image() for _ in range(4)], [_image() for _ in range(4)]]++ payloads = mcp_images.png_payloads_per_result(results)+ assert len(payloads) == mcp_images.LOCAL_MAX_IMAGES_PER_TURN == 1++ out, replay = promote_history_local_for_test(+ [+ {+ "role": "tool",+ "name": "mcp__s__shot",+ "content": _envelope("[4 images returned]", *results[0]),+ }+ ]+ )+ assert len(replay) == 1+ turn = next(m for m in out if isinstance(m.get("content"), list))+ assert sum(1 for p in turn["content"] if p.get("type") == "image") == 1+ note = next(p["text"] for p in turn["content"] if p.get("type") == "text")+ assert "(1 of 4)" in note, note+++def promote_history_local_for_test(messages):+ return mcp_images.promote_history_local(messages, vision = True)+++def test_a_named_non_mcp_result_between_the_images_and_their_turn_detaches_the_note():+ """The early continue for a named non-MCP result skipped the interruption flag, so+ the block landed after web_search or read_file still saying 'the tool call above'."""+ history = [+ {"role": "tool", "name": "mcp__s__shot", "content": _envelope("[1]", _image())},+ {"role": "tool", "name": "web_search", "content": "three results"},+ {"role": "assistant", "content": "done"},+ ]++ out = promote_history(history, vision = True)++ note = next(+ part["text"]+ for message in out+ if isinstance(message.get("content"), list)+ for part in message["content"]+ if part.get("type") == "text" and part["text"].startswith("Images returned by")+ )+ assert note.startswith(mcp_images.DETACHED_IMAGE_TURN_TEXT), note+++def test_the_external_loop_detaches_the_note_for_a_multi_result_batch():+ import inspect++ from core.inference import studio_tool_loop++ body = inspect.getsource(studio_tool_loop)+ assert "_lead = DETACHED_IMAGE_TURN_TEXT if len(tool_messages) != 1 else None" in body++ conversation: list = []+ owned: list = []+ studio_tool_loop._append_mcp_images_owned(+ conversation, [[_image()]], owned, mcp_images.DETACHED_IMAGE_TURN_TEXT+ )+ note = next(+ part["text"]+ for message in conversation+ if isinstance(message.get("content"), list)+ for part in message["content"]+ if part.get("type") == "text"+ )+ assert note.startswith(mcp_images.DETACHED_IMAGE_TURN_TEXT)+++def test_the_attachment_displaces_a_replay_marker_merged_into_its_turn():+ """A replayed picture merges its marker into the following user turn; if that turn+ then owns the attachment, the top-up added a second marker. Non-GGUF messages+ take one image, so that shape failed at render, past the request validation. The+ attachment wins, and the displaced payload drops rather than sliding onto the+ next marker."""+ replay = {"role": "user", "content": [{"type": "image"}, {"type": "text", "text": "and this?"}]}+ conversation = [{"role": "user", "content": "first"}, replay]+ prior = mcp_images.image_marker_parts(conversation)+ assert len(prior) == 1++ topped = mcp_images.top_up_image_markers(conversation, 2, ordinal = 1)++ turn = topped[1]["content"]+ assert sum(1 for p in turn if p.get("type") == "image") == 1, turn+ ordered = mcp_images.pixels_in_marker_order(topped, prior, ["REPLAY"], "ATTACHMENT")+ assert ordered == ["ATTACHMENT"], ordered+++def test_pixels_follow_their_own_marker_not_a_queue():+ """Two replay markers, the first displaced: its payload must vanish, not bind to+ the second marker."""+ a = {"type": "image"}+ b = {"type": "image"}+ conversation = [{"role": "user", "content": [b, {"type": "text", "text": "x"}]}]+ ordered = mcp_images.pixels_in_marker_order(conversation, [a, b], ["PA", "PB"], "NEW")+ assert ordered == ["PB"], ordered+++def test_the_local_batch_shares_one_decode_attempt_budget():+ attempts: list = []+ original = mcp_images._png_data_url+ mcp_images._png_data_url = lambda data: (attempts.append(data), None)[1]+ try:+ results = [+ [{"data": f"j{r}-{i}", "mimeType": "image/png"} for i in range(8)] for r in range(25)+ ]+ assert mcp_images.png_payloads_per_result(results) == []+ finally:+ mcp_images._png_data_url = original+ assert (+ len(attempts) <= mcp_images.LOCAL_MAX_IMAGES_PER_TURN + mcp_images.DECODE_FAILURE_ALLOWANCE+ ), len(attempts)+++def test_the_dispatch_check_never_parses_the_envelope():+ """has_images json-loads the whole array; the async wrappers only need to know+ whether to leave the event loop, and a 12 MB parse on the loop to decide that+ stalled everything beside it."""+ import inspect++ from routes import inference++ assert mcp_images.mentions_images("x\n" + mcp_images.SENTINEL + "[not even json")+ assert not mcp_images.mentions_images("plain text")+ for fn in (+ inference._build_external_messages_async,+ inference._promote_local_mcp_images_async,+ inference._promote_mcp_history_images_async,+ ):+ src = inspect.getsource(fn)+ assert "_messages_mention_mcp_images(messages)" in src, fn.__name__+ assert "_messages_have_mcp_image_envelope(messages)" not in src, fn.__name__+++def test_the_promotable_check_correlates_an_unnamed_result():+ """chat_count_tokens refuses a prompt it reads as carrying a promotable image. An+ unnamed result whose call was read_file is never promoted, so refusing on it+ turned away a countable text prompt."""+ from models.inference import ChatCompletionRequest+ from routes.inference import _request_has_promotable_mcp_images++ def _req(tool):+ return ChatCompletionRequest(+ model = "default",+ messages = [+ _call("call_0", tool),+ {"role": "tool", "tool_call_id": "call_0", "content": _envelope("[1]", _image())},+ ],+ )++ assert not _request_has_promotable_mcp_images(_req("read_file"))+ assert _request_has_promotable_mcp_images(_req("mcp__shot__capture"))+++def test_the_server_tool_route_placement_also_displaces_a_merged_replay_marker():+ """mark_last_user_turn is the server-tool route's placement; it has to follow the+ same one-image rule as the top-up or that path keeps building two-image turns."""+ conversation = [+ {"role": "user", "content": "first"},+ {"role": "user", "content": [{"type": "image"}, {"type": "text", "text": "and this?"}]},+ ]+ marked = mcp_images.mark_last_user_turn(conversation, 1, ordinal = 1)+ assert sum(1 for p in marked[1]["content"] if p.get("type") == "image") == 1, marked[1]+++def _rounds(+ count: int,+ per_result: int,+ *,+ parallel: int = 1,+) -> list[dict]:+ history: list[dict] = [{"role": "user", "content": "start"}]+ for r in range(count):+ calls = [+ {+ "id": f"c{r}_{k}",+ "type": "function",+ "function": {"name": "mcp__s__shot", "arguments": "{}"},+ }+ for k in range(parallel)+ ]+ history.append({"role": "assistant", "content": "", "tool_calls": calls})+ for k in range(parallel):+ history.append(+ {+ "role": "tool",+ "tool_call_id": f"c{r}_{k}",+ "content": _envelope(f"[{per_result}]", *[_image() for _ in range(per_result)]),+ }+ )+ history.append({"role": "assistant", "content": f"round {r}"})+ history.append({"role": "user", "content": "compare them"})+ return history+++def _one_marker_per_turn(conversation: list) -> list[int]:+ counts = [+ sum(1 for part in message["content"] if part.get("type") == "image")+ for message in conversation+ if isinstance(message.get("content"), list)+ ]+ assert all(count <= 1 for count in counts), counts+ return counts+++def test_local_replay_keeps_every_picture_the_live_loop_kept():+ """Eight rounds of four valid PNGs. The live loop keeps one per round, eight in+ all; budgeting four per result for a path that sends one spent the allowance on+ the newest two rounds and replayed three of the eight."""+ history = _rounds(8, 4)+ live = min(+ sum(+ len(mcp_images.png_payloads_per_result([[_image() for _ in range(4)]]))+ for _ in range(8)+ ),+ mcp_images.MAX_TOTAL_MODEL_IMAGES,+ )+ assert live == 8++ out, payloads = mcp_images.promote_history_local(history, vision = True)+ assert len(payloads) == live+ assert sum(_one_marker_per_turn(out)) == live++ eligible = mcp_images.eligible_replay_images(history, local = True)+ tool_positions = [i for i, m in enumerate(history) if m.get("role") == "tool"]+ assert all(eligible.get(i, 0) >= 1 for i in tool_positions), eligible+ # The part paths still take four per result: two rounds fill the budget and a+ # third rides on the spare allowance, so only three results stay eligible there.+ parts = mcp_images.eligible_replay_images(history)+ assert sum(1 for n in parts.values() if n) == 3, parts+++def test_local_replay_charges_a_parallel_batch_once():+ """Three parallel results land as one turn carrying one picture, so the batch is+ charged one, not three: eight such rounds still replay eight pictures."""+ history = _rounds(8, 2, parallel = 3)+ out, payloads = mcp_images.promote_history_local(history, vision = True)+ assert len(payloads) == 8+ assert sum(_one_marker_per_turn(out)) == 8+++def _client_tool_route(monkeypatch, messages, *, tools):+ import asyncio+ from types import SimpleNamespace++ import routes.inference as inf+ from core.inference.api_monitor import ApiMonitor+ from models.inference import ChatCompletionRequest, ChatMessage+ from state.tool_policy import reset_tool_policy++ class Request:+ state = SimpleNamespace()+ url = SimpleNamespace(path = "/v1/chat/completions")+ method = "POST"+ scope: dict = {}+ headers = {"X-Unsloth-Events": "1"}++ async def is_disconnected(self):+ return False++ class Backend:+ active_model_name = "vlm"++ def __init__(self):+ self.models = {+ "vlm": {+ "chat_template_info": {+ "template": "<tool_call> chatml",+ "renders_image": True,+ },+ "context_length": 4096,+ "is_vision": True,+ }+ }+ self.calls: list = []++ def generate_chat_response(+ self,+ *,+ messages,+ tools = None,+ stats_holder = None,+ **kwargs,+ ):+ self.calls.append({"messages": messages, "tools": tools, **kwargs})+ yield "ok"++ def reset_generation_state(self, caller_cancel_event = None):+ return None++ def resize_image(self, image):+ return image++ backend = Backend()+ reset_tool_policy()+ monkeypatch.setattr(inf, "api_monitor", ApiMonitor(max_entries = 8))+ monkeypatch.setattr(+ inf,+ "get_llama_cpp_backend",+ lambda: SimpleNamespace(+ is_loaded = False, supports_tools = False, is_vision = False, context_length = None+ ),+ )+ monkeypatch.setattr(inf, "get_inference_backend", lambda: backend)+ monkeypatch.setattr(+ inf, "_detect_safetensors_features", lambda *a, **k: {"supports_tools": True}+ )+ payload = ChatCompletionRequest(+ model = "default",+ stream = False,+ tools = tools,+ messages = [ChatMessage(**message) for message in messages],+ )++ async def run():+ return await inf.openai_chat_completions(payload, request = Request(), current_subject = "u")++ asyncio.run(run())+ assert len(backend.calls) == 1, backend.calls+ return backend.calls[0]+++_LOOKUP = {+ "type": "function",+ "function": {"name": "lookup", "parameters": {"type": "object", "properties": {}}},+}+++def _shot_round(r: int, *images: dict) -> list[dict]:+ return [+ {+ "role": "assistant",+ "content": None,+ "tool_calls": [+ {+ "id": f"call_{r}",+ "type": "function",+ "function": {"name": "mcp__fs__read_media_file", "arguments": "{}"},+ }+ ],+ },+ {"role": "tool", "tool_call_id": f"call_{r}", "content": _envelope("[1]", *images)},+ ]+++def _no_adjacent_user_turns(messages: list) -> None:+ roles = [message["role"] for message in messages]+ assert all(a != "user" or b != "user" for a, b in zip(roles, roles[1:])), roles+++def test_the_client_tool_rebuild_keeps_one_picture_per_turn(monkeypatch):+ """Two historical image results and a comparison question, with client tools:+ the rebuilt history carries one marker per result at its own position, never two+ on the final user message -- the shape the route refuses on a non-GGUF target."""+ call = _client_tool_route(+ monkeypatch,+ [+ {"role": "user", "content": "read a.png"},+ *_shot_round(0, _image()),+ {"role": "assistant", "content": "a is blue"},+ {"role": "user", "content": "now b.png"},+ *_shot_round(1, _image()),+ {"role": "assistant", "content": "b is red"},+ {"role": "user", "content": "compare them"},+ ],+ tools = [_LOOKUP],+ )+ assert len(call["images"]) == 2+ assert sum(_one_marker_per_turn(call["messages"])) == 2, call["messages"]+ _no_adjacent_user_turns(call["messages"])+ assert isinstance(call["messages"][-1]["content"], str), "the question carries no marker"+ assert call.get("image") is None+++def test_the_client_tool_rebuild_leaves_the_attachment_to_the_backend(monkeypatch):+ """One replayed PNG and a new attachment on the question: no synthetic turn is+ inserted beside the question (two user turns in a row), the replay keeps its+ own marker, and the attachment's marker is the backend's to place by ordinal+ from a snapshot that holds only the replay's."""+ attachment = f"data:image/png;base64,{_png()}"+ question = {+ "role": "user",+ "content": [+ {"type": "text", "text": "and this one?"},+ {"type": "image_url", "image_url": {"url": attachment}},+ ],+ }+ for history in (+ [+ {"role": "user", "content": "read a.png"},+ *_shot_round(0, _image()),+ {"role": "assistant", "content": "a is blue"},+ question,+ ],+ [{"role": "user", "content": "read a.png"}, *_shot_round(0, _image()), question],+ ):+ call = _client_tool_route(monkeypatch, history, tools = [_LOOKUP])+ assert len(call["images"]) == 1+ assert call.get("image") is not None+ assert call.get("image_ordinal") == 1+ _no_adjacent_user_turns(call["messages"])+ assert sum(_one_marker_per_turn(call["messages"])) == 1, call["messages"]+++def test_a_gguf_replay_keeps_its_allowance_beside_the_attachment():+ """llama-server is bounded by its context window, not a per-request image cap:+ the live GGUF loop never reserved the attachment's slot, so the replay of the+ same conversation must not lose a picture to it. A provider reserves it."""+ history = [{"role": "user", "content": "start"}]+ for r in range(9):+ history += [+ {+ "role": "assistant",+ "content": "",+ "tool_calls": [+ {+ "id": f"c{r}",+ "type": "function",+ "function": {"name": "mcp__s__shot", "arguments": "{}"},+ }+ ],+ },+ {"role": "tool", "tool_call_id": f"c{r}", "content": _envelope("[1]", _image())},+ {"role": "assistant", "content": f"round {r}"},+ ]+ history.append(+ {+ "role": "user",+ "content": [+ {"type": "text", "text": "and this one?"},+ {"type": "image_url", "image_url": {"url": f"data:image/png;base64,{_png()}"}},+ ],+ }+ )++ def promoted_parts(out):+ parts = mcp_images._all_image_url_parts(out)+ return sum(1 for part in parts if part["image_url"]["url"].startswith("data:image/png"))++ gguf = promote_history(history, vision = True)+ assert promoted_parts(gguf) == mcp_images.MAX_TOTAL_MODEL_IMAGES + 1+ provider = promote_history(history, vision = True, reserve_for_caller = True)+ assert promoted_parts(provider) == mcp_images.MAX_TOTAL_MODEL_IMAGES+++def test_a_legacy_attachment_displaces_a_merged_replay_marker():+ """No ordinal (the top-level image field has no part to locate) and a replay+ marker already merged into the newest user turn: appending built a two-image+ message. The attachment takes the turn, as the ordinal branch does."""+ conversation = [+ {"role": "user", "content": "read a.png"},+ {"role": "tool", "content": "[1 image returned]"},+ {+ "role": "user",+ "content": [{"type": "image"}, {"type": "text", "text": "and this one?"}],+ },+ ]+ prior = mcp_images.image_marker_parts(conversation)+ topped = mcp_images.top_up_image_markers(conversation, 2, ordinal = None)++ markers = [part for part in topped[2]["content"] if part.get("type") == "image"]+ assert len(markers) == 1, topped[2]+ assert markers[0] is not prior[0], "the replay's marker was displaced, not kept"+ assert isinstance(topped[0]["content"], str)+ ordered = mcp_images.pixels_in_marker_order(topped, prior, ["REPLAY"], "ATTACHMENT")+ assert ordered == ["ATTACHMENT"]+++def test_a_legacy_attachment_skips_a_synthetic_turn():+ """The newest user turn can be a replay's own placeholder; the attachment goes on+ the question, not on the replay's turn."""+ conversation = [+ {"role": "user", "content": "the question"},+ {"role": "tool", "content": "[1 image returned]"},+ mcp_images.placeholder_turn(1, 1),+ ]+ topped = mcp_images.top_up_image_markers(conversation, 2, ordinal = None)+ assert isinstance(topped[0]["content"], list), topped+ assert sum(1 for part in topped[2]["content"] if part.get("type") == "image") == 1+++def test_the_catalog_predicate_never_parses_the_envelope():+ """The dispatch predicate runs on the event loop; the exact check json-loads a+ 12 MB envelope. The substring form must not touch the parser."""+ import routes.inference as inference_route+ from models.inference import ChatCompletionRequest, ChatMessage++ payload = ChatCompletionRequest(+ model = "default",+ messages = [+ ChatMessage(role = "user", content = "look"),+ ChatMessage(+ role = "tool",+ tool_call_id = "c0",+ name = "mcp__fs__shot",+ content = _envelope("[1]", _image()),+ ),+ ],+ )++ def boom(_content):+ raise AssertionError("parsed on the loop")++ original = inference_route.mcp_images_sentinel_in+ inference_route.mcp_images_sentinel_in = boom+ try:+ assert inference_route._request_has_promotable_mcp_images(payload, exact = False)+ finally:+ inference_route.mcp_images_sentinel_in = original+ assert inference_route._request_has_promotable_mcp_images(payload)+++def test_the_client_tool_rebuild_decodes_each_picture_once(monkeypatch):+ """The route promotes the history for the server-tool path and again, from the+ raw payload, for the client-tool rebuild. One request's decodes are shared, so a+ retained raster is decoded once, not once per promotion. Two DIFFERENT pictures:+ the cache is keyed by payload, and identical bytes would decode once anyway."""+ decoded: list = []+ original = mcp_images._png_data_url++ def counting(data):+ decoded.append(data[:16])+ return original(data)++ monkeypatch.setattr(mcp_images, "_png_data_url", counting)+ call = _client_tool_route(+ monkeypatch,+ [+ {"role": "user", "content": "read a.png"},+ *_shot_round(0, _image(_png((8, 8)))),+ {"role": "assistant", "content": "a is blue"},+ {"role": "user", "content": "now b.png"},+ *_shot_round(1, _image(_png((9, 9)))),+ {"role": "assistant", "content": "b is red"},+ {"role": "user", "content": "compare them"},+ ],+ tools = [_LOOKUP],+ )+ assert len(call["images"]) == 2+ assert len(decoded) == 2, decoded+++def test_the_local_loops_detach_the_note_for_a_multi_result_batch():+ """Same rule as the external loop: a batch of several results gets the wording+ that claims no adjacency, since "the tool call above" names whichever ran last."""+ import inspect++ from core.inference import llama_cpp, safetensors_agentic++ rule = (+ "lead = MCP_DETACHED_IMAGE_TURN_TEXT\n"+ " if _batch_results != 1\n"+ " else MCP_IMAGE_TURN_TEXT,"+ )+ assert rule in inspect.getsource(llama_cpp)+ assert rule.replace(+ "\n ", "\n "+ ) in inspect.getsource(safetensors_agentic)+ for module in (llama_cpp, safetensors_agentic):+ body = inspect.getsource(module)+ assert "batch_conversation_start = len(conversation)" in body+ assert "for m in conversation[batch_conversation_start:]" in body+++def test_the_caller_reservation_is_taken_before_any_replay_decode(monkeypatch):+ """Eight attachments leave a provider no room for replay; the eligibility budget+ says so before a single candidate is decoded, instead of decoding eight rasters+ for the trim at the bottom to drop them all."""+ decoded: list = []+ original = mcp_images._png_data_url++ def counting(data):+ decoded.append(1)+ return original(data)++ monkeypatch.setattr(mcp_images, "_png_data_url", counting)+ attachments = [+ {"type": "image_url", "image_url": {"url": f"data:image/png;base64,{_png()}"}}+ for _ in range(mcp_images.MAX_TOTAL_MODEL_IMAGES)+ ]+ history = [+ {"role": "user", "content": [{"type": "text", "text": "these"}, *attachments]},+ {"role": "tool", "name": "mcp__s__shot", "content": _envelope("[1]", _image())},+ {"role": "tool", "name": "mcp__s__shot", "content": _envelope("[1]", _image())},+ {"role": "user", "content": "and the tool's?"},+ ]+ out = promote_history(history, vision = True, reserve_for_caller = True)+ assert decoded == [], "nothing to promote, so nothing to decode"+ assert len(mcp_images._all_image_url_parts(out)) == mcp_images.MAX_TOTAL_MODEL_IMAGES++ out = promote_history(history, vision = True)+ assert len(decoded) == 2, "a GGUF caller keeps the full allowance and decodes both"+ assert len(mcp_images._all_image_url_parts(out)) == mcp_images.MAX_TOTAL_MODEL_IMAGES + 2+++def test_every_async_promotion_wrapper_hops_on_the_marker_alone():+ """Stripping an envelope for a text-only target parses the same 12 MB array as+ promoting it, so the hop off the loop cannot be gated on vision."""+ import inspect++ import routes.inference as inference_route++ for wrapper in (+ inference_route._promote_mcp_history_images_async,+ inference_route._promote_local_mcp_images_async,+ inference_route._build_external_messages_async,+ ):+ body = inspect.getsource(wrapper)+ assert "if _messages_mention_mcp_images(messages):" in body, wrapper.__name__+ assert "vision and _messages_mention" not in body, wrapper.__name__+ assert "promote and _messages_mention" not in body, wrapper.__name__+ count = inspect.getsource(inference_route.anthropic_count_tokens)+ assert "await asyncio.to_thread(_messages_have_promotable_mcp_images, openai_messages)" in count+++def test_live_image_extraction_is_gated_on_the_target_reading_them():+ """mcp_images() json-loads the whole envelope; a text-only target discards the+ result, and the external loop runs on the event loop, so it parses in a thread."""+ import inspect++ import routes.inference as inference_route+ from core.inference import llama_cpp, safetensors_agentic, studio_tool_loop++ assert (+ "await asyncio.to_thread(completion.mcp_images) if run.supports_vision else []"+ in inspect.getsource(studio_tool_loop)+ )+ assert (+ 'if mcp_images_mentioned_in(completion.result or "") and self.is_vision'+ in inspect.getsource(llama_cpp)+ )+ assert "completion.mcp_images() if images_sink is not None else []" in inspect.getsource(+ safetensors_agentic+ )+ count = inspect.getsource(inference_route.chat_count_tokens)+ assert "openai_messages = await _promote_mcp_history_images_async(" in count+ assert "openai_messages = promote_mcp_history_images(" not in count+++def test_an_entry_with_an_unbounded_mime_type_is_not_an_image():+ """A token subtype has no length bound, and metadata is not where megabytes may+ hide from the byte budgets. Mirrors the frontend's MAX_MCP_IMAGE_MIME_CHARS."""+ long_mime = "image/" + "x" * mcp_images.MAX_MCP_IMAGE_MIME_CHARS+ text, images = split_images(_envelope("[2]", _image(mime = long_mime), _image()))+ # The envelope still splits -- or its megabytes would stay in the prompt as text.+ assert len(images) == 2+ assert mcp_images.count_probably_decodable(images) == 1+ assert not mcp_images.probably_decodable(_image(mime = long_mime))+ assert len(mcp_images.png_payloads(images)) == 1, "not decoded either"+++def test_the_monitor_prompt_never_parses_the_envelope(monkeypatch):+ """The monitor row is built on the event loop from the raw messages; cutting at+ the marker shows the same text as the parsed strip without json-loading 12 MB."""+ import routes.inference as inference_route++ content = _envelope("what the tool said", _image())+ assert mcp_images.text_before_envelope(content) == split_images(content)[0]++ def boom(_content):+ raise AssertionError("parsed on the loop")++ monkeypatch.setattr(inference_route, "split_mcp_images", boom)+ prompt = inference_route._monitor_prompt_from_messages(+ [{"role": "tool", "content": content}, {"role": "user", "content": "and?"}]+ )+ assert prompt == "tool: what the tool said\n\nuser: and?"+++def test_the_gguf_admission_estimate_parses_off_the_loop():+ """reserve() binds a waiter to the running loop, so only the ESTIMATE can move: it+ strips every replayed envelope. Every route reservation goes through the async+ form, which hops on the marker."""+ import inspect++ import routes.inference as inference_route++ routes_src = inspect.getsource(inference_route)+ assert "reservation, admission_config = _openai_llama_admission_reserve(" not in routes_src+ assert routes_src.count("await _openai_llama_admission_reserve_async(") >= 7+ wrapper = inspect.getsource(inference_route._openai_llama_admission_reserve_async)+ assert "await asyncio.to_thread(\n _openai_llama_admission_estimate," in wrapper+ assert "_messages_mention_mcp_images(" in wrapper+++def test_a_sixteen_bit_picture_is_scaled_not_clipped():+ """convert("RGB") on I;16 clips 0..65535 to 8 bits, so ordinary 16-bit imagery came+ out nearly white while the tool card showed the real picture."""+ from PIL import Image++ source = Image.new("I;16", (4, 4))+ source.putdata([30_000] * 16)+ flat = mcp_images.flattened_rgb(source)+ assert flat.mode == "RGB"+ value = flat.getpixel((0, 0))[0]+ assert 110 <= value <= 122, value++ buffer = io.BytesIO()+ source.save(buffer, format = "PNG")+ payload = base64.b64encode(buffer.getvalue()).decode("ascii")+ url = mcp_images._png_data_url(payload)+ assert url+ decoded = Image.open(io.BytesIO(base64.b64decode(url.split(",", 1)[1])))+ assert 110 <= decoded.getpixel((0, 0))[0] <= 122+++def test_gguf_admission_charges_replay_beside_the_attachments():+ """The GGUF paths keep the full replay allowance beside the caller's pictures, so one+ attachment plus eight replayed pictures sends nine and must be charged nine."""+ import routes.inference as inference_route++ messages = [+ {+ "role": "user",+ "content": [+ {"type": "text", "text": "this"},+ {"type": "image_url", "image_url": {"url": f"data:image/png;base64,{_png()}"}},+ ],+ },+ ]+ for r in range(9):+ messages.append(+ {+ "role": "tool",+ "name": "mcp__s__shot",+ "tool_call_id": f"c{r}",+ "content": _envelope("[1]", _image()),+ }+ )+ _estimate, image_parts = inference_route._openai_llama_admission_messages_for_estimate(+ messages, vision = True+ )+ assert image_parts == 1 + mcp_images.MAX_TOTAL_MODEL_IMAGES+++def test_the_anthropic_server_tool_gate_reads_caller_attachments_only():+ """A promoted replay sets _has_image too; gating on it routed a follow-up away from+ the server tools it selected because an earlier tool had returned a picture."""+ import inspect++ import routes.inference as inference_route++ body = inspect.getsource(inference_route.anthropic_messages)+ assert "and not _anthropic_has_image\n" in body+ assert "supports_tools and not _has_image" not in body+++def test_the_safetensors_loop_stamps_a_round_id_on_every_decision():+ """Replay groups consecutive results as one batch; the frontend keeps them+ consecutive only when the cards share a round_id."""+ import inspect++ from core.inference import safetensors_agentic++ body = inspect.getsource(safetensors_agentic)+ assert 'decision.provenance["round_id"] = iteration' in body+++def test_a_merge_into_a_trailing_nudge_still_says_where_the_pictures_came_from():+ """A deferred no-op nudge is a trailing user turn the batch's pictures merge into+ (two user turns in a row is what a strict template rejects). Merged bare, they read+ as attachments to the nudge; the note names the tool they came from."""+ nudge = {"role": "user", "content": "Tool `lookup` is not available; continue."}++ conversation = [dict(nudge)]+ mcp_images.append_placeholder_turn(conversation, 1, 1)+ [merged] = conversation+ kinds = [part["type"] for part in merged["content"]]+ assert kinds == ["text", "image", "text"], kinds+ assert merged["content"][-1]["text"].startswith(mcp_images.IMAGE_TURN_TEXT)++ conversation = [dict(nudge)]+ mcp_images.append_image_turn(+ conversation, [[_image()]], per_result = True, lead = mcp_images.DETACHED_IMAGE_TURN_TEXT+ )+ [merged] = conversation+ kinds = [part["type"] for part in merged["content"]]+ assert kinds == ["text", "image_url", "text"], kinds+ assert merged["content"][-1]["text"].startswith(mcp_images.DETACHED_IMAGE_TURN_TEXT)+++def test_a_replay_merged_into_the_question_still_says_where_the_pictures_came_from():+ """Promotion merges a batch into the user turn that follows it (two user turns in a+ row is what a strict template rejects). Merged bare, the pictures read as ones the+ user attached; the note names the tool, detached wording when another tool's result+ sat between. And the question stays the user's turn for the attachment's ordinal."""+ history = [+ {"role": "tool", "name": "mcp__s__shot", "content": _envelope("[1]", _image())},+ {"role": "tool", "name": "web_search", "content": "three results"},+ {"role": "user", "content": "what do you see?"},+ ]+ out, payloads = mcp_images.promote_history_local(history, vision = True)+ assert len(payloads) == 1+ question = out[-1]+ kinds = [part["type"] for part in question["content"]]+ assert kinds == ["image", "text", "text"], kinds+ assert question["content"][1]["text"].startswith(mcp_images.DETACHED_IMAGE_TURN_TEXT)+ assert question["content"][2]["text"] == "what do you see?"+ assert not mcp_images.is_synthetic_image_turn(question)++ out = promote_history(history, vision = True)+ question = out[-1]+ kinds = [part["type"] for part in question["content"]]+ assert kinds == ["image_url", "text", "text"], kinds+ assert question["content"][1]["text"].startswith(mcp_images.DETACHED_IMAGE_TURN_TEXT)+ assert not mcp_images.is_synthetic_image_turn(question)+ assert mcp_images.is_synthetic_image_turn(mcp_images.placeholder_turn(1, 1))+++def test_a_camera_jpeg_is_shown_the_way_up_its_exif_says():+ """The coded raster of a portrait photo is landscape with orientation 6 in EXIF;+ re-encoding it as PNG without the transpose showed the model a sideways picture."""+ from PIL import Image++ source = Image.new("RGB", (8, 4), (10, 120, 200))+ exif = Image.Exif()+ exif[0x0112] = 6+ buffer = io.BytesIO()+ source.save(buffer, format = "JPEG", exif = exif.tobytes())+ url = mcp_images._png_data_url(base64.b64encode(buffer.getvalue()).decode("ascii"))+ assert url+ decoded = Image.open(io.BytesIO(base64.b64decode(url.split(",", 1)[1])))+ assert decoded.size == (4, 8), decoded.size+++def test_the_content_part_extractor_keeps_marker_text_and_hops_the_split(monkeypatch):+ """An MCP tool's own text can contain the marker line with no envelope behind it, so+ the strip is the exact split for every result -- and the awaited callers take the+ async form, which runs that split in a worker whenever the marker is present."""+ import asyncio+ import inspect++ import routes.inference as inference_route+ from models.inference import ChatMessage++ docs = "the sentinel is\n" + mcp_images.SENTINEL + " followed by a JSON array"+ for name in ("mcp__fs__shot", "read_file"):+ _system, chat_messages, _image_b64 = inference_route._extract_content_parts(+ [+ ChatMessage(role = "user", content = "look"),+ ChatMessage(role = "tool", tool_call_id = "c1", name = name, content = docs),+ ]+ )+ tool = next(m for m in chat_messages if m.get("role") == "tool")+ assert tool["content"] == docs, name++ envelope = _envelope("what the tool said", _image())+ hops: list = []+ original = inference_route.asyncio.to_thread++ async def counting(fn, *args, **kwargs):+ hops.append(fn.__name__)+ return await original(fn, *args, **kwargs)++ monkeypatch.setattr(inference_route.asyncio, "to_thread", counting)+ _system, chat_messages, _image_b64 = asyncio.run(+ inference_route._extract_content_parts_async(+ [+ ChatMessage(role = "user", content = "look"),+ ChatMessage(role = "tool", tool_call_id = "c0", name = "mcp__fs__shot", content = envelope),+ ]+ )+ )+ assert hops == ["_extract_content_parts"], hops+ tool = next(m for m in chat_messages if m.get("role") == "tool")+ assert tool["content"] == "what the tool said"++ src = inspect.getsource(inference_route)+ for fn in ("generate_audio", "_mlx_count_chat_tokens", "chat_count_tokens"):+ body = inspect.getsource(getattr(inference_route, fn))+ assert "await _extract_content_parts_async(" in body, fn+ assert "= _extract_content_parts(" not in body, fn+ chat = inspect.getsource(inference_route.produce_openai_chat_completions)+ # only the async extraction strips images; all three synchronous paths preserve them.+ assert chat.count("await _extract_content_parts_async(") == 1+ assert chat.count("keep_tool_images = True") == 3+++def test_the_note_counts_only_the_tools_pictures_in_a_mixed_turn():+ """Five caller attachments beside a four-picture result trimmed to three: the note+ must read (3 of 4), not (8 of 9) -- the attachments are not the tool's."""+ attachments = [+ {"type": "image_url", "image_url": {"url": f"data:image/png;base64,A{i}"}} for i in range(5)+ ]+ promoted = [+ {"type": "image_url", "image_url": {"url": f"data:image/png;base64,T{i}"}} for i in range(4)+ ]+ turn = {+ "role": "user",+ "content": [+ *attachments,+ *promoted,+ {"type": "text", "text": mcp_images.IMAGE_TURN_TEXT},+ {"type": "text", "text": "what do you see?"},+ ],+ }+ conversation = [turn]+ mcp_images._drop_oldest_image_parts(conversation, 1, "image_url", only = list(promoted))+ parts = conversation[0]["content"]+ assert sum(1 for p in parts if p.get("type") == "image_url") == 8+ note = next(+ p["text"]+ for p in parts+ if p.get("type") == "text" and p["text"].startswith(mcp_images.IMAGE_TURN_TEXT)+ )+ assert note == f"{mcp_images.IMAGE_TURN_TEXT} (3 of 4)", note+ assert all(p in parts for p in attachments), "the attachments were not the ones dropped"+++def _first_note(conversation: list) -> str:+ return next(+ part["text"]+ for message in conversation+ if isinstance(message.get("content"), list)+ for part in message["content"]+ if part.get("type") == "text" and part["text"].startswith("Images returned by")+ )+++def test_a_replay_batch_of_two_picture_results_is_detached():+ """Two parallel calls both returned pictures and share the turn; "the tool call+ above" would hand every picture to whichever ran last."""+ history = [+ {"role": "tool", "name": "mcp__s__a", "content": _envelope("[1]", _image())},+ {"role": "tool", "name": "mcp__s__b", "content": _envelope("[1]", _image())},+ {"role": "assistant", "content": "two shots"},+ ]+ assert _first_note(promote_history(history, vision = True)).startswith(+ mcp_images.DETACHED_IMAGE_TURN_TEXT+ )+ out, _payloads = mcp_images.promote_history_local(history, vision = True)+ assert _first_note(out).startswith(mcp_images.DETACHED_IMAGE_TURN_TEXT)+++ "mark, count", [(mcp_images.top_up_image_markers, 2), (mcp_images.mark_last_user_turn, 1)]+)+def test_a_displaced_replay_marker_takes_its_note_with_it(mark, count):+ """The replay merged its marker and note into the question that also owns the+ attachment; the attachment displaces the marker, and the note must go too, or it+ says the caller's own picture was returned by the tool."""+ question = {+ "role": "user",+ "content": [+ {"type": "image"},+ {"type": "text", "text": mcp_images.IMAGE_TURN_TEXT},+ {"type": "text", "text": "and this one?"},+ ],+ }+ for ordinal in (1, None):+ conversation = [{"role": "user", "content": "read a.png"}, dict(question)]+ prior = mcp_images.image_marker_parts(conversation)+ topped = mark(conversation, count, ordinal = ordinal)+ parts = topped[1]["content"]+ assert sum(1 for p in parts if p.get("type") == "image") == 1, parts+ texts = [p["text"] for p in parts if p.get("type") == "text"]+ assert texts == ["and this one?"], texts+ assert mcp_images.pixels_in_marker_order(topped, prior, ["REPLAY"], "ATTACHMENT") == [+ "ATTACHMENT"+ ]+++def test_the_note_counts_a_result_the_allowance_left_nothing_of():+ """Three results of three in one batch on the marker path: one picture is shown and+ the note must say (1 of 9), not (1 of 6) -- the result with no admitted candidate+ is still part of what the batch returned."""+ history = [+ {+ "role": "tool",+ "name": "mcp__s__a",+ "content": _envelope("[3]", *[_image() for _ in range(3)]),+ },+ {+ "role": "tool",+ "name": "mcp__s__b",+ "content": _envelope("[3]", *[_image() for _ in range(3)]),+ },+ {+ "role": "tool",+ "name": "mcp__s__c",+ "content": _envelope("[3]", *[_image() for _ in range(3)]),+ },+ {"role": "assistant", "content": "three shots"},+ ]+ eligible = mcp_images.eligible_replay_images(history, local = True)+ assert min(eligible.values()) == 0, eligible+ out, payloads = mcp_images.promote_history_local(history, vision = True)+ assert len(payloads) == 1+ assert _first_note(out) == f"{mcp_images.DETACHED_IMAGE_TURN_TEXT} (1 of 9)"+++def test_local_replay_cap_keeps_caller_pixels_with_their_markers():+ history = [+ {"role": "user", "content": [{"type": "image"}, {"type": "text", "text": "first"}]},+ {"role": "assistant", "content": "noted"},+ ]+ for index in range(MAX_MODEL_IMAGES * 3):+ history.extend(+ [+ {+ "role": "tool",+ "name": "mcp__test__image",+ "content": _envelope(str(index), _image()),+ },+ {"role": "assistant", "content": "noted"},+ ]+ )+ history.append(+ {"role": "user", "content": [{"type": "image"}, {"type": "text", "text": "last"}]}+ )+ first, last = object(), object()++ messages, pixels = mcp_images.promote_history_local(+ history, vision = True, caller_images = [first, last]+ )++ assert len(pixels) == mcp_images.MAX_TOTAL_MODEL_IMAGES+ assert pixels[0] is first+ assert pixels[-1] is last+ assert len(mcp_images.image_marker_parts(messages)) == len(pixels)+ assert messages[0] is history[0]+ assert messages[-1] is history[-1]+++def test_local_replay_does_not_add_a_second_image_to_an_attachment_turn():+ attachment = object()+ user = {"role": "user", "content": [{"type": "image"}, {"type": "text", "text": "compare"}]}+ history = [+ {"role": "tool", "name": "mcp__test__image", "content": _envelope("shot", _image())},+ user,+ ]++ messages, pixels = mcp_images.promote_history_local(+ history, vision = True, caller_images = [attachment]+ )++ assert pixels == [attachment]+ assert messages[-1] is user+ assert len(mcp_images.image_marker_parts(messages)) == 1diff --git a/studio/backend/tests/test_mlx_inference_backend.py b/studio/backend/tests/test_mlx_inference_backend.pyindex abf19393b71..b15a495956c 100644--- a/studio/backend/tests/test_mlx_inference_backend.py+++ b/studio/backend/tests/test_mlx_inference_backend.py@@ -1014,7 +1014,7 @@ def test_mlx_vlm_generation_selects_renderer_by_capability(monkeypatch):assert calls["stream"][-1][0][2] == "<image> healthy generic"state["generic"] = "generic prompt"text_messages = [{"role": "user", "content": "hello"}]- assert list(backend._generate_vlm(*((text_messages, None) + args[2:]), tools = tools)) == ["ok"]+ assert list(backend._generate_vlm(*((text_messages, []) + args[2:]), tools = tools)) == ["ok"]assert calls["generic"][-1]["tools"] == toolsassert calls["stream"][-1][0][2] == "generic prompt"two_images = [{"role": "user", "content": [{"type": "image"}, {"type": "image"}]}]@@ -5196,6 +5196,195 @@ def test_vlm_add_special_tokens_falls_back_to_the_inline_rule(monkeypatch):assert rule("gemma4", template) == "mlx-vlm's answer"+def test_mlx_vlm_pixels_follow_the_order_the_markers_appear_in():+ """MLX binds pixels to markers positionally. The replayed MCP pictures carry+ their markers on earlier turns, and the attachment's marker goes on the last+ user turn, so the attachment's pixels have to come last."""+ from core.inference.mlx_inference import MLXInferenceBackend++ backend = MLXInferenceBackend()+ backend._model = object()+ backend._is_vlm = True+ backend._multi_image_marker = "<image>"+ captured = []+ backend._generate_vlm = lambda messages, attached, *_args, **_kwargs: (+ captured.append((messages, attached)) or iter(())+ )+ replayed_first, replayed_second, attachment = object(), object(), object()+ messages = [+ {"role": "user", "content": [{"type": "image"}, {"type": "text", "text": "a"}]},+ {"role": "assistant", "content": "seen"},+ {"role": "user", "content": [{"type": "image"}, {"type": "text", "text": "b"}]},+ {"role": "assistant", "content": "seen"},+ {"role": "user", "content": "and this one?"},+ ]++ list(+ backend.generate_chat_response(+ messages,+ image = attachment,+ images = [replayed_first, replayed_second],+ )+ )++ sent_messages, attached = captured[0]+ assert attached == [replayed_first, replayed_second, attachment]+ # The attachment's own marker is the last one in the conversation.+ marker_turns = [+ index+ for index, message in enumerate(sent_messages)+ if isinstance(message.get("content"), list)+ and any(part.get("type") == "image" for part in message["content"])+ ]+ assert marker_turns[-1] == len(sent_messages) - 1+++def test_mlx_attachment_displaces_a_replayed_image_on_the_same_turn():+ """A local turn takes one picture, so the attachment replaces the replay+ marker and its pixels together."""+ from core.inference.mlx_inference import MLXInferenceBackend, _count_vlm_images++ backend = MLXInferenceBackend()+ backend._model = object()+ backend._is_vlm = True+ backend._multi_image_marker = "<image>"+ captured = []+ backend._generate_vlm = lambda messages, attached, *_a, **_k: (+ captured.append((messages, attached)) or iter(())+ )+ replayed, attachment = object(), object()+ messages = [+ {+ "role": "user",+ "content": [{"type": "image"}, {"type": "text", "text": "and this one?"}],+ }+ ]++ list(backend.generate_chat_response(messages, image = attachment, images = [replayed]))++ sent, attached = captured[0]+ markers = sum(_count_vlm_images(m.get("content")) for m in sent)+ assert attached == [attachment]+ assert markers == len(attached), f"{markers} marker(s) for {len(attached)} images"+ # The attachment's pixels are last, so its marker has to be too.+ assert sent[-1]["content"][-1] == {"type": "image"}+++def test_mlx_does_not_double_mark_an_attachment_the_client_already_marked():+ from core.inference.mlx_inference import MLXInferenceBackend, _count_vlm_images++ backend = MLXInferenceBackend()+ backend._model = object()+ backend._is_vlm = True+ backend._multi_image_marker = "<image>"+ captured = []+ backend._generate_vlm = lambda messages, attached, *_a, **_k: (+ captured.append((messages, attached)) or iter(())+ )+ messages = [{"role": "user", "content": [{"type": "image"}, {"type": "text", "text": "x"}]}]++ list(backend.generate_chat_response(messages, image = object()))++ sent, attached = captured[0]+ assert sum(_count_vlm_images(m.get("content")) for m in sent) == len(attached) == 1+++def test_mlx_normalizes_a_replay_only_conversation_for_a_processor_template():+ """A processor template needs every turn in part shape. A replay-only request+ (images without an attachment) is a mix of strings and marker lists, and gating+ the normalization on the singular image left it failing at render."""+ from types import SimpleNamespace++ from core.inference.mlx_inference import MLXInferenceBackend, _count_vlm_images++ backend = MLXInferenceBackend()+ backend._model = object()+ backend._is_vlm = True+ backend._multi_image_marker = "<image>"+ # A processor carrying its own template is what chat_render_target selects.+ backend._processor = SimpleNamespace(+ apply_chat_template = lambda *a, **k: "prompt", chat_template = "t"+ )+ captured = []+ backend._generate_vlm = lambda messages, attached, *a, **k: (+ captured.append((messages, attached)) or iter(())+ )+ messages = [+ {"role": "tool", "content": "[1 image returned]"},+ {"role": "user", "content": [{"type": "image"}, {"type": "text", "text": "what was it"}]},+ ]++ list(backend.generate_chat_response(messages, images = [object()]))++ sent, attached = captured[0]+ assert all(isinstance(message.get("content"), list) for message in sent), sent+ markers = sum(_count_vlm_images(message.get("content")) for message in sent)+ assert markers == len(attached) == 1+++def test_mlx_binds_an_earlier_attachment_to_its_own_turn_not_a_newer_replay():+ """The attachment's turn can PRECEDE a tool's picture. The route used to pre-add+ its marker, so the backend counted that marker as history's, the top-up became a+ no-op, and the two pixels bound to each other's turns -- the model was shown the+ screenshot where the user's own diagram belonged."""+ from core.inference.mcp_images import placeholder_turn+ from core.inference.mlx_inference import MLXInferenceBackend++ backend = MLXInferenceBackend()+ backend._model = object()+ backend._is_vlm = True+ backend._multi_image_marker = "<image>"+ captured = []+ backend._generate_vlm = lambda messages, attached, *_args, **_kwargs: (+ captured.append((messages, attached)) or iter(())+ )+ attachment, replayed = object(), object()++ # What the plain route hands the backend once the fix stops it pre-marking: the+ # attachment's turn is still a plain string, and the only marker in the+ # conversation is the replayed picture's.+ messages = [+ {"role": "user", "content": "here is my diagram"},+ {"role": "assistant", "content": "noted"},+ {+ "role": "assistant",+ "content": "",+ "tool_calls": [+ {+ "id": "c1",+ "type": "function",+ "function": {"name": "mcp__s__shot", "arguments": "{}"},+ }+ ],+ },+ {"role": "tool", "tool_call_id": "c1", "content": "[1 image returned]"},+ placeholder_turn(1, 1),+ {"role": "user", "content": "which one is bluer?"},+ ]++ list(+ backend.generate_chat_response(+ messages,+ image = attachment,+ images = [replayed],+ image_ordinal = 0,+ )+ )++ sent_messages, attached = captured[0]+ marker_turns = [+ index+ for index, message in enumerate(sent_messages)+ if isinstance(message.get("content"), list)+ and any(part.get("type") == "image" for part in message["content"])+ ]+ assert len(marker_turns) == 2, sent_messages+ # The attachment's marker is on the FIRST turn, the replay's on the placeholder+ # after it -- so the pixels have to arrive in that same order.+ assert marker_turns[0] < marker_turns[1]+ assert attached == [attachment, replayed], "the pixels bound to each other's markers"++def _run_mlx_reasoning_stream(monkeypatch,turn,@@ -5560,6 +5749,59 @@ def test_mlx_unset_budget_falls_back_when_the_prompt_cannot_be_counted(monkeypatassert backend._unset_generation_budget("P") == UNSET_GENERATION_BUDGET+def test_mlx_count_strips_replayed_mcp_images_off_the_event_loop(monkeypatch):+ import base64+ import io+ import json+ import threading+ from PIL import Image+ from core.inference import mcp_images+ from routes import inference as route++ buffer = io.BytesIO()+ Image.new("RGB", (4, 4), "blue").save(buffer, format = "PNG")+ envelope = (+ "screenshot result\n"+ + mcp_images.SENTINEL+ + json.dumps(+ [{"data": base64.b64encode(buffer.getvalue()).decode(), "mimeType": "image/png"}]+ )+ )+ history = [+ {"role": "user", "content": "Read the screenshot"},+ {+ "role": "assistant",+ "content": "",+ "tool_calls": [+ {+ "id": "c1",+ "type": "function",+ "function": {"name": "mcp__screen__shot", "arguments": "{}"},+ }+ ],+ },+ {"role": "tool", "tool_call_id": "c1", "name": "mcp__screen__shot", "content": envelope},+ {"role": "user", "content": "Describe it"},+ ]+ loop_thread = threading.get_ident()+ split_threads = []+ original_split = mcp_images.split_images++ def track_split(content):+ if mcp_images.SENTINEL in content:+ split_threads.append(threading.get_ident())+ return original_split(content)++ monkeypatch.setattr(mcp_images, "split_images", track_split)+ monkeypatch.setattr(route, "split_mcp_images", track_split)+ backend = _RenderRecordingBackend()+ response = _count_route(monkeypatch, backend, messages = history, enable_tools = False)+ assert response.status_code == 200+ assert split_threads and all(thread != loop_thread for thread in split_threads)+ assert backend.messages[1]["tool_calls"][0]["function"]["name"] == "mcp__screen__shot"+ assert backend.messages[2]["content"] == "screenshot result"++_CLIP_B64 = "AAAAGGZ0eXBtcDQy" # a bare mp4 box header, decoded byte-for-byte by the backend@@ -6182,10 +6424,9 @@ def test_several_images_reach_the_runtime_in_the_order_they_were_attached(monkey@pytest.mark.parametrize("marker, render, error",[- (None, None, "one image per request"),("<|image|>", "<|image|>only one", "marked 1 image"),],- ids = ["no marker was classified", "the render collapsed a marker"],+ ids = ["the render collapsed a marker"],)def test_a_render_that_cannot_bind_every_image_is_refused(monkeypatch, marker, render, error):"""One image is still taken either way; too few markers would describe one picture twice."""@@ -6196,3 +6437,78 @@ def test_a_render_that_cannot_bind_every_image_is_refused(monkeypatch, marker, rturn = [{"role": "user", "content": "compare"}]with pytest.raises((ValueError, RuntimeError), match = error):list(backend.generate_chat_response(turn, images = [object(), object()]))+++def test_a_single_image_template_is_served_one_picture_not_refused(monkeypatch):+ """A template that marks one image cannot bind several, but a resumed chat carrying+ replayed MCP pictures must still be answered: the caller's attachment, or else the+ newest picture, is served with its own marker and the others are left out."""+ backend, calls = _vlm_runtime(monkeypatch, marker = None)+ older, newer = object(), object()+ turn = [{"role": "user", "content": "compare"}]++ assert list(backend.generate_chat_response(turn, images = [older, newer])) == ["ok"]+ assert calls[-1]["images"] == [newer]+ assert calls[-1]["prompt"].count("<|image|>") == 1++ attachment = object()+ list(backend.generate_chat_response(turn, image = attachment, images = [older]))+ assert calls[-1]["images"] == [attachment]+ assert calls[-1]["prompt"].count("<|image|>") == 1++[email protected]("merged_question", [False, True])+def test_single_image_replay_notes_describe_only_retained_pixels(monkeypatch, merged_question):+ from core.inference.mcp_images import IMAGE_TURN_TEXT, image_marker_parts, placeholder_turn++ backend, calls = _vlm_runtime(monkeypatch, marker = None)+ older, newer = object(), object()+ previous_image = placeholder_turn(1)+ if merged_question:+ previous_image["content"].append({"type": "text", "text": "Keep this user question."})+ history = [+ {"role": "user", "content": "Inspect the first screenshot."},+ {+ "role": "assistant",+ "content": "",+ "tool_calls": [+ {+ "id": "first",+ "type": "function",+ "function": {+ "name": "mcp__test__screenshot",+ "arguments": "{}",+ },+ }+ ],+ },+ {"role": "tool", "tool_call_id": "first", "content": "[1 image returned]"},+ previous_image,+ {"role": "assistant", "content": "The first screenshot was inspected."},+ {"role": "user", "content": "Inspect the next screenshot."},+ {+ "role": "assistant",+ "content": "",+ "tool_calls": [+ {+ "id": "second",+ "type": "function",+ "function": {+ "name": "mcp__test__screenshot",+ "arguments": "{}",+ },+ }+ ],+ },+ {"role": "tool", "tool_call_id": "second", "content": "[1 image returned]"},+ placeholder_turn(1),+ ]+ assert list(backend.generate_chat_response(history, images = [older, newer])) == ["ok"]+ assert calls[-1]["images"] == [newer]+ assert calls[-1]["prompt"].count("<|image|>") == 1+ if merged_question:+ assert "Keep this user question." in calls[-1]["prompt"]+ assert "(0 of 1)" in calls[-1]["prompt"]+ else:+ assert calls[-1]["prompt"].count(IMAGE_TURN_TEXT) == 1+ assert len(image_marker_parts(history)) == 2diff --git a/studio/backend/tests/test_openai_tool_passthrough.py b/studio/backend/tests/test_openai_tool_passthrough.pyindex 9b209e56a27..0e5b4bdf8dd 100644--- a/studio/backend/tests/test_openai_tool_passthrough.py+++ b/studio/backend/tests/test_openai_tool_passthrough.py@@ -4298,6 +4298,259 @@ class TestGgufVisionMessages:{"role": "system", "content": "Use tools."}]+ def _mcp_tool_history(self, text = "[1 image returned]"):+ from core.inference import mcp_images+ envelope = json.dumps([{"data": self._PNG_B64, "mimeType": "image/png"}])+ return [+ {"role": "user", "content": "what does the file look like"},+ {+ "role": "assistant",+ "content": "",+ "tool_calls": [+ {+ "id": "call_0",+ "type": "function",+ "function": {"name": "mcp__fs__read_media_file", "arguments": "{}"},+ }+ ],+ },+ {+ "role": "tool",+ "tool_call_id": "call_0",+ "content": text + "\n" + mcp_images.SENTINEL + envelope,+ },+ {"role": "assistant", "content": "a blue square"},+ {"role": "user", "content": "what colour was it again"},+ ]++ def test_replayed_mcp_images_become_an_image_turn_for_a_vision_model(self):+ req = ChatCompletionRequest(model = "default", messages = self._mcp_tool_history())++ messages, _ = _openai_messages_for_gguf_chat(req, is_vision = True)++ tool_message = next(m for m in messages if m["role"] == "tool")+ assert tool_message["content"] == "[1 image returned]"+ promoted = messages[messages.index(tool_message) + 1]+ assert promoted["role"] == "user"+ assert promoted["content"][1]["image_url"]["url"].startswith("data:image/png;base64,")++ def test_replayed_mcp_images_are_stripped_for_a_text_only_model(self):+ req = ChatCompletionRequest(model = "default", messages = self._mcp_tool_history())++ messages, _ = _openai_messages_for_gguf_chat(req, is_vision = False)++ assert [m["role"] for m in messages] == ["user", "assistant", "tool", "assistant", "user"]+ assert messages[2]["content"] == "[1 image returned]"++ def test_passthrough_never_relays_the_image_envelope_as_tool_text(self):+ req = ChatCompletionRequest(model = "default", messages = self._mcp_tool_history())++ messages = _openai_messages_for_passthrough(req)++ assert all("__MCP_IMAGES__" not in str(m.get("content")) for m in messages)++ def test_passthrough_promotes_the_images_for_a_vision_model(self):+ req = ChatCompletionRequest(model = "default", messages = self._mcp_tool_history())++ messages = _openai_messages_for_passthrough(req, vision = True)++ promoted = messages[3]+ assert promoted["role"] == "user"+ assert promoted["content"][1]["image_url"]["url"].startswith("data:image/png;base64,")++ def test_the_native_path_never_templates_the_image_envelope(self):+ req = ChatCompletionRequest(model = "default", messages = self._mcp_tool_history())++ _, chat_messages, _ = _extract_content_parts(req.messages)++ # The name is correlated from the call now, so the local path can run the+ # provenance gate on a result that arrived unnamed.+ assert chat_messages[2] == {+ "role": "tool",+ "content": "[1 image returned]",+ "name": "mcp__fs__read_media_file",+ }++ def test_a_replayed_envelope_alone_does_not_demand_a_vision_model(self):+ """The capability preflight runs before promote_history strips the envelope+ for a text-only target, so counting history there strands the conversation:+ the switch is refused for a picture that model would never be shown."""+ from routes.inference import _request_has_attached_image, _request_has_image++ req = ChatCompletionRequest(model = "default", messages = self._mcp_tool_history())++ assert not _request_has_attached_image(req)+ # Still an image for the paths that decode one: the work stays off the loop.+ assert _request_has_image(req)++ def test_an_attached_image_still_demands_a_vision_model(self):+ from routes.inference import _request_has_attached_image++ history = self._mcp_tool_history()+ history[-1] = {+ "role": "user",+ "content": [+ {"type": "text", "text": "and this one"},+ {+ "type": "image_url",+ "image_url": {"url": f"data:image/png;base64,{self._PNG_B64}"},+ },+ ],+ }+ req = ChatCompletionRequest(model = "default", messages = history)++ assert _request_has_attached_image(req)++ def test_admission_charges_a_replayed_envelope_as_images_not_text(self):+ """Admission prices the payload before the chat path promotes the envelope.+ Left as text it reserves megabytes of prompt for bytes never sent; ignored+ it charges zero KV for projector images that really are."""+ from routes.inference import _openai_llama_admission_messages_for_estimate++ estimate, image_parts = _openai_llama_admission_messages_for_estimate(+ [+ m.model_dump(exclude_none = True)+ for m in ChatCompletionRequest(+ model = "default", messages = self._mcp_tool_history()+ ).messages+ ]+ )++ assert image_parts == 1+ assert self._PNG_B64 not in json.dumps(estimate)+ assert estimate[2]["content"] == "[1 image returned]"++ def test_admission_never_charges_past_the_conversation_cap(self):+ from core.inference import mcp_images+ from routes.inference import _openai_llama_admission_messages_for_estimate++ envelope = json.dumps([{"data": self._PNG_B64, "mimeType": "image/png"}] * 4)+ turns = []+ for _ in range(6):+ turns.append(+ {+ "role": "tool",+ "tool_call_id": "c",+ "content": "[4 images returned]\n" + mcp_images.SENTINEL + envelope,+ }+ )+ _, image_parts = _openai_llama_admission_messages_for_estimate(turns)++ assert image_parts == mcp_images.MAX_TOTAL_MODEL_IMAGES++ def test_the_envelope_cap_never_discounts_really_attached_images(self):+ """#9842 charges per-image KV so parallel vision chats are admitted+ honestly. The envelope cap bounds what promotion adds, not what the+ caller actually sent."""+ from core.inference import mcp_images+ from routes.inference import _openai_llama_admission_messages_for_estimate++ attached = {+ "role": "user",+ "content": [{"type": "text", "text": "compare these"}]+ + [+ {+ "type": "image_url",+ "image_url": {"url": f"data:image/png;base64,{self._PNG_B64}"},+ }+ for _ in range(mcp_images.MAX_TOTAL_MODEL_IMAGES + 4)+ ],+ }++ _, image_parts = _openai_llama_admission_messages_for_estimate([attached])++ assert image_parts == mcp_images.MAX_TOTAL_MODEL_IMAGES + 4++ def test_a_replayed_envelope_is_priced_without_its_base64(self):+ """The count endpoint renders the messages verbatim, so an envelope left in+ would be priced as thousands of tokens of base64 the completion never sends."""+ from routes.inference import promote_mcp_history_images++ raw = [+ m.model_dump(exclude_none = True)+ for m in ChatCompletionRequest(+ model = "default", messages = self._mcp_tool_history()+ ).messages+ ]++ priced = promote_mcp_history_images(raw, vision = False)++ assert all("__MCP_IMAGES__" not in str(m.get("content")) for m in priced)+ assert all(self._PNG_B64 not in str(m.get("content")) for m in priced)++ def test_a_mixed_catalog_provider_still_forwards_an_attached_image(self):+ """openrouter and huggingface are vision-capable for the family and name no+ model, so the MCP gate says no for every model on them. Passed as the general+ vision flag it also stripped the picture the caller attached to a chosen+ vision model, which main forwarded."""+ from core.inference.providers import get_provider_info+ from routes.inference import _build_external_messages, _external_takes_mcp_images++ attached = [+ ChatMessage(+ role = "user",+ content = [+ {"type": "text", "text": "what colour is this"},+ {+ "type": "image_url",+ "image_url": {"url": f"data:image/png;base64,{self._PNG_B64}"},+ },+ ],+ )+ ]++ for provider, model in (+ ("openrouter", "openai/gpt-4o"),+ ("huggingface", "Qwen/Qwen2.5-VL-7B-Instruct"),+ ("mistral", "pixtral-large-latest"),+ ("qwen", "qwen-vl-max"),+ ):+ info = get_provider_info(provider) or {}+ vision = info.get("supports_vision", False)+ gate = _external_takes_mcp_images(provider, vision, model, info)+ assert gate is False, f"{provider} is expected to gate MCP promotion off"++ built = _build_external_messages(+ attached,+ vision,+ provider_type = provider,+ promote_mcp_images = gate,+ )++ parts = built[0]["content"]+ assert [p["type"] for p in parts] == ["text", "image_url"], provider++ # The shape the regression had: one flag answering both questions.+ # The image is gone and the lone text part collapses back to a string.+ conflated = _build_external_messages(attached, gate, provider_type = provider)+ assert (+ conflated[0]["content"] == "what colour is this"+ ), f"{provider}: the two decisions must stay separate"++ def test_the_mcp_gate_still_keeps_an_envelope_off_a_mixed_catalog_provider(self):+ from core.inference.providers import get_provider_info+ from routes.inference import _build_external_messages, _external_takes_mcp_images++ history = [ChatMessage(**m) for m in self._mcp_tool_history()]+ info = get_provider_info("openrouter") or {}+ vision = info.get("supports_vision", False)++ built = _build_external_messages(+ history,+ vision,+ provider_type = "openrouter",+ promote_mcp_images = _external_takes_mcp_images(+ "openrouter", vision, "openai/gpt-4o", info+ ),+ )++ assert all("__MCP_IMAGES__" not in str(m.get("content")) for m in built)+ assert not any(+ isinstance(m.get("content"), list)+ and any(p.get("type") == "image_url" for p in m["content"])+ for m in built+ )+def test_tool_nudge_system_update_dedupes_non_leading_system(self):messages = [{"role": "user", "content": "earlier"},@@ -10588,3 +10841,359 @@ class TestPassthroughImageNormalization:parts = [p for p in messages[0]["content"] if p.get("type") == "image_url"]assert len(parts) == 1assert parts[0]["image_url"]["url"].startswith("data:image/png;base64,")+++class TestMcpImagesOnTheClientToolPassthrough:+ """The client-tool passthrough rebuilds its prompt from payload.messages and+ flattens every content part to text. The MCP payloads survive in gen_kwargs,+ so the markers have to be put back or the two disagree."""++ def test_one_marker_is_restored_per_retained_payload(self):+ from core.inference.mcp_images import append_placeholder_turn++ # What the rebuild leaves: plain string content, no parts at all.+ messages = [+ {"role": "user", "content": "what does the file look like"},+ {"role": "tool", "tool_call_id": "call_0", "content": "[2 images returned]"},+ ]+ payloads = ["AAAA", "BBBB"]++ append_placeholder_turn(messages, len(payloads), len(payloads))++ markers = [+ part+ for message in messages+ if isinstance(message.get("content"), list)+ for part in message["content"]+ if part.get("type") == "image"+ ]+ assert len(markers) == len(payloads)+ assert messages[-1]["role"] == "user"++ def test_the_route_restores_them_on_that_branch(self):+ """Scoped to the client-tool branch, not a character window: the code+ around it moves with every upstream change, the branch does not."""+ import inspect++ import routes.inference as inference_route++ body = inspect.getsource(inference_route.produce_openai_chat_completions)+ branch = body[body.index("if _sf_client_tools:") :]+ assert "_flatten_content_parts_for_local_template" in branch+ rebuild = branch.index("_flatten_content_parts_for_local_template")+ promote = branch.index("_promote_local_mcp_images_async(")+ assert promote < rebuild, (+ "the flatten has to run inside the promotion's argument, so the markers "+ "are put back after the flatten that removed them"+ )+ assert (+ "promote_mcp_images = False" in branch+ ), "the passthrough must leave the envelopes on for the promotion to find"+ # One marker per batch at its own position, never one block of them all.+ assert "mark_mcp_image_turn_local(" not in branch+ assert "insert_mcp_image_turn_before(" not in branch+++class TestCodexVisionCapability:+ def test_an_mcp_run_resolves_the_catalog_before_pinning_the_vision_flag(self):+ """The flag is fixed once for the whole run. A text-only opening turn on a+ cold worker would pin it False and discard a picture an MCP tool returns+ later, on a model the picker offered as vision-capable."""+ import inspect++ import routes.inference as inference_route++ body = inspect.getsource(inference_route._proxy_to_external_provider)+ assert "_may_receive_image = (" in body+ assert "if _may_receive_image and model not in capabilities" in body++ def test_a_replay_only_turn_also_resolves_the_catalog(self):+ """MCP can be switched off while prior image turns remain. The capability+ still has to be resolved, or the replay is stripped from a model that+ could have read it."""+ import inspect++ import routes.inference as inference_route++ body = inspect.getsource(inference_route._proxy_to_external_provider)+ start = body.index("_may_receive_image = (")+ predicate = body[start : body.index("if _may_receive_image", start)]+ assert "image_requested" in predicate+ assert "mcp_enabled" in predicate+ # The promotable form, so a named non-MCP result ending in a valid envelope+ # does not buy a catalog fetch nothing needs.+ assert "_request_has_promotable_mcp_images(payload, exact = False)" in predicate+++_MCP_ADMISSION_PNG = (+ "iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAIAAACQd1PeAAAADUlEQVR42mNk"+ "+M9QDwADhgGAWjR9awAAAABJRU5ErkJggg=="+)+++class TestMcpImageAdmissionAndCaps:+ _PNG = _MCP_ADMISSION_PNG++ def _envelope_turn(+ self,+ count = 1,+ name = "mcp__fs__read_media_file",+ ):+ from core.inference import mcp_images+ images = json.dumps([{"data": self._PNG, "mimeType": "image/png"}] * count)+ return {+ "role": "tool",+ "tool_call_id": "call_0",+ "name": name,+ "content": f"[{count} images returned]\n" + mcp_images.SENTINEL + images,+ }++ def test_a_text_only_backend_is_charged_no_kv_for_replayed_images(self):+ """Generation calls promote_history(vision=False) and sends no projector+ images, so charging them here reserves the cache for pixels llama-server+ never sees and serialises small text-only chats."""+ from routes.inference import _openai_llama_admission_messages_for_estimate++ turns = [self._envelope_turn(4), self._envelope_turn(4)]++ _, vision_parts = _openai_llama_admission_messages_for_estimate(turns, vision = True)+ _, text_only_parts = _openai_llama_admission_messages_for_estimate(turns, vision = False)++ assert vision_parts == 8+ assert text_only_parts == 0++ def test_the_base64_is_stripped_from_the_estimate_either_way(self):+ from routes.inference import _openai_llama_admission_messages_for_estimate+ for vision in (True, False):+ estimate, _ = _openai_llama_admission_messages_for_estimate(+ [self._envelope_turn()], vision = vision+ )+ assert self._PNG not in json.dumps(estimate), vision++ def test_an_attachment_counts_against_the_replay_cap(self):+ """promote_history_local caps the replay alone; the attachment lands after it.+ Its marker goes on the newest user turn -- here a placeholder already holding+ a replay marker -- and a non-GGUF message carries one picture, so the replay+ marker is displaced and pixels_in_marker_order drops its payload rather than+ sliding it onto the next marker. Markers and pixels stay in step at the cap."""+ from core.inference import mcp_images++ conversation = []+ payloads = []+ for _ in range(mcp_images.MAX_TOTAL_MODEL_IMAGES):+ conversation.append(mcp_images.placeholder_turn(1, 1))+ payloads.append("AAAA")+ prior = mcp_images.image_marker_parts(conversation)+ # what the route does when an image is attached on top+ conversation = mcp_images.mark_last_user_turn(conversation, 1)+ pixels = mcp_images.pixels_in_marker_order(conversation, prior, payloads, "BBBB")+ mcp_images.trim_image_turns(conversation, pixels)++ markers = mcp_images.count_image_parts(conversation, "image")+ assert len(pixels) == mcp_images.MAX_TOTAL_MODEL_IMAGES+ assert markers == len(pixels)+ # The attachment is the one that survives on its turn, and it is the newest.+ assert pixels[-1] == "BBBB"+ assert pixels.count("AAAA") == mcp_images.MAX_TOTAL_MODEL_IMAGES - 1++ def test_the_route_trims_both_local_paths(self):+ import inspect++ import routes.inference as inference_route++ body = inspect.getsource(inference_route.produce_openai_chat_completions)+ assert body.count("trim_mcp_image_turns(") == 3, (+ "the tool-loop path, the plain path and the client-tool rebuild all have "+ "to reserve the attachment's slot"+ )++ def test_the_local_replay_promotion_is_off_the_event_loop(self):+ import inspect++ import routes.inference as inference_route++ body = inspect.getsource(inference_route.produce_openai_chat_completions)+ assert "await _promote_local_mcp_images_async(" in body+ helper = inspect.getsource(inference_route._promote_local_mcp_images_async)+ assert "asyncio.to_thread" in helper+++class TestMcpImageProvenanceAndCounting:+ _PNG = _MCP_ADMISSION_PNG++ def _turn(self, name):+ from core.inference import mcp_images++ images = json.dumps([{"data": self._PNG, "mimeType": "image/png"}])+ msg = {+ "role": "tool",+ "tool_call_id": "call_0",+ "content": "output\n" + mcp_images.SENTINEL + images,+ }+ if name is not None:+ msg["name"] = name+ return msg++ def test_admission_strips_a_non_mcp_envelope_but_charges_it_nothing(self):+ """Generation strips the suffix for everyone, so pricing it as text would+ over-reserve; and it is never promoted, so charging image KV would reserve+ the cache for pixels llama-server never sees."""+ from routes.inference import _openai_llama_admission_messages_for_estimate++ estimate, parts = _openai_llama_admission_messages_for_estimate(+ [self._turn("bash")], vision = True+ )++ assert parts == 0+ assert self._PNG not in json.dumps(estimate)++ def test_admission_still_charges_an_mcp_envelope_as_images(self):+ from routes.inference import _openai_llama_admission_messages_for_estimate++ estimate, parts = _openai_llama_admission_messages_for_estimate(+ [self._turn("mcp__fs__read_media_file")], vision = True+ )++ assert parts == 1+ assert self._PNG not in json.dumps(estimate)++ def test_the_estimator_and_generation_agree_on_provenance(self):+ """The two must read the same rule, or the reservation and the prompt+ disagree in whichever direction the mismatch runs."""+ from core.inference.mcp_images import promote_history+ from routes.inference import _openai_llama_admission_messages_for_estimate++ for name, promoted in (("bash", False), ("mcp__fs__read_media_file", True)):+ turn = self._turn(name)+ out = promote_history([dict(turn)], vision = True)+ estimate, parts = _openai_llama_admission_messages_for_estimate(+ [dict(turn)], vision = True+ )+ # Stripped on both sides regardless of provenance...+ assert out[0]["content"] != turn["content"], name+ assert self._PNG not in json.dumps(estimate), name+ # ...and charged exactly where it is promoted.+ assert (len(out) > 1) is promoted, name+ assert (parts > 0) is promoted, name+++class TestMcpReplayDetectorProvenance:+ _PNG = _MCP_ADMISSION_PNG++ def _req(self, name):+ from core.inference import mcp_images++ images = json.dumps([{"data": self._PNG, "mimeType": "image/png"}])+ msg = {+ "role": "tool",+ "tool_call_id": "call_0",+ "content": "output\n" + mcp_images.SENTINEL + images,+ }+ if name is not None:+ msg["name"] = name+ return ChatCompletionRequest(model = "default", messages = [msg])++ def test_only_a_promotable_envelope_counts_as_a_replayed_image(self):+ """Promotion preserves a named non-MCP result as plain text, so reading its+ suffix as images refuses a countable prompt and buys a catalog fetch that+ nothing needs."""+ from routes.inference import _request_has_promotable_mcp_images++ assert _request_has_promotable_mcp_images(self._req("mcp__fs__read_media_file"))+ assert not _request_has_promotable_mcp_images(self._req("bash"))+ # No name is not evidence against it: older stored turns carry none.+ assert _request_has_promotable_mcp_images(self._req(None))++ def test_the_count_guard_runs_before_the_mlx_dispatch(self):+ """_mlx_count_chat_tokens returns its own count, so a guard placed after it+ never sees a resident MLX vision model."""+ import inspect++ import routes.inference as inference_route++ body = inspect.getsource(inference_route.chat_count_tokens)+ guard = body.index("_resident_model_reads_images()")+ dispatch = body.index("_mlx_count_chat_tokens(")+ assert guard < dispatch, "the replay guard has to run before MLX answers"++ def test_the_capability_probe_asks_both_backends(self):+ import inspect++ import routes.inference as inference_route++ probe = inspect.getsource(inference_route._resident_model_reads_images)+ assert "get_llama_cpp_backend()" in probe+ assert "get_inference_backend" in probe+ assert 'entry.get("is_vision")' in probe+++def test_a_text_only_qwen_model_is_never_sent_an_mcp_picture():+ """qwen reports supports_vision for the family and names no model, and every one+ of its four registry defaults is a text model -- the DashScope vision line is+ qwen-vl-*. Appending an image there fails the run upstream rather than degrading."""+ from core.inference.providers import get_provider_info+ from routes.inference import _external_takes_mcp_images++ info = get_provider_info("qwen") or {}+ vision = info.get("supports_vision", False)+ assert vision is True, "the premise is that the family flag is permissive"+ assert not (info.get("model_capabilities") or {}), "and that it names no model"++ for model in info.get("default_models") or []:+ assert _external_takes_mcp_images("qwen", vision, model, info) is False, model+++def test_kimi_stays_permissive_because_its_allowlist_already_narrowed_it():+ """The opposite case, so the conservative set does not grow by reflex: kimi's+ model_id_allowlist admits only the multimodal pair its own entry documents."""+ from core.inference.providers import get_provider_info+ from routes.inference import _external_takes_mcp_images++ info = get_provider_info("kimi") or {}+ vision = info.get("supports_vision", False)++ for model in info.get("default_models") or []:+ assert _external_takes_mcp_images("kimi", vision, model, info) is True, model+++def test_admission_prices_replay_against_what_generation_really_sends():+ """The GGUF paths keep the full replay allowance BESIDE the caller's own picture+ (only a provider reserves the caller's room), so this request sends nine and is+ charged nine. Subtracting the attachment admitted a second such request into KV+ the first had already taken; charging a second full allowance on top of it would+ over-reserve instead, so the replay is still capped at MAX_TOTAL_MODEL_IMAGES."""+ import json as _json++ from core.inference import mcp_images+ from routes.inference import _openai_llama_admission_messages_for_estimate++ _PNG = (+ "iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAIAAACQd1PeAAAADUlEQVR42mNk"+ "+M9QDwADhgGAWjR9awAAAABJRU5ErkJggg=="+ )+ envelope = _json.dumps([{"data": _PNG, "mimeType": "image/png"} for _ in range(4)])+ history = []+ for index in range(2):+ history.append(+ {+ "role": "tool",+ "name": f"mcp__s__shot{index}",+ "content": "[4 images returned]\n" + mcp_images.SENTINEL + envelope,+ }+ )+ history.append(+ {+ "role": "user",+ "content": [+ {"type": "text", "text": "and this one"},+ {"type": "image_url", "image_url": {"url": f"data:image/png;base64,{_PNG}"}},+ ],+ }+ )++ _, image_parts = _openai_llama_admission_messages_for_estimate(history)++ assert image_parts == mcp_images.MAX_TOTAL_MODEL_IMAGES + 1, (+ f"reserved {image_parts} image embeddings for a request that sends "+ f"{mcp_images.MAX_TOTAL_MODEL_IMAGES + 1}"+ )diff --git a/studio/backend/tests/test_orchestrator_unload_cancel.py b/studio/backend/tests/test_orchestrator_unload_cancel.pyindex 33ffd7c6f3a..d64f0534060 100644--- a/studio/backend/tests/test_orchestrator_unload_cancel.py+++ b/studio/backend/tests/test_orchestrator_unload_cancel.py@@ -3484,3 +3484,51 @@ def test_every_long_lived_spawner_consults_the_shutdown_latch():f"{rel} never rechecks the latch after adopting, so a spawn that raced it ""is left outside a sweep that has already finished")+++def test_the_adapter_path_accepts_the_same_images_kwarg_as_the_locked_one():+ """The tool loop hands both paths one common_kwargs. Without `images` on the+ dispatched path an adapter-controlled vision chat raises TypeError before it+ generates, and the markers left in the conversation have no pixels behind."""+ import inspect++ from core.inference.orchestrator import InferenceOrchestrator++ dispatched = inspect.signature(InferenceOrchestrator._generate_dispatched).parameters+ locked = inspect.signature(InferenceOrchestrator._generate_inner).parameters++ assert "images" in locked+ assert "images" in dispatched, (+ "generate_with_adapter_control forwards **common_kwargs straight into "+ "_generate_dispatched, so a missing `images` is an immediate TypeError"+ )+++def test_the_adapter_path_forwards_images_to_the_worker_command():+ import inspect++ from core.inference.orchestrator import InferenceOrchestrator++ body = inspect.getsource(InferenceOrchestrator._generate_dispatched)+ assert (+ "images_b64 = images" in body+ ), "_build_generate_cmd carries the list to the worker as images_b64"+++def test_both_generation_paths_forward_the_image_ordinal():+ """The dispatched path forwarded it and the locked one dropped it, so ordinary+ non-adapter generation reached the worker with None. The marker top-up then put+ the attachment's marker on the newest user turn rather than the turn that+ supplied it, which changes which question appears to own the picture."""+ import inspect++ from core.inference.orchestrator import InferenceOrchestrator++ for method in (+ InferenceOrchestrator._generate_dispatched,+ InferenceOrchestrator._generate_inner,+ ):+ assert "image_ordinal" in inspect.signature(method).parameters, method.__name__+ assert "image_ordinal = image_ordinal" in inspect.getsource(method), (+ f"{method.__name__} accepts the ordinal but does not put it on the " "worker command"+ )diff --git a/studio/backend/tests/test_safetensors_tool_loop.py b/studio/backend/tests/test_safetensors_tool_loop.pyindex 3f959e82a8a..afe5eba4076 100644--- a/studio/backend/tests/test_safetensors_tool_loop.py+++ b/studio/backend/tests/test_safetensors_tool_loop.py@@ -5152,6 +5152,343 @@ class TestStreamingDisplayStripStillMatchesTheExportedHelper:), f"diverged at offset {i}"+def test_mcp_images_reach_a_vision_model_through_the_sink(monkeypatch):+ import base64+ import io+ import json++ from PIL import Image++ from core.inference import mcp_images++ buffer = io.BytesIO()+ Image.new("RGB", (6, 6), (10, 120, 200)).save(buffer, format = "PNG")+ envelope = json.dumps(+ [{"data": base64.b64encode(buffer.getvalue()).decode(), "mimeType": "image/png"}]+ )+ result = "[1 image returned]\n" + mcp_images.SENTINEL + envelope++ turns = [+ '<tool_call>{"name": "mcp__fs__read_media_file", "arguments": {}}</tool_call>',+ "A tabby cat.",+ ]+ seen: list[list] = []++ def single_turn(conversation, *, active_tools = None):+ seen.append([dict(message) for message in conversation])+ yield turns[len(seen) - 1]++ sink: list = []+ list(+ run_safetensors_tool_loop(+ single_turn = single_turn,+ messages = [{"role": "user", "content": "describe the image"}],+ tools = [{"type": "function", "function": {"name": "mcp__fs__read_media_file"}}],+ execute_tool = lambda name, args, **kwargs: result,+ max_tool_iterations = 2,+ images_sink = sink,+ )+ )++ assert len(sink) == 1+ assert base64.b64decode(sink[0])[:8] == b"\x89PNG\r\n\x1a\n"+ second_turn = seen[1]+ assert "__MCP_IMAGES__" not in json.dumps(second_turn)+ assert second_turn[-1]["role"] == "user"+ assert second_turn[-1]["content"][0] == {"type": "image"}+++def test_mcp_images_are_left_out_without_a_sink(monkeypatch):+ import json++ from core.inference import mcp_images++ envelope = json.dumps([{"data": "QUJD", "mimeType": "image/png"}])+ result = "[1 image returned]\n" + mcp_images.SENTINEL + envelope+ turns = [+ '<tool_call>{"name": "mcp__fs__read_media_file", "arguments": {}}</tool_call>',+ "A tabby cat.",+ ]+ seen: list[list] = []++ def single_turn(conversation, *, active_tools = None):+ seen.append([dict(message) for message in conversation])+ yield turns[len(seen) - 1]++ list(+ run_safetensors_tool_loop(+ single_turn = single_turn,+ messages = [{"role": "user", "content": "describe the image"}],+ tools = [{"type": "function", "function": {"name": "mcp__fs__read_media_file"}}],+ execute_tool = lambda name, args, **kwargs: result,+ max_tool_iterations = 2,+ )+ )++ assert seen[1][-1]["role"] == "tool"+ assert "__MCP_IMAGES__" not in json.dumps(seen[1])+++def test_mcp_image_markers_merge_into_a_deferred_noop_turn(monkeypatch):+ """A batch holding both an internal no-op and an image-returning call used to+ append two role=user turns in a row, which a strict VLM template rejects."""+ import base64+ import io+ import json++ from PIL import Image++ from core.inference import mcp_images++ buffer = io.BytesIO()+ Image.new("RGB", (6, 6), (10, 120, 200)).save(buffer, format = "PNG")+ envelope = json.dumps(+ [{"data": base64.b64encode(buffer.getvalue()).decode(), "mimeType": "image/png"}]+ )+ result = "[1 image returned]\n" + mcp_images.SENTINEL + envelope++ # One image-returning call plus one call to a tool that is not enabled: the+ # second is suppressed as an internal no-op and lands as a deferred nudge+ # after the batch's results, which is the turn the markers must join.+ real = '{"name": "mcp__fs__read_media_file", "arguments": {}}'+ bogus = '{"name": "mcp__fs__delete_everything", "arguments": {}}'+ turns = [+ f"<tool_call>{real}</tool_call><tool_call>{bogus}</tool_call>",+ "A tabby cat.",+ ]+ seen: list[list] = []++ def single_turn(conversation, *, active_tools = None):+ seen.append([dict(message) for message in conversation])+ yield turns[len(seen) - 1]++ sink: list = []+ list(+ run_safetensors_tool_loop(+ single_turn = single_turn,+ messages = [{"role": "user", "content": "describe the image"}],+ tools = [{"type": "function", "function": {"name": "mcp__fs__read_media_file"}}],+ execute_tool = lambda name, args, **kwargs: result,+ max_tool_iterations = 2,+ images_sink = sink,+ )+ )++ second_turn = seen[1]+ roles = [message["role"] for message in second_turn]+ # Guard: without a suppressed call there is no nudge and the test proves nothing.+ assert any(+ message["role"] == "user" and "not executed" in str(message.get("content"))+ for message in second_turn+ ), f"no deferred no-op nudge in {second_turn}"+ assert not any(+ roles[i] == "user" and roles[i + 1] == "user" for i in range(len(roles) - 1)+ ), f"consecutive user turns: {roles}"+ markers = [+ part+ for message in second_turn+ if isinstance(message.get("content"), list)+ for part in message["content"]+ if part.get("type") == "image"+ ]+ assert len(markers) == len(sink) == 1+++def test_the_loop_cap_never_evicts_the_caller_s_own_attachment():+ """The sink is seeded with what the caller attached, and the MCP payloads land+ around it. A cap that trims from the front deleted that picture and its marker,+ so a model asked to compare against it was handed screenshots instead."""+ import base64+ import io+ import json++ from PIL import Image++ from core.inference import mcp_images++ def _png(colour):+ buffer = io.BytesIO()+ Image.new("RGB", (6, 6), colour).save(buffer, format = "PNG")+ return base64.b64encode(buffer.getvalue()).decode()++ attachment = _png((255, 0, 0))+ envelope = json.dumps([{"data": _png((0, 0, 255)), "mimeType": "image/png"} for _ in range(4)])+ result = "[4 images returned]\n" + mcp_images.SENTINEL + envelope++ def _call(n):+ return '<tool_call>{"name": "mcp__fs__shot", "arguments": {"n": %d}}</tool_call>' % n++ # DISTINCT calls: an identical repeat is suppressed as a no-op. Eight of them,+ # because a local placeholder turn carries one picture, and the cap has to be+ # reached for the test to prove anything.+ turns = [_call(n) for n in range(1, 9)] + ["done."]+ seen: list[list] = []++ def single_turn(conversation, *, active_tools = None):+ seen.append([dict(message) for message in conversation])+ yield turns[len(seen) - 1]++ sink = [attachment]+ list(+ run_safetensors_tool_loop(+ single_turn = single_turn,+ messages = [+ {+ "role": "user",+ "content": [{"type": "image"}, {"type": "text", "text": "compare with this"}],+ }+ ],+ tools = [{"type": "function", "function": {"name": "mcp__fs__shot"}}],+ execute_tool = lambda name, args, **kwargs: result,+ max_tool_iterations = 9,+ images_sink = sink,+ caller_image_indexes = (0,),+ )+ )++ assert attachment in sink, "the caller's attachment was trimmed away"+ assert (+ len(sink) == mcp_images.MAX_TOTAL_MODEL_IMAGES+ ), "the attachment is spared, but it still counts against the cap"+ final = seen[-1]+ assert final[0]["content"][0] == {"type": "image"}, "its marker went with it"+ markers = sum(+ 1+ for message in final+ if isinstance(message.get("content"), list)+ for part in message["content"]+ if part.get("type") == "image"+ )+ assert markers == len(sink), "one marker per pixel, or the processor miscounts"+++def test_a_resumed_chat_s_replayed_images_still_count_against_the_cap():+ """Only the caller's own attachment is spared. Sparing the whole seeded sink+ exempted a resumed chat's replayed pictures too, so the prompt could carry a+ second full allowance on top of them."""+ import base64+ import io+ import json++ from PIL import Image++ from core.inference import mcp_images++ def _png(colour):+ buffer = io.BytesIO()+ Image.new("RGB", (6, 6), colour).save(buffer, format = "PNG")+ return base64.b64encode(buffer.getvalue()).decode()++ replayed = [_png((0, 200 - index * 10, 0)) for index in range(4)]+ envelope = json.dumps([{"data": _png((0, 0, 255)), "mimeType": "image/png"} for _ in range(4)])+ result = "[4 images returned]\n" + mcp_images.SENTINEL + envelope++ def _call(n):+ return '<tool_call>{"name": "mcp__fs__shot", "arguments": {"n": %d}}</tool_call>' % n++ # One picture per batch on a local path, so five batches on four replayed is+ # nine against a cap of eight.+ turns = [_call(n) for n in range(1, 6)] + ["done."]+ seen: list[list] = []++ def single_turn(conversation, *, active_tools = None):+ seen.append([dict(message) for message in conversation])+ yield turns[len(seen) - 1]++ sink = list(replayed)+ list(+ run_safetensors_tool_loop(+ single_turn = single_turn,+ messages = [mcp_images.placeholder_turn(4, 4)],+ tools = [{"type": "function", "function": {"name": "mcp__fs__shot"}}],+ execute_tool = lambda name, args, **kwargs: result,+ max_tool_iterations = 6,+ images_sink = sink,+ # No attachment on this request; every seeded pixel is replay.+ caller_image_indexes = (),+ )+ )++ assert (+ len(sink) == mcp_images.MAX_TOTAL_MODEL_IMAGES+ ), f"replay was exempted from the cap: {len(sink)} images in the prompt"+ assert replayed[0] not in sink, "the oldest replayed picture should have gone first"+ final = seen[-1]+ markers = sum(+ 1+ for message in final+ if isinstance(message.get("content"), list)+ for part in message["content"]+ if part.get("type") == "image"+ )+ assert markers == len(sink)+++def test_the_protected_attachment_index_is_rebased_between_batches():+ """The index names a position, and trimming earlier entries moves it. Reusing the+ original across iterations protected the wrong payload and deleted the very+ attachment the argument exists to keep."""+ import base64+ import io+ import json++ from PIL import Image++ from core.inference import mcp_images++ def _png(colour):+ buffer = io.BytesIO()+ Image.new("RGB", (6, 6), colour).save(buffer, format = "PNG")+ return base64.b64encode(buffer.getvalue()).decode()++ replayed = [_png((0, 200 - index * 12, 0)) for index in range(7)]+ attachment = _png((255, 0, 0))+ envelope = json.dumps([{"data": _png((0, 0, 255)), "mimeType": "image/png"} for _ in range(4)])+ result = "[4 images returned]\n" + mcp_images.SENTINEL + envelope++ def _call(n):+ return '<tool_call>{"name": "mcp__fs__shot", "arguments": {"n": %d}}</tool_call>' % n++ turns = [_call(1), _call(2), _call(3), "done."]+ seen: list[list] = []++ def single_turn(conversation, *, active_tools = None):+ seen.append([dict(message) for message in conversation])+ yield turns[len(seen) - 1]++ # Replay first, the caller's attachment last: index 7 on the way in.+ sink = [*replayed, attachment]+ list(+ run_safetensors_tool_loop(+ single_turn = single_turn,+ messages = [+ mcp_images.placeholder_turn(7, 7),+ {+ "role": "user",+ "content": [{"type": "image"}, {"type": "text", "text": "compare with this"}],+ },+ ],+ tools = [{"type": "function", "function": {"name": "mcp__fs__shot"}}],+ execute_tool = lambda name, args, **kwargs: result,+ max_tool_iterations = 4,+ images_sink = sink,+ caller_image_indexes = (7,),+ )+ )++ assert attachment in sink, "the attachment was deleted by a stale protected index"+ assert len(sink) == mcp_images.MAX_TOTAL_MODEL_IMAGES+ final = seen[-1]+ markers = sum(+ 1+ for message in final+ if isinstance(message.get("content"), list)+ for part in message["content"]+ if part.get("type") == "image"+ )+ assert markers == len(sink), "one marker per pixel"++class TestBlockedGemmaChainHold:"""A promotable Gemma call behind a blocked one must not stream before it is promoted."""diff --git a/studio/backend/tests/test_sf_client_tools_passthrough.py b/studio/backend/tests/test_sf_client_tools_passthrough.pyindex eafbf7b6855..d76de9f13d7 100644--- a/studio/backend/tests/test_sf_client_tools_passthrough.py+++ b/studio/backend/tests/test_sf_client_tools_passthrough.py@@ -166,6 +166,86 @@ def _json_body(response):return json.loads(response.body if hasattr(response, "body") else response.content)[email protected]("supports_tools", [False, True])+def test_mcp_replay_preserves_multiple_caller_attachments(monkeypatch, supports_tools):+ import base64+ import io++ from PIL import Image++ def encoded(color):+ buffer = io.BytesIO()+ Image.new("RGB", (8, 8), color).save(buffer, format = "PNG")+ return base64.b64encode(buffer.getvalue()).decode("ascii")++ def user(color, text):+ return ChatMessage(+ role = "user",+ content = [+ {+ "type": "image_url",+ "image_url": {"url": "data:image/png;base64," + encoded(color)},+ },+ {"type": "text", "text": text},+ ],+ )++ backend = _ScriptedBackend(_fixed("done"))+ backend.models["sf-model"].update(+ is_vision = True,+ chat_template_info = {+ "template": _TOKENIZER_TEMPLATE_WITH_TOOLS,+ "processor_template": _TOKENIZER_TEMPLATE_WITH_TOOLS,+ "renders_image": True,+ "accepts_multiple_images": True,+ },+ )+ payload = _request(+ messages = [+ user("red", "Remember this image."),+ ChatMessage(+ role = "assistant",+ content = "",+ tool_calls = [+ {+ "id": "call_1",+ "type": "function",+ "function": {"name": "mcp__test__image", "arguments": "{}"},+ }+ ],+ ),+ ChatMessage(+ role = "tool",+ tool_call_id = "call_1",+ name = "mcp__test__image",+ content = "[1 image returned]\n__MCP_IMAGES__:"+ + json.dumps([{"mimeType": "image/png", "data": encoded("green")}]),+ ),+ ChatMessage(role = "assistant", content = "I have the tool image."),+ user("blue", "Compare all three images."),+ ],+ enable_tools = False,+ stream = False,+ )++ _call(payload, monkeypatch, backend, supports_tools = supports_tools)++ [call] = backend.calls+ colors = []+ for image in call["images"]:+ if isinstance(image, str):+ image = Image.open(io.BytesIO(base64.b64decode(image)))+ colors.append(image.getpixel((0, 0)))+ assert colors == [(255, 0, 0), (0, 128, 0), (0, 0, 255)]+ image_turns = [+ message+ for message in call["messages"]+ if isinstance(message.get("content"), list)+ and any(part.get("type") == "image" for part in message["content"])+ ]+ assert len(image_turns) == 3++def _collect_sse(response):async def _run():return [c async for c in response.body_iterator]@@ -1224,6 +1304,11 @@ _PNG_1x1 = ("+M9QDwADhgGAWjR9awAAAABJRU5ErkJggg==")+_PNG_B64 = (+ "iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAIAAACQd1PeAAAADUlEQVR42mNk"+ "+M9QDwADhgGAWjR9awAAAABJRU5ErkJggg=="+)+def _vision_backend(*snapshots):backend = _ScriptedBackend(_fixed(*snapshots))@@ -1267,6 +1352,344 @@ def test_legacy_image_field_keeps_the_client_tool_catalog(monkeypatch):assert backend.calls[0]["image"] is not None+class _VisionToolLoopBackend(_ToolLoopBackend):+ def __init__(self, responder, **kwargs):+ super().__init__(responder, **kwargs)+ self.models["sf-model"]["is_vision"] = True++ @staticmethod+ def resize_image(image):+ return image+++def test_an_attached_image_still_reaches_the_tool_loop(monkeypatch):+ backend = _VisionToolLoopBackend(_fixed("done"))+ payload = _request(+ messages = [+ ChatMessage(+ role = "user",+ content = [+ {"type": "text", "text": "what is in this picture"},+ {+ "type": "image_url",+ "image_url": {"url": f"data:image/png;base64,{_PNG_B64}"},+ },+ ],+ )+ ],+ enable_tools = True,+ stream = False,+ )++ _call(payload, monkeypatch, backend)++ [call] = backend.calls+ assert call["tools"], "tools must survive an attached image on a vision model"+ assert call["images"], "the attachment has to ride into the loop"+ markers = [+ part+ for message in call["messages"]+ if isinstance(message.get("content"), list)+ for part in message["content"]+ if part.get("type") == "image"+ ]+ assert len(markers) == len(call["images"])+++_TOKENIZER_TEMPLATE_WITH_TOOLS = (+ "{% if tools %}<|im_start|>system\n"+ "{% for t in tools %}{{ t.function.name }}\n{% endfor %}<|im_end|>\n{% endif %}"+ "{% for m in messages %}<|im_start|>{{ m['role'] }}\n{{ m['content'] }}<|im_end|>\n{% endfor %}"+)+_PROCESSOR_TEMPLATE_WITHOUT_TOOLS = (+ "{% for m in messages %}<|im_start|>{{ m['role'] }}\n{{ m['content'] }}<|im_end|>\n{% endfor %}"+)+++def test_the_image_tool_loop_is_gated_on_the_body_that_renders(monkeypatch):+ """The loop now runs on a vision model with an attachment, and generation renders+ through the PROCESSOR template. Classifying its tool support from the tokenizer body+ started the loop on a model whose prompt carries no schemas at all."""+ import routes.inference as inf++ backend = _VisionToolLoopBackend(_fixed("done"))+ backend.models["sf-model"]["chat_template_info"] = {+ "template": _TOKENIZER_TEMPLATE_WITH_TOOLS,+ "processor_template": _PROCESSOR_TEMPLATE_WITHOUT_TOOLS,+ "renders_image": True,+ }+ payload = _request(+ messages = [+ ChatMessage(+ role = "user",+ content = [+ {"type": "text", "text": "what is in this picture"},+ {+ "type": "image_url",+ "image_url": {"url": f"data:image/png;base64,{_PNG_B64}"},+ },+ ],+ )+ ],+ enable_tools = True,+ stream = False,+ )++ _install(monkeypatch, backend)+ monkeypatch.setattr(+ inf,+ "_detect_safetensors_features",+ lambda _backend, template, **k: {+ "supports_tools": template == _TOKENIZER_TEMPLATE_WITH_TOOLS+ },+ )++ async def _run():+ return await openai_chat_completions(payload, request = _Request(), current_subject = "u")++ asyncio.run(_run())++ assert backend.calls, "generation never ran"+ assert not any(+ call.get("tools") for call in backend.calls+ ), "the tool loop was driven from a template the image render never selects"+++def test_a_client_catalog_keeps_an_image_out_of_the_server_loop(monkeypatch):+ """Letting the loop take a picture must not take the request with it: #10092 routes+ image-plus-tools to the passthrough so the CALLER's schemas are the ones rendered.+ Claiming it here answered the client with Unsloth's built-ins instead."""+ backend = _VisionToolLoopBackend(_fixed("a plain answer"))+ payload = _request(+ messages = [+ ChatMessage(+ role = "user",+ content = [+ {"type": "text", "text": "what is in this picture"},+ {+ "type": "image_url",+ "image_url": {"url": f"data:image/png;base64,{_PNG_B64}"},+ },+ ],+ )+ ],+ tools = [LOOKUP_TOOL],+ enable_tools = True,+ stream = False,+ )++ _call(payload, monkeypatch, backend)++ assert backend.calls, "generation never ran"+ assert backend.calls[0]["tools"] == [LOOKUP_TOOL]+ # The passthrough renders one turn; the loop would have run the built-ins instead.+ assert "images" not in backend.calls[0] or not backend.calls[0].get("images")+++def test_a_replayed_picture_sits_beside_the_result_that_produced_it(monkeypatch):+ """The passthrough flatten costs the markers their positions. Promoting after it+ puts each batch's turn straight after its result, so "the tool call above" names+ the right call -- one detached block of every payload, placed wherever the+ attachment's turn happened to be, did not."""+ import base64+ import io+ import json++ from PIL import Image++ from core.inference import mcp_images++ buffer = io.BytesIO()+ Image.new("RGB", (6, 6), (10, 120, 200)).save(buffer, format = "PNG")+ envelope = json.dumps(+ [{"data": base64.b64encode(buffer.getvalue()).decode(), "mimeType": "image/png"}]+ )++ backend = _ScriptedBackend(_fixed("a plain answer"))+ backend.models["sf-model"]["is_vision"] = True+ backend.models["sf-model"]["chat_template_info"] = {+ "template": "{% for m in messages %}{{ m['content'] }}{% endfor %}",+ "renders_image": True,+ }+ payload = _request(+ messages = [+ ChatMessage(role = "user", content = "take a shot"),+ ChatMessage(+ role = "assistant",+ content = "",+ tool_calls = [+ {+ "id": "call_0",+ "type": "function",+ "function": {"name": "mcp__fs__shot", "arguments": "{}"},+ }+ ],+ ),+ ChatMessage(+ role = "tool",+ tool_call_id = "call_0",+ content = "[1 image returned]\n" + mcp_images.SENTINEL + envelope,+ ),+ ChatMessage(role = "assistant", content = "a blue square"),+ ChatMessage(role = "user", content = "and what about now"),+ ],+ tools = [LOOKUP_TOOL],+ stream = False,+ )++ _call(payload, monkeypatch, backend)++ [call] = backend.calls+ sent = call["messages"]+ assert call["images"] and len(call["images"]) == 1+ [marker_at] = [+ index+ for index, message in enumerate(sent)+ if isinstance(message.get("content"), list)+ and any(part.get("type") == "image" for part in message["content"])+ ]+ assert sent[marker_at - 1]["role"] == "tool", [m["role"] for m in sent]+ lead = next(part["text"] for part in sent[marker_at]["content"] if part.get("type") == "text")+ assert lead.startswith(mcp_images.IMAGE_TURN_TEXT), lead+ roles = [m["role"] for m in sent]+ assert all(a != "user" or b != "user" for a, b in zip(roles, roles[1:])), roles+++def test_a_replay_only_image_turn_also_keeps_the_client_catalog(monkeypatch):+ """The gate read `image is None`, so a resumed chat whose only pictures are+ replayed MCP ones looked image-free and the loop took the request -- swapping the+ caller's schemas for Unsloth's on exactly the path this change adds."""+ import base64+ import io+ import json++ from PIL import Image++ from core.inference import mcp_images++ buffer = io.BytesIO()+ Image.new("RGB", (6, 6), (10, 120, 200)).save(buffer, format = "PNG")+ envelope = json.dumps(+ [{"data": base64.b64encode(buffer.getvalue()).decode(), "mimeType": "image/png"}]+ )++ backend = _VisionToolLoopBackend(_fixed("a plain answer"))+ payload = _request(+ messages = [+ ChatMessage(role = "user", content = "take a shot"),+ ChatMessage(+ role = "assistant",+ content = "",+ tool_calls = [+ {+ "id": "call_0",+ "type": "function",+ "function": {"name": "mcp__fs__shot", "arguments": "{}"},+ }+ ],+ ),+ ChatMessage(+ role = "tool",+ tool_call_id = "call_0",+ content = "[1 image returned]\n" + mcp_images.SENTINEL + envelope,+ ),+ ChatMessage(role = "user", content = "what colour was it"),+ ],+ tools = [LOOKUP_TOOL],+ enable_tools = True,+ stream = False,+ )++ _call(payload, monkeypatch, backend)++ assert backend.calls, "generation never ran"+ assert backend.calls[0]["tools"] == [LOOKUP_TOOL]+++def test_the_plain_route_leaves_the_attachment_marker_to_the_backend(monkeypatch):+ """The backends snapshot the conversation's existing markers as history's before+ topping up, so a marker the ROUTE pre-added is counted as a replayed picture's.+ With the attachment on an earlier turn than a tool's picture the two pixels then+ bind to each other's turns, and the model reads the screenshot as the diagram."""+ import base64+ import io+ import json++ from PIL import Image++ from core.inference import mcp_images++ buffer = io.BytesIO()+ Image.new("RGB", (8, 8), (10, 120, 200)).save(buffer, format = "PNG")+ envelope = json.dumps(+ [{"data": base64.b64encode(buffer.getvalue()).decode(), "mimeType": "image/png"}]+ )++ backend = _ScriptedBackend(_fixed("a plain answer"))+ backend.models["sf-model"]["is_vision"] = True+ backend.models["sf-model"]["chat_template_info"] = {+ "template": "{% for m in messages %}{{ m['content'] }}{% endfor %}",+ "renders_image": True,+ }+ payload = _request(+ messages = [+ # The attachment rides on an EARLIER turn than the tool's picture.+ ChatMessage(+ role = "user",+ content = [+ {"type": "text", "text": "here is my diagram"},+ {+ "type": "image_url",+ "image_url": {"url": f"data:image/png;base64,{_PNG_B64}"},+ },+ ],+ ),+ ChatMessage(role = "assistant", content = "noted"),+ ChatMessage(+ role = "assistant",+ content = "",+ tool_calls = [+ {+ "id": "c1",+ "type": "function",+ "function": {"name": "mcp__s__shot", "arguments": "{}"},+ }+ ],+ ),+ ChatMessage(+ role = "tool",+ tool_call_id = "c1",+ content = "[1 image returned]\n" + mcp_images.SENTINEL + envelope,+ ),+ ChatMessage(role = "user", content = "which one is bluer?"),+ ],+ stream = False,+ )++ # supports_tools=False is what makes this the PLAIN path: the replayed picture+ # arrives with tool history, and a template that renders tools would send the+ # request to the client-tool passthrough, which orders the pixels itself.+ _call(payload, monkeypatch, backend, supports_tools = False)++ [call] = backend.calls+ sent, replayed = call["messages"], call["images"] or []+ assert call["image"] is not None, "the attachment reached the backend"+ assert len(replayed) == 1, "so did the replayed picture"++ # Exactly what the backends do next. Given a conversation the route did not+ # pre-mark, the top-up places the attachment's marker at its own ordinal and+ # the history/attachment split survives.+ prior = mcp_images.image_marker_parts(sent)+ topped = mcp_images.top_up_image_markers(sent, len(replayed) + 1, ordinal = call["image_ordinal"])+ ordered = mcp_images.pixels_in_marker_order(topped, prior, ["MCP"], "ATTACHMENT")++ assert len(mcp_images.image_marker_parts(topped)) == 2, topped+ assert ordered == [+ "ATTACHMENT",+ "MCP",+ ], "the route pre-marked, so the pixels bound to each other's markers"++def test_video_turn_with_tools_enabled_keeps_the_client_tool_catalog(monkeypatch):"""A clip rules out the server loop like an image; the passthrough keeps catalog and clip."""backend = _vision_backend(_CALL_XML)diff --git a/studio/backend/tests/test_studio_tool_loop.py b/studio/backend/tests/test_studio_tool_loop.pyindex 3926934fd88..4fa391ea649 100644--- a/studio/backend/tests/test_studio_tool_loop.py+++ b/studio/backend/tests/test_studio_tool_loop.py@@ -136,6 +136,7 @@ def _run(tools = None,tool_choice = None,messages = None,+ supports_vision = False,**policy_kwargs,):policy_fields = {@@ -159,6 +160,7 @@ def _run(session_id = "s1",thread_id = "t1",tool_choice = tool_choice,+ supports_vision = supports_vision,),policy = ToolLoopPolicy(**policy_fields),cancel_event = cancel_event,@@ -1258,3 +1260,80 @@ def test_a_fragment_naming_its_call_goes_back_to_that_call(executed):_run(transport)assert [call["arguments"] for call in executed] == [{"query": "first"}, {"query": "second"}]+++# ── MCP images ────────────────────────────────────────────────────+++def _mcp_image_transport() -> FakeTransport:+ return FakeTransport(+ [+ [+ _sse(+ {+ "tool_calls": [+ {+ "index": 0,+ "id": "call_img",+ "function": {+ "name": "mcp__fs__read_media_file",+ "arguments": '{"path":"cat.png"}',+ },+ }+ ]+ }+ ),+ _sse(finish = "tool_calls"),+ _DONE,+ ],+ [_sse({"content": "A tabby cat."}), _sse(finish = "stop"), _DONE],+ ]+ )+++def mcp_image_result(monkeypatch):+ import base64+ import io++ from PIL import Image++ from core.inference import mcp_images++ buffer = io.BytesIO()+ Image.new("RGB", (6, 6), (10, 120, 200)).save(buffer, format = "PNG")+ envelope = json.dumps(+ [{"data": base64.b64encode(buffer.getvalue()).decode(), "mimeType": "image/png"}]+ )+ monkeypatch.setattr(+ loop_mod,+ "execute_tool",+ lambda name, arguments, **kwargs: "[1 image returned]\n" + mcp_images.SENTINEL + envelope,+ )+ monkeypatch.setattr(loop_mod, "build_rag_autoinject", lambda *a, **k: None)+ monkeypatch.setattr(loop_mod, "is_high_risk_tool_call", lambda name, args: False)+++def test_mcp_images_reach_a_vision_provider_as_their_own_user_turn(mcp_image_result):+ transport = _mcp_image_transport()++ _run(+ transport,+ tools = [_tool("mcp__fs__read_media_file")],+ supports_vision = True,+ )++ follow_up = transport.requests[1]["messages"]+ assert [message["role"] for message in follow_up[-3:]] == ["assistant", "tool", "user"]+ assert "__MCP_IMAGES__" not in follow_up[-2]["content"]+ assert follow_up[-1]["content"][1]["image_url"]["url"].startswith("data:image/png;base64,")+++def test_mcp_images_are_not_sent_to_a_text_only_provider(mcp_image_result):+ transport = _mcp_image_transport()++ _run(transport, tools = [_tool("mcp__fs__read_media_file")])++ follow_up = transport.requests[1]["messages"]+ assert follow_up[-1]["role"] == "tool"+ assert "__MCP_IMAGES__" not in follow_up[-1]["content"]diff --git a/studio/backend/tests/test_video_attachment_part.py b/studio/backend/tests/test_video_attachment_part.pyindex 4ec0ecc6ad4..6c10c67f143 100644--- a/studio/backend/tests/test_video_attachment_part.py+++ b/studio/backend/tests/test_video_attachment_part.py@@ -197,7 +197,9 @@ def test_a_non_gguf_model_takes_the_clip_through_one_gate_and_hands_it_to_generaassert gate < speech and gate < audio_inputassert 'gen_kwargs["video"] = _video_clip' in sourceuse_tools = source.index("_sf_use_tools = (", handler)- assert "and _video_clip is None" in source[use_tools : use_tools + 400]+ tools_block = source[use_tools : source.index("\n )", use_tools)]+ tools_block = "\n".join(line.split("#")[0] for line in tools_block.splitlines())+ assert "and _video_clip is None" in tools_block# Structural, not literal. This pinned the exact# "(image is not None or _video_clip is not None) and not _sf_use_tools"; #10970# widened the image half to `_sf_has_image`, a superset, so the clause still fires@@ -215,10 +217,10 @@ def test_a_non_gguf_model_takes_the_clip_through_one_gate_and_hands_it_to_genera# is not enough: `and` for `or` stops an image-only or video-only request entering the# passthrough, and `image is None` / `not _sf_has_image` invert the condition. Either# order, since which side reads first is arbitrary.- image = r"(?:\bimage is not None\b|\b_sf_has_image\b)"+ image = r"(?:\bimage is not None\b|\b_sf_has_(?:any_)?image\b)"clip = r"\b_video_clip is not None\b"assert re.search(rf"{image}\s+or\s+{clip}|{clip}\s+or\s+{image}", escape), escape- assert not re.search(r"\bimage is None\b|\bnot\s+_sf_has_image\b", escape), escape+ assert not re.search(r"\bimage is None\b|\bnot\s+_sf_has_(?:any_)?image\b", escape), escape# Settled at the gate: a model without audio input never enters the audio-input path.conflict = source.index("if payload.audio_base64:", gate)assert conflict < speechdiff --git a/studio/backend/tests/test_vision_client_tools.py b/studio/backend/tests/test_vision_client_tools.pyindex 56a4dea194b..20d905c8543 100644--- a/studio/backend/tests/test_vision_client_tools.py+++ b/studio/backend/tests/test_vision_client_tools.py@@ -171,7 +171,11 @@ def _vision_probe(chat_template = _CHATML_WITH_TOOLS):return "PROMPT-WITH-TOOLS"return "PROMPT"- def __call__(self, *_args, **_kwargs):+ def __call__(self, *args, **_kwargs):+ # The pixels arrive as the first positional argument, one object when+ # there is a single image and the list itself when there are several.+ if args:+ seen["images"] = args[0]return Batch({"input_ids": torch.zeros((1, 1), dtype = torch.long)})class Model:@@ -1531,6 +1535,103 @@ def test_the_nudge_retry_skips_the_image_marker_on_a_text_only_fallback():assert not any(p.get("type") == "image" for p in body), message+def test_an_earlier_attachment_keeps_its_own_turn_against_a_newer_replay():+ """The attachment's turn can PRECEDE a tool's picture. The plain route used to+ pre-add its marker, so this backend counted that marker as history's, the top-up+ became a no-op, and pixels_in_marker_order could no longer tell the two apart --+ the model was shown the screenshot where the user's own diagram belonged."""+ from core.inference.mcp_images import placeholder_turn++ backend, seen = _vision_probe()+ attachment, replayed = object(), object()++ # What the plain route hands the backend once it stops pre-marking: the+ # attachment's turn is a plain string and the only marker is the replay's.+ messages = [+ {"role": "user", "content": "here is my diagram"},+ {"role": "assistant", "content": "noted"},+ {+ "role": "assistant",+ "content": "",+ "tool_calls": [+ {+ "id": "c1",+ "type": "function",+ "function": {"name": "mcp__s__shot", "arguments": "{}"},+ }+ ],+ },+ {"role": "tool", "tool_call_id": "c1", "content": "[1 image returned]"},+ placeholder_turn(1, 1),+ {"role": "user", "content": "which one is bluer?"},+ ]++ _drain(+ backend,+ messages = messages,+ image = attachment,+ images = [replayed],+ image_ordinal = 0,+ )++ rendered = seen["messages"]+ marker_turns = [+ index+ for index, message in enumerate(rendered)+ if isinstance(message.get("content"), list)+ and any(part.get("type") == "image" for part in message["content"])+ ]+ assert len(marker_turns) == 2, rendered+ assert marker_turns[0] < marker_turns[1]+ assert seen["images"] == [attachment, replayed], "the pixels bound to each other's markers"+++ "initial, requested", [(True, False), (False, True), ("first", "second"), ("first", None)]+)+def test_replayed_vision_images_honor_adapter_selection_under_lock(initial, requested):+ backend, seen = _vision_probe()+ backend.models[backend.active_model_name]["is_vision"] = True+ model = backend.models[backend.active_model_name]["model"]+ state = {"active": initial, "applied": []}++ def apply(value):+ assert backend._generation_lock.locked()+ # The real helper's contract: None leaves the loaded state alone.+ if value is None:+ return+ state["applied"].append(value)+ state["active"] = value++ def generate(**kwargs):+ assert backend._generation_lock.locked()+ state["generated_with"] = state["active"]++ backend._apply_adapter_state = apply+ model.generate = generate+ replay = object()+ list(+ backend.generate_with_adapter_control(+ use_adapter = requested,+ messages = [+ {+ "role": "user",+ "content": [+ {"type": "image"},+ {"type": "text", "text": "Describe the tool image"},+ ],+ }+ ],+ image = None,+ images = [replay],+ max_new_tokens = 1,+ )+ )+ assert seen["images"] is replay+ assert state["generated_with"] == (initial if requested is None else requested)+ assert state["applied"] == ([] if requested is None else [requested])++def test_each_compare_pane_applies_its_own_adapter_state_on_an_image_turn(monkeypatch):backend, _ = _vision_probe()info = backend.models["vision-tools"]diff --git a/studio/frontend/src/features/chat/api/chat-adapter.ts b/studio/frontend/src/features/chat/api/chat-adapter.tsindex caec120789f..0094fa5aa78 100644--- a/studio/frontend/src/features/chat/api/chat-adapter.ts+++ b/studio/frontend/src/features/chat/api/chat-adapter.ts@@ -72,6 +72,13 @@ import {sandboxSessionIdFor,} from "@/components/assistant-ui/sandbox-files";import { apiUrl } from "@/lib/api-base";+import { isMcpToolName } from "../utils/mcp-tool-name";+import {+ type McpImage,+ planMcpImageBound,+ mcpImagesEnvelope,+ splitMcpImages,+} from "./mcp-images";import {answerTextFromParts,extractSearchImages,@@ -125,12 +132,14 @@ import {parseExternalModelId,providerModelSupportsStudioTools,providerModelSupportsVision,+ providerModelTakesMcpImages,supportsProviderPromptCacheTtl,supportsProviderPromptCaching,toExternalBackendProviderType,} from "../external-providers";import {+ localToolExchangeIndexes,addCodexReasoning,codexLocalToolRoundId,codexReasoningForToolCalls,@@ -1020,6 +1029,13 @@ function isSandboxWrapper(return isSandboxToolResult(result);}+/** Exactly the live parser's wrapper: text and images, no other field. */+export function isBareMcpImageWrapper(val: unknown): boolean {+ if (!isMcpImageToolResult(val)) return false;+ const keys = Object.keys(val as object).filter((key) => key !== "text" && key !== "images");+ return keys.length === 0;+}+export function isMcpImageToolResult(val: unknown): val is McpImageToolResult {if (typeof val !== "object" || val === null) {return false;@@ -1057,7 +1073,14 @@ function serializeToolResultPart(// Backend ChatMessage rejects role="tool" with empty content; a sentinel JSON round-trips it.content = result.length > 0 ? result : JSON.stringify({ result: "" });} else if (- isMcpImageToolResult(result) ||+ // The wrapper the live parser builds -- {text, images} and nothing else -- from an+ // MCP result, or from any tool whose raw output ends in a valid envelope. Those+ // are unwrapped by shape, since JSON.stringify below would replay the whole base64+ // array as ordinary prompt text; provenance gates the ENVELOPE, a few lines down.+ // A client tool's own structured result that merely carries text and images among+ // OTHER fields is not that wrapper, and keeps its normal JSON serialization.+ (isMcpImageToolResult(result) &&+ (isMcpToolName(tc.toolName) || isBareMcpImageWrapper(result))) ||isSearchImagesToolResult(result) ||isSandboxWrapper(result, tc.toolName ?? "")) {@@ -1067,6 +1090,12 @@ function serializeToolResultPart(? stripSearchImageTokens(result.text): result.text;content = replayText.length > 0 ? replayText : JSON.stringify({ result: "" });+ // Gated on the mcp__ id the backend stamps, not on shape alone: a client tool+ // is free to answer {text, images:[{data, mimeType}]}, and appending the+ // envelope would hand its bytes to the model as image input.+ if (isMcpImageToolResult(result) && isMcpToolName(tc.toolName)) {+ content += mcpImagesEnvelope(result.images);+ }} else {try {content = JSON.stringify(result);@@ -1687,24 +1716,127 @@ export const CANVAS_TOOL_INSTRUCTION =export const CANVAS_FALLBACK_INSTRUCTION ="When the user asks for an HTML, CSS, or JavaScript canvas, return one complete self-contained fenced html code block. Embed CSS and JavaScript inside the document. Do not emit tool-call syntax.";+/** The MCP image bound applied to the run's own tool results, BEFORE they are+ * serialized. Bounding the OpenAI history afterwards meant every result's image+ * array was stringified into an envelope on the main thread and then parsed again+ * to be cut or dropped -- on every send, for pictures the request would not carry.+ * A result that keeps no picture becomes its plain text, which is what the+ * serializer emits for a stripped envelope. */+function boundMcpImageResults(+ messages: RunMessages,+ { readsImages, localMarkers }: { readsImages: boolean; localMarkers: boolean },+): RunMessages {+ type Carrier = { message: number; part: number; images: McpImage[] };+ // One entry per replay EXCHANGE, not per message: a local run accumulates every+ // round's tool calls in one assistant message and the serializer splits them by+ // round id into the exchanges the backend sees, and a batch is one exchange's+ // results. Partitioned by the serializer's own rule so the two cannot drift.+ const perExchange: Carrier[][] = [];+ messages.forEach((message, m) => {+ const parts = message.content;+ if (!Array.isArray(parts)) return;+ const toolParts: { index: number; part: ToolCallMessagePart }[] = [];+ parts.forEach((part, p) => {+ if (part.type === "tool-call") {+ toolParts.push({ index: p, part: part as ToolCallMessagePart });+ }+ });+ const exchanges = localToolExchangeIndexes(+ toolParts,+ ({ part }) => codexLocalToolRoundId(getToolReplayProvenance(part)),+ ({ part }) => shouldFlushCompletedLocalToolPair(part),+ );+ const byExchange = new Map<number, Carrier[]>();+ toolParts.forEach(({ index, part }, k) => {+ const tc = part as { toolName?: string; result?: unknown };+ if (!isMcpImageToolResult(tc.result) || !isMcpToolName(tc.toolName)) return;+ const carrier = { message: m, part: index, images: tc.result.images };+ const batch = byExchange.get(exchanges[k]) ?? [];+ batch.push(carrier);+ byExchange.set(exchanges[k], batch);+ });+ for (const key of [...byExchange.keys()].sort((a, b) => a - b)) {+ perExchange.push(byExchange.get(key)!);+ }+ });+ if (perExchange.length === 0) return messages;+ const batches = localMarkers+ ? perExchange+ : perExchange.flatMap((batch) => batch.map((carrier) => [carrier]));+ const plan = readsImages+ ? planMcpImageBound(+ batches.map((batch) => batch.map((carrier) => carrier.images)),+ { localMarkers },+ )+ : batches.map((batch) => batch.map((): McpImage[] => []));+ const edits = new Map<number, Map<number, McpImage[]>>();+ batches.forEach((batch, b) =>+ batch.forEach((carrier, r) => {+ const kept = plan[b][r];+ if (kept.length === carrier.images.length) return;+ const forMessage = edits.get(carrier.message) ?? new Map<number, McpImage[]>();+ forMessage.set(carrier.part, kept);+ edits.set(carrier.message, forMessage);+ }),+ );+ if (edits.size === 0) return messages;+ return messages.map((message, m) => {+ const forMessage = edits.get(m);+ if (!forMessage || !Array.isArray(message.content)) return message;+ const content = message.content.map((part, p) => {+ const kept = forMessage.get(p);+ if (!kept) return part;+ const result = (part as { result: McpImageToolResult }).result;+ return {+ ...part,+ result: kept.length > 0 ? { ...result, images: kept } : result.text,+ };+ });+ return { ...message, content } as typeof message;+ }) as RunMessages;+}++/** Whether the local target reads an MCP picture: the SELECTED model's vision flag,+ * in either direction, since a queued send can target a vision model while the+ * resident one is text-only and its loadedIsMultimodal is stale. The loaded state is+ * the fallback only when the selection's capability is unknown; unknown keeps them.+ * Not loadedIsMultimodal alone either way: an audio-only model also sets it. */+function localTargetReadsImages(+ state: Pick<ChatRuntimeState, "models" | "params" | "loadedIsMultimodal">,+): boolean {+ const activeModel = state.models.find(+ (model) => model.id === state.params.checkpoint,+ );+ if (typeof activeModel?.isVision === "boolean") return activeModel.isVision;+ return state.loadedIsMultimodal !== false;+}+/** The OpenAI-form history a completion would send. The tool catalog is priced server-side,* since --enable-tools can inject schemas the client cannot see. */export async function buildLocalTokenCountHistory(- messages: RunMessages,+ rawMessages: RunMessages,threadId: string | undefined,): Promise<{messages: OpenAIChatMessage[];studio_tool_history?: true;}> {+ const runtimeState = useChatRuntimeStore.getState();+ const { params, artifactsEnabled, supportsTools } = runtimeState;+ const activeModel = runtimeState.models.find(+ (model) => model.id === runtimeState.params.checkpoint,+ );+ // Apply send-path target limits before serializing this background recount;+ // backend limits run after parsing and cannot bound the uploaded body.+ const messages = boundMcpImageResults(rawMessages, {+ readsImages: localTargetReadsImages(runtimeState),+ localMarkers: activeModel?.isGguf === false,+ });const survivingMessages = pruneOutboundHistory(messages, true);const outboundMessages = survivingMessages.flatMap((message) => toOpenAIMessages(message, true)).filter((message): message is NonNullable<typeof message> =>Boolean(message),);-- const { params, artifactsEnabled, supportsTools } =- useChatRuntimeStore.getState();const safeSystemPrompt =typeof params.systemPrompt === "string"? resolveSystemPromptVariables(@@ -4005,7 +4137,7 @@ export function createOpenAIStreamAdapter(): ChatModelAdapter {const adapter = {async *run({- messages,+ messages: rawMessages,runConfig,abortSignal,unstable_threadId,@@ -4193,9 +4325,9 @@ export function createOpenAIStreamAdapter(.reverse().find((m) => m.role === "user");if (!userMessage) throw new Error("Research requires a user message.");- const userMessageIndex = messages.indexOf(userMessage);+ const userMessageIndex = rawMessages.indexOf(userMessage);const userMessageParentId =- userMessageIndex > 0 ? messages[userMessageIndex - 1]!.id : null;+ userMessageIndex > 0 ? rawMessages[userMessageIndex - 1]!.id : null;const { params } = runtime;await persistResolvedQueuedModel(params.checkpoint,@@ -4765,13 +4897,35 @@ export function createOpenAIStreamAdapter(throw new Error("Image generation edit unavailable.");}+ // Resolve before the outbound build's standalone tests/studio slice. Match+ // backend provider/model gates and local vision flags (audio-only models can+ // be multimodal), so targets that strip MCP images never upload the envelopes.+ const targetReadsImages = isExternalRequest+ ? providerModelTakesMcpImages(+ externalProvider?.providerType,+ externalSelection?.modelId,+ )+ : localTargetReadsImages(runtime);+ // A local target that is not a GGUF renders replayed pictures as markers, one per+ // tool batch, so the upload is bounded to what that path can use (the bound runs+ // below, after the slice). Unknown format keeps the part paths' four per result.+ const mcpImagesLocalMarkers =+ !isExternalRequest &&+ runtime.models.find((model) => model.id === runtime.params.checkpoint)+ ?.isGguf === false;+ // Bounded on the run's own results, before any envelope is built: what the+ // request will not carry is never stringified, and never parsed back.+ const messages = boundMcpImageResults(rawMessages, {+ readsImages: targetReadsImages,+ localMarkers: mcpImagesLocalMarkers,+ });const survivingMessages = pruneOutboundHistory(messages,!isExternalRequest,);// toOpenAIMessages emits assistant tool_calls plus role="tool" follow-ups; the backend Gemini// translator rebuilds the functionCall/functionResponse parts.- const outboundMessages = survivingMessages+ let outboundMessages = survivingMessages.flatMap((message) => toOpenAIMessages(message, !isExternalRequest)).filter((message): message is NonNullable<typeof message> =>Boolean(message),@@ -6868,8 +7022,6 @@ export function createOpenAIStreamAdapter()? rawResult.lastIndexOf(imgMarker): -1;- const mcpImgMarker = "\n__MCP_IMAGES__:";- const mcpImgIdx = rawResult.lastIndexOf(mcpImgMarker);let parsedResult:| string| {@@ -6891,22 +7043,9 @@ export function createOpenAIStreamAdapter(const imageB64 = toolEvent.image_b64 as string | undefined;// A valid MCP image envelope wins; an invalid marker falls through so a sandbox __IMAGES__// suffix still renders.- let mcpImages: McpImageToolResult | null = null;- if (mcpImgIdx !== -1) {- try {- const images = JSON.parse(- rawResult.slice(mcpImgIdx + mcpImgMarker.length),- );- const candidate = {- text: rawResult.slice(0, mcpImgIdx),- images,- };- if (isMcpImageToolResult(candidate))- mcpImages = candidate;- } catch {- // Not a valid envelope; fall through below.- }- }+ const mcpCandidate = splitMcpImages(rawResult);+ const mcpImages: McpImageToolResult | null =+ isMcpImageToolResult(mcpCandidate) ? mcpCandidate : null;if (toolCallParts[idx].toolName === "image_generation" &&typeof imageB64 === "string" &&diff --git a/studio/frontend/src/features/chat/api/mcp-images.ts b/studio/frontend/src/features/chat/api/mcp-images.tsnew file mode 100644index 00000000000..60fd450742c--- /dev/null+++ b/studio/frontend/src/features/chat/api/mcp-images.ts@@ -0,0 +1,220 @@+// SPDX-License-Identifier: AGPL-3.0-only+// Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0++// The envelope the backend appends to a tool result that returned images.+// Validated, so tool text that merely mentions the marker is never truncated.++export const MCP_IMAGES_MARKER = "\n__MCP_IMAGES__:";++export interface McpImage {+ data: string;+ mimeType: string;+ // Set on the first entry once this side has shortened the array, so the backend's+ // note can still say how many the tool returned rather than how many were uploaded.+ returned?: number;+}++export function isMcpImageArray(value: unknown): value is McpImage[] {+ return (+ Array.isArray(value) &&+ value.length > 0 &&+ value.every(+ (image) =>+ typeof image === "object" &&+ image !== null &&+ typeof (image as Record<string, unknown>).data === "string" &&+ typeof (image as Record<string, unknown>).mimeType === "string",+ )+ );+}++export function splitMcpImages(result: string): {+ text: string;+ images: McpImage[];+} {+ const idx = result.lastIndexOf(MCP_IMAGES_MARKER);+ if (idx === -1) return { text: result, images: [] };+ let images: unknown;+ try {+ images = JSON.parse(result.slice(idx + MCP_IMAGES_MARKER.length));+ } catch {+ return { text: result, images: [] };+ }+ if (!isMcpImageArray(images)) return { text: result, images: [] };+ return { text: result.slice(0, idx), images };+}++// Re-attached on replay: the backend promotes it into an image turn for a vision+// model, and strips it for every other one.+export function mcpImagesEnvelope(images: McpImage[]): string {+ return MCP_IMAGES_MARKER + JSON.stringify(images);+}++// The backend keeps the newest eight pictures of a conversation. It can only do+// that after the request is parsed, so without the same bound here every past+// screenshot is uploaded again on every turn and the body grows without limit.+export const MAX_TOTAL_MCP_IMAGES = 8;+// Mirrors the backend's per-result promotion limit.+export const MAX_MODEL_IMAGES = 4;+// Mirrors LOCAL_MAX_IMAGES_PER_TURN in studio/backend/core/inference/mcp_images.py: the+// marker paths (safetensors, MLX) render a tool batch as one turn carrying one picture.+export const LOCAL_MAX_IMAGES_PER_TURN = 1;+// Spare candidates carried past that limit, because the backend's quota counts+// images that DECODE and this side cannot tell which will. Bounded, so a result+// of unreadable blobs still cannot grow the request without limit.+export const DECODE_FAILURE_ALLOWANCE = 4;+// Bound total replay base64 to the backend's per-result payload ceiling.+// The count cap alone permits twelve 12 MB candidates (~144 MB per turn).+export const MAX_TOTAL_MCP_IMAGE_CHARS = 12_000_000;+// The data budget matches mcp_client.MAX_IMAGE_PAYLOAD_CHARS so accepted live+// images fit their replay. Bound metadata separately, matching mcp_images.py+// MAX_MCP_IMAGE_MIME_CHARS, to prevent oversized mimeType values bypassing it.+export const MAX_MCP_IMAGE_MIME_CHARS = 256;++const MCP_TOOL_PREFIX = "mcp__";++interface EnvelopeCarrier {+ role?: string;+ name?: string;+ content?: unknown;+}++/** The bound itself, over batches of results in document order (oldest first): what+ * each result may still carry. A batch is the results one model turn produced in+ * parallel; the marker paths render it as one turn carrying one picture, so with+ * localMarkers the batch is charged once and its further candidates are decode+ * fallbacks. The part paths take MAX_MODEL_IMAGES per result, each result its own batch.+ *+ * Both carriers of an envelope -- the serialized OpenAI history and the run's own tool+ * results -- plan through here, so the two never drift. */+export function planMcpImageBound(+ batches: readonly (readonly (readonly McpImage[])[])[],+ { localMarkers = false }: { localMarkers?: boolean } = {},+): McpImage[][][] {+ let budget = MAX_TOTAL_MCP_IMAGES;+ // Shared, so at most MAX_TOTAL_MCP_IMAGES + DECODE_FAILURE_ALLOWANCE candidates+ // ever leave here however many results there are.+ let spare = DECODE_FAILURE_ALLOWANCE;+ let charsLeft = MAX_TOTAL_MCP_IMAGE_CHARS;+ const perResult = localMarkers ? LOCAL_MAX_IMAGES_PER_TURN : MAX_MODEL_IMAGES;+ const out: McpImage[][][] = batches.map((batch) => batch.map(() => []));+ // Newest first: those are the ones the backend would have kept.+ for (let b = batches.length - 1; b >= 0; b--) {+ const batch = batches[b];+ const room = Math.min(budget, perResult);+ // Charge only what the backend can promote; excess entries must not evict+ // older results. Since only successful decodes count and this side cannot+ // decode, keep shared fallback candidates beyond the remaining image budget.+ // Spending or resetting spares per result can strand valid older images+ // behind corrupt entries even when decoding would leave room.+ let allowance = room + spare;+ let taken = 0;+ for (let r = batch.length - 1; r >= 0; r--) {+ const images = batch[r];+ // Scan until enough candidates fit: slicing first could discard smaller+ // images behind entries that exceed the remaining byte budget.+ const keep: McpImage[] = [];+ for (const image of images) {+ if (keep.length >= allowance) break;+ if (image.mimeType.length > MAX_MCP_IMAGE_MIME_CHARS) continue;+ const cost = image.data.length;+ // Keep looking for smaller images that fit, matching the live-result budget.+ if (charsLeft - cost < 0) continue;+ charsLeft -= cost;+ keep.push(image);+ }+ allowance -= keep.length;+ taken += keep.length;+ if (keep.length === images.length) {+ out[b][r] = images.slice();+ continue;+ }+ // Carry the count the tool actually returned, or a prior bound's record of it,+ // so the backend's note does not describe this upload as the whole result.+ const returned = Math.max(images[0]?.returned ?? 0, images.length);+ out[b][r] = keep.length > 0 ? [{ ...keep[0], returned }, ...keep.slice(1)] : [];+ }+ // Charge only the batch's usable slots. Fallback candidates consume spares+ // so they cannot independently evict older results.+ const charged = Math.min(taken, room);+ budget -= charged;+ spare -= taken - charged;+ }+ return out;+}++/** The bound over an already serialized OpenAI history. The send path bounds the+ * run's own tool results before serializing them (chat-adapter's+ * boundMcpImageResults), so a discarded envelope is never even built; this form+ * serves callers that only hold the wire shape. */+export function boundMcpImageEnvelopes<T extends EnvelopeCarrier>(+ messages: readonly T[],+ { localMarkers = false }: { localMarkers?: boolean } = {},+): T[] {+ const out = messages.slice();+ type Carrier = { index: number; text: string; images: McpImage[] };+ const carriers: Carrier[] = [];+ for (let i = 0; i < out.length; i++) {+ const message = out[i];+ if (!message || message.role !== "tool") continue;+ if (typeof message.content !== "string") continue;+ const { text, images } = splitMcpImages(message.content);+ if (images.length === 0) continue;+ // Match backend provenance: strip named non-MCP envelopes so their payloads+ // cannot bypass the replay bounds and be uploaded again.+ if (+ typeof message.name === "string" &&+ message.name &&+ !message.name.startsWith(MCP_TOOL_PREFIX)+ ) {+ out[i] = { ...message, content: text };+ continue;+ }+ carriers.push({ index: i, text, images });+ }+ // Consecutive tool results are one batch on a marker target.+ const batches: Carrier[][] = [];+ for (const carrier of carriers) {+ const previous = batches[batches.length - 1];+ const last = previous?.[previous.length - 1];+ let sameBatch = localMarkers && last !== undefined;+ for (let j = (last?.index ?? 0) + 1; sameBatch && j < carrier.index; j++) {+ if (out[j]?.role !== "tool") sameBatch = false;+ }+ if (sameBatch && previous) previous.push(carrier);+ else batches.push([carrier]);+ }+ const plan = planMcpImageBound(+ batches.map((batch) => batch.map((carrier) => carrier.images)),+ { localMarkers },+ );+ batches.forEach((batch, b) =>+ batch.forEach((carrier, r) => {+ const kept = plan[b][r];+ if (kept.length === carrier.images.length) return;+ out[carrier.index] = {+ ...out[carrier.index],+ content:+ kept.length > 0+ ? carrier.text + mcpImagesEnvelope(kept)+ : carrier.text,+ };+ }),+ );+ return out;+}++// For a target known not to read images. The backend strips these envelopes without+// sending a pixel, so leaving them on the wire re-uploaded up to 12 million characters+// of base64 on every text turn after a switch to a text-only model. The stored+// history keeps them, for a later switch back.+export function stripMcpImageEnvelopes<T extends EnvelopeCarrier>(+ messages: readonly T[],+): T[] {+ return messages.map((message) => {+ if (!message || message.role !== "tool") return message;+ if (typeof message.content !== "string") return message;+ const { text, images } = splitMcpImages(message.content);+ return images.length === 0 ? message : { ...message, content: text };+ });+}diff --git a/studio/frontend/src/features/chat/codex-reasoning.ts b/studio/frontend/src/features/chat/codex-reasoning.tsindex e40dc98793a..46e626312dd 100644--- a/studio/frontend/src/features/chat/codex-reasoning.ts+++ b/studio/frontend/src/features/chat/codex-reasoning.ts@@ -95,3 +95,40 @@ export function codexReasoningForToolCalls(}return undefined;}++/** Which replay exchange each tool-call part of one assistant message lands in, by the+ * serializer's own rule: a round id that changes starts a new exchange, and a part with+ * no round id that completes a local pair is an exchange of its own. The MCP image+ * bound groups a message's results by this, so it shares one marker turn's allowance+ * across exactly the results the backend will see as one batch -- grouping the whole+ * message let a later round's picture evict an earlier round's. */+export function localToolExchangeIndexes<TPart>(+ parts: readonly TPart[],+ roundIdOf: (part: TPart) => number | null,+ flushesPair: (part: TPart) => boolean,+): number[] {+ const indexes: number[] = [];+ let exchange = 0;+ let pending = 0;+ let pendingRoundId: number | null = null;+ for (const part of parts) {+ const roundId = roundIdOf(part);+ if (pending > 0 && startsNewCodexToolRound(pendingRoundId, roundId)) {+ exchange += 1;+ pending = 0;+ }+ if (roundId !== null) pendingRoundId = roundId;+ const flushLocalPair = roundId === null && flushesPair(part);+ if (flushLocalPair && pending > 0) {+ exchange += 1;+ pending = 0;+ }+ indexes.push(exchange);+ pending += 1;+ if (flushLocalPair) {+ exchange += 1;+ pending = 0;+ }+ }+ return indexes;+}diff --git a/studio/frontend/src/features/chat/external-providers.ts b/studio/frontend/src/features/chat/external-providers.tsindex 3aeafbae755..8ea2c745373 100644--- a/studio/frontend/src/features/chat/external-providers.ts+++ b/studio/frontend/src/features/chat/external-providers.ts@@ -216,6 +216,29 @@ export function providerModelSupportsVision(}+// Mirrors _MIXED_CATALOG_PROVIDER_TYPES in studio/backend/routes/inference.py: vision for+// the family, unknown for a given model, so an MCP picture is not sent to one the+// registry says nothing about.+const MIXED_CATALOG_PROVIDER_TYPES = new Set(["huggingface", "openrouter", "qwen"]);++/** Whether the backend's external tool loop will hand this model an MCP picture -- the+ * same rule as its _external_takes_mcp_images, so envelopes the backend would strip+ * anyway are not re-uploaded on every turn. Stricter than providerModelSupportsVision:+ * a model-level answer wins, a mixed catalog with none is a no, unknown elsewhere is a yes. */+export function providerModelTakesMcpImages(+ providerType: string | null | undefined,+ modelId: string | null | undefined,+): boolean {+ if (providerTypeSupportsVision(providerType) === false) return false;+ hydrateProviderModelCapabilities();+ if (providerType && modelId) {+ const capability = REGISTRY_MODEL_CAPABILITIES.get(providerType)?.[modelId];+ if (typeof capability?.vision === "boolean") return capability.vision;+ }+ if (providerType && MIXED_CATALOG_PROVIDER_TYPES.has(providerType)) return false;+ return true;+}+/** Provider-level capability key. Self-hosted model ids are user-supplied, so there is no* per-model entry: the registry declares the capability once for the whole provider type. */export const PROVIDER_CAPABILITY_WILDCARD = "*";diff --git a/studio/frontend/src/features/chat/hooks/use-chat-search-index.ts b/studio/frontend/src/features/chat/hooks/use-chat-search-index.tsindex 18972b7f9d0..57a0fe2ec52 100644--- a/studio/frontend/src/features/chat/hooks/use-chat-search-index.ts+++ b/studio/frontend/src/features/chat/hooks/use-chat-search-index.ts@@ -13,6 +13,7 @@ import {CHAT_HISTORY_UPDATED_EVENT,batchListChatMessages,} from "../api/chat-api";+import { splitMcpImages } from "../api/mcp-images";import type { MessageRecord } from "../types";import { isCoalescedHistoryEvent } from "../utils/chat-history-revision";import {@@ -52,38 +53,11 @@ const ROW_RELEASE_DELAY_MS = 300;// Keys whose values are base64 image/audio payloads, not searchable text.const BINARY_KEY = /b64|base64|^(images?|audio|video)$/i;-// Drop a trailing __MCP_IMAGES__ envelope only when it is the valid JSON image array the-// backend appended, so tool text merely mentioning the marker stays searchable.-function stripMcpImageSuffix(value: string): string {- const marker = "\n__MCP_IMAGES__:";- const idx = value.lastIndexOf(marker);- if (idx === -1) return value;- try {- const images: unknown = JSON.parse(value.slice(idx + marker.length));- if (- Array.isArray(images) &&- images.length > 0 &&- images.every(- (img) =>- typeof img === "object" &&- img !== null &&- typeof (img as Record<string, unknown>).data === "string" &&- typeof (img as Record<string, unknown>).mimeType === "string",- )- ) {- return value.slice(0, idx);- }- } catch {- // Not a valid envelope; leave the text intact.- }- return value;-}-// Readable text from tool args/results, dropping base64 image/audio blobs so they never// bloat the index.function searchableText(value: unknown, depth = 0): string {if (typeof value === "string") {- let text = stripMcpImageSuffix(value);+ let text = splitMcpImages(value).text;const cut = text.indexOf("\n__IMAGES__:");if (cut !== -1) text = text.slice(0, cut);return textdiff --git a/studio/frontend/src/features/chat/utils/mcp-tool-name.ts b/studio/frontend/src/features/chat/utils/mcp-tool-name.tsindex aeebdc49969..a648de77886 100644--- a/studio/frontend/src/features/chat/utils/mcp-tool-name.ts+++ b/studio/frontend/src/features/chat/utils/mcp-tool-name.ts@@ -3,6 +3,11 @@const MCP_TOOL_PREFIX = "mcp__";+/** Whether a tool call came from an MCP server, by the id the backend stamps. */+export function isMcpToolName(toolName: string | undefined): boolean {+ return typeof toolName === "string" && toolName.startsWith(MCP_TOOL_PREFIX);+}+function provenanceString(provenance: unknown,key: "mcp_server" | "mcp_tool",diff --git a/studio/frontend/tests/mcp-images.test.ts b/studio/frontend/tests/mcp-images.test.tsnew file mode 100644index 00000000000..3d3b8216de1--- /dev/null+++ b/studio/frontend/tests/mcp-images.test.ts@@ -0,0 +1,589 @@+// SPDX-License-Identifier: AGPL-3.0-only+// Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0++import assert from "node:assert/strict";+import { readFileSync } from "node:fs";+import test from "node:test";++import {+ DECODE_FAILURE_ALLOWANCE,+ MAX_MODEL_IMAGES,+ MAX_TOTAL_MCP_IMAGES,+ MAX_MCP_IMAGE_MIME_CHARS,+ MAX_TOTAL_MCP_IMAGE_CHARS,+ planMcpImageBound,+ MCP_IMAGES_MARKER,+ boundMcpImageEnvelopes,+ mcpImagesEnvelope,+ splitMcpImages,+ stripMcpImageEnvelopes,+} from "../src/features/chat/api/mcp-images.ts";+import { providerModelTakesMcpImages } from "../src/features/chat/external-providers.ts";+import { localToolExchangeIndexes } from "../src/features/chat/codex-reasoning.ts";+import { isMcpToolName } from "../src/features/chat/utils/mcp-tool-name.ts";++const IMAGES = [{ data: "QUJD", mimeType: "image/png" }];+const RESULT = `[1 image returned]${mcpImagesEnvelope(IMAGES)}`;++const adapter = readFileSync(+ new URL("../src/features/chat/api/chat-adapter.ts", import.meta.url),+ "utf8",+);++test("a valid envelope splits into the text and its images", () => {+ assert.deepEqual(splitMcpImages(RESULT), {+ text: "[1 image returned]",+ images: IMAGES,+ });+});++test("text that only mentions the marker is left whole", () => {+ const result = `the marker is${MCP_IMAGES_MARKER} and nothing follows`;+ assert.deepEqual(splitMcpImages(result), { text: result, images: [] });+});++test("an envelope that is not an image array is left whole", () => {+ const result = `log${MCP_IMAGES_MARKER}["not", "image", "dicts"]`;+ assert.deepEqual(splitMcpImages(result), { text: result, images: [] });+});++test("an unparseable envelope is left whole", () => {+ const result = `log${MCP_IMAGES_MARKER}{oops`;+ assert.deepEqual(splitMcpImages(result), { text: result, images: [] });+});++test("replaying a tool result re-attaches its images for the backend", () => {+ assert.match(adapter, /content \+= mcpImagesEnvelope\(result\.images\);/);+});++test("only an MCP tool call carries the privileged envelope", () => {+ // A client tool is free to answer {text, images:[{data, mimeType}]}, which is+ // the shape isMcpImageToolResult accepts. Without the provenance gate its bytes+ // would be promoted into model image input on the next request.+ assert.equal(isMcpToolName("mcp__fs__read_media_file"), true);+ assert.equal(isMcpToolName("render_chart"), false);+ assert.equal(isMcpToolName(undefined), false);+ // The envelope is gated...+ assert.match(+ adapter,+ /if \(isMcpImageToolResult\(result\) && isMcpToolName\(tc\.toolName\)\) \{\n\s*content \+= mcpImagesEnvelope\(result\.images\);/,+ );+ // ...but the wrapper branch is NOT: excluding a non-MCP wrapper there dropped it+ // into JSON.stringify, which replays the whole base64 array as prompt text.+ assert.match(adapter, /^\s*isMcpImageToolResult\(result\) \|\|$/m);+});++const shot = (n: number, name = "mcp__fs__screenshot") => ({+ role: "tool",+ name,+ content:+ `[3 images returned]` ++ mcpImagesEnvelope([+ { data: `A${n}`, mimeType: "image/png" },+ { data: `B${n}`, mimeType: "image/png" },+ { data: `C${n}`, mimeType: "image/png" },+ ]),+});++const countImages = (messages: { content?: unknown }[]) =>+ messages.reduce(+ (n, m) =>+ n ++ (typeof m.content === "string"+ ? splitMcpImages(m.content).images.length+ : 0),+ 0,+ );++test("history is bounded before it is uploaded, not after", () => {+ // The backend's cap runs after the body is parsed, so it cannot bound transport.+ // The ceiling counts CANDIDATES, which sit above the promotable budget so the+ // backend can still scan past entries it cannot decode.+ const messages = [0, 1, 2, 3, 4].map((n) => shot(n));++ const bounded = boundMcpImageEnvelopes(messages);++ assert.equal(countImages(messages), 15);+ assert.ok(countImages(bounded) < countImages(messages), "nothing was bounded");+ assert.ok(+ countImages(bounded) <= MAX_TOTAL_MCP_IMAGES + DECODE_FAILURE_ALLOWANCE,+ `uploaded ${countImages(bounded)} candidates`,+ );+ // The oldest result is still dropped outright.+ assert.equal(splitMcpImages(bounded[0].content).images.length, 0);+});++test("the newest pictures are the ones kept", () => {+ const bounded = boundMcpImageEnvelopes([0, 1, 2, 3, 4].map((n) => shot(n)));++ assert.equal(splitMcpImages(bounded[4].content).images.length, 3);+ assert.equal(splitMcpImages(bounded[0].content).images.length, 0);+ // The note survives even when its payload does not.+ assert.match(bounded[0].content, /^\[3 images returned\]$/);+});++test("a conversation inside the budget is left byte-identical", () => {+ const messages = [0, 1].map((n) => shot(n));++ assert.deepEqual(boundMcpImageEnvelopes(messages), messages);+});++test("a non-MCP tool result keeps its text and loses only the envelope", () => {+ // The backend never promotes a named non-MCP result and strips its envelope+ // regardless, so leaving it whole here only re-uploaded megabytes of base64 on+ // every later turn. The text the model reads is unchanged.+ const message = {+ role: "tool",+ name: "read_file",+ content: "plain text" + mcpImagesEnvelope([{ data: "QUJD", mimeType: "image/png" }]),+ };++ const [bounded] = boundMcpImageEnvelopes([message]);++ assert.equal(bounded.content, "plain text");+ assert.equal(bounded.name, "read_file");+});++const shotOf = (n: string, count: number) => ({+ role: "tool",+ name: "mcp__fs__screenshot",+ content:+ `[${count} images returned]` ++ mcpImagesEnvelope(+ Array.from({ length: count }, (_, i) => ({+ data: `${n}${i}`,+ mimeType: "image/png",+ })),+ ),+});++test("a fat newest result does not evict older usable images", () => {+ // Budget is charged at the promotable rate, so an oversized newest envelope+ // cannot spend the whole history allowance on images the backend would drop.+ const bounded = boundMcpImageEnvelopes([shotOf("old", 4), shotOf("new", 8)]);++ assert.equal(splitMcpImages(bounded[0].content).images.length, 4);+});++test("no single result uploads more than the backend could ever decode", () => {+ // Bounded, but above the quota: the backend counts successful decodes and this+ // side cannot tell which entries will decode.+ const bounded = boundMcpImageEnvelopes([shotOf("only", 12)]);++ assert.equal(+ splitMcpImages(bounded[0].content).images.length,+ MAX_MODEL_IMAGES + DECODE_FAILURE_ALLOWANCE,+ );+});++test("candidates survive the transport bound for the backend to decode", () => {+ // The backend's quota counts SUCCESSFUL decodes and scans past failures. This+ // side cannot decode, so cutting at the quota drops valid PNGs sitting behind+ // formats Pillow rejects -- shown on the first turn, lost on the replay.+ const bounded = boundMcpImageEnvelopes([shotOf("a", 8)]);++ assert.equal(+ splitMcpImages(bounded[0].content).images.length,+ MAX_MODEL_IMAGES + DECODE_FAILURE_ALLOWANCE,+ );+});++test("the spare candidates do not evict an older result", () => {+ const bounded = boundMcpImageEnvelopes([shotOf("old", 4), shotOf("new", 8)]);++ assert.equal(splitMcpImages(bounded[0].content).images.length, 4);+});++test("a partly-spent budget still leaves an older result its decode spares", () => {+ // A newer one-image result leaves 7 of 8. The older result has corrupt entries+ // ahead of valid PNGs, so slicing it to the remaining budget alone strands the+ // last valid one even though both quotas had room.+ const bounded = boundMcpImageEnvelopes([shotOf("old", 8), shotOf("new", 1)]);++ assert.equal(+ splitMcpImages(bounded[0].content).images.length,+ MAX_MODEL_IMAGES + DECODE_FAILURE_ALLOWANCE,+ );+ assert.equal(splitMcpImages(bounded[1].content).images.length, 1);+});++test("the decode-failure allowance survives an undecodable newest result", () => {+ // Four entries the frontend cannot tell apart from PNGs, then two results of four+ // real ones. Per-result, the first spends the whole room and the oldest envelope is+ // dropped outright, so the backend can find only four decodable pictures out of the+ // eight the cap allows.+ const undecodable = Array.from({ length: 4 }, (_, i) => ({+ data: `SVG${i}`,+ mimeType: "image/svg+xml",+ }));+ const valid = (tag: string) =>+ Array.from({ length: 4 }, (_, i) => ({ data: `${tag}${i}`, mimeType: "image/png" }));++ const messages = [+ { role: "tool", name: "mcp__s__a", content: "oldest" + mcpImagesEnvelope(valid("A")) },+ { role: "tool", name: "mcp__s__b", content: "middle" + mcpImagesEnvelope(valid("B")) },+ { role: "tool", name: "mcp__s__c", content: "newest" + mcpImagesEnvelope(undecodable) },+ ];++ const bounded = boundMcpImageEnvelopes(messages);+ const kept = bounded.map((m) => splitMcpImages(m.content as string).images);++ assert.equal(kept[2].length, 4, "the newest result is kept as candidates");+ assert.equal(kept[1].length, 4, "the middle result keeps its four real pictures");+ assert.equal(+ kept[0].length,+ 4,+ "the oldest result still ships candidates on the shared allowance",+ );++ const total = kept.reduce((n, images) => n + images.length, 0);+ assert.ok(+ total <= MAX_TOTAL_MCP_IMAGES + DECODE_FAILURE_ALLOWANCE,+ `bounded: ${total} candidates uploaded`,+ );+});++test("a replayed history is bounded by bytes, not only by image count", () => {+ // One authentic result may carry MAX_IMAGE_PAYLOAD_CHARS (12M) of base64, so the+ // twelve candidates the count bound allows could re-upload ~144MB on every later+ // turn of an image-heavy chat.+ const huge = "A".repeat(4_000_000);+ const messages = Array.from({ length: 6 }, (_, i) => ({+ role: "tool",+ name: `mcp__s__shot${i}`,+ content:+ `result ${i}` ++ mcpImagesEnvelope([{ data: huge, mimeType: "image/png" }]),+ }));++ const bounded = boundMcpImageEnvelopes(messages);+ const kept = bounded.flatMap((m) => splitMcpImages(m.content as string).images);+ const chars = kept.reduce((n, image) => n + image.data.length, 0);++ assert.ok(+ chars <= MAX_TOTAL_MCP_IMAGE_CHARS,+ `${chars} characters uploaded against a budget of ${MAX_TOTAL_MCP_IMAGE_CHARS}`,+ );+ assert.ok(kept.length > 0, "the budget must not starve the newest result");+ // The newest results are the ones kept, as everywhere else here.+ const newest = splitMcpImages(bounded[5].content as string).images;+ assert.equal(newest.length, 1, "the newest result keeps its picture");+ assert.equal(+ splitMcpImages(bounded[0].content as string).images.length,+ 0,+ "the oldest gives its bytes up first",+ );+});++test("an ordinary conversation is untouched by the byte budget", () => {+ const small = "B".repeat(2048);+ const messages = Array.from({ length: 2 }, (_, i) => ({+ role: "tool",+ name: `mcp__s__shot${i}`,+ content: `r${i}` + mcpImagesEnvelope([{ data: small, mimeType: "image/png" }]),+ }));++ const bounded = boundMcpImageEnvelopes(messages);++ assert.deepEqual(bounded, messages, "nothing was rewritten");+});++test("an oversized replay image is skipped, not the rest of its result", () => {+ // Breaking on the first candidate that does not fit threw away three 1MB pictures+ // sitting behind a 5MB one, which the backend could have replayed.+ const newest = "N".repeat(9_000_000);+ const oversized = "X".repeat(5_000_000);+ const small = (tag: string) => tag.repeat(900_000);++ const messages = [+ {+ role: "tool",+ name: "mcp__s__old",+ content:+ "old" ++ mcpImagesEnvelope([+ { data: oversized, mimeType: "image/png" },+ { data: small("a"), mimeType: "image/png" },+ { data: small("b"), mimeType: "image/png" },+ ]),+ },+ { role: "tool", name: "mcp__s__new", content: "new" + mcpImagesEnvelope([{ data: newest, mimeType: "image/png" }]) },+ ];++ const bounded = boundMcpImageEnvelopes(messages);+ const older = splitMcpImages(bounded[0].content as string).images;++ assert.ok(+ !older.some((image) => image.data === oversized),+ "the oversized candidate should not fit",+ );+ assert.equal(older.length, 2, "the two that DO fit must survive it");+ const chars = bounded+ .flatMap((m) => splitMcpImages(m.content as string).images)+ .reduce((n, image) => n + image.data.length, 0);+ assert.ok(chars <= MAX_TOTAL_MCP_IMAGE_CHARS, `${chars} characters uploaded`);+});++test("a bounded envelope records how many the tool returned", () => {+ const images = Array.from({ length: 20 }, (_, i) => ({ data: `IMG${i}`, mimeType: "image/png" }));+ const [bounded] = boundMcpImageEnvelopes([+ { role: "tool", name: "mcp__s__shot", content: "r" + mcpImagesEnvelope(images) },+ ]);+ const kept = splitMcpImages(bounded.content as string).images;+ assert.ok(kept.length < 20, "the bound must actually have cut");+ assert.equal(kept[0].returned, 20, "the original count survives on the first entry");+});++test("a named non-MCP result has its envelope stripped, not re-uploaded", () => {+ const huge = "Z".repeat(3_000_000);+ const [bounded] = boundMcpImageEnvelopes([+ { role: "tool", name: "read_file", content: "here" + mcpImagesEnvelope([{ data: huge, mimeType: "image/png" }]) },+ ]);+ assert.equal(bounded.content, "here");+});++test("a text-only target is sent no envelopes at all", () => {+ const messages = [+ { role: "user", content: "what did it show" },+ {+ role: "tool",+ name: "mcp__s__shot",+ content: "[1 image returned]" + mcpImagesEnvelope([{ data: "A".repeat(50_000), mimeType: "image/png" }]),+ },+ { role: "tool", name: "read_file", content: "plain" },+ ];++ const stripped = stripMcpImageEnvelopes(messages);++ assert.equal(stripped[1].content, "[1 image returned]");+ assert.equal(stripped[0], messages[0], "non-tool messages are the same object");+ assert.equal(stripped[2], messages[2], "a tool result with no envelope is untouched");+});++const round = (n: number, results = 1, perResult = 3) => [+ { role: "assistant", content: `call ${n}` },+ ...Array.from({ length: results }, (_, k) => ({+ role: "tool",+ name: "mcp__fs__screenshot",+ content:+ `[${perResult} images returned]` ++ mcpImagesEnvelope(+ Array.from({ length: perResult }, (_, m) => ({+ data: `R${n}K${k}M${m}`,+ mimeType: "image/png",+ })),+ ),+ })),+ { role: "assistant", content: `answer ${n}` },+];++const imagesPerToolResult = (messages: { role?: string; content?: unknown }[]) =>+ messages+ .filter((m) => m.role === "tool" && typeof m.content === "string")+ .map((m) => splitMcpImages(m.content as string).images.length);++test("a marker target is charged one picture per round, so every round keeps one", () => {+ // Eight rounds of three: the backend's local path replays one per round, eight in+ // all. Four per result spent the budget on the newest rounds and stripped the rest.+ const messages = Array.from({ length: 8 }, (_, n) => round(n)).flat();++ const parts = imagesPerToolResult(boundMcpImageEnvelopes(messages));+ assert.ok(parts.some((n) => n === 0), `default still strips old rounds: ${parts}`);++ const markers = imagesPerToolResult(+ boundMcpImageEnvelopes(messages, { localMarkers: true }),+ );+ assert.equal(markers.length, 8);+ assert.ok(markers.every((n) => n >= 1), `every round keeps a candidate: ${markers}`);+ // Spares ride with the newest rounds only; the rest carry exactly their one.+ assert.equal(markers[markers.length - 1], 3);+ assert.equal(markers[0], 1);+});++test("consecutive results are one batch on a marker target and share one charge", () => {+ // Three parallel results land as one turn carrying one picture: charged once,+ // not three times, so eight such rounds all keep a candidate.+ const messages = Array.from({ length: 8 }, (_, n) => round(n, 3, 2)).flat();++ const markers = imagesPerToolResult(+ boundMcpImageEnvelopes(messages, { localMarkers: true }),+ );+ assert.equal(markers.length, 24);+ for (let n = 0; n < 8; n++) {+ const batch = markers.slice(n * 3, n * 3 + 3);+ assert.ok(batch.some((k) => k >= 1), `round ${n} keeps a candidate: ${batch}`);+ }+});++test("the send path bounds the run's own results before serializing them", () => {+ // Decided before the slice tests/studio executes standalone, applied to the raw+ // results, so no envelope the request will not carry is ever built or parsed back.+ assert.match(+ adapter,+ /const mcpImagesLocalMarkers =\n\s*!isExternalRequest &&\n\s*runtime\.models\.find\(\(model\) => model\.id === runtime\.params\.checkpoint\)\n\s*\?\.isGguf === false;/,+ );+ assert.match(+ adapter,+ /const messages = boundMcpImageResults\(rawMessages, \{\n\s*readsImages: targetReadsImages,\n\s*localMarkers: mcpImagesLocalMarkers,\n\s*\}\);\n\s*const survivingMessages = pruneOutboundHistory\(\n\s*messages,/,+ );+ assert.doesNotMatch(adapter, /boundMcpImageEnvelopes\(outboundMessages/);+ assert.doesNotMatch(adapter, /stripMcpImageEnvelopes\(outboundMessages/);+});++test("the upload gate is the backend's external MCP gate, not provider-level vision", () => {+ // A mixed catalog says nothing about most models: the backend sends no picture+ // there, so uploading the envelopes only had it strip 12 MB again on every turn.+ assert.equal(providerModelTakesMcpImages("openrouter", "some/unknown-model"), false);+ assert.equal(providerModelTakesMcpImages("huggingface", "org/unknown"), false);+ assert.equal(providerModelTakesMcpImages("qwen", "qwen2.5-72b-instruct"), false);+ // Known text-only endpoints stay a no; a vision family with a plain catalog is a yes.+ assert.equal(providerModelTakesMcpImages("mistral", "mistral-large"), false);+ assert.equal(providerModelTakesMcpImages("anthropic", "claude-x"), true);+ // Unknown provider type keeps them: the backend decides.+ assert.equal(providerModelTakesMcpImages(undefined, undefined), true);+});++test("both local paths read the model's vision flag, not just multimodal", () => {+ assert.match(+ adapter,+ /function localTargetReadsImages\([\s\S]*?if \(typeof activeModel\?\.isVision === "boolean"\) return activeModel\.isVision;\n\s*return state\.loadedIsMultimodal !== false;/,+ );+ assert.match(adapter, /: localTargetReadsImages\(runtime\);/);+ assert.match(+ adapter,+ /const messages = boundMcpImageResults\(rawMessages, \{\n\s*readsImages: localTargetReadsImages\(runtimeState\),/,+ );+ assert.match(+ adapter,+ /\? providerModelTakesMcpImages\(\n\s*externalProvider\?\.providerType,\n\s*externalSelection\?\.modelId,\n\s*\)/,+ );+});++test("an entry with an unbounded mimeType is dropped, not charged", () => {+ // A token subtype has no length bound, so a tiny picture can carry megabytes of+ // mimeType; charging data alone let that envelope through on every turn.+ const heavy = (n: number) => ({+ role: "tool",+ name: "mcp__fs__screenshot",+ content:+ "[1 image returned]" ++ mcpImagesEnvelope([+ { data: `A${n}`, mimeType: `image/${"x".repeat(MAX_MCP_IMAGE_MIME_CHARS + 1)}` },+ { data: `B${n}`, mimeType: "image/png" },+ ]),+ });+ const bounded = boundMcpImageEnvelopes([0, 1].map(heavy));+ for (const message of bounded) {+ const kept = splitMcpImages(message.content).images;+ assert.deepEqual(+ kept.map((image) => image.mimeType),+ ["image/png"],+ "the bounded-mime entry stays, the unbounded one goes",+ );+ }+});++test("a picture the live turn accepted at the limit still fits its own replay", () => {+ // The live limit is on data; charging the serialized entry pushed a picture of+ // exactly that size over the identically sized replay budget, and the only+ // picture in the conversation vanished from the next request.+ const messages = [+ {+ role: "tool",+ name: "mcp__fs__screenshot",+ content:+ "[1 image returned]" ++ mcpImagesEnvelope([+ { data: "A".repeat(MAX_TOTAL_MCP_IMAGE_CHARS), mimeType: "image/png" },+ ]),+ },+ ];+ const [bounded] = boundMcpImageEnvelopes(messages);+ assert.equal(splitMcpImages(bounded.content).images.length, 1);+});++test("the byte filter scans past candidates that do not fit", () => {+ // A newer result leaves about 1 MB. The older result's first eight candidates are+ // 1.4 MB each; slicing to the allowance first inspected only those and dropped the+ // small ninth, which fits and which the backend could have replayed.+ const big = (tag: string, n: number, size: number) =>+ Array.from({ length: n }, (_, i) => ({ data: tag.repeat(size / tag.length) + i, mimeType: "image/png" }));+ const messages = [+ {+ role: "tool",+ name: "mcp__fs__a",+ content: "older" + mcpImagesEnvelope([...big("O", 8, 1_400_000), { data: "tiny", mimeType: "image/png" }]),+ },+ { role: "assistant", content: "next" },+ {+ role: "tool",+ name: "mcp__fs__b",+ content: "newer" + mcpImagesEnvelope(big("N", 1, 11_000_000)),+ },+ ];+ const bounded = boundMcpImageEnvelopes(messages);+ const older = splitMcpImages(bounded[0].content).images;+ assert.deepEqual(older.map((image) => image.data), ["tiny"]);+});++test("the planner is the bound the envelope form applies", () => {+ // Both carriers plan through one function, so the raw-result bound the send path+ // takes and the envelope bound agree result for result.+ const rounds = Array.from({ length: 8 }, (_, n) => round(n, 3, 2)).flat();+ const envelopes = boundMcpImageEnvelopes(rounds, { localMarkers: true });+ const viaEnvelopes = imagesPerToolResult(envelopes);+ const batches = Array.from({ length: 8 }, (_, n) =>+ rounds+ .slice(n * 5 + 1, n * 5 + 4)+ .map((m) => splitMcpImages(m.content as string).images),+ );+ const viaPlanner = planMcpImageBound(batches, { localMarkers: true })+ .flat()+ .map((kept) => kept.length);+ assert.deepEqual(viaPlanner, viaEnvelopes);+ // A result that keeps nothing is an empty plan, which the raw bound turns into text.+ const dropped = planMcpImageBound([[[{ data: "x", mimeType: "image/png" }]], [[Array.from({ length: 12 }, (_, i) => ({ data: `${i}`, mimeType: "image/png" }))].flat()]]);+ assert.equal(dropped[0][0].length, 1);+});++test("a message's results are batched by replay exchange, not as one block", () => {+ // Three local rounds accumulate in one assistant message, told apart by round_id;+ // the serializer emits them as three exchanges, so the bound treats them as three+ // batches (one picture each), not one batch sharing a single picture's allowance.+ type P = { round: number | null; flush: boolean };+ const parts: P[] = [+ { round: 0, flush: false },+ { round: 0, flush: false },+ { round: 1, flush: false },+ { round: 2, flush: false },+ { round: null, flush: true },+ { round: null, flush: true },+ ];+ const indexes = localToolExchangeIndexes(+ parts,+ (p) => p.round,+ (p) => p.flush,+ );+ assert.deepEqual(indexes, [0, 0, 1, 2, 3, 4]);+ // The bound's partition uses the serializer's own conditions.+ assert.match(adapter, /startsNewCodexToolRound\(pendingLocalToolRoundId, localRoundId\)/);+ assert.match(adapter, /localRoundId === null && shouldFlushCompletedLocalToolPair\(toolPart\)/);+ assert.match(+ adapter,+ /localToolExchangeIndexes\(\n\s*toolParts,\n\s*\(\{ part \}\) => codexLocalToolRoundId\(getToolReplayProvenance\(part\)\),\n\s*\(\{ part \}\) => shouldFlushCompletedLocalToolPair\(part\),/,+ );+});++test("a client tool's structured result is not unwrapped as the MCP wrapper", () => {+ // Unwrapping by shape alone reduced {text, images, ...} from a non-MCP client tool+ // to its text, silently dropping every other field. The bare live-parser wrapper+ // is still unwrapped, since JSON of it would replay base64 as prompt text.+ assert.match(+ adapter,+ /\(isMcpImageToolResult\(result\) &&\n\s*\(isMcpToolName\(tc\.toolName\) \|\| isBareMcpImageWrapper\(result\)\)\) \|\|/,+ );+ assert.match(+ adapter,+ /export function isBareMcpImageWrapper\(val: unknown\): boolean \{\n\s*if \(!isMcpImageToolResult\(val\)\) return false;\n\s*const keys = Object\.keys\(val as object\)\.filter\(\(key\) => key !== "text" && key !== "images"\);\n\s*return keys\.length === 0;/,+ );+});diff --git a/tests/studio/test_backend_ci_parallel_isolation.py b/tests/studio/test_backend_ci_parallel_isolation.pyindex 3a3a7edc750..be7970e2bf0 100644--- a/tests/studio/test_backend_ci_parallel_isolation.py+++ b/tests/studio/test_backend_ci_parallel_isolation.py@@ -177,6 +177,7 @@ BACKEND_ISOLATED = [),("tests/test_web_fetch_extraction.py", "compares parse time at two input sizes"),("tests/test_tool_call_parser_strict.py", "compares parse time at two nesting depths"),+ ("tests/test_pr5624_regressions.py", "R1 parser's 1s bound exceeded under CPU contention"),# Found by staging rather than by the scan, and the scan cannot find it: see below.("tests/test_tunnel_safe_long_post.py",diff --git a/tests/studio/test_deep_research_frontend_contract.py b/tests/studio/test_deep_research_frontend_contract.pyindex 54d44b6856e..e3495a51b84 100644--- a/tests/studio/test_deep_research_frontend_contract.py+++ b/tests/studio/test_deep_research_frontend_contract.py@@ -143,9 +143,9 @@ def test_research_presave_keeps_the_follow_up_parent() -> None:"const createdRun = await createResearchRun({", 1)[0]- assert "const userMessageIndex = messages.indexOf(userMessage);" in presave+ assert "const userMessageIndex = rawMessages.indexOf(userMessage);" in presaveassert "const userMessageParentId =" in presave- assert "userMessageIndex > 0 ? messages[userMessageIndex - 1]!.id : null" in presave+ assert "userMessageIndex > 0 ? rawMessages[userMessageIndex - 1]!.id : null" in presave# A stored null is an edited root; `??` would reparent it under the predecessor.assert "storedUserMessage && storedUserMessage.parentId !== undefined" in presaveassert "? storedUserMessage.parentId" in presavediff --git a/tests/studio/test_token_count_prompt_parity.py b/tests/studio/test_token_count_prompt_parity.pyindex 28695ef4b9d..4d3f9f0388e 100644--- a/tests/studio/test_token_count_prompt_parity.py+++ b/tests/studio/test_token_count_prompt_parity.py@@ -122,6 +122,7 @@ HARNESS = """// Fixtures the sliced builder reads through. Everything below the PRELUDE marker is// copied verbatim out of studio/frontend/src/features/chat/api/chat-adapter.ts.const state: any = {+ models: [],params: { systemPrompt: "", systemVariables: "" },artifactsEnabled: false,supportsTools: false,diff --git a/tests/test_ci_backend_pytest_shards.py b/tests/test_ci_backend_pytest_shards.pyindex d70b0e9ec7e..0b3fd9d78c7 100644--- a/tests/test_ci_backend_pytest_shards.py+++ b/tests/test_ci_backend_pytest_shards.py@@ -347,14 +347,12 @@ class TestTheSplitKeepsTheSelectionItInherited:returnraise AssertionError(f"the {_SERIAL_STEP!r} step is gone")- def test_the_serial_step_still_runs_the_same_twelve_files(self):- """Not eleven and not thirteen. Whether any of them could go back into the parallel- run was audited separately and the answer was no, so the set is carried across the- split unchanged; a file leaving it here is a coverage change wearing a refactor's- clothes."""+ def test_the_serial_step_runs_the_thirteen_isolated_files(self):+ """The serial selection includes the timing-sensitive R1 parser regressions."""paths = _serial_paths()- assert len(paths) == 12, f"the serial step runs {len(paths)} files, not 12: {paths}"- assert len(set(paths)) == 12, f"the serial step names a file twice: {paths}"+ assert len(paths) == 13, f"the serial step runs {len(paths)} files, not 13: {paths}"+ assert len(set(paths)) == 13, f"the serial step names a file twice: {paths}"+ assert "tests/test_pr5624_regressions.py" in pathsmissing = [path for path in paths if not (_BACKEND_TESTS.parent / path).is_file()]assert not missing, f"the serial step names files that do not exist: {missing}"