Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

studio: send mcp tool images to vision models (#10088)

Nilay992849427e93
diff --git a/.github/workflows/studio-backend-ci.yml b/.github/workflows/studio-backend-ci.yml
index 0dfed4699bc..2321d6d36e0 100644
--- a/.github/workflows/studio-backend-ci.yml
+++ b/.github/workflows/studio-backend-ci.yml
@@ -339,6 +339,7 @@ jobs:
# fails that guard instead of flaking here.
#
# test_media_auto_switch has a 0.4s budget that xdist scheduling can exceed.
+ # test_pr5624_regressions exceeded its 1s R1 parser bound in two contended runs.
#
# --timeout: a per-test wall-clock cap, so a test that never returns fails with its
# own name in the summary instead of taking the job's 45 minutes down with it and
@@ -387,6 +388,7 @@ jobs:
--ignore=tests/test_anthropic_messages.py \
--ignore=tests/test_profile_stats.py \
--ignore=tests/test_media_auto_switch.py \
+ --ignore=tests/test_pr5624_regressions.py \
-k 'not llama_cpp_load_progress_live and not TestGpuAutoSelection and not TestPreSpawnGpuResolution and not TestPerGpuFitGuardAllCounts and not TestTransformersIntrospection and not test_returns_cuda_when_cuda_available and not test_calls_cuda_cache_when_cuda'
- name: Backend tests that cannot share a worker
@@ -412,7 +414,8 @@ jobs:
tests/test_scan_loras_off_event_loop.py \
tests/test_anthropic_messages.py \
tests/test_profile_stats.py \
- tests/test_media_auto_switch.py
+ tests/test_media_auto_switch.py \
+ tests/test_pr5624_regressions.py
- name: Pre-3.12 branches, on the newest interpreter that takes them
if: matrix.scope == 'floor-spot-check'
diff --git a/studio/backend/core/inference/chat_template_helpers.py b/studio/backend/core/inference/chat_template_helpers.py
index 5dd45247d1d..4dc24338af5 100644
--- a/studio/backend/core/inference/chat_template_helpers.py
+++ b/studio/backend/core/inference/chat_template_helpers.py
@@ -3078,6 +3078,10 @@ def render_prompt_with_boundary(
the kwarg get a manual splice, taking the partial from *messages* (which the caller already
swept) rather than a separate copy: a raw partial could close the turn or open another role
instead of resuming (#7066)."""
+ from core.inference.mcp_images import prepare_image_turn_boundaries
+
+ for template in _selected_chat_template_strings(processor, tools):
+ messages = prepare_image_turn_boundaries(messages, template)
extra = {"tools": tools} if tools else {}
partial = trailing_assistant_text(messages) if continue_final_message else None
if not partial:
@@ -3128,6 +3132,10 @@ def apply_chat_template_for_generation(
inside the trailing assistant turn, so the model resumes the partial instead of restarting
it."""
# Shared choke point for the transformers and MLX backends (#7066).
+ from core.inference.mcp_images import prepare_image_turn_boundaries
+
+ for template in _selected_chat_template_strings(tokenizer, tools):
+ messages = prepare_image_turn_boundaries(messages, template)
messages, tools, _markup = neutralize_for_render(tokenizer, messages, tools)
reasoning_kwargs: dict = {}
if enable_thinking is not None:
diff --git a/studio/backend/core/inference/inference.py b/studio/backend/core/inference/inference.py
index 3f3824f967b..c9b95fe987e 100644
--- a/studio/backend/core/inference/inference.py
+++ b/studio/backend/core/inference/inference.py
@@ -471,6 +471,34 @@ def _prompt_already_has_bos(tokenizer, prompt):
return bool(len(ids)) and ids[0] == bos_token_id
+def _without_image_parts(messages) -> list[dict]:
+ """The same conversation with every image placeholder dropped, collapsing a
+ turn back to its text when that is all it held."""
+ out = []
+ for message in messages:
+ content = message.get("content")
+ if not isinstance(content, list):
+ out.append(message)
+ continue
+ kept = [
+ part
+ for part in content
+ if not (
+ isinstance(part, dict) and part.get("type") in ("image", "image_url", "input_image")
+ )
+ ]
+ if len(kept) == len(content):
+ out.append(message)
+ continue
+ texts = [
+ part.get("text", "")
+ for part in kept
+ if isinstance(part, dict) and part.get("type") == "text"
+ ]
+ out.append({**message, "content": "\n".join(texts) if len(kept) == len(texts) else kept})
+ return out
+
+
class InferenceBackend:
"""Unified inference backend supporting text, vision, and LoRA models"""
@@ -1219,6 +1247,8 @@ class InferenceBackend:
messages: list,
system_prompt: str,
image = None,
+ images: Optional[list] = None,
+ image_ordinal: Optional[int] = None,
temperature: float = 0.7,
top_p: float = 0.9,
top_k: int = 40,
@@ -1234,8 +1264,6 @@ class InferenceBackend:
presence_penalty: float = 0.0,
tool_protocol_active: Optional[bool] = None,
stop: Optional[list] = None,
- *,
- images = None,
) -> Generator[str, None, None]:
"""Generate response for text or vision models (lock held by background thread).
@@ -1249,6 +1277,7 @@ class InferenceBackend:
system_prompt = system_prompt,
image = image,
images = images,
+ image_ordinal = image_ordinal,
temperature = temperature,
top_p = top_p,
top_k = top_k,
@@ -1271,6 +1300,8 @@ class InferenceBackend:
messages: list,
system_prompt: str = "",
image = None,
+ images: Optional[list] = None,
+ image_ordinal: Optional[int] = None,
temperature: float = 0.7,
top_p: float = 0.9,
top_k: int = 40,
@@ -1287,8 +1318,6 @@ class InferenceBackend:
presence_penalty: float = 0.0,
tool_protocol_active: Optional[bool] = None,
stop: Optional[list] = None,
- *,
- images = None,
) -> Generator[str, None, None]:
"""Inner generation logic.
@@ -1302,22 +1331,13 @@ class InferenceBackend:
if not self.active_model_name:
raise RuntimeError("No active model")
- # Every generation entry point funnels through here, so the list collapses here.
- if images:
- if len(images) > 1:
- raise ValueError(
- "The transformers backend takes one image per request; "
- f"{len(images)} were supplied."
- )
- image = images[0]
-
model_info = self.models[self.active_model_name]
is_vision = model_info.get("is_vision", False)
tokenizer = model_info.get("tokenizer") or model_info.get("processor")
tokenizer = getattr(tokenizer, "tokenizer", tokenizer)
top_k = self._normalize_top_k(top_k)
- if is_vision and image:
+ if is_vision and (image or images):
# Verify the stored processor can handle images; FastVisionModel may
# return a raw tokenizer instead of a ProcessorMixin (e.g. Gemma-3).
from transformers import ProcessorMixin
@@ -1342,11 +1362,13 @@ class InferenceBackend:
presence_penalty = presence_penalty,
continue_final_message = continue_final_message,
tools = tools,
- tool_protocol_active = tool_protocol_active,
- stop = stop,
+ images = images,
+ image_ordinal = image_ordinal,
enable_thinking = enable_thinking,
reasoning_effort = reasoning_effort,
preserve_thinking = preserve_thinking,
+ tool_protocol_active = tool_protocol_active,
+ stop = stop,
)
return
else:
@@ -1355,6 +1377,10 @@ class InferenceBackend:
f"({type(processor).__name__}) has no image_processor — "
f"falling back to text-only generation (image will be ignored)."
)
+ # Really text-only: the promoted history still carries {"type": "image"}
+ # placeholders, and a text template either rejects the list content or
+ # renders image tokens with no pixels behind them.
+ messages = _without_image_parts(messages)
# Text path: messages are already in ChatML format from eval.py.
@@ -1497,11 +1523,13 @@ class InferenceBackend:
presence_penalty: float = 0.0,
continue_final_message: bool = False,
tools: Optional[list] = None,
- tool_protocol_active: Optional[bool] = None,
- stop: Optional[list] = None,
+ images: Optional[list] = None,
+ image_ordinal: Optional[int] = None,
enable_thinking: Optional[bool] = None,
reasoning_effort: Optional[str] = None,
preserve_thinking: Optional[bool] = None,
+ tool_protocol_active: Optional[bool] = None,
+ stop: Optional[list] = None,
) -> Generator[str, None, None]:
"""Handle vision model generation with true token-by-token streaming."""
# Reset so a failed or uncountable run cannot surface stale stats.
@@ -1524,14 +1552,22 @@ class InferenceBackend:
trailing_assistant_text,
vlm_prompt_issue,
)
+ from core.inference.mcp_images import (
+ image_marker_parts,
+ pixels_in_marker_order,
+ top_up_image_markers,
+ )
+ # History first, the attachment last: the pixels bind to the markers in
+ # order, and the attachment's marker sits on the newest user turn.
+ attached = list(images or []) + ([image] if image is not None else [])
user_message = last_user_text(messages)
continue_partial = trailing_assistant_text(messages) if continue_final_message else None
if not user_message:
- user_message = "Describe this image." if image else "Hello"
+ user_message = "Describe this image." if attached else "Hello"
- if image:
+ if attached:
has_tool_history = messages_have_tool_history(messages)
# Client-tools route signature: tool_choice="none" and a forced unknown name
# also arrive tools=None, and the catalog alone missed them (#10092).
@@ -1546,6 +1582,16 @@ class InferenceBackend:
fallback_user_text = user_message,
structured_content = True,
)
+ # The helper leaves existing markers alone, which is right for a retry but
+ # not for replayed MCP pictures: those markers are not the attachment's.
+ _prior_markers = image_marker_parts(vision_messages)
+ vision_messages = top_up_image_markers(
+ vision_messages, len(attached), ordinal = image_ordinal
+ )
+ if image is not None:
+ attached = pixels_in_marker_order(
+ vision_messages, _prior_markers, list(images or []), image
+ )
if bool(tools) or has_tool_history or folded_system:
# The conversation the LAST render used, not the no-tools probe's (#10092).
rendered_with: dict = {"messages": vision_messages}
@@ -1644,7 +1690,7 @@ class InferenceBackend:
else:
raise
inputs = processor(
- image,
+ attached[0] if len(attached) == 1 else attached,
input_text,
add_special_tokens = False,
return_tensors = "pt",
@@ -1686,7 +1732,11 @@ class InferenceBackend:
# this request's response protocol. Passing *tools* matches the
# render: a named template selects "tool_use", not "default".
reasoning_channel_markers = detect_reasoning_channel_markers(processor, tools = tools)
- if image
+ # Every attached image, not just a bare attachment: a replay-only
+ # turn renders through this processor too, and resolving it to no
+ # markers suppresses the fallback detection and lets native
+ # reasoning output through as visible answer text.
+ if attached
else None,
reasoning_channel_markers_resolved = True,
prompt = prompt_text,
diff --git a/studio/backend/core/inference/llama_cpp.py b/studio/backend/core/inference/llama_cpp.py
index f8056933562..592fdf37805 100644
--- a/studio/backend/core/inference/llama_cpp.py
+++ b/studio/backend/core/inference/llama_cpp.py
@@ -455,6 +455,12 @@ from core.inference.tool_call_parser import (
)
from core.inference.passthrough_healing import nudge_enabled as _nudge_enabled
from core.inference.repetition_guard import is_repetition_dominated
+from core.inference.mcp_images import (
+ DETACHED_IMAGE_TURN_TEXT as MCP_DETACHED_IMAGE_TURN_TEXT,
+ IMAGE_TURN_TEXT as MCP_IMAGE_TURN_TEXT,
+ append_image_turn as append_mcp_image_turn,
+ mentions_images as mcp_images_mentioned_in,
+)
from core.inference.tool_loop_controller import (
_WORKSPACE_TOOLS,
ToolLoopController,
@@ -33352,6 +33358,17 @@ class LlamaCppBackend:
httpx.Client + auth headers for the stream's lifetime; raises
RuntimeError on a non-200. Shared scaffold for the streaming consumers,
which differ only in how they parse the SSE body."""
+ from core.inference.mcp_images import prepare_image_turn_boundaries
+
+ if "messages" in payload:
+ payload = {
+ **payload,
+ "messages": prepare_image_turn_boundaries(
+ payload["messages"],
+ getattr(self, "_chat_template_override", None)
+ or getattr(self, "_chat_template", None),
+ ),
+ }
stream_timeout = httpx.Timeout(connect = 10, read = 0.5, write = 10, pool = 10)
with httpx.Client(
timeout = stream_timeout,
@@ -34264,6 +34281,7 @@ class LlamaCppBackend:
self,
messages: list[dict],
tools: list[dict],
+ replayed_image_parts: "tuple" = (),
temperature: float = 0.6,
top_p: float = 0.95,
top_k: int = 20,
@@ -34805,6 +34823,13 @@ class LlamaCppBackend:
_continuation_credits = 0
_MAX_CONTINUATION_CREDITS = _MAX_LENGTH_CONTINUATIONS * max(1, max_tool_iterations)
iteration = -1
+ # Outside the loop: the cap is across the whole run, and a per-iteration list
+ # would only ever see the current batch, leaving every earlier iteration's
+ # images in the conversation untrimmed.
+ # Seeded with what promotion already put in the conversation, not empty. The cap
+ # is across the CONVERSATION, so a resumed chat whose history already carries the
+ # allowance would otherwise get a second one for this run and send both.
+ loop_mcp_image_parts: list = list(replayed_image_parts)
while True:
iteration += 1
# Here rather than at each append: six sites grow the conversation and all of
@@ -36468,6 +36493,11 @@ class LlamaCppBackend:
# Which tools those no-ops were about, so the flush below can tell
# whether the trailing result belongs to the same tool.
deferred_noop_tools: set = set()
+ # Per result, so a parallel batch is not squeezed into one
+ # result's worth of images.
+ batch_mcp_images: list = []
+ # Where this batch's results start, for the image turn\'s wording.
+ batch_conversation_start = len(conversation)
# The text-path provisional card uses the parser's default id ("call_0");
# a Mistral-style call carries its own id and would open a duplicate. Reuse
@@ -37293,6 +37323,15 @@ class LlamaCppBackend:
_forced_choice_resolved = True
yield completion.tool_end_event()
conversation.append(completion.tool_message())
+ # Parsed only for a result that carries them and a model that will
+ # be shown them; the marker test first, so a text result never asks.
+ _completion_images = (
+ completion.mcp_images()
+ if mcp_images_mentioned_in(completion.result or "") and self.is_vision
+ else []
+ )
+ if _completion_images:
+ batch_mcp_images.append(_completion_images)
if _compact_after_execution and decision.tool_call_id:
# The promise the gate made when it let this run. Applied here
# rather than on the next pass because the next pass may not
@@ -37439,6 +37478,26 @@ class LlamaCppBackend:
conversation.append({"role": "user", "content": _limit_text})
_limit_notice_in_user_turn = True
+ if batch_mcp_images and self.is_vision:
+ # One block after the whole batch. With a single result "the tool
+ # call above" is exact; with several it names whichever ran last,
+ # which may have returned no picture at all, so the block says so
+ # instead -- the external loop's rule.
+ _batch_results = sum(
+ 1
+ for m in conversation[batch_conversation_start:]
+ if isinstance(m, dict) and m.get("role") == "tool"
+ )
+ append_mcp_image_turn(
+ conversation,
+ batch_mcp_images,
+ per_result = True,
+ owned = loop_mcp_image_parts,
+ lead = MCP_DETACHED_IMAGE_TURN_TEXT
+ if _batch_results != 1
+ else MCP_IMAGE_TURN_TEXT,
+ )
+
# Close provisional cards not resolved by execution/no-op handling.
for _pid, _pname in provisional_started_tool_calls.items():
if _pid not in resolved_provisional_tool_call_ids:
@@ -38448,7 +38507,13 @@ class LlamaCppBackend:
return len(tokens)
# 1. Try /apply-template to render the real chat prompt.
- template_messages = list(messages) if messages else []
+ from core.inference.mcp_images import prepare_image_turn_boundaries
+
+ template_messages = prepare_image_turn_boundaries(
+ list(messages) if messages else [],
+ getattr(self, "_chat_template_override", None)
+ or getattr(self, "_chat_template", None),
+ )
if system_text:
template_messages = [
{"role": "system", "content": system_text}
diff --git a/studio/backend/core/inference/mcp_client.py b/studio/backend/core/inference/mcp_client.py
index 2c1f7b76df6..bda78139472 100644
--- a/studio/backend/core/inference/mcp_client.py
+++ b/studio/backend/core/inference/mcp_client.py
@@ -35,6 +35,8 @@ from utils.account_context import (
is_owner_context,
)
+from core.inference import mcp_images
+
logger = get_logger(__name__)
MCP_TOOL_PREFIX = "mcp__"
@@ -1518,7 +1520,7 @@ def invalidate_tool_cache(server_id: Optional[str] = None) -> None:
_probe_cooloff_until.pop(_account_key(server_id), None)
-MCP_IMAGES_SENTINEL = "__MCP_IMAGES__:"
+MCP_IMAGES_SENTINEL = mcp_images.SENTINEL
MAX_IMAGE_PAYLOAD_CHARS = 12_000_000
@@ -1739,7 +1741,7 @@ def _flatten_result(result: Any) -> str:
notes = []
if images:
n = len(images)
- notes.append(f"{n} image{'s' if n > 1 else ''} attached; displayed to the user")
+ notes.append(f"{n} image{'s' if n > 1 else ''} returned")
if omitted:
notes.append(f"{omitted} image{'s' if omitted > 1 else ''} omitted (too large)")
notes.extend(unshown)
diff --git a/studio/backend/core/inference/mcp_images.py b/studio/backend/core/inference/mcp_images.py
new file mode 100644
index 00000000000..5d072d960c2
--- /dev/null
+++ b/studio/backend/core/inference/mcp_images.py
@@ -0,0 +1,1244 @@
+# SPDX-License-Identifier: AGPL-3.0-only
+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
+
+from __future__ import annotations
+
+import base64
+import binascii
+import io
+import json
+import re
+from typing import Any, Sequence
+
+from loggers import get_logger
+
+logger = get_logger(__name__)
+
+SENTINEL = "__MCP_IMAGES__:"
+# Stamped by mcp_client on every tool it registers; the provenance the envelope
+# is trusted on, on the replay side as well as the live one.
+MCP_TOOL_PREFIX = "mcp__"
+IMAGE_TURN_TEXT = "Images returned by the tool call above:"
+# The same pictures where the turn cannot sit beside the result that produced them.
+# The local client-tool passthrough flattens every content part to text before the
+# markers are rebuilt, so they come back as one block rather than at the positions
+# they were taken from, and "above" would name whatever turn happens to precede it.
+DETACHED_IMAGE_TURN_TEXT = "Images returned by earlier tool calls in this conversation:"
+
+MAX_MODEL_IMAGES = 4
+MAX_TOTAL_MODEL_IMAGES = 8
+# Safetensors and MLX require one image per message, including promoted tool
+# batches. GGUF uses image_url parts and is exempt from this marker-path cap.
+LOCAL_MAX_IMAGES_PER_TURN = 1
+# Candidates carried past the cap when choosing what to decode, because which
+# entries a decoder accepts is not known until it has tried. Mirrors
+# DECODE_FAILURE_ALLOWANCE in studio/frontend/src/features/chat/api/mcp-images.ts.
+DECODE_FAILURE_ALLOWANCE = 4
+# The entry's other field. A token subtype has no length bound, and an entry's
+# metadata must not be where megabytes hide from the byte budgets on both sides.
+# Mirrors MAX_MCP_IMAGE_MIME_CHARS in studio/frontend/src/features/chat/api/mcp-images.ts.
+MAX_MCP_IMAGE_MIME_CHARS = 256
+MAX_IMAGE_EDGE = 1024
+# A PNG stays small while its raster does not: 12 MB of encoded payload can hold
+# tens of gigapixels. Bounded off the header, before a pixel is allocated.
+MAX_IMAGE_PIXELS = 40_000_000
+
+
+def split_images(result: str) -> tuple[str, list[dict]]:
+ """Validated, so tool text that merely mentions the marker is not truncated."""
+ head, sep, payload = result.rpartition("\n" + SENTINEL)
+ if not sep:
+ return result, []
+ try:
+ images = json.loads(payload)
+ except (ValueError, RecursionError):
+ return result, []
+ if not isinstance(images, list) or not images:
+ return result, []
+ if not all(_is_image(image) for image in images):
+ return result, []
+ return head.rstrip(), images
+
+
+def _is_image(image: Any) -> bool:
+ return (
+ isinstance(image, dict)
+ and isinstance(image.get("data"), str)
+ and isinstance(image.get("mimeType"), str)
+ )
+
+
+# Deny known non-image markup. An image-format allowlist could miss formats
+# Pillow supports and under-reserve KV for images promotion actually sends;
+# false positives here only over-reserve.
+_UNDECODABLE_PREFIXES = (
+ b"<svg",
+ b"<?xml",
+ b"<!DOCTYPE",
+ b"<html",
+ b"{",
+ b"[",
+)
+
+
+def _mime_is_bounded(image: Any) -> bool:
+ """Metadata is not where megabytes may hide from the byte budgets on either side;
+ an entry whose mimeType runs past the bound is not a picture this path sends."""
+ mime = image.get("mimeType") if isinstance(image, dict) else None
+ return not (isinstance(mime, str) and len(mime) > MAX_MCP_IMAGE_MIME_CHARS)
+
+
+def probably_decodable(image: Any) -> bool:
+ """Whether this entry could become a picture.
+
+ Header sniff only: admission runs per request and must not decode rasters to
+ price them. Answers True unless the payload is plainly not an image, so a
+ format Pillow can open is never charged nothing.
+ """
+ data = image.get("data") if isinstance(image, dict) else None
+ if not isinstance(data, str) or not data:
+ return False
+ if not _mime_is_bounded(image):
+ return False
+ try:
+ head = base64.b64decode(data[:32], validate = False)
+ except (binascii.Error, ValueError, TypeError):
+ return False
+ if not head:
+ return False
+ return not head.lstrip()[:16].startswith(_UNDECODABLE_PREFIXES)
+
+
+def count_probably_decodable(images: Sequence[dict]) -> int:
+ return sum(1 for image in images if probably_decodable(image))
+
+
+def text_before_envelope(result: str) -> str:
+ """The text with a trailing envelope cut off, WITHOUT parsing it -- for display
+ only. split_images json-loads the whole array to decide whether the suffix is a
+ real envelope; a monitor row built on the event loop cannot afford that on 12 MB,
+ and showing a little less of a malformed suffix costs nothing."""
+ index = result.rfind("\n" + SENTINEL)
+ return result if index == -1 else result[:index]
+
+
+def has_images(result: str) -> bool:
+ return bool(split_images(result)[1])
+
+
+def mentions_images(result: str) -> bool:
+ """Substring only, for deciding WHERE to run: has_images json-parses the whole
+ array, and doing that on the event loop to decide whether to leave the event
+ loop parsed a permitted 12 MB envelope right there. A false positive here costs
+ a thread hop; the worker validates for real."""
+ return ("\n" + SENTINEL) in result
+
+
+def _decoded_urls(
+ images: Sequence[dict],
+ limit: int = MAX_MODEL_IMAGES,
+ attempts: "list | None" = None,
+ cache: "dict | None" = None,
+) -> list[str]:
+ """Up to *limit* data URLs, counting only what decoded.
+
+ Slicing first would spend the quota on formats Pillow cannot read -- an SVG
+ _flatten_result accepted, say -- and drop the real PNGs behind them. So the
+ ATTEMPTS are bounded instead: the payload-byte budget caps the size of a result,
+ never the number of entries in it, and a server answering with thousands of tiny
+ unreadable ones would otherwise hold an inference worker open on Pillow for a
+ turn that can show four pictures at most. Same allowance the replay side spends,
+ for the same reason -- enough slack to look past a run of rejects.
+
+ *attempts* is a one-element budget shared across a parallel batch. Per result it
+ resets, and a 25-call turn of malformed results then buys 25 x 8 decodes of
+ attacker-chosen rasters against a cap of eight pictures.
+ """
+ urls = []
+ own = [limit + DECODE_FAILURE_ALLOWANCE] if attempts is None else attempts
+ for image in images:
+ if len(urls) >= limit or own[0] <= 0:
+ break
+ own[0] -= 1
+ if not _mime_is_bounded(image):
+ continue
+ data = image.get("data", "")
+ # *cache* is one request's decodes, keyed by the payload: a route that
+ # promotes the same history twice must not pay Pillow twice for it.
+ if cache is not None and data in cache:
+ url = cache[data]
+ else:
+ url = _png_data_url(data)
+ if cache is not None:
+ cache[data] = url
+ if url:
+ urls.append(url)
+ return urls
+
+
+def _decoded_urls_per_result(results: Sequence[Sequence[dict]]) -> list[str]:
+ """MAX_MODEL_IMAGES from EACH result, then the conversation cap over the whole.
+
+ A parallel batch arrives as several results concatenated. Applying the
+ per-result quota to the concatenation gives the first result the whole
+ allowance and delivers none of the second, though the conversation cap has
+ room for both.
+
+ Filled from the NEWEST result back when the batch cannot fit. Everything else
+ that bounds these pictures keeps the newest -- the conversation trim, and the
+ frontend's replay bound -- so filling from the front would show this turn
+ results 1 and 2 and the next turn results 2 and 3, off the same history.
+ """
+ chosen: list[list[str]] = []
+ room = MAX_TOTAL_MODEL_IMAGES
+ # One budget for the whole batch, not one per result: room only falls on a
+ # SUCCESSFUL decode, so results that fail late in Pillow never close the loop
+ # and a parallel turn could pay for the allowance again on every call it made.
+ attempts = [MAX_TOTAL_MODEL_IMAGES + DECODE_FAILURE_ALLOWANCE]
+ for images in reversed(results):
+ if room <= 0 or attempts[0] <= 0:
+ break
+ urls = _decoded_urls(images, min(MAX_MODEL_IMAGES, room), attempts = attempts)
+ room -= len(urls)
+ chosen.append(urls)
+ # Back into document order: the parts are positional and a batch's own results
+ # must still read in the order the model made the calls.
+ return [url for urls in reversed(chosen) for url in urls]
+
+
+def eligible_replay_images(
+ messages: Sequence[dict],
+ *,
+ local: bool = False,
+ budget: int = MAX_TOTAL_MODEL_IMAGES,
+) -> dict:
+ """Which envelope entries can still be in the prompt once the cap has run.
+
+ The decoders are the expensive part -- a permitted raster is 40 megapixels and
+ a few hundred bytes of base64 can ask for one -- and until now every envelope in
+ a replayed history was decoded before the eight-image trim threw nearly all of
+ it away. A caller that posts its own history therefore chose how much Pillow
+ work one bounded request did. This picks the survivors first, from the message
+ list alone, so the count of decodes is a property of the cap rather than of the
+ history.
+
+ Keyed by position in *messages*; the value is how many leading entries of that
+ result may be decoded. Mirrors the frontend's boundMcpImageEnvelopes, spare
+ allowance included: this side cannot know which entries decode either, so a
+ result whose images all fail must not strand the valid ones behind it.
+
+ *local* budgets the way the marker paths spend: a tool batch -- consecutive
+ results -- lands as one turn carrying one picture, so the batch is charged once
+ and its further candidates are decode fallbacks, not a charge. Budgeting four per
+ result here spent the conversation's allowance on pictures the local path never
+ sends and dropped older batches that still had room.
+ """
+ eligible: dict = {}
+ # Keep fallback candidates because header-only selection cannot know which
+ # images decode. With no image budget, failures cannot free room for spares.
+ spare = DECODE_FAILURE_ALLOWANCE if budget > 0 else 0
+ per_result = LOCAL_MAX_IMAGES_PER_TURN if local else MAX_MODEL_IMAGES
+ # Use promotion's provenance rules so unnamed non-MCP results cannot consume
+ # the replay allowance ahead of genuine MCP results.
+ call_names = resolve_tool_names(messages)
+
+ def _is_tool(position: int) -> bool:
+ message = messages[position]
+ return isinstance(message, dict) and message.get("role") == "tool"
+
+ def _mcp_images_at(position: int) -> "list | None":
+ content = messages[position].get("content")
+ if not isinstance(content, str):
+ return None
+ name = messages[position].get("name") or call_names.get(position)
+ if isinstance(name, str) and name and not name.startswith(MCP_TOOL_PREFIX):
+ return None
+ _text, images = split_images(content)
+ return images or None
+
+ index = len(messages) - 1
+ while index >= 0:
+ if not _is_tool(index):
+ index -= 1
+ continue
+ start = index
+ if local:
+ while start > 0 and _is_tool(start - 1):
+ start -= 1
+ # Newest first within the batch, the order the local decoder tries them.
+ batch = [position for position in range(index, start - 1, -1)]
+ index = start - 1
+ room = min(budget, per_result)
+ allowance = room + spare
+ taken = 0
+ for position in batch:
+ images = _mcp_images_at(position)
+ if images is None:
+ continue
+ take = min(len(images), allowance)
+ if take <= 0:
+ eligible[position] = 0
+ continue
+ eligible[position] = take
+ allowance -= take
+ taken += take
+ charged = min(taken, room)
+ budget -= charged
+ spare -= taken - charged
+ return eligible
+
+
+def content_parts(images: Sequence[dict]) -> list[dict]:
+ return [{"type": "image_url", "image_url": {"url": url}} for url in _decoded_urls(images)]
+
+
+def content_parts_per_result(results: Sequence[Sequence[dict]]) -> list[dict]:
+ """content_parts for a batch, keeping each result's own quota."""
+ return [
+ {"type": "image_url", "image_url": {"url": url}}
+ for url in _decoded_urls_per_result(results)
+ ]
+
+
+def png_payloads_per_result(
+ results: Sequence[Sequence[dict]], cache: "dict | None" = None
+) -> list[str]:
+ """For the local marker paths: at most LOCAL_MAX_IMAGES_PER_TURN pictures, taken
+ from the NEWEST result that decodes, since a batch lands as one turn and a
+ non-GGUF message takes one image."""
+ # One attempt budget across the batch, as _decoded_urls_per_result keeps: reset
+ # per result, a 25-call turn of malformed results bought 25 allowances of Pillow
+ # decodes for a path that keeps a single picture.
+ attempts = [LOCAL_MAX_IMAGES_PER_TURN + DECODE_FAILURE_ALLOWANCE]
+ for images in reversed(list(results)):
+ if attempts[0] <= 0:
+ break
+ urls = _decoded_urls(images, LOCAL_MAX_IMAGES_PER_TURN, attempts = attempts, cache = cache)
+ if urls:
+ return [url.split(",", 1)[1] for url in urls]
+ return []
+
+
+def flattened_rgb(image):
+ """RGB with any transparency composited onto white, not simply dropped.
+
+ ``convert("RGB")`` keeps whatever colour sits UNDER the alpha, and a tool that
+ never painted a background leaves that black -- so a transparent screenshot's
+ dark text or line art converts to black on black and the model is handed a
+ blank rectangle. Only images that actually carry alpha take the composite.
+ """
+ from PIL import Image
+
+ # Match routes/inference.py's _image_bytes_to_png_b64: scale declared I;16
+ # values to 8-bit before RGB conversion clips them. I;16B/I;16L must pass
+ # through "I" because they reject point().
+ if image.mode.startswith("I;16"):
+ if image.mode != "I;16":
+ image = image.convert("I")
+ image = image.point(lambda v: v * (1.0 / 257), mode = "L")
+ has_alpha = image.mode in ("RGBA", "LA", "PA") or (
+ image.mode == "P" and "transparency" in image.info
+ )
+ if not has_alpha:
+ return image.convert("RGB")
+ rgba = image.convert("RGBA")
+ canvas = Image.new("RGB", rgba.size, (255, 255, 255))
+ canvas.paste(rgba, mask = rgba.getchannel("A"))
+ return canvas
+
+
+def _png_data_url(data: str) -> str | None:
+ # PNG regardless of what the server sent: llama-server's stb_image reads only
+ # a few formats, and MCP servers commonly answer with WebP.
+ try:
+ raw = base64.b64decode(data, validate = True)
+ except (binascii.Error, ValueError, TypeError):
+ logger.debug("MCP image payload is not base64")
+ return None
+ try:
+ from PIL import Image
+
+ # open() parses the header only, so the size is known before the raster
+ # exists. load() below is what allocates it.
+ image = Image.open(io.BytesIO(raw))
+ width, height = image.size
+ if width * height > MAX_IMAGE_PIXELS:
+ logger.debug("MCP image is %dx%d, past the pixel budget", width, height)
+ return None
+ # JPEG decodes straight to a smaller raster; a no-op for every other format.
+ image.draft("RGB", (MAX_IMAGE_EDGE, MAX_IMAGE_EDGE))
+ image.load()
+ # A camera JPEG carries its display orientation in EXIF; re-encoded as PNG
+ # without applying it, the model was shown the picture sideways.
+ from PIL import ImageOps
+
+ image = ImageOps.exif_transpose(image) or image
+ if max(image.size) > MAX_IMAGE_EDGE:
+ image.thumbnail((MAX_IMAGE_EDGE, MAX_IMAGE_EDGE), Image.Resampling.LANCZOS)
+ buffer = io.BytesIO()
+ flattened_rgb(image).save(buffer, format = "PNG")
+ except Exception:
+ logger.debug("MCP image could not be decoded", exc_info = True)
+ return None
+ return "data:image/png;base64," + base64.b64encode(buffer.getvalue()).decode("ascii")
+
+
+def png_payloads(images: Sequence[dict]) -> list[str]:
+ """Normalized PNG base64, for backends that take images as objects rather
+ than as data URLs inside the prompt."""
+ return [url.split(",", 1)[1] for url in _decoded_urls(images)]
+
+
+def _turn_text(
+ shown: int,
+ total: int,
+ lead: str = IMAGE_TURN_TEXT,
+) -> str:
+ # The tool result's own note counts every image it returned, so a turn that
+ # carries fewer has to say so rather than let the model wait for the rest.
+ if total > shown:
+ # Report counts: batches keep newest results and trims remove oldest parts,
+ # so a positional label could identify images the model never saw.
+ return f"{lead} ({shown} of {total})"
+ return lead
+
+
+def placeholder_turn(
+ count: int,
+ total: "int | None" = None,
+ lead: str = IMAGE_TURN_TEXT,
+) -> dict:
+ """The user turn a local processor renders: ``{"type": "image"}`` markers the
+ template turns into image tokens, with the pixels passed alongside."""
+ return {
+ "role": "user",
+ "content": [
+ *({"type": "image"} for _ in range(count)),
+ {
+ "type": "text",
+ "text": _turn_text(count, count if total is None else total, lead),
+ },
+ ],
+ }
+
+
+def _relabelled(
+ kept: list,
+ part_type: str,
+ original: int,
+ owned: "set | None" = None,
+) -> list:
+ """The turn's note rewritten for what actually survived a partial trim.
+
+ The label exists to tell the model which of the returned images it was really
+ shown, so a turn left holding two while still saying it carries four defeats it.
+ *owned* names the promoted parts; the caller's own attachments in the same turn
+ are not the tool's and are not counted into its note.
+ """
+ remaining = sum(
+ 1
+ for part in kept
+ if isinstance(part, dict)
+ and part.get("type") == part_type
+ and (owned is None or id(part) in owned)
+ )
+ out = []
+ for part in kept:
+ if (
+ isinstance(part, dict)
+ and part.get("type") == "text"
+ and _is_image_turn_note(part.get("text"))
+ ):
+ lead = (
+ DETACHED_IMAGE_TURN_TEXT
+ if str(part.get("text", "")).startswith(DETACHED_IMAGE_TURN_TEXT)
+ else IMAGE_TURN_TEXT
+ )
+ total = _note_total(part.get("text"), original)
+ out.append({**part, "text": _turn_text(remaining, total, lead)})
+ continue
+ out.append(part)
+ return out
+
+
+def _note_total(text, original: int) -> int:
+ """How many the tool returned, for a turn that no longer carries them all.
+
+ The note's own "of N" wins, so a second trim does not re-baseline to whatever is
+ left. A turn that arrived COMPLETE carries no such suffix, and falling back to
+ the post-trim count there told the model nothing had been dropped -- the one
+ thing the note exists to say. Fall back to what the turn held before this trim.
+ """
+ match = re.search(r"\((?:first )?\d+ of (\d+)\)\s*$", str(text or ""))
+ if match:
+ return int(match.group(1))
+ return original
+
+
+def _is_image_turn_note(text) -> bool:
+ """Either lead the placeholder turn can carry, so a turn whose last picture went
+ is recognised as having nothing left to say on both paths."""
+ value = str(text or "")
+ return value.startswith(IMAGE_TURN_TEXT) or value.startswith(DETACHED_IMAGE_TURN_TEXT)
+
+
+def _image_parts(conversation: Sequence[dict], part_type: str):
+ for message in conversation:
+ content = message.get("content")
+ if isinstance(content, list):
+ yield from (
+ part for part in content if isinstance(part, dict) and part.get("type") == part_type
+ )
+
+
+def _all_image_url_parts(conversation: Sequence[dict]) -> list:
+ return list(_image_parts(conversation, "image_url"))
+
+
+def count_image_parts(conversation: Sequence[dict], part_type: str) -> int:
+ return sum(1 for _ in _image_parts(conversation, part_type))
+
+
+def _drop_oldest_image_parts(
+ conversation: list,
+ excess: int,
+ part_type: str,
+ only: "list | None" = None,
+) -> None:
+ """Drop the *excess* oldest image parts, and any turn they emptied.
+
+ With *only*, a list of the exact part objects promotion created, nothing else
+ is touched: a caller that attaches more than the cap keeps every one of its own
+ images, which admission has already charged for.
+ """
+ owned = {id(part) for part in only} if only is not None else None
+ ordinals = set()
+ for ordinal, part in enumerate(_image_parts(conversation, part_type)):
+ if len(ordinals) >= excess:
+ break
+ if owned is None or id(part) in owned:
+ ordinals.add(ordinal)
+ _drop_image_parts_at(conversation, ordinals, part_type, owned = owned)
+
+
+def _drop_image_parts_at(
+ conversation: list,
+ ordinals: set,
+ part_type: str,
+ *,
+ owned: "set | None" = None,
+) -> None:
+ """Drop the image parts at the given positions in document order.
+
+ Positional rather than oldest-first: a protected pixel can sit anywhere in the
+ sink, because the attachment's marker belongs to the turn that supplied it and
+ that turn can precede a tool's pictures. Dropping by count would take the wrong
+ marker and hand every later pixel to the one before it.
+ """
+ if not ordinals:
+ return
+ seen = 0
+ drained = []
+ for index, message in enumerate(conversation):
+ content = message.get("content")
+ if not isinstance(content, list):
+ continue
+ original = sum(
+ 1
+ for part in content
+ if isinstance(part, dict)
+ and part.get("type") == part_type
+ and (owned is None or id(part) in owned)
+ )
+ kept = []
+ for part in content:
+ if isinstance(part, dict) and part.get("type") == part_type:
+ ordinal = seen
+ seen += 1
+ if ordinal in ordinals:
+ continue
+ kept.append(part)
+ if len(kept) == len(content):
+ continue
+ if not kept or (
+ len(kept) == 1
+ and kept[0].get("type") == "text"
+ and _is_image_turn_note(kept[0].get("text"))
+ ):
+ drained.append(index)
+ else:
+ conversation[index] = {
+ **message,
+ "content": _relabelled(kept, part_type, original, owned = owned),
+ }
+ for index in reversed(drained):
+ del conversation[index]
+
+
+def trim_image_turns(
+ conversation: list,
+ payloads: list,
+ limit: int = MAX_TOTAL_MODEL_IMAGES,
+ keep: "Sequence[int] | None" = None,
+) -> tuple:
+ """Keep the newest *limit* pictures: a loop that keeps calling an image tool
+ otherwise re-sends every one it has seen. Markers and their own pixels go
+ together, or the processor counts image tokens it was given none for.
+
+ *keep* holds indexes into *payloads* that must survive -- the caller's own
+ attachment, which this cap has no business evicting: it is about what a tool
+ loop RE-SENDS, and dropping the picture the question was asked about answers a
+ different question. Bare markers carry no identity the way promoted
+ ``image_url`` parts do, so the position in the sink is what names them.
+
+ They still COUNT against *limit*. Exempting them would let a resumed chat's
+ replayed pictures carry a second full allowance and put twice the cap in the
+ prompt; the route reserves the attachment's slot by trimming replay to
+ ``limit - 1`` before interleaving it, which is where that reservation belongs.
+
+ Returns *keep* rebased onto the payload list this call leaves behind.
+ """
+ protected = sorted(index for index in (keep or ()) if 0 <= index < len(payloads))
+ excess = len(payloads) - limit
+ if excess <= 0:
+ return tuple(protected)
+ drop = [index for index in range(len(payloads)) if index not in set(protected)][:excess]
+ if not drop:
+ return tuple(protected)
+ _drop_image_parts_at(conversation, set(drop), "image")
+ for index in reversed(drop):
+ del payloads[index]
+ # Rebase protected positions after deletion; callers reuse them on later
+ # trims, where stale indices could leave the attachment unprotected.
+ dropped = set(drop)
+ return tuple(index - sum(1 for gone in dropped if gone < index) for index in protected)
+
+
+def trim_image_url_turns(
+ conversation: list,
+ limit: int = MAX_TOTAL_MODEL_IMAGES,
+ only: "list | None" = None,
+) -> None:
+ """The same cap where the pixels ride in the prompt as data URLs.
+
+ GGUF and the external providers carry no separate payload list, so the parts
+ themselves are the budget: without this a screenshot loop resends every
+ picture it has ever taken on every later turn.
+
+ *only* scopes the cap to the parts promotion created. This cap is about what
+ REPLAY re-sends; a caller's own attachments are not counted against it and are
+ never deleted by it.
+ """
+ if only is not None:
+ # Prune before counting: the rolling-context fitter may have evicted whole
+ # turns, and stale parts would cause surviving images to be over-trimmed.
+ present = {id(part) for part in _all_image_url_parts(conversation)}
+ only[:] = [part for part in only if id(part) in present]
+ counted = len(only) if only is not None else count_image_parts(conversation, "image_url")
+ excess = counted - limit
+ if excess <= 0:
+ return
+ _drop_oldest_image_parts(conversation, excess, "image_url", only = only)
+ if only is not None:
+ # Drop what the trim removed, or the next call counts parts that are no
+ # longer in the conversation and cuts far more than the cap asks for.
+ still_present = {id(part) for part in _all_image_url_parts(conversation)}
+ only[:] = [part for part in only if id(part) in still_present]
+
+
+def _merge_into_trailing_user_turn(conversation: list, parts: list[dict]) -> bool:
+ """Fold *parts* into a trailing ``role=user`` turn, if there is one.
+
+ A deferred no-op nudge lands as exactly such a turn, and appending after it
+ would put two user messages in a row -- which a strict VLM template rejects.
+ """
+ last = conversation[-1] if conversation else None
+ if not isinstance(last, dict) or last.get("role") != "user":
+ return False
+ content = last.get("content")
+ own = list(content) if isinstance(content, list) else [{"type": "text", "text": content or ""}]
+ conversation[-1] = {**last, "content": [*own, *parts]}
+ return True
+
+
+def append_image_turn(
+ conversation: list,
+ images: Sequence,
+ *,
+ limit: "int | None" = MAX_TOTAL_MODEL_IMAGES,
+ per_result: bool = False,
+ owned: "list | None" = None,
+ reserve_caller_images: bool = False,
+ returned: "int | None" = None,
+ lead: str = IMAGE_TURN_TEXT,
+) -> None:
+ """A user turn, not the ``role=tool`` result they came with: tool messages take
+ no image parts, and local templates render tool content as a string.
+
+ With *per_result*, *images* is a list of per-result lists and each keeps its own
+ MAX_MODEL_IMAGES quota; flattened first, a parallel batch would spend the whole
+ allowance on its first call.
+ """
+ parts = content_parts_per_result(images) if per_result else content_parts(images)
+ if not parts:
+ return
+ # What the TOOL returned. The caller passes it when the candidates it hands over
+ # have already been sliced by admission, since counting those would describe the
+ # admission pass rather than the result the note sits beside.
+ total = (
+ returned
+ if returned is not None
+ else (sum(len(result) for result in images) if per_result else len(images))
+ )
+ if owned is not None:
+ # Everything this loop has appended, across turns. The cap counts what the
+ # tools returned and never a caller's own attachments, which admission has
+ # already charged for and which are not this cap's business.
+ owned.extend(parts)
+ # The note rides along on the merge too: merged bare into a deferred no-op nudge,
+ # the pictures read as attachments to that nudge rather than as the tool's output,
+ # and the next turn misattributes them.
+ note = {"type": "text", "text": _turn_text(len(parts), total, lead)}
+ if not _merge_into_trailing_user_turn(conversation, [*parts, note]):
+ conversation.append(
+ {
+ "role": "user",
+ "content": [
+ {"type": "text", "text": _turn_text(len(parts), total, lead)},
+ *parts,
+ ],
+ }
+ )
+ if limit is not None:
+ if reserve_caller_images:
+ # Remote providers cap all images in document order, so reserve attachment
+ # slots or their cap may drop the newest tool result. Replay does the same.
+ # This is opt-in: local loops use a context window instead of a provider cap.
+ limit = max(0, limit - (len(_all_image_url_parts(conversation)) - len(owned or ())))
+ trim_image_url_turns(conversation, limit, only = owned)
+
+
+def insert_placeholder_turn(
+ conversation: list,
+ index: int,
+ count: int,
+ total: "int | None" = None,
+ lead: str = IMAGE_TURN_TEXT,
+) -> None:
+ """A marker-only user turn placed *before* ``index``.
+
+ The pixels arrive history-first with the current attachment last, and a
+ positional VLM processor binds them to markers in document order. So the
+ replayed markers have to sit ahead of the turn that owns the attachment,
+ not after it.
+ """
+ if count > 0:
+ conversation.insert(index, placeholder_turn(count, total, lead))
+
+
+def append_placeholder_turn(
+ conversation: list,
+ count: int,
+ total: "int | None" = None,
+ lead: str = IMAGE_TURN_TEXT,
+) -> None:
+ """The marker-only form of the above, for backends taking pixels alongside.
+
+ The note rides along on the merge as well. Merged bare into a deferred no-op
+ nudge, the markers read as pictures attached to that nudge rather than as the
+ tool's output, and the next turn misattributes them; merged into a real question
+ they read as ones the user sent.
+ """
+ markers = [{"type": "image"} for _ in range(count)]
+ if not markers:
+ return
+ note = {"type": "text", "text": _turn_text(count, count if total is None else total, lead)}
+ if not _merge_into_trailing_user_turn(conversation, [*markers, note]):
+ conversation.append(placeholder_turn(count, total, lead))
+
+
+def _without_replay_parts(content: list) -> list:
+ """The turn without a replay's marker AND its note. A non-GGUF message takes one
+ image, so the attachment displaces a replay marker merged into its turn; the
+ note that came with it would then say the caller's own picture was the tool's."""
+ return [
+ part
+ for part in content
+ if not (
+ isinstance(part, dict)
+ and (
+ part.get("type") == "image"
+ or (part.get("type") == "text" and _is_image_turn_note(part.get("text")))
+ )
+ )
+ ]
+
+
+def top_up_image_markers(
+ messages: Sequence[dict],
+ total: int,
+ *,
+ ordinal: "int | None" = None,
+) -> list[dict]:
+ """Give the conversation exactly *total* image markers, adding any shortfall
+ to the newest user turn.
+
+ ``messages_with_attached_image`` leaves a conversation that already carries
+ markers alone, which is right for a nudge retry but wrong for replayed MCP
+ pictures: those markers belong to earlier turns, not to the attachment. The
+ top-up goes last, matching where the attachment sits in the pixel list.
+ """
+ out = list(messages)
+ have = sum(
+ 1
+ for message in out
+ if isinstance(message, dict) and isinstance(message.get("content"), list)
+ for part in message["content"]
+ if isinstance(part, dict) and part.get("type") in ("image", "image_url", "input_image")
+ )
+ missing = total - have
+ if missing <= 0:
+ return out
+ markers = [{"type": "image"} for _ in range(missing)]
+ if ordinal is not None:
+ # The turn that supplied the attachment, which need not be the newest: a
+ # later text-only question would otherwise be shown as carrying it.
+ seen = 0
+ for index, message in enumerate(out):
+ if not isinstance(message, dict) or message.get("role") != "user":
+ continue
+ if is_synthetic_image_turn(message):
+ continue
+ if seen == ordinal:
+ out[index] = _with_attachment_markers(message, markers, after_text = True)
+ return out
+ seen += 1
+ # The legacy top-level image has no ordinal; use the newest real user turn.
+ # Displace any replay marker there to preserve the non-GGUF one-image limit.
+ candidates = [
+ index
+ for index in range(len(out) - 1, -1, -1)
+ if isinstance(out[index], dict) and out[index].get("role") == "user"
+ ]
+ real = [index for index in candidates if not is_synthetic_image_turn(out[index])]
+ for index in real or candidates:
+ out[index] = _with_attachment_markers(out[index], markers, after_text = True)
+ break
+ return out
+
+
+def image_marker_parts(conversation: Sequence[dict]) -> list:
+ """Every ``{"type": "image"}`` marker part, in document order."""
+ return list(_image_parts(conversation, "image"))
+
+
+def pixels_in_marker_order(
+ conversation: Sequence[dict],
+ prior_markers: Sequence[dict],
+ prior_payloads: Sequence,
+ new_payload,
+ placed_at: "list | None" = None,
+) -> list:
+ """The pixel list ordered the way its markers actually appear.
+
+ A positional VLM binds the Nth pixel to the Nth marker, so "history first,
+ attachment last" is only right when the attachment's turn is last. It is not
+ when the attachment came with an earlier question and a tool returned pictures
+ after it, so the order is read off the conversation rather than assumed.
+ """
+ # Each history payload rides with ITS marker, by identity. Popping from the front
+ # let a displaced marker's payload slide onto the marker after it, which is the
+ # off-by-one the whole positional scheme exists to prevent.
+ by_marker = {id(part): payload for part, payload in zip(prior_markers, prior_payloads)}
+ ordered = []
+ placed_new = False
+ for part in image_marker_parts(conversation):
+ # A marker that predates the top-up belongs to history when history has a
+ # pixel for it. A pre-existing marker with none is the attachment's own --
+ # the client may have marked it before the request ever reached this path.
+ if id(part) in by_marker:
+ ordered.append(by_marker[id(part)])
+ elif not placed_new:
+ # Where the attachment landed, so the loop's cap can leave it alone: it
+ # is not always last, and the sink's positions are all that name it.
+ if placed_at is not None:
+ placed_at.append(len(ordered))
+ ordered.append(new_payload)
+ placed_new = True
+ return ordered
+
+
+def is_synthetic_image_turn(message) -> bool:
+ """Whether promotion inserted this user turn, rather than the caller sending it.
+
+ An ordinal computed against the ORIGINAL history counts only real user turns,
+ so resolving it against the promoted list has to skip the turns promotion added
+ or the attachment's marker lands on a historical picture's turn.
+ """
+ if not isinstance(message, dict) or message.get("role") != "user":
+ return False
+ content = message.get("content")
+ if not isinstance(content, list):
+ return False
+ # Both leads: a detached block is promotion's turn just as much, and counting it
+ # as a real user turn put the attachment's marker on it instead of the question.
+ # ALL of its text, though: a replay merged into the user's question carries the
+ # note beside the question's own text, and that turn is still the user's.
+ texts = [
+ part.get("text")
+ for part in content
+ if isinstance(part, dict) and part.get("type") == "text"
+ ]
+ return bool(texts) and all(_is_image_turn_note(text) for text in texts)
+
+
+def prepare_image_turn_boundaries(messages: list, template: str | None) -> list:
+ """Balance inserted image turns for Ministral's tool-skipping alternation check."""
+ if not isinstance(template, str) or (
+ "conversation roles must alternate user and assistant roles except for tool calls and results."
+ not in template
+ ):
+ return messages
+ out = []
+ expects_user = True
+ changed = False
+ for message in messages:
+ role = message.get("role")
+ if role == "user":
+ if not expects_user and is_synthetic_image_turn(message):
+ out.append({"role": "assistant", "content": "The tool returned image content."})
+ changed = True
+ expects_user = False
+ elif role == "assistant" and not message.get("tool_calls"):
+ expects_user = True
+ out.append(message)
+ return out if changed else messages
+
+
+def _with_attachment_markers(
+ message: dict,
+ markers: list[dict],
+ *,
+ after_text: bool = False,
+) -> dict:
+ """Replace a replay's marker and attribution with the caller's attachment."""
+ content = message.get("content")
+ own = (
+ _without_replay_parts(content)
+ if isinstance(content, list)
+ else [{"type": "text", "text": content or ""}]
+ )
+ parts = [*own, *markers] if after_text and isinstance(content, list) else [*markers, *own]
+ return {**message, "content": parts}
+
+
+def mark_last_user_turn(
+ messages: Sequence[dict],
+ count: int,
+ *,
+ ordinal: "int | None" = None,
+) -> list[dict]:
+ """Mark the user turn carrying ``count`` images, where an attachment belongs.
+
+ *ordinal* names that turn counted among user turns. The newest user turn is
+ only the right guess when the attachment came with the newest question: the
+ extractor takes the newest user image from anywhere in the thread, so a
+ text-only latest turn would otherwise be told it supplied an older picture.
+
+ A replay marker already merged into that turn is displaced: a non-GGUF message
+ takes one image, and the attachment is the one the question is about. The
+ displaced payload drops in pixels_in_marker_order rather than sliding on.
+ """
+ out = list(messages)
+ markers = [{"type": "image"} for _ in range(count)]
+ if ordinal is not None:
+ seen = 0
+ for index, message in enumerate(out):
+ # Promotion inserts its own user turns ahead of this, and the ordinal was
+ # counted before they existed.
+ if message.get("role") != "user" or is_synthetic_image_turn(message):
+ continue
+ if seen == ordinal:
+ out[index] = _with_attachment_markers(message, markers)
+ return out
+ seen += 1
+ for index in range(len(out) - 1, -1, -1):
+ if out[index].get("role") == "user":
+ out[index] = _with_attachment_markers(out[index], markers)
+ break
+ return out
+
+
+def promote_history(
+ messages: Sequence[dict],
+ *,
+ vision: bool,
+ promoted_out: "list | None" = None,
+ reserve_for_caller: bool = False,
+) -> list[dict]:
+ """Rebuild image turns from replayed envelopes. The envelope leaves the tool
+ text either way: a text-only model must not be shown its base64.
+
+ *promoted_out* collects the exact image parts this call created, so a tool loop
+ resuming the conversation can seed its own cap with them instead of starting
+ from zero and letting the history's images through uncounted.
+ """
+ out, _payloads, promoted = _promote(
+ messages, vision, local = False, reserve_for_caller = reserve_for_caller
+ )
+ if promoted_out is not None:
+ promoted_out.extend(promoted)
+ return out
+
+
+def promote_history_local(
+ messages: Sequence[dict],
+ *,
+ vision: bool,
+ decode_cache: "dict | None" = None,
+ caller_images: Sequence = (),
+) -> tuple[list[dict], list[str]]:
+ """The same, for backends that take the pixels beside the prompt: the turns
+ carry markers and the payloads come back with them. ``caller_images`` supplies
+ the pixels for existing markers, which retain their positions and survive trimming."""
+ out, payloads, _promoted = _promote(
+ messages, vision, local = True, decode_cache = decode_cache, caller_images = caller_images
+ )
+ return out, payloads
+
+
+def _field(message, key):
+ return message.get(key) if isinstance(message, dict) else getattr(message, key, None)
+
+
+def resolve_tool_names(messages: Sequence) -> dict:
+ """Which tool each role="tool" message answers, keyed by its POSITION.
+
+ A tool message carries no ``name`` in plain OpenAI (the field is optional) or in
+ anything translated from Anthropic, and an absent name is read as legacy MCP
+ history that may be trusted. Correlating the call is what lets the mcp__ gate
+ run on those wire formats at all.
+
+ Each result is paired with the nearest UNMATCHED preceding call bearing its id,
+ not with a conversation-wide last-wins lookup: the backend itself restarts ids
+ like ``call_0`` every response, so one dict let a later non-MCP ``call_0``
+ rename an earlier MCP result -- or the reverse.
+ """
+ open_calls: dict = {}
+ names: dict = {}
+ for index, message in enumerate(messages or ()):
+ for call in _field(message, "tool_calls") or ():
+ if not isinstance(call, dict):
+ continue
+ function = call.get("function")
+ call_id = call.get("id")
+ if isinstance(function, dict) and isinstance(call_id, str):
+ name = function.get("name")
+ if isinstance(name, str) and name:
+ open_calls.setdefault(call_id, []).append(name)
+ if _field(message, "role") == "tool":
+ call_id = _field(message, "tool_call_id")
+ stack = open_calls.get(call_id) if isinstance(call_id, str) else None
+ if stack:
+ names[index] = stack.pop()
+ return names
+
+
+def _returned_count(images: Sequence[dict]) -> int:
+ """How many the tool returned. An upstream bound may already have shortened the
+ array; it leaves the original length on the first entry when it does."""
+ first = images[0] if images else None
+ stated = first.get("returned") if isinstance(first, dict) else None
+ if isinstance(stated, int) and stated >= len(images):
+ return stated
+ return len(images)
+
+
+def _promote(
+ messages,
+ vision: bool,
+ *,
+ local: bool,
+ reserve_for_caller: bool = False,
+ decode_cache: "dict | None" = None,
+ caller_images: Sequence = (),
+) -> tuple[list[dict], list[str], list[dict]]:
+ out: list[dict] = []
+ caller_payloads = (
+ dict(zip(map(id, image_marker_parts(messages)), caller_images)) if caller_images else {}
+ )
+ # Resolved once for the whole conversation, so the provenance gate below works on
+ # every wire format rather than only the ones that happen to send ``name``.
+ call_names = resolve_tool_names(messages)
+ # Set while an image result waits for its turn and a LATER, image-free result of
+ # the same batch is appended after it: "the tool call above" would then name the
+ # wrong call, so the turn takes the wording that claims no adjacency.
+ interrupted = [False]
+ # Resolved before a single decode runs: the trim at the bottom keeps the newest
+ # eight, and decoding a whole replayed history to throw nearly all of it away is
+ # work a caller's own message list gets to choose the size of.
+ # The caller's room is reserved HERE, ahead of the decodes, not only by the trim
+ # at the bottom: with eight attachments every replay candidate was decoded and
+ # re-encoded -- a permitted raster is 40 megapixels -- to be dropped whole.
+ # Before promotion every image_url part in the list is the caller's own.
+ _reserved = (
+ len(caller_payloads)
+ if local
+ else (len(_all_image_url_parts(messages)) if vision and reserve_for_caller else 0)
+ )
+ eligible = (
+ eligible_replay_images(
+ messages, local = local, budget = max(0, MAX_TOTAL_MODEL_IMAGES - _reserved)
+ )
+ if vision
+ else {}
+ )
+ # One entry per tool result, not flattened: two parallel calls each returning
+ # four images would otherwise share a single result's quota and replay only the
+ # first call's four.
+ pending: list[list[dict]] = []
+ # One per entry in *pending*: how many that result really returned.
+ returned_totals: list[int] = []
+ payloads: list[str] = []
+ # The exact part objects promotion creates, so the cap below can leave a
+ # caller's own attachments alone.
+ promoted: list[dict] = []
+
+ def flush(into: "dict | None" = None) -> "dict | None":
+ if not pending or not vision:
+ pending.clear()
+ returned_totals.clear()
+ return into
+ returned = sum(returned_totals) or sum(len(result) for result in pending)
+ # Detached wording for a batch of several results too, not only an interrupted
+ # one: two parallel calls both returning pictures share the turn, and "the tool
+ # call above" would hand every picture to whichever ran last. The loops apply
+ # the same rule to a live batch.
+ lead = DETACHED_IMAGE_TURN_TEXT if interrupted[0] or len(pending) > 1 else IMAGE_TURN_TEXT
+ interrupted[0] = False
+ if local:
+ # the caller's attachment takes the single image slot on its turn.
+ if into is not None and any(
+ id(part) in caller_payloads for part in image_marker_parts([into])
+ ):
+ pending.clear()
+ returned_totals.clear()
+ return into
+ encoded = png_payloads_per_result(pending, cache = decode_cache)
+ pending.clear()
+ returned_totals.clear()
+ if not encoded:
+ return into
+ payloads.extend(encoded)
+ markers = [{"type": "image"} for _ in encoded]
+ if into is None:
+ out.append(placeholder_turn(len(encoded), returned, lead))
+ return None
+ # The note rides along on the merge: merged bare into the question, the
+ # markers read as pictures the user attached, and with a result of another
+ # tool between, as that tool's output.
+ note = {"type": "text", "text": _turn_text(len(encoded), returned, lead)}
+ return _with_parts(into, [*markers, note])
+ results = list(pending)
+ pending.clear()
+ returned_totals.clear()
+ if into is None:
+ before = {id(part) for part in _all_image_url_parts(out)}
+ append_image_turn(
+ out, results, per_result = True, limit = None, returned = returned, lead = lead
+ )
+ promoted.extend(part for part in _all_image_url_parts(out) if id(part) not in before)
+ return None
+ parts = content_parts_per_result(results)
+ promoted.extend(parts)
+ if not parts:
+ return into
+ note = {"type": "text", "text": _turn_text(len(parts), returned, lead)}
+ return _with_parts(into, [*parts, note])
+
+ for position, message in enumerate(messages):
+ content = message.get("content")
+ if message.get("role") == "tool" and isinstance(content, str):
+ text, images = split_images(content)
+ # The suffix always comes off -- it is megabytes of base64 and the model
+ # must never read it as text. Provenance decides only whether it becomes
+ # IMAGE input: a named non-MCP tool that happens to end in a valid
+ # envelope is not one an MCP server served.
+ name = message.get("name") or call_names.get(position)
+ if isinstance(name, str) and name and not name.startswith(MCP_TOOL_PREFIX):
+ # A non-MCP result sitting between the images and their turn makes
+ # "the tool call above" name web_search or read_file.
+ if pending:
+ interrupted[0] = True
+ out.append(
+ {**message, "content": text or "[image returned]"} if images else message
+ )
+ continue
+ if images:
+ # Only the entries the cap can still admit. The suffix comes off the
+ # text either way, above; this decides how many are decoded.
+ admitted = images[: eligible.get(position, len(images))]
+ if admitted:
+ pending.append(admitted)
+ # What the TOOL returned, which the slice above has already lost.
+ # Summing the admitted candidates instead made a 100-image result
+ # read "(4 of 8)" beside a tool result saying 100 -- the note
+ # describing the admission pass rather than the tool. The frontend
+ # bounds the envelope before it ever gets here and records the
+ # count it started from on the first entry; honour that too.
+ # Counted whether or not anything of it was admitted: a result the
+ # allowance left nothing of is still part of what the batch returned,
+ # and leaving it out read "(1 of 6)" for three results of three.
+ returned_totals.append(_returned_count(images))
+ elif pending:
+ interrupted[0] = True
+ out.append({**message, "content": text or "[image returned]"} if images else message)
+ continue
+ if pending and vision and message.get("role") == "user":
+ # Merged, not inserted ahead of it: two user turns in a row is what
+ # a strict template rejects.
+ out.append(flush(message))
+ continue
+ flush()
+ out.append(message)
+ flush()
+ # A replay carries every image turn the conversation ever had; the cap has to
+ # hold here too or the whole history is re-sent on every later turn.
+ if local:
+ protected = ()
+ if caller_payloads:
+ markers = image_marker_parts(out)
+ replay_payloads = dict(
+ zip((id(part) for part in markers if id(part) not in caller_payloads), payloads)
+ )
+ by_marker = {**replay_payloads, **caller_payloads}
+ payloads = [by_marker[id(part)] for part in markers]
+ protected = tuple(
+ index for index, part in enumerate(markers) if id(part) in caller_payloads
+ )
+ trim_image_turns(out, payloads, keep = protected)
+ else:
+ # The cap says attachments are never counted against it, which is right for
+ # what THIS cap protects, and llama-server is bounded by its context window
+ # rather than a fixed image count: a GGUF replay keeps the full allowance
+ # beside the caller's picture, as its live loop did.
+ #
+ # A provider is different. It applies its own per-request cap in document
+ # order, and promotion prepends the replay to the user turn: on Gemini (8
+ # images, later ones dropped silently) eight replayed screenshots evicted the
+ # picture the current question was about, and the model answered it from
+ # stale tool output. So the external caller reserves the attachment's room,
+ # the way the local route reserves its slot before interleaving it.
+ _caller_parts = len(_all_image_url_parts(out)) - len(promoted) if reserve_for_caller else 0
+ trim_image_url_turns(
+ out,
+ limit = max(0, MAX_TOTAL_MODEL_IMAGES - _caller_parts),
+ only = promoted,
+ )
+ return out, payloads, promoted
+
+
+def _with_parts(message: dict, parts: list[dict]) -> dict:
+ content = message.get("content")
+ own = list(content) if isinstance(content, list) else [{"type": "text", "text": content or ""}]
+ return {**message, "content": [*parts, *own]}
diff --git a/studio/backend/core/inference/mlx_inference.py b/studio/backend/core/inference/mlx_inference.py
index bae62f65fe8..0762d3a80b6 100644
--- a/studio/backend/core/inference/mlx_inference.py
+++ b/studio/backend/core/inference/mlx_inference.py
@@ -44,6 +44,12 @@ from core.inference.chat_template_helpers import (
trailing_assistant_text,
vlm_prompt_issue as _vlm_prompt_issue,
)
+from core.inference.mcp_images import (
+ image_marker_parts,
+ pixels_in_marker_order,
+ top_up_image_markers,
+ trim_image_turns,
+)
from utils.models.model_config import is_audio_input_type
from loggers import get_logger
@@ -3217,6 +3223,8 @@ class MLXInferenceBackend:
messages,
system_prompt = "",
image = None,
+ images = None,
+ image_ordinal = None,
temperature = 0.7,
top_p = 0.9,
top_k = 40,
@@ -3240,8 +3248,6 @@ class MLXInferenceBackend:
# cannot tell that the wrappers below still have to survive decoding.
tool_protocol_active = None,
video = None,
- *,
- images = None,
) -> Generator[str, None, None]:
if self._model is None:
raise RuntimeError("No model loaded")
@@ -3254,19 +3260,17 @@ class MLXInferenceBackend:
# Reset so a failed run cannot surface stale stats.
self.last_generation_stats = None
- images = list(images) if images else ([image] if image is not None else [])
- if len(images) > 1 and not self._multi_image_marker:
- raise ValueError(
- f"'{self.active_model_name}' takes one image per request: its chat template does "
- f"not mark {len(images)} images separately."
- )
+ full_messages = self._with_system_prompt(messages, system_prompt)
+ # History first, the attachment last: MLX binds pixels to markers in order.
+ attached = list(images or []) + ([image] if image is not None else [])
# Shared with the transformers vision path so both render the same turns (#10092).
- if self._is_vlm and (images or video is not None):
+ if self._is_vlm and (attached or video is not None):
# Processor templates want part lists, the tokenizer fallback wants strings.
from core.inference.chat_template_helpers import (
chat_render_target as _chat_render_target,
)
+
_renders_via_processor = (
self._processor is not None
and _chat_render_target(self._processor) is self._processor
@@ -3275,16 +3279,41 @@ class MLXInferenceBackend:
messages,
system_prompt = system_prompt,
structured_content = _renders_via_processor,
- image = len(images or ()),
+ image = len(attached),
video = video is not None,
)
- else:
- full_messages = self._with_system_prompt(messages, system_prompt)
+ # That helper leaves a conversation that already carries markers alone,
+ # which is right for a retry but not for replayed MCP pictures: those
+ # markers are not the attachment's, and the render would be one short
+ # for two pixels.
+ _prior_markers = image_marker_parts(full_messages)
+ full_messages = top_up_image_markers(
+ full_messages, len(attached), ordinal = image_ordinal
+ )
+ if image is not None:
+ # Read off the conversation: the attachment's turn can precede a
+ # replayed picture, and pixels bind to markers in document order.
+ attached = pixels_in_marker_order(
+ full_messages, _prior_markers, list(images or []), image
+ )
+ if len(attached) > 1 and not self._multi_image_marker:
+ # The template marks one image, so replayed pictures cannot each have their
+ # own. Serve the caller's attachment, or else the newest picture, with its marker.
+ keep = next(
+ (i for i, one in enumerate(attached) if one is image), len(attached) - 1
+ )
+ logger.info(
+ "Serving 1 of %d images: '%s' marks one image per request.",
+ len(attached),
+ self.active_model_name,
+ )
+ full_messages = list(full_messages)
+ trim_image_turns(full_messages, attached, limit = 1, keep = (keep,))
if self._is_vlm:
stream = self._generate_vlm(
full_messages,
- images,
+ attached,
temperature,
top_p,
top_k,
diff --git a/studio/backend/core/inference/openai_codex_tool_loop.py b/studio/backend/core/inference/openai_codex_tool_loop.py
index 03e39d2f848..8b0b51ecd9f 100644
--- a/studio/backend/core/inference/openai_codex_tool_loop.py
+++ b/studio/backend/core/inference/openai_codex_tool_loop.py
@@ -36,6 +36,8 @@ class CodexRunContext:
response_format: dict[str, Any] | None = None
tool_choice: Any = None
continue_final_message: bool = False
+ supports_vision: bool = False
+ promoted_image_parts: tuple = ()
@dataclass(frozen = True)
@@ -106,6 +108,8 @@ def stream_codex_with_studio_tools(
model = run.model,
tool_choice = run.tool_choice,
continue_final_message = run.continue_final_message,
+ supports_vision = run.supports_vision,
+ promoted_image_parts = run.promoted_image_parts,
),
policy = ToolLoopPolicy(
tools = policy.tools,
diff --git a/studio/backend/core/inference/orchestrator.py b/studio/backend/core/inference/orchestrator.py
index fb819812572..08725f71972 100644
--- a/studio/backend/core/inference/orchestrator.py
+++ b/studio/backend/core/inference/orchestrator.py
@@ -329,11 +329,9 @@ def _summed_tool_loop_stats(total, turn):
return summed
-def _request_images(image, images):
- """One request's images in render order; ``image`` is the older single-image spelling."""
- if images:
- return list(images)
- return [image] if image is not None else []
+def _encoded_images(images, to_base64) -> list:
+ """Replayed MCP pictures arrive already PNG-encoded; a caller's decoded list does not."""
+ return [one if isinstance(one, str) else to_base64(one) for one in images or ()]
def _mirrored_model_entry(model_info: dict, model_name: str) -> dict:
@@ -1222,8 +1220,10 @@ class InferenceOrchestrator:
def _build_generate_cmd(
self,
request_id: str,
- images_b64: list,
+ image_b64: Optional[str],
*,
+ images_b64: Optional[list] = None,
+ image_ordinal: Optional[int] = None,
messages: list = None,
system_prompt: str = "",
temperature: float = 0.7,
@@ -1252,7 +1252,9 @@ class InferenceOrchestrator:
"request_id": request_id,
"messages": messages or [],
"system_prompt": system_prompt,
- "images_base64": images_b64,
+ "image_base64": image_b64,
+ "images_base64": images_b64 or None,
+ "image_ordinal": image_ordinal,
"temperature": temperature,
"top_p": top_p,
"top_k": top_k,
@@ -1454,6 +1456,8 @@ class InferenceOrchestrator:
messages: list = None,
system_prompt: str = "",
image = None,
+ images: Optional[list] = None,
+ image_ordinal: Optional[int] = None,
temperature: float = 0.7,
top_p: float = 0.9,
top_k: int = 40,
@@ -1475,8 +1479,6 @@ class InferenceOrchestrator:
logit_bias: Optional[dict] = None,
stop: Optional[list] = None,
video: Optional[str] = None,
- *,
- images = None,
) -> Generator[str, None, None]:
"""Dispatched generation, sending the command without holding _gen_lock. Uses a per-request
mailbox for tokens so two compare-mode requests can be queued at once. The subprocess
@@ -1512,11 +1514,14 @@ class InferenceOrchestrator:
request_id = str(uuid.uuid4())
- images_b64 = [self._pil_to_base64(one) for one in _request_images(image, images)]
+ image_b64 = self._pil_to_base64(image) if image is not None else None
+ images_b64 = _encoded_images(images, self._pil_to_base64)
cmd = self._build_generate_cmd(
request_id,
- images_b64,
+ image_b64,
+ images_b64 = images_b64,
+ image_ordinal = image_ordinal,
messages = messages,
system_prompt = system_prompt,
temperature = temperature,
@@ -2296,6 +2301,8 @@ class InferenceOrchestrator:
messages: list,
system_prompt: str = "",
image = None,
+ images: Optional[list] = None,
+ image_ordinal: Optional[int] = None,
temperature: float = 0.7,
top_p: float = 0.9,
top_k: int = 40,
@@ -2316,8 +2323,6 @@ class InferenceOrchestrator:
logit_bias: Optional[dict] = None,
stop: Optional[list] = None,
video: Optional[str] = None,
- *,
- images = None,
) -> Generator[str, None, None]:
"""Generate response, streaming tokens from subprocess. ``tools`` / ``enable_thinking`` /
``reasoning_effort`` / ``preserve_thinking`` are forwarded so the template can render
@@ -2330,6 +2335,7 @@ class InferenceOrchestrator:
system_prompt = system_prompt,
image = image,
images = images,
+ image_ordinal = image_ordinal,
temperature = temperature,
top_p = top_p,
top_k = top_k,
@@ -2358,6 +2364,7 @@ class InferenceOrchestrator:
messages: list,
tools: list,
system_prompt: str = "",
+ images: Optional[list] = None,
temperature: float = 0.7,
top_p: float = 0.9,
top_k: int = 40,
@@ -2387,6 +2394,7 @@ class InferenceOrchestrator:
stop: Optional[list] = None,
reasoning_prefilled: bool = False,
seed: Optional[int] = None,
+ caller_image_indexes: "tuple[int, ...]" = (),
**_unused,
):
"""Run the safetensors agentic tool loop in the parent process, calling the worker for each
@@ -2397,6 +2405,13 @@ class InferenceOrchestrator:
# None lets the backend size an unset limit once it has counted the prompt.
max_new_tokens = max_tokens if max_tokens and max_tokens > 0 else None
+ # Only a model that reads images gets a sink; the loop leaves MCP pictures
+ # out of the prompt without one.
+ loop_images: Optional[list] = (
+ list(images or [])
+ if self.models.get(self.active_model_name, {}).get("is_vision")
+ else None
+ )
# The worker's usage for the LATEST turn only. Hoisted out of the turn so the loop can size a conversation
# search against a real prompt count, and cleared on the way in rather than on each way out, so a turn that
@@ -2417,6 +2432,7 @@ class InferenceOrchestrator:
messages = conv,
system_prompt = "",
image = None,
+ images = list(loop_images) if loop_images else None,
temperature = temperature,
top_p = top_p,
top_k = top_k,
@@ -2516,6 +2532,11 @@ class InferenceOrchestrator:
context_length = _model_info.get("context_length"),
max_tokens = max_new_tokens,
generation_stats_holder = turn_stats,
+ images_sink = loop_images,
+ # Which sink entries are the caller's own attachment, so the loop's cap
+ # never evicts it. Empty when the model reads no images, since there is
+ # then no sink to protect anything in.
+ caller_image_indexes = tuple(caller_image_indexes) if loop_images else (),
)
def generate_with_adapter_control(
@@ -2552,6 +2573,8 @@ class InferenceOrchestrator:
messages: list = None,
system_prompt: str = "",
image = None,
+ images: Optional[list] = None,
+ image_ordinal: Optional[int] = None,
temperature: float = 0.7,
top_p: float = 0.9,
top_k: int = 40,
@@ -2573,8 +2596,6 @@ class InferenceOrchestrator:
logit_bias: Optional[dict] = None,
stop: Optional[list] = None,
video: Optional[str] = None,
- *,
- images = None,
) -> Generator[str, None, None]:
"""Inner generation logic: sends the command to the subprocess and yields tokens. Serialized
by _gen_lock (one generation at a time) so concurrent readers don't consume each other's
@@ -2599,10 +2620,13 @@ class InferenceOrchestrator:
if cancel_event is not None and cancel_event.is_set():
return
request_id = str(uuid.uuid4())
- images_b64 = [self._pil_to_base64(one) for one in _request_images(image, images)]
+ image_b64 = self._pil_to_base64(image) if image is not None else None
+ images_b64 = _encoded_images(images, self._pil_to_base64)
cmd = self._build_generate_cmd(
request_id,
- images_b64,
+ image_b64,
+ images_b64 = images_b64,
+ image_ordinal = image_ordinal,
messages = messages,
system_prompt = system_prompt,
temperature = temperature,
diff --git a/studio/backend/core/inference/safetensors_agentic.py b/studio/backend/core/inference/safetensors_agentic.py
index 24ba0ebcd3e..c288054bcca 100644
--- a/studio/backend/core/inference/safetensors_agentic.py
+++ b/studio/backend/core/inference/safetensors_agentic.py
@@ -53,6 +53,13 @@ from core.tool_healing import (
_think_spans_outside_tool_markup,
strip_outside_think,
)
+from core.inference.mcp_images import (
+ DETACHED_IMAGE_TURN_TEXT as MCP_DETACHED_IMAGE_TURN_TEXT,
+ IMAGE_TURN_TEXT as MCP_IMAGE_TURN_TEXT,
+ append_placeholder_turn,
+ png_payloads_per_result,
+ trim_image_turns,
+)
from core.inference.tool_loop_controller import (
_WORKSPACE_TOOLS,
ToolLoopController,
@@ -600,6 +607,8 @@ def run_safetensors_tool_loop(
context_length: Optional[int] = None,
max_tokens: Optional[int] = None,
generation_stats_holder: Optional[dict] = None,
+ images_sink: Optional[list] = None,
+ caller_image_indexes: "tuple[int, ...]" = (),
) -> Generator[dict, None, None]:
"""Drive an agentic tool loop on top of a cumulative-text generator.
@@ -622,6 +631,12 @@ def run_safetensors_tool_loop(
* ``{"type": "tool_end", "tool_name", "tool_call_id", "result"}``
"""
conversation = list(messages)
+ # Where the caller's own attachment sits in the seeded sink. The cap is about what
+ # the loop RE-SENDS, so that entry is never the one it drops -- but it is not the
+ # whole seed: a resumed chat seeds replayed pictures too, and exempting those would
+ # let the prompt carry a second full allowance. The route reserves the attachment's
+ # slot by trimming replay to limit - 1 before interleaving it.
+ caller_images = tuple(caller_image_indexes)
# The branch this request is on, before the loop appends anything. A GGUF-compacted
# thread keeps its archive across a switch to safetensors, so search_conversation is
# advertised here too and needs the same filtering: the stored rows are the whole
@@ -1354,6 +1369,10 @@ def run_safetensors_tool_loop(
# Collect no-op nudges and flush them after the batch, so a no-op doesn't
# abort it and drop the parallel calls that follow.
deferred_noop_msgs: list = []
+ # Per result; see append_image_turn's per_result note.
+ batch_mcp_images: list = []
+ # Where this batch's results start, for the image turn\'s wording.
+ batch_conversation_start = len(conversation)
for _call_index, tc in enumerate(tool_calls or []):
func = tc.get("function", {}) or {}
@@ -1364,6 +1383,12 @@ def run_safetensors_tool_loop(
and tc.get("id", "") == provisional_render_html_id
)
decision = tool_controller.prepare_call(tc, provisional = provisional_match)
+ # The frontend keeps a round's tool cards together by this id
+ # (codexLocalToolRoundId) and otherwise flushes each completed pair on its
+ # own. This loop shows a parallel batch as ONE picture, and replay groups
+ # consecutive results the same way -- so without the id a batch persisted
+ # as separate pairs replayed as several pictures on the next request.
+ decision.provenance["round_id"] = iteration
if not decision.should_execute:
if content_text and not assistant_appended:
@@ -1608,6 +1633,10 @@ def run_safetensors_tool_loop(
_turn_executed_real_tool = True
yield completion.tool_end_event()
conversation.append(completion.tool_message())
+ # Parsed only when there is a sink for them.
+ _completion_images = completion.mcp_images() if images_sink is not None else []
+ if _completion_images:
+ batch_mcp_images.append(_completion_images)
# The turn that ends the loop offers no tools again, so do not ask for a retry there.
over_cap_final = bool(over_cap) and (
@@ -1630,6 +1659,33 @@ def run_safetensors_tool_loop(
)
)
append_deferred_nudges(conversation, deferred_noop_msgs)
+ if batch_mcp_images and images_sink is not None:
+ # A sink only when the loaded model reads images; the pixels ride
+ # beside the prompt, so the turn carries markers rather than data.
+ encoded = png_payloads_per_result(batch_mcp_images)
+ if encoded:
+ images_sink.extend(encoded)
+ # Merged into the deferred nudge above when there was one: two
+ # user turns in a row is what a strict VLM template rejects.
+ # Same wording rule as the other loops: several results, and "the
+ # tool call above" names whichever ran last.
+ _batch_results = sum(
+ 1
+ for m in conversation[batch_conversation_start:]
+ if isinstance(m, dict) and m.get("role") == "tool"
+ )
+ append_placeholder_turn(
+ conversation,
+ len(encoded),
+ sum(len(r) for r in batch_mcp_images),
+ lead = MCP_DETACHED_IMAGE_TURN_TEXT
+ if _batch_results != 1
+ else MCP_IMAGE_TURN_TEXT,
+ )
+ # Rebased on the way out: this trim deletes entries before the
+ # attachment, and reusing the original index on the next batch
+ # would protect the wrong payload and delete the attachment.
+ caller_images = trim_image_turns(conversation, images_sink, keep = caller_images)
yield {"type": "status", "text": ""}
diff --git a/studio/backend/core/inference/studio_tool_loop.py b/studio/backend/core/inference/studio_tool_loop.py
index 80d99859061..3be1beca53a 100644
--- a/studio/backend/core/inference/studio_tool_loop.py
+++ b/studio/backend/core/inference/studio_tool_loop.py
@@ -55,6 +55,32 @@ from core.inference.tool_call_parser import (
reprompt_to_act_message,
strip_tool_markup,
)
+from core.inference.mcp_images import append_image_turn as append_mcp_image_turn
+
+
+def _append_mcp_images_owned(
+ conversation,
+ results,
+ owned,
+ lead = None,
+):
+ """append_image_turn with the loop's own part list, for asyncio.to_thread.
+
+ Reserving here and not on the GGUF loop: this one talks to a remote provider that
+ applies its own per-request image cap in document order, so an attachment beside a
+ full allowance of tool results silently loses the newest result -- the one the
+ model just asked for. llama-server is local and answers to the context window.
+ """
+ append_mcp_image_turn(
+ conversation,
+ results,
+ per_result = True,
+ owned = owned,
+ reserve_caller_images = True,
+ **({"lead": lead} if lead else {}),
+ )
+
+
from core.inference.tool_loop_controller import (
ToolLoopController,
awaiting_approval_status,
@@ -339,6 +365,11 @@ class ToolLoopRun:
model: str | None = None
tool_choice: Any = None
continue_final_message: bool = False
+ supports_vision: bool = False
+ # Image parts an earlier promotion already put in `messages`. The loop's cap
+ # counts these too: starting from zero lets a resumed image-heavy chat carry a
+ # full history through and then add a whole batch on top.
+ promoted_image_parts: tuple = ()
@dataclass(frozen = True)
@@ -1180,6 +1211,10 @@ async def stream_with_studio_tools(
) -> AsyncIterator[str]:
"""Stream a provider, execute requested Unsloth tools, continue to a final answer."""
conversation = [dict(message) for message in run.messages]
+ # The image parts this run appends, so its cap never counts a caller's own
+ # attachments. Run-scoped, not turn-scoped: the cap is across the whole loop,
+ # and seeded with what promotion already put in the conversation.
+ loop_mcp_image_parts: list = list(run.promoted_image_parts)
# Kept before the loop appends anything: this is the branch the request is on.
request_branch = list(run.messages)
remaining = policy.max_calls
@@ -1545,6 +1580,9 @@ async def stream_with_studio_tools(
assistant_tool_calls: list[dict[str, Any]] = []
tool_messages: list[dict[str, Any]] = []
noop_messages: list[dict[str, Any]] = []
+ # One entry per tool result, not flattened: the per-result image quota
+ # would otherwise be spent entirely on the first call of a parallel batch.
+ turn_mcp_images: list[list[dict[str, Any]]] = []
turn_executed_real_tool = False
for call in calls:
@@ -1790,6 +1828,13 @@ async def stream_with_studio_tools(
last_reprompt_text = ""
yield _sse(completion.tool_end_event())
tool_messages.append(completion.tool_message())
+ # Only for a target that reads them, and off the loop: the envelope is
+ # a 12 MB json-load, and a text-only run discards the result anyway.
+ _completion_images = (
+ await asyncio.to_thread(completion.mcp_images) if run.supports_vision else []
+ )
+ if _completion_images:
+ turn_mcp_images.append(_completion_images)
yield _status_sse("")
@@ -1828,6 +1873,20 @@ async def stream_with_studio_tools(
conversation,
"\n\n".join(dict.fromkeys(message["content"] for message in noop_messages)),
)
+ if turn_mcp_images and run.supports_vision:
+ # One block after the whole batch. With a single result "the tool call
+ # above" is exact; with several it names whichever ran last, which may
+ # have returned no picture at all, so the block says so instead.
+ from core.inference.mcp_images import DETACHED_IMAGE_TURN_TEXT
+ _lead = DETACHED_IMAGE_TURN_TEXT if len(tool_messages) != 1 else None
+ # Off the event loop: each image is decoded and re-encoded.
+ await asyncio.to_thread(
+ _append_mcp_images_owned,
+ conversation,
+ turn_mcp_images,
+ loop_mcp_image_parts,
+ _lead,
+ )
if turn_executed_real_tool:
max_reprompts = _MAX_POST_TOOL_REPROMPTS
diff --git a/studio/backend/core/inference/tool_loop_controller.py b/studio/backend/core/inference/tool_loop_controller.py
index eba04f01e48..4a8a5d49452 100644
--- a/studio/backend/core/inference/tool_loop_controller.py
+++ b/studio/backend/core/inference/tool_loop_controller.py
@@ -21,6 +21,11 @@ from typing import Any, Collection, Literal, Mapping, Sequence
from urllib.parse import urlparse
from core.inference.llama_tool_schema import unrelaxed
+from core.inference.mcp_images import split_images as split_mcp_images
+
+# Stamped by mcp_client on every tool it registers; the provenance the envelope
+# is trusted on.
+MCP_TOOL_PREFIX = "mcp__"
from core.inference.tool_call_parser import TOOL_ERROR_NUDGE, TOOL_ERROR_PREFIXES
@@ -332,6 +337,17 @@ class ToolCallCompletion:
message["tool_call_id"] = self.decision.tool_call_id
return message
+ def mcp_images(self) -> list[dict]:
+ """Images this call returned, and only for a call an MCP server served.
+
+ The envelope is a plain suffix, so any tool whose output happens to end in
+ one -- terminal output, a fetched page -- would otherwise have its bytes
+ decoded and attached as model image input.
+ """
+ if not self.executed or not self.decision.tool_name.startswith(MCP_TOOL_PREFIX):
+ return []
+ return split_mcp_images(self.result)[1]
+
@dataclass(frozen = True)
class _ToolCallRecord:
@@ -817,29 +833,6 @@ def is_tool_error(result: str) -> bool:
return isinstance(result, str) and result.lstrip().startswith(TOOL_ERROR_PREFIXES)
-def _strip_mcp_image_suffix(result: str) -> str:
- """Drop a trailing __MCP_IMAGES__ envelope only when it is the valid JSON
- image array appended by _flatten_result, so legit tool text that merely
- mentions the marker is not truncated."""
- head, sep, payload = result.rpartition("\n__MCP_IMAGES__:")
- if not sep:
- return result
- try:
- images = json.loads(payload)
- except (ValueError, RecursionError):
- return result
- if not isinstance(images, list) or not images:
- return result
- if not all(
- isinstance(img, dict)
- and isinstance(img.get("data"), str)
- and isinstance(img.get("mimeType"), str)
- for img in images
- ):
- return result
- return head.rstrip()
-
-
def _strip_files_sentinel(result: str) -> str:
"""Drop a trailing ``__FILES__`` envelope, and only that.
@@ -990,7 +983,10 @@ def strip_result_for_model(
if tool_name is None or tool_name == "web_search":
from .search_images import strip_images_suffix
result = strip_images_suffix(result)
- result = _strip_mcp_image_suffix(result)
+ # Always, whoever produced it: these bytes run to megabytes and the model must
+ # never be shown them as text. Provenance decides whether they become IMAGE
+ # input, which is a separate question answered in mcp_images._promote.
+ result = split_mcp_images(result)[0]
if tool_name is None or tool_name in _SANDBOX_TOOLS:
result = _strip_files_sentinel(result)
if tool_name is None or tool_name in _IMAGE_SENTINEL_TOOLS:
diff --git a/studio/backend/core/inference/worker.py b/studio/backend/core/inference/worker.py
index 877afed0ae8..644f892d0a5 100644
--- a/studio/backend/core/inference/worker.py
+++ b/studio/backend/core/inference/worker.py
@@ -676,6 +676,10 @@ def _handle_generate(backend, cmd: dict, resp_queue: Any, cancel_event) -> None:
request_id = cmd.get("request_id", "")
try:
+ image = None
+ image_b64 = cmd.get("image_base64")
+ if image_b64:
+ image = _resize_image(_decode_image(image_b64))
images = [
_resize_image(_decode_image(encoded))
for encoded in cmd.get("images_base64") or ()
@@ -685,6 +689,7 @@ def _handle_generate(backend, cmd: dict, resp_queue: Any, cancel_event) -> None:
gen_kwargs = {
"messages": cmd["messages"],
"system_prompt": cmd.get("system_prompt", ""),
+ "image": image,
"images": images,
"temperature": cmd.get("temperature", 0.7),
"top_p": cmd.get("top_p", 0.9),
@@ -707,6 +712,9 @@ def _handle_generate(backend, cmd: dict, resp_queue: Any, cancel_event) -> None:
if opt_key in cmd:
gen_kwargs[opt_key] = cmd[opt_key]
+ if cmd.get("image_ordinal") is not None and _backend_declares(backend, "image_ordinal"):
+ gen_kwargs["image_ordinal"] = cmd["image_ordinal"]
+
# Not every backend declares these (transformers declares only ``stop``)
# and none takes **kwargs, so forwarding unconditionally would turn a
# backend's documented "ignores them" behavior into a TypeError.
diff --git a/studio/backend/routes/inference.py b/studio/backend/routes/inference.py
index 7b236f9c822..3af42c342d0 100644
--- a/studio/backend/routes/inference.py
+++ b/studio/backend/routes/inference.py
@@ -2015,7 +2015,9 @@ def _openai_llama_admission_compact_image_part(part: dict) -> dict:
return {"type": "image_url", "image_url": compact_image_url}
-def _openai_llama_admission_messages_for_estimate(messages) -> tuple[list[dict], int]:
+def _openai_llama_admission_messages_for_estimate(
+ messages, *, vision: bool = True
+) -> tuple[list[dict], int]:
"""Remove image bytes before estimating the textual part of a prompt.
mtmd turns an image into model-specific embedding tokens, so its base64 transport
@@ -2024,7 +2026,15 @@ def _openai_llama_admission_messages_for_estimate(messages) -> tuple[list[dict],
"""
estimate_messages = []
image_parts = 0
- for message in messages:
+ # Counted apart from the parts already on the request: promote_history caps how
+ # many an envelope becomes, but a caller that really attaches more images than
+ # that must still be charged for every one (#9842).
+ envelope_image_parts = 0
+ # Same provenance resolution generation uses: an unnamed result correlated to a
+ # non-MCP call is never promoted, so charging its envelope reserves up to eight
+ # embeddings the request never sends.
+ _resolved = _mcp_resolve_tool_names(messages)
+ for _index, message in enumerate(messages):
# exclude_none like every other dump here, including the generation paths this
# predicts: the five unset optionals on ChatMessage otherwise serialise as
# `"name": null` and get priced as prompt text (34 tokens for a two-key message
@@ -2034,6 +2044,36 @@ def _openai_llama_admission_messages_for_estimate(messages) -> tuple[list[dict],
)
estimate_message = dict(message_dict)
content = message_dict.get("content")
+ if message_dict.get("role") == "tool" and isinstance(content, str):
+ # An MCP envelope is still base64 text here, but the chat path promotes
+ # it into projector images before generation. Priced as text it would
+ # reserve megabytes of prompt for bytes never sent; ignored entirely it
+ # charges zero KV for images that are. Strip it and charge the images.
+ head, envelope_images = split_mcp_images(content)
+ if envelope_images:
+ # Generation strips the suffix for everyone, so it is never priced
+ # as prompt text.
+ estimate_message["content"] = head
+ # Charged only where generation will really send pixels: a text-only
+ # model has them stripped, and a named non-MCP result is not promoted
+ # at all, so charging either reserves KV for images never sent.
+ _correlated = _resolved.get(_index)
+ _non_mcp = _names_a_non_mcp_tool(message_dict) or (
+ isinstance(_correlated, str)
+ and bool(_correlated)
+ and not _correlated.startswith("mcp__")
+ )
+ if vision and not _non_mcp:
+ # Only entries that could become a picture: _flatten_result
+ # accepts formats Pillow cannot read, promotion drops them, and
+ # four such entries would reserve most of a small window for
+ # images llama-server is never sent.
+ envelope_image_parts += min(
+ _mcp_count_probably_decodable(envelope_images),
+ _MCP_MAX_MODEL_IMAGES,
+ )
+ estimate_messages.append(estimate_message)
+ continue
if isinstance(content, list):
estimate_content = []
for part in content:
@@ -2077,7 +2117,10 @@ def _openai_llama_admission_messages_for_estimate(messages) -> tuple[list[dict],
estimate_content.append(_openai_llama_admission_compact_image_part(part))
estimate_message["content"] = estimate_content
estimate_messages.append(estimate_message)
- return estimate_messages, image_parts
+ # Match GGUF promotion: cap replay images separately from caller attachments.
+ # One attachment plus eight replay images needs nine KV charges; reserving
+ # attachment slots within this cap would undercharge concurrent requests.
+ return estimate_messages, image_parts + min(envelope_image_parts, _MCP_MAX_TOTAL_MODEL_IMAGES)
def _conversation_video_clips(messages) -> list[str]:
@@ -2181,6 +2224,8 @@ def _openai_llama_admission_tokens(
image_tokens: int = _OPENAI_LLAMA_ADMISSION_IMAGE_TOKENS,
injected_tools = None,
context_window: Optional[int] = None,
+ vision: bool = True,
+ messages_override = None,
) -> Optional[int]:
"""KV a request will occupy: what is sent, plus what it may generate.
@@ -2194,11 +2239,16 @@ def _openai_llama_admission_tokens(
"""
if not budget:
return None
- messages = getattr(payload, "messages", None)
+ # Use the translated OpenAI messages for Anthropic: its raw user/tool_result
+ # blocks bypass envelope handling, mispricing base64 as text and omitting
+ # image KV charges, which can consume the budget and serialize requests.
+ messages = (
+ messages_override if messages_override is not None else getattr(payload, "messages", None)
+ )
if isinstance(messages, list) and messages:
try:
estimate_messages, message_image_parts = _openai_llama_admission_messages_for_estimate(
- messages
+ messages, vision = vision
)
prompt_tokens = estimate_messages_tokens_dense(estimate_messages)
prompt_tokens += _openai_llama_admission_extra_prompt_tokens(payload)
@@ -2266,6 +2316,36 @@ def _openai_llama_admission_tokens(
return max(1, min(budget, prompt_tokens + output_tokens))
+def _openai_llama_admission_estimate(
+ *,
+ request: Optional[Request],
+ llama_backend,
+ payload,
+ tool_loop: bool = False,
+ injected_tools = None,
+ messages_override = None,
+):
+ """The token estimate a reservation is sized by. Split from the reservation so a
+ caller on the event loop can compute it in a worker: it strips every replayed
+ envelope, which json-loads a permitted 12 MB array per result."""
+ capacity = _openai_llama_admission_capacity(request, llama_backend)
+ budget = _openai_llama_admission_budget(llama_backend)
+ return _openai_llama_admission_tokens(
+ payload,
+ budget = budget,
+ capacity = capacity,
+ tool_loop = tool_loop,
+ image_tokens = _openai_llama_admission_image_tokens(llama_backend),
+ injected_tools = injected_tools,
+ context_window = _openai_llama_admission_context_window(llama_backend),
+ vision = bool(getattr(llama_backend, "is_vision", False)),
+ messages_override = messages_override,
+ )
+
+
+_TOKENS_UNSET = object()
+
+
def _openai_llama_admission_reserve(
*,
request: Optional[Request],
@@ -2273,28 +2353,70 @@ def _openai_llama_admission_reserve(
payload = None,
tool_loop: bool = False,
injected_tools = None,
+ messages_override = None,
+ tokens = _TOKENS_UNSET,
) -> tuple[LlamaAdmissionReservation, LlamaAdmissionConfig]:
config = llama_admission_config_from_env()
capacity = _openai_llama_admission_capacity(request, llama_backend)
key = str(getattr(llama_backend, "base_url", "llama-server"))
budget = _openai_llama_admission_budget(llama_backend)
+ if tokens is _TOKENS_UNSET:
+ tokens = (
+ _openai_llama_admission_estimate(
+ request = request,
+ llama_backend = llama_backend,
+ payload = payload,
+ tool_loop = tool_loop,
+ injected_tools = injected_tools,
+ messages_override = messages_override,
+ )
+ if payload is not None
+ else None
+ )
+ # The reservation itself stays on the loop: reserve() binds a waiter to the
+ # running loop, so only the estimate above may be computed elsewhere.
reservation = get_llama_admission_queue(key).reserve(
capacity = capacity,
config = config,
budget = budget,
- tokens = _openai_llama_admission_tokens(
- payload,
- budget = budget,
- capacity = capacity,
+ tokens = tokens,
+ )
+ return reservation, config
+
+
+async def _openai_llama_admission_reserve_async(
+ *,
+ request: Optional[Request],
+ llama_backend,
+ payload = None,
+ tool_loop: bool = False,
+ injected_tools = None,
+ messages_override = None,
+) -> tuple[LlamaAdmissionReservation, LlamaAdmissionConfig]:
+ """The same reservation, with the estimate off the loop when a replayed envelope
+ would make it parse one."""
+ tokens = _TOKENS_UNSET
+ if payload is not None and _messages_mention_mcp_images(
+ messages_override if messages_override is not None else payload.messages
+ ):
+ tokens = await asyncio.to_thread(
+ _openai_llama_admission_estimate,
+ request = request,
+ llama_backend = llama_backend,
+ payload = payload,
tool_loop = tool_loop,
- image_tokens = _openai_llama_admission_image_tokens(llama_backend),
injected_tools = injected_tools,
- context_window = _openai_llama_admission_context_window(llama_backend),
+ messages_override = messages_override,
)
- if payload is not None
- else None,
+ return _openai_llama_admission_reserve(
+ request = request,
+ llama_backend = llama_backend,
+ payload = payload,
+ tool_loop = tool_loop,
+ injected_tools = injected_tools,
+ messages_override = messages_override,
+ tokens = tokens,
)
- return reservation, config
def _openai_llama_admission_recost(
@@ -2344,7 +2466,8 @@ def _openai_llama_admission_recost(
# and since the callback fires at the top of round zero, before any growth, it
# would hand back room llama-server is already using.
estimate_messages, message_image_parts = _openai_llama_admission_messages_for_estimate(
- conversation
+ conversation,
+ vision = bool(getattr(llama_backend, "is_vision", False)),
)
prompt_tokens = estimate_messages_tokens_dense(estimate_messages)
# Re-sent every round, so it belongs in every re-costing, not just the opening one.
@@ -3425,6 +3548,23 @@ from state.tool_approvals import resolve_tool_decision
from core.inference.model_ids import display_model_name, model_id_matches, public_model_id
from core.inference.api_monitor import api_monitor
from core.inference.llama_http import nonstreaming_client
+from core.inference.mcp_images import (
+ MAX_MODEL_IMAGES as _MCP_MAX_MODEL_IMAGES,
+ count_probably_decodable as _mcp_count_probably_decodable,
+ image_marker_parts as mcp_image_marker_parts,
+ flattened_rgb as _mcp_flattened_rgb,
+ resolve_tool_names as _mcp_resolve_tool_names,
+ pixels_in_marker_order as mcp_pixels_in_marker_order,
+ trim_image_turns as trim_mcp_image_turns,
+ MAX_TOTAL_MODEL_IMAGES as _MCP_MAX_TOTAL_MODEL_IMAGES,
+ has_images as mcp_images_sentinel_in,
+ mentions_images as mcp_images_mentioned_in,
+ text_before_envelope as mcp_text_before_envelope,
+ mark_last_user_turn as mark_mcp_image_turn,
+ promote_history as promote_mcp_history_images,
+ promote_history_local as promote_mcp_history_images_local,
+ split_images as split_mcp_images,
+)
from core.inference.tool_call_parser import (
_strip_function_xml_calls,
_strip_gemma_wrapperless_calls,
@@ -6028,7 +6168,11 @@ def _monitor_content_text(content) -> str:
if content is None:
return ""
if isinstance(content, str):
- return content
+ # A replayed envelope is megabytes of base64 in a plain string. Generation
+ # and the search index both strip it; the monitor copied it into the prompt
+ # entry, exposing image bytes in inspection and truncating away the later
+ # conversation behind them. Cut, not parsed: this runs on the event loop.
+ return mcp_text_before_envelope(content)
if isinstance(content, list):
parts: list[str] = []
for part in content:
@@ -8285,12 +8429,145 @@ def _messages_have_remote_image(messages) -> bool:
)
-def _request_has_image(payload) -> bool:
+def _request_has_attached_image(payload) -> bool:
+ """An image the request itself carries, which the target model has to be able
+ to read. History is not counted: a replayed MCP envelope is stripped for a
+ text-only model, so it must not veto a switch to one."""
if getattr(payload, "image_base64", None):
return True
return _messages_have_image(payload.messages)
+def _names_a_non_mcp_tool(message) -> bool:
+ """Whether the message names a tool that is definitely not an MCP one.
+
+ An absent name is not evidence against it: older stored turns carry none, and
+ the envelope only ever came from an MCP server.
+ """
+ name = message.get("name") if isinstance(message, dict) else getattr(message, "name", None)
+ return isinstance(name, str) and bool(name) and not name.startswith("mcp__")
+
+
+# Mirrors NON_VISION_PROVIDER_TYPES in studio/frontend/src/features/chat/
+# external-providers.ts. The backend registry marks these providers
+# supports_vision=True for the family, but their main chat endpoint is text-only,
+# and appending a picture there fails upstream mid-run.
+_TEXT_ONLY_PROVIDER_TYPES = frozenset({"cohere", "deepseek", "mistral"})
+
+
+# Mixed catalogs require model-level vision capability; family flags do not
+# identify which models accept images. Qwen has unrestricted text defaults;
+# its vision models use separate qwen-vl-* or Qwen2.5-VL IDs. Kimi is excluded
+# because its allowlist permits only multimodal kimi-k2.5/k2.6.
+# Single-endpoint types (custom, vllm, ollama, llamacpp, lmstudio) retain the
+# registry default: no model catalog exists to check the configured endpoint.
+_MIXED_CATALOG_PROVIDER_TYPES = frozenset({"huggingface", "openrouter", "qwen"})
+
+
+def _external_takes_mcp_images(
+ provider_type,
+ supports_vision: bool,
+ model: "str | None" = None,
+ info: "dict | None" = None,
+) -> bool:
+ """Whether an MCP picture may be appended to this external loop.
+
+ Deliberately stricter than the provider-wide flag: an image the endpoint
+ cannot take does not degrade, it fails the run as soon as a tool returns one.
+ A model-level answer is used wherever the registry has one; where the catalog
+ mixes modalities and says nothing about this model, the answer is no.
+ """
+ if provider_type in _TEXT_ONLY_PROVIDER_TYPES:
+ return False
+ if not supports_vision:
+ return False
+ capabilities = ((info or {}).get("model_capabilities") or {}) if info else {}
+ if model and model in capabilities:
+ return bool(capabilities[model].get("vision"))
+ if provider_type in _MIXED_CATALOG_PROVIDER_TYPES:
+ # Vision for the family, unknown for this model: an image would be sent to
+ # something that may not take one, and the failure lands mid-run.
+ return False
+ return True
+
+
+def _messages_mention_mcp_images(messages) -> bool:
+ """Dispatch only -- whether the work belongs off the loop. Substring, never a
+ parse: the exact check json-loads the whole array, and doing that on the loop
+ to decide whether to leave it parsed a 12 MB envelope right there."""
+ for message in messages or ():
+ role = message.get("role") if isinstance(message, dict) else getattr(message, "role", None)
+ if role != "tool":
+ continue
+ content = (
+ message.get("content")
+ if isinstance(message, dict)
+ else getattr(message, "content", None)
+ )
+ if isinstance(content, str) and mcp_images_mentioned_in(content):
+ return True
+ return False
+
+
+def _messages_have_promotable_mcp_images(messages) -> bool:
+ """The exact envelope check, minus results generation would never promote.
+
+ For the refusals: a named non-MCP result has its suffix stripped and sends no
+ pixel, so refusing a count on it turned away a countable prompt. Callers stamp
+ names from the calls first (_named_anthropic_tool_results), so an unnamed result
+ here really is one nothing can attribute, and legacy history keeps its trust.
+ """
+ for message in messages or ():
+ if not isinstance(message, dict) or message.get("role") != "tool":
+ continue
+ content = message.get("content")
+ if not isinstance(content, str) or not mcp_images_sentinel_in(content):
+ continue
+ if _names_a_non_mcp_tool(message):
+ continue
+ return True
+ return False
+
+
+def _request_has_replayed_mcp_images(payload) -> bool:
+ return _messages_mention_mcp_images(payload.messages)
+
+
+def _request_has_promotable_mcp_images(payload, *, exact: bool = True) -> bool:
+ """Envelopes generation would really promote.
+
+ ``_promote`` preserves a named non-MCP result verbatim, so treating its
+ trailing suffix as images here refuses a countable prompt and can buy a
+ model-catalog fetch nothing needs.
+
+ ``exact=False`` is the substring form, for a decision made on the event loop:
+ the exact check json-loads the whole array, and a permitted envelope is 12 MB.
+ A false positive there costs a catalog fetch; the worker validates for real.
+ """
+ present = mcp_images_sentinel_in if exact else mcp_images_mentioned_in
+ # The same positional correlation generation applies: an unnamed result whose
+ # call was a non-MCP tool is never promoted, and reading it as promotable here
+ # refused a countable prompt and bought a catalog fetch for nothing.
+ names = _mcp_resolve_tool_names(payload.messages)
+ for index, message in enumerate(payload.messages):
+ if getattr(message, "role", None) != "tool":
+ continue
+ content = getattr(message, "content", None)
+ if not isinstance(content, str) or not present(content):
+ continue
+ name = getattr(message, "name", None) or names.get(index)
+ if isinstance(name, str) and name and not name.startswith("mcp__"):
+ continue
+ return True
+ return False
+
+
+def _request_has_image(payload) -> bool:
+ # A replayed envelope is decoded and re-encoded like any other image, so the
+ # work belongs off the event loop even when no picture is attached.
+ return _request_has_attached_image(payload) or _request_has_replayed_mcp_images(payload)
+
+
def _anthropic_image_blocks(payload, *, tool_results: bool = True):
for msg in getattr(payload, "messages", None) or ():
content = msg.get("content") if isinstance(msg, dict) else msg.content
@@ -19015,7 +19292,7 @@ async def generate_audio(
)
# Extract text from the last user message
- _, chat_messages, _ = _extract_content_parts(payload.messages)
+ _, chat_messages, _ = await _extract_content_parts_async(payload.messages)
if not chat_messages:
raise HTTPException(status_code = 400, detail = "No messages provided.")
last_user_msg = next((m for m in reversed(chat_messages) if m["role"] == "user"), None)
@@ -21317,8 +21594,7 @@ def _video_scheme_rejection(clip: str) -> Optional[tuple[int, str]]:
return None
return (
400,
- f"Unsupported video URL scheme ('{head.split(':', 1)[0]}:'). "
- "Send the clip as a data URI.",
+ f"Unsupported video URL scheme ('{head.split(':', 1)[0]}:'). Send the clip as a data URI.",
)
@@ -21445,7 +21721,10 @@ def _inject_audio_part(messages: list[dict], audio_b64: str, audio_format: str)
def _extract_content_parts(
- messages: list, structured: bool = False
+ messages: list,
+ structured: bool = False,
+ *,
+ keep_tool_images: bool = False,
) -> tuple[str, list[dict], "Union[Optional[str], list[str]]"]:
"""
Parse OpenAI-format messages into components the inference backend expects.
@@ -21466,7 +21745,8 @@ def _extract_content_parts(
latest_user_image_b64: Optional[str] = None
served_images: list[str] = []
- for msg in messages:
+ _resolved_tool_names = _mcp_resolve_tool_names(messages)
+ for _msg_index, msg in enumerate(messages):
# ── System / developer messages → extract as system_prompt ────────
if msg.role in ("system", "developer"):
if isinstance(msg.content, str):
@@ -21526,9 +21806,26 @@ def _extract_content_parts(
if combined_text is None:
continue
+ if msg.role == "tool" and not keep_tool_images:
+ # The exact split, always: an MCP tool's own text can contain the marker
+ # line without an envelope behind it (the envelope is appended only when
+ # there were images), and a cut at the marker truncated such a result.
+ # The json-load this costs on a permitted 12 MB envelope is why the
+ # awaited callers go through _extract_content_parts_async.
+ combined_text = split_mcp_images(combined_text)[0]
chat_message = {"role": msg.role, "content": combined_text}
+ # Carried through: promote_history reads it to decide whether an envelope
+ # came from an MCP server, and dropping it here made an unnamed tool
+ # message that bypasses the check entirely. Resolved from the call when the
+ # result itself is unnamed: this rebuild drops tool_call_id and the calls,
+ # so the correlation has to happen here or the local path cannot run the
+ # provenance gate at all.
if msg.name:
chat_message["name"] = msg.name
+ if msg.role == "tool":
+ _tool_name = getattr(msg, "name", None) or _resolved_tool_names.get(_msg_index)
+ if isinstance(_tool_name, str) and _tool_name:
+ chat_message["name"] = _tool_name
if msg.role == "assistant" and msg.reasoning_content:
chat_message["reasoning_content"] = msg.reasoning_content
chat_messages.append(chat_message)
@@ -21542,6 +21839,15 @@ def _extract_content_parts(
)
+async def _extract_content_parts_async(messages: list, **kwargs):
+ """The same, off the shared loop when a tool result carries the envelope marker:
+ stripping it is the exact split, a json-load of a permitted 12 MB array, and the
+ awaited audio, count and safetensors callers must not do that on the loop."""
+ if _messages_mention_mcp_images(messages):
+ return await asyncio.to_thread(_extract_content_parts, messages, **kwargs)
+ return _extract_content_parts(messages, **kwargs)
+
+
def _user_ordinal_supplying_the_image(messages: list) -> Optional[int]:
"""Which user turn, counted among user turns, the selected image came from.
@@ -21566,15 +21872,27 @@ def _user_ordinal_supplying_the_image(messages: list) -> Optional[int]:
return ordinal
-def _mark_image_owner_turn(messages: list, ordinal: int) -> list:
+def _mark_image_owner_turn(
+ messages: list,
+ ordinal: int,
+ marked_at: "list | None" = None,
+) -> list:
"""Give the *ordinal*-th user turn a structured image part ahead of its text: the
- renderers attach it to the newest turn unless one already carries it."""
+ renderers attach it to the newest turn unless one already carries it.
+
+ *marked_at* collects that turn's index. A replayed MCP picture has to be placed
+ ahead of it -- the pixels go history-first with the attachment last, and a
+ positional processor binds them in document order -- so the caller needs to know
+ which turn owns the attachment, not just that one does.
+ """
marked = list(messages)
seen = 0
for index, message in enumerate(marked):
if not isinstance(message, dict) or message.get("role") != "user":
continue
if seen == ordinal:
+ if marked_at is not None:
+ marked_at.append(index)
body = message.get("content")
if isinstance(body, str):
marked[index] = {
@@ -21868,6 +22186,8 @@ def _build_external_messages(
supports_vision: bool,
provider_type: Optional[str] = None,
base_url: Optional[str] = None,
+ promoted_out: "Optional[list]" = None,
+ promote_mcp_images: "Optional[bool]" = None,
) -> list[dict]:
"""
Convert ChatMessage list to OpenAI-compatible dicts for external providers.
@@ -21875,6 +22195,10 @@ def _build_external_messages(
Behaviour per content-part type:
- `text`: always preserved.
- `image_url`: preserved on vision providers; stripped on non-vision.
+ `promote_mcp_images` governs only whether a replayed MCP envelope becomes
+ image input; it defaults to `supports_vision` and must not be passed as
+ that flag, or a stricter MCP answer would also strip what the caller
+ attached.
- `input_document`: preserved ONLY when the provider's stream helper has
explicit translation logic (Anthropic + OpenAI today, see
``_INPUT_DOCUMENT_PROVIDERS``). Stripped for every other provider so the
@@ -22186,7 +22510,69 @@ def _build_external_messages(
entry["tool_calls"] = _replay_tool_call_ids(entry["tool_calls"], replay_ids)
if isinstance(entry.get("tool_call_id"), str):
entry["tool_call_id"] = replay_ids.get(entry["tool_call_id"], entry["tool_call_id"])
- return result
+ promote = supports_vision if promote_mcp_images is None else promote_mcp_images
+ # A provider applies its own per-request cap in document order, so the replay
+ # leaves the caller's own pictures room; llama-server is bounded by its context
+ # window instead, and the GGUF callers keep the full allowance.
+ return promote_mcp_history_images(
+ result, vision = promote, promoted_out = promoted_out, reserve_for_caller = True
+ )
+
+
+async def _promote_mcp_history_images_async(
+ messages,
+ *,
+ vision: bool,
+ promoted_out = None,
+):
+ """Promotion off the shared loop when there is really an envelope to rebuild.
+ Same hop, and the same reason, as the local and external replay paths take.
+ On marker presence alone, not on vision: a text-only target still has the
+ envelope stripped, and stripping json-loads the whole 12 MB array."""
+ if _messages_mention_mcp_images(messages):
+ return await asyncio.to_thread(
+ promote_mcp_history_images, messages, vision = vision, promoted_out = promoted_out
+ )
+ return promote_mcp_history_images(messages, vision = vision, promoted_out = promoted_out)
+
+
+async def _promote_local_mcp_images_async(
+ messages,
+ *,
+ vision: bool,
+ decode_cache = None,
+ caller_images = (),
+):
+ """Rebuilding a replayed envelope decodes and re-encodes every picture in it.
+ A permitted image runs to 40 megapixels, so that belongs off the shared loop --
+ the same hop the GGUF and external replay paths already take. Taken on marker
+ presence alone: a text-only target still has the envelope stripped, which parses
+ the same 12 MB array."""
+ if _messages_mention_mcp_images(messages):
+ return await asyncio.to_thread(
+ promote_mcp_history_images_local,
+ messages,
+ vision = vision,
+ decode_cache = decode_cache,
+ caller_images = caller_images,
+ )
+ return promote_mcp_history_images_local(
+ messages, vision = vision, decode_cache = decode_cache, caller_images = caller_images
+ )
+
+
+async def _build_external_messages_async(messages, supports_vision, **kwargs) -> list[dict]:
+ """Also collects what promotion created when the caller passes promoted_out."""
+ """Promoting a replayed envelope decodes and re-encodes every picture in it,
+ which is PIL work on the shared loop. Hop to a thread only when there is an
+ envelope to promote, so an ordinary text turn keeps its direct call."""
+ # On marker presence alone, not on whether promotion is on: a text-only target
+ # still has the envelope stripped, and stripping parses the same 12 MB array.
+ if _messages_mention_mcp_images(messages):
+ return await asyncio.to_thread(
+ _build_external_messages, messages, supports_vision, **kwargs
+ )
+ return _build_external_messages(messages, supports_vision, **kwargs)
async def _proxy_to_external_provider(
@@ -22427,11 +22813,25 @@ async def _proxy_to_external_provider(
if model in capabilities
else offered_subscription_model(payload.provider_id, model)
)
- if image_requested and model not in capabilities and listed_model is None:
+ # An MCP tool can hand this loop a picture on a later turn, and the vision
+ # flag is fixed here for the whole run: a text-only opening turn would
+ # otherwise pin it to False and discard that picture on a capable model.
+ _may_receive_image = (
+ image_requested
+ # A tool can hand this loop a picture on a later turn...
+ or bool(getattr(payload, "mcp_enabled", False))
+ # ...and a conversation that already carries one needs the capability
+ # resolved even when MCP has since been switched off, or the replay is
+ # stripped from a model that could have read it. Substring form: this
+ # runs on the loop, and the exact check parses the whole envelope.
+ or _request_has_promotable_mcp_images(payload, exact = False)
+ )
+ if _may_receive_image and model not in capabilities and listed_model is None:
# Admitted straight off the saved row, so no catalog read happened this
# process and nothing here knows the modalities. Defaulting to text-only
- # would refuse an image for a model the picker offered as capable. Only an
- # actual image is worth the fetch; a text turn should not pay for one.
+ # would refuse an image for a model the picker offered as capable. Only a
+ # request that can carry one is worth the fetch; a plain text turn with no
+ # tools should not pay for it.
try:
await ensure_subscription_models(payload.provider_id)
except (CodexAuthError, CodexReauthorizationError) as exc:
@@ -22473,11 +22873,13 @@ async def _proxy_to_external_provider(
mark_subscription_catalog_stale(payload.provider_id)
if model not in _allowed_codex_models():
raise HTTPException(status_code = 400, detail = "Choose a curated Codex model.")
- chat_messages = _build_external_messages(
+ _codex_promoted_parts: list = []
+ chat_messages = await _build_external_messages_async(
payload.messages,
model_supports_vision,
provider_type = provider_type,
base_url = base_url,
+ promoted_out = _codex_promoted_parts,
)
tool_payloads = [
tool.model_dump(exclude_none = True) if hasattr(tool, "model_dump") else tool
@@ -22561,6 +22963,8 @@ async def _proxy_to_external_provider(
response_format = _extract_response_format(payload),
tool_choice = payload.tool_choice,
continue_final_message = _continue_final_message(payload),
+ supports_vision = model_supports_vision,
+ promoted_image_parts = tuple(_codex_promoted_parts),
)
policy = (
CodexToolPolicy(
@@ -22790,11 +23194,21 @@ async def _proxy_to_external_provider(
_pinfo = _get_provider_info(provider_type) or {}
_supports_vision = _pinfo.get("supports_vision", False)
- chat_messages = _build_external_messages(
+ _external_promoted_parts: list = []
+ chat_messages = await _build_external_messages_async(
payload.messages,
+ # The provider-wide flag, so an image the caller attached is preserved
+ # exactly as it was before MCP promotion existed. The stricter
+ # model-level answer gates promotion alone: openrouter and huggingface
+ # are vision-capable for the family and name no model, so folding the
+ # two together stripped a picture sent to a chosen vision model.
_supports_vision,
provider_type = provider_type,
base_url = base_url,
+ promoted_out = _external_promoted_parts,
+ promote_mcp_images = _external_takes_mcp_images(
+ provider_type, _supports_vision, model, _pinfo
+ ),
)
monitor_id = None
if not getattr(request.state, "skip_api_monitor", False):
@@ -22927,6 +23341,10 @@ async def _proxy_to_external_provider(
model = model,
tool_choice = payload.tool_choice,
continue_final_message = _continue_final_message(payload),
+ supports_vision = _external_takes_mcp_images(
+ provider_type, _supports_vision, model, _pinfo
+ ),
+ promoted_image_parts = tuple(_external_promoted_parts),
),
policy = ToolLoopPolicy(
tools = external_studio_tools,
@@ -23564,7 +23982,7 @@ async def produce_openai_chat_completions(
_preprepared_audio = None
_image_preflight = None
if _should_validate_before_switch():
- _pre_parsed = _extract_content_parts(payload.messages)
+ _pre_parsed = _extract_content_parts(payload.messages, keep_tool_images = True)
if not _pre_parsed[1]:
raise HTTPException(
status_code = 400, detail = "At least one non-system message is required."
@@ -23670,7 +24088,7 @@ async def produce_openai_chat_completions(
# audio-only request asks for the projector alone, since an audio model's
# projector carries no vision tower. A safetensors or MLX checkpoint
# declares audio input separately, so it is tracked apart as well.
- _needs_image = bool(_pre_parsed[2]) or _request_has_image(payload)
+ _needs_image = bool(_pre_parsed[2]) or _request_has_attached_image(payload)
# Video rides that projector too. Its own /props gate can only run after
# the load, so this at least keeps a text-only target from evicting a
# working model to serve a clip it could never take.
@@ -23959,7 +24377,9 @@ async def produce_openai_chat_completions(
if _predecoded_audio is not None
else _decode_audio_base64(payload.audio_base64)
)
- system_prompt, chat_messages, _ = _extract_content_parts(payload.messages)
+ system_prompt, chat_messages, _ = await _extract_content_parts_async(
+ payload.messages
+ )
system_prompt = _apply_current_date_prompt(system_prompt, request)
except _DecodedAudioTooLongError as e:
# A limit the caller can act on, not a server fault.
@@ -24391,7 +24811,9 @@ async def produce_openai_chat_completions(
if _pre_parsed is not None:
system_prompt, chat_messages, extracted_image_b64 = _pre_parsed
else:
- system_prompt, chat_messages, extracted_image_b64 = _extract_content_parts(payload.messages)
+ system_prompt, chat_messages, extracted_image_b64 = _extract_content_parts(
+ payload.messages, keep_tool_images = True
+ )
# applied once so both backends inherit it, with or without tools, and never state it twice.
system_prompt = _apply_current_date_prompt(system_prompt, request)
@@ -24446,9 +24868,15 @@ async def produce_openai_chat_completions(
video_b64, _ = _video_b64_rejection(payload.video_base64)
video_b64 = await _shrink_clip_for_llama(video_b64, llama_backend)
+ # The parts promotion created from replayed envelopes. The loop's cap counts
+ # what MCP tools have returned across the whole conversation, so a resumed chat
+ # has to hand it these: seeded empty it counted only the current run and let a
+ # replayed eight sit beside a fresh eight.
+ _gguf_replayed_image_parts: list = []
gguf_messages, _ = await _openai_messages_for_gguf_chat_async(
payload,
llama_backend.is_vision,
+ _gguf_replayed_image_parts,
)
gguf_messages = _set_or_prepend_system_message(gguf_messages, system_prompt)
image_b64 = None
@@ -24692,6 +25120,7 @@ async def produce_openai_chat_completions(
def gguf_generate_with_tools():
return llama_backend.generate_chat_completion_with_tools(
messages = gguf_messages,
+ replayed_image_parts = tuple(_gguf_replayed_image_parts),
tools = tools_to_use,
temperature = payload.temperature,
top_p = payload.top_p,
@@ -24740,7 +25169,7 @@ async def produce_openai_chat_completions(
_tool_admission_mode = "chat_tool_stream" if payload.stream else "chat_tool_nonstream"
try:
- reservation, admission_config = _openai_llama_admission_reserve(
+ reservation, admission_config = await _openai_llama_admission_reserve_async(
request = request,
llama_backend = llama_backend,
payload = payload,
@@ -25569,7 +25998,7 @@ async def produce_openai_chat_completions(
_tracker = _TrackedCancel.for_payload(cancel_event, payload, *_cancel_keys)
_tracker.__enter__()
try:
- reservation, admission_config = _openai_llama_admission_reserve(
+ reservation, admission_config = await _openai_llama_admission_reserve_async(
request = request,
llama_backend = llama_backend,
payload = payload,
@@ -25913,7 +26342,7 @@ async def produce_openai_chat_completions(
)
else:
try:
- reservation, admission_config = _openai_llama_admission_reserve(
+ reservation, admission_config = await _openai_llama_admission_reserve_async(
request = request,
llama_backend = llama_backend,
payload = payload,
@@ -26198,7 +26627,9 @@ async def produce_openai_chat_completions(
served_images: list[str] = []
images: list = []
if _serves_several_images(backend):
- _, _msgs, _payloads = _extract_content_parts(payload.messages, structured = True)
+ _, _msgs, _payloads = _extract_content_parts(
+ payload.messages, structured = True, keep_tool_images = True
+ )
_legacy_distinct = _legacy_image_is_distinct(payload)
# A legacy image the messages do not carry joins the newest user turn, as GGUF splices it.
if _legacy_distinct:
@@ -26274,6 +26705,18 @@ async def produce_openai_chat_completions(
# Classify capability flags from the loaded template.
_sf_model_info = backend.models.get(backend.active_model_name, {})
+ # Strips for a text-only model, rebuilds the picture as a marker turn for one
+ # that reads images; either way no envelope reaches the template.
+ # One request's decodes, shared with the client-tool rebuild below, which promotes
+ # the same history again from the raw payload: a retained picture can be a 40
+ # megapixel raster, and it is decoded once, not once per promotion.
+ _sf_mcp_decode_cache: dict = {}
+ chat_messages, sf_mcp_images = await _promote_local_mcp_images_async(
+ chat_messages,
+ vision = bool(_sf_model_info.get("is_vision")),
+ decode_cache = _sf_mcp_decode_cache,
+ caller_images = served_images,
+ )
_sf_tpl = (_sf_model_info.get("chat_template_info") or {}).get("template")
# Resolve the tool policy BEFORE the protocol is classified: the template
# branch chosen here must be the one generation renders. Reading the raw
@@ -26397,6 +26840,32 @@ async def produce_openai_chat_completions(
_sf_template_tools
)
+ # A replayed MCP picture sends generation down the processor path just as an
+ # attachment does, so protocol classification has to read the same template it
+ # will render with -- otherwise tool schemas and reasoning markers are derived
+ # from the tokenizer template while the prompt comes from the processor one.
+ _sf_has_any_image = image is not None or bool(sf_mcp_images) or bool(images)
+ # Resolved BEFORE the capability gate below, which classifies from this body (#10092).
+ _sf_image_tpl = (
+ (_sf_model_info.get("chat_template_info") or {}).get("processor_template")
+ # A clip renders through the processor as an image does.
+ if _sf_has_any_image or _video_clip is not None
+ else None
+ )
+ # Differs from processor_template: a template-less processor still places the image.
+ _sf_renders_image = _sf_has_any_image and bool(
+ (_sf_model_info.get("chat_template_info") or {}).get("renders_image")
+ )
+ if _sf_image_tpl is not None:
+ # The WHOLE protocol, not just tool support: a processor body can carry a reasoning
+ # channel the tokenizer never declares, whose <think> markup then leaked as visible
+ # content. prefer_tool_use is off: a processor never selects "tool_use" (#10092).
+ _sf_features, _sf_parse_think, _sf_reasoning_prefilled = _sf_response_protocol(
+ _sf_template_tools,
+ template = _sf_image_tpl,
+ prefer_tool_use = False,
+ )
+
# A continued turn renders no generation prompt, so nothing is prefilled and the
# resumed text is the visible answer. Only the first turn continues; later tool-loop
# turns render a fresh generation prompt and prefill as usual.
@@ -26437,7 +26906,22 @@ async def produce_openai_chat_completions(
_sf_use_tools = (
(_sf_tools_on or _sf_mcp_allowed)
and _sf_features.get("supports_tools", False)
- and image is None
+ # An attachment used to withdraw the tools: the loop had no way to carry
+ # a picture. It has one now, so only a model that cannot read images does.
+ #
+ # Except when the caller sent its own catalog. #10092 routes image-plus-tools
+ # to the client passthrough so those schemas are the ones rendered; claiming
+ # the request here would answer it with Unsloth's built-ins instead, which is
+ # the client's tools silently gone. Studio's own chat names tools through
+ # enabled_tools rather than sending schemas, so the picture still reaches the
+ # loop there -- which is the request this whole path exists for.
+ #
+ # Gated on _sf_has_any_image, not on the attachment: a resumed chat whose
+ # only pictures are replayed MCP ones carries image input just the same, and
+ # reading `image is None` there let the loop take the request and swap the
+ # caller's schemas out on exactly the path this PR adds.
+ and (not _sf_has_any_image or (bool(_sf_model_info.get("is_vision")) and not payload.tools))
+ # A caller's several images and a video clip stay out of the loop, as on main.
and not images
and _video_clip is None
and not _sf_is_gptoss
@@ -26533,6 +27017,46 @@ async def produce_openai_chat_completions(
else:
_sf_chat_messages.append(_msg)
+ # The attachment rides in last, matching where its marker sits: the MCP
+ # pictures came from earlier turns.
+ _sf_loop_images = list(sf_mcp_images)
+ # Where the attachment lands among the replayed pixels: its marker sits on the
+ # turn that supplied it, which can precede a tool's pictures, so the position
+ # is read off the conversation rather than assumed to be last.
+ _sf_attachment_at: list[int] = []
+ if image is not None:
+ # The turn that supplied it, not simply the newest one: the extractor
+ # takes the newest user image from anywhere in the thread, so a
+ # text-only latest question would otherwise be told it carried an older
+ # picture -- and with replayed images in between, the marker order stops
+ # matching the pixel order too.
+ # Trimmed BEFORE the attachment's marker exists, which is what the
+ # ordering here is for. The trim drops by marker ORDINAL against a payload
+ # list that holds replay only, so with the attachment's turn ahead of the
+ # replayed pictures it deleted ordinal 0 -- the attachment's own marker --
+ # while charging replay payload 0, and every later pixel shifted onto the
+ # marker before it.
+ trim_mcp_image_turns(
+ _sf_chat_messages, sf_mcp_images, limit = _MCP_MAX_TOTAL_MODEL_IMAGES - 1
+ )
+ # Snapshot after the trim, so "history" names the markers that survived it.
+ _sf_prior_markers = mcp_image_marker_parts(_sf_chat_messages)
+ # Read off the conversation, not assumed: the attachment's marker sits
+ # on the turn that supplied it, which can precede a tool's pictures, and
+ # a positional VLM binds the Nth pixel to the Nth marker.
+ _sf_chat_messages = mark_mcp_image_turn(
+ _sf_chat_messages,
+ 1,
+ ordinal = _user_ordinal_supplying_the_image(payload.messages),
+ )
+ _sf_loop_images = mcp_pixels_in_marker_order(
+ _sf_chat_messages,
+ _sf_prior_markers,
+ sf_mcp_images,
+ _pil_to_png_b64(image),
+ placed_at = _sf_attachment_at,
+ )
+
# Request-scoped usage/timings receptacle (filled at gen_done).
_sf_stats_holder: dict = {}
@@ -26541,6 +27065,10 @@ async def produce_openai_chat_completions(
messages = _sf_chat_messages,
tools = _sf_tools_to_use,
system_prompt = _sf_system_prompt or "",
+ images = _sf_loop_images or None,
+ # The loop's cap trims the replay it grows; the caller's own picture
+ # is not the loop's to evict, so it is named by position.
+ caller_image_indexes = tuple(_sf_attachment_at),
temperature = payload.temperature,
top_p = payload.top_p,
top_k = payload.top_k,
@@ -26935,12 +27463,22 @@ async def produce_openai_chat_completions(
finally:
_sf_tracker.__exit__(None, None, None)
+ # The attachment is passed beside the replayed list and both reach the backend
+ # as one `attached` sequence, so the cap has to cover the pair. Reserve its slot
+ # by trimming the replay, oldest first, with the markers that go with them.
+ if image is not None and sf_mcp_images:
+ trim_mcp_image_turns(chat_messages, sf_mcp_images, limit = _MCP_MAX_TOTAL_MODEL_IMAGES - 1)
+
# Shared generation kwargs
gen_kwargs = dict(
messages = chat_messages,
system_prompt = system_prompt,
image = image,
- images = images,
+ images = sf_mcp_images or images or None,
+ # Which user turn actually supplied the attachment: the extractor takes the
+ # newest user image from anywhere in the thread, so the backends cannot
+ # assume it belongs to the latest question.
+ image_ordinal = _user_ordinal_supplying_the_image(payload.messages),
temperature = payload.temperature,
top_p = payload.top_p,
top_k = payload.top_k,
@@ -26971,27 +27509,10 @@ async def produce_openai_chat_completions(
# tools into the template, generate one turn, heal text-form calls (#6801).
# supports_tools=False falls through to plain relay (GGUF gate parity).
_sf_has_tool_msgs = any(m.role == "tool" or m.tool_calls for m in payload.messages)
- # Resolved BEFORE the capability gate below, which classifies from this body (#10092).
- _sf_has_image = image is not None or bool(images)
- # A clip renders through the processor as an image does.
- _sf_image_tpl = (
- (_sf_model_info.get("chat_template_info") or {}).get("processor_template")
- if _sf_has_image or _video_clip is not None
- else None
- )
- # Differs from processor_template: a template-less processor still places the image.
- _sf_renders_image = _sf_has_image and bool(
- (_sf_model_info.get("chat_template_info") or {}).get("renders_image")
- )
- if _sf_image_tpl is not None:
- # The WHOLE protocol, not just tool support: a processor body can carry a reasoning
- # channel the tokenizer never declares, whose <think> markup then leaked as visible
- # content. prefer_tool_use is off: a processor never selects "tool_use" (#10092).
- _sf_features, _sf_parse_think, _sf_reasoning_prefilled = _sf_response_protocol(
- _sf_template_tools,
- template = _sf_image_tpl,
- prefer_tool_use = False,
- )
+ # Classified against the processor body above, before the server loop's own gate:
+ # that gate reads the same flag, and deciding it here left the loop enabled on a
+ # tokenizer body that renders tools while generation used a processor body that
+ # does not, so the model was never shown the schemas the loop was driving.
_sf_supports_tools = _sf_features.get("supports_tools", False)
# Gate on _sf_use_tools (did the server-side path claim the request?), not
# raw mcp_enabled: an empty MCP registry must not silently drop client tools.
@@ -26999,8 +27520,9 @@ async def produce_openai_chat_completions(
# Read the resolved value, not a fresh _effective_enable_tools: the gate
# above withdraws the launcher default for exactly these requests, and
# recomputing here would hide that and drop the client catalog.
- # An image or a clip rules out the server loop, so the passthrough takes the request (#10092).
- (not _sf_tools_on or ((_sf_has_image or _video_clip is not None) and not _sf_use_tools))
+ # Once an image or a clip rules out the server loop the passthrough takes the
+ # request (#10092). _sf_has_any_image counts replayed MCP pictures too.
+ (not _sf_tools_on or ((_sf_has_any_image or _video_clip is not None) and not _sf_use_tools))
and not _sf_use_tools
and not _sf_is_gptoss
and _sf_supports_tools
@@ -27064,29 +27586,53 @@ async def produce_openai_chat_completions(
if served_images:
# One pass over the conversation this renders, so markers and payloads stay in step.
_sf_rebuilt, _sf_payloads = _conversation_with_image_markers(
- _openai_messages_for_passthrough(payload, normalize_images = False)
+ _openai_messages_for_passthrough(
+ payload, normalize_images = False, promote_mcp_images = False
+ )
+ )
+ _sf_rebuilt, _sf_rebuilt_images = await _promote_local_mcp_images_async(
+ _sf_rebuilt,
+ vision = _sf_renders_image,
+ decode_cache = _sf_mcp_decode_cache,
+ caller_images = _sf_payloads,
)
# The same pictures the extraction already decoded, unless the rebuild dropped one.
gen_kwargs["images"] = await _decode_request_images(
- backend, _sf_payloads, dict(zip(served_images, images)), reject = _reject
+ backend, _sf_rebuilt_images, dict(zip(served_images, images)), reject = _reject
)
gen_kwargs["messages"] = _set_or_prepend_system_message(
_structured_tool_history_for_local_template(_sf_rebuilt), system_prompt
)
else:
- gen_kwargs["messages"] = _set_or_prepend_system_message(
+ #
+ # Promote after flattening, which drops image parts. Preserve one marker per
+ # batch at its original position or merge it into the following user turn;
+ # a detached block can violate non-GGUF image limits and turn alternation.
+ _sf_rebuilt, _sf_rebuilt_images = await _promote_local_mcp_images_async(
_structured_tool_history_for_local_template(
_flatten_content_parts_for_local_template(
- # Not a llama-server body: the flatten below drops image parts.
- _openai_messages_for_passthrough(payload, normalize_images = False)
+ _openai_messages_for_passthrough(
+ payload, normalize_images = False, promote_mcp_images = False
+ )
)
),
- system_prompt,
- )
+ vision = _sf_renders_image,
+ decode_cache = _sf_mcp_decode_cache,
+ )
+ if image is not None and _sf_rebuilt_images:
+ # The attachment rides beside the replay and both reach the backend as
+ # one sequence, so its slot is reserved here as the server-tool path does.
+ trim_mcp_image_turns(
+ _sf_rebuilt, _sf_rebuilt_images, limit = _MCP_MAX_TOTAL_MODEL_IMAGES - 1
+ )
+ gen_kwargs["messages"] = _set_or_prepend_system_message(_sf_rebuilt, system_prompt)
+ gen_kwargs["images"] = _sf_rebuilt_images or None
# Mark the turn that owns the image so the newest-user-turn scan does not move an
# older picture onto a later question. Gated on _sf_renders_image, not on an image:
- # a text-template render must not be handed part lists (#10092). Kept markers need none.
- if _sf_renders_image and not served_images:
+ # a text-template render must not be handed part lists (#10092). Not when pictures
+ # were replayed or kept in place: the backends then place the attachment's marker
+ # themselves, by ordinal, and a marker set here would bind a replayed picture.
+ if _sf_renders_image and not _sf_rebuilt_images and not served_images:
_sf_image_ordinal = _user_ordinal_supplying_the_image(payload.messages)
if _sf_image_ordinal is not None:
gen_kwargs["messages"] = _mark_image_owner_turn(
@@ -27113,8 +27659,15 @@ async def produce_openai_chat_completions(
] or None
else:
gen_kwargs["tools"] = payload.tools
- elif _sf_renders_image:
+ elif _sf_renders_image and not sf_mcp_images:
# The plain route too: later turns then share the prefix that holds the image.
+ #
+ # Only when nothing was replayed. Both backends snapshot the conversation's
+ # existing markers as "history" before topping up, so a marker added HERE is
+ # counted as a replayed picture's and the pixels bind to the wrong turns: with
+ # the attachment on an earlier turn than a tool's picture, the model is shown
+ # the screenshot where its own diagram belongs. The top-up places the marker
+ # at the same ordinal anyway, so the only thing lost is the ambiguity.
_sf_image_ordinal = _user_ordinal_supplying_the_image(payload.messages)
if _sf_image_ordinal is not None:
gen_kwargs["messages"] = _mark_image_owner_turn(
@@ -30818,7 +31371,7 @@ async def _responses_stream(
cancel_event = threading.Event()
_tracker = _TrackedCancel.for_payload(cancel_event, payload, resp_id)
try:
- reservation, admission_config = _openai_llama_admission_reserve(
+ reservation, admission_config = await _openai_llama_admission_reserve_async(
request = request,
llama_backend = llama_backend,
# chat_req, not payload: a ResponsesRequest carries `input` and
@@ -32119,6 +32672,16 @@ def _scaled_from_16_bit(image):
return image.point(lambda v: v * (1.0 / 257), mode = "L")
+def _pil_to_png_b64(img) -> str:
+ buf = io.BytesIO()
+ # Composited, not merely converted: convert("RGB") keeps whatever colour sits
+ # under the alpha, so a transparent attachment whose background was never painted
+ # reaches the worker as black -- and its dark text with it. The ordinary IPC path
+ # carries the PNG's alpha through, so only this serialisation flattened it.
+ _mcp_flattened_rgb(img).save(buf, format = "PNG")
+ return base64.b64encode(buf.getvalue()).decode("ascii")
+
+
def _image_bytes_to_png_b64(raw: bytes) -> str:
"""Decode raw image bytes and re-encode to a base64-ascii PNG string.
@@ -32407,6 +32970,25 @@ def _resident_context_satisfies(model_info: dict, max_seq_length: Any) -> bool:
return _positive_int_or_none(max_seq_length) == _positive_int_or_none(recorded)
+async def _resident_model_reads_images() -> bool:
+ """Whether whatever is resident would be handed pixels for a promoted envelope.
+
+ Both backends are asked: llama.cpp answers for a loaded GGUF, and the inference
+ backend for a resident MLX or safetensors model, which serves its own count and
+ would otherwise never reach a llama-only guard.
+ """
+ if bool(getattr(get_llama_cpp_backend(), "is_vision", False)):
+ return True
+ # Off the loop for the same reason the MLX count is: building the singleton runs
+ # the torch import.
+ backend = await asyncio.to_thread(get_inference_backend)
+ active = getattr(backend, "active_model_name", None)
+ if not active:
+ return False
+ entry = (getattr(backend, "models", {}) or {}).get(active) or {}
+ return bool(entry.get("is_vision"))
+
+
async def _mlx_count_chat_tokens(payload, request = None) -> Optional[JSONResponse]:
"""Count with the resident MLX model's tokenizer, or None if MLX is not serving one.
@@ -32434,7 +33016,7 @@ async def _mlx_count_chat_tokens(payload, request = None) -> Optional[JSONRespon
# The completion's own helper: rebuilding it here is how a count prices a prompt
# nobody sends.
- system_prompt, messages, _image = _extract_content_parts(payload.messages)
+ system_prompt, messages, _image = await _extract_content_parts_async(payload.messages)
# The completion applies this once for both non-GGUF backends before it branches.
# Only with a request: the helper's requestless mode injects the date unconditionally.
if request is not None:
@@ -32539,12 +33121,17 @@ async def _mlx_count_chat_tokens(payload, request = None) -> Optional[JSONRespon
] or None
else:
_tools_to_use = payload.tools or None
- messages = _set_or_prepend_system_message(
+ messages = await _promote_mcp_history_images_async(
_structured_tool_history_for_local_template(
- _flatten_content_parts_for_local_template(_openai_messages_for_passthrough(payload))
+ _flatten_content_parts_for_local_template(
+ _openai_messages_for_passthrough(
+ payload, normalize_images = False, promote_mcp_images = False
+ )
+ )
),
- system_prompt,
+ vision = False,
)
+ messages = _set_or_prepend_system_message(messages, system_prompt)
system_prompt = ""
elif _tools_to_use:
# A PENDING turn is the shape this loop answers from exactly these messages, splicing
@@ -32725,7 +33312,7 @@ async def chat_count_tokens(
_tool["type"] = "function"
# /apply-template swaps each image for a short media marker, so refuse rather than undercount.
- if _request_has_image(payload):
+ if _request_has_attached_image(payload):
raise HTTPException(
status_code = 503,
detail = "Cannot count tokens for messages containing images.",
@@ -32743,6 +33330,23 @@ async def chat_count_tokens(
detail = "Cannot count tokens for messages containing video.",
)
+ # A replay envelope is countable on a text-only model, where generation strips it
+ # and sends no pixels. On a model that reads images the completion promotes it, so
+ # counting would underreport the prompt by every image-embedding token -- the same
+ # reason a direct attachment is refused above. Checked ahead of the MLX dispatch,
+ # which returns its own count and would otherwise never reach the guard.
+ # The exact check parses the envelope, so it runs off the loop, and only once the
+ # substring form says there is one to parse.
+ if (
+ _request_has_promotable_mcp_images(payload, exact = False)
+ and await asyncio.to_thread(_request_has_promotable_mcp_images, payload)
+ and await _resident_model_reads_images()
+ ):
+ raise HTTPException(
+ status_code = 503,
+ detail = "Cannot count tokens for messages containing images.",
+ )
+
llama_backend = get_llama_cpp_backend()
if not llama_backend.is_loaded:
# The refusals above are about the request and so are shared; the ones below are
@@ -32762,12 +33366,22 @@ async def chat_count_tokens(
_count_messages = payload.messages
if _folds_studio_tool_history(payload, llama_backend):
_count_messages = _folded_studio_tool_messages(_count_messages)
- openai_messages = _strip_provider_synthetic_tool_history(
- _drop_empty_assistant_sentinels([m.model_dump(exclude_none = True) for m in _count_messages])
+ # Awaited: stripping parses the whole envelope, and this recount runs in the
+ # background on every turn, so it must not do that on the shared loop.
+ openai_messages = await _promote_mcp_history_images_async(
+ _strip_provider_synthetic_tool_history(
+ _drop_empty_assistant_sentinels(
+ [m.model_dump(exclude_none = True) for m in _count_messages]
+ )
+ ),
+ # Never as pixels: /apply-template prices a media marker, not the image.
+ # This drops the base64 the completion path would have promoted, which is
+ # the one part a count must not price as text.
+ vision = False,
)
if not _takes_passthrough:
openai_messages = _coalesce_consecutive_user_turns(openai_messages)
- _system_prompt, _, _ = _extract_content_parts(payload.messages)
+ _system_prompt, _, _ = await _extract_content_parts_async(payload.messages)
# the verbatim passthrough carries no date line, so counting one here would overcount it.
if not _takes_passthrough:
_system_prompt = _apply_current_date_prompt(_system_prompt, request)
@@ -32976,6 +33590,36 @@ async def anthropic_count_tokens(
# Apply the same sanitization /messages does before generation, so the count
# matches the prompt the real request would build (otherwise empty-assistant
# sentinels / synthetic tool history inflate the count or hit the fallback).
+ # Named and judged BEFORE the sanitizer, in the same order /v1/messages uses: it
+ # folds every role="tool" into a user message on a template without tool-role
+ # support, and both the refusal and _promote only look at tool messages.
+ openai_messages = _named_anthropic_tool_results(openai_messages)
+
+ # Like /chat/count_tokens, refuse promoted images: /apply-template counts
+ # media markers without the projector tokens generation uses. Check the
+ # translated, named results so Anthropic blocks are recognized and non-MCP
+ # envelopes remain countable. Parse off the event loop after a substring check.
+ if (
+ llama_backend.is_vision
+ and _messages_mention_mcp_images(openai_messages)
+ and await asyncio.to_thread(_messages_have_promotable_mcp_images, openai_messages)
+ ):
+ raise HTTPException(
+ status_code = 400,
+ detail = "Cannot count tokens for messages containing images.",
+ )
+
+ # Priced as the images the request really sends, not as the base64 the envelope
+ # carries: rendered verbatim a replayed screenshot counts thousands of tokens the
+ # completion never sends, and the two endpoints must agree on the same prompt.
+ # Awaited, like the completion path: promotion is Pillow work on a permitted
+ # 40-megapixel raster, and running it inline stalls every other request on the loop.
+ openai_messages = await _promote_mcp_history_images_async(
+ openai_messages, vision = llama_backend.is_vision
+ )
+
+ # Apply the same sanitization /messages does before generation, so the count
+ # matches the prompt the real request would build.
openai_messages = _sanitize_anthropic_openai_messages(openai_messages, llama_backend)
# /apply-template fetches remote media even though token counting only needs its marker.
_placeholder_remote_images_for_count(openai_messages)
@@ -33289,11 +33933,26 @@ async def anthropic_messages(
# builders apply the same strip; without it an Anthropic /v1/messages caller
# replaying a prior provider-side tool_use forwards fake builtin tool
# history to a backend with no matching function declarations.
+ # BEFORE the sanitizer. On a template without tool-role support it folds every
+ # role="tool" into a user message, and _promote only looks at tool messages -- so
+ # the envelope survived as JSON in the prompt and the model read megabytes of
+ # base64 while being shown no picture at all.
+ openai_messages = _named_anthropic_tool_results(openai_messages)
+ _anthropic_replayed_image_parts: list = []
+ openai_messages = await _promote_mcp_history_images_async(
+ openai_messages,
+ vision = llama_backend.is_vision,
+ promoted_out = _anthropic_replayed_image_parts,
+ )
+
openai_messages = _sanitize_anthropic_openai_messages(openai_messages, llama_backend)
# Enforce vision guard + re-encode embedded images to PNG so the Anthropic
# endpoint matches /v1/chat/completions.
- if _anthropic_has_image:
+ # Promoted parts count too: _anthropic_has_image was read off the original
+ # blocks, so a replay-only request took the synchronous branch and re-decoded up
+ # to eight promoted PNGs on the shared loop.
+ if _anthropic_has_image or _anthropic_replayed_image_parts:
_has_image = await asyncio.to_thread(
_normalize_anthropic_openai_images,
openai_messages,
@@ -33342,9 +34001,9 @@ async def anthropic_messages(
# 1. enable_tools=true → server-side execution of built-in tools (Unsloth shorthand)
# 2. tools=[...] only → client-side pass-through (standard Anthropic behavior)
# 3. neither → plain chat
- # The server-side agentic loop doesn't support multimodal input -- matches
- # the `not image_b64` gate in /v1/chat/completions. requested_studio_tools and
- # the mixed-mode rejection were computed before the switch above.
+ # Match /v1/chat/completions: server tools reject caller attachments but
+ # accept replayed images. Check original blocks because promotion also sets
+ # _has_image. Tool selection and mixed-mode rejection preceded the switch.
openai_client_tools = [
tool
for tool in anthropic_tools_to_openai(payload.tools or [])
@@ -33356,7 +34015,9 @@ async def anthropic_messages(
# enable_tools=false). Explicit False always wins. Same predicate as the
# permission gate above: deciding "did this request select server tools"
# twice is what let the gate reject requests the router then served.
- server_tools = _selects_server_tools and llama_backend.supports_tools and not _has_image
+ server_tools = (
+ _selects_server_tools and llama_backend.supports_tools and not _anthropic_has_image
+ )
# One short-circuiting chain: a backend whose supports_tools raises must not turn a plain
# no-tools turn into a 500.
client_tools = (
@@ -33560,13 +34221,17 @@ async def anthropic_messages(
async def _admitted_anthropic(coro, *, tool_loop: bool = False):
try:
- reservation, admission_config = _openai_llama_admission_reserve(
+ reservation, admission_config = await _openai_llama_admission_reserve_async(
request = request,
llama_backend = llama_backend,
payload = payload,
tool_loop = tool_loop,
# Only the tool branch resolves a catalogue; the plain branch sends none.
injected_tools = openai_tools if tool_loop else None,
+ # The list generation really runs on: already translated out of the
+ # Anthropic block shape and already promoted, so a replayed screenshot
+ # is charged as the image it becomes rather than as its base64.
+ messages_override = openai_messages,
)
if tool_loop:
_anthropic_admission_hold["reservation"] = reservation
@@ -33815,6 +34480,7 @@ async def anthropic_messages(
return llama_backend.generate_chat_completion_with_tools(
reasoning_provenance = _think_prov,
messages = openai_messages,
+ replayed_image_parts = tuple(_anthropic_replayed_image_parts),
tools = openai_tools,
temperature = temperature,
top_p = top_p,
@@ -35586,6 +36252,24 @@ def _template_supports_tools(backend) -> bool:
return True
+def _named_anthropic_tool_results(messages: list[dict]) -> list[dict]:
+ """Stamp each role="tool" message with the tool it answers.
+
+ anthropic_messages_to_openai renders a tool_result block as tool_call_id plus
+ content and no ``name``, and _promote reads an absent name as legacy MCP history
+ it may trust. Delegates to the positional resolver rather than a conversation-wide
+ id map: the backend restarts ids like call_0 every response, and a last-wins map
+ renamed every earlier result with that id after the newest call.
+ """
+ names = _mcp_resolve_tool_names(messages)
+ return [
+ {**message, "name": names[index]}
+ if message.get("role") == "tool" and not message.get("name") and index in names
+ else message
+ for index, message in enumerate(messages)
+ ]
+
+
def _sanitize_anthropic_openai_messages(messages: list[dict], backend) -> list[dict]:
"""Post-conversion chain shared by /v1/messages and its token counter, so a
diverging list can never make input_tokens describe a different prompt.
@@ -35775,7 +36459,12 @@ def _splice_image_into_last_user(messages: list[dict], image_part: dict) -> None
messages.append({"role": "user", "content": [image_part]})
-def _openai_messages_for_passthrough(payload, normalize_images: bool = True) -> list[dict]:
+def _openai_messages_for_passthrough(
+ payload,
+ vision: bool = False,
+ normalize_images: bool = True,
+ promote_mcp_images: bool = True,
+) -> list[dict]:
"""Build OpenAI-format message dicts for the /v1/chat/completions
passthrough path.
@@ -35805,6 +36494,10 @@ def _openai_messages_for_passthrough(payload, normalize_images: bool = True) ->
messages = _strip_provider_synthetic_tool_history(
_drop_empty_assistant_sentinels([m.model_dump(exclude_none = True) for m in payload.messages])
)
+ # Left on for the local-template rebuild, which promotes them itself once its
+ # flatten has run: promoted here, the flatten would drop the parts again.
+ if promote_mcp_images:
+ messages = promote_mcp_history_images(messages, vision = vision)
if normalize_images:
_normalize_openai_image_parts_to_png(messages)
@@ -35881,7 +36574,11 @@ def _structured_tool_history_for_local_template(messages: list[dict]) -> list[di
return out
-def _openai_messages_for_gguf_chat(payload, is_vision: bool) -> tuple[list[dict], bool]:
+def _openai_messages_for_gguf_chat(
+ payload,
+ is_vision: bool,
+ promoted_out: "Optional[list]" = None,
+) -> tuple[list[dict], bool]:
"""Build llama-server messages for the standard GGUF chat path.
llama-server accepts OpenAI multimodal content parts directly. Preserve all
@@ -35911,13 +36608,22 @@ def _openai_messages_for_gguf_chat(payload, is_vision: bool) -> tuple[list[dict]
}
_splice_image_into_last_user(messages, image_part)
has_image = _normalize_anthropic_openai_images(messages, is_vision)
- return messages, has_image
+ return (
+ promote_mcp_history_images(messages, vision = is_vision, promoted_out = promoted_out),
+ has_image,
+ )
-async def _openai_messages_for_gguf_chat_async(payload, is_vision: bool) -> tuple[list[dict], bool]:
+async def _openai_messages_for_gguf_chat_async(
+ payload,
+ is_vision: bool,
+ promoted_out: "Optional[list]" = None,
+) -> tuple[list[dict], bool]:
if _request_has_image(payload):
- return await asyncio.to_thread(_openai_messages_for_gguf_chat, payload, is_vision)
- return _openai_messages_for_gguf_chat(payload, is_vision)
+ return await asyncio.to_thread(
+ _openai_messages_for_gguf_chat, payload, is_vision, promoted_out
+ )
+ return _openai_messages_for_gguf_chat(payload, is_vision, promoted_out)
def _extract_response_format(payload):
@@ -35955,7 +36661,9 @@ def _build_openai_passthrough_body(
extensions (``enable_tools``, ``enabled_tools``, ``session_id``, ...) never
leak to the backend.
"""
- messages = _openai_messages_for_passthrough(payload)
+ messages = _openai_messages_for_passthrough(
+ payload, vision = bool(getattr(llama_backend, "is_vision", False))
+ )
system_prompt, _, _ = _extract_content_parts(payload.messages)
messages = _set_or_prepend_system_message(messages, system_prompt)
# Markup is broken in _build_passthrough_payload, shared with both /v1/messages (#7066).
@@ -36036,7 +36744,7 @@ async def _openai_passthrough_stream(
_tracker = _TrackedCancel.for_payload(cancel_event, payload, *_cancel_keys)
_tracker.__enter__()
try:
- reservation, admission_config = _openai_llama_admission_reserve(
+ reservation, admission_config = await _openai_llama_admission_reserve_async(
request = request,
llama_backend = llama_backend,
payload = payload,
@@ -37097,7 +37805,7 @@ async def _openai_passthrough_non_streaming(
):
"""Non-streaming pass-through guarded by local llama-server admission."""
try:
- reservation, admission_config = _openai_llama_admission_reserve(
+ reservation, admission_config = await _openai_llama_admission_reserve_async(
request = request,
llama_backend = llama_backend,
payload = payload,
diff --git a/studio/backend/tests/test_anthropic_messages.py b/studio/backend/tests/test_anthropic_messages.py
index cbc53e29311..a4f08335bfe 100644
--- a/studio/backend/tests/test_anthropic_messages.py
+++ b/studio/backend/tests/test_anthropic_messages.py
@@ -5490,3 +5490,357 @@ def test_x_unsloth_effort_still_outranks_thinking_when_sent_explicitly():
)
assert args["enable_thinking"] is True
assert args["reasoning_effort"] == "high"
+
+
+def test_the_anthropic_paths_promote_a_replayed_mcp_envelope():
+ """A client replaying an Anthropic history sends the envelope back inside the
+ tool_result. Without promotion the model reads megabytes of base64 as text and is
+ shown no picture -- the very defect this feature exists to remove, on the endpoint
+ Claude Code actually uses."""
+ import inspect
+
+ from routes import inference
+
+ generate = inspect.getsource(inference.anthropic_messages)
+ assert (
+ "_promote_mcp_history_images_async(" in generate
+ ), "the /v1/messages path never promotes the replayed envelope"
+ assert (
+ "replayed_image_parts = tuple(_anthropic_replayed_image_parts)" in generate
+ ), "the loop's conversation cap has to start from what promotion put back"
+
+ count = inspect.getsource(inference.anthropic_count_tokens)
+ assert "await _promote_mcp_history_images_async(" in count, (
+ "the count endpoint prices the base64 the completion never sends, and must "
+ "not do the Pillow work inline on the event loop"
+ )
+ assert "messages_override = openai_messages" in generate, (
+ "admission has to reserve against the translated list, or an Anthropic "
+ "tool_result block prices its envelope as dense text"
+ )
+
+
+def test_an_anthropic_replay_hands_the_model_the_picture_not_the_base64():
+ import asyncio
+ import base64
+ import io
+ import json
+
+ from PIL import Image
+
+ from core.inference import mcp_images
+ from routes.inference import _promote_mcp_history_images_async
+
+ buffer = io.BytesIO()
+ Image.new("RGB", (12, 12), (24, 90, 219)).save(buffer, format = "PNG")
+ payload = base64.b64encode(buffer.getvalue()).decode()
+ envelope = json.dumps([{"data": payload, "mimeType": "image/png"}])
+
+ # The shape anthropic_messages_to_openai produces from a replayed tool_result.
+ messages = [
+ {"role": "user", "content": "take a screenshot"},
+ {
+ "role": "assistant",
+ "content": "",
+ "tool_calls": [
+ {
+ "id": "toolu_0",
+ "type": "function",
+ "function": {"name": "mcp__shot__capture", "arguments": "{}"},
+ }
+ ],
+ },
+ {
+ "role": "tool",
+ "tool_call_id": "toolu_0",
+ "name": "mcp__shot__capture",
+ "content": "[1 image returned]\n" + mcp_images.SENTINEL + envelope,
+ },
+ {"role": "user", "content": "what colour was it"},
+ ]
+
+ promoted: list = []
+ out = asyncio.run(
+ _promote_mcp_history_images_async(messages, vision = True, promoted_out = promoted)
+ )
+
+ text = "".join(
+ message["content"] if isinstance(message.get("content"), str) else "" for message in out
+ ) + "".join(
+ part.get("text", "")
+ for message in out
+ if isinstance(message.get("content"), list)
+ for part in message["content"]
+ if part.get("type") == "text"
+ )
+ assert mcp_images.SENTINEL not in text
+ assert payload not in text, "the base64 reached the model as prompt text"
+ assert len(promoted) == 1
+ assert (
+ sum(
+ 1
+ for message in out
+ if isinstance(message.get("content"), list)
+ for part in message["content"]
+ if part.get("type") == "image_url"
+ )
+ == 1
+ )
+
+
+def test_a_text_only_anthropic_model_is_not_shown_the_envelope_either():
+ import asyncio
+ import json
+
+ from core.inference import mcp_images
+ from routes.inference import _promote_mcp_history_images_async
+
+ envelope = json.dumps([{"data": "QUJD", "mimeType": "image/png"}])
+ messages = [
+ {
+ "role": "tool",
+ "tool_call_id": "toolu_0",
+ "name": "mcp__shot__capture",
+ "content": "[1 image returned]\n" + mcp_images.SENTINEL + envelope,
+ }
+ ]
+
+ out = asyncio.run(_promote_mcp_history_images_async(messages, vision = False))
+
+ assert mcp_images.SENTINEL not in json.dumps(out)
+ assert not any(isinstance(message.get("content"), list) for message in out)
+
+
+def test_the_anthropic_count_refuses_a_promoted_image_rather_than_undercount():
+ """count_chat_tokens renders /apply-template, which swaps each image for a short
+ media marker. Counting a promoted envelope there reports none of the projector
+ tokens /v1/messages really spends, and an undercount is what a client sizes its
+ context against -- so the OpenAI counter refuses this shape and so must this one."""
+ import inspect
+
+ from routes import inference
+
+ count = inspect.getsource(inference.anthropic_count_tokens)
+ refusal = count.index(
+ "asyncio.to_thread(_messages_have_promotable_mcp_images, openai_messages)"
+ )
+ promotion = count.index("_promote_mcp_history_images_async(")
+ assert refusal < promotion, (
+ "the refusal has to come BEFORE promotion, or the envelope is already "
+ "image parts by the time it is checked"
+ )
+ assert "Cannot count tokens for messages containing images." in count
+ assert (
+ "llama_backend.is_vision\n"
+ " and _messages_mention_mcp_images(openai_messages)\n"
+ " and await asyncio.to_thread(_messages_have_promotable_mcp_images, openai_messages)"
+ ) in count, (
+ "a text-only model has the envelope stripped and sends no pixels, so it "
+ "must still be counted rather than refused"
+ )
+
+
+def test_the_anthropic_envelope_is_promoted_before_tool_roles_are_folded_away():
+ """A template without tool-role support has the sanitizer fold every role="tool"
+ into a user message. _promote only looks at tool messages, so promoting after it
+ left the envelope as JSON in the prompt: megabytes of base64 read as text, and no
+ picture shown at all."""
+ import inspect
+
+ from routes import inference
+
+ for name, source in (
+ ("generation", inspect.getsource(inference.anthropic_messages)),
+ ("count", inspect.getsource(inference.anthropic_count_tokens)),
+ ):
+ promote = source.index("_promote_mcp_history_images_async(")
+ sanitize = source.index("_sanitize_anthropic_openai_messages(openai_messages")
+ assert (
+ promote < sanitize
+ ), f"{name}: the fold runs first and the envelope never reaches promotion"
+ named = source.index("_named_anthropic_tool_results(openai_messages)")
+ assert named < promote, f"{name}: provenance has to be restored first"
+
+
+def test_an_anthropic_client_tool_is_not_trusted_as_an_mcp_image_source():
+ """anthropic_messages_to_openai renders a tool_result with tool_call_id and no
+ name, and _promote reads an absent name as legacy MCP history it may trust. An
+ ordinary client tool whose output merely ends in a valid suffix was therefore
+ promoted as image input on the strength of nothing."""
+ import base64
+ import io
+ import json
+
+ from PIL import Image
+
+ from core.inference import mcp_images
+ from routes.inference import _named_anthropic_tool_results
+
+ buffer = io.BytesIO()
+ Image.new("RGB", (8, 8), (10, 120, 200)).save(buffer, format = "PNG")
+ envelope = json.dumps(
+ [{"data": base64.b64encode(buffer.getvalue()).decode(), "mimeType": "image/png"}]
+ )
+ translated = [
+ {
+ "role": "assistant",
+ "content": "",
+ "tool_calls": [
+ {
+ "id": "toolu_0",
+ "type": "function",
+ "function": {"name": "read_file", "arguments": "{}"},
+ }
+ ],
+ },
+ {
+ "role": "tool",
+ "tool_call_id": "toolu_0",
+ "content": "here you go\n" + mcp_images.SENTINEL + envelope,
+ },
+ ]
+
+ named = _named_anthropic_tool_results(translated)
+ assert named[1]["name"] == "read_file", "the result was not correlated to its call"
+
+ out = mcp_images.promote_history(named, vision = True)
+ assert not any(
+ isinstance(m.get("content"), list) for m in out
+ ), "a non-mcp__ tool was promoted as trusted image input"
+ # The suffix still comes off the text for everyone, MCP or not.
+ assert mcp_images.SENTINEL not in json.dumps(out)
+
+
+def test_a_real_mcp_tool_still_promotes_through_the_anthropic_naming():
+ import base64
+ import io
+ import json
+
+ from PIL import Image
+
+ from core.inference import mcp_images
+ from routes.inference import _named_anthropic_tool_results
+
+ buffer = io.BytesIO()
+ Image.new("RGB", (8, 8), (10, 120, 200)).save(buffer, format = "PNG")
+ envelope = json.dumps(
+ [{"data": base64.b64encode(buffer.getvalue()).decode(), "mimeType": "image/png"}]
+ )
+ translated = [
+ {
+ "role": "assistant",
+ "content": "",
+ "tool_calls": [
+ {
+ "id": "toolu_0",
+ "type": "function",
+ "function": {"name": "mcp__shot__capture", "arguments": "{}"},
+ }
+ ],
+ },
+ {
+ "role": "tool",
+ "tool_call_id": "toolu_0",
+ "content": "[1 image returned]\n" + mcp_images.SENTINEL + envelope,
+ },
+ ]
+
+ out = mcp_images.promote_history(_named_anthropic_tool_results(translated), vision = True)
+
+ assert (
+ sum(
+ 1
+ for message in out
+ if isinstance(message.get("content"), list)
+ for part in message["content"]
+ if part.get("type") == "image_url"
+ )
+ == 1
+ )
+
+
+def test_repeated_anthropic_call_ids_are_paired_positionally():
+ """A conversation-wide id map renamed every earlier result with that id after the
+ newest call, suppressing an earlier MCP picture or trusting an earlier non-MCP one."""
+ from routes.inference import _named_anthropic_tool_results
+
+ def _call(name):
+ return {
+ "role": "assistant",
+ "content": "",
+ "tool_calls": [
+ {"id": "call_0", "type": "function", "function": {"name": name, "arguments": "{}"}}
+ ],
+ }
+
+ named = _named_anthropic_tool_results(
+ [
+ _call("mcp__shot__capture"),
+ {"role": "tool", "tool_call_id": "call_0", "content": "first"},
+ _call("read_file"),
+ {"role": "tool", "tool_call_id": "call_0", "content": "second"},
+ ]
+ )
+
+ assert [m.get("name") for m in named if m["role"] == "tool"] == [
+ "mcp__shot__capture",
+ "read_file",
+ ]
+
+
+def test_promoted_parts_take_the_anthropic_normalizer_off_the_loop():
+ """_anthropic_has_image was read off the original blocks, so a replay-only request
+ took the synchronous branch and re-decoded up to eight promoted PNGs on the loop."""
+ import inspect
+
+ from routes import inference
+
+ src = inspect.getsource(inference.anthropic_messages)
+ assert "if _anthropic_has_image or _anthropic_replayed_image_parts:" in src
+
+
+def test_the_anthropic_count_refusal_is_name_aware():
+ """The names are stamped from the calls right before the check, so a client tool
+ whose output merely ends in a valid envelope -- never promoted, suffix stripped --
+ must not turn a countable prompt into a 400."""
+ import inspect
+ import json
+
+ from core.inference import mcp_images
+ from routes import inference
+ from routes.inference import _messages_have_promotable_mcp_images, _named_anthropic_tool_results
+
+ src = inspect.getsource(inference.anthropic_count_tokens)
+ assert "asyncio.to_thread(_messages_have_promotable_mcp_images, openai_messages)" in src
+ assert "_messages_have_mcp_image_envelope(openai_messages)" not in src
+
+ envelope = json.dumps([{"data": "QUJD", "mimeType": "image/png"}])
+
+ def _translated(tool):
+ return _named_anthropic_tool_results(
+ [
+ {
+ "role": "assistant",
+ "content": "",
+ "tool_calls": [
+ {
+ "id": "toolu_0",
+ "type": "function",
+ "function": {"name": tool, "arguments": "{}"},
+ }
+ ],
+ },
+ {
+ "role": "tool",
+ "tool_call_id": "toolu_0",
+ "content": "out\n" + mcp_images.SENTINEL + envelope,
+ },
+ ]
+ )
+
+ assert not _messages_have_promotable_mcp_images(_translated("read_file"))
+ assert _messages_have_promotable_mcp_images(_translated("mcp__shot__capture"))
+ # Unnamed and uncorrelated is still legacy history, and still refused.
+ assert _messages_have_promotable_mcp_images(
+ [{"role": "tool", "tool_call_id": "x", "content": "out\n" + mcp_images.SENTINEL + envelope}]
+ )
diff --git a/studio/backend/tests/test_chat_template_tool_arguments.py b/studio/backend/tests/test_chat_template_tool_arguments.py
index 8a927ea93ce..276f8b48ce5 100644
--- a/studio/backend/tests/test_chat_template_tool_arguments.py
+++ b/studio/backend/tests/test_chat_template_tool_arguments.py
@@ -309,3 +309,75 @@ def test_lenient_template_never_sees_a_split_conversation():
apply_chat_template_for_generation(_Lenient(), _parallel_conv())
assert seen["n"] == 4 # unsplit
+
+
+_MINISTRAL_IMAGE_TEMPLATE = """
+{% set ns = namespace(index=0) %}
+{% for message in messages %}
+{% if message.role == 'user' or (message.role == 'assistant' and not message.tool_calls) %}
+{% if (message.role == 'user') != (ns.index % 2 == 0) %}
+{{ raise_exception('After the optional system message, conversation roles must alternate user and assistant roles except for tool calls and results.') }}
+{% endif %}
+{% set ns.index = ns.index + 1 %}
+{% endif %}
+{% for part in message.content if message.content is not string %}
+{% if part.type == 'image' %}[IMG]{% endif %}
+{% endfor %}
+{% endfor %}
+"""
+
+
+class _MinistralImageTokenizer:
+ chat_template = _MINISTRAL_IMAGE_TEMPLATE
+
+ def apply_chat_template(self, messages, **kwargs):
+ from transformers.utils.chat_template_utils import _compile_jinja_template
+ self.messages = messages
+ return _compile_jinja_template(self.chat_template).render(messages = messages)
+
+
[email protected]("processor", [False, True])
+def test_tool_image_turn_renders_with_ministral_alternation(processor):
+ from core.inference.chat_template_helpers import render_prompt_with_boundary
+ from core.inference.mcp_images import placeholder_turn
+
+ conversation = _conv({})
+ conversation.append(placeholder_turn(1))
+ conversation.append({"role": "assistant", "content": "The image is red."})
+ conversation.append({"role": "user", "content": "Another screenshot, please."})
+ conversation.extend(_conv({})[1:])
+ conversation.append(placeholder_turn(1))
+ tokenizer = _MinistralImageTokenizer()
+ render = render_prompt_with_boundary if processor else apply_chat_template_for_generation
+ assert render(tokenizer, conversation).count("[IMG]") == 2
+ assert [m["role"] for m in tokenizer.messages] == [
+ "user",
+ "assistant",
+ "tool",
+ "assistant",
+ "user",
+ "assistant",
+ "user",
+ "assistant",
+ "tool",
+ "assistant",
+ "user",
+ ]
+ assert len(conversation) == 9
+
+
+def test_image_boundary_repair_preserves_markers_and_working_templates():
+ from core.inference.mcp_images import placeholder_turn, prepare_image_turn_boundaries
+
+ conversation = _conv({}) + [placeholder_turn(1)]
+ assert prepare_image_turn_boundaries(conversation, None) is conversation
+ assert prepare_image_turn_boundaries(conversation, "{{ messages }}") is conversation
+ fixed = prepare_image_turn_boundaries(conversation, _MINISTRAL_IMAGE_TEMPLATE)
+ assert fixed[-1] is conversation[-1]
+ assert prepare_image_turn_boundaries(fixed, _MINISTRAL_IMAGE_TEMPLATE) is fixed
+
+
+def test_image_boundary_repair_does_not_hide_invalid_caller_roles():
+ conversation = _conv({}) + [{"role": "user", "content": "A second user turn."}]
+ with pytest.raises(Exception, match = "conversation roles must alternate"):
+ apply_chat_template_for_generation(_MinistralImageTokenizer(), conversation)
diff --git a/studio/backend/tests/test_external_tool_call_id_replay.py b/studio/backend/tests/test_external_tool_call_id_replay.py
index 86eb892d5ab..249af5e9cab 100644
--- a/studio/backend/tests/test_external_tool_call_id_replay.py
+++ b/studio/backend/tests/test_external_tool_call_id_replay.py
@@ -162,3 +162,111 @@ def test_replay_is_idempotent_for_every_provider():
once, _ = _replayed_ids(MINTED, provider_type = provider)
twice, paired = _replayed_ids(once, provider_type = provider)
assert twice == once == paired, (provider, once, twice)
+
+
+def _mcp_image_history():
+ import base64
+ import io
+ import json
+
+ from PIL import Image
+
+ from core.inference import mcp_images
+
+ buffer = io.BytesIO()
+ Image.new("RGB", (6, 6), (10, 120, 200)).save(buffer, format = "PNG")
+ envelope = json.dumps(
+ [{"data": base64.b64encode(buffer.getvalue()).decode(), "mimeType": "image/png"}]
+ )
+ return [
+ ChatMessage.model_validate(
+ {
+ "role": "assistant",
+ "content": "",
+ "tool_calls": [
+ {
+ "id": ORIGINAL,
+ "type": "function",
+ "function": {"name": "mcp__fs__read_media_file", "arguments": "{}"},
+ }
+ ],
+ }
+ ),
+ ChatMessage.model_validate(
+ {
+ "role": "tool",
+ "tool_call_id": ORIGINAL,
+ "content": "[1 image returned]\n" + mcp_images.SENTINEL + envelope,
+ }
+ ),
+ ]
+
+
+def test_replayed_mcp_images_become_an_image_turn_on_a_vision_provider():
+ out = _build_external_messages(
+ _mcp_image_history(), supports_vision = True, provider_type = "openai"
+ )
+
+ assert [message["role"] for message in out] == ["assistant", "tool", "user"]
+ assert out[1]["content"] == "[1 image returned]"
+ assert out[2]["content"][1]["image_url"]["url"].startswith("data:image/png;base64,")
+
+
+def test_replayed_mcp_images_are_stripped_on_a_text_only_provider():
+ out = _build_external_messages(
+ _mcp_image_history(), supports_vision = False, provider_type = "openai"
+ )
+
+ assert [message["role"] for message in out] == ["assistant", "tool"]
+ assert out[1]["content"] == "[1 image returned]"
+
+
+def test_replay_promotion_is_moved_off_the_event_loop(monkeypatch):
+ """Promoting an envelope decodes and re-encodes every picture in it. That is
+ PIL work, and _proxy_to_external_provider calls the builder from the loop."""
+ import asyncio
+
+ import routes.inference as inference_route
+
+ hops: list = []
+ real_to_thread = asyncio.to_thread
+
+ async def counting_to_thread(fn, *a, **kw):
+ hops.append(getattr(fn, "__name__", str(fn)))
+ return await real_to_thread(fn, *a, **kw)
+
+ monkeypatch.setattr(inference_route.asyncio, "to_thread", counting_to_thread)
+
+ out = asyncio.run(
+ inference_route._build_external_messages_async(
+ _mcp_image_history(), True, provider_type = "openai"
+ )
+ )
+
+ assert hops == ["_build_external_messages"]
+ assert out[-1]["content"][1]["image_url"]["url"].startswith("data:image/png;base64,")
+
+
+def test_a_plain_conversation_keeps_its_direct_call(monkeypatch):
+ """No envelope, no PIL work: the thread hop would be pure overhead."""
+ import asyncio
+
+ import routes.inference as inference_route
+
+ hops: list = []
+
+ async def counting_to_thread(fn, *a, **kw):
+ hops.append(getattr(fn, "__name__", str(fn)))
+ return await asyncio.to_thread(fn, *a, **kw)
+
+ monkeypatch.setattr(inference_route.asyncio, "to_thread", counting_to_thread)
+
+ asyncio.run(
+ inference_route._build_external_messages_async(
+ [ChatMessage.model_validate({"role": "user", "content": "hello"})],
+ True,
+ provider_type = "openai",
+ )
+ )
+
+ assert hops == []
diff --git a/studio/backend/tests/test_llama_cpp_tool_loop.py b/studio/backend/tests/test_llama_cpp_tool_loop.py
index 32e93c68f73..44f96462701 100644
--- a/studio/backend/tests/test_llama_cpp_tool_loop.py
+++ b/studio/backend/tests/test_llama_cpp_tool_loop.py
@@ -115,6 +115,9 @@ def _run_tool_loop(
)
+_MCP_PNG_B64 = "iVBORw0KGgoAAAANSUhEUgAAAAYAAAAGCAIAAABvrngfAAAAFElEQVR4nGM8ISfHgAqY0PhUFgIAdYgBEED+8ToAAAAASUVORK5CYII="
+
+
def _sse(delta: dict) -> str:
return "data: " + json.dumps({"choices": [{"index": 0, "delta": delta}]}) + "\n"
@@ -6488,6 +6491,58 @@ def test_the_synthesized_final_pass_is_recosted_before_it_is_sent(monkeypatch):
)
+def _mcp_image_result() -> str:
+ from core.inference import mcp_images
+ image = {"data": _MCP_PNG_B64, "mimeType": "image/png"}
+ return "[1 image returned]\n" + mcp_images.SENTINEL + json.dumps([image])
+
+
+def _vision_backend(monkeypatch, streams, payloads, *, vision: bool):
+ backend = _make_backend(monkeypatch, streams, payloads)
+ backend._is_vision = vision
+ backend._mmproj_accepts_image = vision
+ return backend
+
+
+def _run_mcp_image_turn(monkeypatch, *, vision: bool) -> list[dict]:
+ streams = [
+ _structured_tool_call("mcp__fs__read_media_file", {"path": "cat.png"}, "call_mcp"),
+ [_sse({"content": "A tabby cat."}), _done()],
+ ]
+ payloads: list[dict] = []
+ backend = _vision_backend(monkeypatch, streams, payloads, vision = vision)
+ monkeypatch.setattr(
+ "core.inference.tools.execute_tool",
+ lambda name, arguments, **_kwargs: _mcp_image_result(),
+ )
+
+ list(
+ backend.generate_chat_completion_with_tools(
+ messages = [{"role": "user", "content": "describe the image"}],
+ tools = [{"type": "function", "function": {"name": "mcp__fs__read_media_file"}}],
+ max_tool_iterations = 2,
+ )
+ )
+ return payloads[1]["messages"]
+
+
+def test_mcp_images_reach_a_vision_model_as_their_own_user_turn(monkeypatch):
+ messages = _run_mcp_image_turn(monkeypatch, vision = True)
+
+ tool_message = next(m for m in messages if m["role"] == "tool")
+ assert "__MCP_IMAGES__" not in tool_message["content"]
+ assert messages[-1]["role"] == "user"
+ assert messages[-1]["content"][1]["type"] == "image_url"
+ assert messages[-1]["content"][1]["image_url"]["url"].startswith("data:image/png;base64,")
+
+
+def test_mcp_images_are_not_sent_to_a_text_only_model(monkeypatch):
+ messages = _run_mcp_image_turn(monkeypatch, vision = False)
+
+ assert [m["role"] for m in messages] == ["user", "assistant", "tool"]
+ assert "__MCP_IMAGES__" not in messages[-1]["content"]
+
+
@pytest.mark.parametrize("wanted", [True, False])
def test_the_decode_slot_is_asked_for_and_read_only_when_a_caller_wants_it(monkeypatch, wanted):
"""Verbose alone would attach the whole prompt, so it travels with the narrowing."""
diff --git a/studio/backend/tests/test_mcp_flatten_result.py b/studio/backend/tests/test_mcp_flatten_result.py
index 3f368442c03..82343862169 100644
--- a/studio/backend/tests/test_mcp_flatten_result.py
+++ b/studio/backend/tests/test_mcp_flatten_result.py
@@ -79,28 +79,28 @@ def test_text_only_result_unchanged():
def test_image_only_result_keeps_image_and_notes_model():
flat = _flatten_result(_result(_image()))
body, payload = flat.split("\n" + MCP_IMAGES_SENTINEL, 1)
- assert body == "[1 image attached; displayed to the user]"
+ assert body == "[1 image returned]"
assert json.loads(payload) == [{"data": PNG_B64, "mimeType": "image/png"}]
def test_text_plus_image_keeps_both():
flat = _flatten_result(_result(_text("Took a screenshot"), _image()))
body, payload = flat.split("\n" + MCP_IMAGES_SENTINEL, 1)
- assert body == "Took a screenshot\n[1 image attached; displayed to the user]"
+ assert body == "Took a screenshot\n[1 image returned]"
assert json.loads(payload)[0]["mimeType"] == "image/png"
def test_multiple_images_pluralized():
flat = _flatten_result(_result(_image(), _image(mime = "image/jpeg")))
body, payload = flat.split("\n" + MCP_IMAGES_SENTINEL, 1)
- assert "[2 images attached; displayed to the user]" in body
+ assert "[2 images returned]" in body
assert [img["mimeType"] for img in json.loads(payload)] == ["image/png", "image/jpeg"]
def test_strip_result_for_model_drops_image_payload():
flat = _flatten_result(_result(_text("Took a screenshot"), _image()))
stripped = strip_result_for_model(flat)
- assert stripped == "Took a screenshot\n[1 image attached; displayed to the user]"
+ assert stripped == "Took a screenshot\n[1 image returned]"
assert PNG_B64 not in stripped
@@ -120,12 +120,11 @@ def test_strip_preserves_non_image_json_after_marker():
def test_strip_removes_only_valid_terminal_envelope():
text = (
"Earlier mention: __MCP_IMAGES__: is documented here"
- "\n[1 image attached; displayed to the user]"
+ "\n[1 image returned]"
'\n__MCP_IMAGES__:[{"data": "AAAA", "mimeType": "image/png"}]'
)
assert strip_result_for_model(text) == (
- "Earlier mention: __MCP_IMAGES__: is documented here"
- "\n[1 image attached; displayed to the user]"
+ "Earlier mention: __MCP_IMAGES__: is documented here\n[1 image returned]"
)
@@ -143,7 +142,7 @@ def test_error_result_keeps_error_prefix_and_images():
def test_image_only_error_no_longer_reports_no_content():
flat = _flatten_result(_result(_image(), is_error = True))
- assert flat.startswith("Error: [1 image attached")
+ assert flat.startswith("Error: [1 image returned")
assert "tool returned no content" not in flat
@@ -158,7 +157,7 @@ def test_oversized_budget_shared_across_images():
big = "A" * (MAX_IMAGE_PAYLOAD_CHARS - 10)
flat = _flatten_result(_result(_image(data = big), _image()))
body, payload = flat.split("\n" + MCP_IMAGES_SENTINEL, 1)
- assert "1 image attached" in body
+ assert "1 image returned" in body
assert "1 image omitted (too large)" in body
images = json.loads(payload)
assert len(images) == 1 and images[0]["data"] == big
@@ -187,7 +186,7 @@ def test_image_mirrored_in_structured_content_is_not_dumped():
structured = {"content": [{"type": "image", "data": PNG_B64, "mimeType": "image/png"}]}
flat = _flatten_result(_result(_image(), structured = structured))
body, payload = flat.split("\n" + MCP_IMAGES_SENTINEL, 1)
- assert body == "[1 image attached; displayed to the user]"
+ assert body == "[1 image returned]"
assert json.loads(payload) == [{"data": PNG_B64, "mimeType": "image/png"}]
assert PNG_B64 not in strip_result_for_model(flat)
huge = "A" * (MAX_IMAGE_PAYLOAD_CHARS + 1)
@@ -200,16 +199,14 @@ def test_image_mirrored_in_structured_content_is_not_dumped():
def test_image_and_audio_share_one_note():
flat = _flatten_result(_result(_image(), _audio()))
body, payload = flat.split("\n" + MCP_IMAGES_SENTINEL, 1)
- assert body == (
- "[1 image attached; displayed to the user; audio attachment (audio/wav) not shown to the model]"
- )
+ assert body == ("[1 image returned; audio attachment (audio/wav) not shown to the model]")
assert json.loads(payload) == [{"data": PNG_B64, "mimeType": "image/png"}]
def test_independent_structured_content_is_kept_beside_attachments():
flat = _flatten_result(_result(_image(), structured = {"rows": 3, "max": 41.5}))
body, payload = flat.split("\n" + MCP_IMAGES_SENTINEL, 1)
- assert body == "{'rows': 3, 'max': 41.5}\n[1 image attached; displayed to the user]"
+ assert body == "{'rows': 3, 'max': 41.5}\n[1 image returned]"
assert json.loads(payload) == [{"data": PNG_B64, "mimeType": "image/png"}]
assert _flatten_result(_result(_audio(), structured = {"duration_s": 1.0})) == (
"{'duration_s': 1.0}\n[audio attachment (audio/wav) not shown to the model]"
@@ -374,14 +371,14 @@ def test_stdio_session_call_also_passes_raise_on_error_false(monkeypatch):
def test_embedded_resource_image_is_rendered():
flat = _flatten_result(_result(_blob_resource()))
body, payload = flat.split("\n" + MCP_IMAGES_SENTINEL, 1)
- assert body == "[1 image attached; displayed to the user]"
+ assert body == "[1 image returned]"
assert json.loads(payload) == [{"data": PNG_B64, "mimeType": "image/png"}]
def test_embedded_resource_image_shares_budget_with_image_content():
flat = _flatten_result(_result(_text("rendered"), _image(), _blob_resource(mime = "image/webp")))
body, payload = flat.split("\n" + MCP_IMAGES_SENTINEL, 1)
- assert body == "rendered\n[2 images attached; displayed to the user]"
+ assert body == "rendered\n[2 images returned]"
assert [img["mimeType"] for img in json.loads(payload)] == ["image/png", "image/webp"]
@@ -441,7 +438,7 @@ def test_fastmcp_file_format_png_is_rendered():
# fastmcp File(data=..., format="png") labels the blob application/png
flat = _flatten_result(_result(_blob_resource(mime = "application/png")))
body, payload = flat.split("\n" + MCP_IMAGES_SENTINEL, 1)
- assert body == "[1 image attached; displayed to the user]"
+ assert body == "[1 image returned]"
assert json.loads(payload) == [{"data": PNG_B64, "mimeType": "image/png"}]
diff --git a/studio/backend/tests/test_mcp_images.py b/studio/backend/tests/test_mcp_images.py
new file mode 100644
index 00000000000..b341a283f61
--- /dev/null
+++ b/studio/backend/tests/test_mcp_images.py
@@ -0,0 +1,2827 @@
+# SPDX-License-Identifier: AGPL-3.0-only
+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
+
+from __future__ import annotations
+
+import base64
+import importlib
+import importlib.machinery
+import io
+import json
+import json as _json
+import sys
+import types
+from pathlib import Path
+from unittest.mock import MagicMock
+
+import pytest
+
+_STUBBED = {}
+
+_BACKEND_DIR = str(Path(__file__).resolve().parent.parent)
+if _BACKEND_DIR not in sys.path:
+ sys.path.insert(0, _BACKEND_DIR)
+
+
+# core.inference.inference imports unsloth and trl at module scope; studio-backend-ci.yml
+# does not install them, so stub them here rather than rely on an earlier test file having done so.
+def _stub_if_missing(
+ name,
+ attrs = (),
+ named_spec = False,
+):
+ if name in sys.modules:
+ return
+ try:
+ importlib.import_module(name)
+ return
+ except Exception: # noqa: BLE001
+ pass
+ module = types.ModuleType(name)
+ module.__spec__ = importlib.machinery.ModuleSpec(name, None) if named_spec else None
+ module.__version__ = "0.0.0"
+ module.__getattr__ = lambda _attr: MagicMock()
+ for attr in attrs:
+ setattr(module, attr, MagicMock())
+ sys.modules[name] = module
+ _STUBBED[name] = module
+ parent, _, child = name.rpartition(".")
+ if parent and parent in sys.modules:
+ setattr(sys.modules[parent], child, module)
+
+
[email protected](scope = "module", autouse = True)
+def _optional_inference_dependencies():
+ try:
+ for _torchao in (
+ "torchao",
+ "torchao.prototype",
+ "torchao.prototype.safetensors",
+ "torchao.prototype.safetensors.safetensors_support",
+ "torchao.prototype.safetensors.safetensors_utils",
+ "torchao.quantization",
+ "torchao.dtypes",
+ "torchao.float8",
+ "torchao.utils",
+ ):
+ _stub_if_missing(_torchao, named_spec = True)
+
+ _stub_if_missing(
+ "unsloth", ("FastLanguageModel", "FastVisionModel", "is_bfloat16_supported")
+ )
+ _stub_if_missing("unsloth.chat_templates", ("get_chat_template",))
+ _stub_if_missing("unsloth_zoo")
+ _stub_if_missing("trl", ("SFTTrainer", "SFTConfig"))
+ yield
+ finally:
+ for name, module in reversed(list(_STUBBED.items())):
+ if sys.modules.get(name) is module:
+ sys.modules.pop(name)
+ parent, _, child = name.rpartition(".")
+ if parent in sys.modules and vars(sys.modules[parent]).get(child) is module:
+ delattr(sys.modules[parent], child)
+ _STUBBED.clear()
+
+
+from PIL import Image
+
+from core.inference import mcp_images
+from core.inference.mcp_images import (
+ IMAGE_TURN_TEXT,
+ MAX_IMAGE_EDGE,
+ MAX_MODEL_IMAGES,
+ append_image_turn,
+ content_parts,
+ promote_history,
+ split_images,
+)
+
+
+def _png(size = (8, 8), fmt = "PNG") -> str:
+ from PIL import Image
+
+ buffer = io.BytesIO()
+ Image.new("RGB", size, (10, 120, 200)).save(buffer, format = fmt)
+ return base64.b64encode(buffer.getvalue()).decode("ascii")
+
+
+def _image(data = None, mime = "image/png") -> dict:
+ return {"data": data if data is not None else _png(), "mimeType": mime}
+
+
+def _envelope(text: str, *images: dict) -> str:
+ return text + "\n" + mcp_images.SENTINEL + json.dumps(list(images))
+
+
+def _decode(part: dict):
+ from PIL import Image
+
+ url = part["image_url"]["url"]
+ assert url.startswith("data:image/png;base64,")
+ return Image.open(io.BytesIO(base64.b64decode(url.split(",", 1)[1])))
+
+
+def test_split_returns_text_and_images():
+ text, images = split_images(_envelope("a screenshot", _image()))
+
+ assert text == "a screenshot"
+ assert len(images) == 1
+
+
+def test_split_leaves_a_result_that_only_mentions_the_marker():
+ result = "docs say the marker is\n__MCP_IMAGES__: and nothing follows"
+
+ assert split_images(result) == (result, [])
+
+
+def test_split_leaves_an_envelope_that_is_not_an_image_array():
+ result = 'log\n__MCP_IMAGES__:["not", "image", "dicts"]'
+
+ assert split_images(result) == (result, [])
+
+
+def test_content_parts_reencode_to_png():
+ parts = content_parts([_image(data = _png(fmt = "WEBP"), mime = "image/webp")])
+
+ assert len(parts) == 1
+ assert _decode(parts[0]).format == "PNG"
+
+
+def test_content_parts_downscale_a_large_image():
+ parts = content_parts([_image(data = _png(size = (MAX_IMAGE_EDGE * 2, MAX_IMAGE_EDGE)))])
+
+ assert max(_decode(parts[0]).size) == MAX_IMAGE_EDGE
+
+
+def test_content_parts_cap_how_many_reach_the_model():
+ parts = content_parts([_image() for _ in range(MAX_MODEL_IMAGES + 3)])
+
+ assert len(parts) == MAX_MODEL_IMAGES
+
+
+def test_content_parts_drop_an_undecodable_payload():
+ assert content_parts([_image(data = "not base64 at all")]) == []
+ assert content_parts([_image(data = base64.b64encode(b"nope").decode())]) == []
+
+
+def test_image_turn_is_its_own_user_message():
+ conversation = [{"role": "tool", "name": "mcp__fs__read", "content": "[1 image returned]"}]
+
+ append_image_turn(conversation, [_image()])
+
+ assert conversation[-1]["role"] == "user"
+ assert conversation[-1]["content"][0] == {"type": "text", "text": IMAGE_TURN_TEXT}
+ assert conversation[-1]["content"][1]["type"] == "image_url"
+ assert conversation[1] is conversation[-1]
+
+
+def test_image_turn_merges_into_a_trailing_user_turn():
+ conversation = [{"role": "user", "content": "a nudge"}]
+
+ append_image_turn(conversation, [_image()])
+
+ assert len(conversation) == 1
+ assert conversation[0]["content"][0] == {"type": "text", "text": "a nudge"}
+ assert conversation[0]["content"][1]["type"] == "image_url"
+
+
+def test_image_turn_is_skipped_when_nothing_decodes():
+ conversation = [{"role": "tool", "content": "[1 image returned]"}]
+
+ append_image_turn(conversation, [_image(data = "///")])
+
+ assert len(conversation) == 1
+
+
+def test_history_promotes_a_replayed_envelope():
+ messages = promote_history(
+ [
+ {"role": "user", "content": "what is in the file"},
+ {"role": "assistant", "content": "", "tool_calls": [{"id": "call_0"}]},
+ {
+ "role": "tool",
+ "tool_call_id": "call_0",
+ "content": _envelope("[1 image returned]", _image()),
+ },
+ {"role": "assistant", "content": "a blue square"},
+ ],
+ vision = True,
+ )
+
+ assert messages[2]["content"] == "[1 image returned]"
+ assert messages[3]["role"] == "user"
+ assert messages[3]["content"][1]["type"] == "image_url"
+ assert messages[4]["content"] == "a blue square"
+
+
+def test_history_flushes_after_the_whole_batch_of_tool_results():
+ messages = promote_history(
+ [
+ {"role": "assistant", "content": "", "tool_calls": [{"id": "a"}, {"id": "b"}]},
+ {"role": "tool", "tool_call_id": "a", "content": _envelope("first", _image())},
+ {"role": "tool", "tool_call_id": "b", "content": _envelope("second", _image())},
+ ],
+ vision = True,
+ )
+
+ assert [message["role"] for message in messages] == ["assistant", "tool", "tool", "user"]
+ assert len(messages[3]["content"]) == 3
+
+
+def test_history_strips_the_envelope_for_a_model_that_cannot_see_it():
+ messages = promote_history(
+ [{"role": "tool", "content": _envelope("[1 image returned]", _image())}],
+ vision = False,
+ )
+
+ assert messages == [{"role": "tool", "content": "[1 image returned]"}]
+
+
+def test_history_keeps_an_image_only_result_from_emptying_its_tool_message():
+ messages = promote_history(
+ [{"role": "tool", "content": _envelope("", _image())}],
+ vision = False,
+ )
+
+ assert messages[0]["content"] == "[image returned]"
+
+
+def test_history_leaves_other_messages_untouched():
+ original = [
+ {"role": "user", "content": "hello"},
+ {"role": "tool", "content": "plain result"},
+ ]
+
+ assert promote_history(original, vision = True) == original
+
+
+def test_history_merges_into_a_following_user_turn():
+ messages = promote_history(
+ [
+ {"role": "tool", "content": _envelope("[1 image returned]", _image())},
+ {"role": "user", "content": "what colour was it"},
+ ],
+ vision = True,
+ )
+
+ assert [message["role"] for message in messages] == ["tool", "user"]
+ assert messages[1]["content"][0]["type"] == "image_url"
+ # The note rides along on the merge, ahead of the question's own text.
+ assert messages[1]["content"][1]["text"].startswith(mcp_images.IMAGE_TURN_TEXT)
+ assert messages[1]["content"][2] == {"type": "text", "text": "what colour was it"}
+
+
+def test_local_history_carries_markers_and_payloads():
+ messages, payloads = mcp_images.promote_history_local(
+ [
+ {"role": "user", "content": "what is in the file"},
+ {"role": "tool", "content": _envelope("[1 image returned]", _image())},
+ {"role": "assistant", "content": "a blue square"},
+ ],
+ vision = True,
+ )
+
+ assert messages[2]["role"] == "user"
+ assert messages[2]["content"][0] == {"type": "image"}
+ assert len(payloads) == 1
+ assert base64.b64decode(payloads[0])[:8] == b"\x89PNG\r\n\x1a\n"
+
+
+def test_local_history_merges_markers_into_a_following_user_turn():
+ messages, payloads = mcp_images.promote_history_local(
+ [
+ {"role": "tool", "content": _envelope("[1 image returned]", _image())},
+ {"role": "user", "content": "what colour was it"},
+ ],
+ vision = True,
+ )
+
+ assert [message["role"] for message in messages] == ["tool", "user"]
+ assert messages[1]["content"][0] == {"type": "image"}
+ assert len(payloads) == 1
+
+
+def test_local_history_strips_without_vision():
+ messages, payloads = mcp_images.promote_history_local(
+ [{"role": "tool", "content": _envelope("[1 image returned]", _image())}],
+ vision = False,
+ )
+
+ assert messages == [{"role": "tool", "content": "[1 image returned]"}]
+ assert payloads == []
+
+
+def test_placeholder_turn_marks_one_image_per_payload():
+ turn = mcp_images.placeholder_turn(2)
+
+ assert [part["type"] for part in turn["content"]] == ["image", "image", "text"]
+
+
+def test_an_oversized_raster_is_rejected_off_the_header():
+ from PIL import Image
+
+ buffer = io.BytesIO()
+ # Uniform colour, so a 9000x9000 raster still encodes to a few kilobytes and
+ # clears every payload-size gate ahead of the decode.
+ Image.new("RGB", (9000, 9000), (0, 0, 0)).save(buffer, format = "PNG")
+ encoded = base64.b64encode(buffer.getvalue()).decode("ascii")
+ from core.inference.mcp_client import MAX_IMAGE_PAYLOAD_CHARS
+
+ assert len(encoded) < MAX_IMAGE_PAYLOAD_CHARS
+
+ assert content_parts([{"data": encoded, "mimeType": "image/png"}]) == []
+
+
+def test_an_image_inside_the_pixel_budget_still_downscales():
+ parts = content_parts([_image(data = _png(size = (2000, 1000)))])
+
+ assert max(_decode(parts[0]).size) == MAX_IMAGE_EDGE
+
+
+def test_local_history_caps_the_images_a_replay_resends():
+ turns = []
+ for _ in range(mcp_images.MAX_TOTAL_MODEL_IMAGES + 3):
+ turns.append({"role": "tool", "content": _envelope("[1 image returned]", _image())})
+ turns.append({"role": "assistant", "content": "noted"})
+
+ messages, payloads = mcp_images.promote_history_local(turns, vision = True)
+ markers = sum(
+ 1
+ for message in messages
+ if isinstance(message.get("content"), list)
+ for part in message["content"]
+ if part.get("type") == "image"
+ )
+
+ assert len(payloads) == mcp_images.MAX_TOTAL_MODEL_IMAGES
+ # A marker with no pixels behind it makes the processor count image tokens it
+ # was given nothing for, so the two have to come down together.
+ assert markers == len(payloads)
+
+
+def _svg_image() -> dict:
+ # _flatten_result accepts image/svg+xml, and Pillow cannot decode it.
+ svg = b'<svg xmlns="http://www.w3.org/2000/svg"><rect width="8" height="8"/></svg>'
+ return {"data": base64.b64encode(svg).decode(), "mimeType": "image/svg+xml"}
+
+
+def test_undecodable_images_do_not_spend_the_quota():
+ images = [_svg_image() for _ in range(MAX_MODEL_IMAGES)] + [_image(), _image()]
+
+ parts = content_parts(images)
+
+ assert len(parts) == 2
+ assert len(mcp_images.png_payloads(images)) == 2
+
+
+def test_the_data_url_history_is_capped_too():
+ turns = []
+ for _ in range(mcp_images.MAX_TOTAL_MODEL_IMAGES + 3):
+ turns.append({"role": "tool", "content": _envelope("[1 image returned]", _image())})
+ turns.append({"role": "assistant", "content": "noted"})
+
+ messages = promote_history(turns, vision = True)
+ parts = mcp_images.count_image_parts(messages, "image_url")
+
+ assert parts == mcp_images.MAX_TOTAL_MODEL_IMAGES
+
+
+def test_a_live_image_turn_merges_into_a_trailing_nudge():
+ """append_deferred_nudges leaves a role=user turn; a second one in a row is
+ what a strict VLM template rejects."""
+ conversation = [
+ {"role": "user", "content": "take a screenshot"},
+ {"role": "assistant", "content": "", "tool_calls": [{"id": "c"}]},
+ {"role": "tool", "tool_call_id": "c", "content": "[1 image returned]"},
+ {"role": "user", "content": "One earlier request was not executed."},
+ ]
+
+ append_image_turn(conversation, [_image()])
+
+ assert [m["role"] for m in conversation] == ["user", "assistant", "tool", "user"]
+ assert conversation[-1]["content"][0] == {
+ "type": "text",
+ "text": "One earlier request was not executed.",
+ }
+ assert conversation[-1]["content"][1]["type"] == "image_url"
+
+
+def test_a_marker_turn_merges_into_a_trailing_nudge():
+ conversation = [
+ {"role": "tool", "tool_call_id": "c", "content": "[1 image returned]"},
+ {"role": "user", "content": "One earlier request was not executed."},
+ ]
+
+ mcp_images.append_placeholder_turn(conversation, 2, 2)
+
+ assert [m["role"] for m in conversation] == ["tool", "user"]
+ # The note rides along: merged bare, the markers read as pictures attached to the
+ # nudge rather than as the tool's output.
+ assert [p["type"] for p in conversation[-1]["content"]] == ["text", "image", "image", "text"]
+ assert conversation[-1]["content"][-1]["text"].startswith(mcp_images.IMAGE_TURN_TEXT)
+
+
+def test_a_marker_turn_still_opens_its_own_turn_after_a_tool_result():
+ conversation = [{"role": "tool", "tool_call_id": "c", "content": "[1 image returned]"}]
+
+ mcp_images.append_placeholder_turn(conversation, 1, 1)
+
+ assert [m["role"] for m in conversation] == ["tool", "user"]
+ assert conversation[-1]["content"][0] == {"type": "image"}
+
+
+def test_a_non_mcp_tool_never_has_its_output_read_as_images():
+ """The envelope is a plain text suffix, so terminal output or a fetched page
+ can end in a syntactically valid one. Only an mcp__ call is trusted."""
+ from core.inference.tool_loop_controller import ToolCallCompletion, ToolCallDecision
+
+ payload = _envelope("$ cat notes.txt", _image())
+
+ def completion(tool_name):
+ decision = ToolCallDecision(
+ action = "execute",
+ tool_name = tool_name,
+ arguments = {},
+ tool_call_id = "call_0",
+ card_call_id = "",
+ key = tool_name,
+ provenance = {},
+ noop_result = "",
+ )
+ return ToolCallCompletion(decision = decision, result = payload, executed = True)
+
+ assert completion("mcp__fs__read_media_file").mcp_images()
+ assert completion("bash").mcp_images() == []
+ assert completion("web_fetch").mcp_images() == []
+
+
+def test_image_parts_are_dropped_for_a_text_only_fallback():
+ from core.inference.inference import _without_image_parts
+
+ messages = [
+ {"role": "user", "content": "what is in the file"},
+ {"role": "tool", "content": "[1 image returned]"},
+ {
+ "role": "user",
+ "content": [{"type": "image"}, {"type": "text", "text": IMAGE_TURN_TEXT}],
+ },
+ {"role": "user", "content": [{"type": "image"}]},
+ ]
+
+ out = _without_image_parts(messages)
+
+ # And the fallback branch really calls it, not just defines it.
+ import inspect
+
+ from core.inference import inference as inference_module
+
+ body = inspect.getsource(inference_module.InferenceBackend._generate_chat_response_inner)
+ assert "messages = _without_image_parts(messages)" in body
+
+ assert out[0] == messages[0]
+ assert out[1] == messages[1]
+ # Collapsed back to a plain string, which is what a text template takes.
+ assert out[2]["content"] == IMAGE_TURN_TEXT
+ assert out[3]["content"] == ""
+
+
+def test_a_replay_only_turn_takes_the_vision_render():
+ """The reasoning-channel markers follow the vision render, and #10092 made that
+ unconditional on the vision path -- so what this PR has to keep true is that a
+ replay-only turn reaches that path at all, with no attachment to trigger it."""
+ import inspect
+
+ from core.inference import inference as inference_module
+
+ inner = inspect.getsource(inference_module.InferenceBackend._generate_chat_response_inner)
+ assert (
+ "if is_vision and (image or images):" in inner
+ ), "a conversation whose only pictures are replayed still has to render as vision"
+ vision = inspect.getsource(inference_module.InferenceBackend._generate_vision_response)
+ assert "if attached:" in vision, "and the branch inside keys off every attached image"
+
+
+def test_a_named_non_mcp_tool_result_is_not_promoted_on_replay():
+ """The live loop checks the call's tool name; the replay path reads the
+ message's own `name`. Terminal output ending in a valid envelope must not
+ become image input on the next request either."""
+ history = [
+ {
+ "role": "tool",
+ "tool_call_id": "call_0",
+ "name": "bash",
+ "content": _envelope("$ cat notes.txt", _image()),
+ }
+ ]
+
+ out = promote_history(history, vision = True)
+
+ # The suffix still comes off -- it is base64 and the model must not read it as
+ # text -- but it never becomes image input.
+ assert out[0]["content"] == "$ cat notes.txt"
+ assert len(out) == 1, "a bash result was promoted into image input"
+
+
+def test_an_mcp_named_result_is_still_promoted_on_replay():
+ history = [
+ {
+ "role": "tool",
+ "tool_call_id": "call_0",
+ "name": "mcp__fs__read_media_file",
+ "content": _envelope("[1 image returned]", _image()),
+ }
+ ]
+
+ out = promote_history(history, vision = True)
+
+ assert out[0]["content"] == "[1 image returned]"
+ assert out[1]["content"][1]["type"] == "image_url"
+
+
+def test_an_unnamed_tool_result_keeps_working():
+ """Older stored turns carry no name; the envelope only ever came from an MCP
+ server, so an absent name is not evidence against it."""
+ history = [{"role": "tool", "content": _envelope("[1 image returned]", _image())}]
+
+ out = promote_history(history, vision = True)
+
+ assert len(out) == 2
+ assert out[1]["content"][1]["type"] == "image_url"
+
+
+def test_stripping_is_unconditional_and_only_promotion_is_gated():
+ """Two different rules, and conflating them breaks one or the other.
+
+ The suffix runs to megabytes of base64, so it comes off every model-facing
+ text path whoever produced it -- that is a context-window property, and
+ test_tool_result_fits_window depends on it. Provenance answers a separate
+ question: whether those bytes are trusted enough to become IMAGE input.
+ """
+ from core.inference.mcp_images import promote_history
+ from core.inference.tool_loop_controller import (
+ ToolCallCompletion,
+ ToolCallDecision,
+ strip_result_for_model,
+ )
+
+ payload = _envelope("$ cat notes.txt", _image())
+
+ def completion(tool_name):
+ decision = ToolCallDecision(
+ action = "execute",
+ tool_name = tool_name,
+ arguments = {},
+ tool_call_id = "call_0",
+ card_call_id = "",
+ key = tool_name,
+ provenance = {},
+ noop_result = "",
+ )
+ return ToolCallCompletion(decision = decision, result = payload, executed = True)
+
+ for name in ("mcp__fs__read", "bash", "web_search", "mcp"):
+ # 1. never as text, on the live path or the replay
+ assert strip_result_for_model(payload, name) == "$ cat notes.txt", name
+ replayed = promote_history(
+ [{"role": "tool", "name": name, "content": payload}], vision = True
+ )
+ assert replayed[0]["content"] == "$ cat notes.txt", name
+
+ # 2. as image input only from a tool an MCP server served
+ assert completion("mcp__fs__read").mcp_images()
+ assert completion("bash").mcp_images() == []
+ promoted = promote_history(
+ [{"role": "tool", "name": "mcp__fs__read", "content": payload}], vision = True
+ )
+ assert any(isinstance(m.get("content"), list) for m in promoted)
+ not_promoted = promote_history(
+ [{"role": "tool", "name": "bash", "content": payload}], vision = True
+ )
+ assert not any(isinstance(m.get("content"), list) for m in not_promoted)
+
+
+def test_a_parallel_batch_keeps_each_result_its_own_quota():
+ """Two calls returning four images each deliver eight, not four: flattening
+ first hands the whole per-result allowance to the first call."""
+ from core.inference.mcp_images import content_parts_per_result
+
+ one = [_image() for _ in range(MAX_MODEL_IMAGES)]
+ two = [_image() for _ in range(MAX_MODEL_IMAGES)]
+
+ assert len(content_parts(one + two)) == MAX_MODEL_IMAGES
+ assert len(content_parts_per_result([one, two])) == 2 * MAX_MODEL_IMAGES
+
+
+def test_a_parallel_batch_still_stops_at_the_conversation_cap():
+ from core.inference.mcp_images import content_parts_per_result
+ results = [[_image() for _ in range(MAX_MODEL_IMAGES)] for _ in range(4)]
+
+ assert len(content_parts_per_result(results)) == mcp_images.MAX_TOTAL_MODEL_IMAGES
+
+
+def test_an_emptied_image_turn_is_dropped_not_left_blank():
+ """content: [] is an empty user message, which strict provider APIs and chat
+ templates reject rather than serving the newer images."""
+ conversation = [
+ {"role": "user", "content": [{"type": "image"}]},
+ {"role": "user", "content": [{"type": "image"}]},
+ ]
+ payloads = ["a", "b"]
+
+ mcp_images.trim_image_turns(conversation, payloads, limit = 1)
+
+ assert len(conversation) == 1
+ assert all(message.get("content") != [] for message in conversation)
+
+
+def test_replayed_markers_sit_ahead_of_the_attachment_marker():
+ """Pixels go history-first with the attachment last, and a positional VLM
+ processor binds them in document order."""
+ messages = [
+ {"role": "user", "content": [{"type": "image"}, {"type": "text", "text": "this one"}]},
+ ]
+
+ mcp_images.insert_placeholder_turn(messages, 0, 2, 2)
+
+ replayed = [
+ index
+ for index, message in enumerate(messages)
+ for part in message["content"]
+ if part.get("type") == "image"
+ ]
+ # Both replayed markers come before the attachment's turn.
+ assert replayed[:2] == [0, 0]
+ assert replayed[-1] == 1
+
+
+def test_the_attachment_is_marked_on_the_turn_that_supplied_it():
+ """_extract_content_parts takes the newest user image from anywhere in the
+ thread, so marking the newest TURN tells a text-only latest question that it
+ carried an older picture."""
+ conversation = [
+ {"role": "user", "content": "here is the screenshot"},
+ {"role": "assistant", "content": "noted"},
+ {"role": "user", "content": "now a text-only follow-up"},
+ ]
+
+ marked = mcp_images.mark_last_user_turn(conversation, 1, ordinal = 0)
+
+ assert isinstance(marked[0]["content"], list)
+ assert marked[2]["content"] == "now a text-only follow-up"
+
+
+def test_without_an_ordinal_the_newest_user_turn_still_takes_it():
+ conversation = [
+ {"role": "user", "content": "first"},
+ {"role": "user", "content": "second"},
+ ]
+
+ marked = mcp_images.mark_last_user_turn(conversation, 1)
+
+ assert marked[0]["content"] == "first"
+ assert isinstance(marked[1]["content"], list)
+
+
+def test_the_provenance_name_survives_local_extraction():
+ """promote_history reads message['name'] to decide whether an envelope came
+ from an MCP server; an extraction that drops it bypasses the check."""
+ import inspect
+
+ import routes.inference as inference_route
+
+ body = inspect.getsource(inference_route._extract_content_parts)
+ assert (
+ 'chat_message["name"] = _tool_name' in body
+ ), "the tool message's name has to reach promote_history"
+
+
+def test_a_parallel_batch_survives_the_replay_too():
+ """The live loops kept result boundaries; the replay still flattened them, so
+ a conversation resumed after two parallel calls lost the second call's images."""
+ history = [
+ {
+ "role": "tool",
+ "name": "mcp__a__shot",
+ "content": _envelope("[4 images returned]", *[_image() for _ in range(4)]),
+ },
+ {
+ "role": "tool",
+ "name": "mcp__b__shot",
+ "content": _envelope("[4 images returned]", *[_image() for _ in range(4)]),
+ },
+ ]
+
+ promoted = promote_history(history, vision = True)
+
+ assert mcp_images.count_image_parts(promoted, "image_url") == 2 * MAX_MODEL_IMAGES
+
+
+def test_the_local_replay_keeps_markers_and_payloads_in_step_across_results():
+ history = [
+ {
+ "role": "tool",
+ "name": f"mcp__{server}__shot",
+ "content": _envelope("[4 images returned]", *[_image() for _ in range(4)]),
+ }
+ for server in ("a", "b")
+ ]
+
+ messages, payloads = mcp_images.promote_history_local(history, vision = True)
+
+ # Both results flush as one batch, and a local placeholder turn is one message,
+ # which on a non-GGUF model takes one picture. Parity is the invariant.
+ assert len(payloads) == mcp_images.LOCAL_MAX_IMAGES_PER_TURN
+ assert mcp_images.count_image_parts(messages, "image") == len(payloads)
+
+
+def test_the_ordinal_skips_the_turns_promotion_inserted():
+ """The ordinal is counted on the ORIGINAL history, but applied to the promoted
+ one, which has synthetic image turns in it. Counting those would put the
+ attachment's marker on a historical picture's turn."""
+ promoted = [
+ {"role": "user", "content": "here is the shot"},
+ mcp_images.placeholder_turn(1, 1),
+ {"role": "assistant", "content": "ok"},
+ {"role": "user", "content": "and this new one"},
+ ]
+
+ out = mcp_images.mark_last_user_turn(promoted, 1, ordinal = 1)
+
+ # The second REAL user turn, not the second user turn in the promoted list.
+ assert isinstance(out[3]["content"], list)
+ assert out[3]["content"][-1] == {"type": "text", "text": "and this new one"}
+
+
+def test_a_synthetic_image_turn_is_recognised():
+ assert mcp_images.is_synthetic_image_turn(mcp_images.placeholder_turn(2, 2))
+ assert not mcp_images.is_synthetic_image_turn({"role": "user", "content": "hello"})
+ assert not mcp_images.is_synthetic_image_turn(
+ {"role": "user", "content": [{"type": "text", "text": "hello"}]}
+ )
+
+
+def test_undecodable_entries_are_not_counted_as_images():
+ """_flatten_result accepts formats Pillow cannot read, and promotion drops
+ them, so charging their KV reserves cache for images never sent."""
+ svg = {
+ "data": base64.b64encode(b'<svg xmlns="http://www.w3.org/2000/svg"/>').decode(),
+ "mimeType": "image/svg+xml",
+ }
+
+ assert mcp_images.count_probably_decodable([svg] * 4) == 0
+ assert mcp_images.count_probably_decodable([_image()] * 3) == 3
+ assert mcp_images.count_probably_decodable([svg, _image(), svg, _image()]) == 2
+ # And the sniff never claims something it cannot open.
+ assert not mcp_images.probably_decodable({"data": "not base64 at all!!", "mimeType": "x"})
+ assert not mcp_images.probably_decodable({})
+
+
+def test_the_replay_cap_never_deletes_a_callers_own_attachments():
+ """promote_history runs on every request now, so an unscoped cap silently
+ dropped the oldest attachments of a caller that simply sent many images."""
+ attachments = [
+ {"type": "image_url", "image_url": {"url": f"data:image/png;base64,{_png()}"}}
+ for _ in range(mcp_images.MAX_TOTAL_MODEL_IMAGES + 4)
+ ]
+ conversation = [
+ {"role": "user", "content": [{"type": "text", "text": "compare these"}, *attachments]}
+ ]
+
+ out = promote_history(conversation, vision = True)
+
+ assert mcp_images.count_image_parts(out, "image_url") == len(attachments)
+
+
+def test_the_replay_cap_still_bounds_what_promotion_adds():
+ turns = []
+ for _ in range(mcp_images.MAX_TOTAL_MODEL_IMAGES + 3):
+ turns.append({"role": "tool", "name": "mcp__a__s", "content": _envelope("[1]", _image())})
+ turns.append({"role": "assistant", "content": "noted"})
+
+ out = promote_history(turns, vision = True)
+
+ assert mcp_images.count_image_parts(out, "image_url") == mcp_images.MAX_TOTAL_MODEL_IMAGES
+
+
+def test_the_sniff_charges_every_format_promotion_can_decode():
+ """A deny list, not an allow list: a format Pillow gains that an allow list
+ missed would be promoted and reserved nothing for, which overcommits the KV
+ cache. Guessing the other way only over-reserves."""
+ for fmt in ("PNG", "JPEG", "GIF", "BMP", "TIFF", "WEBP", "ICO"):
+ buffer = io.BytesIO()
+ Image.new("RGB", (8, 8), (1, 2, 3)).save(buffer, format = fmt)
+ encoded = base64.b64encode(buffer.getvalue()).decode()
+ assert mcp_images.probably_decodable({"data": encoded}), fmt
+
+ svg = base64.b64encode(b'<svg xmlns="http://www.w3.org/2000/svg"/>').decode()
+ assert not mcp_images.probably_decodable({"data": svg})
+ assert not mcp_images.probably_decodable({"data": base64.b64encode(b'{"a": 1}').decode()})
+
+
+def test_the_live_cap_also_leaves_callers_attachments_alone():
+ """The replay cap was scoped first; the live loops kept the unscoped one, so a
+ tool loop that starts from caller attachments still deleted them."""
+ attachments = [
+ {"type": "image_url", "image_url": {"url": f"data:image/png;base64,{_png()}"}}
+ for _ in range(6)
+ ]
+ conversation = [
+ {"role": "user", "content": [{"type": "text", "text": "compare"}, *attachments]}
+ ]
+ owned: list = []
+ for _ in range(3):
+ conversation.append({"role": "tool", "content": "[4 images returned]"})
+ append_image_turn(
+ conversation, [[_image() for _ in range(4)]], per_result = True, owned = owned
+ )
+
+ survived = sum(
+ 1
+ for part in attachments
+ for message in conversation
+ if isinstance(message.get("content"), list)
+ for other in message["content"]
+ if other is part
+ )
+ assert survived == len(attachments), "a caller attachment was deleted by the loop's cap"
+ assert len(owned) == mcp_images.MAX_TOTAL_MODEL_IMAGES
+
+
+def test_pixels_follow_the_markers_when_the_attachment_came_first():
+ """ "History first, attachment last" only holds when the attachment is on the
+ newest turn. It is not when an earlier question carried it and a tool returned
+ pictures afterwards, and a positional VLM binds the Nth pixel to the Nth marker."""
+ conversation = [
+ {"role": "user", "content": "look at this"},
+ {"role": "tool", "content": "[1 image returned]"},
+ mcp_images.placeholder_turn(1, 1),
+ ]
+ prior = mcp_images.image_marker_parts(conversation)
+ conversation = mcp_images.mark_last_user_turn(conversation, 1, ordinal = 0)
+
+ ordered = mcp_images.pixels_in_marker_order(conversation, prior, ["MCP"], "ATTACH")
+
+ assert ordered == ["ATTACH", "MCP"]
+
+
+def test_pixels_stay_history_first_when_the_attachment_is_newest():
+ conversation = [
+ {"role": "tool", "content": "[1 image returned]"},
+ mcp_images.placeholder_turn(1, 1),
+ {"role": "user", "content": "and this one"},
+ ]
+ prior = mcp_images.image_marker_parts(conversation)
+ conversation = mcp_images.mark_last_user_turn(conversation, 1, ordinal = 1)
+
+ ordered = mcp_images.pixels_in_marker_order(conversation, prior, ["MCP"], "ATTACH")
+
+ assert ordered == ["MCP", "ATTACH"]
+
+
+def test_the_vision_streamer_resolves_markers_for_a_replay_only_turn():
+ """The streamer marks its markers resolved either way, so gating them on the
+ singular attachment resolves a replay-only turn to none AND suppresses the
+ fallback detection -- native reasoning output then reaches the user as answer
+ text. Route classification keys off every image; generation has to as well."""
+ import inspect
+
+ from core.inference import inference as inference_module
+
+ body = inspect.getsource(inference_module.InferenceBackend._generate_vision_response)
+ call = body.index("detect_reasoning_channel_markers(processor")
+ window = body[call : call + 400]
+ assert "if attached" in window, window[:200]
+ assert "if image\n" not in window
+
+
+def test_the_gguf_loop_caps_images_across_iterations_not_per_batch():
+ """A per-iteration ownership list only ever sees the current batch, so three
+ sequential four-image results leave twelve images in the conversation against
+ a cap of eight."""
+ import inspect
+
+ from core.inference import llama_cpp
+
+ body = inspect.getsource(llama_cpp.LlamaCppBackend.generate_chat_completion_with_tools)
+ declared = body.index("loop_mcp_image_parts: list = list(replayed_image_parts)")
+ loop_start = body.index("while True:")
+ assert declared < loop_start, (
+ "the ownership list has to outlive the iteration, or the cap only ever "
+ "counts the newest batch"
+ )
+
+
+def test_promotion_reports_the_parts_it_created():
+ """A resumed chat already carries promoted images; a loop starting its cap at
+ zero lets the whole history through and then adds a batch on top."""
+ history = [
+ {"role": "tool", "name": "mcp__a__s", "content": _envelope("[1]", _image())},
+ {"role": "assistant", "content": "noted"},
+ ]
+ promoted: list = []
+
+ out = promote_history(history, vision = True, promoted_out = promoted)
+
+ assert len(promoted) == 1
+ live = [
+ part
+ for message in out
+ if isinstance(message.get("content"), list)
+ for part in message["content"]
+ if part.get("type") == "image_url"
+ ]
+ assert all(any(part is other for other in live) for part in promoted)
+
+
+def test_the_plain_vision_path_marks_the_turn_that_supplied_the_attachment():
+ """messages_with_attached_image returns early when markers exist, so the top-up
+ placed the attachment's marker on the newest turn -- showing a later text-only
+ question as carrying an older image."""
+ conversation = [
+ {"role": "user", "content": "here it is"},
+ {"role": "assistant", "content": "ok"},
+ mcp_images.placeholder_turn(1, 1),
+ {"role": "user", "content": "a later text question"},
+ ]
+ prior = mcp_images.image_marker_parts(conversation)
+
+ out = mcp_images.top_up_image_markers(conversation, 2, ordinal = 0)
+
+ assert isinstance(out[0]["content"], list)
+ assert out[3]["content"] == "a later text question"
+ assert mcp_images.pixels_in_marker_order(out, prior, ["MCP"], "ATTACH") == ["ATTACH", "MCP"]
+
+
+def test_the_attachment_is_not_the_one_the_cap_drops():
+ """Trimming the combined list removes whatever is first, and the attachment is
+ first whenever its turn precedes the replayed pictures."""
+ conversation = [mcp_images.placeholder_turn(1, 1) for _ in range(8)]
+ payloads = [f"mcp{i}" for i in range(8)]
+
+ # what the route now does: trim the replay to leave room, THEN interleave
+ mcp_images.trim_image_turns(conversation, payloads, limit = mcp_images.MAX_TOTAL_MODEL_IMAGES - 1)
+
+ assert len(payloads) == mcp_images.MAX_TOTAL_MODEL_IMAGES - 1
+ assert payloads[-1] == "mcp7", "the newest replayed image survived"
+ assert "mcp0" not in payloads, "the oldest replayed image went, not the attachment"
+
+
+def test_a_client_marked_attachment_still_gets_its_pixel():
+ """A marker that predates the top-up is history's only while history has
+ pixels left. When the client marked the attachment itself, the top-up adds
+ nothing and that pre-existing marker is the attachment's."""
+ conversation = [{"role": "user", "content": [{"type": "image"}, {"type": "text", "text": "x"}]}]
+ prior = mcp_images.image_marker_parts(conversation)
+
+ ordered = mcp_images.pixels_in_marker_order(conversation, prior, [], "ATTACH")
+
+ assert ordered == ["ATTACH"], "the attachment's pixel was dropped"
+
+
+def test_history_still_takes_its_own_markers_first():
+ conversation = [
+ mcp_images.placeholder_turn(2, 2),
+ {"role": "user", "content": [{"type": "image"}, {"type": "text", "text": "and this"}]},
+ ]
+ prior = mcp_images.image_marker_parts(conversation)
+
+ ordered = mcp_images.pixels_in_marker_order(conversation, prior, ["A", "B"], "ATTACH")
+
+ assert ordered == ["A", "B", "ATTACH"]
+
+
+def test_the_gguf_loop_starts_its_cap_from_the_replayed_parts():
+ """The list is seeded from the conversation, not from zero: a resumed chat whose
+ history already carries the allowance would otherwise get a second one for this
+ run and send both."""
+ import inspect
+
+ from core.inference import llama_cpp
+
+ signature = inspect.signature(llama_cpp.LlamaCppBackend.generate_chat_completion_with_tools)
+ assert "replayed_image_parts" in signature.parameters
+
+ body = inspect.getsource(llama_cpp.LlamaCppBackend.generate_chat_completion_with_tools)
+ assert "loop_mcp_image_parts: list = list(replayed_image_parts)" in body
+
+
+def test_the_gguf_route_hands_the_loop_what_promotion_created():
+ import inspect
+
+ from routes import inference
+
+ builder = inspect.signature(inference._openai_messages_for_gguf_chat)
+ assert "promoted_out" in builder.parameters
+
+ route = inspect.getsource(inference.produce_openai_chat_completions)
+ assert "_gguf_replayed_image_parts: list = []" in route
+ assert "replayed_image_parts = tuple(_gguf_replayed_image_parts)" in route
+
+
+def test_a_replayed_history_is_not_decoded_past_what_the_cap_can_keep():
+ """A permitted raster is 40 megapixels and a few hundred bytes of base64 can ask
+ for one, so decoding every envelope in a caller-supplied history before the trim
+ let the request choose how much Pillow work it cost."""
+ decoded: list = []
+ real = _png()
+
+ def _counting_url(data):
+ decoded.append(data)
+ return "data:image/png;base64," + real
+
+ history: list = []
+ for index in range(60):
+ history.append(
+ {
+ "role": "tool",
+ "name": f"mcp__s__shot{index}",
+ "content": _envelope("[4]", *[_image() for _ in range(4)]),
+ }
+ )
+ history.append({"role": "assistant", "content": f"turn {index}"})
+ history.append({"role": "user", "content": "again"})
+
+ original = mcp_images._png_data_url
+ mcp_images._png_data_url = _counting_url
+ try:
+ out = promote_history(history, vision = True)
+ finally:
+ mcp_images._png_data_url = original
+
+ live = sum(
+ 1
+ for message in out
+ if isinstance(message.get("content"), list)
+ for part in message["content"]
+ if part.get("type") == "image_url"
+ )
+ assert live == mcp_images.MAX_TOTAL_MODEL_IMAGES, live
+ assert len(decoded) <= (
+ mcp_images.MAX_TOTAL_MODEL_IMAGES + mcp_images.DECODE_FAILURE_ALLOWANCE
+ ), f"{len(decoded)} decodes for a history of 240 images"
+
+
+def test_the_eligible_pass_keeps_candidates_behind_an_undecodable_result():
+ """It cannot decode either, so a newest result of formats Pillow rejects must not
+ starve the valid pictures behind it."""
+ history = [
+ {
+ "role": "tool",
+ "name": "mcp__s__a",
+ "content": _envelope("[4]", *[_image() for _ in range(4)]),
+ },
+ {
+ "role": "tool",
+ "name": "mcp__s__b",
+ "content": _envelope("[4]", *[_image() for _ in range(4)]),
+ },
+ {
+ "role": "tool",
+ "name": "mcp__s__c",
+ "content": _envelope("[4]", *[_image() for _ in range(4)]),
+ },
+ ]
+
+ eligible = mcp_images.eligible_replay_images(history)
+
+ assert eligible[2] == 4 and eligible[1] == 4, eligible
+ assert (
+ eligible[0] == mcp_images.DECODE_FAILURE_ALLOWANCE
+ ), "the spare allowance keeps the oldest result as candidates"
+ assert sum(eligible.values()) <= (
+ mcp_images.MAX_TOTAL_MODEL_IMAGES + mcp_images.DECODE_FAILURE_ALLOWANCE
+ )
+
+
+def test_an_oversized_parallel_batch_keeps_its_newest_results():
+ """Everything else that bounds these keeps the newest, so filling the batch from
+ the front showed the model results 1-2 on this turn and 2-3 on the next."""
+ # Distinct sizes so each result's decoded URLs are distinguishable.
+ results = [
+ [{"data": _png(size = (4 + index, 4 + index)), "mimeType": "image/png"} for _ in range(4)]
+ for index in range(3)
+ ]
+ per_result = [set(mcp_images._decoded_urls(images, 4)) for images in results]
+
+ urls = mcp_images._decoded_urls_per_result(results)
+
+ assert len(urls) == mcp_images.MAX_TOTAL_MODEL_IMAGES
+ kept = set(urls)
+ assert per_result[2] <= kept, "the newest result must survive"
+ assert per_result[1] <= kept, "so must the one before it"
+ assert not (per_result[0] & kept), "the OLDEST result is the one that goes"
+ # Within the batch the surviving results still read in call order.
+ assert set(urls[:4]) == per_result[1] and set(urls[4:]) == per_result[2]
+
+
+def test_a_partially_trimmed_turn_stops_claiming_the_images_it_lost():
+ """The note exists to say which of the returned pictures the model was really
+ shown, so a turn left holding two while still reading "first 4 of 8" defeats it."""
+ conversation = [
+ {
+ "role": "user",
+ "content": [
+ {"type": "image"},
+ {"type": "image"},
+ {"type": "image"},
+ {"type": "image"},
+ {"type": "text", "text": mcp_images._turn_text(4, 8)},
+ ],
+ },
+ mcp_images.placeholder_turn(4, 4),
+ mcp_images.placeholder_turn(2, 2),
+ ]
+ payloads = [f"p{i}" for i in range(10)]
+
+ mcp_images.trim_image_turns(conversation, payloads)
+
+ partial = conversation[0]
+ remaining = sum(1 for part in partial["content"] if part.get("type") == "image")
+ note = next(part["text"] for part in partial["content"] if part.get("type") == "text")
+ assert remaining == 2, remaining
+ assert f"({remaining} of 8)" in note, note
+ assert "first" not in note, "the survivors are the newest images, not the first ones"
+
+
+def test_the_owned_list_drops_evicted_parts_before_it_counts():
+ """A rolling-context fitter evicts whole turns mid-loop without telling this list.
+ Counting parts that already left over-trims the ones still in it."""
+ owned: list = []
+ conversation: list = []
+ mcp_images.append_image_turn(
+ conversation,
+ [[{"data": _png(), "mimeType": "image/png"} for _ in range(4)]],
+ per_result = True,
+ owned = owned,
+ )
+ mcp_images.append_image_turn(
+ conversation,
+ [[{"data": _png(), "mimeType": "image/png"} for _ in range(4)]],
+ per_result = True,
+ owned = owned,
+ )
+ assert len(owned) == 8
+
+ # What truncation does: the turns go, the ownership list is not told.
+ conversation.clear()
+
+ mcp_images.append_image_turn(
+ conversation,
+ [[{"data": _png(), "mimeType": "image/png"} for _ in range(4)]],
+ per_result = True,
+ owned = owned,
+ )
+
+ live = sum(
+ 1
+ for message in conversation
+ if isinstance(message.get("content"), list)
+ for part in message["content"]
+ if part.get("type") == "image_url"
+ )
+ assert live == 4, f"the fresh result was over-trimmed to {live}"
+
+
+def test_a_live_result_of_unreadable_entries_bounds_its_decode_attempts():
+ """The payload-byte budget caps how BIG a result is, never how many entries it
+ holds, so a server answering with thousands of tiny unreadable ones could hold an
+ inference worker open on Pillow for a turn that can show four pictures at most."""
+ attempts: list = []
+
+ def _counting_url(data):
+ attempts.append(data)
+ return None
+
+ junk = [{"data": f"not-an-image-{i}", "mimeType": "image/png"} for i in range(5000)]
+
+ original = mcp_images._png_data_url
+ mcp_images._png_data_url = _counting_url
+ try:
+ urls = mcp_images._decoded_urls(junk)
+ finally:
+ mcp_images._png_data_url = original
+
+ assert urls == []
+ assert len(attempts) <= (
+ mcp_images.MAX_MODEL_IMAGES + mcp_images.DECODE_FAILURE_ALLOWANCE
+ ), f"{len(attempts)} Pillow opens for one tool result"
+
+
+def test_the_allowance_still_looks_past_a_run_of_rejects():
+ """The bound must not defeat what it is bounding: the whole reason entries are not
+ sliced first is that valid PNGs sit behind formats Pillow rejects."""
+ seen: list = []
+
+ def _url(data):
+ seen.append(data)
+ return None if data.startswith("bad") else "data:image/png;base64," + _png()
+
+ images = [{"data": f"bad{i}", "mimeType": "image/svg+xml"} for i in range(3)]
+ images += [{"data": f"good{i}", "mimeType": "image/png"} for i in range(4)]
+
+ original = mcp_images._png_data_url
+ mcp_images._png_data_url = _url
+ try:
+ urls = mcp_images._decoded_urls(images)
+ finally:
+ mcp_images._png_data_url = original
+
+ assert (
+ len(urls) == mcp_images.MAX_MODEL_IMAGES
+ ), f"the bound cut into the real pictures: {len(urls)}"
+
+
+def test_the_note_never_claims_the_survivors_are_the_first_ones():
+ """An oversized parallel batch keeps its NEWEST results, so a note reading
+ 'first 8 of 12' names images the model was never shown."""
+ results = [
+ [{"data": _png(size = (4 + index, 4 + index)), "mimeType": "image/png"} for _ in range(4)]
+ for index in range(3)
+ ]
+ conversation: list = []
+
+ mcp_images.append_image_turn(conversation, results, per_result = True, limit = None)
+
+ note = next(
+ part["text"]
+ for message in conversation
+ if isinstance(message.get("content"), list)
+ for part in message["content"]
+ if part.get("type") == "text"
+ )
+ assert "(8 of 12)" in note, note
+ assert "first" not in note, note
+
+
+def test_a_parallel_batch_shares_one_decode_attempt_budget():
+ """room falls only on a SUCCESSFUL decode, so results that fail late in Pillow
+ never close the loop. Per result the allowance reset, and a 25-call turn of
+ malformed results bought 25 x 8 decodes of attacker-chosen rasters against a
+ conversation cap of eight pictures."""
+ attempts: list = []
+ original = mcp_images._png_data_url
+ mcp_images._png_data_url = lambda data: (attempts.append(data), None)[1]
+ try:
+ results = [
+ [{"data": f"junk{r}-{i}", "mimeType": "image/png"} for i in range(8)] for r in range(25)
+ ]
+ assert mcp_images._decoded_urls_per_result(results) == []
+ finally:
+ mcp_images._png_data_url = original
+
+ assert len(attempts) <= (
+ mcp_images.MAX_TOTAL_MODEL_IMAGES + mcp_images.DECODE_FAILURE_ALLOWANCE
+ ), f"{len(attempts)} Pillow opens for one parallel turn"
+
+
+def test_the_shared_budget_still_reaches_a_good_result_behind_bad_ones():
+ """The bound must not defeat what it protects: the newest result failing must
+ still leave enough attempts to find the real pictures behind it."""
+
+ def _url(data):
+ return None if data.startswith("bad") else "data:image/png;base64," + _png()
+
+ original = mcp_images._png_data_url
+ mcp_images._png_data_url = _url
+ try:
+ results = [
+ [{"data": f"good{i}", "mimeType": "image/png"} for i in range(4)],
+ [{"data": f"bad{i}", "mimeType": "image/svg+xml"} for i in range(4)],
+ ]
+ urls = mcp_images._decoded_urls_per_result(results)
+ finally:
+ mcp_images._png_data_url = original
+
+ assert len(urls) == 4, f"the older result's real pictures were lost: {len(urls)}"
+
+
+def test_a_complete_turn_keeps_its_total_when_the_cap_first_trims_it():
+ """A turn that arrived complete carries no "(n of m)" suffix. Falling back to the
+ post-trim count there told the model nothing had been dropped -- the one thing
+ the note exists to say."""
+ conversation = [
+ mcp_images.placeholder_turn(4, 4),
+ mcp_images.placeholder_turn(4, 4),
+ mcp_images.placeholder_turn(2, 2),
+ ]
+ payloads = [f"p{i}" for i in range(10)]
+
+ mcp_images.trim_image_turns(conversation, payloads)
+
+ kept = sum(1 for part in conversation[0]["content"] if part.get("type") == "image")
+ note = next(part["text"] for part in conversation[0]["content"] if part.get("type") == "text")
+ assert kept == 2
+ assert f"({kept} of 4)" in note, note
+
+
+def test_a_transparent_screenshot_is_composited_rather_than_flattened_to_black():
+ """convert("RGB") keeps whatever colour sits UNDER the alpha, and a tool that
+ never painted a background leaves that black -- so a transparent screenshot's
+ dark text converted to black on black and the model saw a blank rectangle."""
+ import base64 as _b64
+ import io as _io
+
+ from PIL import Image
+
+ def _roundtrip(image):
+ buffer = _io.BytesIO()
+ image.save(buffer, format = "PNG")
+ url = mcp_images._png_data_url(_b64.b64encode(buffer.getvalue()).decode())
+ assert url, "the image did not decode at all"
+ return Image.open(_io.BytesIO(_b64.b64decode(url.split(",", 1)[1])))
+
+ rgba = Image.new("RGBA", (8, 8), (0, 0, 0, 0))
+ rgba.putpixel((4, 4), (0, 0, 0, 255))
+ palette = Image.new("P", (8, 8), 0)
+ palette.putpalette([0, 0, 0] * 256)
+ palette.info["transparency"] = 0
+ palette.putpixel((4, 4), 1)
+ grey = Image.new("LA", (8, 8), (0, 0))
+ grey.putpixel((4, 4), (0, 255))
+
+ for name, image in (("RGBA", rgba), ("P", palette), ("LA", grey)):
+ out = _roundtrip(image)
+ assert out.getpixel((0, 0)) == (255, 255, 255), f"{name}: background not composited"
+ assert out.getpixel((4, 4)) == (0, 0, 0), f"{name}: the drawn pixel was lost"
+
+
+def test_replay_leaves_room_for_the_pictures_the_caller_attached():
+ """Providers apply their own per-request cap in document order, and promotion
+ PREPENDS the replay to the user turn. On Gemini (8 images, later ones dropped
+ silently) eight replayed screenshots evicted the picture the current question
+ was about."""
+ attachment = {
+ "type": "image_url",
+ "image_url": {"url": "data:image/png;base64," + _png()},
+ }
+ history = []
+ for index in range(2):
+ history.append(
+ {
+ "role": "assistant",
+ "content": "",
+ "tool_calls": [
+ {
+ "id": f"c{index}",
+ "type": "function",
+ "function": {"name": "mcp__s__shot", "arguments": "{}"},
+ }
+ ],
+ }
+ )
+ history.append(
+ {
+ "role": "tool",
+ "tool_call_id": f"c{index}",
+ "name": "mcp__s__shot",
+ "content": _envelope("[4]", *[_image() for _ in range(4)]),
+ }
+ )
+ history.append(
+ {
+ "role": "user",
+ "content": [{"type": "text", "text": "and how does this compare?"}, attachment],
+ }
+ )
+
+ out = promote_history(history, vision = True, reserve_for_caller = True)
+ urls = [
+ part["image_url"]["url"]
+ for message in out
+ if isinstance(message.get("content"), list)
+ for part in message["content"]
+ if part.get("type") == "image_url"
+ ]
+
+ assert len(urls) <= mcp_images.MAX_TOTAL_MODEL_IMAGES, len(urls)
+ assert attachment["image_url"]["url"] in urls, "the caller's own picture was trimmed"
+ # And it survives a provider that keeps only the first MAX_TOTAL_MODEL_IMAGES.
+ assert attachment["image_url"]["url"] in urls[: mcp_images.MAX_TOTAL_MODEL_IMAGES]
+
+
+def test_the_replay_trim_runs_before_the_attachment_marker_exists():
+ """The trim drops by marker ORDINAL against a payload list holding replay only.
+ With the attachment's turn AHEAD of the replayed pictures, running it after the
+ mark deleted ordinal 0 -- the attachment's own marker -- while charging replay
+ payload 0, and every later pixel shifted onto the marker before it."""
+
+ def _scene():
+ return [
+ {"role": "user", "content": "here is my diagram"},
+ mcp_images.placeholder_turn(4, 4),
+ mcp_images.placeholder_turn(4, 4),
+ {"role": "user", "content": "which is bluer?"},
+ ]
+
+ replay = [f"MCP{i}" for i in range(8)]
+
+ # The order the route now uses: trim, snapshot, then mark.
+ conversation, payloads = _scene(), list(replay)
+ mcp_images.trim_image_turns(conversation, payloads, limit = mcp_images.MAX_TOTAL_MODEL_IMAGES - 1)
+ prior = mcp_images.image_marker_parts(conversation)
+ conversation = mcp_images.mark_last_user_turn(conversation, 1, ordinal = 0)
+ ordered = mcp_images.pixels_in_marker_order(conversation, prior, payloads, "ATTACHMENT")
+
+ owner = conversation[0]["content"]
+ assert isinstance(owner, list) and any(
+ part.get("type") == "image" for part in owner
+ ), "the attachment's own marker was trimmed away"
+ assert ordered[0] == "ATTACHMENT", ordered
+ assert len(mcp_images.image_marker_parts(conversation)) == len(ordered)
+
+
+def test_the_route_trims_replay_before_marking_the_attachment():
+ import inspect
+
+ from routes import inference
+
+ body = inspect.getsource(inference.produce_openai_chat_completions)
+ trim = body.index("trim_mcp_image_turns(\n _sf_chat_messages")
+ mark = body.index("_sf_chat_messages = mark_mcp_image_turn(")
+ prior = body.index("_sf_prior_markers = mcp_image_marker_parts(_sf_chat_messages)")
+ assert trim < prior < mark, (
+ "trim, then snapshot history, then mark -- any other order binds the "
+ "attachment to a replayed picture's marker"
+ )
+
+
+def test_a_live_result_leaves_room_for_the_pictures_the_caller_attached():
+ """owned holds only what the loop appended, so the cap allowed the attachment
+ PLUS a full eight tool images. A provider keeping the first eight in document
+ order then drops the newest result -- the one the model just asked for."""
+ conversation = [
+ {
+ "role": "user",
+ "content": [
+ {"type": "text", "text": "compare with this"},
+ {"type": "image_url", "image_url": {"url": "data:image/png;base64," + _png()}},
+ ],
+ }
+ ]
+ owned: list = []
+ for _ in range(3):
+ mcp_images.append_image_turn(
+ conversation,
+ [[{"data": _png(), "mimeType": "image/png"} for _ in range(4)]],
+ per_result = True,
+ owned = owned,
+ # What the EXTERNAL loop passes. The GGUF loop does not: it answers to a
+ # context window rather than to a provider's per-request image count.
+ reserve_caller_images = True,
+ )
+
+ total = sum(
+ 1
+ for message in conversation
+ if isinstance(message.get("content"), list)
+ for part in message["content"]
+ if part.get("type") == "image_url"
+ )
+ assert total <= mcp_images.MAX_TOTAL_MODEL_IMAGES, (
+ f"{total} images in the conversation against a cap of "
+ f"{mcp_images.MAX_TOTAL_MODEL_IMAGES}"
+ )
+ # And the caller's own picture is not what made room.
+ assert any(
+ part.get("type") == "image_url" for part in conversation[0]["content"]
+ ), "the attachment was trimmed to fit the tool's results"
+
+
+def test_the_local_loop_keeps_its_own_allowance_beside_an_attachment():
+ """The reservation is the EXTERNAL loop's, asked for rather than assumed: a
+ remote provider counts images per request, llama-server answers to a context
+ window. Reserving here would let six attachments squeeze the tool results the
+ model asked for down to two."""
+ attachments = [
+ {"type": "image_url", "image_url": {"url": f"data:image/png;base64,{_png()}"}}
+ for _ in range(6)
+ ]
+ conversation = [
+ {"role": "user", "content": [{"type": "text", "text": "compare"}, *attachments]}
+ ]
+ owned: list = []
+ for _ in range(3):
+ mcp_images.append_image_turn(
+ conversation,
+ [[{"data": _png(), "mimeType": "image/png"} for _ in range(4)]],
+ per_result = True,
+ owned = owned,
+ )
+
+ assert len(owned) == mcp_images.MAX_TOTAL_MODEL_IMAGES
+ survived = sum(
+ 1
+ for part in attachments
+ for message in conversation
+ if isinstance(message.get("content"), list)
+ for other in message["content"]
+ if other is part
+ )
+ assert survived == len(attachments), "the loop's cap is still not the caller's"
+
+
+def test_an_unnamed_tool_result_is_judged_by_the_call_that_made_it():
+ """role="tool" carries no name in plain OpenAI (the field is optional) or in
+ anything translated from Anthropic, and an absent name is read as legacy MCP
+ history that may be trusted -- so any client tool whose output merely ends in a
+ valid envelope was promoted as image input."""
+ for tool, promotes in (("read_file", False), ("mcp__shot__capture", True)):
+ history = [
+ {
+ "role": "assistant",
+ "content": "",
+ "tool_calls": [
+ {
+ "id": "call_0",
+ "type": "function",
+ "function": {"name": tool, "arguments": "{}"},
+ }
+ ],
+ },
+ # No "name" on the result, as the wire format allows.
+ {"role": "tool", "tool_call_id": "call_0", "content": _envelope("[1]", _image())},
+ ]
+
+ out = promote_history(history, vision = True)
+ parts = sum(
+ 1
+ for message in out
+ if isinstance(message.get("content"), list)
+ for part in message["content"]
+ if part.get("type") == "image_url"
+ )
+ assert bool(parts) is promotes, f"{tool}: promoted={bool(parts)}"
+ # Either way the suffix comes off the text; only IMAGE input is gated.
+ assert mcp_images.SENTINEL not in _json.dumps(out)
+
+
+def test_the_note_reports_what_the_tool_returned_not_what_admission_allowed():
+ """The admission pass slices the candidates before promotion sees them, so
+ summing those made a 100-image result read as a handful -- the note describing
+ the admission pass rather than the tool, next to a result saying 100."""
+ history = [
+ {
+ "role": "tool",
+ "name": "mcp__s__shot",
+ "content": _envelope("[100 images returned]", *[_image() for _ in range(100)]),
+ }
+ ]
+
+ out = promote_history(history, vision = True)
+
+ note = next(
+ part["text"]
+ for message in out
+ if isinstance(message.get("content"), list)
+ for part in message["content"]
+ if part.get("type") == "text"
+ )
+ assert "of 100)" in note, note
+
+
+def test_the_caller_attachment_is_composited_on_the_way_to_the_worker():
+ """The server-tool path serialises the attachment to base64 for the worker. Plain
+ convert("RGB") kept whatever colour sat under the alpha, so a transparent
+ attachment whose background was never painted arrived black -- with its dark text
+ gone. The ordinary IPC path carries the PNG's alpha through untouched."""
+ import base64 as _b64
+ import io as _io
+
+ from PIL import Image
+
+ from routes.inference import _pil_to_png_b64
+
+ image = Image.new("RGBA", (8, 8), (0, 0, 0, 0))
+ image.putpixel((4, 4), (0, 0, 0, 255))
+
+ out = Image.open(_io.BytesIO(_b64.b64decode(_pil_to_png_b64(image))))
+
+ assert out.getpixel((0, 0)) == (255, 255, 255), "transparency flattened to black"
+ assert out.getpixel((4, 4)) == (0, 0, 0), "the drawn pixel was lost"
+
+
+def _call(call_id, name):
+ return {
+ "role": "assistant",
+ "content": "",
+ "tool_calls": [
+ {"id": call_id, "type": "function", "function": {"name": name, "arguments": "{}"}}
+ ],
+ }
+
+
+def test_repeated_call_ids_pair_each_result_with_its_own_call():
+ """The backend restarts ids like call_0 every response. A conversation-wide
+ last-wins lookup let a later non-MCP call_0 rename an earlier MCP result -- and
+ the reverse -- so an earlier picture was suppressed or a later envelope trusted."""
+ history = [
+ _call("call_0", "mcp__shot__capture"),
+ {"role": "tool", "tool_call_id": "call_0", "content": _envelope("[1]", _image())},
+ {"role": "assistant", "content": "a blue square"},
+ _call("call_0", "read_file"),
+ {"role": "tool", "tool_call_id": "call_0", "content": _envelope("[1]", _image())},
+ ]
+
+ names = mcp_images.resolve_tool_names(history)
+ assert names == {1: "mcp__shot__capture", 4: "read_file"}, names
+
+ out = promote_history(history, vision = True)
+ promoted = sum(
+ 1
+ for message in out
+ if isinstance(message.get("content"), list)
+ for part in message["content"]
+ if part.get("type") == "image_url"
+ )
+ assert promoted == 1, "exactly the MCP result's picture, not the client tool's"
+
+
+def test_a_batch_with_a_later_text_result_does_not_say_the_tool_call_above():
+ """pending stays open across a later image-free result of the same batch, so the
+ image turn lands after it and 'the tool call above' named the wrong call."""
+ history = [
+ {"role": "tool", "name": "mcp__s__shot", "content": _envelope("[1]", _image())},
+ {"role": "tool", "name": "mcp__s__list", "content": "three files"},
+ # An assistant follows, so the turn is INSERTED after the batch rather than
+ # merged into a user message -- the case that carries a note at all.
+ {"role": "assistant", "content": "done"},
+ ]
+
+ out = promote_history(history, vision = True)
+
+ note = next(
+ part["text"]
+ for message in out
+ if isinstance(message.get("content"), list)
+ for part in message["content"]
+ if part.get("type") == "text" and part["text"].startswith("Images returned by")
+ )
+ assert note.startswith(mcp_images.DETACHED_IMAGE_TURN_TEXT), note
+
+
+def test_the_note_honours_a_returned_count_the_frontend_left_on_the_envelope():
+ """The frontend bounds the envelope before it is ever seen here; it records the
+ length it started from on the first entry so the note can still say so."""
+ images = [_image() for _ in range(2)]
+ images[0]["returned"] = 100
+ history = [{"role": "tool", "name": "mcp__s__shot", "content": _envelope("[100]", *images)}]
+
+ out = promote_history(history, vision = True)
+
+ note = next(
+ part["text"]
+ for message in out
+ if isinstance(message.get("content"), list)
+ for part in message["content"]
+ if part.get("type") == "text"
+ )
+ assert "of 100)" in note, note
+
+
+def test_the_local_rebuild_names_an_unnamed_result_from_its_call():
+ """_extract_content_parts drops tool_call_id and the calls, so the local path
+ could not run the provenance gate on an unnamed result at all."""
+ from models.inference import ChatMessage
+ from routes.inference import _extract_content_parts
+
+ messages = [
+ ChatMessage(**_call("call_0", "read_file")),
+ ChatMessage(role = "tool", tool_call_id = "call_0", content = _envelope("[1]", _image())),
+ ]
+
+ _, chat_messages, _ = _extract_content_parts(messages, keep_tool_images = True)
+
+ tool = next(message for message in chat_messages if message["role"] == "tool")
+ assert tool.get("name") == "read_file", tool
+
+
+def test_admission_does_not_charge_an_envelope_a_non_mcp_call_produced():
+ from routes.inference import _openai_llama_admission_messages_for_estimate
+
+ history = [
+ _call("call_0", "read_file"),
+ {
+ "role": "tool",
+ "tool_call_id": "call_0",
+ "content": _envelope("[4]", *[_image() for _ in range(4)]),
+ },
+ ]
+
+ _, image_parts = _openai_llama_admission_messages_for_estimate(history)
+
+ assert image_parts == 0, f"reserved {image_parts} embeddings generation never sends"
+
+
+def test_the_monitor_prompt_never_retains_envelope_bytes():
+ from routes.inference import _monitor_prompt_from_messages
+
+ payload = _png()
+ text = _monitor_prompt_from_messages(
+ [
+ {
+ "role": "tool",
+ "content": _envelope(
+ "[1 image returned]", {"data": payload, "mimeType": "image/png"}
+ ),
+ }
+ ]
+ )
+
+ assert payload not in text
+ assert mcp_images.SENTINEL not in text
+ assert "[1 image returned]" in text
+
+
+def test_eligibility_is_not_spent_on_results_a_non_mcp_call_produced():
+ """The preselection read only the explicit name, so two newer unnamed results
+ correlated to non-MCP calls took the allowance and a genuine older MCP result was
+ left an eligibility of zero -- nothing useful replayed with capacity to spare."""
+ history = [
+ _call("call_0", "mcp__shot__capture"),
+ {
+ "role": "tool",
+ "tool_call_id": "call_0",
+ "content": _envelope("[4]", *[_image() for _ in range(4)]),
+ },
+ _call("call_1", "read_file"),
+ {
+ "role": "tool",
+ "tool_call_id": "call_1",
+ "content": _envelope("[4]", *[_image() for _ in range(4)]),
+ },
+ _call("call_2", "read_file"),
+ {
+ "role": "tool",
+ "tool_call_id": "call_2",
+ "content": _envelope("[4]", *[_image() for _ in range(4)]),
+ },
+ ]
+
+ eligible = mcp_images.eligible_replay_images(history)
+
+ assert 3 not in eligible and 5 not in eligible, eligible
+ assert eligible.get(1) == 4, eligible
+
+
+def test_a_detached_image_turn_is_synthetic_too():
+ """Counting the detached block as a real user turn put the attachment's marker on
+ it instead of on the question that supplied the picture."""
+ detached = mcp_images.placeholder_turn(1, 1, mcp_images.DETACHED_IMAGE_TURN_TEXT)
+ assert mcp_images.is_synthetic_image_turn(detached)
+
+ conversation = [
+ {"role": "user", "content": "first question"},
+ detached,
+ {"role": "user", "content": "the question with the picture"},
+ ]
+ topped = mcp_images.top_up_image_markers(conversation, 2, ordinal = 1)
+
+ assert isinstance(topped[2]["content"], list), topped
+ assert any(part.get("type") == "image" for part in topped[2]["content"])
+ assert isinstance(topped[0]["content"], str), "the first question gained no marker"
+
+
+def test_a_local_placeholder_turn_carries_one_picture():
+ """The route refuses a caller message with more than one image on every non-GGUF
+ target -- "This model takes one image per message" -- and no processor is known
+ to take several. Promotion was building exactly that shape, so a processor with
+ the limit failed mid-generation after the tool had already run."""
+ results = [[_image() for _ in range(4)], [_image() for _ in range(4)]]
+
+ payloads = mcp_images.png_payloads_per_result(results)
+ assert len(payloads) == mcp_images.LOCAL_MAX_IMAGES_PER_TURN == 1
+
+ out, replay = promote_history_local_for_test(
+ [
+ {
+ "role": "tool",
+ "name": "mcp__s__shot",
+ "content": _envelope("[4 images returned]", *results[0]),
+ }
+ ]
+ )
+ assert len(replay) == 1
+ turn = next(m for m in out if isinstance(m.get("content"), list))
+ assert sum(1 for p in turn["content"] if p.get("type") == "image") == 1
+ note = next(p["text"] for p in turn["content"] if p.get("type") == "text")
+ assert "(1 of 4)" in note, note
+
+
+def promote_history_local_for_test(messages):
+ return mcp_images.promote_history_local(messages, vision = True)
+
+
+def test_a_named_non_mcp_result_between_the_images_and_their_turn_detaches_the_note():
+ """The early continue for a named non-MCP result skipped the interruption flag, so
+ the block landed after web_search or read_file still saying 'the tool call above'."""
+ history = [
+ {"role": "tool", "name": "mcp__s__shot", "content": _envelope("[1]", _image())},
+ {"role": "tool", "name": "web_search", "content": "three results"},
+ {"role": "assistant", "content": "done"},
+ ]
+
+ out = promote_history(history, vision = True)
+
+ note = next(
+ part["text"]
+ for message in out
+ if isinstance(message.get("content"), list)
+ for part in message["content"]
+ if part.get("type") == "text" and part["text"].startswith("Images returned by")
+ )
+ assert note.startswith(mcp_images.DETACHED_IMAGE_TURN_TEXT), note
+
+
+def test_the_external_loop_detaches_the_note_for_a_multi_result_batch():
+ import inspect
+
+ from core.inference import studio_tool_loop
+
+ body = inspect.getsource(studio_tool_loop)
+ assert "_lead = DETACHED_IMAGE_TURN_TEXT if len(tool_messages) != 1 else None" in body
+
+ conversation: list = []
+ owned: list = []
+ studio_tool_loop._append_mcp_images_owned(
+ conversation, [[_image()]], owned, mcp_images.DETACHED_IMAGE_TURN_TEXT
+ )
+ note = next(
+ part["text"]
+ for message in conversation
+ if isinstance(message.get("content"), list)
+ for part in message["content"]
+ if part.get("type") == "text"
+ )
+ assert note.startswith(mcp_images.DETACHED_IMAGE_TURN_TEXT)
+
+
+def test_the_attachment_displaces_a_replay_marker_merged_into_its_turn():
+ """A replayed picture merges its marker into the following user turn; if that turn
+ then owns the attachment, the top-up added a second marker. Non-GGUF messages
+ take one image, so that shape failed at render, past the request validation. The
+ attachment wins, and the displaced payload drops rather than sliding onto the
+ next marker."""
+ replay = {"role": "user", "content": [{"type": "image"}, {"type": "text", "text": "and this?"}]}
+ conversation = [{"role": "user", "content": "first"}, replay]
+ prior = mcp_images.image_marker_parts(conversation)
+ assert len(prior) == 1
+
+ topped = mcp_images.top_up_image_markers(conversation, 2, ordinal = 1)
+
+ turn = topped[1]["content"]
+ assert sum(1 for p in turn if p.get("type") == "image") == 1, turn
+ ordered = mcp_images.pixels_in_marker_order(topped, prior, ["REPLAY"], "ATTACHMENT")
+ assert ordered == ["ATTACHMENT"], ordered
+
+
+def test_pixels_follow_their_own_marker_not_a_queue():
+ """Two replay markers, the first displaced: its payload must vanish, not bind to
+ the second marker."""
+ a = {"type": "image"}
+ b = {"type": "image"}
+ conversation = [{"role": "user", "content": [b, {"type": "text", "text": "x"}]}]
+ ordered = mcp_images.pixels_in_marker_order(conversation, [a, b], ["PA", "PB"], "NEW")
+ assert ordered == ["PB"], ordered
+
+
+def test_the_local_batch_shares_one_decode_attempt_budget():
+ attempts: list = []
+ original = mcp_images._png_data_url
+ mcp_images._png_data_url = lambda data: (attempts.append(data), None)[1]
+ try:
+ results = [
+ [{"data": f"j{r}-{i}", "mimeType": "image/png"} for i in range(8)] for r in range(25)
+ ]
+ assert mcp_images.png_payloads_per_result(results) == []
+ finally:
+ mcp_images._png_data_url = original
+ assert (
+ len(attempts) <= mcp_images.LOCAL_MAX_IMAGES_PER_TURN + mcp_images.DECODE_FAILURE_ALLOWANCE
+ ), len(attempts)
+
+
+def test_the_dispatch_check_never_parses_the_envelope():
+ """has_images json-loads the whole array; the async wrappers only need to know
+ whether to leave the event loop, and a 12 MB parse on the loop to decide that
+ stalled everything beside it."""
+ import inspect
+
+ from routes import inference
+
+ assert mcp_images.mentions_images("x\n" + mcp_images.SENTINEL + "[not even json")
+ assert not mcp_images.mentions_images("plain text")
+ for fn in (
+ inference._build_external_messages_async,
+ inference._promote_local_mcp_images_async,
+ inference._promote_mcp_history_images_async,
+ ):
+ src = inspect.getsource(fn)
+ assert "_messages_mention_mcp_images(messages)" in src, fn.__name__
+ assert "_messages_have_mcp_image_envelope(messages)" not in src, fn.__name__
+
+
+def test_the_promotable_check_correlates_an_unnamed_result():
+ """chat_count_tokens refuses a prompt it reads as carrying a promotable image. An
+ unnamed result whose call was read_file is never promoted, so refusing on it
+ turned away a countable text prompt."""
+ from models.inference import ChatCompletionRequest
+ from routes.inference import _request_has_promotable_mcp_images
+
+ def _req(tool):
+ return ChatCompletionRequest(
+ model = "default",
+ messages = [
+ _call("call_0", tool),
+ {"role": "tool", "tool_call_id": "call_0", "content": _envelope("[1]", _image())},
+ ],
+ )
+
+ assert not _request_has_promotable_mcp_images(_req("read_file"))
+ assert _request_has_promotable_mcp_images(_req("mcp__shot__capture"))
+
+
+def test_the_server_tool_route_placement_also_displaces_a_merged_replay_marker():
+ """mark_last_user_turn is the server-tool route's placement; it has to follow the
+ same one-image rule as the top-up or that path keeps building two-image turns."""
+ conversation = [
+ {"role": "user", "content": "first"},
+ {"role": "user", "content": [{"type": "image"}, {"type": "text", "text": "and this?"}]},
+ ]
+ marked = mcp_images.mark_last_user_turn(conversation, 1, ordinal = 1)
+ assert sum(1 for p in marked[1]["content"] if p.get("type") == "image") == 1, marked[1]
+
+
+def _rounds(
+ count: int,
+ per_result: int,
+ *,
+ parallel: int = 1,
+) -> list[dict]:
+ history: list[dict] = [{"role": "user", "content": "start"}]
+ for r in range(count):
+ calls = [
+ {
+ "id": f"c{r}_{k}",
+ "type": "function",
+ "function": {"name": "mcp__s__shot", "arguments": "{}"},
+ }
+ for k in range(parallel)
+ ]
+ history.append({"role": "assistant", "content": "", "tool_calls": calls})
+ for k in range(parallel):
+ history.append(
+ {
+ "role": "tool",
+ "tool_call_id": f"c{r}_{k}",
+ "content": _envelope(f"[{per_result}]", *[_image() for _ in range(per_result)]),
+ }
+ )
+ history.append({"role": "assistant", "content": f"round {r}"})
+ history.append({"role": "user", "content": "compare them"})
+ return history
+
+
+def _one_marker_per_turn(conversation: list) -> list[int]:
+ counts = [
+ sum(1 for part in message["content"] if part.get("type") == "image")
+ for message in conversation
+ if isinstance(message.get("content"), list)
+ ]
+ assert all(count <= 1 for count in counts), counts
+ return counts
+
+
+def test_local_replay_keeps_every_picture_the_live_loop_kept():
+ """Eight rounds of four valid PNGs. The live loop keeps one per round, eight in
+ all; budgeting four per result for a path that sends one spent the allowance on
+ the newest two rounds and replayed three of the eight."""
+ history = _rounds(8, 4)
+ live = min(
+ sum(
+ len(mcp_images.png_payloads_per_result([[_image() for _ in range(4)]]))
+ for _ in range(8)
+ ),
+ mcp_images.MAX_TOTAL_MODEL_IMAGES,
+ )
+ assert live == 8
+
+ out, payloads = mcp_images.promote_history_local(history, vision = True)
+ assert len(payloads) == live
+ assert sum(_one_marker_per_turn(out)) == live
+
+ eligible = mcp_images.eligible_replay_images(history, local = True)
+ tool_positions = [i for i, m in enumerate(history) if m.get("role") == "tool"]
+ assert all(eligible.get(i, 0) >= 1 for i in tool_positions), eligible
+ # The part paths still take four per result: two rounds fill the budget and a
+ # third rides on the spare allowance, so only three results stay eligible there.
+ parts = mcp_images.eligible_replay_images(history)
+ assert sum(1 for n in parts.values() if n) == 3, parts
+
+
+def test_local_replay_charges_a_parallel_batch_once():
+ """Three parallel results land as one turn carrying one picture, so the batch is
+ charged one, not three: eight such rounds still replay eight pictures."""
+ history = _rounds(8, 2, parallel = 3)
+ out, payloads = mcp_images.promote_history_local(history, vision = True)
+ assert len(payloads) == 8
+ assert sum(_one_marker_per_turn(out)) == 8
+
+
+def _client_tool_route(monkeypatch, messages, *, tools):
+ import asyncio
+ from types import SimpleNamespace
+
+ import routes.inference as inf
+ from core.inference.api_monitor import ApiMonitor
+ from models.inference import ChatCompletionRequest, ChatMessage
+ from state.tool_policy import reset_tool_policy
+
+ class Request:
+ state = SimpleNamespace()
+ url = SimpleNamespace(path = "/v1/chat/completions")
+ method = "POST"
+ scope: dict = {}
+ headers = {"X-Unsloth-Events": "1"}
+
+ async def is_disconnected(self):
+ return False
+
+ class Backend:
+ active_model_name = "vlm"
+
+ def __init__(self):
+ self.models = {
+ "vlm": {
+ "chat_template_info": {
+ "template": "<tool_call> chatml",
+ "renders_image": True,
+ },
+ "context_length": 4096,
+ "is_vision": True,
+ }
+ }
+ self.calls: list = []
+
+ def generate_chat_response(
+ self,
+ *,
+ messages,
+ tools = None,
+ stats_holder = None,
+ **kwargs,
+ ):
+ self.calls.append({"messages": messages, "tools": tools, **kwargs})
+ yield "ok"
+
+ def reset_generation_state(self, caller_cancel_event = None):
+ return None
+
+ def resize_image(self, image):
+ return image
+
+ backend = Backend()
+ reset_tool_policy()
+ monkeypatch.setattr(inf, "api_monitor", ApiMonitor(max_entries = 8))
+ monkeypatch.setattr(
+ inf,
+ "get_llama_cpp_backend",
+ lambda: SimpleNamespace(
+ is_loaded = False, supports_tools = False, is_vision = False, context_length = None
+ ),
+ )
+ monkeypatch.setattr(inf, "get_inference_backend", lambda: backend)
+ monkeypatch.setattr(
+ inf, "_detect_safetensors_features", lambda *a, **k: {"supports_tools": True}
+ )
+ payload = ChatCompletionRequest(
+ model = "default",
+ stream = False,
+ tools = tools,
+ messages = [ChatMessage(**message) for message in messages],
+ )
+
+ async def run():
+ return await inf.openai_chat_completions(payload, request = Request(), current_subject = "u")
+
+ asyncio.run(run())
+ assert len(backend.calls) == 1, backend.calls
+ return backend.calls[0]
+
+
+_LOOKUP = {
+ "type": "function",
+ "function": {"name": "lookup", "parameters": {"type": "object", "properties": {}}},
+}
+
+
+def _shot_round(r: int, *images: dict) -> list[dict]:
+ return [
+ {
+ "role": "assistant",
+ "content": None,
+ "tool_calls": [
+ {
+ "id": f"call_{r}",
+ "type": "function",
+ "function": {"name": "mcp__fs__read_media_file", "arguments": "{}"},
+ }
+ ],
+ },
+ {"role": "tool", "tool_call_id": f"call_{r}", "content": _envelope("[1]", *images)},
+ ]
+
+
+def _no_adjacent_user_turns(messages: list) -> None:
+ roles = [message["role"] for message in messages]
+ assert all(a != "user" or b != "user" for a, b in zip(roles, roles[1:])), roles
+
+
+def test_the_client_tool_rebuild_keeps_one_picture_per_turn(monkeypatch):
+ """Two historical image results and a comparison question, with client tools:
+ the rebuilt history carries one marker per result at its own position, never two
+ on the final user message -- the shape the route refuses on a non-GGUF target."""
+ call = _client_tool_route(
+ monkeypatch,
+ [
+ {"role": "user", "content": "read a.png"},
+ *_shot_round(0, _image()),
+ {"role": "assistant", "content": "a is blue"},
+ {"role": "user", "content": "now b.png"},
+ *_shot_round(1, _image()),
+ {"role": "assistant", "content": "b is red"},
+ {"role": "user", "content": "compare them"},
+ ],
+ tools = [_LOOKUP],
+ )
+ assert len(call["images"]) == 2
+ assert sum(_one_marker_per_turn(call["messages"])) == 2, call["messages"]
+ _no_adjacent_user_turns(call["messages"])
+ assert isinstance(call["messages"][-1]["content"], str), "the question carries no marker"
+ assert call.get("image") is None
+
+
+def test_the_client_tool_rebuild_leaves_the_attachment_to_the_backend(monkeypatch):
+ """One replayed PNG and a new attachment on the question: no synthetic turn is
+ inserted beside the question (two user turns in a row), the replay keeps its
+ own marker, and the attachment's marker is the backend's to place by ordinal
+ from a snapshot that holds only the replay's."""
+ attachment = f"data:image/png;base64,{_png()}"
+ question = {
+ "role": "user",
+ "content": [
+ {"type": "text", "text": "and this one?"},
+ {"type": "image_url", "image_url": {"url": attachment}},
+ ],
+ }
+ for history in (
+ [
+ {"role": "user", "content": "read a.png"},
+ *_shot_round(0, _image()),
+ {"role": "assistant", "content": "a is blue"},
+ question,
+ ],
+ [{"role": "user", "content": "read a.png"}, *_shot_round(0, _image()), question],
+ ):
+ call = _client_tool_route(monkeypatch, history, tools = [_LOOKUP])
+ assert len(call["images"]) == 1
+ assert call.get("image") is not None
+ assert call.get("image_ordinal") == 1
+ _no_adjacent_user_turns(call["messages"])
+ assert sum(_one_marker_per_turn(call["messages"])) == 1, call["messages"]
+
+
+def test_a_gguf_replay_keeps_its_allowance_beside_the_attachment():
+ """llama-server is bounded by its context window, not a per-request image cap:
+ the live GGUF loop never reserved the attachment's slot, so the replay of the
+ same conversation must not lose a picture to it. A provider reserves it."""
+ history = [{"role": "user", "content": "start"}]
+ for r in range(9):
+ history += [
+ {
+ "role": "assistant",
+ "content": "",
+ "tool_calls": [
+ {
+ "id": f"c{r}",
+ "type": "function",
+ "function": {"name": "mcp__s__shot", "arguments": "{}"},
+ }
+ ],
+ },
+ {"role": "tool", "tool_call_id": f"c{r}", "content": _envelope("[1]", _image())},
+ {"role": "assistant", "content": f"round {r}"},
+ ]
+ history.append(
+ {
+ "role": "user",
+ "content": [
+ {"type": "text", "text": "and this one?"},
+ {"type": "image_url", "image_url": {"url": f"data:image/png;base64,{_png()}"}},
+ ],
+ }
+ )
+
+ def promoted_parts(out):
+ parts = mcp_images._all_image_url_parts(out)
+ return sum(1 for part in parts if part["image_url"]["url"].startswith("data:image/png"))
+
+ gguf = promote_history(history, vision = True)
+ assert promoted_parts(gguf) == mcp_images.MAX_TOTAL_MODEL_IMAGES + 1
+ provider = promote_history(history, vision = True, reserve_for_caller = True)
+ assert promoted_parts(provider) == mcp_images.MAX_TOTAL_MODEL_IMAGES
+
+
+def test_a_legacy_attachment_displaces_a_merged_replay_marker():
+ """No ordinal (the top-level image field has no part to locate) and a replay
+ marker already merged into the newest user turn: appending built a two-image
+ message. The attachment takes the turn, as the ordinal branch does."""
+ conversation = [
+ {"role": "user", "content": "read a.png"},
+ {"role": "tool", "content": "[1 image returned]"},
+ {
+ "role": "user",
+ "content": [{"type": "image"}, {"type": "text", "text": "and this one?"}],
+ },
+ ]
+ prior = mcp_images.image_marker_parts(conversation)
+ topped = mcp_images.top_up_image_markers(conversation, 2, ordinal = None)
+
+ markers = [part for part in topped[2]["content"] if part.get("type") == "image"]
+ assert len(markers) == 1, topped[2]
+ assert markers[0] is not prior[0], "the replay's marker was displaced, not kept"
+ assert isinstance(topped[0]["content"], str)
+ ordered = mcp_images.pixels_in_marker_order(topped, prior, ["REPLAY"], "ATTACHMENT")
+ assert ordered == ["ATTACHMENT"]
+
+
+def test_a_legacy_attachment_skips_a_synthetic_turn():
+ """The newest user turn can be a replay's own placeholder; the attachment goes on
+ the question, not on the replay's turn."""
+ conversation = [
+ {"role": "user", "content": "the question"},
+ {"role": "tool", "content": "[1 image returned]"},
+ mcp_images.placeholder_turn(1, 1),
+ ]
+ topped = mcp_images.top_up_image_markers(conversation, 2, ordinal = None)
+ assert isinstance(topped[0]["content"], list), topped
+ assert sum(1 for part in topped[2]["content"] if part.get("type") == "image") == 1
+
+
+def test_the_catalog_predicate_never_parses_the_envelope():
+ """The dispatch predicate runs on the event loop; the exact check json-loads a
+ 12 MB envelope. The substring form must not touch the parser."""
+ import routes.inference as inference_route
+ from models.inference import ChatCompletionRequest, ChatMessage
+
+ payload = ChatCompletionRequest(
+ model = "default",
+ messages = [
+ ChatMessage(role = "user", content = "look"),
+ ChatMessage(
+ role = "tool",
+ tool_call_id = "c0",
+ name = "mcp__fs__shot",
+ content = _envelope("[1]", _image()),
+ ),
+ ],
+ )
+
+ def boom(_content):
+ raise AssertionError("parsed on the loop")
+
+ original = inference_route.mcp_images_sentinel_in
+ inference_route.mcp_images_sentinel_in = boom
+ try:
+ assert inference_route._request_has_promotable_mcp_images(payload, exact = False)
+ finally:
+ inference_route.mcp_images_sentinel_in = original
+ assert inference_route._request_has_promotable_mcp_images(payload)
+
+
+def test_the_client_tool_rebuild_decodes_each_picture_once(monkeypatch):
+ """The route promotes the history for the server-tool path and again, from the
+ raw payload, for the client-tool rebuild. One request's decodes are shared, so a
+ retained raster is decoded once, not once per promotion. Two DIFFERENT pictures:
+ the cache is keyed by payload, and identical bytes would decode once anyway."""
+ decoded: list = []
+ original = mcp_images._png_data_url
+
+ def counting(data):
+ decoded.append(data[:16])
+ return original(data)
+
+ monkeypatch.setattr(mcp_images, "_png_data_url", counting)
+ call = _client_tool_route(
+ monkeypatch,
+ [
+ {"role": "user", "content": "read a.png"},
+ *_shot_round(0, _image(_png((8, 8)))),
+ {"role": "assistant", "content": "a is blue"},
+ {"role": "user", "content": "now b.png"},
+ *_shot_round(1, _image(_png((9, 9)))),
+ {"role": "assistant", "content": "b is red"},
+ {"role": "user", "content": "compare them"},
+ ],
+ tools = [_LOOKUP],
+ )
+ assert len(call["images"]) == 2
+ assert len(decoded) == 2, decoded
+
+
+def test_the_local_loops_detach_the_note_for_a_multi_result_batch():
+ """Same rule as the external loop: a batch of several results gets the wording
+ that claims no adjacency, since "the tool call above" names whichever ran last."""
+ import inspect
+
+ from core.inference import llama_cpp, safetensors_agentic
+
+ rule = (
+ "lead = MCP_DETACHED_IMAGE_TURN_TEXT\n"
+ " if _batch_results != 1\n"
+ " else MCP_IMAGE_TURN_TEXT,"
+ )
+ assert rule in inspect.getsource(llama_cpp)
+ assert rule.replace(
+ "\n ", "\n "
+ ) in inspect.getsource(safetensors_agentic)
+ for module in (llama_cpp, safetensors_agentic):
+ body = inspect.getsource(module)
+ assert "batch_conversation_start = len(conversation)" in body
+ assert "for m in conversation[batch_conversation_start:]" in body
+
+
+def test_the_caller_reservation_is_taken_before_any_replay_decode(monkeypatch):
+ """Eight attachments leave a provider no room for replay; the eligibility budget
+ says so before a single candidate is decoded, instead of decoding eight rasters
+ for the trim at the bottom to drop them all."""
+ decoded: list = []
+ original = mcp_images._png_data_url
+
+ def counting(data):
+ decoded.append(1)
+ return original(data)
+
+ monkeypatch.setattr(mcp_images, "_png_data_url", counting)
+ attachments = [
+ {"type": "image_url", "image_url": {"url": f"data:image/png;base64,{_png()}"}}
+ for _ in range(mcp_images.MAX_TOTAL_MODEL_IMAGES)
+ ]
+ history = [
+ {"role": "user", "content": [{"type": "text", "text": "these"}, *attachments]},
+ {"role": "tool", "name": "mcp__s__shot", "content": _envelope("[1]", _image())},
+ {"role": "tool", "name": "mcp__s__shot", "content": _envelope("[1]", _image())},
+ {"role": "user", "content": "and the tool's?"},
+ ]
+ out = promote_history(history, vision = True, reserve_for_caller = True)
+ assert decoded == [], "nothing to promote, so nothing to decode"
+ assert len(mcp_images._all_image_url_parts(out)) == mcp_images.MAX_TOTAL_MODEL_IMAGES
+
+ out = promote_history(history, vision = True)
+ assert len(decoded) == 2, "a GGUF caller keeps the full allowance and decodes both"
+ assert len(mcp_images._all_image_url_parts(out)) == mcp_images.MAX_TOTAL_MODEL_IMAGES + 2
+
+
+def test_every_async_promotion_wrapper_hops_on_the_marker_alone():
+ """Stripping an envelope for a text-only target parses the same 12 MB array as
+ promoting it, so the hop off the loop cannot be gated on vision."""
+ import inspect
+
+ import routes.inference as inference_route
+
+ for wrapper in (
+ inference_route._promote_mcp_history_images_async,
+ inference_route._promote_local_mcp_images_async,
+ inference_route._build_external_messages_async,
+ ):
+ body = inspect.getsource(wrapper)
+ assert "if _messages_mention_mcp_images(messages):" in body, wrapper.__name__
+ assert "vision and _messages_mention" not in body, wrapper.__name__
+ assert "promote and _messages_mention" not in body, wrapper.__name__
+ count = inspect.getsource(inference_route.anthropic_count_tokens)
+ assert "await asyncio.to_thread(_messages_have_promotable_mcp_images, openai_messages)" in count
+
+
+def test_live_image_extraction_is_gated_on_the_target_reading_them():
+ """mcp_images() json-loads the whole envelope; a text-only target discards the
+ result, and the external loop runs on the event loop, so it parses in a thread."""
+ import inspect
+
+ import routes.inference as inference_route
+ from core.inference import llama_cpp, safetensors_agentic, studio_tool_loop
+
+ assert (
+ "await asyncio.to_thread(completion.mcp_images) if run.supports_vision else []"
+ in inspect.getsource(studio_tool_loop)
+ )
+ assert (
+ 'if mcp_images_mentioned_in(completion.result or "") and self.is_vision'
+ in inspect.getsource(llama_cpp)
+ )
+ assert "completion.mcp_images() if images_sink is not None else []" in inspect.getsource(
+ safetensors_agentic
+ )
+ count = inspect.getsource(inference_route.chat_count_tokens)
+ assert "openai_messages = await _promote_mcp_history_images_async(" in count
+ assert "openai_messages = promote_mcp_history_images(" not in count
+
+
+def test_an_entry_with_an_unbounded_mime_type_is_not_an_image():
+ """A token subtype has no length bound, and metadata is not where megabytes may
+ hide from the byte budgets. Mirrors the frontend's MAX_MCP_IMAGE_MIME_CHARS."""
+ long_mime = "image/" + "x" * mcp_images.MAX_MCP_IMAGE_MIME_CHARS
+ text, images = split_images(_envelope("[2]", _image(mime = long_mime), _image()))
+ # The envelope still splits -- or its megabytes would stay in the prompt as text.
+ assert len(images) == 2
+ assert mcp_images.count_probably_decodable(images) == 1
+ assert not mcp_images.probably_decodable(_image(mime = long_mime))
+ assert len(mcp_images.png_payloads(images)) == 1, "not decoded either"
+
+
+def test_the_monitor_prompt_never_parses_the_envelope(monkeypatch):
+ """The monitor row is built on the event loop from the raw messages; cutting at
+ the marker shows the same text as the parsed strip without json-loading 12 MB."""
+ import routes.inference as inference_route
+
+ content = _envelope("what the tool said", _image())
+ assert mcp_images.text_before_envelope(content) == split_images(content)[0]
+
+ def boom(_content):
+ raise AssertionError("parsed on the loop")
+
+ monkeypatch.setattr(inference_route, "split_mcp_images", boom)
+ prompt = inference_route._monitor_prompt_from_messages(
+ [{"role": "tool", "content": content}, {"role": "user", "content": "and?"}]
+ )
+ assert prompt == "tool: what the tool said\n\nuser: and?"
+
+
+def test_the_gguf_admission_estimate_parses_off_the_loop():
+ """reserve() binds a waiter to the running loop, so only the ESTIMATE can move: it
+ strips every replayed envelope. Every route reservation goes through the async
+ form, which hops on the marker."""
+ import inspect
+
+ import routes.inference as inference_route
+
+ routes_src = inspect.getsource(inference_route)
+ assert "reservation, admission_config = _openai_llama_admission_reserve(" not in routes_src
+ assert routes_src.count("await _openai_llama_admission_reserve_async(") >= 7
+ wrapper = inspect.getsource(inference_route._openai_llama_admission_reserve_async)
+ assert "await asyncio.to_thread(\n _openai_llama_admission_estimate," in wrapper
+ assert "_messages_mention_mcp_images(" in wrapper
+
+
+def test_a_sixteen_bit_picture_is_scaled_not_clipped():
+ """convert("RGB") on I;16 clips 0..65535 to 8 bits, so ordinary 16-bit imagery came
+ out nearly white while the tool card showed the real picture."""
+ from PIL import Image
+
+ source = Image.new("I;16", (4, 4))
+ source.putdata([30_000] * 16)
+ flat = mcp_images.flattened_rgb(source)
+ assert flat.mode == "RGB"
+ value = flat.getpixel((0, 0))[0]
+ assert 110 <= value <= 122, value
+
+ buffer = io.BytesIO()
+ source.save(buffer, format = "PNG")
+ payload = base64.b64encode(buffer.getvalue()).decode("ascii")
+ url = mcp_images._png_data_url(payload)
+ assert url
+ decoded = Image.open(io.BytesIO(base64.b64decode(url.split(",", 1)[1])))
+ assert 110 <= decoded.getpixel((0, 0))[0] <= 122
+
+
+def test_gguf_admission_charges_replay_beside_the_attachments():
+ """The GGUF paths keep the full replay allowance beside the caller's pictures, so one
+ attachment plus eight replayed pictures sends nine and must be charged nine."""
+ import routes.inference as inference_route
+
+ messages = [
+ {
+ "role": "user",
+ "content": [
+ {"type": "text", "text": "this"},
+ {"type": "image_url", "image_url": {"url": f"data:image/png;base64,{_png()}"}},
+ ],
+ },
+ ]
+ for r in range(9):
+ messages.append(
+ {
+ "role": "tool",
+ "name": "mcp__s__shot",
+ "tool_call_id": f"c{r}",
+ "content": _envelope("[1]", _image()),
+ }
+ )
+ _estimate, image_parts = inference_route._openai_llama_admission_messages_for_estimate(
+ messages, vision = True
+ )
+ assert image_parts == 1 + mcp_images.MAX_TOTAL_MODEL_IMAGES
+
+
+def test_the_anthropic_server_tool_gate_reads_caller_attachments_only():
+ """A promoted replay sets _has_image too; gating on it routed a follow-up away from
+ the server tools it selected because an earlier tool had returned a picture."""
+ import inspect
+
+ import routes.inference as inference_route
+
+ body = inspect.getsource(inference_route.anthropic_messages)
+ assert "and not _anthropic_has_image\n" in body
+ assert "supports_tools and not _has_image" not in body
+
+
+def test_the_safetensors_loop_stamps_a_round_id_on_every_decision():
+ """Replay groups consecutive results as one batch; the frontend keeps them
+ consecutive only when the cards share a round_id."""
+ import inspect
+
+ from core.inference import safetensors_agentic
+
+ body = inspect.getsource(safetensors_agentic)
+ assert 'decision.provenance["round_id"] = iteration' in body
+
+
+def test_a_merge_into_a_trailing_nudge_still_says_where_the_pictures_came_from():
+ """A deferred no-op nudge is a trailing user turn the batch's pictures merge into
+ (two user turns in a row is what a strict template rejects). Merged bare, they read
+ as attachments to the nudge; the note names the tool they came from."""
+ nudge = {"role": "user", "content": "Tool `lookup` is not available; continue."}
+
+ conversation = [dict(nudge)]
+ mcp_images.append_placeholder_turn(conversation, 1, 1)
+ [merged] = conversation
+ kinds = [part["type"] for part in merged["content"]]
+ assert kinds == ["text", "image", "text"], kinds
+ assert merged["content"][-1]["text"].startswith(mcp_images.IMAGE_TURN_TEXT)
+
+ conversation = [dict(nudge)]
+ mcp_images.append_image_turn(
+ conversation, [[_image()]], per_result = True, lead = mcp_images.DETACHED_IMAGE_TURN_TEXT
+ )
+ [merged] = conversation
+ kinds = [part["type"] for part in merged["content"]]
+ assert kinds == ["text", "image_url", "text"], kinds
+ assert merged["content"][-1]["text"].startswith(mcp_images.DETACHED_IMAGE_TURN_TEXT)
+
+
+def test_a_replay_merged_into_the_question_still_says_where_the_pictures_came_from():
+ """Promotion merges a batch into the user turn that follows it (two user turns in a
+ row is what a strict template rejects). Merged bare, the pictures read as ones the
+ user attached; the note names the tool, detached wording when another tool's result
+ sat between. And the question stays the user's turn for the attachment's ordinal."""
+ history = [
+ {"role": "tool", "name": "mcp__s__shot", "content": _envelope("[1]", _image())},
+ {"role": "tool", "name": "web_search", "content": "three results"},
+ {"role": "user", "content": "what do you see?"},
+ ]
+ out, payloads = mcp_images.promote_history_local(history, vision = True)
+ assert len(payloads) == 1
+ question = out[-1]
+ kinds = [part["type"] for part in question["content"]]
+ assert kinds == ["image", "text", "text"], kinds
+ assert question["content"][1]["text"].startswith(mcp_images.DETACHED_IMAGE_TURN_TEXT)
+ assert question["content"][2]["text"] == "what do you see?"
+ assert not mcp_images.is_synthetic_image_turn(question)
+
+ out = promote_history(history, vision = True)
+ question = out[-1]
+ kinds = [part["type"] for part in question["content"]]
+ assert kinds == ["image_url", "text", "text"], kinds
+ assert question["content"][1]["text"].startswith(mcp_images.DETACHED_IMAGE_TURN_TEXT)
+ assert not mcp_images.is_synthetic_image_turn(question)
+ assert mcp_images.is_synthetic_image_turn(mcp_images.placeholder_turn(1, 1))
+
+
+def test_a_camera_jpeg_is_shown_the_way_up_its_exif_says():
+ """The coded raster of a portrait photo is landscape with orientation 6 in EXIF;
+ re-encoding it as PNG without the transpose showed the model a sideways picture."""
+ from PIL import Image
+
+ source = Image.new("RGB", (8, 4), (10, 120, 200))
+ exif = Image.Exif()
+ exif[0x0112] = 6
+ buffer = io.BytesIO()
+ source.save(buffer, format = "JPEG", exif = exif.tobytes())
+ url = mcp_images._png_data_url(base64.b64encode(buffer.getvalue()).decode("ascii"))
+ assert url
+ decoded = Image.open(io.BytesIO(base64.b64decode(url.split(",", 1)[1])))
+ assert decoded.size == (4, 8), decoded.size
+
+
+def test_the_content_part_extractor_keeps_marker_text_and_hops_the_split(monkeypatch):
+ """An MCP tool's own text can contain the marker line with no envelope behind it, so
+ the strip is the exact split for every result -- and the awaited callers take the
+ async form, which runs that split in a worker whenever the marker is present."""
+ import asyncio
+ import inspect
+
+ import routes.inference as inference_route
+ from models.inference import ChatMessage
+
+ docs = "the sentinel is\n" + mcp_images.SENTINEL + " followed by a JSON array"
+ for name in ("mcp__fs__shot", "read_file"):
+ _system, chat_messages, _image_b64 = inference_route._extract_content_parts(
+ [
+ ChatMessage(role = "user", content = "look"),
+ ChatMessage(role = "tool", tool_call_id = "c1", name = name, content = docs),
+ ]
+ )
+ tool = next(m for m in chat_messages if m.get("role") == "tool")
+ assert tool["content"] == docs, name
+
+ envelope = _envelope("what the tool said", _image())
+ hops: list = []
+ original = inference_route.asyncio.to_thread
+
+ async def counting(fn, *args, **kwargs):
+ hops.append(fn.__name__)
+ return await original(fn, *args, **kwargs)
+
+ monkeypatch.setattr(inference_route.asyncio, "to_thread", counting)
+ _system, chat_messages, _image_b64 = asyncio.run(
+ inference_route._extract_content_parts_async(
+ [
+ ChatMessage(role = "user", content = "look"),
+ ChatMessage(role = "tool", tool_call_id = "c0", name = "mcp__fs__shot", content = envelope),
+ ]
+ )
+ )
+ assert hops == ["_extract_content_parts"], hops
+ tool = next(m for m in chat_messages if m.get("role") == "tool")
+ assert tool["content"] == "what the tool said"
+
+ src = inspect.getsource(inference_route)
+ for fn in ("generate_audio", "_mlx_count_chat_tokens", "chat_count_tokens"):
+ body = inspect.getsource(getattr(inference_route, fn))
+ assert "await _extract_content_parts_async(" in body, fn
+ assert "= _extract_content_parts(" not in body, fn
+ chat = inspect.getsource(inference_route.produce_openai_chat_completions)
+ # only the async extraction strips images; all three synchronous paths preserve them.
+ assert chat.count("await _extract_content_parts_async(") == 1
+ assert chat.count("keep_tool_images = True") == 3
+
+
+def test_the_note_counts_only_the_tools_pictures_in_a_mixed_turn():
+ """Five caller attachments beside a four-picture result trimmed to three: the note
+ must read (3 of 4), not (8 of 9) -- the attachments are not the tool's."""
+ attachments = [
+ {"type": "image_url", "image_url": {"url": f"data:image/png;base64,A{i}"}} for i in range(5)
+ ]
+ promoted = [
+ {"type": "image_url", "image_url": {"url": f"data:image/png;base64,T{i}"}} for i in range(4)
+ ]
+ turn = {
+ "role": "user",
+ "content": [
+ *attachments,
+ *promoted,
+ {"type": "text", "text": mcp_images.IMAGE_TURN_TEXT},
+ {"type": "text", "text": "what do you see?"},
+ ],
+ }
+ conversation = [turn]
+ mcp_images._drop_oldest_image_parts(conversation, 1, "image_url", only = list(promoted))
+ parts = conversation[0]["content"]
+ assert sum(1 for p in parts if p.get("type") == "image_url") == 8
+ note = next(
+ p["text"]
+ for p in parts
+ if p.get("type") == "text" and p["text"].startswith(mcp_images.IMAGE_TURN_TEXT)
+ )
+ assert note == f"{mcp_images.IMAGE_TURN_TEXT} (3 of 4)", note
+ assert all(p in parts for p in attachments), "the attachments were not the ones dropped"
+
+
+def _first_note(conversation: list) -> str:
+ return next(
+ part["text"]
+ for message in conversation
+ if isinstance(message.get("content"), list)
+ for part in message["content"]
+ if part.get("type") == "text" and part["text"].startswith("Images returned by")
+ )
+
+
+def test_a_replay_batch_of_two_picture_results_is_detached():
+ """Two parallel calls both returned pictures and share the turn; "the tool call
+ above" would hand every picture to whichever ran last."""
+ history = [
+ {"role": "tool", "name": "mcp__s__a", "content": _envelope("[1]", _image())},
+ {"role": "tool", "name": "mcp__s__b", "content": _envelope("[1]", _image())},
+ {"role": "assistant", "content": "two shots"},
+ ]
+ assert _first_note(promote_history(history, vision = True)).startswith(
+ mcp_images.DETACHED_IMAGE_TURN_TEXT
+ )
+ out, _payloads = mcp_images.promote_history_local(history, vision = True)
+ assert _first_note(out).startswith(mcp_images.DETACHED_IMAGE_TURN_TEXT)
+
+
+ "mark, count", [(mcp_images.top_up_image_markers, 2), (mcp_images.mark_last_user_turn, 1)]
+)
+def test_a_displaced_replay_marker_takes_its_note_with_it(mark, count):
+ """The replay merged its marker and note into the question that also owns the
+ attachment; the attachment displaces the marker, and the note must go too, or it
+ says the caller's own picture was returned by the tool."""
+ question = {
+ "role": "user",
+ "content": [
+ {"type": "image"},
+ {"type": "text", "text": mcp_images.IMAGE_TURN_TEXT},
+ {"type": "text", "text": "and this one?"},
+ ],
+ }
+ for ordinal in (1, None):
+ conversation = [{"role": "user", "content": "read a.png"}, dict(question)]
+ prior = mcp_images.image_marker_parts(conversation)
+ topped = mark(conversation, count, ordinal = ordinal)
+ parts = topped[1]["content"]
+ assert sum(1 for p in parts if p.get("type") == "image") == 1, parts
+ texts = [p["text"] for p in parts if p.get("type") == "text"]
+ assert texts == ["and this one?"], texts
+ assert mcp_images.pixels_in_marker_order(topped, prior, ["REPLAY"], "ATTACHMENT") == [
+ "ATTACHMENT"
+ ]
+
+
+def test_the_note_counts_a_result_the_allowance_left_nothing_of():
+ """Three results of three in one batch on the marker path: one picture is shown and
+ the note must say (1 of 9), not (1 of 6) -- the result with no admitted candidate
+ is still part of what the batch returned."""
+ history = [
+ {
+ "role": "tool",
+ "name": "mcp__s__a",
+ "content": _envelope("[3]", *[_image() for _ in range(3)]),
+ },
+ {
+ "role": "tool",
+ "name": "mcp__s__b",
+ "content": _envelope("[3]", *[_image() for _ in range(3)]),
+ },
+ {
+ "role": "tool",
+ "name": "mcp__s__c",
+ "content": _envelope("[3]", *[_image() for _ in range(3)]),
+ },
+ {"role": "assistant", "content": "three shots"},
+ ]
+ eligible = mcp_images.eligible_replay_images(history, local = True)
+ assert min(eligible.values()) == 0, eligible
+ out, payloads = mcp_images.promote_history_local(history, vision = True)
+ assert len(payloads) == 1
+ assert _first_note(out) == f"{mcp_images.DETACHED_IMAGE_TURN_TEXT} (1 of 9)"
+
+
+def test_local_replay_cap_keeps_caller_pixels_with_their_markers():
+ history = [
+ {"role": "user", "content": [{"type": "image"}, {"type": "text", "text": "first"}]},
+ {"role": "assistant", "content": "noted"},
+ ]
+ for index in range(MAX_MODEL_IMAGES * 3):
+ history.extend(
+ [
+ {
+ "role": "tool",
+ "name": "mcp__test__image",
+ "content": _envelope(str(index), _image()),
+ },
+ {"role": "assistant", "content": "noted"},
+ ]
+ )
+ history.append(
+ {"role": "user", "content": [{"type": "image"}, {"type": "text", "text": "last"}]}
+ )
+ first, last = object(), object()
+
+ messages, pixels = mcp_images.promote_history_local(
+ history, vision = True, caller_images = [first, last]
+ )
+
+ assert len(pixels) == mcp_images.MAX_TOTAL_MODEL_IMAGES
+ assert pixels[0] is first
+ assert pixels[-1] is last
+ assert len(mcp_images.image_marker_parts(messages)) == len(pixels)
+ assert messages[0] is history[0]
+ assert messages[-1] is history[-1]
+
+
+def test_local_replay_does_not_add_a_second_image_to_an_attachment_turn():
+ attachment = object()
+ user = {"role": "user", "content": [{"type": "image"}, {"type": "text", "text": "compare"}]}
+ history = [
+ {"role": "tool", "name": "mcp__test__image", "content": _envelope("shot", _image())},
+ user,
+ ]
+
+ messages, pixels = mcp_images.promote_history_local(
+ history, vision = True, caller_images = [attachment]
+ )
+
+ assert pixels == [attachment]
+ assert messages[-1] is user
+ assert len(mcp_images.image_marker_parts(messages)) == 1
diff --git a/studio/backend/tests/test_mlx_inference_backend.py b/studio/backend/tests/test_mlx_inference_backend.py
index abf19393b71..b15a495956c 100644
--- a/studio/backend/tests/test_mlx_inference_backend.py
+++ b/studio/backend/tests/test_mlx_inference_backend.py
@@ -1014,7 +1014,7 @@ def test_mlx_vlm_generation_selects_renderer_by_capability(monkeypatch):
assert calls["stream"][-1][0][2] == "<image> healthy generic"
state["generic"] = "generic prompt"
text_messages = [{"role": "user", "content": "hello"}]
- assert list(backend._generate_vlm(*((text_messages, None) + args[2:]), tools = tools)) == ["ok"]
+ assert list(backend._generate_vlm(*((text_messages, []) + args[2:]), tools = tools)) == ["ok"]
assert calls["generic"][-1]["tools"] == tools
assert calls["stream"][-1][0][2] == "generic prompt"
two_images = [{"role": "user", "content": [{"type": "image"}, {"type": "image"}]}]
@@ -5196,6 +5196,195 @@ def test_vlm_add_special_tokens_falls_back_to_the_inline_rule(monkeypatch):
assert rule("gemma4", template) == "mlx-vlm's answer"
+def test_mlx_vlm_pixels_follow_the_order_the_markers_appear_in():
+ """MLX binds pixels to markers positionally. The replayed MCP pictures carry
+ their markers on earlier turns, and the attachment's marker goes on the last
+ user turn, so the attachment's pixels have to come last."""
+ from core.inference.mlx_inference import MLXInferenceBackend
+
+ backend = MLXInferenceBackend()
+ backend._model = object()
+ backend._is_vlm = True
+ backend._multi_image_marker = "<image>"
+ captured = []
+ backend._generate_vlm = lambda messages, attached, *_args, **_kwargs: (
+ captured.append((messages, attached)) or iter(())
+ )
+ replayed_first, replayed_second, attachment = object(), object(), object()
+ messages = [
+ {"role": "user", "content": [{"type": "image"}, {"type": "text", "text": "a"}]},
+ {"role": "assistant", "content": "seen"},
+ {"role": "user", "content": [{"type": "image"}, {"type": "text", "text": "b"}]},
+ {"role": "assistant", "content": "seen"},
+ {"role": "user", "content": "and this one?"},
+ ]
+
+ list(
+ backend.generate_chat_response(
+ messages,
+ image = attachment,
+ images = [replayed_first, replayed_second],
+ )
+ )
+
+ sent_messages, attached = captured[0]
+ assert attached == [replayed_first, replayed_second, attachment]
+ # The attachment's own marker is the last one in the conversation.
+ marker_turns = [
+ index
+ for index, message in enumerate(sent_messages)
+ if isinstance(message.get("content"), list)
+ and any(part.get("type") == "image" for part in message["content"])
+ ]
+ assert marker_turns[-1] == len(sent_messages) - 1
+
+
+def test_mlx_attachment_displaces_a_replayed_image_on_the_same_turn():
+ """A local turn takes one picture, so the attachment replaces the replay
+ marker and its pixels together."""
+ from core.inference.mlx_inference import MLXInferenceBackend, _count_vlm_images
+
+ backend = MLXInferenceBackend()
+ backend._model = object()
+ backend._is_vlm = True
+ backend._multi_image_marker = "<image>"
+ captured = []
+ backend._generate_vlm = lambda messages, attached, *_a, **_k: (
+ captured.append((messages, attached)) or iter(())
+ )
+ replayed, attachment = object(), object()
+ messages = [
+ {
+ "role": "user",
+ "content": [{"type": "image"}, {"type": "text", "text": "and this one?"}],
+ }
+ ]
+
+ list(backend.generate_chat_response(messages, image = attachment, images = [replayed]))
+
+ sent, attached = captured[0]
+ markers = sum(_count_vlm_images(m.get("content")) for m in sent)
+ assert attached == [attachment]
+ assert markers == len(attached), f"{markers} marker(s) for {len(attached)} images"
+ # The attachment's pixels are last, so its marker has to be too.
+ assert sent[-1]["content"][-1] == {"type": "image"}
+
+
+def test_mlx_does_not_double_mark_an_attachment_the_client_already_marked():
+ from core.inference.mlx_inference import MLXInferenceBackend, _count_vlm_images
+
+ backend = MLXInferenceBackend()
+ backend._model = object()
+ backend._is_vlm = True
+ backend._multi_image_marker = "<image>"
+ captured = []
+ backend._generate_vlm = lambda messages, attached, *_a, **_k: (
+ captured.append((messages, attached)) or iter(())
+ )
+ messages = [{"role": "user", "content": [{"type": "image"}, {"type": "text", "text": "x"}]}]
+
+ list(backend.generate_chat_response(messages, image = object()))
+
+ sent, attached = captured[0]
+ assert sum(_count_vlm_images(m.get("content")) for m in sent) == len(attached) == 1
+
+
+def test_mlx_normalizes_a_replay_only_conversation_for_a_processor_template():
+ """A processor template needs every turn in part shape. A replay-only request
+ (images without an attachment) is a mix of strings and marker lists, and gating
+ the normalization on the singular image left it failing at render."""
+ from types import SimpleNamespace
+
+ from core.inference.mlx_inference import MLXInferenceBackend, _count_vlm_images
+
+ backend = MLXInferenceBackend()
+ backend._model = object()
+ backend._is_vlm = True
+ backend._multi_image_marker = "<image>"
+ # A processor carrying its own template is what chat_render_target selects.
+ backend._processor = SimpleNamespace(
+ apply_chat_template = lambda *a, **k: "prompt", chat_template = "t"
+ )
+ captured = []
+ backend._generate_vlm = lambda messages, attached, *a, **k: (
+ captured.append((messages, attached)) or iter(())
+ )
+ messages = [
+ {"role": "tool", "content": "[1 image returned]"},
+ {"role": "user", "content": [{"type": "image"}, {"type": "text", "text": "what was it"}]},
+ ]
+
+ list(backend.generate_chat_response(messages, images = [object()]))
+
+ sent, attached = captured[0]
+ assert all(isinstance(message.get("content"), list) for message in sent), sent
+ markers = sum(_count_vlm_images(message.get("content")) for message in sent)
+ assert markers == len(attached) == 1
+
+
+def test_mlx_binds_an_earlier_attachment_to_its_own_turn_not_a_newer_replay():
+ """The attachment's turn can PRECEDE a tool's picture. The route used to pre-add
+ its marker, so the backend counted that marker as history's, the top-up became a
+ no-op, and the two pixels bound to each other's turns -- the model was shown the
+ screenshot where the user's own diagram belonged."""
+ from core.inference.mcp_images import placeholder_turn
+ from core.inference.mlx_inference import MLXInferenceBackend
+
+ backend = MLXInferenceBackend()
+ backend._model = object()
+ backend._is_vlm = True
+ backend._multi_image_marker = "<image>"
+ captured = []
+ backend._generate_vlm = lambda messages, attached, *_args, **_kwargs: (
+ captured.append((messages, attached)) or iter(())
+ )
+ attachment, replayed = object(), object()
+
+ # What the plain route hands the backend once the fix stops it pre-marking: the
+ # attachment's turn is still a plain string, and the only marker in the
+ # conversation is the replayed picture's.
+ messages = [
+ {"role": "user", "content": "here is my diagram"},
+ {"role": "assistant", "content": "noted"},
+ {
+ "role": "assistant",
+ "content": "",
+ "tool_calls": [
+ {
+ "id": "c1",
+ "type": "function",
+ "function": {"name": "mcp__s__shot", "arguments": "{}"},
+ }
+ ],
+ },
+ {"role": "tool", "tool_call_id": "c1", "content": "[1 image returned]"},
+ placeholder_turn(1, 1),
+ {"role": "user", "content": "which one is bluer?"},
+ ]
+
+ list(
+ backend.generate_chat_response(
+ messages,
+ image = attachment,
+ images = [replayed],
+ image_ordinal = 0,
+ )
+ )
+
+ sent_messages, attached = captured[0]
+ marker_turns = [
+ index
+ for index, message in enumerate(sent_messages)
+ if isinstance(message.get("content"), list)
+ and any(part.get("type") == "image" for part in message["content"])
+ ]
+ assert len(marker_turns) == 2, sent_messages
+ # The attachment's marker is on the FIRST turn, the replay's on the placeholder
+ # after it -- so the pixels have to arrive in that same order.
+ assert marker_turns[0] < marker_turns[1]
+ assert attached == [attachment, replayed], "the pixels bound to each other's markers"
+
+
def _run_mlx_reasoning_stream(
monkeypatch,
turn,
@@ -5560,6 +5749,59 @@ def test_mlx_unset_budget_falls_back_when_the_prompt_cannot_be_counted(monkeypat
assert backend._unset_generation_budget("P") == UNSET_GENERATION_BUDGET
+def test_mlx_count_strips_replayed_mcp_images_off_the_event_loop(monkeypatch):
+ import base64
+ import io
+ import json
+ import threading
+ from PIL import Image
+ from core.inference import mcp_images
+ from routes import inference as route
+
+ buffer = io.BytesIO()
+ Image.new("RGB", (4, 4), "blue").save(buffer, format = "PNG")
+ envelope = (
+ "screenshot result\n"
+ + mcp_images.SENTINEL
+ + json.dumps(
+ [{"data": base64.b64encode(buffer.getvalue()).decode(), "mimeType": "image/png"}]
+ )
+ )
+ history = [
+ {"role": "user", "content": "Read the screenshot"},
+ {
+ "role": "assistant",
+ "content": "",
+ "tool_calls": [
+ {
+ "id": "c1",
+ "type": "function",
+ "function": {"name": "mcp__screen__shot", "arguments": "{}"},
+ }
+ ],
+ },
+ {"role": "tool", "tool_call_id": "c1", "name": "mcp__screen__shot", "content": envelope},
+ {"role": "user", "content": "Describe it"},
+ ]
+ loop_thread = threading.get_ident()
+ split_threads = []
+ original_split = mcp_images.split_images
+
+ def track_split(content):
+ if mcp_images.SENTINEL in content:
+ split_threads.append(threading.get_ident())
+ return original_split(content)
+
+ monkeypatch.setattr(mcp_images, "split_images", track_split)
+ monkeypatch.setattr(route, "split_mcp_images", track_split)
+ backend = _RenderRecordingBackend()
+ response = _count_route(monkeypatch, backend, messages = history, enable_tools = False)
+ assert response.status_code == 200
+ assert split_threads and all(thread != loop_thread for thread in split_threads)
+ assert backend.messages[1]["tool_calls"][0]["function"]["name"] == "mcp__screen__shot"
+ assert backend.messages[2]["content"] == "screenshot result"
+
+
_CLIP_B64 = "AAAAGGZ0eXBtcDQy" # a bare mp4 box header, decoded byte-for-byte by the backend
@@ -6182,10 +6424,9 @@ def test_several_images_reach_the_runtime_in_the_order_they_were_attached(monkey
@pytest.mark.parametrize(
"marker, render, error",
[
- (None, None, "one image per request"),
("<|image|>", "<|image|>only one", "marked 1 image"),
],
- ids = ["no marker was classified", "the render collapsed a marker"],
+ ids = ["the render collapsed a marker"],
)
def test_a_render_that_cannot_bind_every_image_is_refused(monkeypatch, marker, render, error):
"""One image is still taken either way; too few markers would describe one picture twice."""
@@ -6196,3 +6437,78 @@ def test_a_render_that_cannot_bind_every_image_is_refused(monkeypatch, marker, r
turn = [{"role": "user", "content": "compare"}]
with pytest.raises((ValueError, RuntimeError), match = error):
list(backend.generate_chat_response(turn, images = [object(), object()]))
+
+
+def test_a_single_image_template_is_served_one_picture_not_refused(monkeypatch):
+ """A template that marks one image cannot bind several, but a resumed chat carrying
+ replayed MCP pictures must still be answered: the caller's attachment, or else the
+ newest picture, is served with its own marker and the others are left out."""
+ backend, calls = _vlm_runtime(monkeypatch, marker = None)
+ older, newer = object(), object()
+ turn = [{"role": "user", "content": "compare"}]
+
+ assert list(backend.generate_chat_response(turn, images = [older, newer])) == ["ok"]
+ assert calls[-1]["images"] == [newer]
+ assert calls[-1]["prompt"].count("<|image|>") == 1
+
+ attachment = object()
+ list(backend.generate_chat_response(turn, image = attachment, images = [older]))
+ assert calls[-1]["images"] == [attachment]
+ assert calls[-1]["prompt"].count("<|image|>") == 1
+
+
[email protected]("merged_question", [False, True])
+def test_single_image_replay_notes_describe_only_retained_pixels(monkeypatch, merged_question):
+ from core.inference.mcp_images import IMAGE_TURN_TEXT, image_marker_parts, placeholder_turn
+
+ backend, calls = _vlm_runtime(monkeypatch, marker = None)
+ older, newer = object(), object()
+ previous_image = placeholder_turn(1)
+ if merged_question:
+ previous_image["content"].append({"type": "text", "text": "Keep this user question."})
+ history = [
+ {"role": "user", "content": "Inspect the first screenshot."},
+ {
+ "role": "assistant",
+ "content": "",
+ "tool_calls": [
+ {
+ "id": "first",
+ "type": "function",
+ "function": {
+ "name": "mcp__test__screenshot",
+ "arguments": "{}",
+ },
+ }
+ ],
+ },
+ {"role": "tool", "tool_call_id": "first", "content": "[1 image returned]"},
+ previous_image,
+ {"role": "assistant", "content": "The first screenshot was inspected."},
+ {"role": "user", "content": "Inspect the next screenshot."},
+ {
+ "role": "assistant",
+ "content": "",
+ "tool_calls": [
+ {
+ "id": "second",
+ "type": "function",
+ "function": {
+ "name": "mcp__test__screenshot",
+ "arguments": "{}",
+ },
+ }
+ ],
+ },
+ {"role": "tool", "tool_call_id": "second", "content": "[1 image returned]"},
+ placeholder_turn(1),
+ ]
+ assert list(backend.generate_chat_response(history, images = [older, newer])) == ["ok"]
+ assert calls[-1]["images"] == [newer]
+ assert calls[-1]["prompt"].count("<|image|>") == 1
+ if merged_question:
+ assert "Keep this user question." in calls[-1]["prompt"]
+ assert "(0 of 1)" in calls[-1]["prompt"]
+ else:
+ assert calls[-1]["prompt"].count(IMAGE_TURN_TEXT) == 1
+ assert len(image_marker_parts(history)) == 2
diff --git a/studio/backend/tests/test_openai_tool_passthrough.py b/studio/backend/tests/test_openai_tool_passthrough.py
index 9b209e56a27..0e5b4bdf8dd 100644
--- a/studio/backend/tests/test_openai_tool_passthrough.py
+++ b/studio/backend/tests/test_openai_tool_passthrough.py
@@ -4298,6 +4298,259 @@ class TestGgufVisionMessages:
{"role": "system", "content": "Use tools."}
]
+ def _mcp_tool_history(self, text = "[1 image returned]"):
+ from core.inference import mcp_images
+ envelope = json.dumps([{"data": self._PNG_B64, "mimeType": "image/png"}])
+ return [
+ {"role": "user", "content": "what does the file look like"},
+ {
+ "role": "assistant",
+ "content": "",
+ "tool_calls": [
+ {
+ "id": "call_0",
+ "type": "function",
+ "function": {"name": "mcp__fs__read_media_file", "arguments": "{}"},
+ }
+ ],
+ },
+ {
+ "role": "tool",
+ "tool_call_id": "call_0",
+ "content": text + "\n" + mcp_images.SENTINEL + envelope,
+ },
+ {"role": "assistant", "content": "a blue square"},
+ {"role": "user", "content": "what colour was it again"},
+ ]
+
+ def test_replayed_mcp_images_become_an_image_turn_for_a_vision_model(self):
+ req = ChatCompletionRequest(model = "default", messages = self._mcp_tool_history())
+
+ messages, _ = _openai_messages_for_gguf_chat(req, is_vision = True)
+
+ tool_message = next(m for m in messages if m["role"] == "tool")
+ assert tool_message["content"] == "[1 image returned]"
+ promoted = messages[messages.index(tool_message) + 1]
+ assert promoted["role"] == "user"
+ assert promoted["content"][1]["image_url"]["url"].startswith("data:image/png;base64,")
+
+ def test_replayed_mcp_images_are_stripped_for_a_text_only_model(self):
+ req = ChatCompletionRequest(model = "default", messages = self._mcp_tool_history())
+
+ messages, _ = _openai_messages_for_gguf_chat(req, is_vision = False)
+
+ assert [m["role"] for m in messages] == ["user", "assistant", "tool", "assistant", "user"]
+ assert messages[2]["content"] == "[1 image returned]"
+
+ def test_passthrough_never_relays_the_image_envelope_as_tool_text(self):
+ req = ChatCompletionRequest(model = "default", messages = self._mcp_tool_history())
+
+ messages = _openai_messages_for_passthrough(req)
+
+ assert all("__MCP_IMAGES__" not in str(m.get("content")) for m in messages)
+
+ def test_passthrough_promotes_the_images_for_a_vision_model(self):
+ req = ChatCompletionRequest(model = "default", messages = self._mcp_tool_history())
+
+ messages = _openai_messages_for_passthrough(req, vision = True)
+
+ promoted = messages[3]
+ assert promoted["role"] == "user"
+ assert promoted["content"][1]["image_url"]["url"].startswith("data:image/png;base64,")
+
+ def test_the_native_path_never_templates_the_image_envelope(self):
+ req = ChatCompletionRequest(model = "default", messages = self._mcp_tool_history())
+
+ _, chat_messages, _ = _extract_content_parts(req.messages)
+
+ # The name is correlated from the call now, so the local path can run the
+ # provenance gate on a result that arrived unnamed.
+ assert chat_messages[2] == {
+ "role": "tool",
+ "content": "[1 image returned]",
+ "name": "mcp__fs__read_media_file",
+ }
+
+ def test_a_replayed_envelope_alone_does_not_demand_a_vision_model(self):
+ """The capability preflight runs before promote_history strips the envelope
+ for a text-only target, so counting history there strands the conversation:
+ the switch is refused for a picture that model would never be shown."""
+ from routes.inference import _request_has_attached_image, _request_has_image
+
+ req = ChatCompletionRequest(model = "default", messages = self._mcp_tool_history())
+
+ assert not _request_has_attached_image(req)
+ # Still an image for the paths that decode one: the work stays off the loop.
+ assert _request_has_image(req)
+
+ def test_an_attached_image_still_demands_a_vision_model(self):
+ from routes.inference import _request_has_attached_image
+
+ history = self._mcp_tool_history()
+ history[-1] = {
+ "role": "user",
+ "content": [
+ {"type": "text", "text": "and this one"},
+ {
+ "type": "image_url",
+ "image_url": {"url": f"data:image/png;base64,{self._PNG_B64}"},
+ },
+ ],
+ }
+ req = ChatCompletionRequest(model = "default", messages = history)
+
+ assert _request_has_attached_image(req)
+
+ def test_admission_charges_a_replayed_envelope_as_images_not_text(self):
+ """Admission prices the payload before the chat path promotes the envelope.
+ Left as text it reserves megabytes of prompt for bytes never sent; ignored
+ it charges zero KV for projector images that really are."""
+ from routes.inference import _openai_llama_admission_messages_for_estimate
+
+ estimate, image_parts = _openai_llama_admission_messages_for_estimate(
+ [
+ m.model_dump(exclude_none = True)
+ for m in ChatCompletionRequest(
+ model = "default", messages = self._mcp_tool_history()
+ ).messages
+ ]
+ )
+
+ assert image_parts == 1
+ assert self._PNG_B64 not in json.dumps(estimate)
+ assert estimate[2]["content"] == "[1 image returned]"
+
+ def test_admission_never_charges_past_the_conversation_cap(self):
+ from core.inference import mcp_images
+ from routes.inference import _openai_llama_admission_messages_for_estimate
+
+ envelope = json.dumps([{"data": self._PNG_B64, "mimeType": "image/png"}] * 4)
+ turns = []
+ for _ in range(6):
+ turns.append(
+ {
+ "role": "tool",
+ "tool_call_id": "c",
+ "content": "[4 images returned]\n" + mcp_images.SENTINEL + envelope,
+ }
+ )
+ _, image_parts = _openai_llama_admission_messages_for_estimate(turns)
+
+ assert image_parts == mcp_images.MAX_TOTAL_MODEL_IMAGES
+
+ def test_the_envelope_cap_never_discounts_really_attached_images(self):
+ """#9842 charges per-image KV so parallel vision chats are admitted
+ honestly. The envelope cap bounds what promotion adds, not what the
+ caller actually sent."""
+ from core.inference import mcp_images
+ from routes.inference import _openai_llama_admission_messages_for_estimate
+
+ attached = {
+ "role": "user",
+ "content": [{"type": "text", "text": "compare these"}]
+ + [
+ {
+ "type": "image_url",
+ "image_url": {"url": f"data:image/png;base64,{self._PNG_B64}"},
+ }
+ for _ in range(mcp_images.MAX_TOTAL_MODEL_IMAGES + 4)
+ ],
+ }
+
+ _, image_parts = _openai_llama_admission_messages_for_estimate([attached])
+
+ assert image_parts == mcp_images.MAX_TOTAL_MODEL_IMAGES + 4
+
+ def test_a_replayed_envelope_is_priced_without_its_base64(self):
+ """The count endpoint renders the messages verbatim, so an envelope left in
+ would be priced as thousands of tokens of base64 the completion never sends."""
+ from routes.inference import promote_mcp_history_images
+
+ raw = [
+ m.model_dump(exclude_none = True)
+ for m in ChatCompletionRequest(
+ model = "default", messages = self._mcp_tool_history()
+ ).messages
+ ]
+
+ priced = promote_mcp_history_images(raw, vision = False)
+
+ assert all("__MCP_IMAGES__" not in str(m.get("content")) for m in priced)
+ assert all(self._PNG_B64 not in str(m.get("content")) for m in priced)
+
+ def test_a_mixed_catalog_provider_still_forwards_an_attached_image(self):
+ """openrouter and huggingface are vision-capable for the family and name no
+ model, so the MCP gate says no for every model on them. Passed as the general
+ vision flag it also stripped the picture the caller attached to a chosen
+ vision model, which main forwarded."""
+ from core.inference.providers import get_provider_info
+ from routes.inference import _build_external_messages, _external_takes_mcp_images
+
+ attached = [
+ ChatMessage(
+ role = "user",
+ content = [
+ {"type": "text", "text": "what colour is this"},
+ {
+ "type": "image_url",
+ "image_url": {"url": f"data:image/png;base64,{self._PNG_B64}"},
+ },
+ ],
+ )
+ ]
+
+ for provider, model in (
+ ("openrouter", "openai/gpt-4o"),
+ ("huggingface", "Qwen/Qwen2.5-VL-7B-Instruct"),
+ ("mistral", "pixtral-large-latest"),
+ ("qwen", "qwen-vl-max"),
+ ):
+ info = get_provider_info(provider) or {}
+ vision = info.get("supports_vision", False)
+ gate = _external_takes_mcp_images(provider, vision, model, info)
+ assert gate is False, f"{provider} is expected to gate MCP promotion off"
+
+ built = _build_external_messages(
+ attached,
+ vision,
+ provider_type = provider,
+ promote_mcp_images = gate,
+ )
+
+ parts = built[0]["content"]
+ assert [p["type"] for p in parts] == ["text", "image_url"], provider
+
+ # The shape the regression had: one flag answering both questions.
+ # The image is gone and the lone text part collapses back to a string.
+ conflated = _build_external_messages(attached, gate, provider_type = provider)
+ assert (
+ conflated[0]["content"] == "what colour is this"
+ ), f"{provider}: the two decisions must stay separate"
+
+ def test_the_mcp_gate_still_keeps_an_envelope_off_a_mixed_catalog_provider(self):
+ from core.inference.providers import get_provider_info
+ from routes.inference import _build_external_messages, _external_takes_mcp_images
+
+ history = [ChatMessage(**m) for m in self._mcp_tool_history()]
+ info = get_provider_info("openrouter") or {}
+ vision = info.get("supports_vision", False)
+
+ built = _build_external_messages(
+ history,
+ vision,
+ provider_type = "openrouter",
+ promote_mcp_images = _external_takes_mcp_images(
+ "openrouter", vision, "openai/gpt-4o", info
+ ),
+ )
+
+ assert all("__MCP_IMAGES__" not in str(m.get("content")) for m in built)
+ assert not any(
+ isinstance(m.get("content"), list)
+ and any(p.get("type") == "image_url" for p in m["content"])
+ for m in built
+ )
+
def test_tool_nudge_system_update_dedupes_non_leading_system(self):
messages = [
{"role": "user", "content": "earlier"},
@@ -10588,3 +10841,359 @@ class TestPassthroughImageNormalization:
parts = [p for p in messages[0]["content"] if p.get("type") == "image_url"]
assert len(parts) == 1
assert parts[0]["image_url"]["url"].startswith("data:image/png;base64,")
+
+
+class TestMcpImagesOnTheClientToolPassthrough:
+ """The client-tool passthrough rebuilds its prompt from payload.messages and
+ flattens every content part to text. The MCP payloads survive in gen_kwargs,
+ so the markers have to be put back or the two disagree."""
+
+ def test_one_marker_is_restored_per_retained_payload(self):
+ from core.inference.mcp_images import append_placeholder_turn
+
+ # What the rebuild leaves: plain string content, no parts at all.
+ messages = [
+ {"role": "user", "content": "what does the file look like"},
+ {"role": "tool", "tool_call_id": "call_0", "content": "[2 images returned]"},
+ ]
+ payloads = ["AAAA", "BBBB"]
+
+ append_placeholder_turn(messages, len(payloads), len(payloads))
+
+ markers = [
+ part
+ for message in messages
+ if isinstance(message.get("content"), list)
+ for part in message["content"]
+ if part.get("type") == "image"
+ ]
+ assert len(markers) == len(payloads)
+ assert messages[-1]["role"] == "user"
+
+ def test_the_route_restores_them_on_that_branch(self):
+ """Scoped to the client-tool branch, not a character window: the code
+ around it moves with every upstream change, the branch does not."""
+ import inspect
+
+ import routes.inference as inference_route
+
+ body = inspect.getsource(inference_route.produce_openai_chat_completions)
+ branch = body[body.index("if _sf_client_tools:") :]
+ assert "_flatten_content_parts_for_local_template" in branch
+ rebuild = branch.index("_flatten_content_parts_for_local_template")
+ promote = branch.index("_promote_local_mcp_images_async(")
+ assert promote < rebuild, (
+ "the flatten has to run inside the promotion's argument, so the markers "
+ "are put back after the flatten that removed them"
+ )
+ assert (
+ "promote_mcp_images = False" in branch
+ ), "the passthrough must leave the envelopes on for the promotion to find"
+ # One marker per batch at its own position, never one block of them all.
+ assert "mark_mcp_image_turn_local(" not in branch
+ assert "insert_mcp_image_turn_before(" not in branch
+
+
+class TestCodexVisionCapability:
+ def test_an_mcp_run_resolves_the_catalog_before_pinning_the_vision_flag(self):
+ """The flag is fixed once for the whole run. A text-only opening turn on a
+ cold worker would pin it False and discard a picture an MCP tool returns
+ later, on a model the picker offered as vision-capable."""
+ import inspect
+
+ import routes.inference as inference_route
+
+ body = inspect.getsource(inference_route._proxy_to_external_provider)
+ assert "_may_receive_image = (" in body
+ assert "if _may_receive_image and model not in capabilities" in body
+
+ def test_a_replay_only_turn_also_resolves_the_catalog(self):
+ """MCP can be switched off while prior image turns remain. The capability
+ still has to be resolved, or the replay is stripped from a model that
+ could have read it."""
+ import inspect
+
+ import routes.inference as inference_route
+
+ body = inspect.getsource(inference_route._proxy_to_external_provider)
+ start = body.index("_may_receive_image = (")
+ predicate = body[start : body.index("if _may_receive_image", start)]
+ assert "image_requested" in predicate
+ assert "mcp_enabled" in predicate
+ # The promotable form, so a named non-MCP result ending in a valid envelope
+ # does not buy a catalog fetch nothing needs.
+ assert "_request_has_promotable_mcp_images(payload, exact = False)" in predicate
+
+
+_MCP_ADMISSION_PNG = (
+ "iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAIAAACQd1PeAAAADUlEQVR42mNk"
+ "+M9QDwADhgGAWjR9awAAAABJRU5ErkJggg=="
+)
+
+
+class TestMcpImageAdmissionAndCaps:
+ _PNG = _MCP_ADMISSION_PNG
+
+ def _envelope_turn(
+ self,
+ count = 1,
+ name = "mcp__fs__read_media_file",
+ ):
+ from core.inference import mcp_images
+ images = json.dumps([{"data": self._PNG, "mimeType": "image/png"}] * count)
+ return {
+ "role": "tool",
+ "tool_call_id": "call_0",
+ "name": name,
+ "content": f"[{count} images returned]\n" + mcp_images.SENTINEL + images,
+ }
+
+ def test_a_text_only_backend_is_charged_no_kv_for_replayed_images(self):
+ """Generation calls promote_history(vision=False) and sends no projector
+ images, so charging them here reserves the cache for pixels llama-server
+ never sees and serialises small text-only chats."""
+ from routes.inference import _openai_llama_admission_messages_for_estimate
+
+ turns = [self._envelope_turn(4), self._envelope_turn(4)]
+
+ _, vision_parts = _openai_llama_admission_messages_for_estimate(turns, vision = True)
+ _, text_only_parts = _openai_llama_admission_messages_for_estimate(turns, vision = False)
+
+ assert vision_parts == 8
+ assert text_only_parts == 0
+
+ def test_the_base64_is_stripped_from_the_estimate_either_way(self):
+ from routes.inference import _openai_llama_admission_messages_for_estimate
+ for vision in (True, False):
+ estimate, _ = _openai_llama_admission_messages_for_estimate(
+ [self._envelope_turn()], vision = vision
+ )
+ assert self._PNG not in json.dumps(estimate), vision
+
+ def test_an_attachment_counts_against_the_replay_cap(self):
+ """promote_history_local caps the replay alone; the attachment lands after it.
+ Its marker goes on the newest user turn -- here a placeholder already holding
+ a replay marker -- and a non-GGUF message carries one picture, so the replay
+ marker is displaced and pixels_in_marker_order drops its payload rather than
+ sliding it onto the next marker. Markers and pixels stay in step at the cap."""
+ from core.inference import mcp_images
+
+ conversation = []
+ payloads = []
+ for _ in range(mcp_images.MAX_TOTAL_MODEL_IMAGES):
+ conversation.append(mcp_images.placeholder_turn(1, 1))
+ payloads.append("AAAA")
+ prior = mcp_images.image_marker_parts(conversation)
+ # what the route does when an image is attached on top
+ conversation = mcp_images.mark_last_user_turn(conversation, 1)
+ pixels = mcp_images.pixels_in_marker_order(conversation, prior, payloads, "BBBB")
+ mcp_images.trim_image_turns(conversation, pixels)
+
+ markers = mcp_images.count_image_parts(conversation, "image")
+ assert len(pixels) == mcp_images.MAX_TOTAL_MODEL_IMAGES
+ assert markers == len(pixels)
+ # The attachment is the one that survives on its turn, and it is the newest.
+ assert pixels[-1] == "BBBB"
+ assert pixels.count("AAAA") == mcp_images.MAX_TOTAL_MODEL_IMAGES - 1
+
+ def test_the_route_trims_both_local_paths(self):
+ import inspect
+
+ import routes.inference as inference_route
+
+ body = inspect.getsource(inference_route.produce_openai_chat_completions)
+ assert body.count("trim_mcp_image_turns(") == 3, (
+ "the tool-loop path, the plain path and the client-tool rebuild all have "
+ "to reserve the attachment's slot"
+ )
+
+ def test_the_local_replay_promotion_is_off_the_event_loop(self):
+ import inspect
+
+ import routes.inference as inference_route
+
+ body = inspect.getsource(inference_route.produce_openai_chat_completions)
+ assert "await _promote_local_mcp_images_async(" in body
+ helper = inspect.getsource(inference_route._promote_local_mcp_images_async)
+ assert "asyncio.to_thread" in helper
+
+
+class TestMcpImageProvenanceAndCounting:
+ _PNG = _MCP_ADMISSION_PNG
+
+ def _turn(self, name):
+ from core.inference import mcp_images
+
+ images = json.dumps([{"data": self._PNG, "mimeType": "image/png"}])
+ msg = {
+ "role": "tool",
+ "tool_call_id": "call_0",
+ "content": "output\n" + mcp_images.SENTINEL + images,
+ }
+ if name is not None:
+ msg["name"] = name
+ return msg
+
+ def test_admission_strips_a_non_mcp_envelope_but_charges_it_nothing(self):
+ """Generation strips the suffix for everyone, so pricing it as text would
+ over-reserve; and it is never promoted, so charging image KV would reserve
+ the cache for pixels llama-server never sees."""
+ from routes.inference import _openai_llama_admission_messages_for_estimate
+
+ estimate, parts = _openai_llama_admission_messages_for_estimate(
+ [self._turn("bash")], vision = True
+ )
+
+ assert parts == 0
+ assert self._PNG not in json.dumps(estimate)
+
+ def test_admission_still_charges_an_mcp_envelope_as_images(self):
+ from routes.inference import _openai_llama_admission_messages_for_estimate
+
+ estimate, parts = _openai_llama_admission_messages_for_estimate(
+ [self._turn("mcp__fs__read_media_file")], vision = True
+ )
+
+ assert parts == 1
+ assert self._PNG not in json.dumps(estimate)
+
+ def test_the_estimator_and_generation_agree_on_provenance(self):
+ """The two must read the same rule, or the reservation and the prompt
+ disagree in whichever direction the mismatch runs."""
+ from core.inference.mcp_images import promote_history
+ from routes.inference import _openai_llama_admission_messages_for_estimate
+
+ for name, promoted in (("bash", False), ("mcp__fs__read_media_file", True)):
+ turn = self._turn(name)
+ out = promote_history([dict(turn)], vision = True)
+ estimate, parts = _openai_llama_admission_messages_for_estimate(
+ [dict(turn)], vision = True
+ )
+ # Stripped on both sides regardless of provenance...
+ assert out[0]["content"] != turn["content"], name
+ assert self._PNG not in json.dumps(estimate), name
+ # ...and charged exactly where it is promoted.
+ assert (len(out) > 1) is promoted, name
+ assert (parts > 0) is promoted, name
+
+
+class TestMcpReplayDetectorProvenance:
+ _PNG = _MCP_ADMISSION_PNG
+
+ def _req(self, name):
+ from core.inference import mcp_images
+
+ images = json.dumps([{"data": self._PNG, "mimeType": "image/png"}])
+ msg = {
+ "role": "tool",
+ "tool_call_id": "call_0",
+ "content": "output\n" + mcp_images.SENTINEL + images,
+ }
+ if name is not None:
+ msg["name"] = name
+ return ChatCompletionRequest(model = "default", messages = [msg])
+
+ def test_only_a_promotable_envelope_counts_as_a_replayed_image(self):
+ """Promotion preserves a named non-MCP result as plain text, so reading its
+ suffix as images refuses a countable prompt and buys a catalog fetch that
+ nothing needs."""
+ from routes.inference import _request_has_promotable_mcp_images
+
+ assert _request_has_promotable_mcp_images(self._req("mcp__fs__read_media_file"))
+ assert not _request_has_promotable_mcp_images(self._req("bash"))
+ # No name is not evidence against it: older stored turns carry none.
+ assert _request_has_promotable_mcp_images(self._req(None))
+
+ def test_the_count_guard_runs_before_the_mlx_dispatch(self):
+ """_mlx_count_chat_tokens returns its own count, so a guard placed after it
+ never sees a resident MLX vision model."""
+ import inspect
+
+ import routes.inference as inference_route
+
+ body = inspect.getsource(inference_route.chat_count_tokens)
+ guard = body.index("_resident_model_reads_images()")
+ dispatch = body.index("_mlx_count_chat_tokens(")
+ assert guard < dispatch, "the replay guard has to run before MLX answers"
+
+ def test_the_capability_probe_asks_both_backends(self):
+ import inspect
+
+ import routes.inference as inference_route
+
+ probe = inspect.getsource(inference_route._resident_model_reads_images)
+ assert "get_llama_cpp_backend()" in probe
+ assert "get_inference_backend" in probe
+ assert 'entry.get("is_vision")' in probe
+
+
+def test_a_text_only_qwen_model_is_never_sent_an_mcp_picture():
+ """qwen reports supports_vision for the family and names no model, and every one
+ of its four registry defaults is a text model -- the DashScope vision line is
+ qwen-vl-*. Appending an image there fails the run upstream rather than degrading."""
+ from core.inference.providers import get_provider_info
+ from routes.inference import _external_takes_mcp_images
+
+ info = get_provider_info("qwen") or {}
+ vision = info.get("supports_vision", False)
+ assert vision is True, "the premise is that the family flag is permissive"
+ assert not (info.get("model_capabilities") or {}), "and that it names no model"
+
+ for model in info.get("default_models") or []:
+ assert _external_takes_mcp_images("qwen", vision, model, info) is False, model
+
+
+def test_kimi_stays_permissive_because_its_allowlist_already_narrowed_it():
+ """The opposite case, so the conservative set does not grow by reflex: kimi's
+ model_id_allowlist admits only the multimodal pair its own entry documents."""
+ from core.inference.providers import get_provider_info
+ from routes.inference import _external_takes_mcp_images
+
+ info = get_provider_info("kimi") or {}
+ vision = info.get("supports_vision", False)
+
+ for model in info.get("default_models") or []:
+ assert _external_takes_mcp_images("kimi", vision, model, info) is True, model
+
+
+def test_admission_prices_replay_against_what_generation_really_sends():
+ """The GGUF paths keep the full replay allowance BESIDE the caller's own picture
+ (only a provider reserves the caller's room), so this request sends nine and is
+ charged nine. Subtracting the attachment admitted a second such request into KV
+ the first had already taken; charging a second full allowance on top of it would
+ over-reserve instead, so the replay is still capped at MAX_TOTAL_MODEL_IMAGES."""
+ import json as _json
+
+ from core.inference import mcp_images
+ from routes.inference import _openai_llama_admission_messages_for_estimate
+
+ _PNG = (
+ "iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAIAAACQd1PeAAAADUlEQVR42mNk"
+ "+M9QDwADhgGAWjR9awAAAABJRU5ErkJggg=="
+ )
+ envelope = _json.dumps([{"data": _PNG, "mimeType": "image/png"} for _ in range(4)])
+ history = []
+ for index in range(2):
+ history.append(
+ {
+ "role": "tool",
+ "name": f"mcp__s__shot{index}",
+ "content": "[4 images returned]\n" + mcp_images.SENTINEL + envelope,
+ }
+ )
+ history.append(
+ {
+ "role": "user",
+ "content": [
+ {"type": "text", "text": "and this one"},
+ {"type": "image_url", "image_url": {"url": f"data:image/png;base64,{_PNG}"}},
+ ],
+ }
+ )
+
+ _, image_parts = _openai_llama_admission_messages_for_estimate(history)
+
+ assert image_parts == mcp_images.MAX_TOTAL_MODEL_IMAGES + 1, (
+ f"reserved {image_parts} image embeddings for a request that sends "
+ f"{mcp_images.MAX_TOTAL_MODEL_IMAGES + 1}"
+ )
diff --git a/studio/backend/tests/test_orchestrator_unload_cancel.py b/studio/backend/tests/test_orchestrator_unload_cancel.py
index 33ffd7c6f3a..d64f0534060 100644
--- a/studio/backend/tests/test_orchestrator_unload_cancel.py
+++ b/studio/backend/tests/test_orchestrator_unload_cancel.py
@@ -3484,3 +3484,51 @@ def test_every_long_lived_spawner_consults_the_shutdown_latch():
f"{rel} never rechecks the latch after adopting, so a spawn that raced it "
"is left outside a sweep that has already finished"
)
+
+
+def test_the_adapter_path_accepts_the_same_images_kwarg_as_the_locked_one():
+ """The tool loop hands both paths one common_kwargs. Without `images` on the
+ dispatched path an adapter-controlled vision chat raises TypeError before it
+ generates, and the markers left in the conversation have no pixels behind."""
+ import inspect
+
+ from core.inference.orchestrator import InferenceOrchestrator
+
+ dispatched = inspect.signature(InferenceOrchestrator._generate_dispatched).parameters
+ locked = inspect.signature(InferenceOrchestrator._generate_inner).parameters
+
+ assert "images" in locked
+ assert "images" in dispatched, (
+ "generate_with_adapter_control forwards **common_kwargs straight into "
+ "_generate_dispatched, so a missing `images` is an immediate TypeError"
+ )
+
+
+def test_the_adapter_path_forwards_images_to_the_worker_command():
+ import inspect
+
+ from core.inference.orchestrator import InferenceOrchestrator
+
+ body = inspect.getsource(InferenceOrchestrator._generate_dispatched)
+ assert (
+ "images_b64 = images" in body
+ ), "_build_generate_cmd carries the list to the worker as images_b64"
+
+
+def test_both_generation_paths_forward_the_image_ordinal():
+ """The dispatched path forwarded it and the locked one dropped it, so ordinary
+ non-adapter generation reached the worker with None. The marker top-up then put
+ the attachment's marker on the newest user turn rather than the turn that
+ supplied it, which changes which question appears to own the picture."""
+ import inspect
+
+ from core.inference.orchestrator import InferenceOrchestrator
+
+ for method in (
+ InferenceOrchestrator._generate_dispatched,
+ InferenceOrchestrator._generate_inner,
+ ):
+ assert "image_ordinal" in inspect.signature(method).parameters, method.__name__
+ assert "image_ordinal = image_ordinal" in inspect.getsource(method), (
+ f"{method.__name__} accepts the ordinal but does not put it on the " "worker command"
+ )
diff --git a/studio/backend/tests/test_safetensors_tool_loop.py b/studio/backend/tests/test_safetensors_tool_loop.py
index 3f959e82a8a..afe5eba4076 100644
--- a/studio/backend/tests/test_safetensors_tool_loop.py
+++ b/studio/backend/tests/test_safetensors_tool_loop.py
@@ -5152,6 +5152,343 @@ class TestStreamingDisplayStripStillMatchesTheExportedHelper:
), f"diverged at offset {i}"
+def test_mcp_images_reach_a_vision_model_through_the_sink(monkeypatch):
+ import base64
+ import io
+ import json
+
+ from PIL import Image
+
+ from core.inference import mcp_images
+
+ buffer = io.BytesIO()
+ Image.new("RGB", (6, 6), (10, 120, 200)).save(buffer, format = "PNG")
+ envelope = json.dumps(
+ [{"data": base64.b64encode(buffer.getvalue()).decode(), "mimeType": "image/png"}]
+ )
+ result = "[1 image returned]\n" + mcp_images.SENTINEL + envelope
+
+ turns = [
+ '<tool_call>{"name": "mcp__fs__read_media_file", "arguments": {}}</tool_call>',
+ "A tabby cat.",
+ ]
+ seen: list[list] = []
+
+ def single_turn(conversation, *, active_tools = None):
+ seen.append([dict(message) for message in conversation])
+ yield turns[len(seen) - 1]
+
+ sink: list = []
+ list(
+ run_safetensors_tool_loop(
+ single_turn = single_turn,
+ messages = [{"role": "user", "content": "describe the image"}],
+ tools = [{"type": "function", "function": {"name": "mcp__fs__read_media_file"}}],
+ execute_tool = lambda name, args, **kwargs: result,
+ max_tool_iterations = 2,
+ images_sink = sink,
+ )
+ )
+
+ assert len(sink) == 1
+ assert base64.b64decode(sink[0])[:8] == b"\x89PNG\r\n\x1a\n"
+ second_turn = seen[1]
+ assert "__MCP_IMAGES__" not in json.dumps(second_turn)
+ assert second_turn[-1]["role"] == "user"
+ assert second_turn[-1]["content"][0] == {"type": "image"}
+
+
+def test_mcp_images_are_left_out_without_a_sink(monkeypatch):
+ import json
+
+ from core.inference import mcp_images
+
+ envelope = json.dumps([{"data": "QUJD", "mimeType": "image/png"}])
+ result = "[1 image returned]\n" + mcp_images.SENTINEL + envelope
+ turns = [
+ '<tool_call>{"name": "mcp__fs__read_media_file", "arguments": {}}</tool_call>',
+ "A tabby cat.",
+ ]
+ seen: list[list] = []
+
+ def single_turn(conversation, *, active_tools = None):
+ seen.append([dict(message) for message in conversation])
+ yield turns[len(seen) - 1]
+
+ list(
+ run_safetensors_tool_loop(
+ single_turn = single_turn,
+ messages = [{"role": "user", "content": "describe the image"}],
+ tools = [{"type": "function", "function": {"name": "mcp__fs__read_media_file"}}],
+ execute_tool = lambda name, args, **kwargs: result,
+ max_tool_iterations = 2,
+ )
+ )
+
+ assert seen[1][-1]["role"] == "tool"
+ assert "__MCP_IMAGES__" not in json.dumps(seen[1])
+
+
+def test_mcp_image_markers_merge_into_a_deferred_noop_turn(monkeypatch):
+ """A batch holding both an internal no-op and an image-returning call used to
+ append two role=user turns in a row, which a strict VLM template rejects."""
+ import base64
+ import io
+ import json
+
+ from PIL import Image
+
+ from core.inference import mcp_images
+
+ buffer = io.BytesIO()
+ Image.new("RGB", (6, 6), (10, 120, 200)).save(buffer, format = "PNG")
+ envelope = json.dumps(
+ [{"data": base64.b64encode(buffer.getvalue()).decode(), "mimeType": "image/png"}]
+ )
+ result = "[1 image returned]\n" + mcp_images.SENTINEL + envelope
+
+ # One image-returning call plus one call to a tool that is not enabled: the
+ # second is suppressed as an internal no-op and lands as a deferred nudge
+ # after the batch's results, which is the turn the markers must join.
+ real = '{"name": "mcp__fs__read_media_file", "arguments": {}}'
+ bogus = '{"name": "mcp__fs__delete_everything", "arguments": {}}'
+ turns = [
+ f"<tool_call>{real}</tool_call><tool_call>{bogus}</tool_call>",
+ "A tabby cat.",
+ ]
+ seen: list[list] = []
+
+ def single_turn(conversation, *, active_tools = None):
+ seen.append([dict(message) for message in conversation])
+ yield turns[len(seen) - 1]
+
+ sink: list = []
+ list(
+ run_safetensors_tool_loop(
+ single_turn = single_turn,
+ messages = [{"role": "user", "content": "describe the image"}],
+ tools = [{"type": "function", "function": {"name": "mcp__fs__read_media_file"}}],
+ execute_tool = lambda name, args, **kwargs: result,
+ max_tool_iterations = 2,
+ images_sink = sink,
+ )
+ )
+
+ second_turn = seen[1]
+ roles = [message["role"] for message in second_turn]
+ # Guard: without a suppressed call there is no nudge and the test proves nothing.
+ assert any(
+ message["role"] == "user" and "not executed" in str(message.get("content"))
+ for message in second_turn
+ ), f"no deferred no-op nudge in {second_turn}"
+ assert not any(
+ roles[i] == "user" and roles[i + 1] == "user" for i in range(len(roles) - 1)
+ ), f"consecutive user turns: {roles}"
+ markers = [
+ part
+ for message in second_turn
+ if isinstance(message.get("content"), list)
+ for part in message["content"]
+ if part.get("type") == "image"
+ ]
+ assert len(markers) == len(sink) == 1
+
+
+def test_the_loop_cap_never_evicts_the_caller_s_own_attachment():
+ """The sink is seeded with what the caller attached, and the MCP payloads land
+ around it. A cap that trims from the front deleted that picture and its marker,
+ so a model asked to compare against it was handed screenshots instead."""
+ import base64
+ import io
+ import json
+
+ from PIL import Image
+
+ from core.inference import mcp_images
+
+ def _png(colour):
+ buffer = io.BytesIO()
+ Image.new("RGB", (6, 6), colour).save(buffer, format = "PNG")
+ return base64.b64encode(buffer.getvalue()).decode()
+
+ attachment = _png((255, 0, 0))
+ envelope = json.dumps([{"data": _png((0, 0, 255)), "mimeType": "image/png"} for _ in range(4)])
+ result = "[4 images returned]\n" + mcp_images.SENTINEL + envelope
+
+ def _call(n):
+ return '<tool_call>{"name": "mcp__fs__shot", "arguments": {"n": %d}}</tool_call>' % n
+
+ # DISTINCT calls: an identical repeat is suppressed as a no-op. Eight of them,
+ # because a local placeholder turn carries one picture, and the cap has to be
+ # reached for the test to prove anything.
+ turns = [_call(n) for n in range(1, 9)] + ["done."]
+ seen: list[list] = []
+
+ def single_turn(conversation, *, active_tools = None):
+ seen.append([dict(message) for message in conversation])
+ yield turns[len(seen) - 1]
+
+ sink = [attachment]
+ list(
+ run_safetensors_tool_loop(
+ single_turn = single_turn,
+ messages = [
+ {
+ "role": "user",
+ "content": [{"type": "image"}, {"type": "text", "text": "compare with this"}],
+ }
+ ],
+ tools = [{"type": "function", "function": {"name": "mcp__fs__shot"}}],
+ execute_tool = lambda name, args, **kwargs: result,
+ max_tool_iterations = 9,
+ images_sink = sink,
+ caller_image_indexes = (0,),
+ )
+ )
+
+ assert attachment in sink, "the caller's attachment was trimmed away"
+ assert (
+ len(sink) == mcp_images.MAX_TOTAL_MODEL_IMAGES
+ ), "the attachment is spared, but it still counts against the cap"
+ final = seen[-1]
+ assert final[0]["content"][0] == {"type": "image"}, "its marker went with it"
+ markers = sum(
+ 1
+ for message in final
+ if isinstance(message.get("content"), list)
+ for part in message["content"]
+ if part.get("type") == "image"
+ )
+ assert markers == len(sink), "one marker per pixel, or the processor miscounts"
+
+
+def test_a_resumed_chat_s_replayed_images_still_count_against_the_cap():
+ """Only the caller's own attachment is spared. Sparing the whole seeded sink
+ exempted a resumed chat's replayed pictures too, so the prompt could carry a
+ second full allowance on top of them."""
+ import base64
+ import io
+ import json
+
+ from PIL import Image
+
+ from core.inference import mcp_images
+
+ def _png(colour):
+ buffer = io.BytesIO()
+ Image.new("RGB", (6, 6), colour).save(buffer, format = "PNG")
+ return base64.b64encode(buffer.getvalue()).decode()
+
+ replayed = [_png((0, 200 - index * 10, 0)) for index in range(4)]
+ envelope = json.dumps([{"data": _png((0, 0, 255)), "mimeType": "image/png"} for _ in range(4)])
+ result = "[4 images returned]\n" + mcp_images.SENTINEL + envelope
+
+ def _call(n):
+ return '<tool_call>{"name": "mcp__fs__shot", "arguments": {"n": %d}}</tool_call>' % n
+
+ # One picture per batch on a local path, so five batches on four replayed is
+ # nine against a cap of eight.
+ turns = [_call(n) for n in range(1, 6)] + ["done."]
+ seen: list[list] = []
+
+ def single_turn(conversation, *, active_tools = None):
+ seen.append([dict(message) for message in conversation])
+ yield turns[len(seen) - 1]
+
+ sink = list(replayed)
+ list(
+ run_safetensors_tool_loop(
+ single_turn = single_turn,
+ messages = [mcp_images.placeholder_turn(4, 4)],
+ tools = [{"type": "function", "function": {"name": "mcp__fs__shot"}}],
+ execute_tool = lambda name, args, **kwargs: result,
+ max_tool_iterations = 6,
+ images_sink = sink,
+ # No attachment on this request; every seeded pixel is replay.
+ caller_image_indexes = (),
+ )
+ )
+
+ assert (
+ len(sink) == mcp_images.MAX_TOTAL_MODEL_IMAGES
+ ), f"replay was exempted from the cap: {len(sink)} images in the prompt"
+ assert replayed[0] not in sink, "the oldest replayed picture should have gone first"
+ final = seen[-1]
+ markers = sum(
+ 1
+ for message in final
+ if isinstance(message.get("content"), list)
+ for part in message["content"]
+ if part.get("type") == "image"
+ )
+ assert markers == len(sink)
+
+
+def test_the_protected_attachment_index_is_rebased_between_batches():
+ """The index names a position, and trimming earlier entries moves it. Reusing the
+ original across iterations protected the wrong payload and deleted the very
+ attachment the argument exists to keep."""
+ import base64
+ import io
+ import json
+
+ from PIL import Image
+
+ from core.inference import mcp_images
+
+ def _png(colour):
+ buffer = io.BytesIO()
+ Image.new("RGB", (6, 6), colour).save(buffer, format = "PNG")
+ return base64.b64encode(buffer.getvalue()).decode()
+
+ replayed = [_png((0, 200 - index * 12, 0)) for index in range(7)]
+ attachment = _png((255, 0, 0))
+ envelope = json.dumps([{"data": _png((0, 0, 255)), "mimeType": "image/png"} for _ in range(4)])
+ result = "[4 images returned]\n" + mcp_images.SENTINEL + envelope
+
+ def _call(n):
+ return '<tool_call>{"name": "mcp__fs__shot", "arguments": {"n": %d}}</tool_call>' % n
+
+ turns = [_call(1), _call(2), _call(3), "done."]
+ seen: list[list] = []
+
+ def single_turn(conversation, *, active_tools = None):
+ seen.append([dict(message) for message in conversation])
+ yield turns[len(seen) - 1]
+
+ # Replay first, the caller's attachment last: index 7 on the way in.
+ sink = [*replayed, attachment]
+ list(
+ run_safetensors_tool_loop(
+ single_turn = single_turn,
+ messages = [
+ mcp_images.placeholder_turn(7, 7),
+ {
+ "role": "user",
+ "content": [{"type": "image"}, {"type": "text", "text": "compare with this"}],
+ },
+ ],
+ tools = [{"type": "function", "function": {"name": "mcp__fs__shot"}}],
+ execute_tool = lambda name, args, **kwargs: result,
+ max_tool_iterations = 4,
+ images_sink = sink,
+ caller_image_indexes = (7,),
+ )
+ )
+
+ assert attachment in sink, "the attachment was deleted by a stale protected index"
+ assert len(sink) == mcp_images.MAX_TOTAL_MODEL_IMAGES
+ final = seen[-1]
+ markers = sum(
+ 1
+ for message in final
+ if isinstance(message.get("content"), list)
+ for part in message["content"]
+ if part.get("type") == "image"
+ )
+ assert markers == len(sink), "one marker per pixel"
+
+
class TestBlockedGemmaChainHold:
"""A promotable Gemma call behind a blocked one must not stream before it is promoted."""
diff --git a/studio/backend/tests/test_sf_client_tools_passthrough.py b/studio/backend/tests/test_sf_client_tools_passthrough.py
index eafbf7b6855..d76de9f13d7 100644
--- a/studio/backend/tests/test_sf_client_tools_passthrough.py
+++ b/studio/backend/tests/test_sf_client_tools_passthrough.py
@@ -166,6 +166,86 @@ def _json_body(response):
return json.loads(response.body if hasattr(response, "body") else response.content)
[email protected]("supports_tools", [False, True])
+def test_mcp_replay_preserves_multiple_caller_attachments(monkeypatch, supports_tools):
+ import base64
+ import io
+
+ from PIL import Image
+
+ def encoded(color):
+ buffer = io.BytesIO()
+ Image.new("RGB", (8, 8), color).save(buffer, format = "PNG")
+ return base64.b64encode(buffer.getvalue()).decode("ascii")
+
+ def user(color, text):
+ return ChatMessage(
+ role = "user",
+ content = [
+ {
+ "type": "image_url",
+ "image_url": {"url": "data:image/png;base64," + encoded(color)},
+ },
+ {"type": "text", "text": text},
+ ],
+ )
+
+ backend = _ScriptedBackend(_fixed("done"))
+ backend.models["sf-model"].update(
+ is_vision = True,
+ chat_template_info = {
+ "template": _TOKENIZER_TEMPLATE_WITH_TOOLS,
+ "processor_template": _TOKENIZER_TEMPLATE_WITH_TOOLS,
+ "renders_image": True,
+ "accepts_multiple_images": True,
+ },
+ )
+ payload = _request(
+ messages = [
+ user("red", "Remember this image."),
+ ChatMessage(
+ role = "assistant",
+ content = "",
+ tool_calls = [
+ {
+ "id": "call_1",
+ "type": "function",
+ "function": {"name": "mcp__test__image", "arguments": "{}"},
+ }
+ ],
+ ),
+ ChatMessage(
+ role = "tool",
+ tool_call_id = "call_1",
+ name = "mcp__test__image",
+ content = "[1 image returned]\n__MCP_IMAGES__:"
+ + json.dumps([{"mimeType": "image/png", "data": encoded("green")}]),
+ ),
+ ChatMessage(role = "assistant", content = "I have the tool image."),
+ user("blue", "Compare all three images."),
+ ],
+ enable_tools = False,
+ stream = False,
+ )
+
+ _call(payload, monkeypatch, backend, supports_tools = supports_tools)
+
+ [call] = backend.calls
+ colors = []
+ for image in call["images"]:
+ if isinstance(image, str):
+ image = Image.open(io.BytesIO(base64.b64decode(image)))
+ colors.append(image.getpixel((0, 0)))
+ assert colors == [(255, 0, 0), (0, 128, 0), (0, 0, 255)]
+ image_turns = [
+ message
+ for message in call["messages"]
+ if isinstance(message.get("content"), list)
+ and any(part.get("type") == "image" for part in message["content"])
+ ]
+ assert len(image_turns) == 3
+
+
def _collect_sse(response):
async def _run():
return [c async for c in response.body_iterator]
@@ -1224,6 +1304,11 @@ _PNG_1x1 = (
"+M9QDwADhgGAWjR9awAAAABJRU5ErkJggg=="
)
+_PNG_B64 = (
+ "iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAIAAACQd1PeAAAADUlEQVR42mNk"
+ "+M9QDwADhgGAWjR9awAAAABJRU5ErkJggg=="
+)
+
def _vision_backend(*snapshots):
backend = _ScriptedBackend(_fixed(*snapshots))
@@ -1267,6 +1352,344 @@ def test_legacy_image_field_keeps_the_client_tool_catalog(monkeypatch):
assert backend.calls[0]["image"] is not None
+class _VisionToolLoopBackend(_ToolLoopBackend):
+ def __init__(self, responder, **kwargs):
+ super().__init__(responder, **kwargs)
+ self.models["sf-model"]["is_vision"] = True
+
+ @staticmethod
+ def resize_image(image):
+ return image
+
+
+def test_an_attached_image_still_reaches_the_tool_loop(monkeypatch):
+ backend = _VisionToolLoopBackend(_fixed("done"))
+ payload = _request(
+ messages = [
+ ChatMessage(
+ role = "user",
+ content = [
+ {"type": "text", "text": "what is in this picture"},
+ {
+ "type": "image_url",
+ "image_url": {"url": f"data:image/png;base64,{_PNG_B64}"},
+ },
+ ],
+ )
+ ],
+ enable_tools = True,
+ stream = False,
+ )
+
+ _call(payload, monkeypatch, backend)
+
+ [call] = backend.calls
+ assert call["tools"], "tools must survive an attached image on a vision model"
+ assert call["images"], "the attachment has to ride into the loop"
+ markers = [
+ part
+ for message in call["messages"]
+ if isinstance(message.get("content"), list)
+ for part in message["content"]
+ if part.get("type") == "image"
+ ]
+ assert len(markers) == len(call["images"])
+
+
+_TOKENIZER_TEMPLATE_WITH_TOOLS = (
+ "{% if tools %}<|im_start|>system\n"
+ "{% for t in tools %}{{ t.function.name }}\n{% endfor %}<|im_end|>\n{% endif %}"
+ "{% for m in messages %}<|im_start|>{{ m['role'] }}\n{{ m['content'] }}<|im_end|>\n{% endfor %}"
+)
+_PROCESSOR_TEMPLATE_WITHOUT_TOOLS = (
+ "{% for m in messages %}<|im_start|>{{ m['role'] }}\n{{ m['content'] }}<|im_end|>\n{% endfor %}"
+)
+
+
+def test_the_image_tool_loop_is_gated_on_the_body_that_renders(monkeypatch):
+ """The loop now runs on a vision model with an attachment, and generation renders
+ through the PROCESSOR template. Classifying its tool support from the tokenizer body
+ started the loop on a model whose prompt carries no schemas at all."""
+ import routes.inference as inf
+
+ backend = _VisionToolLoopBackend(_fixed("done"))
+ backend.models["sf-model"]["chat_template_info"] = {
+ "template": _TOKENIZER_TEMPLATE_WITH_TOOLS,
+ "processor_template": _PROCESSOR_TEMPLATE_WITHOUT_TOOLS,
+ "renders_image": True,
+ }
+ payload = _request(
+ messages = [
+ ChatMessage(
+ role = "user",
+ content = [
+ {"type": "text", "text": "what is in this picture"},
+ {
+ "type": "image_url",
+ "image_url": {"url": f"data:image/png;base64,{_PNG_B64}"},
+ },
+ ],
+ )
+ ],
+ enable_tools = True,
+ stream = False,
+ )
+
+ _install(monkeypatch, backend)
+ monkeypatch.setattr(
+ inf,
+ "_detect_safetensors_features",
+ lambda _backend, template, **k: {
+ "supports_tools": template == _TOKENIZER_TEMPLATE_WITH_TOOLS
+ },
+ )
+
+ async def _run():
+ return await openai_chat_completions(payload, request = _Request(), current_subject = "u")
+
+ asyncio.run(_run())
+
+ assert backend.calls, "generation never ran"
+ assert not any(
+ call.get("tools") for call in backend.calls
+ ), "the tool loop was driven from a template the image render never selects"
+
+
+def test_a_client_catalog_keeps_an_image_out_of_the_server_loop(monkeypatch):
+ """Letting the loop take a picture must not take the request with it: #10092 routes
+ image-plus-tools to the passthrough so the CALLER's schemas are the ones rendered.
+ Claiming it here answered the client with Unsloth's built-ins instead."""
+ backend = _VisionToolLoopBackend(_fixed("a plain answer"))
+ payload = _request(
+ messages = [
+ ChatMessage(
+ role = "user",
+ content = [
+ {"type": "text", "text": "what is in this picture"},
+ {
+ "type": "image_url",
+ "image_url": {"url": f"data:image/png;base64,{_PNG_B64}"},
+ },
+ ],
+ )
+ ],
+ tools = [LOOKUP_TOOL],
+ enable_tools = True,
+ stream = False,
+ )
+
+ _call(payload, monkeypatch, backend)
+
+ assert backend.calls, "generation never ran"
+ assert backend.calls[0]["tools"] == [LOOKUP_TOOL]
+ # The passthrough renders one turn; the loop would have run the built-ins instead.
+ assert "images" not in backend.calls[0] or not backend.calls[0].get("images")
+
+
+def test_a_replayed_picture_sits_beside_the_result_that_produced_it(monkeypatch):
+ """The passthrough flatten costs the markers their positions. Promoting after it
+ puts each batch's turn straight after its result, so "the tool call above" names
+ the right call -- one detached block of every payload, placed wherever the
+ attachment's turn happened to be, did not."""
+ import base64
+ import io
+ import json
+
+ from PIL import Image
+
+ from core.inference import mcp_images
+
+ buffer = io.BytesIO()
+ Image.new("RGB", (6, 6), (10, 120, 200)).save(buffer, format = "PNG")
+ envelope = json.dumps(
+ [{"data": base64.b64encode(buffer.getvalue()).decode(), "mimeType": "image/png"}]
+ )
+
+ backend = _ScriptedBackend(_fixed("a plain answer"))
+ backend.models["sf-model"]["is_vision"] = True
+ backend.models["sf-model"]["chat_template_info"] = {
+ "template": "{% for m in messages %}{{ m['content'] }}{% endfor %}",
+ "renders_image": True,
+ }
+ payload = _request(
+ messages = [
+ ChatMessage(role = "user", content = "take a shot"),
+ ChatMessage(
+ role = "assistant",
+ content = "",
+ tool_calls = [
+ {
+ "id": "call_0",
+ "type": "function",
+ "function": {"name": "mcp__fs__shot", "arguments": "{}"},
+ }
+ ],
+ ),
+ ChatMessage(
+ role = "tool",
+ tool_call_id = "call_0",
+ content = "[1 image returned]\n" + mcp_images.SENTINEL + envelope,
+ ),
+ ChatMessage(role = "assistant", content = "a blue square"),
+ ChatMessage(role = "user", content = "and what about now"),
+ ],
+ tools = [LOOKUP_TOOL],
+ stream = False,
+ )
+
+ _call(payload, monkeypatch, backend)
+
+ [call] = backend.calls
+ sent = call["messages"]
+ assert call["images"] and len(call["images"]) == 1
+ [marker_at] = [
+ index
+ for index, message in enumerate(sent)
+ if isinstance(message.get("content"), list)
+ and any(part.get("type") == "image" for part in message["content"])
+ ]
+ assert sent[marker_at - 1]["role"] == "tool", [m["role"] for m in sent]
+ lead = next(part["text"] for part in sent[marker_at]["content"] if part.get("type") == "text")
+ assert lead.startswith(mcp_images.IMAGE_TURN_TEXT), lead
+ roles = [m["role"] for m in sent]
+ assert all(a != "user" or b != "user" for a, b in zip(roles, roles[1:])), roles
+
+
+def test_a_replay_only_image_turn_also_keeps_the_client_catalog(monkeypatch):
+ """The gate read `image is None`, so a resumed chat whose only pictures are
+ replayed MCP ones looked image-free and the loop took the request -- swapping the
+ caller's schemas for Unsloth's on exactly the path this change adds."""
+ import base64
+ import io
+ import json
+
+ from PIL import Image
+
+ from core.inference import mcp_images
+
+ buffer = io.BytesIO()
+ Image.new("RGB", (6, 6), (10, 120, 200)).save(buffer, format = "PNG")
+ envelope = json.dumps(
+ [{"data": base64.b64encode(buffer.getvalue()).decode(), "mimeType": "image/png"}]
+ )
+
+ backend = _VisionToolLoopBackend(_fixed("a plain answer"))
+ payload = _request(
+ messages = [
+ ChatMessage(role = "user", content = "take a shot"),
+ ChatMessage(
+ role = "assistant",
+ content = "",
+ tool_calls = [
+ {
+ "id": "call_0",
+ "type": "function",
+ "function": {"name": "mcp__fs__shot", "arguments": "{}"},
+ }
+ ],
+ ),
+ ChatMessage(
+ role = "tool",
+ tool_call_id = "call_0",
+ content = "[1 image returned]\n" + mcp_images.SENTINEL + envelope,
+ ),
+ ChatMessage(role = "user", content = "what colour was it"),
+ ],
+ tools = [LOOKUP_TOOL],
+ enable_tools = True,
+ stream = False,
+ )
+
+ _call(payload, monkeypatch, backend)
+
+ assert backend.calls, "generation never ran"
+ assert backend.calls[0]["tools"] == [LOOKUP_TOOL]
+
+
+def test_the_plain_route_leaves_the_attachment_marker_to_the_backend(monkeypatch):
+ """The backends snapshot the conversation's existing markers as history's before
+ topping up, so a marker the ROUTE pre-added is counted as a replayed picture's.
+ With the attachment on an earlier turn than a tool's picture the two pixels then
+ bind to each other's turns, and the model reads the screenshot as the diagram."""
+ import base64
+ import io
+ import json
+
+ from PIL import Image
+
+ from core.inference import mcp_images
+
+ buffer = io.BytesIO()
+ Image.new("RGB", (8, 8), (10, 120, 200)).save(buffer, format = "PNG")
+ envelope = json.dumps(
+ [{"data": base64.b64encode(buffer.getvalue()).decode(), "mimeType": "image/png"}]
+ )
+
+ backend = _ScriptedBackend(_fixed("a plain answer"))
+ backend.models["sf-model"]["is_vision"] = True
+ backend.models["sf-model"]["chat_template_info"] = {
+ "template": "{% for m in messages %}{{ m['content'] }}{% endfor %}",
+ "renders_image": True,
+ }
+ payload = _request(
+ messages = [
+ # The attachment rides on an EARLIER turn than the tool's picture.
+ ChatMessage(
+ role = "user",
+ content = [
+ {"type": "text", "text": "here is my diagram"},
+ {
+ "type": "image_url",
+ "image_url": {"url": f"data:image/png;base64,{_PNG_B64}"},
+ },
+ ],
+ ),
+ ChatMessage(role = "assistant", content = "noted"),
+ ChatMessage(
+ role = "assistant",
+ content = "",
+ tool_calls = [
+ {
+ "id": "c1",
+ "type": "function",
+ "function": {"name": "mcp__s__shot", "arguments": "{}"},
+ }
+ ],
+ ),
+ ChatMessage(
+ role = "tool",
+ tool_call_id = "c1",
+ content = "[1 image returned]\n" + mcp_images.SENTINEL + envelope,
+ ),
+ ChatMessage(role = "user", content = "which one is bluer?"),
+ ],
+ stream = False,
+ )
+
+ # supports_tools=False is what makes this the PLAIN path: the replayed picture
+ # arrives with tool history, and a template that renders tools would send the
+ # request to the client-tool passthrough, which orders the pixels itself.
+ _call(payload, monkeypatch, backend, supports_tools = False)
+
+ [call] = backend.calls
+ sent, replayed = call["messages"], call["images"] or []
+ assert call["image"] is not None, "the attachment reached the backend"
+ assert len(replayed) == 1, "so did the replayed picture"
+
+ # Exactly what the backends do next. Given a conversation the route did not
+ # pre-mark, the top-up places the attachment's marker at its own ordinal and
+ # the history/attachment split survives.
+ prior = mcp_images.image_marker_parts(sent)
+ topped = mcp_images.top_up_image_markers(sent, len(replayed) + 1, ordinal = call["image_ordinal"])
+ ordered = mcp_images.pixels_in_marker_order(topped, prior, ["MCP"], "ATTACHMENT")
+
+ assert len(mcp_images.image_marker_parts(topped)) == 2, topped
+ assert ordered == [
+ "ATTACHMENT",
+ "MCP",
+ ], "the route pre-marked, so the pixels bound to each other's markers"
+
+
def test_video_turn_with_tools_enabled_keeps_the_client_tool_catalog(monkeypatch):
"""A clip rules out the server loop like an image; the passthrough keeps catalog and clip."""
backend = _vision_backend(_CALL_XML)
diff --git a/studio/backend/tests/test_studio_tool_loop.py b/studio/backend/tests/test_studio_tool_loop.py
index 3926934fd88..4fa391ea649 100644
--- a/studio/backend/tests/test_studio_tool_loop.py
+++ b/studio/backend/tests/test_studio_tool_loop.py
@@ -136,6 +136,7 @@ def _run(
tools = None,
tool_choice = None,
messages = None,
+ supports_vision = False,
**policy_kwargs,
):
policy_fields = {
@@ -159,6 +160,7 @@ def _run(
session_id = "s1",
thread_id = "t1",
tool_choice = tool_choice,
+ supports_vision = supports_vision,
),
policy = ToolLoopPolicy(**policy_fields),
cancel_event = cancel_event,
@@ -1258,3 +1260,80 @@ def test_a_fragment_naming_its_call_goes_back_to_that_call(executed):
_run(transport)
assert [call["arguments"] for call in executed] == [{"query": "first"}, {"query": "second"}]
+
+
+# ── MCP images ────────────────────────────────────────────────────
+
+
+def _mcp_image_transport() -> FakeTransport:
+ return FakeTransport(
+ [
+ [
+ _sse(
+ {
+ "tool_calls": [
+ {
+ "index": 0,
+ "id": "call_img",
+ "function": {
+ "name": "mcp__fs__read_media_file",
+ "arguments": '{"path":"cat.png"}',
+ },
+ }
+ ]
+ }
+ ),
+ _sse(finish = "tool_calls"),
+ _DONE,
+ ],
+ [_sse({"content": "A tabby cat."}), _sse(finish = "stop"), _DONE],
+ ]
+ )
+
+
+def mcp_image_result(monkeypatch):
+ import base64
+ import io
+
+ from PIL import Image
+
+ from core.inference import mcp_images
+
+ buffer = io.BytesIO()
+ Image.new("RGB", (6, 6), (10, 120, 200)).save(buffer, format = "PNG")
+ envelope = json.dumps(
+ [{"data": base64.b64encode(buffer.getvalue()).decode(), "mimeType": "image/png"}]
+ )
+ monkeypatch.setattr(
+ loop_mod,
+ "execute_tool",
+ lambda name, arguments, **kwargs: "[1 image returned]\n" + mcp_images.SENTINEL + envelope,
+ )
+ monkeypatch.setattr(loop_mod, "build_rag_autoinject", lambda *a, **k: None)
+ monkeypatch.setattr(loop_mod, "is_high_risk_tool_call", lambda name, args: False)
+
+
+def test_mcp_images_reach_a_vision_provider_as_their_own_user_turn(mcp_image_result):
+ transport = _mcp_image_transport()
+
+ _run(
+ transport,
+ tools = [_tool("mcp__fs__read_media_file")],
+ supports_vision = True,
+ )
+
+ follow_up = transport.requests[1]["messages"]
+ assert [message["role"] for message in follow_up[-3:]] == ["assistant", "tool", "user"]
+ assert "__MCP_IMAGES__" not in follow_up[-2]["content"]
+ assert follow_up[-1]["content"][1]["image_url"]["url"].startswith("data:image/png;base64,")
+
+
+def test_mcp_images_are_not_sent_to_a_text_only_provider(mcp_image_result):
+ transport = _mcp_image_transport()
+
+ _run(transport, tools = [_tool("mcp__fs__read_media_file")])
+
+ follow_up = transport.requests[1]["messages"]
+ assert follow_up[-1]["role"] == "tool"
+ assert "__MCP_IMAGES__" not in follow_up[-1]["content"]
diff --git a/studio/backend/tests/test_video_attachment_part.py b/studio/backend/tests/test_video_attachment_part.py
index 4ec0ecc6ad4..6c10c67f143 100644
--- a/studio/backend/tests/test_video_attachment_part.py
+++ b/studio/backend/tests/test_video_attachment_part.py
@@ -197,7 +197,9 @@ def test_a_non_gguf_model_takes_the_clip_through_one_gate_and_hands_it_to_genera
assert gate < speech and gate < audio_input
assert 'gen_kwargs["video"] = _video_clip' in source
use_tools = source.index("_sf_use_tools = (", handler)
- assert "and _video_clip is None" in source[use_tools : use_tools + 400]
+ tools_block = source[use_tools : source.index("\n )", use_tools)]
+ tools_block = "\n".join(line.split("#")[0] for line in tools_block.splitlines())
+ assert "and _video_clip is None" in tools_block
# Structural, not literal. This pinned the exact
# "(image is not None or _video_clip is not None) and not _sf_use_tools"; #10970
# widened the image half to `_sf_has_image`, a superset, so the clause still fires
@@ -215,10 +217,10 @@ def test_a_non_gguf_model_takes_the_clip_through_one_gate_and_hands_it_to_genera
# is not enough: `and` for `or` stops an image-only or video-only request entering the
# passthrough, and `image is None` / `not _sf_has_image` invert the condition. Either
# order, since which side reads first is arbitrary.
- image = r"(?:\bimage is not None\b|\b_sf_has_image\b)"
+ image = r"(?:\bimage is not None\b|\b_sf_has_(?:any_)?image\b)"
clip = r"\b_video_clip is not None\b"
assert re.search(rf"{image}\s+or\s+{clip}|{clip}\s+or\s+{image}", escape), escape
- assert not re.search(r"\bimage is None\b|\bnot\s+_sf_has_image\b", escape), escape
+ assert not re.search(r"\bimage is None\b|\bnot\s+_sf_has_(?:any_)?image\b", escape), escape
# Settled at the gate: a model without audio input never enters the audio-input path.
conflict = source.index("if payload.audio_base64:", gate)
assert conflict < speech
diff --git a/studio/backend/tests/test_vision_client_tools.py b/studio/backend/tests/test_vision_client_tools.py
index 56a4dea194b..20d905c8543 100644
--- a/studio/backend/tests/test_vision_client_tools.py
+++ b/studio/backend/tests/test_vision_client_tools.py
@@ -171,7 +171,11 @@ def _vision_probe(chat_template = _CHATML_WITH_TOOLS):
return "PROMPT-WITH-TOOLS"
return "PROMPT"
- def __call__(self, *_args, **_kwargs):
+ def __call__(self, *args, **_kwargs):
+ # The pixels arrive as the first positional argument, one object when
+ # there is a single image and the list itself when there are several.
+ if args:
+ seen["images"] = args[0]
return Batch({"input_ids": torch.zeros((1, 1), dtype = torch.long)})
class Model:
@@ -1531,6 +1535,103 @@ def test_the_nudge_retry_skips_the_image_marker_on_a_text_only_fallback():
assert not any(p.get("type") == "image" for p in body), message
+def test_an_earlier_attachment_keeps_its_own_turn_against_a_newer_replay():
+ """The attachment's turn can PRECEDE a tool's picture. The plain route used to
+ pre-add its marker, so this backend counted that marker as history's, the top-up
+ became a no-op, and pixels_in_marker_order could no longer tell the two apart --
+ the model was shown the screenshot where the user's own diagram belonged."""
+ from core.inference.mcp_images import placeholder_turn
+
+ backend, seen = _vision_probe()
+ attachment, replayed = object(), object()
+
+ # What the plain route hands the backend once it stops pre-marking: the
+ # attachment's turn is a plain string and the only marker is the replay's.
+ messages = [
+ {"role": "user", "content": "here is my diagram"},
+ {"role": "assistant", "content": "noted"},
+ {
+ "role": "assistant",
+ "content": "",
+ "tool_calls": [
+ {
+ "id": "c1",
+ "type": "function",
+ "function": {"name": "mcp__s__shot", "arguments": "{}"},
+ }
+ ],
+ },
+ {"role": "tool", "tool_call_id": "c1", "content": "[1 image returned]"},
+ placeholder_turn(1, 1),
+ {"role": "user", "content": "which one is bluer?"},
+ ]
+
+ _drain(
+ backend,
+ messages = messages,
+ image = attachment,
+ images = [replayed],
+ image_ordinal = 0,
+ )
+
+ rendered = seen["messages"]
+ marker_turns = [
+ index
+ for index, message in enumerate(rendered)
+ if isinstance(message.get("content"), list)
+ and any(part.get("type") == "image" for part in message["content"])
+ ]
+ assert len(marker_turns) == 2, rendered
+ assert marker_turns[0] < marker_turns[1]
+ assert seen["images"] == [attachment, replayed], "the pixels bound to each other's markers"
+
+
+ "initial, requested", [(True, False), (False, True), ("first", "second"), ("first", None)]
+)
+def test_replayed_vision_images_honor_adapter_selection_under_lock(initial, requested):
+ backend, seen = _vision_probe()
+ backend.models[backend.active_model_name]["is_vision"] = True
+ model = backend.models[backend.active_model_name]["model"]
+ state = {"active": initial, "applied": []}
+
+ def apply(value):
+ assert backend._generation_lock.locked()
+ # The real helper's contract: None leaves the loaded state alone.
+ if value is None:
+ return
+ state["applied"].append(value)
+ state["active"] = value
+
+ def generate(**kwargs):
+ assert backend._generation_lock.locked()
+ state["generated_with"] = state["active"]
+
+ backend._apply_adapter_state = apply
+ model.generate = generate
+ replay = object()
+ list(
+ backend.generate_with_adapter_control(
+ use_adapter = requested,
+ messages = [
+ {
+ "role": "user",
+ "content": [
+ {"type": "image"},
+ {"type": "text", "text": "Describe the tool image"},
+ ],
+ }
+ ],
+ image = None,
+ images = [replay],
+ max_new_tokens = 1,
+ )
+ )
+ assert seen["images"] is replay
+ assert state["generated_with"] == (initial if requested is None else requested)
+ assert state["applied"] == ([] if requested is None else [requested])
+
+
def test_each_compare_pane_applies_its_own_adapter_state_on_an_image_turn(monkeypatch):
backend, _ = _vision_probe()
info = backend.models["vision-tools"]
diff --git a/studio/frontend/src/features/chat/api/chat-adapter.ts b/studio/frontend/src/features/chat/api/chat-adapter.ts
index caec120789f..0094fa5aa78 100644
--- a/studio/frontend/src/features/chat/api/chat-adapter.ts
+++ b/studio/frontend/src/features/chat/api/chat-adapter.ts
@@ -72,6 +72,13 @@ import {
sandboxSessionIdFor,
} from "@/components/assistant-ui/sandbox-files";
import { apiUrl } from "@/lib/api-base";
+import { isMcpToolName } from "../utils/mcp-tool-name";
+import {
+ type McpImage,
+ planMcpImageBound,
+ mcpImagesEnvelope,
+ splitMcpImages,
+} from "./mcp-images";
import {
answerTextFromParts,
extractSearchImages,
@@ -125,12 +132,14 @@ import {
parseExternalModelId,
providerModelSupportsStudioTools,
providerModelSupportsVision,
+ providerModelTakesMcpImages,
supportsProviderPromptCacheTtl,
supportsProviderPromptCaching,
toExternalBackendProviderType,
} from "../external-providers";
import {
+ localToolExchangeIndexes,
addCodexReasoning,
codexLocalToolRoundId,
codexReasoningForToolCalls,
@@ -1020,6 +1029,13 @@ function isSandboxWrapper(
return isSandboxToolResult(result);
}
+/** Exactly the live parser's wrapper: text and images, no other field. */
+export function isBareMcpImageWrapper(val: unknown): boolean {
+ if (!isMcpImageToolResult(val)) return false;
+ const keys = Object.keys(val as object).filter((key) => key !== "text" && key !== "images");
+ return keys.length === 0;
+}
+
export function isMcpImageToolResult(val: unknown): val is McpImageToolResult {
if (typeof val !== "object" || val === null) {
return false;
@@ -1057,7 +1073,14 @@ function serializeToolResultPart(
// Backend ChatMessage rejects role="tool" with empty content; a sentinel JSON round-trips it.
content = result.length > 0 ? result : JSON.stringify({ result: "" });
} else if (
- isMcpImageToolResult(result) ||
+ // The wrapper the live parser builds -- {text, images} and nothing else -- from an
+ // MCP result, or from any tool whose raw output ends in a valid envelope. Those
+ // are unwrapped by shape, since JSON.stringify below would replay the whole base64
+ // array as ordinary prompt text; provenance gates the ENVELOPE, a few lines down.
+ // A client tool's own structured result that merely carries text and images among
+ // OTHER fields is not that wrapper, and keeps its normal JSON serialization.
+ (isMcpImageToolResult(result) &&
+ (isMcpToolName(tc.toolName) || isBareMcpImageWrapper(result))) ||
isSearchImagesToolResult(result) ||
isSandboxWrapper(result, tc.toolName ?? "")
) {
@@ -1067,6 +1090,12 @@ function serializeToolResultPart(
? stripSearchImageTokens(result.text)
: result.text;
content = replayText.length > 0 ? replayText : JSON.stringify({ result: "" });
+ // Gated on the mcp__ id the backend stamps, not on shape alone: a client tool
+ // is free to answer {text, images:[{data, mimeType}]}, and appending the
+ // envelope would hand its bytes to the model as image input.
+ if (isMcpImageToolResult(result) && isMcpToolName(tc.toolName)) {
+ content += mcpImagesEnvelope(result.images);
+ }
} else {
try {
content = JSON.stringify(result);
@@ -1687,24 +1716,127 @@ export const CANVAS_TOOL_INSTRUCTION =
export const CANVAS_FALLBACK_INSTRUCTION =
"When the user asks for an HTML, CSS, or JavaScript canvas, return one complete self-contained fenced html code block. Embed CSS and JavaScript inside the document. Do not emit tool-call syntax.";
+/** The MCP image bound applied to the run's own tool results, BEFORE they are
+ * serialized. Bounding the OpenAI history afterwards meant every result's image
+ * array was stringified into an envelope on the main thread and then parsed again
+ * to be cut or dropped -- on every send, for pictures the request would not carry.
+ * A result that keeps no picture becomes its plain text, which is what the
+ * serializer emits for a stripped envelope. */
+function boundMcpImageResults(
+ messages: RunMessages,
+ { readsImages, localMarkers }: { readsImages: boolean; localMarkers: boolean },
+): RunMessages {
+ type Carrier = { message: number; part: number; images: McpImage[] };
+ // One entry per replay EXCHANGE, not per message: a local run accumulates every
+ // round's tool calls in one assistant message and the serializer splits them by
+ // round id into the exchanges the backend sees, and a batch is one exchange's
+ // results. Partitioned by the serializer's own rule so the two cannot drift.
+ const perExchange: Carrier[][] = [];
+ messages.forEach((message, m) => {
+ const parts = message.content;
+ if (!Array.isArray(parts)) return;
+ const toolParts: { index: number; part: ToolCallMessagePart }[] = [];
+ parts.forEach((part, p) => {
+ if (part.type === "tool-call") {
+ toolParts.push({ index: p, part: part as ToolCallMessagePart });
+ }
+ });
+ const exchanges = localToolExchangeIndexes(
+ toolParts,
+ ({ part }) => codexLocalToolRoundId(getToolReplayProvenance(part)),
+ ({ part }) => shouldFlushCompletedLocalToolPair(part),
+ );
+ const byExchange = new Map<number, Carrier[]>();
+ toolParts.forEach(({ index, part }, k) => {
+ const tc = part as { toolName?: string; result?: unknown };
+ if (!isMcpImageToolResult(tc.result) || !isMcpToolName(tc.toolName)) return;
+ const carrier = { message: m, part: index, images: tc.result.images };
+ const batch = byExchange.get(exchanges[k]) ?? [];
+ batch.push(carrier);
+ byExchange.set(exchanges[k], batch);
+ });
+ for (const key of [...byExchange.keys()].sort((a, b) => a - b)) {
+ perExchange.push(byExchange.get(key)!);
+ }
+ });
+ if (perExchange.length === 0) return messages;
+ const batches = localMarkers
+ ? perExchange
+ : perExchange.flatMap((batch) => batch.map((carrier) => [carrier]));
+ const plan = readsImages
+ ? planMcpImageBound(
+ batches.map((batch) => batch.map((carrier) => carrier.images)),
+ { localMarkers },
+ )
+ : batches.map((batch) => batch.map((): McpImage[] => []));
+ const edits = new Map<number, Map<number, McpImage[]>>();
+ batches.forEach((batch, b) =>
+ batch.forEach((carrier, r) => {
+ const kept = plan[b][r];
+ if (kept.length === carrier.images.length) return;
+ const forMessage = edits.get(carrier.message) ?? new Map<number, McpImage[]>();
+ forMessage.set(carrier.part, kept);
+ edits.set(carrier.message, forMessage);
+ }),
+ );
+ if (edits.size === 0) return messages;
+ return messages.map((message, m) => {
+ const forMessage = edits.get(m);
+ if (!forMessage || !Array.isArray(message.content)) return message;
+ const content = message.content.map((part, p) => {
+ const kept = forMessage.get(p);
+ if (!kept) return part;
+ const result = (part as { result: McpImageToolResult }).result;
+ return {
+ ...part,
+ result: kept.length > 0 ? { ...result, images: kept } : result.text,
+ };
+ });
+ return { ...message, content } as typeof message;
+ }) as RunMessages;
+}
+
+/** Whether the local target reads an MCP picture: the SELECTED model's vision flag,
+ * in either direction, since a queued send can target a vision model while the
+ * resident one is text-only and its loadedIsMultimodal is stale. The loaded state is
+ * the fallback only when the selection's capability is unknown; unknown keeps them.
+ * Not loadedIsMultimodal alone either way: an audio-only model also sets it. */
+function localTargetReadsImages(
+ state: Pick<ChatRuntimeState, "models" | "params" | "loadedIsMultimodal">,
+): boolean {
+ const activeModel = state.models.find(
+ (model) => model.id === state.params.checkpoint,
+ );
+ if (typeof activeModel?.isVision === "boolean") return activeModel.isVision;
+ return state.loadedIsMultimodal !== false;
+}
+
/** The OpenAI-form history a completion would send. The tool catalog is priced server-side,
* since --enable-tools can inject schemas the client cannot see. */
export async function buildLocalTokenCountHistory(
- messages: RunMessages,
+ rawMessages: RunMessages,
threadId: string | undefined,
): Promise<{
messages: OpenAIChatMessage[];
studio_tool_history?: true;
}> {
+ const runtimeState = useChatRuntimeStore.getState();
+ const { params, artifactsEnabled, supportsTools } = runtimeState;
+ const activeModel = runtimeState.models.find(
+ (model) => model.id === runtimeState.params.checkpoint,
+ );
+ // Apply send-path target limits before serializing this background recount;
+ // backend limits run after parsing and cannot bound the uploaded body.
+ const messages = boundMcpImageResults(rawMessages, {
+ readsImages: localTargetReadsImages(runtimeState),
+ localMarkers: activeModel?.isGguf === false,
+ });
const survivingMessages = pruneOutboundHistory(messages, true);
const outboundMessages = survivingMessages
.flatMap((message) => toOpenAIMessages(message, true))
.filter((message): message is NonNullable<typeof message> =>
Boolean(message),
);
-
- const { params, artifactsEnabled, supportsTools } =
- useChatRuntimeStore.getState();
const safeSystemPrompt =
typeof params.systemPrompt === "string"
? resolveSystemPromptVariables(
@@ -4005,7 +4137,7 @@ export function createOpenAIStreamAdapter(
): ChatModelAdapter {
const adapter = {
async *run({
- messages,
+ messages: rawMessages,
runConfig,
abortSignal,
unstable_threadId,
@@ -4193,9 +4325,9 @@ export function createOpenAIStreamAdapter(
.reverse()
.find((m) => m.role === "user");
if (!userMessage) throw new Error("Research requires a user message.");
- const userMessageIndex = messages.indexOf(userMessage);
+ const userMessageIndex = rawMessages.indexOf(userMessage);
const userMessageParentId =
- userMessageIndex > 0 ? messages[userMessageIndex - 1]!.id : null;
+ userMessageIndex > 0 ? rawMessages[userMessageIndex - 1]!.id : null;
const { params } = runtime;
await persistResolvedQueuedModel(
params.checkpoint,
@@ -4765,13 +4897,35 @@ export function createOpenAIStreamAdapter(
throw new Error("Image generation edit unavailable.");
}
+ // Resolve before the outbound build's standalone tests/studio slice. Match
+ // backend provider/model gates and local vision flags (audio-only models can
+ // be multimodal), so targets that strip MCP images never upload the envelopes.
+ const targetReadsImages = isExternalRequest
+ ? providerModelTakesMcpImages(
+ externalProvider?.providerType,
+ externalSelection?.modelId,
+ )
+ : localTargetReadsImages(runtime);
+ // A local target that is not a GGUF renders replayed pictures as markers, one per
+ // tool batch, so the upload is bounded to what that path can use (the bound runs
+ // below, after the slice). Unknown format keeps the part paths' four per result.
+ const mcpImagesLocalMarkers =
+ !isExternalRequest &&
+ runtime.models.find((model) => model.id === runtime.params.checkpoint)
+ ?.isGguf === false;
+ // Bounded on the run's own results, before any envelope is built: what the
+ // request will not carry is never stringified, and never parsed back.
+ const messages = boundMcpImageResults(rawMessages, {
+ readsImages: targetReadsImages,
+ localMarkers: mcpImagesLocalMarkers,
+ });
const survivingMessages = pruneOutboundHistory(
messages,
!isExternalRequest,
);
// toOpenAIMessages emits assistant tool_calls plus role="tool" follow-ups; the backend Gemini
// translator rebuilds the functionCall/functionResponse parts.
- const outboundMessages = survivingMessages
+ let outboundMessages = survivingMessages
.flatMap((message) => toOpenAIMessages(message, !isExternalRequest))
.filter((message): message is NonNullable<typeof message> =>
Boolean(message),
@@ -6868,8 +7022,6 @@ export function createOpenAIStreamAdapter(
)
? rawResult.lastIndexOf(imgMarker)
: -1;
- const mcpImgMarker = "\n__MCP_IMAGES__:";
- const mcpImgIdx = rawResult.lastIndexOf(mcpImgMarker);
let parsedResult:
| string
| {
@@ -6891,22 +7043,9 @@ export function createOpenAIStreamAdapter(
const imageB64 = toolEvent.image_b64 as string | undefined;
// A valid MCP image envelope wins; an invalid marker falls through so a sandbox __IMAGES__
// suffix still renders.
- let mcpImages: McpImageToolResult | null = null;
- if (mcpImgIdx !== -1) {
- try {
- const images = JSON.parse(
- rawResult.slice(mcpImgIdx + mcpImgMarker.length),
- );
- const candidate = {
- text: rawResult.slice(0, mcpImgIdx),
- images,
- };
- if (isMcpImageToolResult(candidate))
- mcpImages = candidate;
- } catch {
- // Not a valid envelope; fall through below.
- }
- }
+ const mcpCandidate = splitMcpImages(rawResult);
+ const mcpImages: McpImageToolResult | null =
+ isMcpImageToolResult(mcpCandidate) ? mcpCandidate : null;
if (
toolCallParts[idx].toolName === "image_generation" &&
typeof imageB64 === "string" &&
diff --git a/studio/frontend/src/features/chat/api/mcp-images.ts b/studio/frontend/src/features/chat/api/mcp-images.ts
new file mode 100644
index 00000000000..60fd450742c
--- /dev/null
+++ b/studio/frontend/src/features/chat/api/mcp-images.ts
@@ -0,0 +1,220 @@
+// SPDX-License-Identifier: AGPL-3.0-only
+// Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
+
+// The envelope the backend appends to a tool result that returned images.
+// Validated, so tool text that merely mentions the marker is never truncated.
+
+export const MCP_IMAGES_MARKER = "\n__MCP_IMAGES__:";
+
+export interface McpImage {
+ data: string;
+ mimeType: string;
+ // Set on the first entry once this side has shortened the array, so the backend's
+ // note can still say how many the tool returned rather than how many were uploaded.
+ returned?: number;
+}
+
+export function isMcpImageArray(value: unknown): value is McpImage[] {
+ return (
+ Array.isArray(value) &&
+ value.length > 0 &&
+ value.every(
+ (image) =>
+ typeof image === "object" &&
+ image !== null &&
+ typeof (image as Record<string, unknown>).data === "string" &&
+ typeof (image as Record<string, unknown>).mimeType === "string",
+ )
+ );
+}
+
+export function splitMcpImages(result: string): {
+ text: string;
+ images: McpImage[];
+} {
+ const idx = result.lastIndexOf(MCP_IMAGES_MARKER);
+ if (idx === -1) return { text: result, images: [] };
+ let images: unknown;
+ try {
+ images = JSON.parse(result.slice(idx + MCP_IMAGES_MARKER.length));
+ } catch {
+ return { text: result, images: [] };
+ }
+ if (!isMcpImageArray(images)) return { text: result, images: [] };
+ return { text: result.slice(0, idx), images };
+}
+
+// Re-attached on replay: the backend promotes it into an image turn for a vision
+// model, and strips it for every other one.
+export function mcpImagesEnvelope(images: McpImage[]): string {
+ return MCP_IMAGES_MARKER + JSON.stringify(images);
+}
+
+// The backend keeps the newest eight pictures of a conversation. It can only do
+// that after the request is parsed, so without the same bound here every past
+// screenshot is uploaded again on every turn and the body grows without limit.
+export const MAX_TOTAL_MCP_IMAGES = 8;
+// Mirrors the backend's per-result promotion limit.
+export const MAX_MODEL_IMAGES = 4;
+// Mirrors LOCAL_MAX_IMAGES_PER_TURN in studio/backend/core/inference/mcp_images.py: the
+// marker paths (safetensors, MLX) render a tool batch as one turn carrying one picture.
+export const LOCAL_MAX_IMAGES_PER_TURN = 1;
+// Spare candidates carried past that limit, because the backend's quota counts
+// images that DECODE and this side cannot tell which will. Bounded, so a result
+// of unreadable blobs still cannot grow the request without limit.
+export const DECODE_FAILURE_ALLOWANCE = 4;
+// Bound total replay base64 to the backend's per-result payload ceiling.
+// The count cap alone permits twelve 12 MB candidates (~144 MB per turn).
+export const MAX_TOTAL_MCP_IMAGE_CHARS = 12_000_000;
+// The data budget matches mcp_client.MAX_IMAGE_PAYLOAD_CHARS so accepted live
+// images fit their replay. Bound metadata separately, matching mcp_images.py
+// MAX_MCP_IMAGE_MIME_CHARS, to prevent oversized mimeType values bypassing it.
+export const MAX_MCP_IMAGE_MIME_CHARS = 256;
+
+const MCP_TOOL_PREFIX = "mcp__";
+
+interface EnvelopeCarrier {
+ role?: string;
+ name?: string;
+ content?: unknown;
+}
+
+/** The bound itself, over batches of results in document order (oldest first): what
+ * each result may still carry. A batch is the results one model turn produced in
+ * parallel; the marker paths render it as one turn carrying one picture, so with
+ * localMarkers the batch is charged once and its further candidates are decode
+ * fallbacks. The part paths take MAX_MODEL_IMAGES per result, each result its own batch.
+ *
+ * Both carriers of an envelope -- the serialized OpenAI history and the run's own tool
+ * results -- plan through here, so the two never drift. */
+export function planMcpImageBound(
+ batches: readonly (readonly (readonly McpImage[])[])[],
+ { localMarkers = false }: { localMarkers?: boolean } = {},
+): McpImage[][][] {
+ let budget = MAX_TOTAL_MCP_IMAGES;
+ // Shared, so at most MAX_TOTAL_MCP_IMAGES + DECODE_FAILURE_ALLOWANCE candidates
+ // ever leave here however many results there are.
+ let spare = DECODE_FAILURE_ALLOWANCE;
+ let charsLeft = MAX_TOTAL_MCP_IMAGE_CHARS;
+ const perResult = localMarkers ? LOCAL_MAX_IMAGES_PER_TURN : MAX_MODEL_IMAGES;
+ const out: McpImage[][][] = batches.map((batch) => batch.map(() => []));
+ // Newest first: those are the ones the backend would have kept.
+ for (let b = batches.length - 1; b >= 0; b--) {
+ const batch = batches[b];
+ const room = Math.min(budget, perResult);
+ // Charge only what the backend can promote; excess entries must not evict
+ // older results. Since only successful decodes count and this side cannot
+ // decode, keep shared fallback candidates beyond the remaining image budget.
+ // Spending or resetting spares per result can strand valid older images
+ // behind corrupt entries even when decoding would leave room.
+ let allowance = room + spare;
+ let taken = 0;
+ for (let r = batch.length - 1; r >= 0; r--) {
+ const images = batch[r];
+ // Scan until enough candidates fit: slicing first could discard smaller
+ // images behind entries that exceed the remaining byte budget.
+ const keep: McpImage[] = [];
+ for (const image of images) {
+ if (keep.length >= allowance) break;
+ if (image.mimeType.length > MAX_MCP_IMAGE_MIME_CHARS) continue;
+ const cost = image.data.length;
+ // Keep looking for smaller images that fit, matching the live-result budget.
+ if (charsLeft - cost < 0) continue;
+ charsLeft -= cost;
+ keep.push(image);
+ }
+ allowance -= keep.length;
+ taken += keep.length;
+ if (keep.length === images.length) {
+ out[b][r] = images.slice();
+ continue;
+ }
+ // Carry the count the tool actually returned, or a prior bound's record of it,
+ // so the backend's note does not describe this upload as the whole result.
+ const returned = Math.max(images[0]?.returned ?? 0, images.length);
+ out[b][r] = keep.length > 0 ? [{ ...keep[0], returned }, ...keep.slice(1)] : [];
+ }
+ // Charge only the batch's usable slots. Fallback candidates consume spares
+ // so they cannot independently evict older results.
+ const charged = Math.min(taken, room);
+ budget -= charged;
+ spare -= taken - charged;
+ }
+ return out;
+}
+
+/** The bound over an already serialized OpenAI history. The send path bounds the
+ * run's own tool results before serializing them (chat-adapter's
+ * boundMcpImageResults), so a discarded envelope is never even built; this form
+ * serves callers that only hold the wire shape. */
+export function boundMcpImageEnvelopes<T extends EnvelopeCarrier>(
+ messages: readonly T[],
+ { localMarkers = false }: { localMarkers?: boolean } = {},
+): T[] {
+ const out = messages.slice();
+ type Carrier = { index: number; text: string; images: McpImage[] };
+ const carriers: Carrier[] = [];
+ for (let i = 0; i < out.length; i++) {
+ const message = out[i];
+ if (!message || message.role !== "tool") continue;
+ if (typeof message.content !== "string") continue;
+ const { text, images } = splitMcpImages(message.content);
+ if (images.length === 0) continue;
+ // Match backend provenance: strip named non-MCP envelopes so their payloads
+ // cannot bypass the replay bounds and be uploaded again.
+ if (
+ typeof message.name === "string" &&
+ message.name &&
+ !message.name.startsWith(MCP_TOOL_PREFIX)
+ ) {
+ out[i] = { ...message, content: text };
+ continue;
+ }
+ carriers.push({ index: i, text, images });
+ }
+ // Consecutive tool results are one batch on a marker target.
+ const batches: Carrier[][] = [];
+ for (const carrier of carriers) {
+ const previous = batches[batches.length - 1];
+ const last = previous?.[previous.length - 1];
+ let sameBatch = localMarkers && last !== undefined;
+ for (let j = (last?.index ?? 0) + 1; sameBatch && j < carrier.index; j++) {
+ if (out[j]?.role !== "tool") sameBatch = false;
+ }
+ if (sameBatch && previous) previous.push(carrier);
+ else batches.push([carrier]);
+ }
+ const plan = planMcpImageBound(
+ batches.map((batch) => batch.map((carrier) => carrier.images)),
+ { localMarkers },
+ );
+ batches.forEach((batch, b) =>
+ batch.forEach((carrier, r) => {
+ const kept = plan[b][r];
+ if (kept.length === carrier.images.length) return;
+ out[carrier.index] = {
+ ...out[carrier.index],
+ content:
+ kept.length > 0
+ ? carrier.text + mcpImagesEnvelope(kept)
+ : carrier.text,
+ };
+ }),
+ );
+ return out;
+}
+
+// For a target known not to read images. The backend strips these envelopes without
+// sending a pixel, so leaving them on the wire re-uploaded up to 12 million characters
+// of base64 on every text turn after a switch to a text-only model. The stored
+// history keeps them, for a later switch back.
+export function stripMcpImageEnvelopes<T extends EnvelopeCarrier>(
+ messages: readonly T[],
+): T[] {
+ return messages.map((message) => {
+ if (!message || message.role !== "tool") return message;
+ if (typeof message.content !== "string") return message;
+ const { text, images } = splitMcpImages(message.content);
+ return images.length === 0 ? message : { ...message, content: text };
+ });
+}
diff --git a/studio/frontend/src/features/chat/codex-reasoning.ts b/studio/frontend/src/features/chat/codex-reasoning.ts
index e40dc98793a..46e626312dd 100644
--- a/studio/frontend/src/features/chat/codex-reasoning.ts
+++ b/studio/frontend/src/features/chat/codex-reasoning.ts
@@ -95,3 +95,40 @@ export function codexReasoningForToolCalls(
}
return undefined;
}
+
+/** Which replay exchange each tool-call part of one assistant message lands in, by the
+ * serializer's own rule: a round id that changes starts a new exchange, and a part with
+ * no round id that completes a local pair is an exchange of its own. The MCP image
+ * bound groups a message's results by this, so it shares one marker turn's allowance
+ * across exactly the results the backend will see as one batch -- grouping the whole
+ * message let a later round's picture evict an earlier round's. */
+export function localToolExchangeIndexes<TPart>(
+ parts: readonly TPart[],
+ roundIdOf: (part: TPart) => number | null,
+ flushesPair: (part: TPart) => boolean,
+): number[] {
+ const indexes: number[] = [];
+ let exchange = 0;
+ let pending = 0;
+ let pendingRoundId: number | null = null;
+ for (const part of parts) {
+ const roundId = roundIdOf(part);
+ if (pending > 0 && startsNewCodexToolRound(pendingRoundId, roundId)) {
+ exchange += 1;
+ pending = 0;
+ }
+ if (roundId !== null) pendingRoundId = roundId;
+ const flushLocalPair = roundId === null && flushesPair(part);
+ if (flushLocalPair && pending > 0) {
+ exchange += 1;
+ pending = 0;
+ }
+ indexes.push(exchange);
+ pending += 1;
+ if (flushLocalPair) {
+ exchange += 1;
+ pending = 0;
+ }
+ }
+ return indexes;
+}
diff --git a/studio/frontend/src/features/chat/external-providers.ts b/studio/frontend/src/features/chat/external-providers.ts
index 3aeafbae755..8ea2c745373 100644
--- a/studio/frontend/src/features/chat/external-providers.ts
+++ b/studio/frontend/src/features/chat/external-providers.ts
@@ -216,6 +216,29 @@ export function providerModelSupportsVision(
}
+// Mirrors _MIXED_CATALOG_PROVIDER_TYPES in studio/backend/routes/inference.py: vision for
+// the family, unknown for a given model, so an MCP picture is not sent to one the
+// registry says nothing about.
+const MIXED_CATALOG_PROVIDER_TYPES = new Set(["huggingface", "openrouter", "qwen"]);
+
+/** Whether the backend's external tool loop will hand this model an MCP picture -- the
+ * same rule as its _external_takes_mcp_images, so envelopes the backend would strip
+ * anyway are not re-uploaded on every turn. Stricter than providerModelSupportsVision:
+ * a model-level answer wins, a mixed catalog with none is a no, unknown elsewhere is a yes. */
+export function providerModelTakesMcpImages(
+ providerType: string | null | undefined,
+ modelId: string | null | undefined,
+): boolean {
+ if (providerTypeSupportsVision(providerType) === false) return false;
+ hydrateProviderModelCapabilities();
+ if (providerType && modelId) {
+ const capability = REGISTRY_MODEL_CAPABILITIES.get(providerType)?.[modelId];
+ if (typeof capability?.vision === "boolean") return capability.vision;
+ }
+ if (providerType && MIXED_CATALOG_PROVIDER_TYPES.has(providerType)) return false;
+ return true;
+}
+
/** Provider-level capability key. Self-hosted model ids are user-supplied, so there is no
* per-model entry: the registry declares the capability once for the whole provider type. */
export const PROVIDER_CAPABILITY_WILDCARD = "*";
diff --git a/studio/frontend/src/features/chat/hooks/use-chat-search-index.ts b/studio/frontend/src/features/chat/hooks/use-chat-search-index.ts
index 18972b7f9d0..57a0fe2ec52 100644
--- a/studio/frontend/src/features/chat/hooks/use-chat-search-index.ts
+++ b/studio/frontend/src/features/chat/hooks/use-chat-search-index.ts
@@ -13,6 +13,7 @@ import {
CHAT_HISTORY_UPDATED_EVENT,
batchListChatMessages,
} from "../api/chat-api";
+import { splitMcpImages } from "../api/mcp-images";
import type { MessageRecord } from "../types";
import { isCoalescedHistoryEvent } from "../utils/chat-history-revision";
import {
@@ -52,38 +53,11 @@ const ROW_RELEASE_DELAY_MS = 300;
// Keys whose values are base64 image/audio payloads, not searchable text.
const BINARY_KEY = /b64|base64|^(images?|audio|video)$/i;
-// Drop a trailing __MCP_IMAGES__ envelope only when it is the valid JSON image array the
-// backend appended, so tool text merely mentioning the marker stays searchable.
-function stripMcpImageSuffix(value: string): string {
- const marker = "\n__MCP_IMAGES__:";
- const idx = value.lastIndexOf(marker);
- if (idx === -1) return value;
- try {
- const images: unknown = JSON.parse(value.slice(idx + marker.length));
- if (
- Array.isArray(images) &&
- images.length > 0 &&
- images.every(
- (img) =>
- typeof img === "object" &&
- img !== null &&
- typeof (img as Record<string, unknown>).data === "string" &&
- typeof (img as Record<string, unknown>).mimeType === "string",
- )
- ) {
- return value.slice(0, idx);
- }
- } catch {
- // Not a valid envelope; leave the text intact.
- }
- return value;
-}
-
// Readable text from tool args/results, dropping base64 image/audio blobs so they never
// bloat the index.
function searchableText(value: unknown, depth = 0): string {
if (typeof value === "string") {
- let text = stripMcpImageSuffix(value);
+ let text = splitMcpImages(value).text;
const cut = text.indexOf("\n__IMAGES__:");
if (cut !== -1) text = text.slice(0, cut);
return text
diff --git a/studio/frontend/src/features/chat/utils/mcp-tool-name.ts b/studio/frontend/src/features/chat/utils/mcp-tool-name.ts
index aeebdc49969..a648de77886 100644
--- a/studio/frontend/src/features/chat/utils/mcp-tool-name.ts
+++ b/studio/frontend/src/features/chat/utils/mcp-tool-name.ts
@@ -3,6 +3,11 @@
const MCP_TOOL_PREFIX = "mcp__";
+/** Whether a tool call came from an MCP server, by the id the backend stamps. */
+export function isMcpToolName(toolName: string | undefined): boolean {
+ return typeof toolName === "string" && toolName.startsWith(MCP_TOOL_PREFIX);
+}
+
function provenanceString(
provenance: unknown,
key: "mcp_server" | "mcp_tool",
diff --git a/studio/frontend/tests/mcp-images.test.ts b/studio/frontend/tests/mcp-images.test.ts
new file mode 100644
index 00000000000..3d3b8216de1
--- /dev/null
+++ b/studio/frontend/tests/mcp-images.test.ts
@@ -0,0 +1,589 @@
+// SPDX-License-Identifier: AGPL-3.0-only
+// Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
+
+import assert from "node:assert/strict";
+import { readFileSync } from "node:fs";
+import test from "node:test";
+
+import {
+ DECODE_FAILURE_ALLOWANCE,
+ MAX_MODEL_IMAGES,
+ MAX_TOTAL_MCP_IMAGES,
+ MAX_MCP_IMAGE_MIME_CHARS,
+ MAX_TOTAL_MCP_IMAGE_CHARS,
+ planMcpImageBound,
+ MCP_IMAGES_MARKER,
+ boundMcpImageEnvelopes,
+ mcpImagesEnvelope,
+ splitMcpImages,
+ stripMcpImageEnvelopes,
+} from "../src/features/chat/api/mcp-images.ts";
+import { providerModelTakesMcpImages } from "../src/features/chat/external-providers.ts";
+import { localToolExchangeIndexes } from "../src/features/chat/codex-reasoning.ts";
+import { isMcpToolName } from "../src/features/chat/utils/mcp-tool-name.ts";
+
+const IMAGES = [{ data: "QUJD", mimeType: "image/png" }];
+const RESULT = `[1 image returned]${mcpImagesEnvelope(IMAGES)}`;
+
+const adapter = readFileSync(
+ new URL("../src/features/chat/api/chat-adapter.ts", import.meta.url),
+ "utf8",
+);
+
+test("a valid envelope splits into the text and its images", () => {
+ assert.deepEqual(splitMcpImages(RESULT), {
+ text: "[1 image returned]",
+ images: IMAGES,
+ });
+});
+
+test("text that only mentions the marker is left whole", () => {
+ const result = `the marker is${MCP_IMAGES_MARKER} and nothing follows`;
+ assert.deepEqual(splitMcpImages(result), { text: result, images: [] });
+});
+
+test("an envelope that is not an image array is left whole", () => {
+ const result = `log${MCP_IMAGES_MARKER}["not", "image", "dicts"]`;
+ assert.deepEqual(splitMcpImages(result), { text: result, images: [] });
+});
+
+test("an unparseable envelope is left whole", () => {
+ const result = `log${MCP_IMAGES_MARKER}{oops`;
+ assert.deepEqual(splitMcpImages(result), { text: result, images: [] });
+});
+
+test("replaying a tool result re-attaches its images for the backend", () => {
+ assert.match(adapter, /content \+= mcpImagesEnvelope\(result\.images\);/);
+});
+
+test("only an MCP tool call carries the privileged envelope", () => {
+ // A client tool is free to answer {text, images:[{data, mimeType}]}, which is
+ // the shape isMcpImageToolResult accepts. Without the provenance gate its bytes
+ // would be promoted into model image input on the next request.
+ assert.equal(isMcpToolName("mcp__fs__read_media_file"), true);
+ assert.equal(isMcpToolName("render_chart"), false);
+ assert.equal(isMcpToolName(undefined), false);
+ // The envelope is gated...
+ assert.match(
+ adapter,
+ /if \(isMcpImageToolResult\(result\) && isMcpToolName\(tc\.toolName\)\) \{\n\s*content \+= mcpImagesEnvelope\(result\.images\);/,
+ );
+ // ...but the wrapper branch is NOT: excluding a non-MCP wrapper there dropped it
+ // into JSON.stringify, which replays the whole base64 array as prompt text.
+ assert.match(adapter, /^\s*isMcpImageToolResult\(result\) \|\|$/m);
+});
+
+const shot = (n: number, name = "mcp__fs__screenshot") => ({
+ role: "tool",
+ name,
+ content:
+ `[3 images returned]` +
+ mcpImagesEnvelope([
+ { data: `A${n}`, mimeType: "image/png" },
+ { data: `B${n}`, mimeType: "image/png" },
+ { data: `C${n}`, mimeType: "image/png" },
+ ]),
+});
+
+const countImages = (messages: { content?: unknown }[]) =>
+ messages.reduce(
+ (n, m) =>
+ n +
+ (typeof m.content === "string"
+ ? splitMcpImages(m.content).images.length
+ : 0),
+ 0,
+ );
+
+test("history is bounded before it is uploaded, not after", () => {
+ // The backend's cap runs after the body is parsed, so it cannot bound transport.
+ // The ceiling counts CANDIDATES, which sit above the promotable budget so the
+ // backend can still scan past entries it cannot decode.
+ const messages = [0, 1, 2, 3, 4].map((n) => shot(n));
+
+ const bounded = boundMcpImageEnvelopes(messages);
+
+ assert.equal(countImages(messages), 15);
+ assert.ok(countImages(bounded) < countImages(messages), "nothing was bounded");
+ assert.ok(
+ countImages(bounded) <= MAX_TOTAL_MCP_IMAGES + DECODE_FAILURE_ALLOWANCE,
+ `uploaded ${countImages(bounded)} candidates`,
+ );
+ // The oldest result is still dropped outright.
+ assert.equal(splitMcpImages(bounded[0].content).images.length, 0);
+});
+
+test("the newest pictures are the ones kept", () => {
+ const bounded = boundMcpImageEnvelopes([0, 1, 2, 3, 4].map((n) => shot(n)));
+
+ assert.equal(splitMcpImages(bounded[4].content).images.length, 3);
+ assert.equal(splitMcpImages(bounded[0].content).images.length, 0);
+ // The note survives even when its payload does not.
+ assert.match(bounded[0].content, /^\[3 images returned\]$/);
+});
+
+test("a conversation inside the budget is left byte-identical", () => {
+ const messages = [0, 1].map((n) => shot(n));
+
+ assert.deepEqual(boundMcpImageEnvelopes(messages), messages);
+});
+
+test("a non-MCP tool result keeps its text and loses only the envelope", () => {
+ // The backend never promotes a named non-MCP result and strips its envelope
+ // regardless, so leaving it whole here only re-uploaded megabytes of base64 on
+ // every later turn. The text the model reads is unchanged.
+ const message = {
+ role: "tool",
+ name: "read_file",
+ content: "plain text" + mcpImagesEnvelope([{ data: "QUJD", mimeType: "image/png" }]),
+ };
+
+ const [bounded] = boundMcpImageEnvelopes([message]);
+
+ assert.equal(bounded.content, "plain text");
+ assert.equal(bounded.name, "read_file");
+});
+
+const shotOf = (n: string, count: number) => ({
+ role: "tool",
+ name: "mcp__fs__screenshot",
+ content:
+ `[${count} images returned]` +
+ mcpImagesEnvelope(
+ Array.from({ length: count }, (_, i) => ({
+ data: `${n}${i}`,
+ mimeType: "image/png",
+ })),
+ ),
+});
+
+test("a fat newest result does not evict older usable images", () => {
+ // Budget is charged at the promotable rate, so an oversized newest envelope
+ // cannot spend the whole history allowance on images the backend would drop.
+ const bounded = boundMcpImageEnvelopes([shotOf("old", 4), shotOf("new", 8)]);
+
+ assert.equal(splitMcpImages(bounded[0].content).images.length, 4);
+});
+
+test("no single result uploads more than the backend could ever decode", () => {
+ // Bounded, but above the quota: the backend counts successful decodes and this
+ // side cannot tell which entries will decode.
+ const bounded = boundMcpImageEnvelopes([shotOf("only", 12)]);
+
+ assert.equal(
+ splitMcpImages(bounded[0].content).images.length,
+ MAX_MODEL_IMAGES + DECODE_FAILURE_ALLOWANCE,
+ );
+});
+
+test("candidates survive the transport bound for the backend to decode", () => {
+ // The backend's quota counts SUCCESSFUL decodes and scans past failures. This
+ // side cannot decode, so cutting at the quota drops valid PNGs sitting behind
+ // formats Pillow rejects -- shown on the first turn, lost on the replay.
+ const bounded = boundMcpImageEnvelopes([shotOf("a", 8)]);
+
+ assert.equal(
+ splitMcpImages(bounded[0].content).images.length,
+ MAX_MODEL_IMAGES + DECODE_FAILURE_ALLOWANCE,
+ );
+});
+
+test("the spare candidates do not evict an older result", () => {
+ const bounded = boundMcpImageEnvelopes([shotOf("old", 4), shotOf("new", 8)]);
+
+ assert.equal(splitMcpImages(bounded[0].content).images.length, 4);
+});
+
+test("a partly-spent budget still leaves an older result its decode spares", () => {
+ // A newer one-image result leaves 7 of 8. The older result has corrupt entries
+ // ahead of valid PNGs, so slicing it to the remaining budget alone strands the
+ // last valid one even though both quotas had room.
+ const bounded = boundMcpImageEnvelopes([shotOf("old", 8), shotOf("new", 1)]);
+
+ assert.equal(
+ splitMcpImages(bounded[0].content).images.length,
+ MAX_MODEL_IMAGES + DECODE_FAILURE_ALLOWANCE,
+ );
+ assert.equal(splitMcpImages(bounded[1].content).images.length, 1);
+});
+
+test("the decode-failure allowance survives an undecodable newest result", () => {
+ // Four entries the frontend cannot tell apart from PNGs, then two results of four
+ // real ones. Per-result, the first spends the whole room and the oldest envelope is
+ // dropped outright, so the backend can find only four decodable pictures out of the
+ // eight the cap allows.
+ const undecodable = Array.from({ length: 4 }, (_, i) => ({
+ data: `SVG${i}`,
+ mimeType: "image/svg+xml",
+ }));
+ const valid = (tag: string) =>
+ Array.from({ length: 4 }, (_, i) => ({ data: `${tag}${i}`, mimeType: "image/png" }));
+
+ const messages = [
+ { role: "tool", name: "mcp__s__a", content: "oldest" + mcpImagesEnvelope(valid("A")) },
+ { role: "tool", name: "mcp__s__b", content: "middle" + mcpImagesEnvelope(valid("B")) },
+ { role: "tool", name: "mcp__s__c", content: "newest" + mcpImagesEnvelope(undecodable) },
+ ];
+
+ const bounded = boundMcpImageEnvelopes(messages);
+ const kept = bounded.map((m) => splitMcpImages(m.content as string).images);
+
+ assert.equal(kept[2].length, 4, "the newest result is kept as candidates");
+ assert.equal(kept[1].length, 4, "the middle result keeps its four real pictures");
+ assert.equal(
+ kept[0].length,
+ 4,
+ "the oldest result still ships candidates on the shared allowance",
+ );
+
+ const total = kept.reduce((n, images) => n + images.length, 0);
+ assert.ok(
+ total <= MAX_TOTAL_MCP_IMAGES + DECODE_FAILURE_ALLOWANCE,
+ `bounded: ${total} candidates uploaded`,
+ );
+});
+
+test("a replayed history is bounded by bytes, not only by image count", () => {
+ // One authentic result may carry MAX_IMAGE_PAYLOAD_CHARS (12M) of base64, so the
+ // twelve candidates the count bound allows could re-upload ~144MB on every later
+ // turn of an image-heavy chat.
+ const huge = "A".repeat(4_000_000);
+ const messages = Array.from({ length: 6 }, (_, i) => ({
+ role: "tool",
+ name: `mcp__s__shot${i}`,
+ content:
+ `result ${i}` +
+ mcpImagesEnvelope([{ data: huge, mimeType: "image/png" }]),
+ }));
+
+ const bounded = boundMcpImageEnvelopes(messages);
+ const kept = bounded.flatMap((m) => splitMcpImages(m.content as string).images);
+ const chars = kept.reduce((n, image) => n + image.data.length, 0);
+
+ assert.ok(
+ chars <= MAX_TOTAL_MCP_IMAGE_CHARS,
+ `${chars} characters uploaded against a budget of ${MAX_TOTAL_MCP_IMAGE_CHARS}`,
+ );
+ assert.ok(kept.length > 0, "the budget must not starve the newest result");
+ // The newest results are the ones kept, as everywhere else here.
+ const newest = splitMcpImages(bounded[5].content as string).images;
+ assert.equal(newest.length, 1, "the newest result keeps its picture");
+ assert.equal(
+ splitMcpImages(bounded[0].content as string).images.length,
+ 0,
+ "the oldest gives its bytes up first",
+ );
+});
+
+test("an ordinary conversation is untouched by the byte budget", () => {
+ const small = "B".repeat(2048);
+ const messages = Array.from({ length: 2 }, (_, i) => ({
+ role: "tool",
+ name: `mcp__s__shot${i}`,
+ content: `r${i}` + mcpImagesEnvelope([{ data: small, mimeType: "image/png" }]),
+ }));
+
+ const bounded = boundMcpImageEnvelopes(messages);
+
+ assert.deepEqual(bounded, messages, "nothing was rewritten");
+});
+
+test("an oversized replay image is skipped, not the rest of its result", () => {
+ // Breaking on the first candidate that does not fit threw away three 1MB pictures
+ // sitting behind a 5MB one, which the backend could have replayed.
+ const newest = "N".repeat(9_000_000);
+ const oversized = "X".repeat(5_000_000);
+ const small = (tag: string) => tag.repeat(900_000);
+
+ const messages = [
+ {
+ role: "tool",
+ name: "mcp__s__old",
+ content:
+ "old" +
+ mcpImagesEnvelope([
+ { data: oversized, mimeType: "image/png" },
+ { data: small("a"), mimeType: "image/png" },
+ { data: small("b"), mimeType: "image/png" },
+ ]),
+ },
+ { role: "tool", name: "mcp__s__new", content: "new" + mcpImagesEnvelope([{ data: newest, mimeType: "image/png" }]) },
+ ];
+
+ const bounded = boundMcpImageEnvelopes(messages);
+ const older = splitMcpImages(bounded[0].content as string).images;
+
+ assert.ok(
+ !older.some((image) => image.data === oversized),
+ "the oversized candidate should not fit",
+ );
+ assert.equal(older.length, 2, "the two that DO fit must survive it");
+ const chars = bounded
+ .flatMap((m) => splitMcpImages(m.content as string).images)
+ .reduce((n, image) => n + image.data.length, 0);
+ assert.ok(chars <= MAX_TOTAL_MCP_IMAGE_CHARS, `${chars} characters uploaded`);
+});
+
+test("a bounded envelope records how many the tool returned", () => {
+ const images = Array.from({ length: 20 }, (_, i) => ({ data: `IMG${i}`, mimeType: "image/png" }));
+ const [bounded] = boundMcpImageEnvelopes([
+ { role: "tool", name: "mcp__s__shot", content: "r" + mcpImagesEnvelope(images) },
+ ]);
+ const kept = splitMcpImages(bounded.content as string).images;
+ assert.ok(kept.length < 20, "the bound must actually have cut");
+ assert.equal(kept[0].returned, 20, "the original count survives on the first entry");
+});
+
+test("a named non-MCP result has its envelope stripped, not re-uploaded", () => {
+ const huge = "Z".repeat(3_000_000);
+ const [bounded] = boundMcpImageEnvelopes([
+ { role: "tool", name: "read_file", content: "here" + mcpImagesEnvelope([{ data: huge, mimeType: "image/png" }]) },
+ ]);
+ assert.equal(bounded.content, "here");
+});
+
+test("a text-only target is sent no envelopes at all", () => {
+ const messages = [
+ { role: "user", content: "what did it show" },
+ {
+ role: "tool",
+ name: "mcp__s__shot",
+ content: "[1 image returned]" + mcpImagesEnvelope([{ data: "A".repeat(50_000), mimeType: "image/png" }]),
+ },
+ { role: "tool", name: "read_file", content: "plain" },
+ ];
+
+ const stripped = stripMcpImageEnvelopes(messages);
+
+ assert.equal(stripped[1].content, "[1 image returned]");
+ assert.equal(stripped[0], messages[0], "non-tool messages are the same object");
+ assert.equal(stripped[2], messages[2], "a tool result with no envelope is untouched");
+});
+
+const round = (n: number, results = 1, perResult = 3) => [
+ { role: "assistant", content: `call ${n}` },
+ ...Array.from({ length: results }, (_, k) => ({
+ role: "tool",
+ name: "mcp__fs__screenshot",
+ content:
+ `[${perResult} images returned]` +
+ mcpImagesEnvelope(
+ Array.from({ length: perResult }, (_, m) => ({
+ data: `R${n}K${k}M${m}`,
+ mimeType: "image/png",
+ })),
+ ),
+ })),
+ { role: "assistant", content: `answer ${n}` },
+];
+
+const imagesPerToolResult = (messages: { role?: string; content?: unknown }[]) =>
+ messages
+ .filter((m) => m.role === "tool" && typeof m.content === "string")
+ .map((m) => splitMcpImages(m.content as string).images.length);
+
+test("a marker target is charged one picture per round, so every round keeps one", () => {
+ // Eight rounds of three: the backend's local path replays one per round, eight in
+ // all. Four per result spent the budget on the newest rounds and stripped the rest.
+ const messages = Array.from({ length: 8 }, (_, n) => round(n)).flat();
+
+ const parts = imagesPerToolResult(boundMcpImageEnvelopes(messages));
+ assert.ok(parts.some((n) => n === 0), `default still strips old rounds: ${parts}`);
+
+ const markers = imagesPerToolResult(
+ boundMcpImageEnvelopes(messages, { localMarkers: true }),
+ );
+ assert.equal(markers.length, 8);
+ assert.ok(markers.every((n) => n >= 1), `every round keeps a candidate: ${markers}`);
+ // Spares ride with the newest rounds only; the rest carry exactly their one.
+ assert.equal(markers[markers.length - 1], 3);
+ assert.equal(markers[0], 1);
+});
+
+test("consecutive results are one batch on a marker target and share one charge", () => {
+ // Three parallel results land as one turn carrying one picture: charged once,
+ // not three times, so eight such rounds all keep a candidate.
+ const messages = Array.from({ length: 8 }, (_, n) => round(n, 3, 2)).flat();
+
+ const markers = imagesPerToolResult(
+ boundMcpImageEnvelopes(messages, { localMarkers: true }),
+ );
+ assert.equal(markers.length, 24);
+ for (let n = 0; n < 8; n++) {
+ const batch = markers.slice(n * 3, n * 3 + 3);
+ assert.ok(batch.some((k) => k >= 1), `round ${n} keeps a candidate: ${batch}`);
+ }
+});
+
+test("the send path bounds the run's own results before serializing them", () => {
+ // Decided before the slice tests/studio executes standalone, applied to the raw
+ // results, so no envelope the request will not carry is ever built or parsed back.
+ assert.match(
+ adapter,
+ /const mcpImagesLocalMarkers =\n\s*!isExternalRequest &&\n\s*runtime\.models\.find\(\(model\) => model\.id === runtime\.params\.checkpoint\)\n\s*\?\.isGguf === false;/,
+ );
+ assert.match(
+ adapter,
+ /const messages = boundMcpImageResults\(rawMessages, \{\n\s*readsImages: targetReadsImages,\n\s*localMarkers: mcpImagesLocalMarkers,\n\s*\}\);\n\s*const survivingMessages = pruneOutboundHistory\(\n\s*messages,/,
+ );
+ assert.doesNotMatch(adapter, /boundMcpImageEnvelopes\(outboundMessages/);
+ assert.doesNotMatch(adapter, /stripMcpImageEnvelopes\(outboundMessages/);
+});
+
+test("the upload gate is the backend's external MCP gate, not provider-level vision", () => {
+ // A mixed catalog says nothing about most models: the backend sends no picture
+ // there, so uploading the envelopes only had it strip 12 MB again on every turn.
+ assert.equal(providerModelTakesMcpImages("openrouter", "some/unknown-model"), false);
+ assert.equal(providerModelTakesMcpImages("huggingface", "org/unknown"), false);
+ assert.equal(providerModelTakesMcpImages("qwen", "qwen2.5-72b-instruct"), false);
+ // Known text-only endpoints stay a no; a vision family with a plain catalog is a yes.
+ assert.equal(providerModelTakesMcpImages("mistral", "mistral-large"), false);
+ assert.equal(providerModelTakesMcpImages("anthropic", "claude-x"), true);
+ // Unknown provider type keeps them: the backend decides.
+ assert.equal(providerModelTakesMcpImages(undefined, undefined), true);
+});
+
+test("both local paths read the model's vision flag, not just multimodal", () => {
+ assert.match(
+ adapter,
+ /function localTargetReadsImages\([\s\S]*?if \(typeof activeModel\?\.isVision === "boolean"\) return activeModel\.isVision;\n\s*return state\.loadedIsMultimodal !== false;/,
+ );
+ assert.match(adapter, /: localTargetReadsImages\(runtime\);/);
+ assert.match(
+ adapter,
+ /const messages = boundMcpImageResults\(rawMessages, \{\n\s*readsImages: localTargetReadsImages\(runtimeState\),/,
+ );
+ assert.match(
+ adapter,
+ /\? providerModelTakesMcpImages\(\n\s*externalProvider\?\.providerType,\n\s*externalSelection\?\.modelId,\n\s*\)/,
+ );
+});
+
+test("an entry with an unbounded mimeType is dropped, not charged", () => {
+ // A token subtype has no length bound, so a tiny picture can carry megabytes of
+ // mimeType; charging data alone let that envelope through on every turn.
+ const heavy = (n: number) => ({
+ role: "tool",
+ name: "mcp__fs__screenshot",
+ content:
+ "[1 image returned]" +
+ mcpImagesEnvelope([
+ { data: `A${n}`, mimeType: `image/${"x".repeat(MAX_MCP_IMAGE_MIME_CHARS + 1)}` },
+ { data: `B${n}`, mimeType: "image/png" },
+ ]),
+ });
+ const bounded = boundMcpImageEnvelopes([0, 1].map(heavy));
+ for (const message of bounded) {
+ const kept = splitMcpImages(message.content).images;
+ assert.deepEqual(
+ kept.map((image) => image.mimeType),
+ ["image/png"],
+ "the bounded-mime entry stays, the unbounded one goes",
+ );
+ }
+});
+
+test("a picture the live turn accepted at the limit still fits its own replay", () => {
+ // The live limit is on data; charging the serialized entry pushed a picture of
+ // exactly that size over the identically sized replay budget, and the only
+ // picture in the conversation vanished from the next request.
+ const messages = [
+ {
+ role: "tool",
+ name: "mcp__fs__screenshot",
+ content:
+ "[1 image returned]" +
+ mcpImagesEnvelope([
+ { data: "A".repeat(MAX_TOTAL_MCP_IMAGE_CHARS), mimeType: "image/png" },
+ ]),
+ },
+ ];
+ const [bounded] = boundMcpImageEnvelopes(messages);
+ assert.equal(splitMcpImages(bounded.content).images.length, 1);
+});
+
+test("the byte filter scans past candidates that do not fit", () => {
+ // A newer result leaves about 1 MB. The older result's first eight candidates are
+ // 1.4 MB each; slicing to the allowance first inspected only those and dropped the
+ // small ninth, which fits and which the backend could have replayed.
+ const big = (tag: string, n: number, size: number) =>
+ Array.from({ length: n }, (_, i) => ({ data: tag.repeat(size / tag.length) + i, mimeType: "image/png" }));
+ const messages = [
+ {
+ role: "tool",
+ name: "mcp__fs__a",
+ content: "older" + mcpImagesEnvelope([...big("O", 8, 1_400_000), { data: "tiny", mimeType: "image/png" }]),
+ },
+ { role: "assistant", content: "next" },
+ {
+ role: "tool",
+ name: "mcp__fs__b",
+ content: "newer" + mcpImagesEnvelope(big("N", 1, 11_000_000)),
+ },
+ ];
+ const bounded = boundMcpImageEnvelopes(messages);
+ const older = splitMcpImages(bounded[0].content).images;
+ assert.deepEqual(older.map((image) => image.data), ["tiny"]);
+});
+
+test("the planner is the bound the envelope form applies", () => {
+ // Both carriers plan through one function, so the raw-result bound the send path
+ // takes and the envelope bound agree result for result.
+ const rounds = Array.from({ length: 8 }, (_, n) => round(n, 3, 2)).flat();
+ const envelopes = boundMcpImageEnvelopes(rounds, { localMarkers: true });
+ const viaEnvelopes = imagesPerToolResult(envelopes);
+ const batches = Array.from({ length: 8 }, (_, n) =>
+ rounds
+ .slice(n * 5 + 1, n * 5 + 4)
+ .map((m) => splitMcpImages(m.content as string).images),
+ );
+ const viaPlanner = planMcpImageBound(batches, { localMarkers: true })
+ .flat()
+ .map((kept) => kept.length);
+ assert.deepEqual(viaPlanner, viaEnvelopes);
+ // A result that keeps nothing is an empty plan, which the raw bound turns into text.
+ const dropped = planMcpImageBound([[[{ data: "x", mimeType: "image/png" }]], [[Array.from({ length: 12 }, (_, i) => ({ data: `${i}`, mimeType: "image/png" }))].flat()]]);
+ assert.equal(dropped[0][0].length, 1);
+});
+
+test("a message's results are batched by replay exchange, not as one block", () => {
+ // Three local rounds accumulate in one assistant message, told apart by round_id;
+ // the serializer emits them as three exchanges, so the bound treats them as three
+ // batches (one picture each), not one batch sharing a single picture's allowance.
+ type P = { round: number | null; flush: boolean };
+ const parts: P[] = [
+ { round: 0, flush: false },
+ { round: 0, flush: false },
+ { round: 1, flush: false },
+ { round: 2, flush: false },
+ { round: null, flush: true },
+ { round: null, flush: true },
+ ];
+ const indexes = localToolExchangeIndexes(
+ parts,
+ (p) => p.round,
+ (p) => p.flush,
+ );
+ assert.deepEqual(indexes, [0, 0, 1, 2, 3, 4]);
+ // The bound's partition uses the serializer's own conditions.
+ assert.match(adapter, /startsNewCodexToolRound\(pendingLocalToolRoundId, localRoundId\)/);
+ assert.match(adapter, /localRoundId === null && shouldFlushCompletedLocalToolPair\(toolPart\)/);
+ assert.match(
+ adapter,
+ /localToolExchangeIndexes\(\n\s*toolParts,\n\s*\(\{ part \}\) => codexLocalToolRoundId\(getToolReplayProvenance\(part\)\),\n\s*\(\{ part \}\) => shouldFlushCompletedLocalToolPair\(part\),/,
+ );
+});
+
+test("a client tool's structured result is not unwrapped as the MCP wrapper", () => {
+ // Unwrapping by shape alone reduced {text, images, ...} from a non-MCP client tool
+ // to its text, silently dropping every other field. The bare live-parser wrapper
+ // is still unwrapped, since JSON of it would replay base64 as prompt text.
+ assert.match(
+ adapter,
+ /\(isMcpImageToolResult\(result\) &&\n\s*\(isMcpToolName\(tc\.toolName\) \|\| isBareMcpImageWrapper\(result\)\)\) \|\|/,
+ );
+ assert.match(
+ adapter,
+ /export function isBareMcpImageWrapper\(val: unknown\): boolean \{\n\s*if \(!isMcpImageToolResult\(val\)\) return false;\n\s*const keys = Object\.keys\(val as object\)\.filter\(\(key\) => key !== "text" && key !== "images"\);\n\s*return keys\.length === 0;/,
+ );
+});
diff --git a/tests/studio/test_backend_ci_parallel_isolation.py b/tests/studio/test_backend_ci_parallel_isolation.py
index 3a3a7edc750..be7970e2bf0 100644
--- a/tests/studio/test_backend_ci_parallel_isolation.py
+++ b/tests/studio/test_backend_ci_parallel_isolation.py
@@ -177,6 +177,7 @@ BACKEND_ISOLATED = [
),
("tests/test_web_fetch_extraction.py", "compares parse time at two input sizes"),
("tests/test_tool_call_parser_strict.py", "compares parse time at two nesting depths"),
+ ("tests/test_pr5624_regressions.py", "R1 parser's 1s bound exceeded under CPU contention"),
# Found by staging rather than by the scan, and the scan cannot find it: see below.
(
"tests/test_tunnel_safe_long_post.py",
diff --git a/tests/studio/test_deep_research_frontend_contract.py b/tests/studio/test_deep_research_frontend_contract.py
index 54d44b6856e..e3495a51b84 100644
--- a/tests/studio/test_deep_research_frontend_contract.py
+++ b/tests/studio/test_deep_research_frontend_contract.py
@@ -143,9 +143,9 @@ def test_research_presave_keeps_the_follow_up_parent() -> None:
"const createdRun = await createResearchRun({", 1
)[0]
- assert "const userMessageIndex = messages.indexOf(userMessage);" in presave
+ assert "const userMessageIndex = rawMessages.indexOf(userMessage);" in presave
assert "const userMessageParentId =" in presave
- assert "userMessageIndex > 0 ? messages[userMessageIndex - 1]!.id : null" in presave
+ assert "userMessageIndex > 0 ? rawMessages[userMessageIndex - 1]!.id : null" in presave
# A stored null is an edited root; `??` would reparent it under the predecessor.
assert "storedUserMessage && storedUserMessage.parentId !== undefined" in presave
assert "? storedUserMessage.parentId" in presave
diff --git a/tests/studio/test_token_count_prompt_parity.py b/tests/studio/test_token_count_prompt_parity.py
index 28695ef4b9d..4d3f9f0388e 100644
--- a/tests/studio/test_token_count_prompt_parity.py
+++ b/tests/studio/test_token_count_prompt_parity.py
@@ -122,6 +122,7 @@ HARNESS = """
// Fixtures the sliced builder reads through. Everything below the PRELUDE marker is
// copied verbatim out of studio/frontend/src/features/chat/api/chat-adapter.ts.
const state: any = {
+ models: [],
params: { systemPrompt: "", systemVariables: "" },
artifactsEnabled: false,
supportsTools: false,
diff --git a/tests/test_ci_backend_pytest_shards.py b/tests/test_ci_backend_pytest_shards.py
index d70b0e9ec7e..0b3fd9d78c7 100644
--- a/tests/test_ci_backend_pytest_shards.py
+++ b/tests/test_ci_backend_pytest_shards.py
@@ -347,14 +347,12 @@ class TestTheSplitKeepsTheSelectionItInherited:
return
raise AssertionError(f"the {_SERIAL_STEP!r} step is gone")
- def test_the_serial_step_still_runs_the_same_twelve_files(self):
- """Not eleven and not thirteen. Whether any of them could go back into the parallel
- run was audited separately and the answer was no, so the set is carried across the
- split unchanged; a file leaving it here is a coverage change wearing a refactor's
- clothes."""
+ def test_the_serial_step_runs_the_thirteen_isolated_files(self):
+ """The serial selection includes the timing-sensitive R1 parser regressions."""
paths = _serial_paths()
- assert len(paths) == 12, f"the serial step runs {len(paths)} files, not 12: {paths}"
- assert len(set(paths)) == 12, f"the serial step names a file twice: {paths}"
+ assert len(paths) == 13, f"the serial step runs {len(paths)} files, not 13: {paths}"
+ assert len(set(paths)) == 13, f"the serial step names a file twice: {paths}"
+ assert "tests/test_pr5624_regressions.py" in paths
missing = [path for path in paths if not (_BACKEND_TESTS.parent / path).is_file()]
assert not missing, f"the serial step names files that do not exist: {missing}"