Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

Studio: let small models send an attached image to a mapped MCP tool (#13166)

Etherlebc871108da1
diff --git a/studio/backend/core/inference/llama_cpp.py b/studio/backend/core/inference/llama_cpp.py
index 0b831df1205..8047d2113e1 100644
--- a/studio/backend/core/inference/llama_cpp.py
+++ b/studio/backend/core/inference/llama_cpp.py
@@ -36917,8 +36917,10 @@ class LlamaCppBackend:
has_text_only_provisional_card,
is_high_risk_tool_call,
mcp_image_share,
+ mcp_image_targets,
never_needs_approval,
)
+ from core.inference.mcp_image import note_attached_image
# "full" and bypass_permissions are the same switch, whichever arrives
# first wins. "off" keeps the sandbox but never prompts. Unset defaults to
@@ -36944,6 +36946,10 @@ class LlamaCppBackend:
if not self.is_loaded:
raise RuntimeError("llama-server is not loaded")
+ if mcp_image is not None:
+ messages = note_attached_image(
+ messages, mcp_image_targets(_gguf_active_tool_names(tools))
+ )
conversation = list(messages)
# Seeded with tool results an explicit skill load relies on, so fitting keeps them.
_rolling_anchor_ids: set[int] = set(instruction_anchor_ids or ())
@@ -39152,6 +39158,18 @@ class LlamaCppBackend:
provisional = provisional_match,
allowed_tool_names = {forced_name} if forced_name else None,
)
+ image_share = None
+ if decision.should_execute and mcp_image is not None:
+ image_share = mcp_image_share(
+ decision.tool_name, decision.arguments, mcp_image
+ )
+ if image_share is not None:
+ decision = tool_controller.reprepare_call(
+ decision,
+ forced = _forced_tool_call_pending,
+ provisional = provisional_match,
+ allowed_tool_names = {forced_name} if forced_name else None,
+ )
if not decision.should_execute:
if provisional_match:
@@ -39207,7 +39225,6 @@ class LlamaCppBackend:
sandbox_level = sandbox_level,
)
# Sending the user's image always asks, whatever the permission mode.
- image_share = mcp_image_share(decision.tool_name, decision.arguments, mcp_image)
needs_confirm = needs_confirm or image_share is not None
strict_isolation = requires_os_isolation(
confirm_tool_calls = bool(confirm_tool_calls),
diff --git a/studio/backend/core/inference/mcp_image.py b/studio/backend/core/inference/mcp_image.py
index 355016a73ef..956e575808a 100644
--- a/studio/backend/core/inference/mcp_image.py
+++ b/studio/backend/core/inference/mcp_image.py
@@ -180,7 +180,7 @@ def _input_schema(tool: dict) -> dict:
def image_mapping(server: dict, tool: Optional[dict]) -> Optional[dict]:
- """The server's mapping for ``tool`` while its field is still a top-level string."""
+ """return the server mapping only while its field remains a top-level string."""
if not tool:
return None
for mapping in image_input_mappings(server):
@@ -193,8 +193,79 @@ def image_mapping(server: dict, tool: Optional[dict]) -> Optional[dict]:
return None
+def _loose(value: str) -> str:
+ return re.sub(r"[\s-]+", "_", value.strip(" \"'`<>[]{}").lower())
+
+
+def _names_the_image(value) -> bool:
+ """match only the placeholder or a path or URL ending in it, never a mention."""
+ return (
+ isinstance(value, str)
+ and _loose(re.split(r"[/\\]", value.strip().rstrip("/\\"))[-1]) == ATTACHED_IMAGE
+ )
+
+
+def settle_image_call(
+ arguments: dict,
+ field: str,
+ required = (),
+) -> bool:
+ """normalize small-model image arguments in place so approval shows the exact outgoing call."""
+ value = arguments.get(field)
+ if not (
+ value is None or value == "" or isinstance(value, str) and _loose(value) == ATTACHED_IMAGE
+ ):
+ return False
+ for key in [
+ key
+ for key, other in arguments.items()
+ if key != field and key not in required and _names_the_image(other)
+ ]:
+ del arguments[key]
+ arguments[field] = ATTACHED_IMAGE
+ return True
+
+
+IMAGE_NOTE_PREFIX = "[The user attached an image to this message."
+_IMAGE_NOTE_INSTRUCTION = " You cannot see it. To use it, call "
+
+
+def _is_attached_image_note(text: str) -> bool:
+ return text.startswith(IMAGE_NOTE_PREFIX + _IMAGE_NOTE_INSTRUCTION) and text.endswith(".]")
+
+
+def strip_attached_image_note(text: str) -> str:
+ """remove the synthetic image note before exposing user-authored text."""
+ if _is_attached_image_note(text):
+ return ""
+ head, separator, tail = text.rpartition(f"\n\n{IMAGE_NOTE_PREFIX}")
+ note = IMAGE_NOTE_PREFIX + tail
+ return head if separator and _is_attached_image_note(note) else text
+
+
+def note_attached_image(messages: list, targets: list[tuple[str, str]]) -> list:
+ """tell the model which fields accept the attached image without exposing its bytes."""
+ if not targets:
+ return messages
+ calls = " or ".join(
+ f"{name} with {json.dumps({field: ATTACHED_IMAGE})}" for name, field in targets
+ )
+ note = f"{IMAGE_NOTE_PREFIX}{_IMAGE_NOTE_INSTRUCTION}{calls}.]"
+ for index in range(len(messages) - 1, -1, -1):
+ message = messages[index]
+ if not isinstance(message, dict) or message.get("role") != "user":
+ continue
+ content = message.get("content")
+ if isinstance(content, list):
+ content = [*content, {"type": "text", "text": note}]
+ else:
+ content = f"{content}\n\n{note}" if content else note
+ return [*messages[:index], {**message, "content": content}, *messages[index + 1 :]]
+ return messages
+
+
def public_tool(server: dict, tool: dict) -> dict:
- """``tool`` as the model sees it: a mapped field accepts only the placeholder."""
+ """expose a mapped field to the model as a placeholder-only string."""
mapping = image_mapping(server, tool)
if mapping is None:
return tool
@@ -204,7 +275,8 @@ def public_tool(server: dict, tool: dict) -> dict:
"enum": [ATTACHED_IMAGE],
"description": (
f'Pass "{ATTACHED_IMAGE}" to send the image the user attached to their latest '
- "message. Studio inserts it after the user approves; never pass image data or a URL."
+ "message. Unsloth Studio inserts it after the user approves; never pass image data, a "
+ "path or a URL."
),
}
public = {k: v for k, v in tool.items() if k not in ("inputSchema", "input_schema")}
diff --git a/studio/backend/core/inference/safetensors_agentic.py b/studio/backend/core/inference/safetensors_agentic.py
index 481d1463e9b..d1710b4eb46 100644
--- a/studio/backend/core/inference/safetensors_agentic.py
+++ b/studio/backend/core/inference/safetensors_agentic.py
@@ -664,6 +664,10 @@ def run_safetensors_tool_loop(
* ``{"type": "tool_end", "tool_name", "tool_call_id", "result"}``
"""
+ if mcp_image is not None:
+ from core.inference.mcp_image import note_attached_image
+ from core.inference.tools import mcp_image_targets
+ messages = note_attached_image(messages, mcp_image_targets(_active_tool_names(tools)))
conversation = list(messages)
# Where the caller's own attachment sits in the seeded sink. The cap is about what
# the loop RE-SENDS, so that entry is never the one it drops -- but it is not the
@@ -1485,6 +1489,17 @@ def run_safetensors_tool_loop(
# consecutive results the same way -- so without the id a batch persisted
# as separate pairs replayed as several pictures on the next request.
decision.provenance["round_id"] = iteration
+ image_share = None
+ if decision.should_execute and mcp_image is not None:
+ from core.inference.tools import mcp_image_share
+
+ image_share = mcp_image_share(
+ decision.tool_name, decision.arguments, mcp_image
+ )
+ if image_share is not None:
+ decision = tool_controller.reprepare_call(
+ decision, provisional = provisional_match
+ )
if not decision.should_execute:
if content_text and not assistant_appended:
@@ -1530,8 +1545,6 @@ def run_safetensors_tool_loop(
# Bypass wins here too, so a direct internal caller with both flags
# never prompts. "auto" pauses only high-risk calls; "off" pauses only a
# high-risk python/terminal call without OS isolation.
- from core.inference.tools import mcp_image_share
-
needs_confirm = needs_tool_confirmation(
confirm_tool_calls = bool(confirm_tool_calls),
bypass_permissions = bypass_permissions,
@@ -1541,7 +1554,6 @@ def run_safetensors_tool_loop(
sandbox_level = sandbox_level,
)
# Sending the user's image always asks, whatever the permission mode.
- image_share = mcp_image_share(decision.tool_name, decision.arguments, mcp_image)
needs_confirm = needs_confirm or image_share is not None
strict_isolation = requires_os_isolation(
confirm_tool_calls = bool(confirm_tool_calls),
diff --git a/studio/backend/core/inference/studio_tool_loop.py b/studio/backend/core/inference/studio_tool_loop.py
index 0613c095f66..9177d4849fe 100644
--- a/studio/backend/core/inference/studio_tool_loop.py
+++ b/studio/backend/core/inference/studio_tool_loop.py
@@ -56,6 +56,7 @@ from core.inference.tool_call_parser import (
strip_tool_markup,
)
from core.inference.mcp_images import append_image_turn as append_mcp_image_turn
+from core.inference.mcp_image import note_attached_image
def _append_mcp_images_owned(
@@ -64,13 +65,7 @@ def _append_mcp_images_owned(
owned,
lead = None,
):
- """append_image_turn with the loop's own part list, for asyncio.to_thread.
-
- Reserving here and not on the GGUF loop: this one talks to a remote provider that
- applies its own per-request image cap in document order, so an attachment beside a
- full allowance of tool results silently loses the newest result -- the one the
- model just asked for. llama-server is local and answers to the context window.
- """
+ """reserve caller images because remote providers apply the image cap in document order."""
append_mcp_image_turn(
conversation,
results,
@@ -102,6 +97,7 @@ from core.inference.tools import (
execute_tool,
is_high_risk_tool_call,
mcp_image_share,
+ mcp_image_targets,
never_needs_approval,
)
from state.tool_approvals import (
@@ -306,6 +302,25 @@ def _normalized_call(call: dict[str, Any], fallback_id: str = "") -> dict[str, A
return normalized
+def _signed_provider_call_for_replay(call: dict[str, Any]) -> dict[str, Any] | None:
+ extra = call.get("extra_content")
+ google = extra.get("google") if isinstance(extra, dict) else None
+ if not isinstance(google, dict) or not google.get("thought_signature"):
+ return None
+ function = call.get("function")
+ if not isinstance(function, dict):
+ return None
+ return {
+ "id": call["id"],
+ "type": "function",
+ "function": {
+ "name": function["name"],
+ "arguments": function.get("arguments", ""),
+ },
+ "extra_content": extra,
+ }
+
+
def _argument_fragment(value: Any) -> Any:
"""A decoded-object ``arguments`` delta as the text it would have streamed as. llama-server has
shipped a decoded object where a string fragment belongs (ggml-org/llama.cpp#20198).
@@ -1390,15 +1405,15 @@ async def stream_with_studio_tools(
cancel_event: threading.Event,
mcp_image = None,
) -> AsyncIterator[str]:
- """Stream a provider, execute requested Unsloth tools, continue to a final answer."""
conversation = [dict(message) for message in run.messages]
+ if mcp_image is not None:
+ targets = await asyncio.to_thread(mcp_image_targets, sorted(_tool_names(policy.tools)))
+ conversation = note_attached_image(conversation, targets)
openai_compaction: tuple[list[dict[str, Any]], str] | None = None
resumes_partial = run.continue_final_message
- # The image parts this run appends, so its cap never counts a caller's own
- # attachments. Run-scoped, not turn-scoped: the cap is across the whole loop,
- # and seeded with what promotion already put in the conversation.
+ # cap run-owned image parts across the full loop without counting caller attachments.
loop_mcp_image_parts: list = list(run.promoted_image_parts)
- # Kept before the loop appends anything: this is the branch the request is on.
+ # preserve the request branch for tools that search conversation history.
request_branch = list(run.messages)
remaining = policy.max_calls
unlimited = remaining >= 9999
@@ -1840,6 +1855,7 @@ async def stream_with_studio_tools(
break
# Before the gate: an exhausted call is replayed too, and only the decision's replay is guaranteed to parse.
decision = controller.prepare_call(call)
+ signed_provider_call = _signed_provider_call_for_replay(call)
if not unlimited and remaining <= 0:
# Budget spent.
for card_line in _unrun_call_card(
@@ -1853,9 +1869,9 @@ async def stream_with_studio_tools(
# The result below has to be replayed with its call: only the call that spent the last slot reaches
# assistant_tool_calls further down, so this one would arrive as an orphan role="tool" message and
# OpenAI, Anthropic and Gemini all reject that history instead of answering.
- exhausted_call = decision.as_assistant_tool_call()
+ exhausted_call = signed_provider_call or decision.as_assistant_tool_call()
exhausted_extra = call.get("extra_content")
- if isinstance(exhausted_extra, dict) and exhausted_extra:
+ if signed_provider_call is None and isinstance(exhausted_extra, dict) and exhausted_extra:
exhausted_call["extra_content"] = exhausted_extra
assistant_tool_calls.append(exhausted_call)
tool_messages.append(
@@ -1870,10 +1886,19 @@ async def stream_with_studio_tools(
# The frontend groups a round's reasoning by this id (codexLocalToolRoundId), so every tool card the loop
# emits has to carry it, including the budget-exhausted card above.
decision.provenance["round_id"] = round_id
+ image_share = None
+ if decision.should_execute and mcp_image is not None:
+ image_share = await asyncio.to_thread(
+ mcp_image_share, decision.tool_name, decision.arguments, mcp_image
+ )
+ if image_share is not None:
+ decision = controller.reprepare_call(decision)
if not decision.should_execute:
completion = controller.record_noop(decision)
if getattr(transport, "tool_result_only_continuation", False):
- assistant_tool_calls.append(decision.as_assistant_tool_call())
+ assistant_tool_calls.append(
+ signed_provider_call or decision.as_assistant_tool_call()
+ )
tool_messages.append(
{
"role": "tool",
@@ -1903,10 +1928,9 @@ async def stream_with_studio_tools(
):
yield card_line
continue
- assistant_call = decision.as_assistant_tool_call()
+ assistant_call = signed_provider_call or decision.as_assistant_tool_call()
call_extra = call.get("extra_content")
- if isinstance(call_extra, dict) and call_extra:
- # Replayed verbatim: Gemini 3 validates the signature that came back with this exact call
+ if signed_provider_call is None and isinstance(call_extra, dict) and call_extra:
assistant_call["extra_content"] = call_extra
assistant_tool_calls.append(assistant_call)
@@ -1926,11 +1950,6 @@ async def stream_with_studio_tools(
sandbox_level = sandbox_level,
)
# Sending the user's image always asks, whatever the permission mode.
- image_share = (
- await asyncio.to_thread(mcp_image_share, name, arguments, mcp_image)
- if mcp_image is not None
- else None
- )
needs_confirmation = needs_confirmation or image_share is not None
strict_isolation = requires_os_isolation(
confirm_tool_calls = confirm_tool_calls,
diff --git a/studio/backend/core/inference/tool_loop_controller.py b/studio/backend/core/inference/tool_loop_controller.py
index 5210e8675c6..0b0482cee93 100644
--- a/studio/backend/core/inference/tool_loop_controller.py
+++ b/studio/backend/core/inference/tool_loop_controller.py
@@ -1265,6 +1265,33 @@ class ToolLoopController:
noop_result = noop,
)
+ def reprepare_call(
+ self,
+ decision: ToolCallDecision,
+ *,
+ forced: bool = False,
+ provisional: bool = False,
+ allowed_tool_names: Collection[str] | None = None,
+ ) -> ToolCallDecision:
+ """Reclassify a call after a pre-execution normalizer changed its arguments.
+
+ The decision's replay, status and duplicate key must all describe the arguments that
+ execute. A second preparation keeps the ordinary controller rules as the source of truth.
+ """
+ tool_call = decision.as_assistant_tool_call()
+ if decision.card_call_id:
+ tool_call["card_id"] = decision.card_call_id
+ refreshed = self.prepare_call(
+ tool_call,
+ forced = forced,
+ provisional = provisional,
+ allowed_tool_names = allowed_tool_names,
+ )
+ # The second pass receives an argument mapping, so retain provenance from the first parse,
+ # notably whether a local model's bare argument was healed.
+ refreshed.provenance.update(decision.provenance)
+ return refreshed
+
def record_result(self, decision: ToolCallDecision, result: Any) -> ToolCallCompletion:
"""Record a real tool execution and return model/frontend payload helpers."""
result_text = result if isinstance(result, str) else str(result)
diff --git a/studio/backend/core/inference/tools.py b/studio/backend/core/inference/tools.py
index 4748cf829f6..74a626b047e 100644
--- a/studio/backend/core/inference/tools.py
+++ b/studio/backend/core/inference/tools.py
@@ -65,6 +65,8 @@ from core.inference.mcp_image import (
image_input_mappings,
image_mapping,
public_tool,
+ settle_image_call,
+ strip_attached_image_note,
)
from core.inference.mcp_client import (
MCP_TOOL_PREFIX,
@@ -13545,12 +13547,26 @@ def _mcp_image_destination(url: str) -> str:
def mcp_image_share(name, arguments, mcp_image) -> dict | None:
- """Approval-card details plus the image bound to this server when the call would send it, else None."""
+ """Approval-card details plus the image bound to this server when the call would send it, else None.
+
+ A call that would send it has ``arguments`` rewritten in place to what goes out (settle_image_call).
+ """
if mcp_image is None or not isinstance(arguments, dict):
return None
+ from .tool_loop_controller import UNPARSED_ARGUMENTS_KEY # noqa: PLC0415
+
server, tool, tool_name = _mcp_resolve_tool(name)
mapping = image_mapping(server, tool) if server else None
- if mapping is None or arguments.get(mapping["field"]) != ATTACHED_IMAGE:
+ if mapping is None:
+ return None
+ # Arguments that could not be read are not a call to rewrite: they keep their ordinary path.
+ schema = _mcp_input_schema(tool)
+ properties = schema.get("properties") or {}
+ if UNPARSED_ARGUMENTS_KEY in arguments or (
+ set(arguments) == {"raw"} and "raw" not in properties
+ ):
+ return None
+ if not settle_image_call(arguments, mapping["field"], schema.get("required") or ()):
return None
# The fingerprint covers the server's headers, so it stays on the server: only "disclosure" is streamed.
return {
@@ -13573,6 +13589,17 @@ def _mcp_resolve_tool(name) -> "tuple[dict | None, dict | None, str]":
return server, _mcp_cached_tool(server, tool_name) if server else None, tool_name
+def mcp_image_targets(names) -> list[tuple[str, str]]:
+ """(catalog name, field) for each of these tools with a field mapped to the attached image."""
+ targets = []
+ for name in names:
+ server, tool, _ = _mcp_resolve_tool(name)
+ mapping = image_mapping(server, tool) if server else None
+ if mapping:
+ targets.append((name, mapping["field"]))
+ return targets
+
+
def mcp_catalog_takes_image(names) -> bool:
"""Whether any of these catalog tools has a field mapped to the attached image."""
for name in names:
@@ -14698,10 +14725,10 @@ def _last_user_text(conversation: list[dict]) -> str:
continue
content = msg.get("content")
if isinstance(content, str):
- return strip_current_date_update_note(content).strip()
+ return strip_current_date_update_note(strip_attached_image_note(content)).strip()
if isinstance(content, list):
parts = [
- p.get("text", "")
+ strip_attached_image_note(p.get("text", ""))
for p in content
if isinstance(p, dict) and p.get("type") in ("text", "input_text")
]
diff --git a/studio/backend/tests/test_mcp_image.py b/studio/backend/tests/test_mcp_image.py
index da809580aa0..d8bcbe38ee1 100644
--- a/studio/backend/tests/test_mcp_image.py
+++ b/studio/backend/tests/test_mcp_image.py
@@ -18,6 +18,7 @@ from core.inference.mcp_image import (
McpImage,
WITHHELD_RESULT,
McpImageError,
+ note_attached_image,
parse_mcp_image,
public_tool,
)
@@ -358,6 +359,132 @@ def test_unmapped_and_literal_arguments_take_the_ordinary_path(mapped_server):
)
+def test_the_model_is_told_the_image_is_attached_and_where_it_goes():
+ # The image never reaches the model, so without this note small models ask the user to attach one.
+ history = [
+ {"role": "user", "content": "earlier"},
+ {"role": "assistant", "content": "ok"},
+ {"role": "user", "content": "what anime is this?"},
+ ]
+ noted = note_attached_image(history, [("mcp__srv1__lookup", "image")])
+ assert noted[:2] == history[:2] and history[2]["content"] == "what anime is this?"
+ assert noted[2]["content"].startswith("what anime is this?\n\n[The user attached an image")
+ assert 'call mcp__srv1__lookup with {"image": "attached_image"}.]' in noted[2]["content"]
+ parts = [{"type": "text", "text": "which anime?"}]
+ noted = note_attached_image([{"role": "user", "content": parts}], [("t", "f")])
+ assert (
+ noted[0]["content"][0] == parts[0] and "attached an image" in noted[0]["content"][1]["text"]
+ )
+ assert note_attached_image(history, []) is history
+
+
+ "args",
+ [
+ {},
+ {"cut_borders": True},
+ {"image": ""},
+ {"image": "ATTACHED IMAGE"},
+ {"image": "<attached-image>"},
+ ],
+)
+def test_a_call_that_leaves_out_or_misspells_the_field_still_sends_after_approval(
+ mapped_server, args
+):
+ image = McpImage(mime = "image/png", data = _png_bytes())
+ # Without an attached image the call keeps the ordinary path, untouched.
+ before = dict(args)
+ assert tools_mod.mcp_image_share("mcp__srv1__lookup", args, None) is None and args == before
+ share = tools_mod.mcp_image_share("mcp__srv1__lookup", args, image)
+ # The card shows what is sent: the placeholder, in the mapped field.
+ assert share["disclosure"]["tool"] == "lookup" and args["image"] == ATTACHED_IMAGE
+ out = tools_mod.execute_tool("mcp__srv1__lookup", args, mcp_image = share["image"])
+ assert mapped_server[-1]["args"]["image"] == image.encoded("data_url")
+ assert out == "match for [attached image]"
+
+
+def test_siblings_pointed_at_the_attachment_are_not_sent(mapped_server):
+ image = McpImage(mime = "image/png", data = _png_bytes())
+ args = {
+ "path": "attached_image",
+ "url": "https://example.com/attached_image",
+ "question": "Which anime is in the attached image?",
+ "note": "",
+ "cut_borders": True,
+ "other": "image_path",
+ }
+ share = tools_mod.mcp_image_share("mcp__srv1__lookup", args, image)
+ kept = {
+ "question": "Which anime is in the attached image?",
+ "note": "",
+ "cut_borders": True,
+ "other": "image_path",
+ }
+ assert args == {**kept, "image": ATTACHED_IMAGE}
+ tools_mod.execute_tool("mcp__srv1__lookup", args, mcp_image = share["image"])
+ assert mapped_server[-1]["args"] == {**kept, "image": image.encoded("data_url")}
+ # Any other value in the mapped field is the model's own input: no image, nothing rewritten.
+ literal = {"image": "/tmp/a.png", "path": "attached_image"}
+ assert tools_mod.mcp_image_share("mcp__srv1__lookup", literal, image) is None
+ assert literal == {"image": "/tmp/a.png", "path": "attached_image"}
+
+
+def test_a_required_sibling_is_kept_even_when_it_names_the_image(mapped_server):
+ lookup = {
+ **LOOKUP,
+ "inputSchema": {
+ "type": "object",
+ "properties": {"image": {"type": "string"}, "filename": {"type": "string"}},
+ "required": ["image", "filename"],
+ },
+ }
+ mcp_client.cache_tools("srv1", [lookup])
+ args = {"filename": "attached_image"}
+ tools_mod.mcp_image_share(
+ "mcp__srv1__lookup", args, McpImage(mime = "image/png", data = _png_bytes())
+ )
+ assert args == {"filename": "attached_image", "image": ATTACHED_IMAGE}
+
+
+ "args",
+ [
+ {"__unsloth_unparsed_arguments__": '{"image": "attached_image", "cut_borders": tr'},
+ {"raw": '{"image": "attached_image", "cut'},
+ ],
+)
+def test_a_call_whose_arguments_were_cut_off_never_carries_the_image(mapped_server, args):
+ image = McpImage(mime = "image/png", data = _png_bytes())
+ before = dict(args)
+ assert tools_mod.mcp_image_share("mcp__srv1__lookup", args, image) is None
+ assert args == before
+
+
+def test_retrieval_queries_ignore_the_note():
+ noted = note_attached_image(
+ [{"role": "user", "content": "what is this?"}], [("mcp__srv1__lookup", "image")]
+ )
+ assert tools_mod._last_user_text(noted) == "what is this?"
+ parts = note_attached_image(
+ [{"role": "user", "content": [{"type": "text", "text": "what is this?"}]}], [("t", "f")]
+ )
+ assert tools_mod._last_user_text(parts) == "what is this?"
+
+
+def test_retrieval_queries_keep_a_literal_note_prefix_from_the_user():
+ text = (
+ "Explain this literal text: [The user attached an image to this message. "
+ "It came from a log."
+ )
+ assert tools_mod._last_user_text([{"role": "user", "content": text}]) == text
+ noted = note_attached_image(
+ [{"role": "user", "content": text}], [("mcp__srv1__lookup", "image")]
+ )
+ assert tools_mod._last_user_text(noted) == text
+ parts = [{"type": "text", "text": text}]
+ assert tools_mod._last_user_text([{"role": "user", "content": parts}]) == text
+
+
def _one_call_turns():
turns = iter(
[
@@ -408,6 +535,69 @@ def test_safetensors_loop_always_asks_before_sending_the_image(mapped_server, de
assert all(s.recipient for s in seen)
+def test_the_card_shows_the_arguments_the_call_is_sent_with(mapped_server):
+ from core.inference.safetensors_agentic import run_safetensors_tool_loop
+ from state.tool_approvals import resolve_tool_decision
+
+ turns = iter(
+ [
+ '<tool_call>{"name": "mcp__srv1__lookup", "arguments": {"path": "attached_image", "cut_borders": true}}</tool_call>',
+ "done",
+ ]
+ )
+ sent = []
+ starts = []
+ prompts = []
+
+ def single_turn(messages):
+ prompts.append(json.loads(json.dumps(messages)))
+ return iter([next(turns, "")])
+
+ for event in run_safetensors_tool_loop(
+ single_turn = single_turn,
+ messages = [{"role": "user", "content": "what anime is this?"}],
+ tools = [{"type": "function", "function": {"name": "mcp__srv1__lookup"}}],
+ execute_tool = lambda name, arguments, **kwargs: sent.append(dict(arguments)) or "ok",
+ session_id = "s",
+ mcp_image = McpImage(mime = "image/png", data = _png_bytes()),
+ ):
+ if event["type"] == "tool_start":
+ starts.append(event)
+ assert resolve_tool_decision(event["approval_id"], "allow", session_id = "s")
+ assert starts[0]["image_disclosure"]["tool"] == "lookup"
+ assert starts[0]["arguments"] == sent[0] == {"cut_borders": True, "image": ATTACHED_IMAGE}
+ replayed = next(
+ message
+ for message in reversed(prompts[1])
+ if message.get("role") == "assistant" and message.get("tool_calls")
+ )
+ assert json.loads(replayed["tool_calls"][0]["function"]["arguments"]) == sent[0]
+
+
+def test_safetensors_loop_tells_the_model_about_the_image(mapped_server):
+ from core.inference.safetensors_agentic import run_safetensors_tool_loop
+
+ prompts = []
+
+ def single_turn(messages):
+ prompts.append(messages[-1]["content"])
+ yield "done"
+
+ for image in (McpImage(mime = "image/png", data = _png_bytes()), None):
+ list(
+ run_safetensors_tool_loop(
+ single_turn = single_turn,
+ messages = [{"role": "user", "content": "what anime is this?"}],
+ tools = [{"type": "function", "function": {"name": "mcp__srv1__lookup"}}],
+ execute_tool = lambda *a, **k: "ok",
+ session_id = "s",
+ mcp_image = image,
+ )
+ )
+ assert 'call mcp__srv1__lookup with {"image": "attached_image"}' in prompts[0]
+ assert prompts[1] == "what anime is this?"
+
+
def test_route_requires_an_interactive_stream_for_the_image():
from routes.inference import _request_mcp_image
@@ -529,4 +719,12 @@ def test_gguf_loop_gates_and_forwards_the_image_like_the_other_loops():
ast.unparse(node) for node in ast.walk(ast.parse(src)) if isinstance(node, ast.stmt)
}
assert "needs_confirm = needs_confirm or image_share is not None" in statements
+ assert (
+ "messages = note_attached_image(messages, mcp_image_targets(_gguf_active_tool_names(tools)))"
+ in statements
+ )
+ assert "decision = tool_controller.reprepare_call(" in src
+ assert src.index("image_share = mcp_image_share(") < src.index(
+ "decision.as_assistant_tool_call()"
+ )
assert "kwargs['mcp_image'] = image_share['image']" in statements
diff --git a/studio/backend/tests/test_studio_tool_loop.py b/studio/backend/tests/test_studio_tool_loop.py
index 8fce1cf8d6b..da73b29d453 100644
--- a/studio/backend/tests/test_studio_tool_loop.py
+++ b/studio/backend/tests/test_studio_tool_loop.py
@@ -719,13 +719,119 @@ def test_sharing_the_attached_image_asks_even_with_bypass(executed, monkeypatch)
start = _events(lines, "tool_start")[0]
assert start["awaiting_confirmation"] is True and start["image_disclosure"] == disclosure
assert len(asked) == 1 and executed[0]["mcp_image"] == image.approved_for("r1")
- # Without an image the same call keeps the ordinary path: no card, no image.
executed.clear()
lines = _run(FakeTransport(turns), bypass_permissions = True)
assert _events(lines, "tool_start")[0]["awaiting_confirmation"] is False
assert "mcp_image" not in executed[0]
+def test_settled_image_call_is_replayed_and_deduped_with_sent_arguments(
+ executed, monkeypatch
+):
+ from core.inference.mcp_image import ATTACHED_IMAGE, McpImage
+
+ image = McpImage(mime = "image/png", data = b"IMG")
+ shared = {
+ "disclosure": {"server": "Trace", "tool": "lookup", "size_bytes": 3},
+ "image": image.approved_for("r1"),
+ }
+
+ def settle(_name, arguments, _image):
+ arguments["image"] = ATTACHED_IMAGE
+ return shared
+
+ monkeypatch.setattr(loop_mod, "mcp_image_share", settle)
+ monkeypatch.setattr(loop_mod, "begin_tool_decision", lambda *_args: object())
+ monkeypatch.setattr(
+ loop_mod, "wait_tool_decision", lambda *_args, **_kwargs: "allow"
+ )
+ monkeypatch.setattr(loop_mod, "abort_tool_decision", lambda *_args: None)
+ call = {"index": 0, "function": {"name": "python", "arguments": "{}"}}
+ transport = FakeTransport(
+ [
+ [_sse({"tool_calls": [{**call, "id": "c1"}]}), _sse(finish = "tool_calls"), _DONE],
+ [_sse({"tool_calls": [{**call, "id": "c2"}]}), _sse(finish = "tool_calls"), _DONE],
+ [_sse({"content": "done"}), _sse(finish = "stop"), _DONE],
+ ],
+ heals = False,
+ )
+ _run(transport, tools = [PY], mcp_image = image, bypass_permissions = True)
+
+ assert len(executed) == 1
+ replayed = next(
+ message
+ for message in reversed(transport.requests[1]["messages"])
+ if message.get("role") == "assistant" and message.get("tool_calls")
+ )
+ assert json.loads(replayed["tool_calls"][0]["function"]["arguments"]) == {
+ "image": ATTACHED_IMAGE
+ }
+
+
+def test_settled_image_call_keeps_signed_provider_replay_exact(executed, monkeypatch):
+ from core.inference.mcp_image import ATTACHED_IMAGE, McpImage
+
+ image = McpImage(mime = "image/png", data = b"IMG")
+
+ def settle(_name, arguments, _image):
+ arguments["image"] = ATTACHED_IMAGE
+ return {
+ "disclosure": {"server": "Trace", "tool": "lookup", "size_bytes": 3},
+ "image": image.approved_for("r1"),
+ }
+
+ monkeypatch.setattr(loop_mod, "mcp_image_share", settle)
+ monkeypatch.setattr(loop_mod, "begin_tool_decision", lambda *_args: object())
+ monkeypatch.setattr(
+ loop_mod, "wait_tool_decision", lambda *_args, **_kwargs: "allow"
+ )
+ monkeypatch.setattr(loop_mod, "abort_tool_decision", lambda *_args: None)
+ call = {
+ "index": 0,
+ "id": "c1",
+ "function": {"name": "python", "arguments": "{ }"},
+ "extra_content": {"google": {"thought_signature": "SIG-A"}},
+ }
+ transport = FakeTransport(
+ [
+ [_sse({"tool_calls": [call]}), _sse(finish = "tool_calls"), _DONE],
+ [_sse({"content": "done"}), _sse(finish = "stop"), _DONE],
+ ],
+ heals = False,
+ )
+ _run(transport, tools = [PY], mcp_image = image, bypass_permissions = True)
+
+ assert executed[0]["arguments"] == {"image": ATTACHED_IMAGE}
+ replayed = next(
+ message
+ for message in reversed(transport.requests[1]["messages"])
+ if message.get("role") == "assistant" and message.get("tool_calls")
+ )["tool_calls"][0]
+ assert replayed["function"]["arguments"] == "{ }"
+ assert replayed["extra_content"] == {
+ "google": {"thought_signature": "SIG-A"}
+ }
+
+
+def test_the_model_is_told_about_the_attached_image(executed, monkeypatch):
+ from core.inference.mcp_image import McpImage
+
+ monkeypatch.setattr(
+ loop_mod, "mcp_image_targets", lambda names: [("mcp__srv1__lookup", "image")]
+ )
+ turns = [[_sse({"content": "ok"}), _sse(finish = "stop"), _DONE]]
+ messages = [{"role": "user", "content": "what anime is this?"}]
+ transport = FakeTransport(turns)
+ _run(transport, messages = messages, mcp_image = McpImage(mime = "image/png", data = b"IMG"))
+ sent = transport.requests[0]["messages"][-1]["content"]
+ assert sent.startswith("what anime is this?\n\n[The user attached an image")
+ assert 'mcp__srv1__lookup with {"image": "attached_image"}' in sent
+ assert messages == [{"role": "user", "content": "what anime is this?"}]
+ transport = FakeTransport(turns)
+ _run(transport, messages = messages)
+ assert transport.requests[0]["messages"][-1]["content"] == "what anime is this?"
+
+
def test_full_access_disables_the_sandbox_at_execution(executed):
transport = _shared_setup_1()
_run(transport, tools = [PY], bypass_permissions = True)
@@ -738,7 +844,7 @@ def test_sandbox_stays_on_by_default(executed):
_run(transport, tools = [PY])
assert executed[0]["disable_sandbox"] is False
- # Nobody approved it, so the executor keeps the jail even for a host path.
+ # without approval, the executor keeps the jail even for a host path.
assert "host_access_approved" not in executed[0]
diff --git a/studio/backend/tests/test_tool_loop_controller.py b/studio/backend/tests/test_tool_loop_controller.py
index 792dbb1a1eb..953db28b731 100644
--- a/studio/backend/tests/test_tool_loop_controller.py
+++ b/studio/backend/tests/test_tool_loop_controller.py
@@ -190,6 +190,29 @@ def test_successful_duplicate_is_internal_noop_and_keeps_remaining_tools():
_shared_setup_1(controller)
+def test_reprepare_uses_normalized_arguments_for_replay_and_dedup():
+ tool_name = "mcp__srv1__lookup"
+ controller = ToolLoopController(tools = [_tool(tool_name)])
+ call = _call(tool_name, {}, "call_a")
+ call["card_id"] = "painted_a"
+ first = controller.prepare_call(call)
+ first.arguments["image"] = "attached_image"
+ first = controller.reprepare_call(first)
+
+ assert first.card_id == "painted_a"
+ assert first.as_assistant_tool_call()["function"]["arguments"] == (
+ '{"image":"attached_image"}'
+ )
+ assert first.key == canonical_tool_call_key(tool_name, {"image": "attached_image"})
+ controller.record_result(first, "ok")
+
+ repeated = controller.prepare_call(_call(tool_name, {}, "call_b"))
+ repeated.arguments["image"] = "attached_image"
+ repeated = controller.reprepare_call(repeated)
+ assert repeated.action == "duplicate"
+ assert repeated.key == first.key
+
+
def test_web_search_alias_args_share_the_duplicate_key():
controller = ToolLoopController(tools = [_tool("web_search")])
first = controller.prepare_call(_call("web_search", {"q": "unsloth"}, "call_a"))