Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

Studio: only strip a valid trailing __IMAGES__/__RAG_SOURCES__ envelope (#10668)

Nilayd4ef69f848ad
diff --git a/studio/backend/core/inference/tool_loop_controller.py b/studio/backend/core/inference/tool_loop_controller.py
index ce6f38c45c5..e34cf90cefa 100644
--- a/studio/backend/core/inference/tool_loop_controller.py
+++ b/studio/backend/core/inference/tool_loop_controller.py
@@ -869,10 +869,81 @@ def _is_file_entry(entry: object) -> bool:
)
+def _strip_images_sentinel(result: str) -> str:
+ """Drop the trailing ``__IMAGES__`` envelopes, and only those.
+
+ Validated rather than split on sight, like the two above and like
+ ``studio_tool_loop._carries_image_sentinel``: a tool whose own output quotes
+ the marker would otherwise lose everything after it while the card the user
+ reads still shows the whole result.
+
+ Every envelope, not just the last: a Gemini ``code_execution`` turn that drew
+ two figures stacks one per ``inlineData`` part, and stopping after the last
+ would replay the earlier plot's whole base64 data URI to the model.
+
+ Walked by index and cut once at the end. Re-partitioning the shortened string
+ each time copies it again, which is quadratic in the number of markers, and an
+ MCP server answering with 80,000 of them is 1.3 MB of text that held this
+ thread for seconds.
+ """
+ marker = "\n__IMAGES__:"
+ end = len(result)
+ cut = -1
+ while True:
+ start = result.rfind(marker, 0, end)
+ if start == -1:
+ break
+ try:
+ images = json.loads(result[start + len(marker) : end])
+ except (ValueError, RecursionError):
+ break
+ if not isinstance(images, list) or not images:
+ break
+ if not all(isinstance(image, str) and image for image in images):
+ break
+ cut = start
+ end = start
+ return result if cut == -1 else result[:cut].rstrip()
+
+
+def _strip_rag_sources_sentinel(result: str) -> str:
+ """Drop a trailing ``__RAG_SOURCES__`` source map, and only that.
+
+ The retrieval tools append ``RAG_SOURCES_SENTINEL`` plus a JSON list; a
+ result that merely mentions the marker is text.
+
+ Deliberately unbounded, like the walk above. Each source record repeats its whole
+ chunk, and ``search_knowledge_base`` takes the model's ``top_k`` without a ceiling,
+ so a real map has no size worth calling suspicious -- and one refused for being big
+ is a frontend-only blob left in the model's context, which ``_fit_result_to_room``
+ would then truncate into malformed JSON. What keeps unbounded MCP text away from
+ this decode is the gate on the emitting tools, not a length.
+ """
+ head, sep, payload = result.rpartition("\n__RAG_SOURCES__:")
+ if not sep:
+ return result
+ try:
+ sources = json.loads(payload)
+ except (ValueError, RecursionError):
+ return result
+ if not isinstance(sources, list):
+ return result
+ return head.rstrip()
+
+
# Only these emit the file envelope, and only their output is defused first. An MCP tool or a fetched page ending in a
# well-formed __FILES__ line is content, not an envelope, and stripping it would take that line away from the model.
_SANDBOX_TOOLS = frozenset({"python", "terminal"})
+# Same rule for the other two envelopes. The image one is emitted by the sandbox tools
+# through `_created_file_sentinels` and by Gemini's hosted code_execution through the
+# provider; the source map by the retrieval tools that append `RAG_SOURCES_SENTINEL`. A
+# document an MCP tool read, or a page that was fetched, ending in a well-formed one of
+# either is content the model needs, and cutting it leaves the model reasoning over less
+# than the card the user is looking at.
+_IMAGE_SENTINEL_TOOLS = _SANDBOX_TOOLS | {"code_execution"}
+_SOURCE_MAP_TOOLS = frozenset({"search_knowledge_base", "search_conversation"})
+
def strip_result_for_model(result: str, tool_name: "str | None" = None) -> str:
"""Remove frontend-only sentinels (image paths, RAG source map) before
@@ -883,9 +954,10 @@ def strip_result_for_model(result: str, tool_name: "str | None" = None) -> str:
result = _strip_mcp_image_suffix(result)
if tool_name is None or tool_name in _SANDBOX_TOOLS:
result = _strip_files_sentinel(result)
- for sentinel in ("__IMAGES__:", "__RAG_SOURCES__:"):
- if sentinel in result:
- result = result.split(sentinel, 1)[0].rstrip()
+ if tool_name is None or tool_name in _IMAGE_SENTINEL_TOOLS:
+ result = _strip_images_sentinel(result)
+ if tool_name is None or tool_name in _SOURCE_MAP_TOOLS:
+ result = _strip_rag_sources_sentinel(result)
return result
diff --git a/studio/backend/tests/test_mcp_flatten_result.py b/studio/backend/tests/test_mcp_flatten_result.py
index c6fdb187b01..6e2be2307f0 100644
--- a/studio/backend/tests/test_mcp_flatten_result.py
+++ b/studio/backend/tests/test_mcp_flatten_result.py
@@ -125,7 +125,7 @@ def test_strip_removes_only_valid_terminal_envelope():
def test_strip_still_handles_images_and_rag_sentinels():
- assert strip_result_for_model("output\n__IMAGES__:['a.png']") == "output"
+ assert strip_result_for_model('output\n__IMAGES__:["a.png"]') == "output"
assert strip_result_for_model("answer\n__RAG_SOURCES__:[{}]") == "answer"
diff --git a/studio/backend/tests/test_safetensors_tool_loop.py b/studio/backend/tests/test_safetensors_tool_loop.py
index 9d640ae4d7e..152fe061178 100644
--- a/studio/backend/tests/test_safetensors_tool_loop.py
+++ b/studio/backend/tests/test_safetensors_tool_loop.py
@@ -2860,7 +2860,7 @@ class TestLoopBehaviour:
assert "__IMAGES__" in tool_end["result"]
def test_image_sentinel_stripped_with_leading_marker(self):
- # Sentinel at start (no newline) must not leak to the model.
+ # A call that printed nothing is nothing but the envelope; it must not leak to the model.
from core.inference import safetensors_agentic as _sa
captured: list[list[dict]] = []
@@ -2877,7 +2877,7 @@ class TestLoopBehaviour:
single_turn = fake_single_turn,
messages = [{"role": "user", "content": "plot please"}],
tools = [{"function": {"name": "python"}}],
- execute_tool = lambda *_a, **_kw: "__IMAGES__:/tmp/x.png",
+ execute_tool = lambda *_a, **_kw: '\n__IMAGES__:["/tmp/x.png"]',
cancel_event = threading.Event(),
max_tool_iterations = 3,
auto_heal_tool_calls = True,
@@ -2891,7 +2891,7 @@ class TestLoopBehaviour:
assert "__IMAGES__" not in tm["content"], f"sentinel leaked to model: {tm['content']!r}"
def test_image_sentinel_stripped_with_multiple_markers(self):
- # Consecutive sentinels: cut at the first, nothing leaks.
+ # Consecutive markers: the trailing envelope goes, a line the tool printed itself stays.
from core.inference import safetensors_agentic as _sa
captured: list[list[dict]] = []
@@ -2903,7 +2903,7 @@ class TestLoopBehaviour:
else:
yield "done"
- multi = "panel\n__IMAGES__:/tmp/a.png\n__IMAGES__:/tmp/b.png"
+ multi = 'panel\n__IMAGES__:/tmp/a.png\n__IMAGES__:["/tmp/b.png"]'
events = list(
_sa.run_safetensors_tool_loop(
single_turn = fake_single_turn,
@@ -2918,8 +2918,9 @@ class TestLoopBehaviour:
tool_msgs = [m for m in captured[1] if m.get("role") == "tool"]
assert tool_msgs
for tm in tool_msgs:
- assert "__IMAGES__" not in tm["content"], f"second sentinel leaked: {tm['content']!r}"
- assert tm["content"] == "panel", f"expected payload-only 'panel', got {tm['content']!r}"
+ assert (
+ tm["content"] == "panel\n__IMAGES__:/tmp/a.png"
+ ), f"expected the printed line kept, got {tm['content']!r}"
def test_tool_execution_error_is_emitted_but_loop_continues(self):
loop, exec_fn = _make_loop(
diff --git a/studio/backend/tests/test_tool_loop_controller.py b/studio/backend/tests/test_tool_loop_controller.py
index 994893a0ee0..6091f8f02f7 100644
--- a/studio/backend/tests/test_tool_loop_controller.py
+++ b/studio/backend/tests/test_tool_loop_controller.py
@@ -5,6 +5,7 @@ from __future__ import annotations
import json
import sys
+import time
from pathlib import Path
import pytest
@@ -142,9 +143,10 @@ def test_prepare_execute_builds_visible_events_and_model_tool_message():
assert decision.tool_start_event()["type"] == "tool_start"
assert decision.as_assistant_tool_call()["function"]["arguments"] == '{"query":"gpu prices"}'
- completion = controller.record_result(decision, "Search result\n__IMAGES__:{...}")
+ envelope = 'Search result\n__WEB_IMAGES__:[{"id": "a1b2c3d4e5f6", "title": "A chart", "domain": "example.com", "source": "https://example.com/a.png"}]'
+ completion = controller.record_result(decision, envelope)
- assert completion.tool_end_payload()["result"] == "Search result\n__IMAGES__:{...}"
+ assert completion.tool_end_payload()["result"] == envelope
assert completion.tool_end_event()["type"] == "tool_end"
assert completion.tool_message() == {
"role": "tool",
@@ -276,11 +278,134 @@ def test_render_html_success_filters_active_tools_and_repeat_is_internal():
def test_strip_result_for_model_removes_frontend_image_sentinel():
- assert strip_result_for_model('text\n__IMAGES__:{"paths":[]}') == "text"
- assert strip_result_for_model("text __IMAGES__:payload") == "text"
+ assert strip_result_for_model('text\n__IMAGES__:["a.png"]') == "text"
assert strip_result_for_model("plain text") == "plain text"
+def test_a_result_that_only_quotes_the_image_or_source_marker_is_kept_whole():
+ """Only a structurally valid trailing envelope is stripped, the way
+ `_strip_files_sentinel` and `_strip_mcp_image_suffix` beside it already are."""
+ quoted = 'tools.py:16134: out += f"\\n__IMAGES__:{_json.dumps(images)}"\nsecond hit\nthird hit'
+ for name in (None, "terminal", "mcp__fs__read"):
+ assert strip_result_for_model(quoted, name) == quoted
+
+ # What `_defuse_sentinels` leaves behind when a program prints the marker itself.
+ defused = "line one\n __IMAGES__:printed by the program\nline three"
+ assert strip_result_for_model(defused, "python") == defused
+
+ sources = 'line one\nRAG_SOURCES_SENTINEL = "\\n__RAG_SOURCES__:"\nline three'
+ assert strip_result_for_model(sources, "search_knowledge_base") == sources
+
+ assert (
+ strip_result_for_model('text\n__IMAGES__:{"paths":[]}') == 'text\n__IMAGES__:{"paths":[]}'
+ )
+ assert strip_result_for_model('output\n__IMAGES__:["a.png"]', "python") == "output"
+ assert (
+ strip_result_for_model(
+ 'answer\n__RAG_SOURCES__:[{"filename": "a.pdf"}]', "search_knowledge_base"
+ )
+ == "answer"
+ )
+
+
+def test_two_plots_in_one_code_execution_turn_replay_no_base64():
+ """A Gemini `code_execution` turn that draws two figures stacks one envelope per
+ `inlineData` part (external_provider re-appends to the result it just emitted), so
+ peeling once would hand the model the earlier plot's whole data URI."""
+ first = "data:image/png;base64," + "A" * 64
+ second = "data:image/png;base64," + "B" * 64
+ stacked = (
+ "Figures saved."
+ + f"\n__IMAGES__:{json.dumps([first])}"
+ + f"\n__IMAGES__:{json.dumps([second])}"
+ )
+
+ assert strip_result_for_model(stacked, "code_execution") == "Figures saved."
+
+
+def test_a_flood_of_stacked_image_markers_stays_linear():
+ """A hosted result's length is the provider's to choose and this runs on the request
+ thread, so the cost has to follow it. Re-partitioning the shortened string once per
+ marker copies it again every time: quadruple the markers and the work grows about
+ sixteenfold instead of fourfold.
+
+ Measured as CPU time, and as the best of several runs. A shared runner can deschedule
+ the process mid-call, which adds wall clock but no CPU, and taking the minimum drops
+ the samples where it happened -- interference can only ever make a run look slower.
+ """
+
+ def cost(markers: int) -> float:
+ flood = '\n__IMAGES__:["x"]' * markers
+ best = float("inf")
+ for _ in range(5):
+ started = time.process_time()
+ assert strip_result_for_model(flood, "code_execution") == ""
+ best = min(best, time.process_time() - started)
+ return max(best, 1e-4)
+
+ small = cost(20_000)
+ large = cost(80_000)
+
+ assert large / small < 10.0, f"4x the markers cost {large / small:.1f}x the work"
+
+
+def test_a_large_source_map_is_still_taken_off_the_result():
+ """No length bound here either. Every source record repeats its whole chunk and
+ `search_knowledge_base` honours the model's `top_k` without a ceiling, so a real map
+ can be megabytes; one refused for being big would be left for `_fit_result_to_room`
+ to cut into malformed JSON in front of the model."""
+ import json as _json
+
+ chunk = "retrieved text. " * 400
+ sources = [
+ {
+ "citationId": i,
+ "chunkId": f"c{i}",
+ "filename": "handbook.pdf",
+ "page": i,
+ "text": chunk,
+ "score": 0.5,
+ }
+ for i in range(200)
+ ]
+ result = "answer\n__RAG_SOURCES__:" + _json.dumps(sources, ensure_ascii = False)
+ assert len(result) > 1 << 20
+
+ assert strip_result_for_model(result, "search_knowledge_base") == "answer"
+
+
+def test_a_large_plot_is_still_taken_off_the_result():
+ """The image payload has no length bound on purpose: a figure Gemini renders at high
+ DPI is a data URI of whatever size it chose, and leaving it in for being large is the
+ base64 leak this stripper exists to prevent."""
+ big = 'output\n__IMAGES__:["data:image/png;base64,' + "A" * (12 << 20) + '"]'
+ assert len(big) > 8 << 20
+ assert strip_result_for_model(big, "code_execution") == "output"
+
+
+def test_only_the_tools_that_emit_an_envelope_have_one_taken_off():
+ """`_strip_files_sentinel` is already scoped this way. A document an MCP tool read,
+ or a page that was fetched, can end in a well-formed line of either kind, and it is
+ content: the card keeps it, so cutting it leaves the model with less than the user
+ is looking at."""
+ manifest = 'icons/\n__IMAGES__:["icon.png"]'
+ citation = 'notes\n__RAG_SOURCES__:[{"filename": "a.pdf"}]'
+
+ for reader in ("mcp__fs__read_file", "web_search", "fetch_url"):
+ assert strip_result_for_model(manifest, reader) == manifest
+ assert strip_result_for_model(citation, reader) == citation
+
+ # The tools that do emit them are unaffected.
+ for emitter in ("python", "terminal", "code_execution"):
+ assert strip_result_for_model(manifest, emitter) == "icons/"
+ for emitter in ("search_knowledge_base", "search_conversation"):
+ assert strip_result_for_model(citation, emitter) == "notes"
+
+ # An unnamed caller still gets everything stripped, as it did before.
+ assert strip_result_for_model(manifest) == "icons/"
+ assert strip_result_for_model(citation) == "notes"
+
+
def test_the_card_text_keeps_digits_the_browser_would_round():
"""`JSON.parse` reads 9007199254740993 back as ...992, so a card that re-encodes the
parsed arguments in the browser would show a record the tool is not being run with."""
diff --git a/studio/frontend/src/components/assistant-ui/sandbox-files.ts b/studio/frontend/src/components/assistant-ui/sandbox-files.ts
index 8663f799a73..0913f5068b9 100644
--- a/studio/frontend/src/components/assistant-ui/sandbox-files.ts
+++ b/studio/frontend/src/components/assistant-ui/sandbox-files.ts
@@ -12,6 +12,14 @@ const FILES_MARKER = "\n__FILES__:";
/** The tools that emit the file envelope. Nothing else's output is an envelope. */
export const SANDBOX_FILE_TOOLS = new Set(["python", "terminal"]);
+/**
+ * The tools that emit the image envelope: the sandbox ones, and the provider's hosted
+ * code execution, which attaches its plots the same way. The backend keeps a
+ * well-formed `__IMAGES__` line from any other tool as content the model reads, so
+ * the card has to keep it too, or the user sees less than the model was given.
+ */
+export const IMAGE_SENTINEL_TOOLS = new Set([...SANDBOX_FILE_TOOLS, "code_execution"]);
+
function isSandboxFile(entry: unknown): entry is SandboxFile {
if (typeof entry !== "object" || entry === null) return false;
const { name, size } = entry as { name?: unknown; size?: unknown };
diff --git a/studio/frontend/src/features/chat/api/chat-adapter.ts b/studio/frontend/src/features/chat/api/chat-adapter.ts
index dca408f18c9..12c9030b675 100644
--- a/studio/frontend/src/features/chat/api/chat-adapter.ts
+++ b/studio/frontend/src/features/chat/api/chat-adapter.ts
@@ -35,6 +35,7 @@ import { sanitizeStoredExtraArgs } from "@/features/model-picker/model-config/ll
import { usePlatformStore } from "@/config/env";
import { projectHasSources } from "@/features/rag/api/rag-api";
import {
+ IMAGE_SENTINEL_TOOLS,
SANDBOX_FILE_TOOLS,
extractCreatedFiles,
isSandboxFileList,
@@ -6667,7 +6668,14 @@ export function createOpenAIStreamAdapter(
? extractSearchImages(rawResult)
: { text: rawResult, images: [] as SearchImageEntry[] };
const imgMarker = "\n__IMAGES__:";
- const imgIdx = rawResult.lastIndexOf(imgMarker);
+ // Same rule again. The backend keeps this line for the model when the tool is not one that
+ // emits the envelope, so the card keeps it too, rather than hiding it and fetching a
+ // sandbox file that was never written.
+ const imgIdx = IMAGE_SENTINEL_TOOLS.has(
+ toolCallParts[idx].toolName ?? "",
+ )
+ ? rawResult.lastIndexOf(imgMarker)
+ : -1;
const mcpImgMarker = "\n__MCP_IMAGES__:";
const mcpImgIdx = rawResult.lastIndexOf(mcpImgMarker);
let parsedResult:
diff --git a/studio/frontend/tests/image-sentinel-tools.test.ts b/studio/frontend/tests/image-sentinel-tools.test.ts
new file mode 100644
index 00000000000..f20555be520
--- /dev/null
+++ b/studio/frontend/tests/image-sentinel-tools.test.ts
@@ -0,0 +1,42 @@
+// SPDX-License-Identifier: AGPL-3.0-only
+// Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
+
+import assert from "node:assert/strict";
+import test from "node:test";
+
+import { readSrc } from "./helpers/kit.ts";
+
+const { IMAGE_SENTINEL_TOOLS, SANDBOX_FILE_TOOLS } = await import(
+ "../src/components/assistant-ui/sandbox-files.ts"
+);
+
+test("only the tools that emit the image envelope are in the set", () => {
+ for (const emitter of ["python", "terminal", "code_execution"]) {
+ assert.ok(IMAGE_SENTINEL_TOOLS.has(emitter), emitter);
+ }
+ for (const reader of ["mcp__fs__read_file", "web_search", "search_knowledge_base", ""]) {
+ assert.ok(!IMAGE_SENTINEL_TOOLS.has(reader), reader);
+ }
+ for (const sandbox of SANDBOX_FILE_TOOLS) {
+ assert.ok(IMAGE_SENTINEL_TOOLS.has(sandbox), `${sandbox} emits both envelopes`);
+ }
+});
+
+// Asserted against the source like the other chat-adapter tests: importing the adapter
+// drags in the whole app. The backend keeps a well-formed `__IMAGES__` line from a tool
+// that does not emit the envelope as content the model reads, so the card must not slice
+// it off and fetch a sandbox file that was never written.
+test("the adapter slices the image envelope only for the tools that emit it", () => {
+ const source = readSrc("features/chat/api/chat-adapter.ts");
+ const gate = source.indexOf(
+ 'IMAGE_SENTINEL_TOOLS.has(\n toolCallParts[idx].toolName ?? "",\n )',
+ );
+ const slice = source.indexOf("rawResult.lastIndexOf(imgMarker)");
+ assert.ok(gate >= 0, "the __IMAGES__ slice is no longer gated on IMAGE_SENTINEL_TOOLS");
+ assert.ok(slice > gate, "the slice runs before the gate");
+ assert.equal(
+ source.indexOf("rawResult.lastIndexOf(imgMarker)", slice + 1),
+ -1,
+ "a second, ungated __IMAGES__ slice appeared",
+ );
+});