Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
Studio: only strip a valid trailing __IMAGES__/__RAG_SOURCES__ envelope (#10668)
Nilayd4ef69f848ad
diff --git a/studio/backend/core/inference/tool_loop_controller.py b/studio/backend/core/inference/tool_loop_controller.pyindex ce6f38c45c5..e34cf90cefa 100644--- a/studio/backend/core/inference/tool_loop_controller.py+++ b/studio/backend/core/inference/tool_loop_controller.py@@ -869,10 +869,81 @@ def _is_file_entry(entry: object) -> bool:)+def _strip_images_sentinel(result: str) -> str:+ """Drop the trailing ``__IMAGES__`` envelopes, and only those.++ Validated rather than split on sight, like the two above and like+ ``studio_tool_loop._carries_image_sentinel``: a tool whose own output quotes+ the marker would otherwise lose everything after it while the card the user+ reads still shows the whole result.++ Every envelope, not just the last: a Gemini ``code_execution`` turn that drew+ two figures stacks one per ``inlineData`` part, and stopping after the last+ would replay the earlier plot's whole base64 data URI to the model.++ Walked by index and cut once at the end. Re-partitioning the shortened string+ each time copies it again, which is quadratic in the number of markers, and an+ MCP server answering with 80,000 of them is 1.3 MB of text that held this+ thread for seconds.+ """+ marker = "\n__IMAGES__:"+ end = len(result)+ cut = -1+ while True:+ start = result.rfind(marker, 0, end)+ if start == -1:+ break+ try:+ images = json.loads(result[start + len(marker) : end])+ except (ValueError, RecursionError):+ break+ if not isinstance(images, list) or not images:+ break+ if not all(isinstance(image, str) and image for image in images):+ break+ cut = start+ end = start+ return result if cut == -1 else result[:cut].rstrip()+++def _strip_rag_sources_sentinel(result: str) -> str:+ """Drop a trailing ``__RAG_SOURCES__`` source map, and only that.++ The retrieval tools append ``RAG_SOURCES_SENTINEL`` plus a JSON list; a+ result that merely mentions the marker is text.++ Deliberately unbounded, like the walk above. Each source record repeats its whole+ chunk, and ``search_knowledge_base`` takes the model's ``top_k`` without a ceiling,+ so a real map has no size worth calling suspicious -- and one refused for being big+ is a frontend-only blob left in the model's context, which ``_fit_result_to_room``+ would then truncate into malformed JSON. What keeps unbounded MCP text away from+ this decode is the gate on the emitting tools, not a length.+ """+ head, sep, payload = result.rpartition("\n__RAG_SOURCES__:")+ if not sep:+ return result+ try:+ sources = json.loads(payload)+ except (ValueError, RecursionError):+ return result+ if not isinstance(sources, list):+ return result+ return head.rstrip()++# Only these emit the file envelope, and only their output is defused first. An MCP tool or a fetched page ending in a# well-formed __FILES__ line is content, not an envelope, and stripping it would take that line away from the model._SANDBOX_TOOLS = frozenset({"python", "terminal"})+# Same rule for the other two envelopes. The image one is emitted by the sandbox tools+# through `_created_file_sentinels` and by Gemini's hosted code_execution through the+# provider; the source map by the retrieval tools that append `RAG_SOURCES_SENTINEL`. A+# document an MCP tool read, or a page that was fetched, ending in a well-formed one of+# either is content the model needs, and cutting it leaves the model reasoning over less+# than the card the user is looking at.+_IMAGE_SENTINEL_TOOLS = _SANDBOX_TOOLS | {"code_execution"}+_SOURCE_MAP_TOOLS = frozenset({"search_knowledge_base", "search_conversation"})+def strip_result_for_model(result: str, tool_name: "str | None" = None) -> str:"""Remove frontend-only sentinels (image paths, RAG source map) before@@ -883,9 +954,10 @@ def strip_result_for_model(result: str, tool_name: "str | None" = None) -> str:result = _strip_mcp_image_suffix(result)if tool_name is None or tool_name in _SANDBOX_TOOLS:result = _strip_files_sentinel(result)- for sentinel in ("__IMAGES__:", "__RAG_SOURCES__:"):- if sentinel in result:- result = result.split(sentinel, 1)[0].rstrip()+ if tool_name is None or tool_name in _IMAGE_SENTINEL_TOOLS:+ result = _strip_images_sentinel(result)+ if tool_name is None or tool_name in _SOURCE_MAP_TOOLS:+ result = _strip_rag_sources_sentinel(result)return resultdiff --git a/studio/backend/tests/test_mcp_flatten_result.py b/studio/backend/tests/test_mcp_flatten_result.pyindex c6fdb187b01..6e2be2307f0 100644--- a/studio/backend/tests/test_mcp_flatten_result.py+++ b/studio/backend/tests/test_mcp_flatten_result.py@@ -125,7 +125,7 @@ def test_strip_removes_only_valid_terminal_envelope():def test_strip_still_handles_images_and_rag_sentinels():- assert strip_result_for_model("output\n__IMAGES__:['a.png']") == "output"+ assert strip_result_for_model('output\n__IMAGES__:["a.png"]') == "output"assert strip_result_for_model("answer\n__RAG_SOURCES__:[{}]") == "answer"diff --git a/studio/backend/tests/test_safetensors_tool_loop.py b/studio/backend/tests/test_safetensors_tool_loop.pyindex 9d640ae4d7e..152fe061178 100644--- a/studio/backend/tests/test_safetensors_tool_loop.py+++ b/studio/backend/tests/test_safetensors_tool_loop.py@@ -2860,7 +2860,7 @@ class TestLoopBehaviour:assert "__IMAGES__" in tool_end["result"]def test_image_sentinel_stripped_with_leading_marker(self):- # Sentinel at start (no newline) must not leak to the model.+ # A call that printed nothing is nothing but the envelope; it must not leak to the model.from core.inference import safetensors_agentic as _sacaptured: list[list[dict]] = []@@ -2877,7 +2877,7 @@ class TestLoopBehaviour:single_turn = fake_single_turn,messages = [{"role": "user", "content": "plot please"}],tools = [{"function": {"name": "python"}}],- execute_tool = lambda *_a, **_kw: "__IMAGES__:/tmp/x.png",+ execute_tool = lambda *_a, **_kw: '\n__IMAGES__:["/tmp/x.png"]',cancel_event = threading.Event(),max_tool_iterations = 3,auto_heal_tool_calls = True,@@ -2891,7 +2891,7 @@ class TestLoopBehaviour:assert "__IMAGES__" not in tm["content"], f"sentinel leaked to model: {tm['content']!r}"def test_image_sentinel_stripped_with_multiple_markers(self):- # Consecutive sentinels: cut at the first, nothing leaks.+ # Consecutive markers: the trailing envelope goes, a line the tool printed itself stays.from core.inference import safetensors_agentic as _sacaptured: list[list[dict]] = []@@ -2903,7 +2903,7 @@ class TestLoopBehaviour:else:yield "done"- multi = "panel\n__IMAGES__:/tmp/a.png\n__IMAGES__:/tmp/b.png"+ multi = 'panel\n__IMAGES__:/tmp/a.png\n__IMAGES__:["/tmp/b.png"]'events = list(_sa.run_safetensors_tool_loop(single_turn = fake_single_turn,@@ -2918,8 +2918,9 @@ class TestLoopBehaviour:tool_msgs = [m for m in captured[1] if m.get("role") == "tool"]assert tool_msgsfor tm in tool_msgs:- assert "__IMAGES__" not in tm["content"], f"second sentinel leaked: {tm['content']!r}"- assert tm["content"] == "panel", f"expected payload-only 'panel', got {tm['content']!r}"+ assert (+ tm["content"] == "panel\n__IMAGES__:/tmp/a.png"+ ), f"expected the printed line kept, got {tm['content']!r}"def test_tool_execution_error_is_emitted_but_loop_continues(self):loop, exec_fn = _make_loop(diff --git a/studio/backend/tests/test_tool_loop_controller.py b/studio/backend/tests/test_tool_loop_controller.pyindex 994893a0ee0..6091f8f02f7 100644--- a/studio/backend/tests/test_tool_loop_controller.py+++ b/studio/backend/tests/test_tool_loop_controller.py@@ -5,6 +5,7 @@ from __future__ import annotationsimport jsonimport sys+import timefrom pathlib import Pathimport pytest@@ -142,9 +143,10 @@ def test_prepare_execute_builds_visible_events_and_model_tool_message():assert decision.tool_start_event()["type"] == "tool_start"assert decision.as_assistant_tool_call()["function"]["arguments"] == '{"query":"gpu prices"}'- completion = controller.record_result(decision, "Search result\n__IMAGES__:{...}")+ envelope = 'Search result\n__WEB_IMAGES__:[{"id": "a1b2c3d4e5f6", "title": "A chart", "domain": "example.com", "source": "https://example.com/a.png"}]'+ completion = controller.record_result(decision, envelope)- assert completion.tool_end_payload()["result"] == "Search result\n__IMAGES__:{...}"+ assert completion.tool_end_payload()["result"] == envelopeassert completion.tool_end_event()["type"] == "tool_end"assert completion.tool_message() == {"role": "tool",@@ -276,11 +278,134 @@ def test_render_html_success_filters_active_tools_and_repeat_is_internal():def test_strip_result_for_model_removes_frontend_image_sentinel():- assert strip_result_for_model('text\n__IMAGES__:{"paths":[]}') == "text"- assert strip_result_for_model("text __IMAGES__:payload") == "text"+ assert strip_result_for_model('text\n__IMAGES__:["a.png"]') == "text"assert strip_result_for_model("plain text") == "plain text"+def test_a_result_that_only_quotes_the_image_or_source_marker_is_kept_whole():+ """Only a structurally valid trailing envelope is stripped, the way+ `_strip_files_sentinel` and `_strip_mcp_image_suffix` beside it already are."""+ quoted = 'tools.py:16134: out += f"\\n__IMAGES__:{_json.dumps(images)}"\nsecond hit\nthird hit'+ for name in (None, "terminal", "mcp__fs__read"):+ assert strip_result_for_model(quoted, name) == quoted++ # What `_defuse_sentinels` leaves behind when a program prints the marker itself.+ defused = "line one\n __IMAGES__:printed by the program\nline three"+ assert strip_result_for_model(defused, "python") == defused++ sources = 'line one\nRAG_SOURCES_SENTINEL = "\\n__RAG_SOURCES__:"\nline three'+ assert strip_result_for_model(sources, "search_knowledge_base") == sources++ assert (+ strip_result_for_model('text\n__IMAGES__:{"paths":[]}') == 'text\n__IMAGES__:{"paths":[]}'+ )+ assert strip_result_for_model('output\n__IMAGES__:["a.png"]', "python") == "output"+ assert (+ strip_result_for_model(+ 'answer\n__RAG_SOURCES__:[{"filename": "a.pdf"}]', "search_knowledge_base"+ )+ == "answer"+ )+++def test_two_plots_in_one_code_execution_turn_replay_no_base64():+ """A Gemini `code_execution` turn that draws two figures stacks one envelope per+ `inlineData` part (external_provider re-appends to the result it just emitted), so+ peeling once would hand the model the earlier plot's whole data URI."""+ first = "data:image/png;base64," + "A" * 64+ second = "data:image/png;base64," + "B" * 64+ stacked = (+ "Figures saved."+ + f"\n__IMAGES__:{json.dumps([first])}"+ + f"\n__IMAGES__:{json.dumps([second])}"+ )++ assert strip_result_for_model(stacked, "code_execution") == "Figures saved."+++def test_a_flood_of_stacked_image_markers_stays_linear():+ """A hosted result's length is the provider's to choose and this runs on the request+ thread, so the cost has to follow it. Re-partitioning the shortened string once per+ marker copies it again every time: quadruple the markers and the work grows about+ sixteenfold instead of fourfold.++ Measured as CPU time, and as the best of several runs. A shared runner can deschedule+ the process mid-call, which adds wall clock but no CPU, and taking the minimum drops+ the samples where it happened -- interference can only ever make a run look slower.+ """++ def cost(markers: int) -> float:+ flood = '\n__IMAGES__:["x"]' * markers+ best = float("inf")+ for _ in range(5):+ started = time.process_time()+ assert strip_result_for_model(flood, "code_execution") == ""+ best = min(best, time.process_time() - started)+ return max(best, 1e-4)++ small = cost(20_000)+ large = cost(80_000)++ assert large / small < 10.0, f"4x the markers cost {large / small:.1f}x the work"+++def test_a_large_source_map_is_still_taken_off_the_result():+ """No length bound here either. Every source record repeats its whole chunk and+ `search_knowledge_base` honours the model's `top_k` without a ceiling, so a real map+ can be megabytes; one refused for being big would be left for `_fit_result_to_room`+ to cut into malformed JSON in front of the model."""+ import json as _json++ chunk = "retrieved text. " * 400+ sources = [+ {+ "citationId": i,+ "chunkId": f"c{i}",+ "filename": "handbook.pdf",+ "page": i,+ "text": chunk,+ "score": 0.5,+ }+ for i in range(200)+ ]+ result = "answer\n__RAG_SOURCES__:" + _json.dumps(sources, ensure_ascii = False)+ assert len(result) > 1 << 20++ assert strip_result_for_model(result, "search_knowledge_base") == "answer"+++def test_a_large_plot_is_still_taken_off_the_result():+ """The image payload has no length bound on purpose: a figure Gemini renders at high+ DPI is a data URI of whatever size it chose, and leaving it in for being large is the+ base64 leak this stripper exists to prevent."""+ big = 'output\n__IMAGES__:["data:image/png;base64,' + "A" * (12 << 20) + '"]'+ assert len(big) > 8 << 20+ assert strip_result_for_model(big, "code_execution") == "output"+++def test_only_the_tools_that_emit_an_envelope_have_one_taken_off():+ """`_strip_files_sentinel` is already scoped this way. A document an MCP tool read,+ or a page that was fetched, can end in a well-formed line of either kind, and it is+ content: the card keeps it, so cutting it leaves the model with less than the user+ is looking at."""+ manifest = 'icons/\n__IMAGES__:["icon.png"]'+ citation = 'notes\n__RAG_SOURCES__:[{"filename": "a.pdf"}]'++ for reader in ("mcp__fs__read_file", "web_search", "fetch_url"):+ assert strip_result_for_model(manifest, reader) == manifest+ assert strip_result_for_model(citation, reader) == citation++ # The tools that do emit them are unaffected.+ for emitter in ("python", "terminal", "code_execution"):+ assert strip_result_for_model(manifest, emitter) == "icons/"+ for emitter in ("search_knowledge_base", "search_conversation"):+ assert strip_result_for_model(citation, emitter) == "notes"++ # An unnamed caller still gets everything stripped, as it did before.+ assert strip_result_for_model(manifest) == "icons/"+ assert strip_result_for_model(citation) == "notes"++def test_the_card_text_keeps_digits_the_browser_would_round():"""`JSON.parse` reads 9007199254740993 back as ...992, so a card that re-encodes theparsed arguments in the browser would show a record the tool is not being run with."""diff --git a/studio/frontend/src/components/assistant-ui/sandbox-files.ts b/studio/frontend/src/components/assistant-ui/sandbox-files.tsindex 8663f799a73..0913f5068b9 100644--- a/studio/frontend/src/components/assistant-ui/sandbox-files.ts+++ b/studio/frontend/src/components/assistant-ui/sandbox-files.ts@@ -12,6 +12,14 @@ const FILES_MARKER = "\n__FILES__:";/** The tools that emit the file envelope. Nothing else's output is an envelope. */export const SANDBOX_FILE_TOOLS = new Set(["python", "terminal"]);+/**+ * The tools that emit the image envelope: the sandbox ones, and the provider's hosted+ * code execution, which attaches its plots the same way. The backend keeps a+ * well-formed `__IMAGES__` line from any other tool as content the model reads, so+ * the card has to keep it too, or the user sees less than the model was given.+ */+export const IMAGE_SENTINEL_TOOLS = new Set([...SANDBOX_FILE_TOOLS, "code_execution"]);+function isSandboxFile(entry: unknown): entry is SandboxFile {if (typeof entry !== "object" || entry === null) return false;const { name, size } = entry as { name?: unknown; size?: unknown };diff --git a/studio/frontend/src/features/chat/api/chat-adapter.ts b/studio/frontend/src/features/chat/api/chat-adapter.tsindex dca408f18c9..12c9030b675 100644--- a/studio/frontend/src/features/chat/api/chat-adapter.ts+++ b/studio/frontend/src/features/chat/api/chat-adapter.ts@@ -35,6 +35,7 @@ import { sanitizeStoredExtraArgs } from "@/features/model-picker/model-config/llimport { usePlatformStore } from "@/config/env";import { projectHasSources } from "@/features/rag/api/rag-api";import {+ IMAGE_SENTINEL_TOOLS,SANDBOX_FILE_TOOLS,extractCreatedFiles,isSandboxFileList,@@ -6667,7 +6668,14 @@ export function createOpenAIStreamAdapter(? extractSearchImages(rawResult): { text: rawResult, images: [] as SearchImageEntry[] };const imgMarker = "\n__IMAGES__:";- const imgIdx = rawResult.lastIndexOf(imgMarker);+ // Same rule again. The backend keeps this line for the model when the tool is not one that+ // emits the envelope, so the card keeps it too, rather than hiding it and fetching a+ // sandbox file that was never written.+ const imgIdx = IMAGE_SENTINEL_TOOLS.has(+ toolCallParts[idx].toolName ?? "",+ )+ ? rawResult.lastIndexOf(imgMarker)+ : -1;const mcpImgMarker = "\n__MCP_IMAGES__:";const mcpImgIdx = rawResult.lastIndexOf(mcpImgMarker);let parsedResult:diff --git a/studio/frontend/tests/image-sentinel-tools.test.ts b/studio/frontend/tests/image-sentinel-tools.test.tsnew file mode 100644index 00000000000..f20555be520--- /dev/null+++ b/studio/frontend/tests/image-sentinel-tools.test.ts@@ -0,0 +1,42 @@+// SPDX-License-Identifier: AGPL-3.0-only+// Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0++import assert from "node:assert/strict";+import test from "node:test";++import { readSrc } from "./helpers/kit.ts";++const { IMAGE_SENTINEL_TOOLS, SANDBOX_FILE_TOOLS } = await import(+ "../src/components/assistant-ui/sandbox-files.ts"+);++test("only the tools that emit the image envelope are in the set", () => {+ for (const emitter of ["python", "terminal", "code_execution"]) {+ assert.ok(IMAGE_SENTINEL_TOOLS.has(emitter), emitter);+ }+ for (const reader of ["mcp__fs__read_file", "web_search", "search_knowledge_base", ""]) {+ assert.ok(!IMAGE_SENTINEL_TOOLS.has(reader), reader);+ }+ for (const sandbox of SANDBOX_FILE_TOOLS) {+ assert.ok(IMAGE_SENTINEL_TOOLS.has(sandbox), `${sandbox} emits both envelopes`);+ }+});++// Asserted against the source like the other chat-adapter tests: importing the adapter+// drags in the whole app. The backend keeps a well-formed `__IMAGES__` line from a tool+// that does not emit the envelope as content the model reads, so the card must not slice+// it off and fetch a sandbox file that was never written.+test("the adapter slices the image envelope only for the tools that emit it", () => {+ const source = readSrc("features/chat/api/chat-adapter.ts");+ const gate = source.indexOf(+ 'IMAGE_SENTINEL_TOOLS.has(\n toolCallParts[idx].toolName ?? "",\n )',+ );+ const slice = source.indexOf("rawResult.lastIndexOf(imgMarker)");+ assert.ok(gate >= 0, "the __IMAGES__ slice is no longer gated on IMAGE_SENTINEL_TOOLS");+ assert.ok(slice > gate, "the slice runs before the gate");+ assert.equal(+ source.indexOf("rawResult.lastIndexOf(imgMarker)", slice + 1),+ -1,+ "a second, ungated __IMAGES__ slice appeared",+ );+});