Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
Studio: stop runaway tool output from using up memory (#11723)
Nilaya886277ec3fb
diff --git a/studio/backend/core/inference/tools.py b/studio/backend/core/inference/tools.pyindex 2201be4c9a2..8902e48e021 100644--- a/studio/backend/core/inference/tools.py+++ b/studio/backend/core/inference/tools.py@@ -7,6 +7,7 @@ execution, and terminal commands."""import astimport codecsimport copy+from collections import dequeimport fnmatchimport functoolsimport hashlib@@ -19397,11 +19398,16 @@ def _truncate(scope: "str | None" = "",hint: str = "",reserve_tokens: float = 0.0,+ omitted: "tuple[int, int]" = (0, 0),) -> str:# Resolved per call, not bound at import: the default would freeze the constant before any model is loaded, which# is exactly when the window is still unknown.if limit is None:limit = _tool_result_char_budget()+ # `omitted` is the (chars, lines) the drain dropped between its head and tail, so only the head may be shown.+ size = len(text) + omitted[0]+ if omitted[0]:+ limit = min(limit, _SPILL_MAX_BYTES)# Same correction as a fetched page: a character cap reserves its share of the window only for English, and a# command that prints CJK or percent-escaped text costs two to three times what the cap assumed.# Whatever the loop will append to this result once it has it: the tool-error nudge goes on after the tool has@@ -19443,7 +19449,7 @@ def _truncate(# to nothing.if _request_result_room() is not None:limit = _dense_char_limit(text, cap, cost + _RESULT_NOTICE_RESERVE)- if limit <= 0 and len(_zero_room_stub(len(text), None, True)) >= len(text):+ if limit <= 0 and len(_zero_room_stub(size, None, True)) >= len(text):# Decided BEFORE the spill: a result this short is served whole below, and writing a file (and creating the# spill directory) for output that is never cut is a side effect with nothing on the other side of it.return text + hint@@ -19452,7 +19458,7 @@ def _truncate(# No room for a body, so no room for the usual notice either: at this point the notice IS the message, and the# full one costs ~90 tokens of a budget that just reported none. Kept to a line so the thread stays servable# and the next fit can evict older turns and recover, which is the whole reason a stub beats a refusal.- stub = _zero_room_stub(len(text), spill, complete)+ stub = _zero_room_stub(size, spill, complete)# A short result costs less than the notice explaining it is gone, and replacing "done" with a longer sentence# saves nothing and loses the answer.return (stub if len(stub) < len(text) else text) + hint@@ -19461,7 +19467,7 @@ def _truncate(return (head+ (- f"\n\n... (truncated to {limit} chars for the model; {len(text)} chars "+ f"\n\n... (truncated to {limit} chars for the model; {size} chars ""total. The full output is not retained here; any files the code wrote ""persist in the working directory.)")@@ -19476,7 +19482,7 @@ def _truncate(# blank line, where the head is "\n" alone and a count of two makes the hint resume at line 3, skipping the# first line the reader never saw.shown = 0 if not head else head.count("\n") + (0 if head.endswith("\n") else 1)- total = text.count("\n") + 1+ total = text.count("\n") + 1 + omitted[1]resume = f"sed -n '{shown + 1},{shown + max(1, shown)}p' {spill}"where = f"showing lines 1-{shown} of {total}"else:@@ -19491,11 +19497,11 @@ def _truncate(chunk = text[len(head) : len(head) * 2 or None]span = len(chunk.encode("utf-8", "surrogatepass"))resume = f"tail -c +{offset + 1} {spill} | head -c {max(1, span)}"- where = f"showing the first {len(head)} chars of {len(text)}"+ where = f"showing the first {len(head)} chars of {size}"# The workdir sentence stays whatever else the notice says: it is about the files the CODE wrote, not the spill,# and it is the only thing telling the model those survive.common = (- f"\n\n... (truncated to {limit} chars for the model; {where}, {len(text)} chars "+ f"\n\n... (truncated to {limit} chars for the model; {where}, {size} chars "f"total. {_capitalise(_spill_phrase(spill, complete))}, and any files the code ""wrote persist in the working directory")@@ -20453,6 +20459,10 @@ def _missing_path_hint(output: str, workdir: str | None = None) -> str:)+# Kept past the spill's head so a trailing traceback still reaches `_missing_path_hint`; also the drain's read size.+_DRAIN_TAIL_CHARS = 64 * 1024++def _drain_process_output(proc,timeout,@@ -20460,17 +20470,21 @@ def _drain_process_output(cancel_event = None,*,pgid = None,-) -> tuple[str, bool]:- """``proc.communicate(timeout=...)`` equivalent that also streams each stdout line to- ``output_callback`` as it is produced.+) -> "tuple[str, bool, tuple[int, int]]":+ """``proc.communicate(timeout=...)`` equivalent that also streams each stdout line (in+ ``_DRAIN_TAIL_CHARS`` pieces when longer) to ``output_callback`` as it is produced.- Returns ``(output, timed_out)``. The joined output is identical to what ``communicate`` would+ Returns ``(output, timed_out, omitted)``. The joined output is what ``communicate`` wouldreturn: the same TextIOWrapper decodes the stream, so encoding, error replacement, and newline- translation all match. On timeout the process tree is killed (mirroring the non-streaming path).+ translation all match. Past the spill's head only a rolling tail is kept, and ``omitted`` is the+ ``(chars, lines)`` dropped between them. On timeout the process tree is killed (mirroring the+ non-streaming path).With ``timeout=None`` the drain waits for EOF like ``communicate`` would, stopping early onlywhen ``cancel_event`` is set."""chunks: list[str] = []+ tail: "deque[str]" = deque()+ kept = joined = tail_chars = omitted_chars = omitted_lines = 0# Captured before waiting so a stdout-holding grandchild can still be killed after the leader is reaped (getpgid# then fails). Callers pass it in from right after Popen; fall back to capturing here for direct callers.@@ -20478,9 +20492,26 @@ def _drain_process_output(pgid = _capture_process_group(proc)def _reader() -> None:+ nonlocal kept, joined, tail_chars, omitted_chars, omitted_linestry:- for line in iter(proc.stdout.readline, ""):- chunks.append(line)+ # Sized reads: a newline-free stream would otherwise arrive as one unbounded "line".+ for line in iter(lambda: proc.stdout.readline(_DRAIN_TAIL_CHARS), ""):+ # Chars against a byte cap: UTF-8 never has fewer bytes than chars, so the head covers the spill.+ if kept <= _SPILL_MAX_BYTES:+ chunks.append(line)+ kept += len(line)+ # A str per line costs ~50 bytes, so a flood of tiny lines would dwarf the cap unless coalesced.+ if len(chunks) - joined >= 1024:+ chunks[joined:] = ["".join(chunks[joined:])]+ joined += 1+ else:+ tail.append(line)+ tail_chars += len(line)+ while tail_chars > _DRAIN_TAIL_CHARS and len(tail) > 1:+ gone = tail.popleft()+ tail_chars -= len(gone)+ omitted_chars += len(gone)+ omitted_lines += gone.count("\n")if output_callback is not None:try:output_callback(line)@@ -20531,7 +20562,7 @@ def _drain_process_output(breakreader.join(timeout = 0.5)reader.join(timeout = 5)- return "".join(chunks), timed_out+ return "".join(chunks) + "".join(tail), timed_out, (omitted_chars, omitted_lines)_MAX_REPORTED_FILES = 25@@ -20815,7 +20846,11 @@ def _created_file_sentinels(def _timed_out_result(- output: str | None, timeout: int, workdir: str | None, scope: "str | None"+ output: str | None,+ timeout: int,+ workdir: str | None,+ scope: "str | None",+ omitted: "tuple[int, int]" = (0, 0),) -> str:"""Captured output, then the timeout status line.@@ -20833,6 +20868,7 @@ def _timed_out_result(workdir = workdir,scope = scope,reserve_tokens = _text_token_cost(f"\n{ended}", ctx),+ omitted = omitted,)result = f"{head}\n{ended}"# With the retry nudge, a stub or short head served whole can overrun a room the status fits.@@ -20856,7 +20892,7 @@ def _python_exec() -> str:"""Execute Python code in a subprocess sandbox. disable_sandbox (Bypass Permissions): skip thesafety analysis and rlimit pre-exec, and use the host env minus secrets. output_callback:- optional callable(str) streamed each stdout line as it is produced; the returned result is+ optional callable(str) streamed stdout as it is produced; the returned result isunchanged. tool_execution_mode selects automatic or required OS isolation; disable_sandboxkeeps full access as a separate explicit choice."""if not code or not code.strip():@@ -20987,9 +21023,9 @@ def _python_exec(watcher.start()# Always drain via _drain_process_output (output_callback may be None): it kills the captured group on- # cancellation, reaping a grandchild that outlived the leader, and returns bytes identical to communicate() so- # the streaming vs non-streaming result stays byte-identical.- output, timed_out = _drain_process_output(+ # cancellation, reaping a grandchild that outlived the leader, and keeps the same bytes with or without a+ # callback so the streaming vs non-streaming result stays byte-identical.+ output, timed_out, omitted = _drain_process_output(proc, timeout, output_callback, cancel_event, pgid = pgid)if prepared is not None:@@ -21010,7 +21046,7 @@ def _python_exec(# A run that wrote its file and then hung still produced that file, so report it: `printf data > report.csv;# sleep 999` is downloadable.if timed_out:- ended = _timed_out_result(output, timeout, spill_dir, spill_scope)+ ended = _timed_out_result(output, timeout, spill_dir, spill_scope, omitted)return ended + (_created_file_sentinels(workdir, _before, _scratch_name, call_token)if session_id@@ -21037,7 +21073,7 @@ def _python_exec(# envelope.result = _defuse_sentinels(result)result = (- _truncate(result, workdir = spill_dir, scope = spill_scope, hint = hint)+ _truncate(result, workdir = spill_dir, scope = spill_scope, hint = hint, omitted = omitted)if result.strip()else "(no output)" + hint)@@ -21093,8 +21129,8 @@ def _bash_exec() -> str:"""Execute a bash command in a subprocess sandbox. disable_sandbox (Bypass Permissions): skipthe command blocklist and rlimit pre-exec, and use the host env minus secrets.- output_callback: optional callable(str) streamed each stdout line as it is produced; the- returned result is unchanged. tool_execution_mode follows _python_exec."""+ output_callback: optional callable(str) streamed stdout as it is produced; the returned+ result is unchanged. tool_execution_mode follows _python_exec."""if not command or not command.strip():return "No command provided."@@ -21207,8 +21243,8 @@ def _bash_exec(watcher.start()# Always drain via _drain_process_output (see _python_exec): kills the captured group on cancellation and- # returns bytes identical to communicate(), keeping streaming vs non-streaming byte-identical.- output, timed_out = _drain_process_output(+ # keeps the same bytes with or without a callback, so streaming vs non-streaming stays byte-identical.+ output, timed_out, omitted = _drain_process_output(proc, timeout, output_callback, cancel_event, pgid = pgid)if prepared is not None:@@ -21229,7 +21265,7 @@ def _bash_exec(# A run that wrote its file and then hung still produced that file, so report it: `printf data > report.csv;# sleep 999` is downloadable.if timed_out:- ended = _timed_out_result(output, timeout, spill_dir, spill_scope)+ ended = _timed_out_result(output, timeout, spill_dir, spill_scope, omitted)return ended + (_created_file_sentinels(workdir, _before, _scratch_name, call_token)if session_id@@ -21250,7 +21286,7 @@ def _bash_exec(hint = _missing_path_hint(result, workdir)result = _defuse_sentinels(result) # before the fit; see _python_execresult = (- _truncate(result, workdir = spill_dir, scope = spill_scope, hint = hint)+ _truncate(result, workdir = spill_dir, scope = spill_scope, hint = hint, omitted = omitted)if result.strip()else "(no output)" + hint)diff --git a/studio/backend/tests/test_tool_output_streaming.py b/studio/backend/tests/test_tool_output_streaming.pyindex 84420d6ecdb..6aabed82f6b 100644--- a/studio/backend/tests/test_tool_output_streaming.py+++ b/studio/backend/tests/test_tool_output_streaming.py@@ -730,7 +730,7 @@ def test_drain_process_output_without_posix_process_group_apis(monkeypatch):stderr = _sp.STDOUT,text = True,)- output, timed_out = _drain_process_output(proc, 10, lambda _t: None)+ output, timed_out, _ = _drain_process_output(proc, 10, lambda _t: None)assert not timed_outassert "ok-no-pgid" in output@@ -757,7 +757,7 @@ def test_captured_group_survives_fast_leader_reap(tmp_path):assert pgid is not Noneproc.wait() # reap the leader before draining- output, timed_out = _drain_process_output(proc, 0.5, None, pgid = pgid)+ output, timed_out, _ = _drain_process_output(proc, 0.5, None, pgid = pgid)assert timed_outassert "parent-done" in output_assert_grandchild_was_killed(gate, sentinel)@@ -804,7 +804,7 @@ def test_finite_drain_honors_cancel_after_leader_exit(tmp_path):started = time.monotonic()# Large finite timeout (30s); without the cancel poll the drain keeps reading# the grandchild until the pipe closes ~20s later.- output, timed_out = _drain_process_output(proc, 30, lambda _t: None, cancel_event, pgid = pgid)+ output, timed_out, _ = _drain_process_output(proc, 30, lambda _t: None, cancel_event, pgid = pgid)elapsed = time.monotonic() - startedassert elapsed < 5.0, f"finite drain ignored cancel_event (took {elapsed:.1f}s)"# Cancellation is not a timeout: the budget never elapsed.@@ -841,7 +841,7 @@ def test_streamed_wait_timeout_kills_grandchild_when_leader_reaped(tmp_path, monpgid = _capture_process_group(proc)assert pgid is not None- output, timed_out = _drain_process_output(proc, 0.5, None, pgid = pgid)+ output, timed_out, _ = _drain_process_output(proc, 0.5, None, pgid = pgid)assert timed_out_assert_grandchild_was_killed(gate, sentinel)@@ -1005,6 +1005,124 @@ def test_truncated_result_identical_and_notice_neutral_with_streaming():assert "persist in the working directory" in baseline+def test_drain_retains_bounded_head_and_tail_of_runaway_output(monkeypatch):+ import subprocess as _sp++ from core.inference import tools as _tools_mod++ monkeypatch.setattr(_tools_mod, "_SPILL_MAX_BYTES", 1000)+ monkeypatch.setattr(_tools_mod, "_DRAIN_TAIL_CHARS", 500)+ proc = _sp.Popen(+ [sys.executable, "-c", "for i in range(5000): print(f'{i:09d}')"],+ stdout = _sp.PIPE,+ stderr = _sp.STDOUT,+ text = True,+ )+ streamed = []+ output, timed_out, (chars, lines) = _tools_mod._drain_process_output(proc, 30, streamed.append)+ full = "".join(streamed)+ assert not timed_out+ assert len(full) == 50000+ assert len(output) <= 1010 + 510+ assert output.startswith(full[:1000]) and full.endswith(output[-500:])+ assert len(output) + chars == len(full)+ assert output.count("\n") + lines == 5000+++def test_drain_bounds_runaway_output_without_newlines(monkeypatch):+ import subprocess as _sp++ from core.inference import tools as _tools_mod++ monkeypatch.setattr(_tools_mod, "_SPILL_MAX_BYTES", 1000)+ monkeypatch.setattr(_tools_mod, "_DRAIN_TAIL_CHARS", 500)+ proc = _sp.Popen(+ [sys.executable, "-c", "import sys; sys.stdout.write('x' * 50000)"],+ stdout = _sp.PIPE,+ stderr = _sp.STDOUT,+ text = True,+ )+ streamed = []+ output, timed_out, (chars, lines) = _tools_mod._drain_process_output(proc, 30, streamed.append)+ assert not timed_out+ assert "".join(streamed) == "x" * 50000+ assert len(output) <= 1500 + 500+ assert output == "x" * len(output)+ assert len(output) + chars == 50000+ assert lines == 0+++def test_drain_head_memory_tracks_chars_not_line_count(monkeypatch):+ import subprocess as _sp+ import tracemalloc++ from core.inference import tools as _tools_mod++ monkeypatch.setattr(_tools_mod, "_SPILL_MAX_BYTES", 1_000_000)+ monkeypatch.setattr(_tools_mod, "_DRAIN_TAIL_CHARS", 500)+ proc = _sp.Popen(+ [sys.executable, "-c", "import sys\nfor _ in range(600000): sys.stdout.write('x\\n')"],+ stdout = _sp.PIPE,+ stderr = _sp.STDOUT,+ text = True,+ )+ tracemalloc.start()+ try:+ output, timed_out, (chars, lines) = _tools_mod._drain_process_output(proc, 60, None)+ _, peak = tracemalloc.get_traced_memory()+ finally:+ tracemalloc.stop()+ assert not timed_out+ assert len(output) + chars == 1_200_000+ assert output.count("\n") + lines == 600_000+ # ~500k head lines: one str each is ~30 MB, coalesced it is a few copies of the 1 MB head.+ assert peak < 8 * 1024 * 1024, f"drain peaked at {peak / 2**20:.1f} MiB for a 1 MB head"+++def test_runaway_output_reports_true_size_and_keeps_trailing_hint(monkeypatch):+ from core.inference import tools as _tools_mod++ monkeypatch.setattr(_tools_mod, "_SPILL_MAX_BYTES", 2000)+ monkeypatch.setattr(_tools_mod, "_DRAIN_TAIL_CHARS", 2000)+ code = "for i in range(3000): print('x' * 99)\nopen('/mnt/data/x.html')\n"+ streamed = []+ baseline = _python_exec(code, timeout = 60)+ out = _python_exec(code, timeout = 60, output_callback = streamed.append)+ assert out == baseline+ total = len("Exit code 1:\n") + len("".join(streamed))+ assert f"{total} chars total" in out+ assert "'x.html', not '/mnt/data/x.html'" in out+++ "run, code",+ [+ (_python_exec, "while True: print('x' * 99)"),+ (_bash_exec, "while :; do printf '%099d\\n' 0; done"),+ ],+ ids = ["python", "bash"],+)+def test_timed_out_runaway_output_reports_true_size_and_line_count(+ monkeypatch, tmp_path, run, code+):+ from core.inference import tools as _tools_mod++ monkeypatch.setattr(_tools_mod, "_SPILL_MAX_BYTES", 2000)+ monkeypatch.setattr(_tools_mod, "_DRAIN_TAIL_CHARS", 500)+ monkeypatch.setattr(_tools_mod, "_get_workdir", lambda _sid = None: str(tmp_path))+ streamed = []+ out = run(+ code,+ timeout = 2,+ session_id = "runaway",+ output_callback = streamed.append,+ )+ full = "".join(streamed)+ assert len(full) > 100_000+ assert f"of {full.count(chr(10)) + 1}, {len(full)} chars total" in out+ assert out.rstrip().endswith("Execution timed out after 2 seconds.")++def test_result_cap_env_override(monkeypatch):monkeypatch.delenv("UNSLOTH_TOOL_RESULT_MAX_CHARS", raising = False)assert _env_int("UNSLOTH_TOOL_RESULT_MAX_CHARS", 16000) == 16000