Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
Studio: stop exports timing out while they are still working (#10258)
Nilay31330653f97d
diff --git a/studio/backend/core/export/orchestrator.py b/studio/backend/core/export/orchestrator.pyindex d357b1e0998..a6030f58136 100644--- a/studio/backend/core/export/orchestrator.py+++ b/studio/backend/core/export/orchestrator.py@@ -31,6 +31,14 @@ _CTX = mp.get_context("spawn")# Max log lines kept per orchestrator (live log panel scrollback); ~1 MB worst-case._LOG_BUFFER_MAXLEN = 4000+# How long an export op may go without a single log or status line before the worker is treated as+# dead. Not how long an export may run: a reporting worker resets it on every line.+_EXPORT_INACTIVITY_TIMEOUT = 3600.0++# Teardown is a bounded amount of work, so it is capped outright rather than by silence: the log+# gate an export opens is never closed, and teardown chatter would otherwise renew this forever.+_CLEANUP_TIMEOUT = 30.0+class ExportOrchestrator:"""@@ -334,14 +342,25 @@ class ExportOrchestrator:def _wait_response(self,expected_type: str,- timeout: float = 3600.0,+ timeout: float = _EXPORT_INACTIVITY_TIMEOUT,+ max_wait: Optional[float] = None,) -> dict:"""Block until a response of the expected type arrives.- Export ops can take a long time — GGUF conversion for large- models (30B+) easily takes 20-30 minutes. Default timeout 1 hour.+ *timeout* is an **inactivity** timeout: it resets on each log and status message, so a+ large export survives as long as the worker keeps reporting. Matches the inference side.++ *max_wait* additionally caps the total wait, for ops that must fail fast: a short+ inactivity budget alone is not one, since any line printed renews it."""- deadline = time.monotonic() + timeout+ started = time.monotonic()+ hard_deadline = None if max_wait is None else started + max_wait++ def renew() -> float:+ extended = time.monotonic() + timeout+ return extended if hard_deadline is None else min(extended, hard_deadline)++ deadline = renew()while time.monotonic() < deadline:remaining = max(0.1, deadline - time.monotonic())@@ -364,6 +383,7 @@ class ExportOrchestrator:if rtype == "log":# Forwarded stdout/stderr line from the worker.self._append_log(resp)+ deadline = renew()continueif rtype == "status":@@ -379,6 +399,7 @@ class ExportOrchestrator:"ts": resp.get("ts", time.time()),})+ deadline = renew()continue# Other response types during wait - skip.@@ -388,7 +409,13 @@ class ExportOrchestrator:expected_type,)- raise RuntimeError(f"Timeout waiting for '{expected_type}' response after {timeout}s")+ if hard_deadline is not None and time.monotonic() >= hard_deadline:+ raise RuntimeError(+ f"Timeout waiting for '{expected_type}' response (gave up after {max_wait}s)"+ )+ raise RuntimeError(+ f"Timeout waiting for '{expected_type}' response (no activity for {timeout}s)"+ )def _drain_queue(self) -> list:"""Drain all pending responses."""@@ -632,13 +659,15 @@ class ExportOrchestrator:cmd = {"type": "export", "export_type": export_type, **params}try:self._send_cmd(cmd)- # GGUF for a 30B+ model can take 30+ min per quant, and a multi-quant list runs every quant in one- # op off a single merge, so scale the timeout by quant count.+ # Scaled by quant count because the budget is silence, not duration: a+ # multi-quant list runs every pass in one op, and the quant passes emit+ # nothing (Studio leaves UNSLOTH_ENABLE_LOGGING unset, which is what makes+ # save_pretrained_gguf quantize without streaming). One hour per silent pass._qm = params.get("quantization_method")_n = len(_qm) if isinstance(_qm, (list, tuple)) and _qm else 1resp = self._wait_response(f"export_{export_type}_done",- timeout = 3600 * max(1, _n),+ timeout = _EXPORT_INACTIVITY_TIMEOUT * max(1, _n),)op_success = resp.get("success", False)op_message = resp.get("message", "")@@ -667,7 +696,11 @@ class ExportOrchestrator:try:try:self._send_cmd({"type": "cleanup"})- resp = self._wait_response("cleanup_done", timeout = 30)+ resp = self._wait_response(+ "cleanup_done",+ timeout = _CLEANUP_TIMEOUT,+ max_wait = _CLEANUP_TIMEOUT,+ )success = resp.get("success", False)except RuntimeError:success = Falsediff --git a/studio/backend/tests/test_export_wait_inactivity_timeout.py b/studio/backend/tests/test_export_wait_inactivity_timeout.pynew file mode 100644index 00000000000..6b805918864--- /dev/null+++ b/studio/backend/tests/test_export_wait_inactivity_timeout.py@@ -0,0 +1,164 @@+# SPDX-License-Identifier: AGPL-3.0-only+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0++"""The export response wait must be an INACTIVITY timeout, not an absolute deadline.++An absolute deadline cannot tell a busy worker from a hung one, so a large export dies at exactly+one hour and the cleanup that follows SIGKILLs it mid-write, leaving a half-written model on disk.+"""++from __future__ import annotations++import sys+import types+from pathlib import Path++import pytest+++_BACKEND_DIR = Path(__file__).resolve().parent.parent+if str(_BACKEND_DIR) not in sys.path:+ sys.path.insert(0, str(_BACKEND_DIR))++TIMEOUT = 10.0+ONE_HOUR = 3600.0+READ_SECONDS = 4.0+++def waiting_orchestrator(monkeypatch):+ """(orchestrator, clock, script) on a fake clock; a read past the end of *script* is a quiet one."""+ import time as real_time++ from core.export import orchestrator as orchestrator_module++ clock = types.SimpleNamespace(now = 0.0)+ # Swap the module reference, not `time.monotonic` itself: patching the attribute would hand the+ # frozen clock to every other thread in the process for the length of the test.+ fake_time = types.SimpleNamespace(monotonic = lambda: clock.now, time = real_time.time)+ monkeypatch.setattr(orchestrator_module, "time", fake_time)++ orch = orchestrator_module.ExportOrchestrator()+ script: list = []++ def fake_read(timeout = None):+ # A real read blocks for at most the timeout it was given, so charge the clock the same.+ clock.now += min(READ_SECONDS, timeout) if timeout is not None else READ_SECONDS+ return script.pop(0) if script else None++ monkeypatch.setattr(orch, "_read_resp", fake_read)+ monkeypatch.setattr(orch, "_ensure_subprocess_alive", lambda: True)+ return orch, clock, script+++def test_a_worker_that_keeps_logging_survives_past_the_timeout(waiting_orchestrator) -> None:+ orch, clock, script = waiting_orchestrator+ script.extend(+ [+ {"type": "log", "stream": "stdout", "line": f"writing shard {n}", "ts": 0.0}+ for n in range(6)+ ]+ )+ script.append({"type": "export_merged_done", "path": "/out/model"})++ resp = orch._wait_response("export_merged_done", timeout = TIMEOUT)++ assert resp["type"] == "export_merged_done"+ assert clock.now > TIMEOUT, "the fixture must run the wait past the timeout to be meaningful"+++def test_a_status_message_also_resets_the_deadline(waiting_orchestrator) -> None:+ orch, clock, script = waiting_orchestrator+ script.extend(+ [{"type": "status", "message": f"Quantizing block {n}", "ts": 0.0} for n in range(6)]+ )+ script.append({"type": "export_gguf_done", "path": "/out/model.gguf"})++ assert orch._wait_response("export_gguf_done", timeout = TIMEOUT)["type"] == "export_gguf_done"+ assert clock.now > TIMEOUT+++def test_a_quiet_worker_still_times_out(waiting_orchestrator) -> None:+ orch, clock, _script = waiting_orchestrator++ with pytest.raises(RuntimeError):+ orch._wait_response("export_merged_done", timeout = TIMEOUT)++ assert clock.now < TIMEOUT * 2, "a quiet wait must end near the timeout, not run on"+++def test_max_wait_caps_a_chatty_wait(waiting_orchestrator) -> None:+ """Cleanup must fail fast even though the worker is still printing.++ The log gate the worker opens for an export is never closed again, so teardown chatter reaches+ a wait whose short budget exists precisely to give up quickly.+ """+ orch, clock, script = waiting_orchestrator+ script.extend(+ [+ {"type": "log", "stream": "stdout", "line": f"freeing buffer {n}", "ts": 0.0}+ for n in range(50)+ ]+ )++ with pytest.raises(RuntimeError, match = "gave up after"):+ orch._wait_response("cleanup_done", timeout = TIMEOUT, max_wait = TIMEOUT)++ assert clock.now < TIMEOUT * 2, "the cap must hold regardless of how much the worker prints"+ assert script, "the wait must give up with the worker still talking, not drain the script"+++def test_cleanup_passes_a_hard_cap(monkeypatch) -> None:+ """The cap belongs to the cleanup call site, not just to _wait_response."""+ from core.export import orchestrator as orchestrator_module++ seen: list = []+ orch = orchestrator_module.ExportOrchestrator()+ monkeypatch.setattr(orch, "_ensure_subprocess_alive", lambda: True)+ monkeypatch.setattr(orch, "_send_cmd", lambda cmd: None)+ monkeypatch.setattr(orch, "_shutdown_subprocess", lambda *a, **k: True)+ monkeypatch.setattr(+ orch,+ "_wait_response",+ lambda expected_type, timeout = None, max_wait = None: (+ seen.append((timeout, max_wait)) or {"success": True}+ ),+ )++ cap = orchestrator_module._CLEANUP_TIMEOUT+ assert orch.cleanup_memory() is True+ assert seen == [(cap, cap)], seen+++def test_a_multi_quant_export_is_allowed_to_stay_silent_for_the_whole_batch(monkeypatch) -> None:+ """The silence budget scales with quant count, because the batch reports nothing while it runs.++ Studio never sets UNSLOTH_ENABLE_LOGGING, which is the condition save.py needs to run the quant+ passes in parallel, and that branch prints once and then waits on all of them. Flattening this+ to one hour kills a 12-quant export mid-write, which is the failure this file exists to prevent.+ """+ from core.export import orchestrator as orchestrator_module++ # _run_export imports this at call time. Injected per test and undone after: a module-level+ # sys.modules entry would shadow the real utils.transformers_version for the whole session,+ # and the rest of the backend suite imports a dozen names from it.+ tv_stub = types.ModuleType("utils.transformers_version")+ tv_stub.sidecar_swap_in_progress = lambda: False+ tv_stub.SidecarSwapInProgress = type("SidecarSwapInProgress", (RuntimeError,), {})+ monkeypatch.setitem(sys.modules, "utils.transformers_version", tv_stub)++ seen: list = []++ def record(expected_type, timeout = None):+ seen.append(timeout)+ return {"success": True, "message": "", "output_path": None}++ orch = orchestrator_module.ExportOrchestrator()+ monkeypatch.setattr(orch, "_ensure_subprocess_alive", lambda: True)+ monkeypatch.setattr(orch, "_send_cmd", lambda cmd: None)+ monkeypatch.setattr(orch, "_wait_response", record)++ orch._run_export("gguf", {"quantization_method": "Q4_K_M"})+ orch._run_export("gguf", {"quantization_method": ["Q4_K_M"] * 12})++ assert seen == [ONE_HOUR, ONE_HOUR * 12], seen