Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

Studio: stop exports timing out while they are still working (#10258)

Nilay31330653f97d
diff --git a/studio/backend/core/export/orchestrator.py b/studio/backend/core/export/orchestrator.py
index d357b1e0998..a6030f58136 100644
--- a/studio/backend/core/export/orchestrator.py
+++ b/studio/backend/core/export/orchestrator.py
@@ -31,6 +31,14 @@ _CTX = mp.get_context("spawn")
# Max log lines kept per orchestrator (live log panel scrollback); ~1 MB worst-case.
_LOG_BUFFER_MAXLEN = 4000
+# How long an export op may go without a single log or status line before the worker is treated as
+# dead. Not how long an export may run: a reporting worker resets it on every line.
+_EXPORT_INACTIVITY_TIMEOUT = 3600.0
+
+# Teardown is a bounded amount of work, so it is capped outright rather than by silence: the log
+# gate an export opens is never closed, and teardown chatter would otherwise renew this forever.
+_CLEANUP_TIMEOUT = 30.0
+
class ExportOrchestrator:
"""
@@ -334,14 +342,25 @@ class ExportOrchestrator:
def _wait_response(
self,
expected_type: str,
- timeout: float = 3600.0,
+ timeout: float = _EXPORT_INACTIVITY_TIMEOUT,
+ max_wait: Optional[float] = None,
) -> dict:
"""Block until a response of the expected type arrives.
- Export ops can take a long time — GGUF conversion for large
- models (30B+) easily takes 20-30 minutes. Default timeout 1 hour.
+ *timeout* is an **inactivity** timeout: it resets on each log and status message, so a
+ large export survives as long as the worker keeps reporting. Matches the inference side.
+
+ *max_wait* additionally caps the total wait, for ops that must fail fast: a short
+ inactivity budget alone is not one, since any line printed renews it.
"""
- deadline = time.monotonic() + timeout
+ started = time.monotonic()
+ hard_deadline = None if max_wait is None else started + max_wait
+
+ def renew() -> float:
+ extended = time.monotonic() + timeout
+ return extended if hard_deadline is None else min(extended, hard_deadline)
+
+ deadline = renew()
while time.monotonic() < deadline:
remaining = max(0.1, deadline - time.monotonic())
@@ -364,6 +383,7 @@ class ExportOrchestrator:
if rtype == "log":
# Forwarded stdout/stderr line from the worker.
self._append_log(resp)
+ deadline = renew()
continue
if rtype == "status":
@@ -379,6 +399,7 @@ class ExportOrchestrator:
"ts": resp.get("ts", time.time()),
}
)
+ deadline = renew()
continue
# Other response types during wait - skip.
@@ -388,7 +409,13 @@ class ExportOrchestrator:
expected_type,
)
- raise RuntimeError(f"Timeout waiting for '{expected_type}' response after {timeout}s")
+ if hard_deadline is not None and time.monotonic() >= hard_deadline:
+ raise RuntimeError(
+ f"Timeout waiting for '{expected_type}' response (gave up after {max_wait}s)"
+ )
+ raise RuntimeError(
+ f"Timeout waiting for '{expected_type}' response (no activity for {timeout}s)"
+ )
def _drain_queue(self) -> list:
"""Drain all pending responses."""
@@ -632,13 +659,15 @@ class ExportOrchestrator:
cmd = {"type": "export", "export_type": export_type, **params}
try:
self._send_cmd(cmd)
- # GGUF for a 30B+ model can take 30+ min per quant, and a multi-quant list runs every quant in one
- # op off a single merge, so scale the timeout by quant count.
+ # Scaled by quant count because the budget is silence, not duration: a
+ # multi-quant list runs every pass in one op, and the quant passes emit
+ # nothing (Studio leaves UNSLOTH_ENABLE_LOGGING unset, which is what makes
+ # save_pretrained_gguf quantize without streaming). One hour per silent pass.
_qm = params.get("quantization_method")
_n = len(_qm) if isinstance(_qm, (list, tuple)) and _qm else 1
resp = self._wait_response(
f"export_{export_type}_done",
- timeout = 3600 * max(1, _n),
+ timeout = _EXPORT_INACTIVITY_TIMEOUT * max(1, _n),
)
op_success = resp.get("success", False)
op_message = resp.get("message", "")
@@ -667,7 +696,11 @@ class ExportOrchestrator:
try:
try:
self._send_cmd({"type": "cleanup"})
- resp = self._wait_response("cleanup_done", timeout = 30)
+ resp = self._wait_response(
+ "cleanup_done",
+ timeout = _CLEANUP_TIMEOUT,
+ max_wait = _CLEANUP_TIMEOUT,
+ )
success = resp.get("success", False)
except RuntimeError:
success = False
diff --git a/studio/backend/tests/test_export_wait_inactivity_timeout.py b/studio/backend/tests/test_export_wait_inactivity_timeout.py
new file mode 100644
index 00000000000..6b805918864
--- /dev/null
+++ b/studio/backend/tests/test_export_wait_inactivity_timeout.py
@@ -0,0 +1,164 @@
+# SPDX-License-Identifier: AGPL-3.0-only
+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
+
+"""The export response wait must be an INACTIVITY timeout, not an absolute deadline.
+
+An absolute deadline cannot tell a busy worker from a hung one, so a large export dies at exactly
+one hour and the cleanup that follows SIGKILLs it mid-write, leaving a half-written model on disk.
+"""
+
+from __future__ import annotations
+
+import sys
+import types
+from pathlib import Path
+
+import pytest
+
+
+_BACKEND_DIR = Path(__file__).resolve().parent.parent
+if str(_BACKEND_DIR) not in sys.path:
+ sys.path.insert(0, str(_BACKEND_DIR))
+
+TIMEOUT = 10.0
+ONE_HOUR = 3600.0
+READ_SECONDS = 4.0
+
+
+def waiting_orchestrator(monkeypatch):
+ """(orchestrator, clock, script) on a fake clock; a read past the end of *script* is a quiet one."""
+ import time as real_time
+
+ from core.export import orchestrator as orchestrator_module
+
+ clock = types.SimpleNamespace(now = 0.0)
+ # Swap the module reference, not `time.monotonic` itself: patching the attribute would hand the
+ # frozen clock to every other thread in the process for the length of the test.
+ fake_time = types.SimpleNamespace(monotonic = lambda: clock.now, time = real_time.time)
+ monkeypatch.setattr(orchestrator_module, "time", fake_time)
+
+ orch = orchestrator_module.ExportOrchestrator()
+ script: list = []
+
+ def fake_read(timeout = None):
+ # A real read blocks for at most the timeout it was given, so charge the clock the same.
+ clock.now += min(READ_SECONDS, timeout) if timeout is not None else READ_SECONDS
+ return script.pop(0) if script else None
+
+ monkeypatch.setattr(orch, "_read_resp", fake_read)
+ monkeypatch.setattr(orch, "_ensure_subprocess_alive", lambda: True)
+ return orch, clock, script
+
+
+def test_a_worker_that_keeps_logging_survives_past_the_timeout(waiting_orchestrator) -> None:
+ orch, clock, script = waiting_orchestrator
+ script.extend(
+ [
+ {"type": "log", "stream": "stdout", "line": f"writing shard {n}", "ts": 0.0}
+ for n in range(6)
+ ]
+ )
+ script.append({"type": "export_merged_done", "path": "/out/model"})
+
+ resp = orch._wait_response("export_merged_done", timeout = TIMEOUT)
+
+ assert resp["type"] == "export_merged_done"
+ assert clock.now > TIMEOUT, "the fixture must run the wait past the timeout to be meaningful"
+
+
+def test_a_status_message_also_resets_the_deadline(waiting_orchestrator) -> None:
+ orch, clock, script = waiting_orchestrator
+ script.extend(
+ [{"type": "status", "message": f"Quantizing block {n}", "ts": 0.0} for n in range(6)]
+ )
+ script.append({"type": "export_gguf_done", "path": "/out/model.gguf"})
+
+ assert orch._wait_response("export_gguf_done", timeout = TIMEOUT)["type"] == "export_gguf_done"
+ assert clock.now > TIMEOUT
+
+
+def test_a_quiet_worker_still_times_out(waiting_orchestrator) -> None:
+ orch, clock, _script = waiting_orchestrator
+
+ with pytest.raises(RuntimeError):
+ orch._wait_response("export_merged_done", timeout = TIMEOUT)
+
+ assert clock.now < TIMEOUT * 2, "a quiet wait must end near the timeout, not run on"
+
+
+def test_max_wait_caps_a_chatty_wait(waiting_orchestrator) -> None:
+ """Cleanup must fail fast even though the worker is still printing.
+
+ The log gate the worker opens for an export is never closed again, so teardown chatter reaches
+ a wait whose short budget exists precisely to give up quickly.
+ """
+ orch, clock, script = waiting_orchestrator
+ script.extend(
+ [
+ {"type": "log", "stream": "stdout", "line": f"freeing buffer {n}", "ts": 0.0}
+ for n in range(50)
+ ]
+ )
+
+ with pytest.raises(RuntimeError, match = "gave up after"):
+ orch._wait_response("cleanup_done", timeout = TIMEOUT, max_wait = TIMEOUT)
+
+ assert clock.now < TIMEOUT * 2, "the cap must hold regardless of how much the worker prints"
+ assert script, "the wait must give up with the worker still talking, not drain the script"
+
+
+def test_cleanup_passes_a_hard_cap(monkeypatch) -> None:
+ """The cap belongs to the cleanup call site, not just to _wait_response."""
+ from core.export import orchestrator as orchestrator_module
+
+ seen: list = []
+ orch = orchestrator_module.ExportOrchestrator()
+ monkeypatch.setattr(orch, "_ensure_subprocess_alive", lambda: True)
+ monkeypatch.setattr(orch, "_send_cmd", lambda cmd: None)
+ monkeypatch.setattr(orch, "_shutdown_subprocess", lambda *a, **k: True)
+ monkeypatch.setattr(
+ orch,
+ "_wait_response",
+ lambda expected_type, timeout = None, max_wait = None: (
+ seen.append((timeout, max_wait)) or {"success": True}
+ ),
+ )
+
+ cap = orchestrator_module._CLEANUP_TIMEOUT
+ assert orch.cleanup_memory() is True
+ assert seen == [(cap, cap)], seen
+
+
+def test_a_multi_quant_export_is_allowed_to_stay_silent_for_the_whole_batch(monkeypatch) -> None:
+ """The silence budget scales with quant count, because the batch reports nothing while it runs.
+
+ Studio never sets UNSLOTH_ENABLE_LOGGING, which is the condition save.py needs to run the quant
+ passes in parallel, and that branch prints once and then waits on all of them. Flattening this
+ to one hour kills a 12-quant export mid-write, which is the failure this file exists to prevent.
+ """
+ from core.export import orchestrator as orchestrator_module
+
+ # _run_export imports this at call time. Injected per test and undone after: a module-level
+ # sys.modules entry would shadow the real utils.transformers_version for the whole session,
+ # and the rest of the backend suite imports a dozen names from it.
+ tv_stub = types.ModuleType("utils.transformers_version")
+ tv_stub.sidecar_swap_in_progress = lambda: False
+ tv_stub.SidecarSwapInProgress = type("SidecarSwapInProgress", (RuntimeError,), {})
+ monkeypatch.setitem(sys.modules, "utils.transformers_version", tv_stub)
+
+ seen: list = []
+
+ def record(expected_type, timeout = None):
+ seen.append(timeout)
+ return {"success": True, "message": "", "output_path": None}
+
+ orch = orchestrator_module.ExportOrchestrator()
+ monkeypatch.setattr(orch, "_ensure_subprocess_alive", lambda: True)
+ monkeypatch.setattr(orch, "_send_cmd", lambda cmd: None)
+ monkeypatch.setattr(orch, "_wait_response", record)
+
+ orch._run_export("gguf", {"quantization_method": "Q4_K_M"})
+ orch._run_export("gguf", {"quantization_method": ["Q4_K_M"] * 12})
+
+ assert seen == [ONE_HOUR, ONE_HOUR * 12], seen