Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
perf(studio): keep the MLX GPU warm for a bounded time after generation (#12721)
Long Yixingd4bfbd9f8d36
diff --git a/studio/backend/core/inference/worker.py b/studio/backend/core/inference/worker.pyindex 44617d0c1ce..8c33a9c0520 100644--- a/studio/backend/core/inference/worker.py+++ b/studio/backend/core/inference/worker.py@@ -1314,6 +1314,37 @@ def _held_head_leaves_the_hold(batch: "_ResidentBatch", held: list) -> bool:return head.get("type") == "generate" and batch.unavailable_reason(head) is None+class _MLXIdleWarmth:+ def __init__(self):+ self.enabled = os.environ.get("UNSLOTH_MLX_GPU_KEEP_WARM", "1").lower() in (+ "1",+ "true",+ "yes",+ )+ self.deadline = 0.0++ def active(self):+ self.deadline = time.monotonic() + 60.0++ def clear(self):+ self.deadline = 0.0++ def timeout(self, loaded):+ return 0.5 if self.enabled and loaded and time.monotonic() < self.deadline else 1.0++ def tick(self, loaded):+ if self.timeout(loaded) != 0.5:+ return+ try:+ import mlx.core as mx++ # Avoid the first-command stall after the GPU enters its idle power state.+ mx.eval(mx.zeros((1,), dtype = mx.float32) + 1)+ except Exception as exc:+ self.enabled = False+ logger.warning("MLX GPU keep-warm disabled after an idle tick failed: %s", exc)++class _ResidentBatch:"""The replies an MLX worker is decoding at once."""@@ -2025,6 +2056,7 @@ def run_inference_process(logger.info("MLX inference subprocess ready, entering command loop")batch = _ResidentBatch(backend, resp_queue)+ warmth = _MLXIdleWarmth()deferred: list[dict] = []stops = _Stops(stop_ledger, resp_queue, batch, deferred)if stop_ledger is not None:@@ -2033,6 +2065,8 @@ def run_inference_process(stops.answer()tearing_down = pending_teardowns is not None and pending_teardowns.any_in_flight()if not tearing_down:+ if batch.rows_in_flight:+ warmth.active()batch.step()from_deferred = Falseif _held_head_leaves_the_hold(batch, deferred):@@ -2043,9 +2077,18 @@ def run_inference_process(cmd = cmd_queue.get(timeout = 0.0if (batch.rows_in_flight or deferred) and not tearing_down- else 1.0+ else warmth.timeout(getattr(backend, "active_model_name", None)))except _queue.Empty:+ if (+ not batch.rows_in_flight+ and not deferred+ and not (+ pending_teardowns is not None and pending_teardowns.any_in_flight()+ )+ and not (drain_event is not None and drain_event.is_set())+ ):+ warmth.tick(getattr(backend, "active_model_name", None))continueexcept (EOFError, OSError):batch.close()@@ -2053,6 +2096,8 @@ def run_inference_process(if cmd is None:continuecmd_type = cmd.get("type", "")+ if cmd_type in ("load", "unload", "reset", "cancel", "shutdown"):+ warmth.clear()if pending_teardowns is not None and cmd_type in _TEARDOWN_COMMANDS:pending_teardowns.taken()try:@@ -2214,6 +2259,9 @@ def run_inference_process(if cmd_type != "generate":_payload["type"] = "error"_send_response(resp_queue, _payload)+ finally:+ if cmd_type in ("generate", "generate_audio_input"):+ warmth.active()return# Windows Triton check, ahead of the torchao stub below, matching the training and export workers' gate-then-stubdiff --git a/studio/backend/tests/test_mlx_worker_audio_commands.py b/studio/backend/tests/test_mlx_worker_audio_commands.pyindex 33fcb4d5388..610abdcfa70 100644--- a/studio/backend/tests/test_mlx_worker_audio_commands.py+++ b/studio/backend/tests/test_mlx_worker_audio_commands.py@@ -38,7 +38,14 @@ class _RespQueue:self.sent.append(item)-def _run_mlx_loop(monkeypatch, cmds):+def _run_mlx_loop(+ monkeypatch,+ cmds,+ *,+ backend = None,+ command_queue = None,+ **kwargs,+):"""Drive the real MLX command loop with the init short-circuited."""from utils.hardware import hardware as _hw@@ -54,7 +61,11 @@ def _run_mlx_loop(monkeypatch, cmds):import core.inference.mlx_inference as mlx_mod- monkeypatch.setattr(mlx_mod, "MLXInferenceBackend", lambda *a, **k: SimpleNamespace())+ monkeypatch.setattr(+ mlx_mod,+ "MLXInferenceBackend",+ lambda *a, **k: backend if backend is not None else SimpleNamespace(),+ )from loggers.config import LogConfig@@ -62,10 +73,11 @@ def _run_mlx_loop(monkeypatch, cmds):resp = _RespQueue()worker.run_inference_process(- cmd_queue = _CmdQueue([*cmds, {"type": "shutdown"}]),+ cmd_queue = command_queue or _CmdQueue([*cmds, {"type": "shutdown"}]),resp_queue = resp,cancel_event = SimpleNamespace(is_set = lambda: False, clear = lambda: None, set = lambda: None),config = {"model_name": "unsloth/orpheus-3b-0.1-ft"},+ **kwargs,)return resp.sent@@ -118,3 +130,110 @@ def test_every_mlx_command_gets_exactly_one_reply(monkeypatch, cmd_type):addressed = [m for m in sent if m.get("request_id") == "r4"]assert len(addressed) == 1, addressed+++ "setting, enabled",+ [(None, True), ("0", False), ("false", False), ("1", True), ("TRUE", True), ("yes", True)],+)+def test_mlx_idle_warmth_is_default_on_bounded_and_failure_safe(monkeypatch, setting, enabled):+ import sys+ from types import ModuleType++ now, ticks = [100.0], []+ monkeypatch.setattr(worker.time, "monotonic", lambda: now[0])+ if setting is None:+ monkeypatch.delenv("UNSLOTH_MLX_GPU_KEEP_WARM", raising = False)+ else:+ monkeypatch.setenv("UNSLOTH_MLX_GPU_KEEP_WARM", setting)+ core = ModuleType("mlx.core")+ core.float32 = object()+ core.zeros = lambda *a, **k: 0+ core.eval = lambda x: ticks.append(x)+ mlx = ModuleType("mlx")+ mlx.core = core+ monkeypatch.setitem(sys.modules, "mlx", mlx)+ monkeypatch.setitem(sys.modules, "mlx.core", core)+ warmth = worker._MLXIdleWarmth()+ warmth.tick(True)+ assert ticks == []+ warmth.active()+ assert warmth.timeout(True) == (0.5 if enabled else 1.0)+ warmth.tick(False)+ assert ticks == []+ warmth.tick(True)+ assert ticks == ([1] if enabled else [])+ ticks.clear()+ now[0] += 60+ warmth.tick(True)+ assert ticks == []+ warmth.active()+ warmth.clear()+ warmth.tick(True)+ assert ticks == []+ warmth.active()+ core.eval = lambda x: (_ for _ in ()).throw(RuntimeError("tick failure"))+ warmth.tick(True)+ assert not warmth.enabled+ assert warmth.timeout(True) == 1.0++[email protected]("ending", ["unload", "reset", "cancel"])[email protected]("blocked", [None, "drain", "teardown", "busy"])+def test_mlx_loop_ticks_only_when_loaded_recent_and_idle(monkeypatch, blocked, ending):+ now, ticks = [100.0], []+ monkeypatch.setattr(worker.time, "monotonic", lambda: now[0])+ monkeypatch.setenv("UNSLOTH_MLX_GPU_KEEP_WARM", "1")+ monkeypatch.setattr(+ worker._MLXIdleWarmth,+ "tick",+ lambda self, loaded: ticks.append(loaded) if self.timeout(loaded) == 0.5 else None,+ )+ backend = SimpleNamespace(active_model_name = "model", reset_generation_state = lambda: None)+ monkeypatch.setattr(worker, "_dispatch_generate", lambda *a, **k: None)+ monkeypatch.setattr(+ worker, "_handle_unload", lambda *a, **k: setattr(backend, "active_model_name", None)+ )+ blocked_now = [False]+ busy = [False]+ original_batch = worker._ResidentBatch++ class Batch(original_batch):+ @property+ def rows_in_flight(self):+ return int(busy[0])++ def step(self):+ pass++ monkeypatch.setattr(worker, "_ResidentBatch", Batch)++ class Commands:+ step = 0++ def get(self, timeout = None):+ self.step += 1+ if self.step == 1:+ return {"type": "generate", "request_id": "warm"}+ if self.step in (2, 3):+ now[0] += 0.5+ blocked_now[0] = self.step == 2 and blocked in ("drain", "teardown")+ busy[0] = self.step == 2 and blocked == "busy"+ raise _queue.Empty+ if self.step == 4:+ return {"type": ending}+ if self.step == 5:+ raise _queue.Empty+ return {"type": "shutdown"}++ _run_mlx_loop(+ monkeypatch,+ [],+ backend = backend,+ command_queue = Commands(),+ drain_event = SimpleNamespace(is_set = lambda: blocked == "drain" and blocked_now[0]),+ pending_teardowns = SimpleNamespace(+ any_in_flight = lambda: blocked == "teardown" and blocked_now[0], taken = lambda: None+ ),+ )+ assert ticks == (["model", "model"] if blocked is None else ["model"])