Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
feat(studio): idle auto-unload an MLX chat model (#15680)
Long Yixing53795245e012
diff --git a/studio/backend/core/inference/llama_keepwarm.py b/studio/backend/core/inference/llama_keepwarm.pyindex c05dcf7c8a9..8de44b4a892 100644--- a/studio/backend/core/inference/llama_keepwarm.py+++ b/studio/backend/core/inference/llama_keepwarm.py@@ -1,9 +1,9 @@# SPDX-License-Identifier: AGPL-3.0-only# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0-"""Opt-in idle auto-unload (TTL keep-warm) for the local llama.cpp model.+"""Opt-in idle auto-unload (TTL keep-warm) for the local llama.cpp or MLX chat model.-Off by default (idle seconds = 0). A background loop unloads the GGUF after the TTL, and a+Off by default (idle seconds = 0). A background loop unloads the model after the TTL, and apure-ASGI middleware tracks in-flight requests so a long stream is never unloaded mid-response.The resident model is shared, so any account's activity resets the one global idle clock. Thesame loop and middleware drive media_keepwarm.@@ -47,6 +47,7 @@ _unattended: contextvars.ContextVar[bool] = contextvars.ContextVar(# The (id, quant) idle-unload last freed, so an alias/unknown request that would otherwise 503 against an empty backend# can reload it (set on unload, cleared on reload). The quant lets the reload restore the exact freed variant._last_unloaded_model = None+_last_unloaded_mlx = False# Slot KV manifest saved by the idle unload; whoever pops it owns deleting its files._kv_resume = None# Process-wide, not per-loop: the backend slot is shared across every event loop.@@ -536,11 +537,18 @@ def get_last_unloaded_model():return _last_unloaded_model-def _set_last_unloaded(value) -> None:- global _last_unloaded_model, _kv_resume+def stash_is_mlx(identity) -> bool:+ """Whether ``identity``, as get_last_unloaded_model() returned it, is an MLX model idle unload freed."""+ with _lock:+ return _last_unloaded_mlx and _last_unloaded_model is identity+++def _set_last_unloaded(value, mlx: bool = False) -> None:+ global _last_unloaded_model, _last_unloaded_mlx, _kv_resumestale = Nonewith _lock:_last_unloaded_model = value+ _last_unloaded_mlx = mlx and value is not Noneif value is None and _kv_resume is not None:stale, _kv_resume = _kv_resume, Noneif stale:@@ -793,6 +801,21 @@ def _loaded_identity(backend):return (*identity, companion_roots) if companion_roots else identity+def _resident_mlx(standard):+ name = getattr(standard, "active_model_name", None)+ entry = (getattr(standard, "models", None) or {}).get(name) if name else None+ if not isinstance(entry, dict) or not entry.get("is_mlx"):+ return None+ # Speech models (TTS, STT) stay loaded: the reload-stash path skips their audio preflights.+ return None if entry.get("is_audio") else standard+++def _mlx_identity(standard):+ # The auto-switch reload stash shape: no quant, the advertised id keys the saved overrides.+ name = standard.active_model_name+ return (name, None, getattr(standard, "_openai_advertised_id", None) or name)++def loaded_identity():"""The loaded GGUF's reload identity (path, quant, advertised id), or None when none is loaded."""from routes.inference import get_llama_cpp_backend@@ -821,7 +844,7 @@ def _note_idle_unload_event(freed, reason: str = "idle") -> None:async def idle_unload_loop(poll_seconds: float = 15.0) -> None:- """Unload the loaded GGUF once idle past the configured TTL. Inert when off."""+ """Unload the loaded GGUF or MLX model once idle past the configured TTL. Inert when off."""from utils.openai_auto_switch_settings import (get_auto_unload_api_only,get_auto_unload_idle_seconds,@@ -831,7 +854,7 @@ async def idle_unload_loop(poll_seconds: float = 15.0) -> None:def _user_pinned(b) -> bool:"""Whether the setting spares this model. Re-read like the othersettings: a KV save can outlive the user turning this on. getattr keeps- a foreign backend (tests, MLX) on the old unload-everything path."""+ a foreign backend (tests) on the old unload-everything path."""return get_auto_unload_api_only() and getattr(b, "_loaded_by_user_action", False)seen_model = None@@ -848,7 +871,11 @@ async def idle_unload_loop(poll_seconds: float = 15.0) -> None:if ttl <= 0:continuefrom core.inference.model_slots import unload_extra_models- from routes.inference import get_llama_cpp_backend, release_chat_gpu_claim+ from routes.inference import (+ _peek_inference_backend,+ get_llama_cpp_backend,+ release_chat_gpu_claim,+ )backend = get_llama_cpp_backend()# A (re)loaded model counts as activity so it survives one TTL before its first request.@@ -857,33 +884,37 @@ async def idle_unload_loop(poll_seconds: float = 15.0) -> None:unload_extra_models, _user_pinned, True):await asyncio.to_thread(release_chat_gpu_claim)+ mlx = None if backend.is_loaded else _resident_mlx(_peek_inference_backend())+ resident = backend if backend.is_loaded else mlx# Purging the stash mid-reload would race the restore.- current = _loaded_identity(backend)+ current = _loaded_identity(backend) if mlx is None else _mlx_identity(mlx)if current != seen_model:seen_model = currentif current is not None:# The loop's own context: whoever loaded it already stamped whether it was attended._note_activity(attended = False)_set_last_unloaded(None) # a model is loaded; drop stale stash- if backend.is_loaded and await asyncio.to_thread(_user_pinned, backend):+ if resident is not None and await asyncio.to_thread(_user_pinned, resident):# loaded from the UI, so the user wants it resident; only models the API loaded are freedcontinue- if backend.is_loaded and _is_idle(ttl):- freed = _loaded_identity(backend)+ if resident is not None and _is_idle(ttl):+ freed = currentmanifest = Noneif await asyncio.to_thread(get_auto_unload_keep_kv):+ save = (+ backend.save_slots_for_resume+ if mlx is None+ else mlx.save_prompt_cache_for_resume+ )try:- manifest = await asyncio.to_thread(- backend.save_slots_for_resume,- lambda: not _is_idle(ttl),- )+ manifest = await asyncio.to_thread(save, lambda: not _is_idle(ttl))except Exception as exc:- logger.debug("slot save before idle unload failed: %s", exc)+ logger.debug("chat KV save before idle unload failed: %s", exc)ttl = await asyncio.to_thread(get_auto_unload_idle_seconds)if (ttl <= 0or not _is_idle(ttl)- or await asyncio.to_thread(_user_pinned, backend)+ or await asyncio.to_thread(_user_pinned, resident)):if manifest:_delete_resume_files(manifest)@@ -897,15 +928,18 @@ async def idle_unload_loop(poll_seconds: float = 15.0) -> None:_delete_resume_files(manifest)continuetry:- await asyncio.to_thread(backend.unload_model)+ if mlx is None:+ await asyncio.to_thread(backend.unload_model)+ else:+ await asyncio.to_thread(mlx.unload_model, freed[0])except Exception:if manifest:_delete_resume_files(manifest)raise- _set_last_unloaded(freed) # let an alias request reload it+ _set_last_unloaded(freed, mlx = mlx is not None) # let an alias request reload itif manifest and freed:_set_kv_resume({"identity": freed, "origin": "idle", **manifest})- logger.info("Idle auto-unload: saved slot KV for restore on reload")+ logger.info("Idle auto-unload: saved chat KV for restore on reload")elif manifest:_delete_resume_files(manifest)# After the stash, so a failed release never loses the reload identity.@@ -917,7 +951,11 @@ async def idle_unload_loop(poll_seconds: float = 15.0) -> None:await asyncio.to_thread(release_chat_gpu_claim)except Exception as exc: # noqa: BLE001 - the unload already happenedlogger.debug("Idle auto-unload: claim release failed: %s", exc)- logger.info("Idle auto-unload: freed GGUF after %ss idle", ttl)+ logger.info(+ "Idle auto-unload: freed %s after %ss idle",+ "GGUF" if mlx is None else "MLX model",+ ttl,+ )# An idle unload stashes for reload and skips note_model_unloaded._note_idle_unload_event(freed)seen_model = Nonediff --git a/studio/backend/core/inference/orchestrator.py b/studio/backend/core/inference/orchestrator.pyindex 9e2d26c4fe0..7edcc9f9a24 100644--- a/studio/backend/core/inference/orchestrator.py+++ b/studio/backend/core/inference/orchestrator.py@@ -1982,6 +1982,8 @@ class InferenceOrchestrator:# Bumped at publish, not load start, so a same-model reload mid-install is detected.load_generation: int = 0+ # Cleared at each publish; only an explicit UI load sets it, pinning the model against idle unload.+ _loaded_by_user_action: bool = False@_invalidates_gpu_memory("inference load")@holds_inference@@ -2293,6 +2295,7 @@ class InferenceOrchestrator:return Falseself.active_model_name = model_info.get("identifier", model_name)self.load_generation += 1+ self._loaded_by_user_action = False# Fresh subprocess holds only this model; a stale name would unload the wrong one.self.models = {}self.models[self.active_model_name] = _mirrored_model_entry(@@ -2520,6 +2523,7 @@ class InferenceOrchestrator:}self.active_model_name = modelself.load_generation += 1+ self._loaded_by_user_action = Falsereturn Trueexcept Exception:self._shutdown_subprocess()diff --git a/studio/backend/routes/inference.py b/studio/backend/routes/inference.pyindex 5f26790b5f5..f91d21798c3 100644--- a/studio/backend/routes/inference.py+++ b/studio/backend/routes/inference.py@@ -10844,6 +10844,7 @@ async def _maybe_auto_switch_model(model_load_gate,note_admitted_inference,preview_swapped_since_entry,+ stash_is_mlx,)generation_cancel_event = getattr(@@ -10979,6 +10980,7 @@ async def _maybe_auto_switch_model(resolved = Nonerepo_level_companion_resolution = Falsestashed_gguf_companion_roots: tuple[str, ...] = ()+ stashed_mlx = Falseif auto_switch_on and not reload_only:# audio.cpp GGUFs load via the audio worker, not llama-server; the HF cache resolves them.resolved = await asyncio.to_thread(_audio_cpp_switch_target, requested_model)@@ -11035,6 +11037,7 @@ async def _maybe_auto_switch_model(else: # pre-3-tuple stash: fall back to the path as the override keytarget_id, variant = lastoverride_id = target_id+ stashed_mlx = stash_is_mlx(last)# Held-back keyless callers may restore only the model they named; check before loading.if keyless_held_back and not reload_only:requested_base, requested_variant = split_model_ref(requested_model)@@ -11059,8 +11062,11 @@ async def _maybe_auto_switch_model(# Not inferred from the quant; audio.cpp GGUFs are read only by audiocpp_server.from core.inference.audio_cpp_models import looks_like_audio_cpp- target_is_gguf = not looks_like_audio_cpp(target_id) and await asyncio.to_thread(- local_target_is_gguf, target_id, override_id+ # A cold resolver index reads an unknown repo id as GGUF, so trust the stash's own format.+ target_is_gguf = (+ not stashed_mlx+ and not looks_like_audio_cpp(target_id)+ and await asyncio.to_thread(local_target_is_gguf, target_id, override_id))gguf_companion_roots: tuple[str, ...] = ()if target_is_gguf:@@ -17520,6 +17526,9 @@ async def load_model_gated(_note_activity()# Record provenance once resident, here so the fast paths are covered; outside the gate. Preview keeps False: only an explicit UI load pins.get_llama_cpp_backend()._loaded_by_user_action = user_initiated+ standard = _peek_inference_backend()+ if getattr(standard, "active_model_name", None):+ standard._loaded_by_user_action = user_initiated# Every other model this load unloaded is listed too, under its model id: idle kept models,# kept models a vLLM/SGLang load retired and media a chat load freed.evicted += [diff --git a/studio/backend/tests/test_openai_auto_switch.py b/studio/backend/tests/test_openai_auto_switch.pyindex 63f048fda6a..9cf5b9c7838 100644--- a/studio/backend/tests/test_openai_auto_switch.py+++ b/studio/backend/tests/test_openai_auto_switch.py@@ -75,6 +75,7 @@ def _reset_keepwarm():kw._pending = 0kw._last_active = time.monotonic() - 3600kw._last_unloaded_model = None+ kw._last_unloaded_mlx = Falsekw._kv_resume = None@@ -9226,17 +9227,164 @@ def test_the_load_route_pins_and_other_load_surfaces_do_not(monkeypatch):monkeypatch.setattr(inference_route, "_load_model_impl", _already_loaded)request = LoadRequest(model_path = "unsloth/A-GGUF")+ standard = SimpleNamespace(active_model_name = "/models/Qwen-MLX", _loaded_by_user_action = False)+ monkeypatch.setattr(inference_route, "_peek_inference_backend", lambda: standard)+asyncio.run(inference_route.load_model_gated(request, object(), "tester", user_initiated = True))assert backend._loaded_by_user_action is True+ assert standard._loaded_by_user_action is True# Preview shares this helper and must not pin, so it keeps the default.asyncio.run(inference_route.load_model_gated(request, object(), "tester"))assert backend._loaded_by_user_action is False+ assert standard._loaded_by_user_action is Falseimport routes.preview as preview_routeassert "user_initiated" not in inspect.getsource(preview_route._serve_chat)+def test_every_orchestrator_load_publish_clears_the_ui_pin():+ # Loads that bypass the /load route (preview, auto-switch) must not inherit an earlier UI pin.+ from core.inference.orchestrator import InferenceOrchestrator+ for publisher in (InferenceOrchestrator.load_model, InferenceOrchestrator._load_managed_engine):+ assert "self._loaded_by_user_action = False" in inspect.getsource(publisher)+ assert InferenceOrchestrator._loaded_by_user_action is False+++class _FakeMlxOrchestrator:+ def __init__(+ self,+ name,+ *,+ entry = None,+ user_loaded = False,+ root = None,+ ):+ self.active_model_name = name+ self.models = {name: entry or {"is_mlx": True}}+ self._openai_advertised_id = "org/Qwen-MLX-Alias"+ self._loaded_by_user_action = user_loaded+ self.root, self.unloads, self.restores = root, [], []++ def save_prompt_cache_for_resume(self, should_abort):+ if self.root is None:+ return None+ directory = self.root / "resume-mlx"+ directory.mkdir()+ (directory / "text-0.safetensors").write_bytes(b"kv")+ slot = {"filename": "text-0.safetensors", "store": "text"}+ return {"engine": "mlx", "dir": str(directory), "fingerprint": (), "slots": [slot]}++ def restore_prompt_cache_for_resume(self, manifest):+ self.restores.append(manifest)+ return 1++ def unload_model(self, name):+ self.unloads.append(name)+ self.models.pop(name, None)+ self.active_model_name = None+ return True+++def _idle_mlx(monkeypatch, **kwargs):+ """A long-idle MLX primary on the orchestrator, with the llama backend empty."""+ llama = _idle_backend(kw, monkeypatch, user_loaded = False)+ llama.is_loaded = False+ llama.model_identifier = None+ saves = []+ llama.save_slots_for_resume = lambda *_a, **_k: saves.append(1)+ monkeypatch.setattr(settings, "get_auto_unload_keep_kv", lambda: True)+ orch = _FakeMlxOrchestrator("/models/Qwen-MLX", **kwargs)+ monkeypatch.setattr(inference_route, "_peek_inference_backend", lambda: orch)+ return llama, orch, saves+++def test_idle_unload_frees_an_api_loaded_mlx_primary_and_the_next_request_reloads_it(+ monkeypatch, tmp_path+):+ monkeypatch.setattr(settings, "get_auto_unload_api_only", lambda: True)+ llama, orch, saves = _idle_mlx(monkeypatch, root = tmp_path)++ kw._inflight = 1+ _drive_idle_loop(kw)+ assert orch.unloads == [], "a running request lost its MLX model"++ kw._inflight = 0+ _drive_idle_loop(kw, until = lambda: orch.unloads)+ assert orch.unloads == ["/models/Qwen-MLX"]+ assert saves == [], "the MLX prompt cache was saved through llama-server"+ assert kw.get_last_unloaded_model() == ("/models/Qwen-MLX", None, "org/Qwen-MLX-Alias")+ saved = kw._kv_resume+ assert saved["engine"] == "mlx" and saved["origin"] == "idle"+ assert saved["identity"] == ("/models/Qwen-MLX", None, "org/Qwen-MLX-Alias")++ # The reload is an auto-switch load, so the restored model stays idle-unloadable.+ orch._loaded_by_user_action = True+ calls = []++ async def _load(request, *args, **kwargs):+ calls.append(request)+ orch.active_model_name = request.model_path+ orch.models = {request.model_path: {"is_mlx": True}}+ kw.note_model_loaded(orch)++ _wire_on(monkeypatch, resolves_to = None, backend = llama, recorder = _load)+ monkeypatch.setattr(inference_route, "get_inference_backend", lambda: orch)+ monkeypatch.setattr(inference_route, "_reject_unservable_model", _noop_reject)+ # The resolver index is cold, so the stashed id alone would be read as a GGUF.+ assert resolver.local_target_is_gguf("/models/Qwen-MLX", "org/Qwen-MLX-Alias") is True+ _run_hook(inference_route._RELOAD_ONLY_MODEL)++ assert [c.model_path for c in calls] == ["/models/Qwen-MLX"]+ assert orch._openai_advertised_id == "org/Qwen-MLX-Alias"+ assert orch._loaded_by_user_action is False+ assert orch.restores == [saved]+ assert kw._kv_resume is None and not Path(saved["dir"]).exists()+++def test_an_mlx_save_dropped_after_the_save_leaves_no_files(monkeypatch, tmp_path):+ _llama, orch, _saves = _idle_mlx(monkeypatch, root = tmp_path)+ reads = iter([True, False])+ monkeypatch.setattr(settings, "get_auto_unload_keep_kv", lambda: next(reads, False))++ _drive_idle_loop(kw, until = lambda: orch.unloads)+ assert orch.unloads == ["/models/Qwen-MLX"]+ assert kw._kv_resume is None+ assert not (tmp_path / "resume-mlx").exists()+++def test_the_stash_format_belongs_to_that_stash_only():+ mlx_stash = ("mlx-community/Qwen-MLX", None, "mlx-community/Qwen-MLX")+ kw._set_last_unloaded(mlx_stash, mlx = True)+ assert kw.stash_is_mlx(kw.get_last_unloaded_model()) is True+ # An equal tuple from elsewhere (a test, another stash) is not the stash the loop recorded.+ assert kw.stash_is_mlx(tuple(list(mlx_stash))) is False++ gguf_stash = ("unsloth/A-GGUF", "Q4_K_M", "unsloth/A-GGUF")+ kw.stash_for_reload(gguf_stash)+ assert kw.stash_is_mlx(kw.get_last_unloaded_model()) is False+++ "user_loaded, entry",+ [+ (True, {"is_mlx": True}),+ (False, {"is_mlx": False}),+ (False, {"is_mlx": True, "is_audio": True}),+ ],+ ids = ["ui-pinned", "not-mlx", "mlx-speech"],+)+def test_idle_unload_leaves_a_ui_pinned_mlx_or_a_non_mlx_orchestrator_model(+ monkeypatch, user_loaded, entry+):+ monkeypatch.setattr(settings, "get_auto_unload_api_only", lambda: True)+ _llama, orch, _saves = _idle_mlx(monkeypatch, user_loaded = user_loaded, entry = entry)++ _drive_idle_loop(kw)+ assert orch.unloads == []+ assert kw.get_last_unloaded_model() is None++def test_api_only_turned_on_mid_save_still_spares_the_model(monkeypatch, tmp_path):on = {"now": False}monkeypatch.setattr(settings, "get_auto_unload_api_only", lambda: on["now"])