Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

feat(studio): idle auto-unload an MLX chat model (#15680)

Long Yixing53795245e012
diff --git a/studio/backend/core/inference/llama_keepwarm.py b/studio/backend/core/inference/llama_keepwarm.py
index c05dcf7c8a9..8de44b4a892 100644
--- a/studio/backend/core/inference/llama_keepwarm.py
+++ b/studio/backend/core/inference/llama_keepwarm.py
@@ -1,9 +1,9 @@
# SPDX-License-Identifier: AGPL-3.0-only
# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
-"""Opt-in idle auto-unload (TTL keep-warm) for the local llama.cpp model.
+"""Opt-in idle auto-unload (TTL keep-warm) for the local llama.cpp or MLX chat model.
-Off by default (idle seconds = 0). A background loop unloads the GGUF after the TTL, and a
+Off by default (idle seconds = 0). A background loop unloads the model after the TTL, and a
pure-ASGI middleware tracks in-flight requests so a long stream is never unloaded mid-response.
The resident model is shared, so any account's activity resets the one global idle clock. The
same loop and middleware drive media_keepwarm.
@@ -47,6 +47,7 @@ _unattended: contextvars.ContextVar[bool] = contextvars.ContextVar(
# The (id, quant) idle-unload last freed, so an alias/unknown request that would otherwise 503 against an empty backend
# can reload it (set on unload, cleared on reload). The quant lets the reload restore the exact freed variant.
_last_unloaded_model = None
+_last_unloaded_mlx = False
# Slot KV manifest saved by the idle unload; whoever pops it owns deleting its files.
_kv_resume = None
# Process-wide, not per-loop: the backend slot is shared across every event loop.
@@ -536,11 +537,18 @@ def get_last_unloaded_model():
return _last_unloaded_model
-def _set_last_unloaded(value) -> None:
- global _last_unloaded_model, _kv_resume
+def stash_is_mlx(identity) -> bool:
+ """Whether ``identity``, as get_last_unloaded_model() returned it, is an MLX model idle unload freed."""
+ with _lock:
+ return _last_unloaded_mlx and _last_unloaded_model is identity
+
+
+def _set_last_unloaded(value, mlx: bool = False) -> None:
+ global _last_unloaded_model, _last_unloaded_mlx, _kv_resume
stale = None
with _lock:
_last_unloaded_model = value
+ _last_unloaded_mlx = mlx and value is not None
if value is None and _kv_resume is not None:
stale, _kv_resume = _kv_resume, None
if stale:
@@ -793,6 +801,21 @@ def _loaded_identity(backend):
return (*identity, companion_roots) if companion_roots else identity
+def _resident_mlx(standard):
+ name = getattr(standard, "active_model_name", None)
+ entry = (getattr(standard, "models", None) or {}).get(name) if name else None
+ if not isinstance(entry, dict) or not entry.get("is_mlx"):
+ return None
+ # Speech models (TTS, STT) stay loaded: the reload-stash path skips their audio preflights.
+ return None if entry.get("is_audio") else standard
+
+
+def _mlx_identity(standard):
+ # The auto-switch reload stash shape: no quant, the advertised id keys the saved overrides.
+ name = standard.active_model_name
+ return (name, None, getattr(standard, "_openai_advertised_id", None) or name)
+
+
def loaded_identity():
"""The loaded GGUF's reload identity (path, quant, advertised id), or None when none is loaded."""
from routes.inference import get_llama_cpp_backend
@@ -821,7 +844,7 @@ def _note_idle_unload_event(freed, reason: str = "idle") -> None:
async def idle_unload_loop(poll_seconds: float = 15.0) -> None:
- """Unload the loaded GGUF once idle past the configured TTL. Inert when off."""
+ """Unload the loaded GGUF or MLX model once idle past the configured TTL. Inert when off."""
from utils.openai_auto_switch_settings import (
get_auto_unload_api_only,
get_auto_unload_idle_seconds,
@@ -831,7 +854,7 @@ async def idle_unload_loop(poll_seconds: float = 15.0) -> None:
def _user_pinned(b) -> bool:
"""Whether the setting spares this model. Re-read like the other
settings: a KV save can outlive the user turning this on. getattr keeps
- a foreign backend (tests, MLX) on the old unload-everything path."""
+ a foreign backend (tests) on the old unload-everything path."""
return get_auto_unload_api_only() and getattr(b, "_loaded_by_user_action", False)
seen_model = None
@@ -848,7 +871,11 @@ async def idle_unload_loop(poll_seconds: float = 15.0) -> None:
if ttl <= 0:
continue
from core.inference.model_slots import unload_extra_models
- from routes.inference import get_llama_cpp_backend, release_chat_gpu_claim
+ from routes.inference import (
+ _peek_inference_backend,
+ get_llama_cpp_backend,
+ release_chat_gpu_claim,
+ )
backend = get_llama_cpp_backend()
# A (re)loaded model counts as activity so it survives one TTL before its first request.
@@ -857,33 +884,37 @@ async def idle_unload_loop(poll_seconds: float = 15.0) -> None:
unload_extra_models, _user_pinned, True
):
await asyncio.to_thread(release_chat_gpu_claim)
+ mlx = None if backend.is_loaded else _resident_mlx(_peek_inference_backend())
+ resident = backend if backend.is_loaded else mlx
# Purging the stash mid-reload would race the restore.
- current = _loaded_identity(backend)
+ current = _loaded_identity(backend) if mlx is None else _mlx_identity(mlx)
if current != seen_model:
seen_model = current
if current is not None:
# The loop's own context: whoever loaded it already stamped whether it was attended.
_note_activity(attended = False)
_set_last_unloaded(None) # a model is loaded; drop stale stash
- if backend.is_loaded and await asyncio.to_thread(_user_pinned, backend):
+ if resident is not None and await asyncio.to_thread(_user_pinned, resident):
# loaded from the UI, so the user wants it resident; only models the API loaded are freed
continue
- if backend.is_loaded and _is_idle(ttl):
- freed = _loaded_identity(backend)
+ if resident is not None and _is_idle(ttl):
+ freed = current
manifest = None
if await asyncio.to_thread(get_auto_unload_keep_kv):
+ save = (
+ backend.save_slots_for_resume
+ if mlx is None
+ else mlx.save_prompt_cache_for_resume
+ )
try:
- manifest = await asyncio.to_thread(
- backend.save_slots_for_resume,
- lambda: not _is_idle(ttl),
- )
+ manifest = await asyncio.to_thread(save, lambda: not _is_idle(ttl))
except Exception as exc:
- logger.debug("slot save before idle unload failed: %s", exc)
+ logger.debug("chat KV save before idle unload failed: %s", exc)
ttl = await asyncio.to_thread(get_auto_unload_idle_seconds)
if (
ttl <= 0
or not _is_idle(ttl)
- or await asyncio.to_thread(_user_pinned, backend)
+ or await asyncio.to_thread(_user_pinned, resident)
):
if manifest:
_delete_resume_files(manifest)
@@ -897,15 +928,18 @@ async def idle_unload_loop(poll_seconds: float = 15.0) -> None:
_delete_resume_files(manifest)
continue
try:
- await asyncio.to_thread(backend.unload_model)
+ if mlx is None:
+ await asyncio.to_thread(backend.unload_model)
+ else:
+ await asyncio.to_thread(mlx.unload_model, freed[0])
except Exception:
if manifest:
_delete_resume_files(manifest)
raise
- _set_last_unloaded(freed) # let an alias request reload it
+ _set_last_unloaded(freed, mlx = mlx is not None) # let an alias request reload it
if manifest and freed:
_set_kv_resume({"identity": freed, "origin": "idle", **manifest})
- logger.info("Idle auto-unload: saved slot KV for restore on reload")
+ logger.info("Idle auto-unload: saved chat KV for restore on reload")
elif manifest:
_delete_resume_files(manifest)
# After the stash, so a failed release never loses the reload identity.
@@ -917,7 +951,11 @@ async def idle_unload_loop(poll_seconds: float = 15.0) -> None:
await asyncio.to_thread(release_chat_gpu_claim)
except Exception as exc: # noqa: BLE001 - the unload already happened
logger.debug("Idle auto-unload: claim release failed: %s", exc)
- logger.info("Idle auto-unload: freed GGUF after %ss idle", ttl)
+ logger.info(
+ "Idle auto-unload: freed %s after %ss idle",
+ "GGUF" if mlx is None else "MLX model",
+ ttl,
+ )
# An idle unload stashes for reload and skips note_model_unloaded.
_note_idle_unload_event(freed)
seen_model = None
diff --git a/studio/backend/core/inference/orchestrator.py b/studio/backend/core/inference/orchestrator.py
index 9e2d26c4fe0..7edcc9f9a24 100644
--- a/studio/backend/core/inference/orchestrator.py
+++ b/studio/backend/core/inference/orchestrator.py
@@ -1982,6 +1982,8 @@ class InferenceOrchestrator:
# Bumped at publish, not load start, so a same-model reload mid-install is detected.
load_generation: int = 0
+ # Cleared at each publish; only an explicit UI load sets it, pinning the model against idle unload.
+ _loaded_by_user_action: bool = False
@_invalidates_gpu_memory("inference load")
@holds_inference
@@ -2293,6 +2295,7 @@ class InferenceOrchestrator:
return False
self.active_model_name = model_info.get("identifier", model_name)
self.load_generation += 1
+ self._loaded_by_user_action = False
# Fresh subprocess holds only this model; a stale name would unload the wrong one.
self.models = {}
self.models[self.active_model_name] = _mirrored_model_entry(
@@ -2520,6 +2523,7 @@ class InferenceOrchestrator:
}
self.active_model_name = model
self.load_generation += 1
+ self._loaded_by_user_action = False
return True
except Exception:
self._shutdown_subprocess()
diff --git a/studio/backend/routes/inference.py b/studio/backend/routes/inference.py
index 5f26790b5f5..f91d21798c3 100644
--- a/studio/backend/routes/inference.py
+++ b/studio/backend/routes/inference.py
@@ -10844,6 +10844,7 @@ async def _maybe_auto_switch_model(
model_load_gate,
note_admitted_inference,
preview_swapped_since_entry,
+ stash_is_mlx,
)
generation_cancel_event = getattr(
@@ -10979,6 +10980,7 @@ async def _maybe_auto_switch_model(
resolved = None
repo_level_companion_resolution = False
stashed_gguf_companion_roots: tuple[str, ...] = ()
+ stashed_mlx = False
if auto_switch_on and not reload_only:
# audio.cpp GGUFs load via the audio worker, not llama-server; the HF cache resolves them.
resolved = await asyncio.to_thread(_audio_cpp_switch_target, requested_model)
@@ -11035,6 +11037,7 @@ async def _maybe_auto_switch_model(
else: # pre-3-tuple stash: fall back to the path as the override key
target_id, variant = last
override_id = target_id
+ stashed_mlx = stash_is_mlx(last)
# Held-back keyless callers may restore only the model they named; check before loading.
if keyless_held_back and not reload_only:
requested_base, requested_variant = split_model_ref(requested_model)
@@ -11059,8 +11062,11 @@ async def _maybe_auto_switch_model(
# Not inferred from the quant; audio.cpp GGUFs are read only by audiocpp_server.
from core.inference.audio_cpp_models import looks_like_audio_cpp
- target_is_gguf = not looks_like_audio_cpp(target_id) and await asyncio.to_thread(
- local_target_is_gguf, target_id, override_id
+ # A cold resolver index reads an unknown repo id as GGUF, so trust the stash's own format.
+ target_is_gguf = (
+ not stashed_mlx
+ and not looks_like_audio_cpp(target_id)
+ and await asyncio.to_thread(local_target_is_gguf, target_id, override_id)
)
gguf_companion_roots: tuple[str, ...] = ()
if target_is_gguf:
@@ -17520,6 +17526,9 @@ async def load_model_gated(
_note_activity()
# Record provenance once resident, here so the fast paths are covered; outside the gate. Preview keeps False: only an explicit UI load pins.
get_llama_cpp_backend()._loaded_by_user_action = user_initiated
+ standard = _peek_inference_backend()
+ if getattr(standard, "active_model_name", None):
+ standard._loaded_by_user_action = user_initiated
# Every other model this load unloaded is listed too, under its model id: idle kept models,
# kept models a vLLM/SGLang load retired and media a chat load freed.
evicted += [
diff --git a/studio/backend/tests/test_openai_auto_switch.py b/studio/backend/tests/test_openai_auto_switch.py
index 63f048fda6a..9cf5b9c7838 100644
--- a/studio/backend/tests/test_openai_auto_switch.py
+++ b/studio/backend/tests/test_openai_auto_switch.py
@@ -75,6 +75,7 @@ def _reset_keepwarm():
kw._pending = 0
kw._last_active = time.monotonic() - 3600
kw._last_unloaded_model = None
+ kw._last_unloaded_mlx = False
kw._kv_resume = None
@@ -9226,17 +9227,164 @@ def test_the_load_route_pins_and_other_load_surfaces_do_not(monkeypatch):
monkeypatch.setattr(inference_route, "_load_model_impl", _already_loaded)
request = LoadRequest(model_path = "unsloth/A-GGUF")
+ standard = SimpleNamespace(active_model_name = "/models/Qwen-MLX", _loaded_by_user_action = False)
+ monkeypatch.setattr(inference_route, "_peek_inference_backend", lambda: standard)
+
asyncio.run(inference_route.load_model_gated(request, object(), "tester", user_initiated = True))
assert backend._loaded_by_user_action is True
+ assert standard._loaded_by_user_action is True
# Preview shares this helper and must not pin, so it keeps the default.
asyncio.run(inference_route.load_model_gated(request, object(), "tester"))
assert backend._loaded_by_user_action is False
+ assert standard._loaded_by_user_action is False
import routes.preview as preview_route
assert "user_initiated" not in inspect.getsource(preview_route._serve_chat)
+def test_every_orchestrator_load_publish_clears_the_ui_pin():
+ # Loads that bypass the /load route (preview, auto-switch) must not inherit an earlier UI pin.
+ from core.inference.orchestrator import InferenceOrchestrator
+ for publisher in (InferenceOrchestrator.load_model, InferenceOrchestrator._load_managed_engine):
+ assert "self._loaded_by_user_action = False" in inspect.getsource(publisher)
+ assert InferenceOrchestrator._loaded_by_user_action is False
+
+
+class _FakeMlxOrchestrator:
+ def __init__(
+ self,
+ name,
+ *,
+ entry = None,
+ user_loaded = False,
+ root = None,
+ ):
+ self.active_model_name = name
+ self.models = {name: entry or {"is_mlx": True}}
+ self._openai_advertised_id = "org/Qwen-MLX-Alias"
+ self._loaded_by_user_action = user_loaded
+ self.root, self.unloads, self.restores = root, [], []
+
+ def save_prompt_cache_for_resume(self, should_abort):
+ if self.root is None:
+ return None
+ directory = self.root / "resume-mlx"
+ directory.mkdir()
+ (directory / "text-0.safetensors").write_bytes(b"kv")
+ slot = {"filename": "text-0.safetensors", "store": "text"}
+ return {"engine": "mlx", "dir": str(directory), "fingerprint": (), "slots": [slot]}
+
+ def restore_prompt_cache_for_resume(self, manifest):
+ self.restores.append(manifest)
+ return 1
+
+ def unload_model(self, name):
+ self.unloads.append(name)
+ self.models.pop(name, None)
+ self.active_model_name = None
+ return True
+
+
+def _idle_mlx(monkeypatch, **kwargs):
+ """A long-idle MLX primary on the orchestrator, with the llama backend empty."""
+ llama = _idle_backend(kw, monkeypatch, user_loaded = False)
+ llama.is_loaded = False
+ llama.model_identifier = None
+ saves = []
+ llama.save_slots_for_resume = lambda *_a, **_k: saves.append(1)
+ monkeypatch.setattr(settings, "get_auto_unload_keep_kv", lambda: True)
+ orch = _FakeMlxOrchestrator("/models/Qwen-MLX", **kwargs)
+ monkeypatch.setattr(inference_route, "_peek_inference_backend", lambda: orch)
+ return llama, orch, saves
+
+
+def test_idle_unload_frees_an_api_loaded_mlx_primary_and_the_next_request_reloads_it(
+ monkeypatch, tmp_path
+):
+ monkeypatch.setattr(settings, "get_auto_unload_api_only", lambda: True)
+ llama, orch, saves = _idle_mlx(monkeypatch, root = tmp_path)
+
+ kw._inflight = 1
+ _drive_idle_loop(kw)
+ assert orch.unloads == [], "a running request lost its MLX model"
+
+ kw._inflight = 0
+ _drive_idle_loop(kw, until = lambda: orch.unloads)
+ assert orch.unloads == ["/models/Qwen-MLX"]
+ assert saves == [], "the MLX prompt cache was saved through llama-server"
+ assert kw.get_last_unloaded_model() == ("/models/Qwen-MLX", None, "org/Qwen-MLX-Alias")
+ saved = kw._kv_resume
+ assert saved["engine"] == "mlx" and saved["origin"] == "idle"
+ assert saved["identity"] == ("/models/Qwen-MLX", None, "org/Qwen-MLX-Alias")
+
+ # The reload is an auto-switch load, so the restored model stays idle-unloadable.
+ orch._loaded_by_user_action = True
+ calls = []
+
+ async def _load(request, *args, **kwargs):
+ calls.append(request)
+ orch.active_model_name = request.model_path
+ orch.models = {request.model_path: {"is_mlx": True}}
+ kw.note_model_loaded(orch)
+
+ _wire_on(monkeypatch, resolves_to = None, backend = llama, recorder = _load)
+ monkeypatch.setattr(inference_route, "get_inference_backend", lambda: orch)
+ monkeypatch.setattr(inference_route, "_reject_unservable_model", _noop_reject)
+ # The resolver index is cold, so the stashed id alone would be read as a GGUF.
+ assert resolver.local_target_is_gguf("/models/Qwen-MLX", "org/Qwen-MLX-Alias") is True
+ _run_hook(inference_route._RELOAD_ONLY_MODEL)
+
+ assert [c.model_path for c in calls] == ["/models/Qwen-MLX"]
+ assert orch._openai_advertised_id == "org/Qwen-MLX-Alias"
+ assert orch._loaded_by_user_action is False
+ assert orch.restores == [saved]
+ assert kw._kv_resume is None and not Path(saved["dir"]).exists()
+
+
+def test_an_mlx_save_dropped_after_the_save_leaves_no_files(monkeypatch, tmp_path):
+ _llama, orch, _saves = _idle_mlx(monkeypatch, root = tmp_path)
+ reads = iter([True, False])
+ monkeypatch.setattr(settings, "get_auto_unload_keep_kv", lambda: next(reads, False))
+
+ _drive_idle_loop(kw, until = lambda: orch.unloads)
+ assert orch.unloads == ["/models/Qwen-MLX"]
+ assert kw._kv_resume is None
+ assert not (tmp_path / "resume-mlx").exists()
+
+
+def test_the_stash_format_belongs_to_that_stash_only():
+ mlx_stash = ("mlx-community/Qwen-MLX", None, "mlx-community/Qwen-MLX")
+ kw._set_last_unloaded(mlx_stash, mlx = True)
+ assert kw.stash_is_mlx(kw.get_last_unloaded_model()) is True
+ # An equal tuple from elsewhere (a test, another stash) is not the stash the loop recorded.
+ assert kw.stash_is_mlx(tuple(list(mlx_stash))) is False
+
+ gguf_stash = ("unsloth/A-GGUF", "Q4_K_M", "unsloth/A-GGUF")
+ kw.stash_for_reload(gguf_stash)
+ assert kw.stash_is_mlx(kw.get_last_unloaded_model()) is False
+
+
+ "user_loaded, entry",
+ [
+ (True, {"is_mlx": True}),
+ (False, {"is_mlx": False}),
+ (False, {"is_mlx": True, "is_audio": True}),
+ ],
+ ids = ["ui-pinned", "not-mlx", "mlx-speech"],
+)
+def test_idle_unload_leaves_a_ui_pinned_mlx_or_a_non_mlx_orchestrator_model(
+ monkeypatch, user_loaded, entry
+):
+ monkeypatch.setattr(settings, "get_auto_unload_api_only", lambda: True)
+ _llama, orch, _saves = _idle_mlx(monkeypatch, user_loaded = user_loaded, entry = entry)
+
+ _drive_idle_loop(kw)
+ assert orch.unloads == []
+ assert kw.get_last_unloaded_model() is None
+
+
def test_api_only_turned_on_mid_save_still_spares_the_model(monkeypatch, tmp_path):
on = {"now": False}
monkeypatch.setattr(settings, "get_auto_unload_api_only", lambda: on["now"])