Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

Studio: keep the MLX chat context when an image or video model takes the GPU (#15358)

Long Yixing20f3ac001827
diff --git a/studio/backend/core/inference/chat_swap_kv.py b/studio/backend/core/inference/chat_swap_kv.py
index e3c1bfced3a..4bfbc1181fc 100644
--- a/studio/backend/core/inference/chat_swap_kv.py
+++ b/studio/backend/core/inference/chat_swap_kv.py
@@ -3,7 +3,8 @@
"""Keep the chat context when an image or video load takes the GPU from chat.
-Before the media acquire (outside every lock) the chat slots' KV is saved to disk, bounded by
+Before the media acquire (outside every lock) the chat KV (llama-server's slots, or the MLX
+worker's prompt cache) is saved to disk, bounded by
``SWAP_KV_SAVE_BUDGET_S``. The manifest waits here as "pending" until ``gpu_arbiter._evict_chat``
unloads that same model; only then does it become the keep-warm stash that the next chat load
restores. If nothing was evicted, ``SwapSave.close`` drops it and deletes its files.
@@ -304,11 +305,36 @@ def _chat_account() -> Optional[str]:
return gpu_arbiter.owner_account()
-def _save_in_thread(llama, should_abort, abort: threading.Event, loop, future) -> None:
+def mlx_chat_identity(orchestrator):
+ serves_mlx = getattr(orchestrator, "_serves_mlx", None)
+ if not callable(serves_mlx) or not serves_mlx():
+ return None
+ name = orchestrator.active_model_name
+ return (name, None, getattr(orchestrator, "_openai_advertised_id", None) or name, "mlx")
+
+
+def _chat_saver():
+ from core.inference.llama_keepwarm import _loaded_identity
+ from routes.inference import get_llama_cpp_backend
+
+ llama = get_llama_cpp_backend()
+ identity = _loaded_identity(llama)
+ if identity is not None:
+ if not getattr(llama, "_gguf_path", None):
+ return None
+ return identity, llama.save_slots_for_resume
+ from core.inference.orchestrator import peek_inference_backend
+
+ orchestrator = peek_inference_backend()
+ identity = mlx_chat_identity(orchestrator)
+ return None if identity is None else (identity, orchestrator.save_prompt_cache_for_resume)
+
+
+def _save_in_thread(save, should_abort, abort: threading.Event, loop, future) -> None:
"""Runs the blocking save on its own thread, so neither the budget nor the event loop's
shutdown waits on it. A result nobody is waiting for any more is deleted right here."""
try:
- manifest, error = llama.save_slots_for_resume(should_abort), None
+ manifest, error = save(should_abort), None
except BaseException as exc: # noqa: BLE001 - reported to the waiter
manifest, error = None, exc
if abort.is_set():
@@ -345,19 +371,16 @@ async def save_chat_kv_before_swap(
global _pending
try:
from core.inference import gpu_arbiter
- from core.inference.llama_keepwarm import _loaded_identity
from utils.openai_auto_switch_settings import get_auto_unload_keep_kv
if not gpu_arbiter.would_evict(owner, policy = policy, need_mib = need_mib, gpu_ids = gpu_ids):
return SwapSave()
if not await asyncio.to_thread(get_auto_unload_keep_kv):
return SwapSave()
- from routes.inference import get_llama_cpp_backend
-
- llama = get_llama_cpp_backend()
- identity = _loaded_identity(llama)
- if identity is None or not getattr(llama, "_gguf_path", None):
+ saver = _chat_saver()
+ if saver is None:
return SwapSave()
+ identity, save = saver
account = _account_id()
counted = _request_counted()
if _chat_account() != account or _chat_busy(counted):
@@ -389,7 +412,7 @@ async def save_chat_kv_before_swap(
try:
threading.Thread(
target = _save_in_thread,
- args = (llama, should_abort, abort, loop, future),
+ args = (save, should_abort, abort, loop, future),
name = "chat-kv-swap-save",
daemon = True,
).start()
diff --git a/studio/backend/core/inference/gpu_arbiter.py b/studio/backend/core/inference/gpu_arbiter.py
index db341c839d1..e278109bd7a 100644
--- a/studio/backend/core/inference/gpu_arbiter.py
+++ b/studio/backend/core/inference/gpu_arbiter.py
@@ -147,7 +147,10 @@ def _evict_chat() -> None:
_commit_swap_kv(unloaded)
orchestrator = get_inference_backend()
if orchestrator.active_model_name:
+ unloaded = _mlx_chat_identity(orchestrator)
orchestrator.unload_model(orchestrator.active_model_name)
+ if unloaded is not None:
+ _commit_swap_kv(unloaded)
# In-flight loads lack active_model_name; cancel_load must drop the loading marker before teardown.
for pending in list(getattr(orchestrator, "loading_models", ()) or ()):
orchestrator.cancel_load(pending)
@@ -173,6 +176,14 @@ def _chat_identity(llama):
return None
+def _mlx_chat_identity(orchestrator):
+ try:
+ from core.inference.chat_swap_kv import mlx_chat_identity
+ return mlx_chat_identity(orchestrator)
+ except Exception:
+ return None
+
+
def _commit_swap_kv(unloaded) -> None:
"""Promote the chat KV saved before this media load; an eviction never fails on it."""
try:
diff --git a/studio/backend/core/inference/llama_keepwarm.py b/studio/backend/core/inference/llama_keepwarm.py
index 423977531c9..c05dcf7c8a9 100644
--- a/studio/backend/core/inference/llama_keepwarm.py
+++ b/studio/backend/core/inference/llama_keepwarm.py
@@ -14,6 +14,7 @@ from __future__ import annotations
import asyncio
import contextlib
import contextvars
+import shutil
import threading
import time
from pathlib import Path
@@ -552,6 +553,10 @@ def _delete_resume_files(manifest) -> None:
for entry in manifest.get("slots") or []:
with contextlib.suppress(OSError):
(base / str(entry.get("filename"))).unlink()
+ if manifest.get("engine") == "mlx":
+ # Each MLX save has a directory of its own; llama-server's is shared.
+ with contextlib.suppress(OSError):
+ base.rmdir()
except Exception:
pass
@@ -589,6 +594,12 @@ def purge_kv_resume(*, origin: Optional[str] = None) -> None:
def restore_kv_resume(backend, manifest) -> None:
try:
+ if manifest.get("engine") == "mlx":
+ # The worker checks the load matches the save; the identity names only the model.
+ restore = getattr(backend, "restore_prompt_cache_for_resume", None)
+ if callable(restore) and restore(manifest):
+ logger.info("Restored the saved MLX prompt cache onto the reloaded model")
+ return
gguf = manifest.get("gguf")
binary = manifest.get("binary")
current = getattr(backend, "_gguf_path", None)
@@ -613,9 +624,14 @@ def restore_kv_resume(backend, manifest) -> None:
def sweep_slot_save_dir() -> None:
try:
from utils.paths.storage_roots import llama_slot_cache_root
+
for path in llama_slot_cache_root().glob("resume-*.bin"):
with contextlib.suppress(OSError):
path.unlink()
+ from utils.paths.storage_roots import mlx_prompt_cache_root
+
+ for path in mlx_prompt_cache_root().glob("resume-*"):
+ shutil.rmtree(path, ignore_errors = True)
except Exception:
pass
diff --git a/studio/backend/core/inference/mlx_inference.py b/studio/backend/core/inference/mlx_inference.py
index d6d7a06a849..38f4ac3c5ce 100644
--- a/studio/backend/core/inference/mlx_inference.py
+++ b/studio/backend/core/inference/mlx_inference.py
@@ -10,6 +10,7 @@ import importlib
import inspect
import os
import re
+import shutil
import sys
import threading
import time
@@ -630,6 +631,24 @@ class VLMPromptSnapshotStore:
for other in [other for other in self._entries if other[0] != key]:
self.discard(other)
+ def resume_entries(self):
+ """The newest turn's longest boundary snapshot, then its replay."""
+ newest = next(reversed(self._entries), None)
+ if newest is None:
+ return []
+ key = newest[0]
+ replay = self._replays.get(key)
+ boundaries = [i for i in self._entries if i[0] == key and not self._is_replay(i)]
+ # _touch keeps the longest last, but after an edit that can be an older base the replay's
+ # turn no longer extends: prefer the longest one it does.
+ fits = [i for i in boundaries if replay is not None and replay[1][: len(i[1])] == i[1]]
+ boundary = max(fits, key = lambda i: len(i[1])) if fits else next(reversed(boundaries), None)
+ return [
+ (item[0], list(item[1]), self._entries[item][0], self._is_replay(item))
+ for item in (boundary, replay)
+ if item is not None
+ ]
+
def clear(self):
self._entries.clear()
self._bases.clear()
@@ -3090,6 +3109,7 @@ class _MLXPromptCacheHistory:
self._max_bytes = max_bytes
self._lru = lru_cls(max_size = max_entries, max_bytes = max_bytes)
self._oversized = None
+ self._newest = None
def _take_oversized(self, key, tokens):
held, self._oversized = self._oversized, None
@@ -3157,6 +3177,7 @@ class _MLXPromptCacheHistory:
tokens = tokens[:covered]
if not tokens:
return
+ self._newest = (key, tokens)
if nbytes > self._max_bytes:
# In the LRU it would evict itself and every other conversation.
logger.debug(
@@ -3168,6 +3189,86 @@ class _MLXPromptCacheHistory:
return
self._lru.insert_cache(key, tokens, cache)
+ def resume_entries(self):
+ if self._newest is None:
+ return []
+ key, tokens = self._newest
+ held = self._oversized
+ if held is not None and held[0] == key and held[1] == tokens:
+ return [(key, tokens, held[2], False)]
+ try:
+ # The trie's own entry: fetch_nearest_cache would deep-copy it.
+ return [(key, tokens, self._lru._trie.get(key, tokens).prompt_cache, False)]
+ except (AttributeError, KeyError):
+ return []
+
+
+def _resume_save_max_bytes():
+ # Mirrors core.inference.llama_cpp._SLOT_SAVE_MAX_BYTES.
+ try:
+ return int(os.environ.get("UNSLOTH_SLOT_SAVE_MAX_BYTES") or (10 << 30))
+ except ValueError:
+ return 10 << 30
+
+
+_RESUME_CACHE_MODULES = ("mlx.", "mlx_lm.", "mlx_vlm.", "unsloth_zoo.mlx.")
+
+
+def save_resume_cache(path, key, tokens, entries):
+ """Arrays as safetensors tensors, the cache layout pickled in metadata: any cache class fits."""
+ import base64
+ import io
+ import pickle
+
+ import mlx.core as mx
+
+ arrays = {}
+
+ class _Pickler(pickle.Pickler):
+ def persistent_id(self, obj):
+ if isinstance(obj, mx.array):
+ name = f"a{len(arrays)}"
+ arrays[name] = obj
+ return name
+ return None
+
+ layout = io.BytesIO()
+ _Pickler(layout, protocol = pickle.HIGHEST_PROTOCOL).dump((key, list(tokens), entries))
+ # Renamed into place, so a reader never sees a partial file and a removed dir fails the write.
+ partial = f"{path}.partial.safetensors"
+ mx.save_safetensors(
+ partial, arrays, {"layout": base64.b64encode(layout.getvalue()).decode("ascii")}
+ )
+ os.replace(partial, path)
+
+
+def load_resume_cache(path):
+ """``(key, tokens, entries)`` from ``save_resume_cache``; refuses any non-MLX class."""
+ import base64
+ import io
+ import pickle
+
+ import mlx.core as mx
+
+ arrays, metadata = mx.load(path, return_metadata = True)
+
+ class _Unpickler(pickle.Unpickler):
+ def persistent_load(self, name):
+ return arrays[name]
+
+ def find_class(self, module, name):
+ found = None
+ if module.startswith(_RESUME_CACHE_MODULES) and "." not in name:
+ found = super().find_class(module, name)
+ if not isinstance(found, type):
+ raise pickle.UnpicklingError(f"{module}.{name} is not a cache class")
+ return found
+
+ key, tokens, entries = _Unpickler(io.BytesIO(base64.b64decode(metadata["layout"]))).load()
+ # Read now: the caller deletes the file once the restore returns.
+ mx.eval(list(arrays.values()))
+ return key, tokens, entries
+
def _mlx_distributed_rank_size(group = None):
if group is None:
@@ -4401,6 +4502,8 @@ class MLXInferenceBackend:
self._kv_cache_window = None
self._kv_context_budget = None
self._served_context = None
+ self._resume_load_args = None
+ self._resume_files = None
self._template_override = _template_override_status(None, None, None)[1]
self._prompt_cache_history = None
@@ -4442,6 +4545,115 @@ class MLXInferenceBackend:
self._vlm_snapshot_store_unavailable = False
self._vlm_is_diffusion_model = None
+ def _resume_weight_files(self):
+ from pathlib import Path
+
+ name = self.active_model_name
+ files = []
+ for root in sorted(
+ {d for d in (_snapshot_dir(self._model, name), _snapshot_dir(None, name)) if d}
+ ):
+ weights = [*Path(root).glob("*.safetensors"), *Path(root).glob("*.json")]
+ for path in sorted(weights):
+ st = path.stat()
+ files.append((str(path), st.st_size, st.st_mtime_ns))
+ return tuple(files)
+
+ def _resume_fingerprint(self):
+ """What a restore must match: the weights this load read and every KV-affecting setting."""
+ import mlx.core as mx
+
+ files = self._resume_files
+ if files is None:
+ files = self._resume_weight_files()
+ versions = [mx.__version__]
+ for module in ("mlx_lm", "mlx_vlm"):
+ versions.append(getattr(sys.modules.get(module), "__version__", None))
+ return (
+ self.active_model_name,
+ files,
+ tuple(versions),
+ self._is_vlm,
+ self._kv_quant_bits(),
+ bool(getattr(self, "_turboquant", False)),
+ self._kv_cache_window,
+ bool(self._int8_prefill.get("active")),
+ self._prefill_step,
+ self._resume_load_args,
+ )
+
+ def _resume_stores(self):
+ return (("text", self._prompt_cache_history), ("vlm", self._vlm_snapshot_store))
+
+ def save_prompt_cache_for_resume(self, directory):
+ """Saves each store's newest turn into ``directory``; the manifest, or None."""
+ if not self.active_model_name or self._distributed_world_size > 1:
+ return None
+ # The KV was computed from the weights read at load; files replaced since would mislabel it.
+ if self._resume_files is False or (
+ self._resume_files is not None and self._resume_files != self._resume_weight_files()
+ ):
+ return None
+ # Never blocks: the worker loop advancing a batch is the thread that would release it.
+ if not self._generation_lock.acquire(blocking = False):
+ return None
+ try:
+ kept = [
+ (store_name, store.resume_entries() if store is not None else ())
+ for store_name, store in self._resume_stores()
+ ]
+ # llama-slots' bounds: a cap, and 1 GiB of disk left after it.
+ total = sum(cache_entries_nbytes(item[2]) for _, items in kept for item in items)
+ if total > _resume_save_max_bytes() or shutil.disk_usage(directory).free < total + (
+ 1 << 30
+ ):
+ logger.debug("MLX prompt cache: %d bytes do not fit the save budget", total)
+ return None
+ slots = []
+ for store_name, items in kept:
+ for index, (key, tokens, entries, replay) in enumerate(items):
+ filename = f"{store_name}-{index}.safetensors"
+ save_resume_cache(os.path.join(directory, filename), key, tokens, entries)
+ slots.append({"filename": filename, "store": store_name, "replay": replay})
+ if not slots:
+ return None
+ return {
+ "engine": "mlx",
+ "dir": directory,
+ "fingerprint": self._resume_fingerprint(),
+ "slots": slots,
+ }
+ finally:
+ self._generation_lock.release()
+
+ def restore_prompt_cache_for_resume(self, manifest):
+ """Restores a manifest saved under this exact load; the entry count."""
+ if not self.active_model_name or manifest.get("engine") != "mlx":
+ return 0
+ if manifest.get("fingerprint") != self._resume_fingerprint():
+ logger.info("MLX prompt cache: the saved context belongs to another load; not restored")
+ return 0
+ restored = 0
+ with self._generation_lock:
+ for slot in manifest.get("slots") or ():
+ if slot.get("store") == "text":
+ store = self._prompt_cache()
+ elif slot.get("store") == "vlm":
+ store = self._vlm_prompt_cache_store()
+ else:
+ store = None
+ if store is None:
+ continue
+ key, tokens, entries = load_resume_cache(
+ os.path.join(manifest["dir"], slot["filename"])
+ )
+ if slot["store"] == "text":
+ store.insert(key, tokens, entries)
+ else:
+ store.store(key, tokens, entries, replay = bool(slot.get("replay")))
+ restored += 1
+ return restored
+
def _release_vlm_snapshots(self):
store = getattr(self, "_vlm_snapshot_store", None)
if store is not None:
@@ -4975,6 +5187,7 @@ class MLXInferenceBackend:
# Keep the token so the native-template fallback can fetch a gated model's repo template during generation.
self._hf_token = hf_token
+ self._resume_load_args = (bool(load_in_4bit), repr(dtype), repr(tuple(offload or ())))
model_name = config.identifier if hasattr(config, "identifier") else str(config)
is_vision = getattr(config, "is_vision", False)
kv_bits, turboquant = parse_mlx_kv_quant(kv_quant)
@@ -5350,6 +5563,10 @@ class MLXInferenceBackend:
)
self.active_model_name = model_name
+ try:
+ self._resume_files = self._resume_weight_files()
+ except OSError:
+ self._resume_files = False
self.models[model_name] = {
# Per-model token for the native-template fallback (matches transformers).
"hf_token": hf_token,
diff --git a/studio/backend/core/inference/orchestrator.py b/studio/backend/core/inference/orchestrator.py
index 20764d82512..1225f8355b3 100644
--- a/studio/backend/core/inference/orchestrator.py
+++ b/studio/backend/core/inference/orchestrator.py
@@ -3001,6 +3001,90 @@ class InferenceOrchestrator:
raise (ValueError if resp.get("invalid") else RuntimeError)(resp["error"])
return {"ids": list(resp["ids"]), "text": resp["text"], "n_vocab": resp.get("n_vocab")}
+ def _serves_mlx(self) -> bool:
+ if getattr(self, "_managed_engine", None) is not None or not self.active_model_name:
+ return False
+ return bool((self.models.get(self.active_model_name) or {}).get("is_mlx"))
+
+ def _prompt_cache_resume_reply(
+ self,
+ cmd: dict,
+ timeout: float,
+ cancelled = None,
+ ):
+ request_id = str(uuid.uuid4())
+ read_one, _drain, release_mailbox = self._direct_reader(request_id)
+ try:
+ self._send_cmd({**cmd, "request_id": request_id})
+ return self._await_reply(
+ read_one,
+ request_id,
+ f"{cmd['type']}_response",
+ "prompt cache",
+ timeout,
+ cancelled,
+ )
+ finally:
+ release_mailbox()
+
+ def save_prompt_cache_for_resume(
+ self,
+ should_abort: Optional[Callable[[], bool]] = None,
+ timeout: float = 120.0,
+ ) -> Optional[dict]:
+ """The MLX worker's save manifest, or None; a failed or abandoned save's dir is removed."""
+ if not self._serves_mlx() or not self._gen_lock.acquire(blocking = False):
+ return None
+ directory = manifest = None
+ try:
+ with self._mailbox_lock:
+ dispatched = bool(self._mailboxes)
+ if dispatched or self._unload_pending:
+ return None
+ from utils.paths.storage_roots import mlx_prompt_cache_root
+
+ root = mlx_prompt_cache_root()
+ root.mkdir(parents = True, exist_ok = True)
+ # Saved KV encodes chat content; keep it from other local users, as llama-slots does.
+ with contextlib.suppress(OSError):
+ os.chmod(root, 0o700)
+ directory = root / f"resume-{uuid.uuid4().hex}"
+ directory.mkdir(mode = 0o700)
+ resp = self._prompt_cache_resume_reply(
+ {"type": "save_prompt_cache", "directory": str(directory)}, timeout, should_abort
+ )
+ if resp is not None and resp.get("error"):
+ logger.debug("MLX prompt cache save failed: %s", resp["error"])
+ elif resp is not None:
+ manifest = resp.get("result")
+ return manifest
+ finally:
+ self._gen_lock.release()
+ if not manifest and directory is not None:
+ import shutil
+ shutil.rmtree(directory, ignore_errors = True)
+
+ def restore_prompt_cache_for_resume(
+ self,
+ manifest: dict,
+ timeout: float = 120.0,
+ ) -> int:
+ if not self._serves_mlx() or not self._gen_lock.acquire(timeout = timeout):
+ return 0
+ try:
+ resp = self._prompt_cache_resume_reply(
+ {"type": "restore_prompt_cache", "manifest": manifest}, timeout
+ )
+ finally:
+ self._gen_lock.release()
+ if resp is None or resp.get("error"):
+ logger.info(
+ "MLX prompt cache restore failed: %s",
+ "timed out" if resp is None else resp["error"],
+ )
+ return 0
+ return int(resp.get("result") or 0)
+
def generate_completion(
self,
prompt_ids: list,
diff --git a/studio/backend/core/inference/worker.py b/studio/backend/core/inference/worker.py
index f105cf474d6..23aca7aada9 100644
--- a/studio/backend/core/inference/worker.py
+++ b/studio/backend/core/inference/worker.py
@@ -1272,6 +1272,34 @@ def _handle_resolve_prompt(backend, cmd: dict, resp_queue: Any) -> None:
_send_response(resp_queue, reply)
+_PROMPT_CACHE_RESUME_METHODS = {
+ "save_prompt_cache": ("save_prompt_cache_for_resume", "directory"),
+ "restore_prompt_cache": ("restore_prompt_cache_for_resume", "manifest"),
+}
+
+
+def _handle_prompt_cache_resume(
+ backend,
+ cmd: dict,
+ resp_queue: Any,
+ busy: bool = False,
+) -> None:
+ """Every outcome answers: the parent otherwise waits out its timeout."""
+ method, argument = _PROMPT_CACHE_RESUME_METHODS[cmd["type"]]
+ reply = {"type": f"{cmd['type']}_response", "request_id": cmd.get("request_id")}
+ run = getattr(backend, method, None)
+ if busy:
+ reply["error"] = "A generation is in progress."
+ elif not callable(run):
+ reply["error"] = "The loaded backend keeps no prompt cache."
+ else:
+ try:
+ reply["result"] = run(cmd.get(argument))
+ except Exception as exc:
+ reply["error"] = str(exc)
+ _send_response(resp_queue, reply)
+
+
def _handle_count_tokens(backend, cmd: dict, resp_queue: Any) -> None:
"""Count prompt tokens for the loaded model and reply with the total."""
try:
@@ -2272,6 +2300,10 @@ def run_inference_process(
_handle_count_tokens(backend, cmd, resp_queue)
elif cmd_type == "resolve_prompt":
_handle_resolve_prompt(backend, cmd, resp_queue)
+ elif cmd_type in _PROMPT_CACHE_RESUME_METHODS:
+ _handle_prompt_cache_resume(
+ backend, cmd, resp_queue, busy = bool(batch.rows_in_flight)
+ )
elif cmd_type == "share_object":
_handle_share_object(backend, cmd, resp_queue)
elif cmd_type == "load":
@@ -2524,6 +2556,9 @@ def run_inference_process(
elif cmd_type == "resolve_prompt":
_handle_resolve_prompt(backend, cmd, resp_queue)
+ elif cmd_type in _PROMPT_CACHE_RESUME_METHODS:
+ _handle_prompt_cache_resume(backend, cmd, resp_queue)
+
elif cmd_type == "share_object":
_handle_share_object(backend, cmd, resp_queue)
diff --git a/studio/backend/routes/inference.py b/studio/backend/routes/inference.py
index b1042544948..0db8890a1cf 100644
--- a/studio/backend/routes/inference.py
+++ b/studio/backend/routes/inference.py
@@ -19107,11 +19107,11 @@ async def _load_model_impl(
_close_load_event(
_load_event, model_log_label if native_grant_backed else config.identifier, None
)
- # Clear the idle-unload stash so the next /v1 request doesn't resurrect an old GGUF.
+ # Consume the reload stash: an MLX swap's chat context is restored, anything else dropped.
from core.inference.llama_keepwarm import note_model_loaded
if replacing:
- note_model_loaded()
+ await asyncio.to_thread(note_model_loaded, backend)
account_access.publish_resident(
"chat",
request.model_path,
diff --git a/studio/backend/tests/test_mlx_inference_backend.py b/studio/backend/tests/test_mlx_inference_backend.py
index 6190cdd79b8..0f8e03b5c37 100644
--- a/studio/backend/tests/test_mlx_inference_backend.py
+++ b/studio/backend/tests/test_mlx_inference_backend.py
@@ -3860,6 +3860,154 @@ def test_mlx_prompt_cache_covers_hybrid_recurrent_layouts(monkeypatch):
assert tuple(range(30)) in history._lru.entries["hybrid"]
+def test_mlx_prompt_cache_resume_restores_the_newest_entries_under_the_same_load(tmp_path):
+ mx = pytest.importorskip("mlx.core")
+ pytest.importorskip("mlx_lm")
+ from mlx_lm.models.cache import KVCache
+
+ from core.inference.mlx_inference import (
+ MLXInferenceBackend,
+ VLMPromptSnapshotStore,
+ _MLXPromptCacheHistory,
+ )
+
+ def kv(n, seed):
+ entry = KVCache()
+ rows = mx.random.normal((1, 2, n, 4), key = mx.random.key(seed)).astype(mx.float16)
+ entry.update_and_fetch(rows, rows + 1)
+ return [entry]
+
+ model_dir = tmp_path / "model"
+ model_dir.mkdir()
+ (model_dir / "model.safetensors").write_bytes(b"weights")
+
+ def backend(window = None):
+ loaded = MLXInferenceBackend()
+ loaded.active_model_name = str(model_dir)
+ loaded._kv_cache_window = window
+ loaded._prompt_cache_history = _MLXPromptCacheHistory(6, 1 << 30)
+ loaded._vlm_snapshot_store = VLMPromptSnapshotStore(1 << 30)
+ return loaded
+
+ saved = backend()
+ saved._prompt_cache_history.insert("k", list(range(8)), kv(8, 0))
+ newest = kv(12, 1)
+ saved._prompt_cache_history.insert("k|other", list(range(12)), newest)
+ saved._vlm_snapshot_store.store("k|img", list(range(6)), kv(6, 2))
+ saved._vlm_snapshot_store.store("k|img", [9] * 5, kv(5, 3), replay = True)
+ out = tmp_path / "save"
+ out.mkdir()
+ manifest = saved.save_prompt_cache_for_resume(str(out))
+ assert [slot["store"] for slot in manifest["slots"]] == ["text", "vlm", "vlm"]
+
+ restored = backend()
+ assert restored.restore_prompt_cache_for_resume(manifest) == 3
+ [(key, tokens, cache, _)] = restored._prompt_cache_history.resume_entries()
+ assert (key, tokens) == ("k|other", list(range(12)))
+ assert mx.array_equal(cache[0].state[0], newest[0].state[0]).item()
+ with pytest.raises(KeyError):
+ restored._prompt_cache_history._lru._trie.get("k", list(range(8)))
+ vlm = restored._vlm_snapshot_store.resume_entries()
+ assert [(key, prefix, replay) for key, prefix, _, replay in vlm] == [
+ ("k|img", list(range(6)), False),
+ ("k|img", [9] * 5, True),
+ ]
+
+ assert backend(window = 64).restore_prompt_cache_for_resume(manifest) == 0
+ (model_dir / "model.safetensors").write_bytes(b"rewritten weights")
+ assert backend().restore_prompt_cache_for_resume(manifest) == 0
+
+ # A turn over the LRU budget is held outside it, and still kept.
+ over_budget = _MLXPromptCacheHistory(6, 1)
+ over_budget.insert("k", list(range(8)), kv(8, 4))
+ assert [tokens for _, tokens, _, _ in over_budget.resume_entries()] == [list(range(8))]
+
+
+def test_mlx_vlm_resume_keeps_the_boundary_an_edited_turn_extends():
+ mx = pytest.importorskip("mlx.core")
+ pytest.importorskip("mlx_lm")
+ from mlx_lm.models.cache import KVCache
+
+ from core.inference.mlx_inference import VLMPromptSnapshotStore
+
+ def kv(n):
+ entry = KVCache()
+ rows = mx.zeros((1, 2, n, 4), dtype = mx.float16)
+ entry.update_and_fetch(rows, rows)
+ return [entry]
+
+ store = VLMPromptSnapshotStore(1 << 30)
+ store.store("k", list(range(12)), kv(12))
+ store.store("k", list(range(14)), kv(14), replay = True)
+ # The edit keeps the first 4 tokens; the old, longer boundary is then touched last.
+ edited = [*range(4), 90, 91]
+ store.store("k", list(range(4)), kv(4))
+ store.store("k", edited, kv(6), replay = True)
+ store._touch(("k", tuple(range(12))))
+ assert [(prefix, replay) for _, prefix, _, replay in store.resume_entries()] == [
+ (list(range(4)), False),
+ (edited, True),
+ ]
+
+
+def test_mlx_prompt_cache_resume_keys_on_the_weights_the_load_read(tmp_path, monkeypatch):
+ mx = pytest.importorskip("mlx.core")
+ pytest.importorskip("mlx_lm")
+ from mlx_lm.models.cache import KVCache
+
+ from core.inference.mlx_inference import MLXInferenceBackend, _MLXPromptCacheHistory
+
+ model_dir = tmp_path / "model"
+ model_dir.mkdir()
+ weights = model_dir / "model.safetensors"
+ weights.write_bytes(b"weights")
+
+ def loaded():
+ backend = MLXInferenceBackend()
+ backend.active_model_name = str(model_dir)
+ backend._resume_files = backend._resume_weight_files() # what load_model records
+ backend._prompt_cache_history = _MLXPromptCacheHistory(6, 1 << 30)
+ entry = KVCache()
+ rows = mx.zeros((1, 2, 8, 4), dtype = mx.float16)
+ entry.update_and_fetch(rows, rows)
+ backend._prompt_cache_history.insert("k", list(range(8)), [entry])
+ return backend
+
+ def save(backend):
+ out = tmp_path / f"save-{len(list(tmp_path.glob('save-*')))}"
+ out.mkdir()
+ return backend.save_prompt_cache_for_resume(str(out))
+
+ old = loaded()
+ manifest = save(old)
+ assert manifest is not None
+ # Replaced while the old weights are resident: their KV must neither be saved nor restored.
+ weights.write_bytes(b"replaced weights")
+ assert save(old) is None
+ assert loaded().restore_prompt_cache_for_resume(manifest) == 0
+
+ # The same bounds as llama-server's slot save.
+ monkeypatch.setenv("UNSLOTH_SLOT_SAVE_MAX_BYTES", "1")
+ assert save(loaded()) is None
+
+
+def test_mlx_resume_cache_loads_only_mlx_cache_classes(tmp_path):
+ import collections
+ import pickle
+
+ mx = pytest.importorskip("mlx.core")
+ from core.inference.mlx_inference import load_resume_cache, save_resume_cache
+
+ from mlx_lm.models.cache import make_prompt_cache
+
+ path = str(tmp_path / "cache.safetensors")
+ # Outside the MLX modules, and inside them but not a class.
+ for refused in (collections.OrderedDict(), make_prompt_cache):
+ save_resume_cache(path, "k", [1], [mx.zeros(2), refused])
+ with pytest.raises(pickle.UnpicklingError):
+ load_resume_cache(path)
+
+
def _audio_model(module_paths = ("audio_tower",), model_type = "gemma3n"):
"""Model stub shaped like a loaded mlx-vlm tree (families nest differently)."""
return SimpleNamespace(
diff --git a/studio/backend/tests/test_mlx_worker_audio_commands.py b/studio/backend/tests/test_mlx_worker_audio_commands.py
index d04a43b22fb..6fe707522ff 100644
--- a/studio/backend/tests/test_mlx_worker_audio_commands.py
+++ b/studio/backend/tests/test_mlx_worker_audio_commands.py
@@ -3,7 +3,9 @@
"""A dropped audio command costs the caller its full 120s deadline, so the MLX loop must always reply."""
+import os
import queue as _queue
+from pathlib import Path
from types import SimpleNamespace
import pytest
@@ -116,6 +118,75 @@ def test_whisper_on_a_backend_without_asr_explains_itself(monkeypatch):
assert "attribute" not in errors[0]["error"].lower()
+def test_mlx_loop_answers_every_prompt_cache_resume_command(monkeypatch):
+ backend = SimpleNamespace(
+ save_prompt_cache_for_resume = lambda directory: {"dir": directory},
+ restore_prompt_cache_for_resume = lambda manifest: (_ for _ in ()).throw(
+ RuntimeError("unreadable")
+ ),
+ )
+ sent = _run_mlx_loop(
+ monkeypatch,
+ [
+ {"type": "save_prompt_cache", "request_id": "s", "directory": "/d"},
+ {"type": "restore_prompt_cache", "request_id": "r", "manifest": {}},
+ ],
+ backend = backend,
+ )
+ replies = {m["request_id"]: m for m in sent if m.get("request_id") in ("s", "r")}
+ assert replies["s"]["type"] == "save_prompt_cache_response"
+ assert replies["s"]["result"] == {"dir": "/d"}
+ assert replies["r"]["type"] == "restore_prompt_cache_response"
+ assert replies["r"]["error"] == "unreadable"
+
+ resp = _RespQueue()
+ worker._handle_prompt_cache_resume(
+ SimpleNamespace(), {"type": "save_prompt_cache", "request_id": "u"}, resp
+ )
+ assert resp.sent[0]["request_id"] == "u" and "no prompt cache" in resp.sent[0]["error"]
+
+
[email protected]("case", ["saved", "error", "abandoned", "dispatched"])
+def test_a_prompt_cache_save_keeps_its_directory_only_with_a_manifest(monkeypatch, tmp_path, case):
+ import threading
+
+ from core.inference import orchestrator as orch_mod
+ from utils.paths import storage_roots
+
+ root = tmp_path / "mlx-prompt-cache"
+ monkeypatch.setattr(storage_roots, "mlx_prompt_cache_root", lambda: root)
+ orch = orch_mod.InferenceOrchestrator.__new__(orch_mod.InferenceOrchestrator)
+ orch.active_model_name, orch.models = "m", {"m": {"is_mlx": True}}
+ orch._managed_engine, orch._unload_pending = None, False
+ orch._gen_lock, orch._mailbox_lock = threading.Lock(), threading.Lock()
+ orch._mailboxes = {"generation": object()} if case == "dispatched" else {}
+ sent = []
+
+ def reply(
+ cmd,
+ timeout,
+ cancelled = None,
+ ):
+ sent.append(cmd)
+ if case == "saved":
+ return {"result": {"engine": "mlx", "dir": cmd["directory"]}}
+ return {"error": "disk full"} if case == "error" else None
+
+ monkeypatch.setattr(orch, "_prompt_cache_resume_reply", reply)
+ manifest = orch.save_prompt_cache_for_resume()
+ assert not orch._gen_lock.locked() and len(sent) == (case != "dispatched")
+ if case == "saved":
+ assert [str(path) for path in root.iterdir()] == [manifest["dir"]]
+ if os.name == "posix":
+ # Saved KV encodes chat content, so neither level is readable by other local users.
+ assert {
+ (root.stat().st_mode & 0o777),
+ (Path(manifest["dir"]).stat().st_mode & 0o777),
+ } == {0o700}
+ else:
+ assert manifest is None and list(root.iterdir() if root.exists() else ()) == []
+
+
@pytest.mark.parametrize("cmd_type", ["generate_audio", "generate_video"])
def test_every_mlx_command_gets_exactly_one_reply(monkeypatch, cmd_type):
"""One reply, not zero and not a duplicate that would confuse the mailbox."""
diff --git a/studio/backend/tests/test_openai_auto_switch.py b/studio/backend/tests/test_openai_auto_switch.py
index 4aca9950c00..c6b1cd6d4a9 100644
--- a/studio/backend/tests/test_openai_auto_switch.py
+++ b/studio/backend/tests/test_openai_auto_switch.py
@@ -6251,10 +6251,11 @@ def test_unload_route_clears_reload_stash(monkeypatch):
def test_non_gguf_load_clears_reload_stash():
- # A non-GGUF load clears the stash too, so it never lingers when idle-unload is off.
+ # A non-GGUF load clears the stash too, so it never lingers when idle-unload is off; it is
+ # handed the orchestrator, which restores an MLX chat context kept across a media swap.
src = inspect.getsource(inference_route._load_model_impl)
- assert src.count("note_model_loaded()") >= 1
+ assert "to_thread(note_model_loaded, backend)" in src
assert "to_thread(note_model_loaded, llama_backend)" in src
@@ -6963,14 +6964,20 @@ def test_new_idle_save_purges_previous_manifest_files(tmp_path):
def test_sweep_slot_save_dir_removes_only_resume_files(monkeypatch, tmp_path):
from utils.paths import storage_roots
+ mlx_root = tmp_path / "mlx"
monkeypatch.setattr(storage_roots, "llama_slot_cache_root", lambda: tmp_path)
+ monkeypatch.setattr(storage_roots, "mlx_prompt_cache_root", lambda: mlx_root)
stale = tmp_path / "resume-old-slot0.bin"
stale.write_bytes(b"kv")
other = tmp_path / "unrelated.txt"
other.write_text("keep")
+ (mlx_root / "resume-old").mkdir(parents = True)
+ (mlx_root / "resume-old" / "text-0.safetensors").write_bytes(b"kv")
+ (mlx_root / "unrelated").mkdir()
kw.sweep_slot_save_dir()
assert not stale.exists()
assert other.exists()
+ assert not (mlx_root / "resume-old").exists() and (mlx_root / "unrelated").exists()
def test_keep_kv_setting_roundtrip_and_default(monkeypatch):
diff --git a/studio/backend/tests/test_swap_kv_resume.py b/studio/backend/tests/test_swap_kv_resume.py
index bc04d25081f..82d00ba1da9 100644
--- a/studio/backend/tests/test_swap_kv_resume.py
+++ b/studio/backend/tests/test_swap_kv_resume.py
@@ -317,6 +317,68 @@ def test_the_real_evict_chat_promotes_the_identity_read_before_unload(env, monke
assert kw._kv_resume["identity"] == IDENTITY and env.files[0].exists()
+class _FakeMlxOrchestrator:
+ loading_models = ()
+ _managed_engine = None
+ _openai_advertised_id = None
+
+ def __init__(self, tmp_path):
+ self.root, self.active_model_name, self.restores = tmp_path, "unsloth/A-MLX", []
+
+ def _serves_mlx(self):
+ return self.active_model_name is not None
+
+ def save_prompt_cache_for_resume(self, should_abort):
+ directory = self.root / "resume-mlx"
+ directory.mkdir()
+ (directory / "text-0.safetensors").write_bytes(b"kv")
+ slot = {"filename": "text-0.safetensors", "store": "text"}
+ return {"engine": "mlx", "dir": str(directory), "fingerprint": (), "slots": [slot]}
+
+ def restore_prompt_cache_for_resume(self, manifest):
+ self.restores.append(manifest)
+ return 1
+
+ def unload_model(self, name):
+ self.active_model_name = None
+
+ def _shutdown_subprocess(self, timeout):
+ return True
+
+
[email protected]("reload", ["mlx", "gguf"])
+def test_an_mlx_chat_keeps_its_prompt_cache_across_the_swap(env, monkeypatch, tmp_path, reload):
+ import core.inference as inference_pkg
+ import core.inference.llama_cpp as llama_cpp_mod
+ import core.inference.model_slots as model_slots
+ import core.inference.orchestrator as orchestrator_mod
+ from hub.services.models import account_access
+
+ orch = _FakeMlxOrchestrator(tmp_path)
+ env.unload_model()
+ env._wait_for_vram_settle = lambda **k: None
+ monkeypatch.setattr(model_slots, "unload_extra_models", lambda **k: None)
+ monkeypatch.setattr(inference_pkg, "get_inference_backend", lambda: orch)
+ monkeypatch.setattr(orchestrator_mod, "peek_inference_backend", lambda: orch)
+ monkeypatch.setattr(llama_cpp_mod, "chat_load_active", lambda: False)
+ monkeypatch.setattr(account_access, "clear_resident", lambda owner: None)
+ monkeypatch.setitem(arb._EVICTORS, arb.CHAT, arb._evict_chat)
+ _load()
+ stash = kw._kv_resume
+ assert stash["identity"] == ("unsloth/A-MLX", None, "unsloth/A-MLX", "mlx")
+ assert orch.active_model_name is None and Path(stash["dir"]).is_dir()
+ assert [(r["event"], r["reason"]) for r in env.rows] == [("unload", "swap")]
+
+ if reload == "mlx":
+ orch.active_model_name = "unsloth/A-MLX"
+ kw.note_model_loaded(orch)
+ else:
+ env.reload()
+ kw.note_model_loaded(env)
+ assert orch.restores == ([stash] if reload == "mlx" else []) and env.restores == []
+ assert kw._kv_resume is None and not Path(stash["dir"]).exists()
+
+
@pytest.mark.parametrize(
"case",
["generating", "keep_kv_off", "chat_not_resident", "not_gguf", "other_account", "chat_target"],
diff --git a/studio/backend/utils/paths/storage_roots.py b/studio/backend/utils/paths/storage_roots.py
index f14781e6402..958d792e8d3 100644
--- a/studio/backend/utils/paths/storage_roots.py
+++ b/studio/backend/utils/paths/storage_roots.py
@@ -234,6 +234,11 @@ def llama_slot_cache_root() -> Path:
return cache_root() / "llama-slots"
+def mlx_prompt_cache_root() -> Path:
+ """Dir the MLX worker saves its prompt cache in, for the next load of the same model."""
+ return cache_root() / "mlx-prompt-cache"
+
+
def studio_bin_root() -> Path:
"""Dir for Unsloth-managed executables (the `unsloth` shim, downloaded tools like cloudflared)."""
return studio_root() / "bin"