Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
Studio: keep the MLX chat context when an image or video model takes the GPU (#15358)
Long Yixing20f3ac001827
diff --git a/studio/backend/core/inference/chat_swap_kv.py b/studio/backend/core/inference/chat_swap_kv.pyindex e3c1bfced3a..4bfbc1181fc 100644--- a/studio/backend/core/inference/chat_swap_kv.py+++ b/studio/backend/core/inference/chat_swap_kv.py@@ -3,7 +3,8 @@"""Keep the chat context when an image or video load takes the GPU from chat.-Before the media acquire (outside every lock) the chat slots' KV is saved to disk, bounded by+Before the media acquire (outside every lock) the chat KV (llama-server's slots, or the MLX+worker's prompt cache) is saved to disk, bounded by``SWAP_KV_SAVE_BUDGET_S``. The manifest waits here as "pending" until ``gpu_arbiter._evict_chat``unloads that same model; only then does it become the keep-warm stash that the next chat loadrestores. If nothing was evicted, ``SwapSave.close`` drops it and deletes its files.@@ -304,11 +305,36 @@ def _chat_account() -> Optional[str]:return gpu_arbiter.owner_account()-def _save_in_thread(llama, should_abort, abort: threading.Event, loop, future) -> None:+def mlx_chat_identity(orchestrator):+ serves_mlx = getattr(orchestrator, "_serves_mlx", None)+ if not callable(serves_mlx) or not serves_mlx():+ return None+ name = orchestrator.active_model_name+ return (name, None, getattr(orchestrator, "_openai_advertised_id", None) or name, "mlx")+++def _chat_saver():+ from core.inference.llama_keepwarm import _loaded_identity+ from routes.inference import get_llama_cpp_backend++ llama = get_llama_cpp_backend()+ identity = _loaded_identity(llama)+ if identity is not None:+ if not getattr(llama, "_gguf_path", None):+ return None+ return identity, llama.save_slots_for_resume+ from core.inference.orchestrator import peek_inference_backend++ orchestrator = peek_inference_backend()+ identity = mlx_chat_identity(orchestrator)+ return None if identity is None else (identity, orchestrator.save_prompt_cache_for_resume)+++def _save_in_thread(save, should_abort, abort: threading.Event, loop, future) -> None:"""Runs the blocking save on its own thread, so neither the budget nor the event loop'sshutdown waits on it. A result nobody is waiting for any more is deleted right here."""try:- manifest, error = llama.save_slots_for_resume(should_abort), None+ manifest, error = save(should_abort), Noneexcept BaseException as exc: # noqa: BLE001 - reported to the waitermanifest, error = None, excif abort.is_set():@@ -345,19 +371,16 @@ async def save_chat_kv_before_swap(global _pendingtry:from core.inference import gpu_arbiter- from core.inference.llama_keepwarm import _loaded_identityfrom utils.openai_auto_switch_settings import get_auto_unload_keep_kvif not gpu_arbiter.would_evict(owner, policy = policy, need_mib = need_mib, gpu_ids = gpu_ids):return SwapSave()if not await asyncio.to_thread(get_auto_unload_keep_kv):return SwapSave()- from routes.inference import get_llama_cpp_backend-- llama = get_llama_cpp_backend()- identity = _loaded_identity(llama)- if identity is None or not getattr(llama, "_gguf_path", None):+ saver = _chat_saver()+ if saver is None:return SwapSave()+ identity, save = saveraccount = _account_id()counted = _request_counted()if _chat_account() != account or _chat_busy(counted):@@ -389,7 +412,7 @@ async def save_chat_kv_before_swap(try:threading.Thread(target = _save_in_thread,- args = (llama, should_abort, abort, loop, future),+ args = (save, should_abort, abort, loop, future),name = "chat-kv-swap-save",daemon = True,).start()diff --git a/studio/backend/core/inference/gpu_arbiter.py b/studio/backend/core/inference/gpu_arbiter.pyindex db341c839d1..e278109bd7a 100644--- a/studio/backend/core/inference/gpu_arbiter.py+++ b/studio/backend/core/inference/gpu_arbiter.py@@ -147,7 +147,10 @@ def _evict_chat() -> None:_commit_swap_kv(unloaded)orchestrator = get_inference_backend()if orchestrator.active_model_name:+ unloaded = _mlx_chat_identity(orchestrator)orchestrator.unload_model(orchestrator.active_model_name)+ if unloaded is not None:+ _commit_swap_kv(unloaded)# In-flight loads lack active_model_name; cancel_load must drop the loading marker before teardown.for pending in list(getattr(orchestrator, "loading_models", ()) or ()):orchestrator.cancel_load(pending)@@ -173,6 +176,14 @@ def _chat_identity(llama):return None+def _mlx_chat_identity(orchestrator):+ try:+ from core.inference.chat_swap_kv import mlx_chat_identity+ return mlx_chat_identity(orchestrator)+ except Exception:+ return None++def _commit_swap_kv(unloaded) -> None:"""Promote the chat KV saved before this media load; an eviction never fails on it."""try:diff --git a/studio/backend/core/inference/llama_keepwarm.py b/studio/backend/core/inference/llama_keepwarm.pyindex 423977531c9..c05dcf7c8a9 100644--- a/studio/backend/core/inference/llama_keepwarm.py+++ b/studio/backend/core/inference/llama_keepwarm.py@@ -14,6 +14,7 @@ from __future__ import annotationsimport asyncioimport contextlibimport contextvars+import shutilimport threadingimport timefrom pathlib import Path@@ -552,6 +553,10 @@ def _delete_resume_files(manifest) -> None:for entry in manifest.get("slots") or []:with contextlib.suppress(OSError):(base / str(entry.get("filename"))).unlink()+ if manifest.get("engine") == "mlx":+ # Each MLX save has a directory of its own; llama-server's is shared.+ with contextlib.suppress(OSError):+ base.rmdir()except Exception:pass@@ -589,6 +594,12 @@ def purge_kv_resume(*, origin: Optional[str] = None) -> None:def restore_kv_resume(backend, manifest) -> None:try:+ if manifest.get("engine") == "mlx":+ # The worker checks the load matches the save; the identity names only the model.+ restore = getattr(backend, "restore_prompt_cache_for_resume", None)+ if callable(restore) and restore(manifest):+ logger.info("Restored the saved MLX prompt cache onto the reloaded model")+ returngguf = manifest.get("gguf")binary = manifest.get("binary")current = getattr(backend, "_gguf_path", None)@@ -613,9 +624,14 @@ def restore_kv_resume(backend, manifest) -> None:def sweep_slot_save_dir() -> None:try:from utils.paths.storage_roots import llama_slot_cache_root+for path in llama_slot_cache_root().glob("resume-*.bin"):with contextlib.suppress(OSError):path.unlink()+ from utils.paths.storage_roots import mlx_prompt_cache_root++ for path in mlx_prompt_cache_root().glob("resume-*"):+ shutil.rmtree(path, ignore_errors = True)except Exception:passdiff --git a/studio/backend/core/inference/mlx_inference.py b/studio/backend/core/inference/mlx_inference.pyindex d6d7a06a849..38f4ac3c5ce 100644--- a/studio/backend/core/inference/mlx_inference.py+++ b/studio/backend/core/inference/mlx_inference.py@@ -10,6 +10,7 @@ import importlibimport inspectimport osimport re+import shutilimport sysimport threadingimport time@@ -630,6 +631,24 @@ class VLMPromptSnapshotStore:for other in [other for other in self._entries if other[0] != key]:self.discard(other)+ def resume_entries(self):+ """The newest turn's longest boundary snapshot, then its replay."""+ newest = next(reversed(self._entries), None)+ if newest is None:+ return []+ key = newest[0]+ replay = self._replays.get(key)+ boundaries = [i for i in self._entries if i[0] == key and not self._is_replay(i)]+ # _touch keeps the longest last, but after an edit that can be an older base the replay's+ # turn no longer extends: prefer the longest one it does.+ fits = [i for i in boundaries if replay is not None and replay[1][: len(i[1])] == i[1]]+ boundary = max(fits, key = lambda i: len(i[1])) if fits else next(reversed(boundaries), None)+ return [+ (item[0], list(item[1]), self._entries[item][0], self._is_replay(item))+ for item in (boundary, replay)+ if item is not None+ ]+def clear(self):self._entries.clear()self._bases.clear()@@ -3090,6 +3109,7 @@ class _MLXPromptCacheHistory:self._max_bytes = max_bytesself._lru = lru_cls(max_size = max_entries, max_bytes = max_bytes)self._oversized = None+ self._newest = Nonedef _take_oversized(self, key, tokens):held, self._oversized = self._oversized, None@@ -3157,6 +3177,7 @@ class _MLXPromptCacheHistory:tokens = tokens[:covered]if not tokens:return+ self._newest = (key, tokens)if nbytes > self._max_bytes:# In the LRU it would evict itself and every other conversation.logger.debug(@@ -3168,6 +3189,86 @@ class _MLXPromptCacheHistory:returnself._lru.insert_cache(key, tokens, cache)+ def resume_entries(self):+ if self._newest is None:+ return []+ key, tokens = self._newest+ held = self._oversized+ if held is not None and held[0] == key and held[1] == tokens:+ return [(key, tokens, held[2], False)]+ try:+ # The trie's own entry: fetch_nearest_cache would deep-copy it.+ return [(key, tokens, self._lru._trie.get(key, tokens).prompt_cache, False)]+ except (AttributeError, KeyError):+ return []+++def _resume_save_max_bytes():+ # Mirrors core.inference.llama_cpp._SLOT_SAVE_MAX_BYTES.+ try:+ return int(os.environ.get("UNSLOTH_SLOT_SAVE_MAX_BYTES") or (10 << 30))+ except ValueError:+ return 10 << 30+++_RESUME_CACHE_MODULES = ("mlx.", "mlx_lm.", "mlx_vlm.", "unsloth_zoo.mlx.")+++def save_resume_cache(path, key, tokens, entries):+ """Arrays as safetensors tensors, the cache layout pickled in metadata: any cache class fits."""+ import base64+ import io+ import pickle++ import mlx.core as mx++ arrays = {}++ class _Pickler(pickle.Pickler):+ def persistent_id(self, obj):+ if isinstance(obj, mx.array):+ name = f"a{len(arrays)}"+ arrays[name] = obj+ return name+ return None++ layout = io.BytesIO()+ _Pickler(layout, protocol = pickle.HIGHEST_PROTOCOL).dump((key, list(tokens), entries))+ # Renamed into place, so a reader never sees a partial file and a removed dir fails the write.+ partial = f"{path}.partial.safetensors"+ mx.save_safetensors(+ partial, arrays, {"layout": base64.b64encode(layout.getvalue()).decode("ascii")}+ )+ os.replace(partial, path)+++def load_resume_cache(path):+ """``(key, tokens, entries)`` from ``save_resume_cache``; refuses any non-MLX class."""+ import base64+ import io+ import pickle++ import mlx.core as mx++ arrays, metadata = mx.load(path, return_metadata = True)++ class _Unpickler(pickle.Unpickler):+ def persistent_load(self, name):+ return arrays[name]++ def find_class(self, module, name):+ found = None+ if module.startswith(_RESUME_CACHE_MODULES) and "." not in name:+ found = super().find_class(module, name)+ if not isinstance(found, type):+ raise pickle.UnpicklingError(f"{module}.{name} is not a cache class")+ return found++ key, tokens, entries = _Unpickler(io.BytesIO(base64.b64decode(metadata["layout"]))).load()+ # Read now: the caller deletes the file once the restore returns.+ mx.eval(list(arrays.values()))+ return key, tokens, entries+def _mlx_distributed_rank_size(group = None):if group is None:@@ -4401,6 +4502,8 @@ class MLXInferenceBackend:self._kv_cache_window = Noneself._kv_context_budget = Noneself._served_context = None+ self._resume_load_args = None+ self._resume_files = Noneself._template_override = _template_override_status(None, None, None)[1]self._prompt_cache_history = None@@ -4442,6 +4545,115 @@ class MLXInferenceBackend:self._vlm_snapshot_store_unavailable = Falseself._vlm_is_diffusion_model = None+ def _resume_weight_files(self):+ from pathlib import Path++ name = self.active_model_name+ files = []+ for root in sorted(+ {d for d in (_snapshot_dir(self._model, name), _snapshot_dir(None, name)) if d}+ ):+ weights = [*Path(root).glob("*.safetensors"), *Path(root).glob("*.json")]+ for path in sorted(weights):+ st = path.stat()+ files.append((str(path), st.st_size, st.st_mtime_ns))+ return tuple(files)++ def _resume_fingerprint(self):+ """What a restore must match: the weights this load read and every KV-affecting setting."""+ import mlx.core as mx++ files = self._resume_files+ if files is None:+ files = self._resume_weight_files()+ versions = [mx.__version__]+ for module in ("mlx_lm", "mlx_vlm"):+ versions.append(getattr(sys.modules.get(module), "__version__", None))+ return (+ self.active_model_name,+ files,+ tuple(versions),+ self._is_vlm,+ self._kv_quant_bits(),+ bool(getattr(self, "_turboquant", False)),+ self._kv_cache_window,+ bool(self._int8_prefill.get("active")),+ self._prefill_step,+ self._resume_load_args,+ )++ def _resume_stores(self):+ return (("text", self._prompt_cache_history), ("vlm", self._vlm_snapshot_store))++ def save_prompt_cache_for_resume(self, directory):+ """Saves each store's newest turn into ``directory``; the manifest, or None."""+ if not self.active_model_name or self._distributed_world_size > 1:+ return None+ # The KV was computed from the weights read at load; files replaced since would mislabel it.+ if self._resume_files is False or (+ self._resume_files is not None and self._resume_files != self._resume_weight_files()+ ):+ return None+ # Never blocks: the worker loop advancing a batch is the thread that would release it.+ if not self._generation_lock.acquire(blocking = False):+ return None+ try:+ kept = [+ (store_name, store.resume_entries() if store is not None else ())+ for store_name, store in self._resume_stores()+ ]+ # llama-slots' bounds: a cap, and 1 GiB of disk left after it.+ total = sum(cache_entries_nbytes(item[2]) for _, items in kept for item in items)+ if total > _resume_save_max_bytes() or shutil.disk_usage(directory).free < total + (+ 1 << 30+ ):+ logger.debug("MLX prompt cache: %d bytes do not fit the save budget", total)+ return None+ slots = []+ for store_name, items in kept:+ for index, (key, tokens, entries, replay) in enumerate(items):+ filename = f"{store_name}-{index}.safetensors"+ save_resume_cache(os.path.join(directory, filename), key, tokens, entries)+ slots.append({"filename": filename, "store": store_name, "replay": replay})+ if not slots:+ return None+ return {+ "engine": "mlx",+ "dir": directory,+ "fingerprint": self._resume_fingerprint(),+ "slots": slots,+ }+ finally:+ self._generation_lock.release()++ def restore_prompt_cache_for_resume(self, manifest):+ """Restores a manifest saved under this exact load; the entry count."""+ if not self.active_model_name or manifest.get("engine") != "mlx":+ return 0+ if manifest.get("fingerprint") != self._resume_fingerprint():+ logger.info("MLX prompt cache: the saved context belongs to another load; not restored")+ return 0+ restored = 0+ with self._generation_lock:+ for slot in manifest.get("slots") or ():+ if slot.get("store") == "text":+ store = self._prompt_cache()+ elif slot.get("store") == "vlm":+ store = self._vlm_prompt_cache_store()+ else:+ store = None+ if store is None:+ continue+ key, tokens, entries = load_resume_cache(+ os.path.join(manifest["dir"], slot["filename"])+ )+ if slot["store"] == "text":+ store.insert(key, tokens, entries)+ else:+ store.store(key, tokens, entries, replay = bool(slot.get("replay")))+ restored += 1+ return restored+def _release_vlm_snapshots(self):store = getattr(self, "_vlm_snapshot_store", None)if store is not None:@@ -4975,6 +5187,7 @@ class MLXInferenceBackend:# Keep the token so the native-template fallback can fetch a gated model's repo template during generation.self._hf_token = hf_token+ self._resume_load_args = (bool(load_in_4bit), repr(dtype), repr(tuple(offload or ())))model_name = config.identifier if hasattr(config, "identifier") else str(config)is_vision = getattr(config, "is_vision", False)kv_bits, turboquant = parse_mlx_kv_quant(kv_quant)@@ -5350,6 +5563,10 @@ class MLXInferenceBackend:)self.active_model_name = model_name+ try:+ self._resume_files = self._resume_weight_files()+ except OSError:+ self._resume_files = Falseself.models[model_name] = {# Per-model token for the native-template fallback (matches transformers)."hf_token": hf_token,diff --git a/studio/backend/core/inference/orchestrator.py b/studio/backend/core/inference/orchestrator.pyindex 20764d82512..1225f8355b3 100644--- a/studio/backend/core/inference/orchestrator.py+++ b/studio/backend/core/inference/orchestrator.py@@ -3001,6 +3001,90 @@ class InferenceOrchestrator:raise (ValueError if resp.get("invalid") else RuntimeError)(resp["error"])return {"ids": list(resp["ids"]), "text": resp["text"], "n_vocab": resp.get("n_vocab")}+ def _serves_mlx(self) -> bool:+ if getattr(self, "_managed_engine", None) is not None or not self.active_model_name:+ return False+ return bool((self.models.get(self.active_model_name) or {}).get("is_mlx"))++ def _prompt_cache_resume_reply(+ self,+ cmd: dict,+ timeout: float,+ cancelled = None,+ ):+ request_id = str(uuid.uuid4())+ read_one, _drain, release_mailbox = self._direct_reader(request_id)+ try:+ self._send_cmd({**cmd, "request_id": request_id})+ return self._await_reply(+ read_one,+ request_id,+ f"{cmd['type']}_response",+ "prompt cache",+ timeout,+ cancelled,+ )+ finally:+ release_mailbox()++ def save_prompt_cache_for_resume(+ self,+ should_abort: Optional[Callable[[], bool]] = None,+ timeout: float = 120.0,+ ) -> Optional[dict]:+ """The MLX worker's save manifest, or None; a failed or abandoned save's dir is removed."""+ if not self._serves_mlx() or not self._gen_lock.acquire(blocking = False):+ return None+ directory = manifest = None+ try:+ with self._mailbox_lock:+ dispatched = bool(self._mailboxes)+ if dispatched or self._unload_pending:+ return None+ from utils.paths.storage_roots import mlx_prompt_cache_root++ root = mlx_prompt_cache_root()+ root.mkdir(parents = True, exist_ok = True)+ # Saved KV encodes chat content; keep it from other local users, as llama-slots does.+ with contextlib.suppress(OSError):+ os.chmod(root, 0o700)+ directory = root / f"resume-{uuid.uuid4().hex}"+ directory.mkdir(mode = 0o700)+ resp = self._prompt_cache_resume_reply(+ {"type": "save_prompt_cache", "directory": str(directory)}, timeout, should_abort+ )+ if resp is not None and resp.get("error"):+ logger.debug("MLX prompt cache save failed: %s", resp["error"])+ elif resp is not None:+ manifest = resp.get("result")+ return manifest+ finally:+ self._gen_lock.release()+ if not manifest and directory is not None:+ import shutil+ shutil.rmtree(directory, ignore_errors = True)++ def restore_prompt_cache_for_resume(+ self,+ manifest: dict,+ timeout: float = 120.0,+ ) -> int:+ if not self._serves_mlx() or not self._gen_lock.acquire(timeout = timeout):+ return 0+ try:+ resp = self._prompt_cache_resume_reply(+ {"type": "restore_prompt_cache", "manifest": manifest}, timeout+ )+ finally:+ self._gen_lock.release()+ if resp is None or resp.get("error"):+ logger.info(+ "MLX prompt cache restore failed: %s",+ "timed out" if resp is None else resp["error"],+ )+ return 0+ return int(resp.get("result") or 0)+def generate_completion(self,prompt_ids: list,diff --git a/studio/backend/core/inference/worker.py b/studio/backend/core/inference/worker.pyindex f105cf474d6..23aca7aada9 100644--- a/studio/backend/core/inference/worker.py+++ b/studio/backend/core/inference/worker.py@@ -1272,6 +1272,34 @@ def _handle_resolve_prompt(backend, cmd: dict, resp_queue: Any) -> None:_send_response(resp_queue, reply)+_PROMPT_CACHE_RESUME_METHODS = {+ "save_prompt_cache": ("save_prompt_cache_for_resume", "directory"),+ "restore_prompt_cache": ("restore_prompt_cache_for_resume", "manifest"),+}+++def _handle_prompt_cache_resume(+ backend,+ cmd: dict,+ resp_queue: Any,+ busy: bool = False,+) -> None:+ """Every outcome answers: the parent otherwise waits out its timeout."""+ method, argument = _PROMPT_CACHE_RESUME_METHODS[cmd["type"]]+ reply = {"type": f"{cmd['type']}_response", "request_id": cmd.get("request_id")}+ run = getattr(backend, method, None)+ if busy:+ reply["error"] = "A generation is in progress."+ elif not callable(run):+ reply["error"] = "The loaded backend keeps no prompt cache."+ else:+ try:+ reply["result"] = run(cmd.get(argument))+ except Exception as exc:+ reply["error"] = str(exc)+ _send_response(resp_queue, reply)++def _handle_count_tokens(backend, cmd: dict, resp_queue: Any) -> None:"""Count prompt tokens for the loaded model and reply with the total."""try:@@ -2272,6 +2300,10 @@ def run_inference_process(_handle_count_tokens(backend, cmd, resp_queue)elif cmd_type == "resolve_prompt":_handle_resolve_prompt(backend, cmd, resp_queue)+ elif cmd_type in _PROMPT_CACHE_RESUME_METHODS:+ _handle_prompt_cache_resume(+ backend, cmd, resp_queue, busy = bool(batch.rows_in_flight)+ )elif cmd_type == "share_object":_handle_share_object(backend, cmd, resp_queue)elif cmd_type == "load":@@ -2524,6 +2556,9 @@ def run_inference_process(elif cmd_type == "resolve_prompt":_handle_resolve_prompt(backend, cmd, resp_queue)+ elif cmd_type in _PROMPT_CACHE_RESUME_METHODS:+ _handle_prompt_cache_resume(backend, cmd, resp_queue)+elif cmd_type == "share_object":_handle_share_object(backend, cmd, resp_queue)diff --git a/studio/backend/routes/inference.py b/studio/backend/routes/inference.pyindex b1042544948..0db8890a1cf 100644--- a/studio/backend/routes/inference.py+++ b/studio/backend/routes/inference.py@@ -19107,11 +19107,11 @@ async def _load_model_impl(_close_load_event(_load_event, model_log_label if native_grant_backed else config.identifier, None)- # Clear the idle-unload stash so the next /v1 request doesn't resurrect an old GGUF.+ # Consume the reload stash: an MLX swap's chat context is restored, anything else dropped.from core.inference.llama_keepwarm import note_model_loadedif replacing:- note_model_loaded()+ await asyncio.to_thread(note_model_loaded, backend)account_access.publish_resident("chat",request.model_path,diff --git a/studio/backend/tests/test_mlx_inference_backend.py b/studio/backend/tests/test_mlx_inference_backend.pyindex 6190cdd79b8..0f8e03b5c37 100644--- a/studio/backend/tests/test_mlx_inference_backend.py+++ b/studio/backend/tests/test_mlx_inference_backend.py@@ -3860,6 +3860,154 @@ def test_mlx_prompt_cache_covers_hybrid_recurrent_layouts(monkeypatch):assert tuple(range(30)) in history._lru.entries["hybrid"]+def test_mlx_prompt_cache_resume_restores_the_newest_entries_under_the_same_load(tmp_path):+ mx = pytest.importorskip("mlx.core")+ pytest.importorskip("mlx_lm")+ from mlx_lm.models.cache import KVCache++ from core.inference.mlx_inference import (+ MLXInferenceBackend,+ VLMPromptSnapshotStore,+ _MLXPromptCacheHistory,+ )++ def kv(n, seed):+ entry = KVCache()+ rows = mx.random.normal((1, 2, n, 4), key = mx.random.key(seed)).astype(mx.float16)+ entry.update_and_fetch(rows, rows + 1)+ return [entry]++ model_dir = tmp_path / "model"+ model_dir.mkdir()+ (model_dir / "model.safetensors").write_bytes(b"weights")++ def backend(window = None):+ loaded = MLXInferenceBackend()+ loaded.active_model_name = str(model_dir)+ loaded._kv_cache_window = window+ loaded._prompt_cache_history = _MLXPromptCacheHistory(6, 1 << 30)+ loaded._vlm_snapshot_store = VLMPromptSnapshotStore(1 << 30)+ return loaded++ saved = backend()+ saved._prompt_cache_history.insert("k", list(range(8)), kv(8, 0))+ newest = kv(12, 1)+ saved._prompt_cache_history.insert("k|other", list(range(12)), newest)+ saved._vlm_snapshot_store.store("k|img", list(range(6)), kv(6, 2))+ saved._vlm_snapshot_store.store("k|img", [9] * 5, kv(5, 3), replay = True)+ out = tmp_path / "save"+ out.mkdir()+ manifest = saved.save_prompt_cache_for_resume(str(out))+ assert [slot["store"] for slot in manifest["slots"]] == ["text", "vlm", "vlm"]++ restored = backend()+ assert restored.restore_prompt_cache_for_resume(manifest) == 3+ [(key, tokens, cache, _)] = restored._prompt_cache_history.resume_entries()+ assert (key, tokens) == ("k|other", list(range(12)))+ assert mx.array_equal(cache[0].state[0], newest[0].state[0]).item()+ with pytest.raises(KeyError):+ restored._prompt_cache_history._lru._trie.get("k", list(range(8)))+ vlm = restored._vlm_snapshot_store.resume_entries()+ assert [(key, prefix, replay) for key, prefix, _, replay in vlm] == [+ ("k|img", list(range(6)), False),+ ("k|img", [9] * 5, True),+ ]++ assert backend(window = 64).restore_prompt_cache_for_resume(manifest) == 0+ (model_dir / "model.safetensors").write_bytes(b"rewritten weights")+ assert backend().restore_prompt_cache_for_resume(manifest) == 0++ # A turn over the LRU budget is held outside it, and still kept.+ over_budget = _MLXPromptCacheHistory(6, 1)+ over_budget.insert("k", list(range(8)), kv(8, 4))+ assert [tokens for _, tokens, _, _ in over_budget.resume_entries()] == [list(range(8))]+++def test_mlx_vlm_resume_keeps_the_boundary_an_edited_turn_extends():+ mx = pytest.importorskip("mlx.core")+ pytest.importorskip("mlx_lm")+ from mlx_lm.models.cache import KVCache++ from core.inference.mlx_inference import VLMPromptSnapshotStore++ def kv(n):+ entry = KVCache()+ rows = mx.zeros((1, 2, n, 4), dtype = mx.float16)+ entry.update_and_fetch(rows, rows)+ return [entry]++ store = VLMPromptSnapshotStore(1 << 30)+ store.store("k", list(range(12)), kv(12))+ store.store("k", list(range(14)), kv(14), replay = True)+ # The edit keeps the first 4 tokens; the old, longer boundary is then touched last.+ edited = [*range(4), 90, 91]+ store.store("k", list(range(4)), kv(4))+ store.store("k", edited, kv(6), replay = True)+ store._touch(("k", tuple(range(12))))+ assert [(prefix, replay) for _, prefix, _, replay in store.resume_entries()] == [+ (list(range(4)), False),+ (edited, True),+ ]+++def test_mlx_prompt_cache_resume_keys_on_the_weights_the_load_read(tmp_path, monkeypatch):+ mx = pytest.importorskip("mlx.core")+ pytest.importorskip("mlx_lm")+ from mlx_lm.models.cache import KVCache++ from core.inference.mlx_inference import MLXInferenceBackend, _MLXPromptCacheHistory++ model_dir = tmp_path / "model"+ model_dir.mkdir()+ weights = model_dir / "model.safetensors"+ weights.write_bytes(b"weights")++ def loaded():+ backend = MLXInferenceBackend()+ backend.active_model_name = str(model_dir)+ backend._resume_files = backend._resume_weight_files() # what load_model records+ backend._prompt_cache_history = _MLXPromptCacheHistory(6, 1 << 30)+ entry = KVCache()+ rows = mx.zeros((1, 2, 8, 4), dtype = mx.float16)+ entry.update_and_fetch(rows, rows)+ backend._prompt_cache_history.insert("k", list(range(8)), [entry])+ return backend++ def save(backend):+ out = tmp_path / f"save-{len(list(tmp_path.glob('save-*')))}"+ out.mkdir()+ return backend.save_prompt_cache_for_resume(str(out))++ old = loaded()+ manifest = save(old)+ assert manifest is not None+ # Replaced while the old weights are resident: their KV must neither be saved nor restored.+ weights.write_bytes(b"replaced weights")+ assert save(old) is None+ assert loaded().restore_prompt_cache_for_resume(manifest) == 0++ # The same bounds as llama-server's slot save.+ monkeypatch.setenv("UNSLOTH_SLOT_SAVE_MAX_BYTES", "1")+ assert save(loaded()) is None+++def test_mlx_resume_cache_loads_only_mlx_cache_classes(tmp_path):+ import collections+ import pickle++ mx = pytest.importorskip("mlx.core")+ from core.inference.mlx_inference import load_resume_cache, save_resume_cache++ from mlx_lm.models.cache import make_prompt_cache++ path = str(tmp_path / "cache.safetensors")+ # Outside the MLX modules, and inside them but not a class.+ for refused in (collections.OrderedDict(), make_prompt_cache):+ save_resume_cache(path, "k", [1], [mx.zeros(2), refused])+ with pytest.raises(pickle.UnpicklingError):+ load_resume_cache(path)++def _audio_model(module_paths = ("audio_tower",), model_type = "gemma3n"):"""Model stub shaped like a loaded mlx-vlm tree (families nest differently)."""return SimpleNamespace(diff --git a/studio/backend/tests/test_mlx_worker_audio_commands.py b/studio/backend/tests/test_mlx_worker_audio_commands.pyindex d04a43b22fb..6fe707522ff 100644--- a/studio/backend/tests/test_mlx_worker_audio_commands.py+++ b/studio/backend/tests/test_mlx_worker_audio_commands.py@@ -3,7 +3,9 @@"""A dropped audio command costs the caller its full 120s deadline, so the MLX loop must always reply."""+import osimport queue as _queue+from pathlib import Pathfrom types import SimpleNamespaceimport pytest@@ -116,6 +118,75 @@ def test_whisper_on_a_backend_without_asr_explains_itself(monkeypatch):assert "attribute" not in errors[0]["error"].lower()+def test_mlx_loop_answers_every_prompt_cache_resume_command(monkeypatch):+ backend = SimpleNamespace(+ save_prompt_cache_for_resume = lambda directory: {"dir": directory},+ restore_prompt_cache_for_resume = lambda manifest: (_ for _ in ()).throw(+ RuntimeError("unreadable")+ ),+ )+ sent = _run_mlx_loop(+ monkeypatch,+ [+ {"type": "save_prompt_cache", "request_id": "s", "directory": "/d"},+ {"type": "restore_prompt_cache", "request_id": "r", "manifest": {}},+ ],+ backend = backend,+ )+ replies = {m["request_id"]: m for m in sent if m.get("request_id") in ("s", "r")}+ assert replies["s"]["type"] == "save_prompt_cache_response"+ assert replies["s"]["result"] == {"dir": "/d"}+ assert replies["r"]["type"] == "restore_prompt_cache_response"+ assert replies["r"]["error"] == "unreadable"++ resp = _RespQueue()+ worker._handle_prompt_cache_resume(+ SimpleNamespace(), {"type": "save_prompt_cache", "request_id": "u"}, resp+ )+ assert resp.sent[0]["request_id"] == "u" and "no prompt cache" in resp.sent[0]["error"]++[email protected]("case", ["saved", "error", "abandoned", "dispatched"])+def test_a_prompt_cache_save_keeps_its_directory_only_with_a_manifest(monkeypatch, tmp_path, case):+ import threading++ from core.inference import orchestrator as orch_mod+ from utils.paths import storage_roots++ root = tmp_path / "mlx-prompt-cache"+ monkeypatch.setattr(storage_roots, "mlx_prompt_cache_root", lambda: root)+ orch = orch_mod.InferenceOrchestrator.__new__(orch_mod.InferenceOrchestrator)+ orch.active_model_name, orch.models = "m", {"m": {"is_mlx": True}}+ orch._managed_engine, orch._unload_pending = None, False+ orch._gen_lock, orch._mailbox_lock = threading.Lock(), threading.Lock()+ orch._mailboxes = {"generation": object()} if case == "dispatched" else {}+ sent = []++ def reply(+ cmd,+ timeout,+ cancelled = None,+ ):+ sent.append(cmd)+ if case == "saved":+ return {"result": {"engine": "mlx", "dir": cmd["directory"]}}+ return {"error": "disk full"} if case == "error" else None++ monkeypatch.setattr(orch, "_prompt_cache_resume_reply", reply)+ manifest = orch.save_prompt_cache_for_resume()+ assert not orch._gen_lock.locked() and len(sent) == (case != "dispatched")+ if case == "saved":+ assert [str(path) for path in root.iterdir()] == [manifest["dir"]]+ if os.name == "posix":+ # Saved KV encodes chat content, so neither level is readable by other local users.+ assert {+ (root.stat().st_mode & 0o777),+ (Path(manifest["dir"]).stat().st_mode & 0o777),+ } == {0o700}+ else:+ assert manifest is None and list(root.iterdir() if root.exists() else ()) == []++@pytest.mark.parametrize("cmd_type", ["generate_audio", "generate_video"])def test_every_mlx_command_gets_exactly_one_reply(monkeypatch, cmd_type):"""One reply, not zero and not a duplicate that would confuse the mailbox."""diff --git a/studio/backend/tests/test_openai_auto_switch.py b/studio/backend/tests/test_openai_auto_switch.pyindex 4aca9950c00..c6b1cd6d4a9 100644--- a/studio/backend/tests/test_openai_auto_switch.py+++ b/studio/backend/tests/test_openai_auto_switch.py@@ -6251,10 +6251,11 @@ def test_unload_route_clears_reload_stash(monkeypatch):def test_non_gguf_load_clears_reload_stash():- # A non-GGUF load clears the stash too, so it never lingers when idle-unload is off.+ # A non-GGUF load clears the stash too, so it never lingers when idle-unload is off; it is+ # handed the orchestrator, which restores an MLX chat context kept across a media swap.src = inspect.getsource(inference_route._load_model_impl)- assert src.count("note_model_loaded()") >= 1+ assert "to_thread(note_model_loaded, backend)" in srcassert "to_thread(note_model_loaded, llama_backend)" in src@@ -6963,14 +6964,20 @@ def test_new_idle_save_purges_previous_manifest_files(tmp_path):def test_sweep_slot_save_dir_removes_only_resume_files(monkeypatch, tmp_path):from utils.paths import storage_roots+ mlx_root = tmp_path / "mlx"monkeypatch.setattr(storage_roots, "llama_slot_cache_root", lambda: tmp_path)+ monkeypatch.setattr(storage_roots, "mlx_prompt_cache_root", lambda: mlx_root)stale = tmp_path / "resume-old-slot0.bin"stale.write_bytes(b"kv")other = tmp_path / "unrelated.txt"other.write_text("keep")+ (mlx_root / "resume-old").mkdir(parents = True)+ (mlx_root / "resume-old" / "text-0.safetensors").write_bytes(b"kv")+ (mlx_root / "unrelated").mkdir()kw.sweep_slot_save_dir()assert not stale.exists()assert other.exists()+ assert not (mlx_root / "resume-old").exists() and (mlx_root / "unrelated").exists()def test_keep_kv_setting_roundtrip_and_default(monkeypatch):diff --git a/studio/backend/tests/test_swap_kv_resume.py b/studio/backend/tests/test_swap_kv_resume.pyindex bc04d25081f..82d00ba1da9 100644--- a/studio/backend/tests/test_swap_kv_resume.py+++ b/studio/backend/tests/test_swap_kv_resume.py@@ -317,6 +317,68 @@ def test_the_real_evict_chat_promotes_the_identity_read_before_unload(env, monkeassert kw._kv_resume["identity"] == IDENTITY and env.files[0].exists()+class _FakeMlxOrchestrator:+ loading_models = ()+ _managed_engine = None+ _openai_advertised_id = None++ def __init__(self, tmp_path):+ self.root, self.active_model_name, self.restores = tmp_path, "unsloth/A-MLX", []++ def _serves_mlx(self):+ return self.active_model_name is not None++ def save_prompt_cache_for_resume(self, should_abort):+ directory = self.root / "resume-mlx"+ directory.mkdir()+ (directory / "text-0.safetensors").write_bytes(b"kv")+ slot = {"filename": "text-0.safetensors", "store": "text"}+ return {"engine": "mlx", "dir": str(directory), "fingerprint": (), "slots": [slot]}++ def restore_prompt_cache_for_resume(self, manifest):+ self.restores.append(manifest)+ return 1++ def unload_model(self, name):+ self.active_model_name = None++ def _shutdown_subprocess(self, timeout):+ return True++[email protected]("reload", ["mlx", "gguf"])+def test_an_mlx_chat_keeps_its_prompt_cache_across_the_swap(env, monkeypatch, tmp_path, reload):+ import core.inference as inference_pkg+ import core.inference.llama_cpp as llama_cpp_mod+ import core.inference.model_slots as model_slots+ import core.inference.orchestrator as orchestrator_mod+ from hub.services.models import account_access++ orch = _FakeMlxOrchestrator(tmp_path)+ env.unload_model()+ env._wait_for_vram_settle = lambda **k: None+ monkeypatch.setattr(model_slots, "unload_extra_models", lambda **k: None)+ monkeypatch.setattr(inference_pkg, "get_inference_backend", lambda: orch)+ monkeypatch.setattr(orchestrator_mod, "peek_inference_backend", lambda: orch)+ monkeypatch.setattr(llama_cpp_mod, "chat_load_active", lambda: False)+ monkeypatch.setattr(account_access, "clear_resident", lambda owner: None)+ monkeypatch.setitem(arb._EVICTORS, arb.CHAT, arb._evict_chat)+ _load()+ stash = kw._kv_resume+ assert stash["identity"] == ("unsloth/A-MLX", None, "unsloth/A-MLX", "mlx")+ assert orch.active_model_name is None and Path(stash["dir"]).is_dir()+ assert [(r["event"], r["reason"]) for r in env.rows] == [("unload", "swap")]++ if reload == "mlx":+ orch.active_model_name = "unsloth/A-MLX"+ kw.note_model_loaded(orch)+ else:+ env.reload()+ kw.note_model_loaded(env)+ assert orch.restores == ([stash] if reload == "mlx" else []) and env.restores == []+ assert kw._kv_resume is None and not Path(stash["dir"]).exists()++@pytest.mark.parametrize("case",["generating", "keep_kv_off", "chat_not_resident", "not_gguf", "other_account", "chat_target"],diff --git a/studio/backend/utils/paths/storage_roots.py b/studio/backend/utils/paths/storage_roots.pyindex f14781e6402..958d792e8d3 100644--- a/studio/backend/utils/paths/storage_roots.py+++ b/studio/backend/utils/paths/storage_roots.py@@ -234,6 +234,11 @@ def llama_slot_cache_root() -> Path:return cache_root() / "llama-slots"+def mlx_prompt_cache_root() -> Path:+ """Dir the MLX worker saves its prompt cache in, for the next load of the same model."""+ return cache_root() / "mlx-prompt-cache"++def studio_bin_root() -> Path:"""Dir for Unsloth-managed executables (the `unsloth` shim, downloaded tools like cloudflared)."""return studio_root() / "bin"