Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
Studio: keep chat context when switching models (#16493)
Nilay499ee47fbe9a
diff --git a/studio/backend/core/inference/chat_swap_kv.py b/studio/backend/core/inference/chat_swap_kv.pyindex 4bfbc1181fc..e0be23004a9 100644--- a/studio/backend/core/inference/chat_swap_kv.py+++ b/studio/backend/core/inference/chat_swap_kv.py@@ -305,6 +305,20 @@ def _chat_account() -> Optional[str]:return gpu_arbiter.owner_account()+def _unload_account() -> Optional[str]:+ account = _chat_account()+ if account is not None:+ return account+ # A zero-VRAM primary drops its CHAT claim; fall back to the account that loaded it.+ from core.inference.llama_keepwarm import loaded_account+ from routes.inference import get_llama_cpp_backend++ llama = get_llama_cpp_backend()+ if not getattr(llama, "holds_no_vram", False):+ return None+ return loaded_account(getattr(llama, "_gguf_path", None))++def mlx_chat_identity(orchestrator):serves_mlx = getattr(orchestrator, "_serves_mlx", None)if not callable(serves_mlx) or not serves_mlx():@@ -355,6 +369,46 @@ def _save_in_thread(save, should_abort, abort: threading.Event, loop, future) ->_delete(manifest)+async def save_chat_kv_before_unload() -> None:+ from core.inference.llama_keepwarm import stash_unload_kv++ abort = threading.Event()+ loop = asyncio.get_running_loop()+ future = loop.create_future()+ deadline = time.monotonic() + SWAP_KV_SAVE_BUDGET_S+ try:+ saver = _chat_saver()+ if saver is None or saver[0][-1] == "mlx" or not _keep_kv():+ return+ if _unload_account() != _account_id() or _chat_busy():+ return+ threading.Thread(+ target = _save_in_thread,+ args = (+ saver[1],+ lambda: abort.is_set() or time.monotonic() >= deadline or _chat_busy(),+ abort,+ loop,+ future,+ ),+ name = "chat-kv-unload-save",+ daemon = True,+ ).start()+ manifest = await asyncio.wait_for(asyncio.shield(future), SWAP_KV_SAVE_BUDGET_S)+ if manifest and _keep_kv():+ stash_unload_kv(manifest)+ else:+ _delete(manifest)+ except BaseException as exc: # noqa: BLE001+ abort.set()+ if future.done() and not future.cancelled() and future.exception() is None:+ _delete(future.result())+ future.cancel()+ if isinstance(exc, asyncio.CancelledError):+ raise+ logger.debug("chat KV save before unload failed: %s", exc)++async def save_chat_kv_before_swap(owner: str,media: str,diff --git a/studio/backend/core/inference/llama_keepwarm.py b/studio/backend/core/inference/llama_keepwarm.pyindex 33ffe1be3d9..633411cf687 100644--- a/studio/backend/core/inference/llama_keepwarm.py+++ b/studio/backend/core/inference/llama_keepwarm.py@@ -51,6 +51,9 @@ _last_unloaded_model = None_last_unloaded_mlx = False# Slot KV manifest saved by the idle unload; whoever pops it owns deleting its files._kv_resume = None+# Slot saves of manually unloaded GGUFs, oldest first, until that GGUF loads again.+_unload_kv: list = []+_loaded_by = None# Process-wide, not per-loop: the backend slot is shared across every event loop._lifecycle_lock = threading.Lock()@@ -517,9 +520,19 @@ def model_load_gate():def note_model_loaded(backend = None) -> None:"""Stamp activity and synchronously drop any reload stash."""+ global _loaded_by_note_activity()+ gguf = getattr(backend, "_gguf_path", None)+ if gguf:+ from utils.account_context import current_account_id+ _loaded_by = (str(gguf), current_account_id())resume = take_kv_resume()_set_last_unloaded(None)+ kept = _take_unload_kv(gguf)+ if kept is not None and (resume is None or resume.get("gguf") != kept.get("gguf")):+ resume, kept = kept, resume+ if kept:+ _delete_resume_files(kept)if resume is None:returnif backend is not None:@@ -570,15 +583,22 @@ def _delete_resume_files(manifest) -> None:pass+def loaded_account(gguf) -> Optional[str]:+ loaded = _loaded_by+ return loaded[1] if gguf and loaded and loaded[0] == str(gguf) else None++def _set_kv_resume(value) -> None:global _kv_resume- stale = None+ stale = []with _lock:if _kv_resume is not None and _kv_resume is not value:- stale = _kv_resume+ stale.append(_kv_resume)_kv_resume = value- if stale:- _delete_resume_files(stale)+ if value is not None:+ _trim_unload_kv(stale, keep_newest = False)+ for old in stale:+ _delete_resume_files(old)def take_kv_resume():@@ -588,6 +608,50 @@ def take_kv_resume():return manifest+def _resume_bytes(manifest) -> int:+ total = 0+ for entry in manifest.get("slots") or []:+ with contextlib.suppress(OSError):+ total += (Path(manifest.get("dir") or "") / str(entry.get("filename"))).stat().st_size+ return total+++def _trim_unload_kv(stale: list, *, keep_newest: bool) -> None:+ """Caller holds _lock. Evicts the oldest unload saves until every kept save fits the one cap."""+ global _kv_resume+ from core.inference.llama_cpp import _SLOT_SAVE_MAX_BYTES++ def _kept_bytes() -> int:+ resume = _resume_bytes(_kv_resume) if _kv_resume else 0+ return resume + sum(map(_resume_bytes, _unload_kv))++ while len(_unload_kv) > (1 if keep_newest else 0) and _kept_bytes() > _SLOT_SAVE_MAX_BYTES:+ stale.append(_unload_kv.pop(0))+ if keep_newest and _kv_resume is not None and _kept_bytes() > _SLOT_SAVE_MAX_BYTES:+ stale.append(_kv_resume)+ _kv_resume = None+++def stash_unload_kv(manifest) -> None:+ with _lock:+ stale = [m for m in _unload_kv if m.get("gguf") == manifest.get("gguf")]+ _unload_kv[:] = [m for m in _unload_kv if m.get("gguf") != manifest.get("gguf")]+ _unload_kv.append(manifest)+ _trim_unload_kv(stale, keep_newest = True)+ for old in stale:+ _delete_resume_files(old)+++def _take_unload_kv(gguf):+ if not gguf:+ return None+ with _lock:+ for index, manifest in enumerate(_unload_kv):+ if manifest.get("gguf") == str(gguf):+ return _unload_kv.pop(index)+ return None++def purge_kv_resume(*, origin: Optional[str] = None) -> None:"""Drop the stash; with ``origin``, only one of that origin (an untagged stash is "idle")."""global _kv_resume@@ -597,8 +661,13 @@ def purge_kv_resume(*, origin: Optional[str] = None) -> None:_kv_resume = Noneelse:resume = None+ unloaded = [] if origin is not None else _unload_kv[:]+ if origin is None:+ _unload_kv.clear()if resume:_delete_resume_files(resume)+ for manifest in unloaded:+ _delete_resume_files(manifest)def restore_kv_resume(backend, manifest) -> None:diff --git a/studio/backend/routes/inference.py b/studio/backend/routes/inference.pyindex 9add1cdfa25..a1a2792f5df 100644--- a/studio/backend/routes/inference.py+++ b/studio/backend/routes/inference.py@@ -21586,6 +21586,9 @@ async def _unload_model_impl(request: UnloadRequest, current_subject: str):await _drain_and_recancel_before_teardown(force = request.force_cancel_active, action = "Unloading the model")+ from core.inference.chat_swap_kv import save_chat_kv_before_unload++ await save_chat_kv_before_unload()# Off the loop: a long teardown would block the event loop and this route's own padding.await asyncio.to_thread(llama_backend.unload_model)note_model_unloaded()diff --git a/studio/backend/tests/test_swap_kv_resume.py b/studio/backend/tests/test_swap_kv_resume.pyindex 82d00ba1da9..a9ede0d743e 100644--- a/studio/backend/tests/test_swap_kv_resume.py+++ b/studio/backend/tests/test_swap_kv_resume.py@@ -14,6 +14,7 @@ import sysimport threadingimport timefrom pathlib import Path+from types import SimpleNamespaceimport pytest@@ -21,6 +22,7 @@ import core.inference.chat_swap_kv as csimport core.inference.gpu_arbiter as arbimport routes.inference as inference_routesfrom core.inference import llama_keepwarm as kw+from models.inference import UnloadRequestfrom utils import openai_auto_switch_settings as settingsfrom utils.account_context import current_account_id@@ -28,6 +30,7 @@ _TESTS_DIR = str(Path(__file__).resolve().parent)if _TESTS_DIR not in sys.path:sys.path.insert(0, _TESTS_DIR)+from test_active_generations import _route_gate, _stub_unload_backends # noqa: E402from test_diffusion_routes import client # noqa: E402,F401@@ -103,6 +106,7 @@ def env(monkeypatch, tmp_path):monkeypatch.setitem(arb._EVICTORS, owner, lambda: None)monkeypatch.setitem(arb._EVICTORS, arb.CHAT, _evict_chat)monkeypatch.setattr(kw, "_kv_resume", None)+ monkeypatch.setattr(kw, "_loaded_by", None, raising = False)monkeypatch.setattr(cs, "_pending", None)monkeypatch.setattr(cs, "_committed", None)monkeypatch.setattr(cs, "_saving", {})@@ -916,3 +920,157 @@ def test_a_token_unload_names_the_engine_the_preflight_settled_on(client, env, mresp = client.post(f"{_LOADS['image'][0]}/unload", json = {"expected_load_id": "t1"})assert resp.status_code == 200, resp.textassert seen == ["new"]+++def _unload(monkeypatch, llama):+ _route_gate()+ _stub_unload_backends(+ monkeypatch,+ llama = llama,+ backend = SimpleNamespace(+ get_loading_model = lambda: None,+ active_model_name = None,+ models = {},+ unload_model = lambda path: None,+ ),+ )+ request = UnloadRequest(model_path = llama.model_identifier)+ asyncio.run(inference_routes.unload_model(request, "tester"))+ assert not llama.is_loaded+++def test_switching_away_and_back_restores_the_first_models_slots(env, monkeypatch, tmp_path):+ monkeypatch.setattr(kw, "_unload_kv", [], raising = False)+ a_gguf = env._gguf_path+ b_gguf = tmp_path / "other.gguf"+ b_gguf.write_bytes(b"gguf")++ _unload(monkeypatch, env)+ env.reload(b_gguf)+ kw.note_model_loaded(env)+ assert env.restores == []++ _unload(monkeypatch, env)+ env.reload(a_gguf)+ kw.note_model_loaded(env)++ assert [restored["gguf"] for restored in env.restores] == [a_gguf]+++def test_an_idle_save_of_another_model_does_not_cost_the_loading_models_slots(+ env, monkeypatch, tmp_path+):+ monkeypatch.setattr(kw, "_unload_kv", [], raising = False)+ a_gguf = env._gguf_path+ b_gguf = tmp_path / "other.gguf"+ b_gguf.write_bytes(b"gguf")++ _unload(monkeypatch, env)+ env.reload(b_gguf)+ kw.note_model_loaded(env)+ other = dict(env.save_slots_for_resume(lambda: False), gguf = b_gguf)+ kw._kv_resume = other+ env.reload(a_gguf)+ kw.note_model_loaded(env)++ assert [restored["gguf"] for restored in env.restores] == [a_gguf]+++ "guard, value",+ [("_chat_busy", lambda counted = False: True), ("_chat_account", lambda: "someone-else")],+)+def test_a_manual_unload_leaves_a_busy_or_foreign_chat_unsaved(env, monkeypatch, guard, value):+ monkeypatch.setattr(kw, "_unload_kv", [], raising = False)+ monkeypatch.setattr(cs, guard, value)++ _unload(monkeypatch, env)++ assert env.saves == []+++def test_a_manual_unload_saves_nothing_with_keep_context_off(env, monkeypatch):+ monkeypatch.setattr(kw, "_unload_kv", [], raising = False)+ monkeypatch.setattr(settings, "get_auto_unload_keep_kv", lambda: False)++ _unload(monkeypatch, env)++ assert env.saves == []+++def test_turning_keep_context_off_deletes_saved_unloads(env, monkeypatch):+ monkeypatch.setattr(kw, "_unload_kv", [], raising = False)++ _unload(monkeypatch, env)+ assert env.files and not _no_files(env)++ kw.purge_kv_resume()++ assert _no_files(env)+++def test_a_save_that_lands_as_the_unload_times_out_is_deleted(env, monkeypatch):+ monkeypatch.setattr(kw, "_unload_kv", [], raising = False)+ real_wait_for = asyncio.wait_for++ async def _times_out_after_delivery(aw, timeout):+ if timeout != cs.SWAP_KV_SAVE_BUDGET_S:+ return await real_wait_for(aw, timeout)+ await aw+ raise asyncio.TimeoutError++ monkeypatch.setattr(asyncio, "wait_for", _times_out_after_delivery)++ _unload(monkeypatch, env)++ assert len(env.saves) == 1+ assert kw._unload_kv == []+ assert _no_files(env)+++def three_byte_cap(monkeypatch):+ import core.inference.llama_cpp as llama_cpp+ monkeypatch.setattr(llama_cpp, "_SLOT_SAVE_MAX_BYTES", 3)+ monkeypatch.setattr(kw, "_unload_kv", [], raising = False)+++def test_an_idle_or_swap_save_evicts_older_unload_saves_past_the_cap(env, three_byte_cap):+ first = env.save_slots_for_resume(lambda: False)+ kw.stash_unload_kv(first)+ second = env.save_slots_for_resume(lambda: False)++ kw._set_kv_resume({"origin": "idle", **second})++ assert kw._unload_kv == []+ assert not env.files[0].exists()+ assert env.files[1].exists()+++def test_an_unload_save_replaces_an_idle_save_past_the_cap(env, three_byte_cap):+ idle = env.save_slots_for_resume(lambda: False)+ kw._set_kv_resume({"origin": "idle", **idle})+ unloaded = env.save_slots_for_resume(lambda: False)++ kw.stash_unload_kv(unloaded)++ assert kw._kv_resume is None+ assert kw._unload_kv == [unloaded]+ assert not env.files[0].exists()+ assert env.files[1].exists()++[email protected]("loader, saved", [(None, 1), ("someone-else", 0)])+def test_a_cpu_only_primary_is_saved_for_the_account_that_loaded_it(+ env, monkeypatch, loader, saved+):+ monkeypatch.setattr(kw, "_unload_kv", [], raising = False)+ monkeypatch.setattr(cs, "_chat_account", lambda: None)+ env.holds_no_vram = True+ kw.note_model_loaded(env)+ if loader is not None:+ monkeypatch.setattr(kw, "_loaded_by", (env._gguf_path, loader))++ _unload(monkeypatch, env)++ assert len(env.saves) == saveddiff --git a/studio/frontend/src/i18n/locales/en.ts b/studio/frontend/src/i18n/locales/en.tsindex 98203d245c4..d54f9382e4e 100644--- a/studio/frontend/src/i18n/locales/en.ts+++ b/studio/frontend/src/i18n/locales/en.ts@@ -1965,7 +1965,7 @@ export const en = {idleUnsaved: "Not saved yet.",keepKv: "Keep chat context across unloads",keepKvDescription:- "Save the KV cache before an idle unload, or before an image or video model takes the GPU, so resumed chats skip re-reading history. Up to 10 GB on disk.",+ "Save the KV cache before a model is unloaded or switched, or before an image or video model takes the GPU, so resumed chats skip re-reading history. Up to 10 GB on disk.",keepKvKeywords: "kv cache context swap resume image video",apiOnly: "Only unload models loaded by the API",apiOnlyDescription: