Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

Studio: keep chat context when switching models (#16493)

Nilay499ee47fbe9a
diff --git a/studio/backend/core/inference/chat_swap_kv.py b/studio/backend/core/inference/chat_swap_kv.py
index 4bfbc1181fc..e0be23004a9 100644
--- a/studio/backend/core/inference/chat_swap_kv.py
+++ b/studio/backend/core/inference/chat_swap_kv.py
@@ -305,6 +305,20 @@ def _chat_account() -> Optional[str]:
return gpu_arbiter.owner_account()
+def _unload_account() -> Optional[str]:
+ account = _chat_account()
+ if account is not None:
+ return account
+ # A zero-VRAM primary drops its CHAT claim; fall back to the account that loaded it.
+ from core.inference.llama_keepwarm import loaded_account
+ from routes.inference import get_llama_cpp_backend
+
+ llama = get_llama_cpp_backend()
+ if not getattr(llama, "holds_no_vram", False):
+ return None
+ return loaded_account(getattr(llama, "_gguf_path", None))
+
+
def mlx_chat_identity(orchestrator):
serves_mlx = getattr(orchestrator, "_serves_mlx", None)
if not callable(serves_mlx) or not serves_mlx():
@@ -355,6 +369,46 @@ def _save_in_thread(save, should_abort, abort: threading.Event, loop, future) ->
_delete(manifest)
+async def save_chat_kv_before_unload() -> None:
+ from core.inference.llama_keepwarm import stash_unload_kv
+
+ abort = threading.Event()
+ loop = asyncio.get_running_loop()
+ future = loop.create_future()
+ deadline = time.monotonic() + SWAP_KV_SAVE_BUDGET_S
+ try:
+ saver = _chat_saver()
+ if saver is None or saver[0][-1] == "mlx" or not _keep_kv():
+ return
+ if _unload_account() != _account_id() or _chat_busy():
+ return
+ threading.Thread(
+ target = _save_in_thread,
+ args = (
+ saver[1],
+ lambda: abort.is_set() or time.monotonic() >= deadline or _chat_busy(),
+ abort,
+ loop,
+ future,
+ ),
+ name = "chat-kv-unload-save",
+ daemon = True,
+ ).start()
+ manifest = await asyncio.wait_for(asyncio.shield(future), SWAP_KV_SAVE_BUDGET_S)
+ if manifest and _keep_kv():
+ stash_unload_kv(manifest)
+ else:
+ _delete(manifest)
+ except BaseException as exc: # noqa: BLE001
+ abort.set()
+ if future.done() and not future.cancelled() and future.exception() is None:
+ _delete(future.result())
+ future.cancel()
+ if isinstance(exc, asyncio.CancelledError):
+ raise
+ logger.debug("chat KV save before unload failed: %s", exc)
+
+
async def save_chat_kv_before_swap(
owner: str,
media: str,
diff --git a/studio/backend/core/inference/llama_keepwarm.py b/studio/backend/core/inference/llama_keepwarm.py
index 33ffe1be3d9..633411cf687 100644
--- a/studio/backend/core/inference/llama_keepwarm.py
+++ b/studio/backend/core/inference/llama_keepwarm.py
@@ -51,6 +51,9 @@ _last_unloaded_model = None
_last_unloaded_mlx = False
# Slot KV manifest saved by the idle unload; whoever pops it owns deleting its files.
_kv_resume = None
+# Slot saves of manually unloaded GGUFs, oldest first, until that GGUF loads again.
+_unload_kv: list = []
+_loaded_by = None
# Process-wide, not per-loop: the backend slot is shared across every event loop.
_lifecycle_lock = threading.Lock()
@@ -517,9 +520,19 @@ def model_load_gate():
def note_model_loaded(backend = None) -> None:
"""Stamp activity and synchronously drop any reload stash."""
+ global _loaded_by
_note_activity()
+ gguf = getattr(backend, "_gguf_path", None)
+ if gguf:
+ from utils.account_context import current_account_id
+ _loaded_by = (str(gguf), current_account_id())
resume = take_kv_resume()
_set_last_unloaded(None)
+ kept = _take_unload_kv(gguf)
+ if kept is not None and (resume is None or resume.get("gguf") != kept.get("gguf")):
+ resume, kept = kept, resume
+ if kept:
+ _delete_resume_files(kept)
if resume is None:
return
if backend is not None:
@@ -570,15 +583,22 @@ def _delete_resume_files(manifest) -> None:
pass
+def loaded_account(gguf) -> Optional[str]:
+ loaded = _loaded_by
+ return loaded[1] if gguf and loaded and loaded[0] == str(gguf) else None
+
+
def _set_kv_resume(value) -> None:
global _kv_resume
- stale = None
+ stale = []
with _lock:
if _kv_resume is not None and _kv_resume is not value:
- stale = _kv_resume
+ stale.append(_kv_resume)
_kv_resume = value
- if stale:
- _delete_resume_files(stale)
+ if value is not None:
+ _trim_unload_kv(stale, keep_newest = False)
+ for old in stale:
+ _delete_resume_files(old)
def take_kv_resume():
@@ -588,6 +608,50 @@ def take_kv_resume():
return manifest
+def _resume_bytes(manifest) -> int:
+ total = 0
+ for entry in manifest.get("slots") or []:
+ with contextlib.suppress(OSError):
+ total += (Path(manifest.get("dir") or "") / str(entry.get("filename"))).stat().st_size
+ return total
+
+
+def _trim_unload_kv(stale: list, *, keep_newest: bool) -> None:
+ """Caller holds _lock. Evicts the oldest unload saves until every kept save fits the one cap."""
+ global _kv_resume
+ from core.inference.llama_cpp import _SLOT_SAVE_MAX_BYTES
+
+ def _kept_bytes() -> int:
+ resume = _resume_bytes(_kv_resume) if _kv_resume else 0
+ return resume + sum(map(_resume_bytes, _unload_kv))
+
+ while len(_unload_kv) > (1 if keep_newest else 0) and _kept_bytes() > _SLOT_SAVE_MAX_BYTES:
+ stale.append(_unload_kv.pop(0))
+ if keep_newest and _kv_resume is not None and _kept_bytes() > _SLOT_SAVE_MAX_BYTES:
+ stale.append(_kv_resume)
+ _kv_resume = None
+
+
+def stash_unload_kv(manifest) -> None:
+ with _lock:
+ stale = [m for m in _unload_kv if m.get("gguf") == manifest.get("gguf")]
+ _unload_kv[:] = [m for m in _unload_kv if m.get("gguf") != manifest.get("gguf")]
+ _unload_kv.append(manifest)
+ _trim_unload_kv(stale, keep_newest = True)
+ for old in stale:
+ _delete_resume_files(old)
+
+
+def _take_unload_kv(gguf):
+ if not gguf:
+ return None
+ with _lock:
+ for index, manifest in enumerate(_unload_kv):
+ if manifest.get("gguf") == str(gguf):
+ return _unload_kv.pop(index)
+ return None
+
+
def purge_kv_resume(*, origin: Optional[str] = None) -> None:
"""Drop the stash; with ``origin``, only one of that origin (an untagged stash is "idle")."""
global _kv_resume
@@ -597,8 +661,13 @@ def purge_kv_resume(*, origin: Optional[str] = None) -> None:
_kv_resume = None
else:
resume = None
+ unloaded = [] if origin is not None else _unload_kv[:]
+ if origin is None:
+ _unload_kv.clear()
if resume:
_delete_resume_files(resume)
+ for manifest in unloaded:
+ _delete_resume_files(manifest)
def restore_kv_resume(backend, manifest) -> None:
diff --git a/studio/backend/routes/inference.py b/studio/backend/routes/inference.py
index 9add1cdfa25..a1a2792f5df 100644
--- a/studio/backend/routes/inference.py
+++ b/studio/backend/routes/inference.py
@@ -21586,6 +21586,9 @@ async def _unload_model_impl(request: UnloadRequest, current_subject: str):
await _drain_and_recancel_before_teardown(
force = request.force_cancel_active, action = "Unloading the model"
)
+ from core.inference.chat_swap_kv import save_chat_kv_before_unload
+
+ await save_chat_kv_before_unload()
# Off the loop: a long teardown would block the event loop and this route's own padding.
await asyncio.to_thread(llama_backend.unload_model)
note_model_unloaded()
diff --git a/studio/backend/tests/test_swap_kv_resume.py b/studio/backend/tests/test_swap_kv_resume.py
index 82d00ba1da9..a9ede0d743e 100644
--- a/studio/backend/tests/test_swap_kv_resume.py
+++ b/studio/backend/tests/test_swap_kv_resume.py
@@ -14,6 +14,7 @@ import sys
import threading
import time
from pathlib import Path
+from types import SimpleNamespace
import pytest
@@ -21,6 +22,7 @@ import core.inference.chat_swap_kv as cs
import core.inference.gpu_arbiter as arb
import routes.inference as inference_routes
from core.inference import llama_keepwarm as kw
+from models.inference import UnloadRequest
from utils import openai_auto_switch_settings as settings
from utils.account_context import current_account_id
@@ -28,6 +30,7 @@ _TESTS_DIR = str(Path(__file__).resolve().parent)
if _TESTS_DIR not in sys.path:
sys.path.insert(0, _TESTS_DIR)
+from test_active_generations import _route_gate, _stub_unload_backends # noqa: E402
from test_diffusion_routes import client # noqa: E402,F401
@@ -103,6 +106,7 @@ def env(monkeypatch, tmp_path):
monkeypatch.setitem(arb._EVICTORS, owner, lambda: None)
monkeypatch.setitem(arb._EVICTORS, arb.CHAT, _evict_chat)
monkeypatch.setattr(kw, "_kv_resume", None)
+ monkeypatch.setattr(kw, "_loaded_by", None, raising = False)
monkeypatch.setattr(cs, "_pending", None)
monkeypatch.setattr(cs, "_committed", None)
monkeypatch.setattr(cs, "_saving", {})
@@ -916,3 +920,157 @@ def test_a_token_unload_names_the_engine_the_preflight_settled_on(client, env, m
resp = client.post(f"{_LOADS['image'][0]}/unload", json = {"expected_load_id": "t1"})
assert resp.status_code == 200, resp.text
assert seen == ["new"]
+
+
+def _unload(monkeypatch, llama):
+ _route_gate()
+ _stub_unload_backends(
+ monkeypatch,
+ llama = llama,
+ backend = SimpleNamespace(
+ get_loading_model = lambda: None,
+ active_model_name = None,
+ models = {},
+ unload_model = lambda path: None,
+ ),
+ )
+ request = UnloadRequest(model_path = llama.model_identifier)
+ asyncio.run(inference_routes.unload_model(request, "tester"))
+ assert not llama.is_loaded
+
+
+def test_switching_away_and_back_restores_the_first_models_slots(env, monkeypatch, tmp_path):
+ monkeypatch.setattr(kw, "_unload_kv", [], raising = False)
+ a_gguf = env._gguf_path
+ b_gguf = tmp_path / "other.gguf"
+ b_gguf.write_bytes(b"gguf")
+
+ _unload(monkeypatch, env)
+ env.reload(b_gguf)
+ kw.note_model_loaded(env)
+ assert env.restores == []
+
+ _unload(monkeypatch, env)
+ env.reload(a_gguf)
+ kw.note_model_loaded(env)
+
+ assert [restored["gguf"] for restored in env.restores] == [a_gguf]
+
+
+def test_an_idle_save_of_another_model_does_not_cost_the_loading_models_slots(
+ env, monkeypatch, tmp_path
+):
+ monkeypatch.setattr(kw, "_unload_kv", [], raising = False)
+ a_gguf = env._gguf_path
+ b_gguf = tmp_path / "other.gguf"
+ b_gguf.write_bytes(b"gguf")
+
+ _unload(monkeypatch, env)
+ env.reload(b_gguf)
+ kw.note_model_loaded(env)
+ other = dict(env.save_slots_for_resume(lambda: False), gguf = b_gguf)
+ kw._kv_resume = other
+ env.reload(a_gguf)
+ kw.note_model_loaded(env)
+
+ assert [restored["gguf"] for restored in env.restores] == [a_gguf]
+
+
+ "guard, value",
+ [("_chat_busy", lambda counted = False: True), ("_chat_account", lambda: "someone-else")],
+)
+def test_a_manual_unload_leaves_a_busy_or_foreign_chat_unsaved(env, monkeypatch, guard, value):
+ monkeypatch.setattr(kw, "_unload_kv", [], raising = False)
+ monkeypatch.setattr(cs, guard, value)
+
+ _unload(monkeypatch, env)
+
+ assert env.saves == []
+
+
+def test_a_manual_unload_saves_nothing_with_keep_context_off(env, monkeypatch):
+ monkeypatch.setattr(kw, "_unload_kv", [], raising = False)
+ monkeypatch.setattr(settings, "get_auto_unload_keep_kv", lambda: False)
+
+ _unload(monkeypatch, env)
+
+ assert env.saves == []
+
+
+def test_turning_keep_context_off_deletes_saved_unloads(env, monkeypatch):
+ monkeypatch.setattr(kw, "_unload_kv", [], raising = False)
+
+ _unload(monkeypatch, env)
+ assert env.files and not _no_files(env)
+
+ kw.purge_kv_resume()
+
+ assert _no_files(env)
+
+
+def test_a_save_that_lands_as_the_unload_times_out_is_deleted(env, monkeypatch):
+ monkeypatch.setattr(kw, "_unload_kv", [], raising = False)
+ real_wait_for = asyncio.wait_for
+
+ async def _times_out_after_delivery(aw, timeout):
+ if timeout != cs.SWAP_KV_SAVE_BUDGET_S:
+ return await real_wait_for(aw, timeout)
+ await aw
+ raise asyncio.TimeoutError
+
+ monkeypatch.setattr(asyncio, "wait_for", _times_out_after_delivery)
+
+ _unload(monkeypatch, env)
+
+ assert len(env.saves) == 1
+ assert kw._unload_kv == []
+ assert _no_files(env)
+
+
+def three_byte_cap(monkeypatch):
+ import core.inference.llama_cpp as llama_cpp
+ monkeypatch.setattr(llama_cpp, "_SLOT_SAVE_MAX_BYTES", 3)
+ monkeypatch.setattr(kw, "_unload_kv", [], raising = False)
+
+
+def test_an_idle_or_swap_save_evicts_older_unload_saves_past_the_cap(env, three_byte_cap):
+ first = env.save_slots_for_resume(lambda: False)
+ kw.stash_unload_kv(first)
+ second = env.save_slots_for_resume(lambda: False)
+
+ kw._set_kv_resume({"origin": "idle", **second})
+
+ assert kw._unload_kv == []
+ assert not env.files[0].exists()
+ assert env.files[1].exists()
+
+
+def test_an_unload_save_replaces_an_idle_save_past_the_cap(env, three_byte_cap):
+ idle = env.save_slots_for_resume(lambda: False)
+ kw._set_kv_resume({"origin": "idle", **idle})
+ unloaded = env.save_slots_for_resume(lambda: False)
+
+ kw.stash_unload_kv(unloaded)
+
+ assert kw._kv_resume is None
+ assert kw._unload_kv == [unloaded]
+ assert not env.files[0].exists()
+ assert env.files[1].exists()
+
+
[email protected]("loader, saved", [(None, 1), ("someone-else", 0)])
+def test_a_cpu_only_primary_is_saved_for_the_account_that_loaded_it(
+ env, monkeypatch, loader, saved
+):
+ monkeypatch.setattr(kw, "_unload_kv", [], raising = False)
+ monkeypatch.setattr(cs, "_chat_account", lambda: None)
+ env.holds_no_vram = True
+ kw.note_model_loaded(env)
+ if loader is not None:
+ monkeypatch.setattr(kw, "_loaded_by", (env._gguf_path, loader))
+
+ _unload(monkeypatch, env)
+
+ assert len(env.saves) == saved
diff --git a/studio/frontend/src/i18n/locales/en.ts b/studio/frontend/src/i18n/locales/en.ts
index 98203d245c4..d54f9382e4e 100644
--- a/studio/frontend/src/i18n/locales/en.ts
+++ b/studio/frontend/src/i18n/locales/en.ts
@@ -1965,7 +1965,7 @@ export const en = {
idleUnsaved: "Not saved yet.",
keepKv: "Keep chat context across unloads",
keepKvDescription:
- "Save the KV cache before an idle unload, or before an image or video model takes the GPU, so resumed chats skip re-reading history. Up to 10 GB on disk.",
+ "Save the KV cache before a model is unloaded or switched, or before an image or video model takes the GPU, so resumed chats skip re-reading history. Up to 10 GB on disk.",
keepKvKeywords: "kv cache context swap resume image video",
apiOnly: "Only unload models loaded by the API",
apiOnlyDescription: