Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

Studio: serve multiple models at once (#11591)

Nilay8fbd5b6cb2b0
diff --git a/studio/backend/core/inference/gpu_arbiter.py b/studio/backend/core/inference/gpu_arbiter.py
index 9c0641a7216..99a47f51100 100644
--- a/studio/backend/core/inference/gpu_arbiter.py
+++ b/studio/backend/core/inference/gpu_arbiter.py
@@ -38,10 +38,12 @@ def _evict_chat() -> None:
import time
from core.inference import get_inference_backend
+ from core.inference.model_slots import unload_extra_models
from routes.inference import get_llama_cpp_backend
from core.inference.llama_cpp import chat_load_active
+ unload_extra_models(strict = True)
llama = get_llama_cpp_backend()
# is_active (process exists), not is_loaded (exists AND healthy): a chat model still starting up holds VRAM but is
# not healthy. chat_load_active too, since an HF load has no process until its GGUF downloaded. unload_model sets
@@ -179,6 +181,7 @@ def acquire_for(
allow_evict: bool = True,
account_id: Optional[str] = None,
replacing: bool = False,
+ alongside: bool = False,
) -> Any:
"""Make ``owner`` the sole GPU owner, evicting the other if it holds it.
@@ -209,7 +212,7 @@ def acquire_for(
# A resident H3 sd-server sits outside every owner's teardown.
_release_idle_video_servers(f"GPU acquired for {owner}")
# Records who LOADED the model; a plain re-assert must not hand it to whoever asked last.
- claims = _owner != owner or register is not None or replacing
+ claims = _owner != owner or ((register is not None or replacing) and not alongside)
_owner = owner
_owner_epoch += 1
result = register() if register is not None else None
diff --git a/studio/backend/core/inference/llama_cpp.py b/studio/backend/core/inference/llama_cpp.py
index 7d6e8c4bfee..6ddaac42096 100644
--- a/studio/backend/core/inference/llama_cpp.py
+++ b/studio/backend/core/inference/llama_cpp.py
@@ -10,6 +10,7 @@ OpenAI-compatible /v1/chat/completions endpoint.
import ast
import atexit
import bisect
+import weakref
import contextlib
import ctypes
import errno
@@ -537,6 +538,80 @@ class LlamaServerNotFoundError(RuntimeError):
__slots__ = ()
+def _gpu_short_message(
+ need_gb: float, free_gb: float, asked_ctx: int, fitting_ctx: int, capped_only: bool
+) -> str:
+ """What a user reads when a model does not fit next to the loaded ones."""
+ message = (
+ f"This model needs about {need_gb:.1f} GB of GPU memory at a {asked_ctx} context, "
+ f"and {free_gb:.1f} GB is free next to the models already loaded. "
+ )
+ if capped_only:
+ return message + f"It fits with a {fitting_ctx} context."
+ return message + (
+ "Unload one of them, lower the context length, or turn off "
+ '"Keep other models loaded" to replace the current model.'
+ )
+
+
+class GpuMemoryShortError(RuntimeError):
+ """A load that must fit next to the loaded models does not. ``capped``: it would, at a
+ smaller context than asked for. ``gpu_indices``: the GPUs ``short_mib`` was measured on."""
+
+ __slots__ = ("capped", "short_mib", "gpu_indices")
+
+ def __init__(
+ self,
+ message: str,
+ capped: bool = False,
+ short_mib: int = 0,
+ gpu_indices = None,
+ ):
+ super().__init__(message)
+ self.capped = capped
+ self.short_mib = short_mib
+ self.gpu_indices = gpu_indices
+
+
+def _gpu_plan_mib(total_bytes: int, on: list[int], gpus) -> dict[int, int]:
+ """MiB this load holds per GPU: an even share, capped at what each card had free, since
+ --fit on offloads the rest to CPU."""
+ share = total_bytes // (1024 * 1024 * len(on))
+ free = dict(gpus)
+ return {idx: min(share, free.get(idx, share)) for idx in on}
+
+
+def _net_of_held_vram(gpu_mem, held: dict[int, int]):
+ """Cap each GPU's free MiB at its total less what the other loaded models planned. A total of 0
+ is an integrated GPU on shared RAM, whose free reading already counts them: left as is."""
+ if not held:
+ return gpu_mem
+ return [
+ (idx, min(free, max(0, total - held.get(idx, 0))) if total > 0 else free, total)
+ for idx, free, total in gpu_mem
+ ]
+
+
+_serving_backends: "weakref.WeakSet[LlamaCppBackend]" = weakref.WeakSet()
+# A WeakSet is not safe to iterate while another thread adds or drops a backend.
+_serving_backends_lock = threading.Lock()
+
+
+def register_serving_backend(backend) -> None:
+ with _serving_backends_lock:
+ _serving_backends.add(backend)
+
+
+def unregister_serving_backend(backend) -> None:
+ with _serving_backends_lock:
+ _serving_backends.discard(backend)
+
+
+def _serving_backends_snapshot() -> list:
+ with _serving_backends_lock:
+ return list(_serving_backends)
+
+
class GgufDownloadCancelled(RuntimeError):
"""Expected signal from a cancelled GGUF download."""
@@ -629,6 +704,7 @@ class GgufLoadIntent:
# a launch-time rewrite makes the launched and requested lists diverge.
extra_args_inherited: bool = False
preserve_multi_gpu_on_layer: bool = False
+ refuse_partial_gpu_fit: bool = False
compare_mtp_draft: bool = False
force_reload: bool = False
@@ -7864,6 +7940,8 @@ class LlamaCppBackend:
# Monotonic timestamp set in _kill_process; read by load_model
# to decide whether to wait for the VRAM reclaim to finish.
self._last_kill_monotonic: float = 0.0
+ self._planned_vram_mib: dict[int, int] = {}
+ self._pending_plan_mib: dict[int, int] = {}
if manages_processes:
_reaped = self._kill_orphaned_servers()
@@ -16242,9 +16320,13 @@ class LlamaCppBackend:
total_by_idx: Optional[dict[int, int]] = None,
per_device_overhead_bytes: int = 0,
min_gpus: int = 1,
+ shared: frozenset = frozenset(),
) -> tuple[Optional[list[int]], bool]:
"""Pick GPU(s) for a model from estimated VRAM and free memory.
+ ``shared``: GPUs another loaded model runs on. A model that fits one card alone
+ takes a card outside it when one fits, so two models don't share compute.
+
``min_gpus`` (default 1, capped at ``len(gpus)``) keeps a downgraded
tensor/multi-GPU request spread instead of collapsing to one card.
@@ -16294,6 +16376,9 @@ class LlamaCppBackend:
# Try 1 GPU at the usable-VRAM threshold (only when one device is allowed).
if min_gpus <= 1 and _usable(ranked[0][0], ranked[0][1]) >= model_size_mib:
+ for idx, free_mib in ranked:
+ if idx not in shared and _usable(idx, free_mib) >= model_size_mib:
+ return [idx], False
return [ranked[0][0]], False
# Try N GPUs (most-free first); each past the first adds per-device overhead.
@@ -16326,6 +16411,7 @@ class LlamaCppBackend:
per_device_overhead_bytes: int = 0,
min_gpus: int = 1,
split_extra_bytes: int = 0,
+ shared: frozenset = frozenset(),
) -> tuple[Optional[list[int]], bool]:
"""``_select_gpus``, re-checked at the multi-device context-compute rate.
@@ -16345,6 +16431,7 @@ class LlamaCppBackend:
total_by_idx = total_by_idx,
per_device_overhead_bytes = per_device_overhead_bytes,
min_gpus = min_gpus,
+ shared = shared,
)
if use_fit or split_extra_bytes <= 0 or not gpu_indices or len(gpu_indices) < 2:
return gpu_indices, use_fit
@@ -16365,6 +16452,7 @@ class LlamaCppBackend:
total_by_idx = total_by_idx,
per_device_overhead_bytes = per_device_overhead_bytes,
min_gpus = 1,
+ shared = shared,
)
if not single_fit and single is not None and len(single) == 1:
return single, False
@@ -23361,7 +23449,7 @@ class LlamaCppBackend:
_invalidate_gpu_memory("llama-server spawned")
# Cross-session backstop for when parent-death cleanup did not run.
# Under the lock: see _spawn_and_wait.
- self._record_server_pid(_spawned.pid)
+ self._note_server_pid(_spawned.pid)
# The stale check above and the process-wide latch are only atomic for the
# instance run.py tears down, which sets its own flag under this same lock. A
@@ -24946,6 +25034,10 @@ class LlamaCppBackend:
# from the default unless the verdict is recorded on its own.
# Bound before the try for the same reason as _detected_gpus.
_placement_verdict_partial = False
+ _ctx_capped_for_vram = False
+ # A path that never prices the launch must not commit the previous one's plan.
+ self._pending_plan_mib = {}
+ _shared_gpus = frozenset()
# Sized inputs for the tensor-spill planner, None when the fit never
# priced them. Bound before the try like _placement_verdict_partial:
# the except arm restores use_fit=True without rebinding
@@ -25050,6 +25142,9 @@ class LlamaCppBackend:
# so the pin happens anyway. A pinned uncovered GPU is the user's
# call and already reports "device kernel image is invalid".
_gpu_mem = self._get_gpu_memory(binary, for_llama_server = not gpu_ids)
+ _held_by_others = self._other_planned_vram_mib()
+ _shared_gpus = frozenset(_held_by_others)
+ _gpu_mem = _net_of_held_vram(_gpu_mem, _held_by_others)
# Every present device gated out (#7624). Left alone the launch
# takes the `--fit on` arm with `gpu_indices` still None, so no
# mask is written, the child enumerates every unsupported card and
@@ -26514,6 +26609,7 @@ class LlamaCppBackend:
+ _cc_bytes(effective_ctx),
min_gpus = _layer_min_gpus,
split_extra_bytes = _cc_split_extra(effective_ctx),
+ shared = _shared_gpus,
)
# No silent shrink: effective_ctx stays == requested_ctx.
# use_fit = the pin failed and --fit on will offload; say why.
@@ -26602,6 +26698,28 @@ class LlamaCppBackend:
or 1,
),
)
+ _ctx_before_fit = effective_ctx
+ if _shared_gpus and _auto_min_gpus == 1 and effective_ctx > 0:
+ # A card no other model runs on, when it holds this one at the full
+ # context: sharing a card splits its compute between the models.
+ _alone_mib = (
+ _subset_model_size(1)
+ + _kv_bytes(effective_ctx)
+ + _mtp_bytes(effective_ctx)
+ + _cc_bytes(effective_ctx, 1)
+ ) / (1024 * 1024)
+ _unshared = next(
+ (
+ g
+ for g in ranked
+ if g[0] not in _shared_gpus
+ and _gpu_usable(g, pin_fraction) >= _alone_mib
+ ),
+ None,
+ )
+ if _unshared is not None:
+ ranked.remove(_unshared)
+ ranked.insert(0, _unshared)
for n_gpus in range(_auto_min_gpus, len(ranked) + 1):
subset = ranked[:n_gpus]
pool_budget = _pool_budget_mib(subset, pin_fraction)
@@ -26665,6 +26783,7 @@ class LlamaCppBackend:
if footprint_mib > pool_budget:
continue
effective_ctx = capped
+ _ctx_capped_for_vram = 0 < capped < _ctx_before_fit
gpu_indices = sorted(idx for idx, _ in subset)
use_fit = False
break
@@ -26716,6 +26835,7 @@ class LlamaCppBackend:
total_by_idx = total_by_idx,
per_device_overhead_bytes = _pipeline_overhead_bytes,
min_gpus = _layer_min_gpus,
+ shared = _shared_gpus,
)
if use_fit and not explicit_ctx:
# Without KV metadata, llama.cpp owns the fit. Keep the
@@ -27294,6 +27414,11 @@ class LlamaCppBackend:
# --fit flag state, not "does it fit": off means this subset provably fits.
f"GPUs free: {gpus}, selected: {gpu_indices}, --fit: {'on' if use_fit else 'off'}"
)
+ _on = list(gpu_indices or [idx for idx, _ in gpus])
+ if _on:
+ self._pending_plan_mib = _gpu_plan_mib(
+ gguf_size + mmproj_size + kv_cache_bytes, _on, gpus
+ )
if (
not gpus
and not _detected_gpus
@@ -27532,6 +27657,23 @@ class LlamaCppBackend:
raise RuntimeError(_metal_ctx_refusal)
self._record_load_warning(_metal_ctx_warning)
+ if (
+ intent.refuse_partial_gpu_fit
+ and (_placement_verdict_partial or _ctx_capped_for_vram)
+ and gpu_memory_mode != "manual"
+ ):
+ capped_only = _ctx_capped_for_vram and not _placement_verdict_partial
+ asked_ctx = _ctx_before_fit if capped_only else effective_ctx
+ need_mib = (gguf_size + mmproj_size + _kv_bytes(asked_ctx)) // (1024 * 1024)
+ free_mib = sum(free for _, free in gpus)
+ need_gb, free_gb = need_mib / 1024, free_mib / 1024
+ raise GpuMemoryShortError(
+ _gpu_short_message(need_gb, free_gb, asked_ctx, effective_ctx, capped_only),
+ capped = capped_only,
+ short_mib = max(0, need_mib - free_mib),
+ gpu_indices = tuple(idx for idx, _ in gpus),
+ )
+
# An unenumerated explicit Vulkan ordinal can't be pinned; fail loudly
# instead of fitting onto an unselected device. Clear the raw selection
# the early state-publish recorded so it never leaks into gpu_ids (#7239).
@@ -30215,7 +30357,7 @@ class LlamaCppBackend:
# Inside the lock: written after a sweep reaped the child,
# _pid_start_identity yields no start time, and the bare pid
# left behind is one a later launch kills blind.
- self._record_server_pid(_spawned.pid)
+ self._note_server_pid(_spawned.pid)
# mark_process_shutting_down does not take _spawn_lock, so the
# check above is not atomic against it for a helper-owned
# backend. Without this recheck a child spawned in that gap sits
@@ -33200,6 +33342,7 @@ class LlamaCppBackend:
logger.info("load no longer belongs to this lifecycle; not publishing it healthy")
return False
self._healthy = True
+ self._planned_vram_mib = dict(getattr(self, "_pending_plan_mib", {}))
return True
def _close_attempt_log(
@@ -33315,6 +33458,7 @@ class LlamaCppBackend:
self._diffusion_requested_ngl = None
self._child_gpu_physical_ids = None
if self._process is None:
+ self._planned_vram_mib = {}
return
# Not every _process is a Popen: tests stand one in to mean "a server is
# loaded" without spawning anything. Terminating what cannot be terminated
@@ -33431,12 +33575,15 @@ class LlamaCppBackend:
except Exception:
pass
self._process = None
+ self._planned_vram_mib = {}
# Same rule as the lifetime record above: the pidfile is the next launch's
# only handle on a server that outlived this kill, so it is removed once the
# exit is confirmed and not merely attempted. Without a child handle there is
# no exit to confirm and nothing was signalled, so the pidfile is dropped as
# it always was rather than kept forever by a stand-in that cannot answer.
- if _killed_pid is None or _exited or not _owns_child:
+ if getattr(self, "_owns_pidfile", True) and (
+ _killed_pid is None or _exited or not _owns_child
+ ):
self._clear_server_pid()
# Clear healthy so a /load during the replacement's warm-up can't
# short-circuit against the previous server's health (#5401).
@@ -33459,6 +33606,29 @@ class LlamaCppBackend:
reason = "exited" if _exited_first else "killed", exit_code = _exit_code
)
+ def _other_planned_vram_mib(self) -> dict[int, int]:
+ """MiB per GPU the other serving backends' running servers hold."""
+ held: dict[int, int] = {}
+ serving = _serving_backends_snapshot()
+ if self not in serving:
+ return held
+ for backend in serving:
+ if backend is self or not getattr(backend, "is_active", False):
+ continue
+ for idx, mib in getattr(backend, "_planned_vram_mib", {}).items():
+ held[idx] = held.get(idx, 0) + mib
+ return held
+
+ def _note_server_pid(self, pid: int) -> None:
+ if getattr(self, "_owns_pidfile", True):
+ self._record_server_pid(pid)
+ return
+ try:
+ from utils.process_lifetime import adopt_pid
+ adopt_pid(pid)
+ except Exception as e:
+ logger.debug(f"Could not track llama-server for lifetime sweep: {e}")
+
@staticmethod
def _server_pidfile_path() -> Optional[Path]:
"""Pidfile recording the live llama-server PID, under the active studio root
diff --git a/studio/backend/core/inference/llama_keepwarm.py b/studio/backend/core/inference/llama_keepwarm.py
index 3d5b69d856c..1c3928af8b9 100644
--- a/studio/backend/core/inference/llama_keepwarm.py
+++ b/studio/backend/core/inference/llama_keepwarm.py
@@ -55,14 +55,19 @@ _kv_resume = None
_lifecycle_lock = threading.Lock()
+_load_lock = threading.Lock()
+
+
@contextlib.asynccontextmanager
-async def _unload_gate(cancel_event: threading.Event | None = None):
+async def _unload_gate(
+ cancel_event: threading.Event | None = None, lock: threading.Lock = _lifecycle_lock
+):
# Acquire off the loop: non-blocking first (the common uncontended case), else poll a non-blocking acquire off a
# short sleep. Polling keeps the wait off this loop AND cancellation-safe -- a cancel lands during the sleep, when
# the gate is not held, so it never leaks (mirrors the auto-switch swap gate).
acquired = False
try:
- while not _lifecycle_lock.acquire(blocking = False):
+ while not lock.acquire(blocking = False):
if cancel_event is not None and cancel_event.is_set():
raise asyncio.CancelledError()
await asyncio.sleep(0.02)
@@ -72,7 +77,7 @@ async def _unload_gate(cancel_event: threading.Event | None = None):
yield
finally:
if acquired:
- _lifecycle_lock.release()
+ lock.release()
_INFERENCE_PREFIXES = ("/v1/", "/api/inference/")
@@ -500,6 +505,10 @@ def inference_lifecycle_gate():
return _unload_gate()
+def model_load_gate():
+ return _unload_gate(lock = _load_lock)
+
+
def note_model_loaded(backend = None) -> None:
"""Stamp activity and synchronously drop any reload stash."""
_note_activity()
@@ -836,12 +845,17 @@ async def idle_unload_loop(poll_seconds: float = 15.0) -> None:
ttl = await asyncio.to_thread(get_auto_unload_idle_seconds)
if ttl <= 0:
continue
- from routes.inference import get_llama_cpp_backend
+ from core.inference.model_slots import unload_extra_models
+ from routes.inference import get_llama_cpp_backend, release_chat_gpu_claim
backend = get_llama_cpp_backend()
# track by (id, variant): a (re)loaded model counts as activity so it survives one TTL before its first
# request
async with _unload_gate():
+ if _is_idle(ttl) and await asyncio.to_thread(
+ unload_extra_models, _user_pinned, True
+ ):
+ await asyncio.to_thread(release_chat_gpu_claim)
# Purging the stash mid-reload would race the restore.
current = _loaded_identity(backend)
if current != seen_model:
diff --git a/studio/backend/core/inference/model_slots.py b/studio/backend/core/inference/model_slots.py
new file mode 100644
index 00000000000..29776937246
--- /dev/null
+++ b/studio/backend/core/inference/model_slots.py
@@ -0,0 +1,299 @@
+# SPDX-License-Identifier: AGPL-3.0-only
+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
+
+"""Models kept loaded alongside the primary one, each with its own llama-server and orchestrator.
+A task naming one sets ``routed_slot``, so the usual backend getters answer with that slot."""
+
+import asyncio
+import atexit
+import contextvars
+import threading
+import time
+from contextlib import suppress
+from dataclasses import dataclass, field
+from typing import Any, Callable, Optional
+
+from core.inference.orchestrator import routed_slot
+from hub.services.models import account_access
+from loggers import get_logger
+from state import active_generations
+from utils.account_context import current_account_id
+
+logger = get_logger(__name__)
+
+
+@dataclass(eq = False)
+class ExtraSlot:
+ llama: Any
+ orchestrator: Any
+ account: str
+ request: Any = None
+ last_used: float = 0.0
+ generations: set = field(default_factory = set)
+ refs: int = 0
+
+
+slots: list[ExtraSlot] = []
+lock = threading.Lock()
+# (slot, model_path) while a load fills a slot of its own.
+loading: Optional[tuple[ExtraSlot, str]] = None
+# Out of routing, but their server would not stop: still priced for VRAM and retried by every drop.
+stuck: list[ExtraSlot] = []
+
+
+def in_slot(slot: Optional[ExtraSlot], fn: Callable):
+ context = contextvars.copy_context()
+ context.run(routed_slot.set, slot)
+ return context.run(fn)
+
+
+def filling_model() -> Optional[str]:
+ current = loading
+ return current[1] if current else None
+
+
+def any_loading() -> bool:
+ return loading is not None or any(
+ getattr(slot.orchestrator, "loading_models", None) for slot in list(slots)
+ )
+
+
+def in_use(slot: ExtraSlot) -> bool:
+ return bool(slot.llama.is_active or slot.orchestrator.active_model_name)
+
+
+def busy() -> bool:
+ return any_loading() or any(in_use(slot) for slot in list(slots))
+
+
+def resident() -> list[ExtraSlot]:
+ """Every kept slot that may still hold a model: the routed ones and the stuck ones."""
+ return [*slots, *stuck]
+
+
+def _worker_alive(orchestrator) -> bool:
+ worker_alive = getattr(orchestrator, "is_worker_alive", None)
+ return bool(callable(worker_alive) and worker_alive())
+
+
+def holds_vram() -> bool:
+ """A kept model on the GPU or one loading; a llama-server started with no GPU layers is not."""
+ return any_loading() or any(
+ slot.orchestrator.active_model_name
+ or (slot in stuck and _worker_alive(slot.orchestrator))
+ or (slot.llama.is_active and getattr(slot.llama, "_gpu_offload_active", None) is not False)
+ for slot in [*slots, *stuck]
+ )
+
+
+def visible() -> list[ExtraSlot]:
+ if not account_access.managed_account():
+ return list(slots)
+ return [slot for slot in slots if slot.account == current_account_id()]
+
+
+def visible_loading() -> Optional[tuple[ExtraSlot, str]]:
+ current = loading
+ return current if current and current[0] in visible() else None
+
+
+def serving_slot(requested: str, candidates: list[ExtraSlot], satisfies) -> Optional[ExtraSlot]:
+ """The slot among ``candidates`` serving ``requested``; None when the primary does (it wins a tie)."""
+ for slot in (None, *candidates):
+ if in_slot(slot, lambda: satisfies(requested)):
+ return slot
+ return None
+
+
+async def route(requested: Optional[str], satisfies) -> Optional[ExtraSlot]:
+ """Route this task to the slot serving ``requested``, if any, and hold it until the task ends."""
+ routed_slot.set(None)
+ if not (slots and isinstance(requested, str) and requested):
+ return None
+ candidates = visible()
+ if not candidates:
+ return None
+ slot = await asyncio.to_thread(serving_slot, requested, candidates, satisfies)
+ if slot is None or not reserve(slot):
+ return None
+ routed_slot.set(slot)
+ slot.last_used = time.monotonic()
+ return slot
+
+
+def reserve(slot: ExtraSlot) -> bool:
+ """Hold ``slot`` against eviction until the current task ends; False once it is gone.
+ The task, not the request: a durable chat run inherits the POST that queued it, which has
+ already answered 202 by the time the run routes."""
+ with lock:
+ if slot not in slots:
+ return False
+ slot.refs += 1
+
+ def release(_task = None):
+ with lock:
+ slot.refs -= 1
+
+ task = asyncio.current_task()
+ release() if task is None else task.add_done_callback(release)
+ return True
+
+
+def eviction_victims(
+ exclude,
+ short_mib: int,
+ gpu_indices = None,
+) -> list[ExtraSlot]:
+ """Least recently used idle slots, as many as it takes to free ``short_mib`` on ``gpu_indices``;
+ one at a time when a footprint is unknown, since the retry prices the rest. A slot still
+ answering is never a victim."""
+ candidates = [s for s in visible() if s is not exclude and in_use(s)]
+ victims, freed = [], 0
+ for slot in sorted(candidates, key = lambda s: s.last_used):
+ planned = getattr(slot.llama, "_planned_vram_mib", {})
+ held = sum(mib for idx, mib in planned.items() if gpu_indices is None or idx in gpu_indices)
+ if slot.generations or slot.refs or (planned and not held):
+ continue
+ victims.append(slot)
+ freed += held
+ if not held or freed >= short_mib:
+ return victims
+ return []
+
+
+def claim_victim(slot: ExtraSlot) -> bool:
+ """Take ``slot`` out of routing unless a request reached it since it was picked."""
+ with lock:
+ if slot.generations or slot.refs or slot not in slots:
+ return False
+ slots.remove(slot)
+ return True
+
+
+def evict(
+ exclude,
+ short_mib: int,
+ gpu_indices = None,
+) -> list[ExtraSlot]:
+ """Unload the idle slots ``eviction_victims`` picks."""
+ return evict_these(eviction_victims(exclude, short_mib, gpu_indices))
+
+
+def evict_these(victims) -> list[ExtraSlot]:
+ """Unload ``victims``, skipping any a request reached since they were picked."""
+ dropped = []
+ for victim in victims:
+ if claim_victim(victim):
+ drop(victim)
+ dropped.append(victim)
+ return dropped
+
+
+def all_generations() -> set:
+ return {event for slot in list(slots) for event in list(slot.generations)}
+
+
+def routed_generation_count() -> int:
+ """Generations on the model this task is routed to; each model decodes on its own server."""
+ slot = routed_slot.get()
+ if slot is not None:
+ return len(slot.generations)
+ elsewhere = all_generations()
+ return active_generations.count(None, elsewhere) if elsewhere else active_generations.count()
+
+
+def drop(slot: ExtraSlot) -> None:
+ from core.inference.llama_cpp import unregister_serving_backend
+
+ with suppress(ValueError):
+ slots.remove(slot)
+ try:
+ slot.llama.unload_model()
+ except BaseException:
+ if slot not in stuck:
+ stuck.append(slot)
+ slot.orchestrator._cleanup()
+ raise
+ slot.orchestrator._cleanup()
+ # _cleanup does not report a worker that outlived its kill.
+ if _worker_alive(slot.orchestrator):
+ if slot not in stuck:
+ stuck.append(slot)
+ raise RuntimeError("An inference worker kept alongside is still alive")
+ with suppress(ValueError):
+ stuck.remove(slot)
+ unregister_serving_backend(slot.llama)
+ atexit.unregister(slot.llama._cleanup)
+ atexit.unregister(slot.orchestrator._cleanup)
+
+
+def stop_orchestrator_workers() -> None:
+ """Drop every slot running an orchestrator worker, for the transformers sidecar swap. Raises
+ when one survives: it would lazy-import from the swapped package tree."""
+
+ doomed = [
+ s for s in resident() if s.orchestrator.active_model_name or _worker_alive(s.orchestrator)
+ ]
+ for slot in doomed:
+ with suppress(Exception):
+ drop(slot)
+ if any(_worker_alive(slot.orchestrator) for slot in doomed):
+ raise RuntimeError(
+ "An inference worker kept alongside is still alive before the transformers swap"
+ )
+
+
+def _drop_where(predicate, strict: bool = False) -> int:
+ """``strict`` raises once every slot was tried if any teardown failed: a GPU handoff must not
+ proceed past a server that may still hold VRAM."""
+ filling = loading[0] if loading else None
+ doomed = [slot for slot in list(slots) if predicate(slot, slot is filling)]
+ retried = list(stuck)
+ failed = []
+ for slot in doomed:
+ try:
+ drop(slot)
+ except Exception as exc:
+ logger.warning("Could not unload an extra model: %s", exc)
+ failed.append(exc)
+ for slot in retried:
+ try:
+ drop(slot)
+ except Exception as exc:
+ failed.append(exc)
+ if strict and failed:
+ raise RuntimeError(f"Could not unload {len(failed)} model(s) kept alongside") from failed[0]
+ return len(doomed)
+
+
+def unload_extra_models(
+ keep = None,
+ spare_filling: bool = False,
+ strict: bool = False,
+) -> int:
+ """Drop every slot, or with ``keep`` only the loaded ones it does not spare. Returns how many."""
+
+ def doomed(slot, filling):
+ loaded = slot.llama.is_loaded or slot.orchestrator.active_model_name
+ return not (spare_filling and filling) and (
+ keep is None or bool(loaded and not keep(slot.llama))
+ )
+
+ return _drop_where(doomed, strict)
+
+
+def unload_idle() -> int:
+ """Drop every kept model no request is using or loading."""
+ filling = loading[0] if loading else None
+ idle = [s for s in list(slots) if s is not filling and not s.generations and not s.refs]
+ return len(evict_these(idle))
+
+
+def unload_llama_slots(strict: bool = False) -> int:
+ """Drop every slot running or starting a llama-server, a GGUF load still filling one included."""
+ return _drop_where(
+ lambda slot, filling: slot.llama.is_active
+ or slot.llama.is_loaded
+ or (filling and not getattr(slot.orchestrator, "loading_models", None)),
+ strict,
+ )
diff --git a/studio/backend/core/inference/orchestrator.py b/studio/backend/core/inference/orchestrator.py
index 4585e8b14be..06fcaf69efa 100644
--- a/studio/backend/core/inference/orchestrator.py
+++ b/studio/backend/core/inference/orchestrator.py
@@ -8,6 +8,7 @@ Qwen needs 4.57.x), the old subprocess is killed and a new one spawned with the
Pattern follows core/training/training.py."""
import atexit
+import contextvars
import base64
import contextlib
import os
@@ -3717,16 +3718,23 @@ _inference_backend = None
_inference_backend_lock = threading.Lock()
+routed_slot: contextvars.ContextVar = contextvars.ContextVar("routed_slot", default = None)
+
+
def peek_inference_backend() -> Optional["InferenceOrchestrator"]:
"""The orchestrator if one exists, else None. Never constructs one. For callers that only
describe what is already loaded: constructing reaches get_default_models() -> get_device(),
which blocks on the torch import during the warm."""
- return _inference_backend
+ slot = routed_slot.get()
+ return slot.orchestrator if slot is not None else _inference_backend
def get_inference_backend() -> InferenceOrchestrator:
"""Global inference backend instance (orchestrator)."""
global _inference_backend
+ slot = routed_slot.get()
+ if slot is not None:
+ return slot.orchestrator
# Double-checked: the cheap read keeps the hot path lock-free, the recheck picks a builder
if _inference_backend is None:
with _inference_backend_lock:
diff --git a/studio/backend/hub/services/models/deletion.py b/studio/backend/hub/services/models/deletion.py
index 7ae59ab58c6..611e07e4e07 100644
--- a/studio/backend/hub/services/models/deletion.py
+++ b/studio/backend/hub/services/models/deletion.py
@@ -651,27 +651,26 @@ _LOAD_STATE_UNVERIFIABLE_DETAIL = (
def _llama_cpp_blocks_delete(repo_id: str, variant: Optional[str]) -> bool:
"""Whether the llama.cpp backend holds *repo_id* (/variant). Acquiring fails open (import error means nothing loaded); reading load state is unguarded so a raise propagates and the caller fails closed rather than delete a live model."""
try:
+ from core.inference import model_slots
from routes.inference import get_llama_cpp_backend
- backend = get_llama_cpp_backend()
+
+ backends = [get_llama_cpp_backend(), *(slot.llama for slot in model_slots.resident())]
+ filling = model_slots.filling_model()
except Exception as e:
logger.debug(f"llama.cpp backend unavailable during delete guard for {repo_id}: {e}")
return False
- loaded_id = backend.model_identifier
- loaded_variant = getattr(backend, "hf_variant", None)
- if backend.is_active and not backend.is_loaded and loaded_id:
- return _loaded_repo_variant_blocks_delete(
- loaded_id,
- repo_id,
- variant,
- loaded_variant,
- )
- if backend.is_loaded and loaded_id:
- return _loaded_repo_variant_blocks_delete(
- loaded_id,
- repo_id,
- variant,
- loaded_variant,
- )
+ if filling and _loaded_id_matches_repo(filling, repo_id):
+ return True
+ for backend in backends:
+ loaded_id = backend.model_identifier
+ if (backend.is_active or backend.is_loaded) and loaded_id:
+ if _loaded_repo_variant_blocks_delete(
+ loaded_id,
+ repo_id,
+ variant,
+ getattr(backend, "hf_variant", None),
+ ):
+ return True
return False
@@ -679,14 +678,23 @@ def _inference_backend_blocks_delete(repo_id: str) -> bool:
"""Whether the subprocess inference backend holds *repo_id*; same fail-open-on-acquire / surface-on-query contract as :func:`_llama_cpp_blocks_delete`."""
try:
from core.inference.orchestrator import peek_inference_backend
- backend = peek_inference_backend()
+ from core.inference import model_slots
+
+ primary = peek_inference_backend()
+ kept = [slot.orchestrator for slot in model_slots.resident()]
except Exception as e:
logger.debug(f"Inference backend unavailable during delete guard for {repo_id}: {e}")
return False
- if backend is None:
- return False
- active_name = backend.active_model_name
- return bool(active_name) and _loaded_id_matches_repo(active_name, repo_id)
+ for backend in (primary, *kept):
+ if backend is None:
+ continue
+ active_name = backend.active_model_name
+ if active_name and _loaded_id_matches_repo(active_name, repo_id):
+ return True
+ for backend in kept:
+ if any(_loaded_id_matches_repo(m, repo_id) for m in getattr(backend, "loading_models", ())):
+ return True
+ return False
def _diffusion_blocks_delete(repo_id: str) -> Optional[str]:
@@ -729,13 +737,24 @@ def any_model_load_blocks_cache_clear() -> Optional[str]:
different matter, and the caller fails closed on it rather than unlink weights blindly.
"""
try:
+ from core.inference import model_slots
from routes.inference import get_llama_cpp_backend
+
backend = get_llama_cpp_backend()
+ kept = model_slots.resident()
+ kept_loading = model_slots.any_loading()
except Exception as exc: # noqa: BLE001 - unavailable is not "in use"
logger.debug(f"llama.cpp backend unavailable during the cache-clear guard: {exc}")
else:
if (backend.is_loaded or backend.is_active) and backend.model_identifier:
return "Unload the model before clearing the model cache"
+ if any(
+ model_slots.in_use(slot) or slot.llama.is_loaded or slot.llama.is_active
+ for slot in kept
+ ):
+ return "Unload the model before clearing the model cache"
+ if kept_loading:
+ return "A model load is using the cache; wait for it to finish"
# is_active above only covers a live llama-server process, which an HF-backed chat load does
# not have until its GGUF finished downloading: minutes, per chat_load_active's own docstring.
diff --git a/studio/backend/hub/utils/host_paths.py b/studio/backend/hub/utils/host_paths.py
index 7f37ece7042..c8f69ea9ffb 100644
--- a/studio/backend/hub/utils/host_paths.py
+++ b/studio/backend/hub/utils/host_paths.py
@@ -102,7 +102,7 @@ HOST_PATH_IDENTITY_FIELDS = (
"dataset_name",
"base_repo",
)
-HOST_PATH_IDENTITY_LIST_FIELDS = ("loaded", "loading")
+HOST_PATH_IDENTITY_LIST_FIELDS = ("loaded", "loading", "serving", "serving_checkpoints")
# The subset a LOCAL row is named by, referenced on source alone; the others only on value.
HOST_PATH_ROW_IDENTITY_FIELDS = ("id", "load_id")
HOST_PATH_ENCODED_IDENTITY_FIELD = "inventory_id"
diff --git a/studio/backend/models/inference.py b/studio/backend/models/inference.py
index 1e46f682023..24a7f4ba47d 100644
--- a/studio/backend/models/inference.py
+++ b/studio/backend/models/inference.py
@@ -79,6 +79,14 @@ class LoadRequest(LlamaCppConfigFields):
False,
description = "Start a fresh runtime even when the active settings already match",
)
+ alongside: bool = Field(
+ False,
+ description = "Keep the loaded model and serve this one next to it",
+ )
+ force_alongside: bool = Field(
+ False,
+ description = "Load alongside even when it only partly fits the free GPU memory",
+ )
native_path_lease: Optional[str] = Field(
None, description = "Frontend-visible signed native path grant"
)
@@ -1753,6 +1761,10 @@ class LoadResponse(_InferenceRuntimeFields):
"message. Null once the user has dismissed it at this allocation, and on every "
"load where enlarging the allocation would not help. The model still loaded.",
)
+ evicted: list[str] = Field(
+ default_factory = list,
+ description = "Models loaded alongside that were unloaded to make room for this one.",
+ )
class UnloadResponse(BaseModel):
@@ -1877,6 +1889,16 @@ class InferenceStatusResponse(_InferenceRuntimeFields):
)
loading: List[str] = Field(default_factory = list, description = "Models currently being loaded")
loaded: List[str] = Field(default_factory = list, description = "Models currently loaded")
+ serving: List[str] = Field(
+ default_factory = list,
+ description = "Models answering requests: the active one and each kept alongside. "
+ "Unlike loaded, leaves out a model only held in memory behind the active one.",
+ )
+ serving_checkpoints: List[str] = Field(
+ default_factory = list,
+ description = "The checkpoint id each serving entry is loaded from, in the same order: "
+ "the path of a local model, else the same id. Load, select and unload by this one.",
+ )
inference: Optional[Dict[str, Any]] = Field(
None, description = "Recommended inference parameters for the active model"
)
diff --git a/studio/backend/routes/inference.py b/studio/backend/routes/inference.py
index 255390d39e8..6c20e2f8817 100644
--- a/studio/backend/routes/inference.py
+++ b/studio/backend/routes/inference.py
@@ -66,7 +66,7 @@ import contextvars
import threading
import weakref
from concurrent.futures import ThreadPoolExecutor
-from contextlib import ExitStack, asynccontextmanager, contextmanager
+from contextlib import ExitStack, asynccontextmanager, contextmanager, nullcontext
from dataclasses import dataclass, fields as dataclass_fields, replace
@@ -123,9 +123,11 @@ from core.inference.orchestrator import (
AUDIO_GENERATION_MAX_TOKENS,
GenStreamError,
GenStreamErrorRaised,
+ InferenceOrchestrator,
MINIMAX_MUSIC_MAX_FRAMES,
MOSS_TTS_MAX_FRAMES,
_summed_tool_loop_stats,
+ routed_slot,
)
from core.inference.llama_admission import (
LlamaAdmissionCancelled,
@@ -139,11 +141,14 @@ from core.inference.llama_admission import (
peek_llama_admission_snapshot,
)
from core.inference.tool_stream_exec import TOOL_APPROVAL_FLUSH_DELAY_S
+from core.inference import model_slots
from core.inference.llama_cpp import (
_llama_chunk_has_generated_output,
_local_ssl_context,
+ register_serving_backend,
requested_video_fps,
)
+from core.inference.model_slots import ExtraSlot as _ExtraSlot
from core.inference.llama_video_input import shrink_video_for_llama
@@ -5399,6 +5404,7 @@ class _TrackedCancel:
self._active = active_generations.ActiveGeneration(
event, thread_id = thread_id, run_id = run_id, model = model, kind = kind
)
+ self._slot = None
@classmethod
def for_payload(cls, event: threading.Event, payload, *keys):
@@ -5424,6 +5430,10 @@ class _TrackedCancel:
if k and _PENDING_CANCELS.pop(k, None) is not None:
should_cancel = True
self._active.__enter__()
+ self._slot = routed_slot.get()
+ if self._slot is not None:
+ with model_slots.lock:
+ self._slot.generations.add(self.event)
if should_cancel:
self.event.set()
return self.event
@@ -5438,6 +5448,10 @@ class _TrackedCancel:
if not bucket:
_CANCEL_REGISTRY.pop(k, None)
self._active.__exit__(*exc)
+ if self._slot is not None:
+ with model_slots.lock:
+ self._slot.generations.discard(self.event)
+ self._slot = None
return False
@@ -8612,10 +8626,47 @@ def _resolve_model_identifier_for_request(
# GGUF inference backend (llama-server)
_llama_cpp_backend = LlamaCppBackend()
+register_serving_backend(_llama_cpp_backend)
def get_llama_cpp_backend() -> LlamaCppBackend:
- return _llama_cpp_backend
+ slot = routed_slot.get()
+ return slot.llama if slot is not None else _llama_cpp_backend
+
+
+async def _route_to_extra_slot(requested: Optional[str]) -> Optional[_ExtraSlot]:
+ """Route this request to the kept model serving *requested*, if any. The primary wins a tie."""
+ return await model_slots.route(requested, _loaded_satisfies)
+
+
+def _model_key(request: LoadRequest) -> str:
+ return (
+ f"{request.model_path}:{request.gguf_variant}"
+ if request.gguf_variant
+ else request.model_path
+ )
+
+
+def _raise_or_cancel_slot_generations(
+ slot: _ExtraSlot,
+ *,
+ force: bool,
+ cancel: bool = True,
+ action: str = "Unloading this model",
+) -> int:
+ """The primary's 409 for a slot: only the generations this slot serves are in the way."""
+ events = list(slot.generations)
+ if not events:
+ return 0
+ if not force:
+ raise _active_generations_conflict(
+ action, len(events), active_generations.active_thread_ids(None, (), events)
+ )
+ if not cancel:
+ return 0
+ for event in events:
+ event.set()
+ return len(events)
# Serializes opt-in auto-switch loads so two requests can't race a swap. One
@@ -8723,12 +8774,10 @@ async def _wait_for_model_switch_idle(
current_request_counted = current_request_counted,
include_pending = False,
)
+ elsewhere = model_slots.all_generations()
+ active_others -= min(active_others, len(elsewhere))
if cancel_pending:
- cancellable = (
- active_generations.count(account_id)
- if account_id is not None
- else active_generations.count()
- )
+ cancellable = active_generations.count(account_id, elsewhere)
active_others -= min(active_others, cancellable)
if active_others <= queued_switches:
return
@@ -9578,6 +9627,11 @@ def _anthropic_local_image_payloads(payload) -> list[str]:
return encoded_images
+def _auto_switch_opted_out(fastapi_request) -> bool:
+ scope = getattr(fastapi_request, "scope", None)
+ return isinstance(scope, dict) and bool(scope.get(_DISABLE_OPENAI_AUTO_SWITCH_SCOPE_KEY))
+
+
def disable_openai_auto_switch_for_request(scope) -> None:
"""Opt a request out of OpenAI auto-switch. The public preview route uses this:
it always serves its pinned checkpoint, so a caller-supplied model must never
@@ -9766,10 +9820,8 @@ async def _no_model_loaded_error(
"""
from utils.openai_auto_switch_settings import get_openai_auto_switch_enabled
from core.inference.local_model_resolver import resolve_local_gguf
- from core.inference.npu_backend import peek_npu_backend
- _npu = peek_npu_backend()
- _npu_model = _npu.loaded_model if _npu is not None else None
+ _npu_model = _resident_npu_model()
if _npu_model is not None:
return 400, (
f"The loaded NPU model ({_npu_model.model_path}) serves "
@@ -9947,9 +9999,12 @@ def _resident_id_is_namespaced() -> bool:
def _resident_npu_model():
- """The NPU model serving chat, or None. It answers to its own names only."""
+ """The NPU model serving chat, or None. It answers to its own names only, and holds the
+ primary's seat: a request routed to a model kept alongside is not the NPU's."""
from core.inference.npu_backend import peek_npu_backend
+ if routed_slot.get() is not None:
+ return None
npu = peek_npu_backend()
return npu.loaded_model if npu is not None else None
@@ -10614,11 +10669,17 @@ async def _maybe_auto_switch_model(
)
# The reload-only sentinel means an omitted model, not a name.
named_model = requested_model if requested_model != _RELOAD_ONLY_MODEL else None
+ # The public preview serves its pinned checkpoint only, never a model kept alongside.
+ routed = await _route_to_extra_slot(
+ None if _auto_switch_opted_out(fastapi_request) else named_model
+ )
if account_access.managed_account():
if named_model:
await _require_named_model_access(named_model, fastapi_request)
elif account_access.resident_hidden("chat", _loaded_slot_ident()):
raise HTTPException(status_code = 404, detail = "Model not found")
+ if routed is not None:
+ return
async def _refuse_foreign_resident() -> None:
"""Serve a named model only if the resident is the caller's own or answers to the name."""
@@ -10647,6 +10708,7 @@ async def _maybe_auto_switch_model(
from core.inference.llama_keepwarm import (
get_last_unloaded_model,
inference_lifecycle_gate,
+ model_load_gate,
note_admitted_inference,
preview_swapped_since_entry,
)
@@ -10689,8 +10751,7 @@ async def _maybe_auto_switch_model(
return
# The public preview route opts out so a caller cannot switch away from the
# pinned preview checkpoint it just loaded.
- scope = getattr(fastapi_request, "scope", None)
- if isinstance(scope, dict) and scope.get(_DISABLE_OPENAI_AUTO_SWITCH_SCOPE_KEY):
+ if _auto_switch_opted_out(fastapi_request):
return
auto_switch_on = get_openai_auto_switch_enabled()
@@ -11118,7 +11179,7 @@ async def _maybe_auto_switch_model(
try:
# Hold the keep-warm gate across the swap so no new inference can
# start on the model while it is being torn down and replaced.
- async with inference_lifecycle_gate():
+ async with model_load_gate(), inference_lifecycle_gate():
# Re-read under the gate: the snapshot above predates the wait.
if ollama_target:
ollama_source_identity = await asyncio.to_thread(
@@ -11340,6 +11401,8 @@ _preview_resident_ident: Optional[str] = None
def _set_preview_resident(ident: Optional[str]) -> None:
global _preview_resident_ident
+ if routed_slot.get() is not None:
+ return # a model kept alongside never takes the preview seat
with _preview_slot_lock:
_preview_resident_ident = ident
@@ -11408,10 +11471,7 @@ def _loaded_slot_ident() -> Optional[str]:
llama_backend = get_llama_cpp_backend()
if llama_backend.is_loaded and llama_backend.model_identifier:
return str(llama_backend.model_identifier)
- from core.inference.npu_backend import peek_npu_backend
-
- npu = peek_npu_backend()
- npu_model = npu.loaded_model if npu is not None else None
+ npu_model = _resident_npu_model()
return npu_model.model_path if npu_model is not None else None
@@ -11422,6 +11482,8 @@ def release_chat_gpu_claim() -> bool:
from core.inference.llama_cpp import chat_load_active
def chat_idle() -> bool:
+ if model_slots.busy():
+ return False
llama = get_llama_cpp_backend()
# is_active, not is_loaded: a starting model holds VRAM, and an HF load has no process yet.
if llama.is_active or chat_load_active():
@@ -11434,7 +11496,37 @@ def release_chat_gpu_claim() -> bool:
getattr(backend, "loading_models", ()) or ()
)
- return release_if(CHAT, chat_idle)
+ return model_slots.in_slot(None, lambda: release_if(CHAT, chat_idle))
+
+
+def _release_chat_for_zero_vram_primary() -> None:
+ """A primary that holds no VRAM drops CHAT, unless a model kept alongside still holds it."""
+ from core.inference.gpu_arbiter import CHAT, release, release_if
+
+ if not model_slots.slots and not model_slots.stuck and model_slots.loading is None:
+ release(CHAT)
+ return
+ release_if(CHAT, lambda: not model_slots.holds_vram())
+
+
+def release_chat_after_kept_models() -> bool:
+ """Once kept models are gone, drop CHAT if nothing is left or only a primary that holds no VRAM."""
+ from core.inference.gpu_arbiter import CHAT, release_if
+ from core.inference.llama_cpp import chat_load_active
+
+ if release_chat_gpu_claim():
+ return True
+
+ def zero_vram_primary_only() -> bool:
+ llama = get_llama_cpp_backend()
+ return (
+ not model_slots.holds_vram()
+ and llama.is_loaded
+ and llama.holds_no_vram
+ and not chat_load_active()
+ )
+
+ return model_slots.in_slot(None, lambda: release_if(CHAT, zero_vram_primary_only))
def reap_dead_managed_engine(backend) -> None:
@@ -11488,6 +11580,7 @@ async def load_model_for_preview(
account_access.require_idle_other_accounts()
from core.inference.llama_keepwarm import (
inference_lifecycle_gate,
+ model_load_gate,
note_preview_swap,
note_preview_swap_begin,
note_preview_swap_end,
@@ -11504,7 +11597,7 @@ async def load_model_for_preview(
await _acquire_swap_gate()
_swap_begun = False
try:
- async with inference_lifecycle_gate():
+ async with model_load_gate(), inference_lifecycle_gate():
# Preview loads bypass load_model(), so re-apply its sidecar guard:
# a public preview must not complete a load while a transformers
# install has reserved the swap, or the installer later aborts.
@@ -16429,6 +16522,23 @@ def _raise_if_sidecar_swap_in_progress() -> None:
)
+def _active_generations_conflict(action: str, running: int, thread_ids) -> HTTPException:
+ return HTTPException(
+ status_code = 409,
+ detail = {
+ "error": "active_generations",
+ "message": (
+ f"{action} would stop {running} chat"
+ f"{'s' if running != 1 else ''} that "
+ f"{'are' if running != 1 else 'is'} still generating. "
+ "Stop them first, or retry with force_cancel_active."
+ ),
+ "running": running,
+ "thread_ids": thread_ids,
+ },
+ )
+
+
def _raise_or_cancel_active_generations(
*,
force: bool,
@@ -16457,29 +16567,19 @@ def _raise_or_cancel_active_generations(
# first would end the caller's chats for nothing. Keyed on account_scope(), whose count
# drops while a deactivated account's generation still holds the GPU.
require_no_foreign_generations(scope)
- if not active_generations.count(scope):
+ elsewhere = model_slots.all_generations()
+ if not active_generations.count(scope, elsewhere):
return 0
if not force:
- thread_ids = active_generations.active_thread_ids(scope)
- running = active_generations.count(scope)
- raise HTTPException(
- status_code = 409,
- detail = {
- "error": "active_generations",
- "message": (
- f"{action} would stop {running} chat"
- f"{'s' if running != 1 else ''} that "
- f"{'are' if running != 1 else 'is'} still generating. "
- "Stop them first, or retry with force_cancel_active."
- ),
- "running": running,
- "thread_ids": thread_ids,
- },
+ raise _active_generations_conflict(
+ action,
+ active_generations.count(scope, elsewhere),
+ active_generations.active_thread_ids(scope, elsewhere),
)
if not cancel:
# Refusal-only pass: the caller cancels later, once nothing can still reject the load.
return 0
- cancelled = active_generations.cancel_all(scope)
+ cancelled = active_generations.cancel_all(scope, elsewhere)
if cancelled:
logger.info(
"model_swap_cancelled_active_generations",
@@ -16678,7 +16778,9 @@ def _unload_may_evict(model_path: str) -> bool:
@studio_router.get("/active-generations")
async def get_active_generations(
- fastapi_request: Request, current_subject: str = Depends(get_current_subject)
+ fastapi_request: Request,
+ current_subject: str = Depends(get_current_subject),
+ model: Optional[str] = None,
):
"""Conversations currently generating, plus how many can decode at once.
@@ -16688,7 +16790,19 @@ async def get_active_generations(
requested --parallel; chats beyond it queue rather than fail.
"""
scope = account_access.account_scope()
- entries = active_generations.snapshot(scope)
+ # ``model``: only the chats an unload of that model stops, the same split its scoped 409 uses.
+ exclude, only, llama = (), None, None
+ if model and model_slots.slots:
+ slot = await asyncio.to_thread(
+ model_slots.serving_slot, model, model_slots.visible(), _loaded_satisfies
+ )
+ if slot is not None:
+ only, llama = set(slot.generations), slot.llama
+ elif await asyncio.to_thread(model_slots.in_slot, None, lambda: _loaded_satisfies(model)):
+ exclude = model_slots.all_generations()
+ else:
+ only = set() # no loaded model by that name: its unload stops nothing
+ entries = active_generations.snapshot(scope, exclude, only)
# A tracker's model can be a native local path (the legacy stream records active_model_name
# verbatim); redact here, the one place that serialises it.
for _entry in entries:
@@ -16696,13 +16810,13 @@ async def get_active_generations(
_entry["model"] = redact_native_paths(_entry["model"])
slots = 1
try:
- slots = _openai_llama_admission_capacity(fastapi_request, get_llama_cpp_backend())
+ slots = _openai_llama_admission_capacity(fastapi_request, llama or get_llama_cpp_backend())
except Exception:
slots = int(getattr(fastapi_request.app.state, "llama_parallel_slots", 1) or 1)
return {
"active": entries,
"count": len(entries),
- "thread_ids": active_generations.active_thread_ids(scope),
+ "thread_ids": active_generations.active_thread_ids(scope, exclude, only),
"parallel_slots": max(1, int(slots)),
}
@@ -17177,6 +17291,7 @@ async def load_model_gated(
current_subject: str,
*,
user_initiated: bool = False,
+ current_request_counted: bool = False,
):
"""Everything ``POST /load`` does except the tunnel-safe padding.
@@ -17189,8 +17304,11 @@ async def load_model_gated(
# then gets unloaded by the pre-swap teardown. Rechecked under the gate: an
# install can reserve while this request queues on the gate, so the pre-gate
# check alone is only a fast path.
- from core.inference.llama_keepwarm import inference_lifecycle_gate
+ from core.inference.llama_cpp import GpuMemoryShortError
+ from core.inference.llama_keepwarm import inference_lifecycle_gate, model_load_gate
+ extra = None
+ evicted: list[str] = []
attempt = _begin_load_attempt(request, current_subject)
with _scoped_load_attempts_lock:
_pending_load_attempts[attempt.token] = attempt
@@ -17205,32 +17323,209 @@ async def load_model_gated(
# Hold the lifecycle gate across the load so idle auto-unload can't unload the
# model mid-load. Auto-switch calls the tracked impl directly since it already
# holds this gate.
- async with inference_lifecycle_gate():
- _raise_if_sidecar_swap_in_progress()
- # The active-generation gate runs inside _load_model_impl, once it knows this is a real
- # reload, and still under the lifecycle gate so the check stays atomic with the teardown.
- response = await _run_tracked_load_model_impl(
- request,
- fastapi_request,
- current_subject,
- attempt = attempt,
- on_reload_confirmed = lambda *, cancel: _raise_or_cancel_active_generations(
- force = request.force_cancel_active,
- action = "Loading a model",
- cancel = cancel,
- ),
- )
+ async with model_load_gate():
+ # Chosen under the gate, so two loads cannot both claim an empty primary.
+ extra = await _select_load_slot(request)
+ new_slot = extra is not None and extra not in model_slots.slots
+ if new_slot:
+ model_slots.slots.append(extra)
+ if extra is not None:
+ model_slots.loading = (extra, request.model_path)
+ extra.llama._llama_update_in_progress = getattr(
+ _llama_cpp_backend, "_llama_update_in_progress", False
+ )
+ async with nullcontext() if new_slot else inference_lifecycle_gate():
+ _raise_if_sidecar_swap_in_progress()
+ # The 409 gate runs inside _load_model_impl, under the lifecycle gate, atomic with teardown.
+ if new_slot:
+ reload_gate = None
+ elif extra is not None:
+ reload_gate = functools.partial(
+ _raise_or_cancel_slot_generations,
+ extra,
+ force = request.force_cancel_active,
+ action = "Reloading this model",
+ )
+ else:
+ reload_gate = functools.partial(
+ _raise_or_cancel_active_generations,
+ force = request.force_cancel_active,
+ action = "Loading a model",
+ )
+
+ while True:
+ try:
+ response = await _run_tracked_load_model_impl(
+ request,
+ fastapi_request,
+ current_subject,
+ attempt = attempt,
+ current_request_counted = current_request_counted,
+ on_reload_confirmed = reload_gate,
+ )
+ break
+ except (GpuMemoryShortError, HTTPException) as exc:
+ if not isinstance(exc, GpuMemoryShortError):
+ # A non-GGUF load reports no fit up front: its out-of-memory failure
+ # makes room the same way, one idle kept model at a time.
+ if extra is None or not _ran_out_of_memory(exc):
+ raise
+ exc = GpuMemoryShortError(str(exc.detail))
+ # No chat may start on a victim between its pick and teardown.
+ async with inference_lifecycle_gate() if new_slot else nullcontext():
+ dropped = await asyncio.to_thread(
+ model_slots.evict, extra, exc.short_mib, exc.gpu_indices
+ )
+ if not dropped:
+ if exc.capped:
+ request = request.model_copy(update = {"force_alongside": True})
+ continue
+ # Fits nowhere beside the others: replace the active model, as before,
+ # unless its chats refuse that, checked before anything else changes.
+ routed_slot.set(None)
+ _raise_or_cancel_active_generations(
+ force = request.force_cancel_active,
+ action = "Loading a model",
+ cancel = False,
+ )
+ await asyncio.to_thread(model_slots.drop, extra)
+ model_slots.loading, extra = None, None
+ replaced = _primary_model_label()
+ request = request.model_copy(
+ update = {"alongside": False, "force_alongside": False}
+ )
+ async with inference_lifecycle_gate() if new_slot else nullcontext():
+ _raise_or_cancel_active_generations(
+ force = request.force_cancel_active, action = "Loading a model"
+ )
+ # As a pick does before a switch: the active model goes first, so
+ # the new one is placed on a GPU it no longer shares.
+ await asyncio.to_thread(_unload_primary)
+ response = await _run_tracked_load_model_impl(
+ request,
+ fastapi_request,
+ current_subject,
+ attempt = attempt,
+ current_request_counted = current_request_counted,
+ on_reload_confirmed = None,
+ )
+ evicted += [replaced] if replaced else []
+ break
+ evicted += [_model_key(s.request) for s in dropped if s.request is not None]
+ logger.info(
+ "Unloaded %d model(s) loaded alongside to fit %s",
+ len(dropped),
+ request.model_path,
+ )
+ extra.llama._last_kill_monotonic = time.monotonic()
+ if extra is not None:
+ from core.inference.llama_keepwarm import _note_activity
+
+ extra.request = request
+ extra.last_used = time.monotonic()
+ # A fresh load is use: without it the idle loop drops the slot on its next tick.
+ _note_activity()
# Record provenance only once the model is resident, and here rather than
# inside the impl so the already-loaded fast paths are covered too. Preview
# keeps the False default: only an explicit UI load pins. Outside the gate:
# it is a plain attribute write and holding the gate for it would only widen
# the window that blocks unload.
get_llama_cpp_backend()._loaded_by_user_action = user_initiated
+ if evicted and isinstance(response, LoadResponse):
+ response.evicted = evicted
return response
finally:
- with _scoped_load_attempts_lock:
- _pending_load_attempts.pop(attempt.token, None)
- _finish_load_attempt(attempt)
+ try:
+ if extra is not None:
+ model_slots.loading = None
+ if extra not in model_slots.slots or not model_slots.in_use(extra):
+ await asyncio.to_thread(model_slots.drop, extra)
+ await asyncio.to_thread(release_chat_gpu_claim)
+ finally:
+ # A slot that would not stop stays stuck, but this attempt is over either way.
+ with _scoped_load_attempts_lock:
+ _pending_load_attempts.pop(attempt.token, None)
+ _finish_load_attempt(attempt)
+
+
+def _ran_out_of_memory(exc: HTTPException) -> bool:
+ return exc.status_code == 500 and "out of memory" in str(exc.detail).lower()
+
+
+def _gate_kept_models(request: LoadRequest, cancel: bool = True) -> None:
+ for slot in list(model_slots.slots):
+ _raise_or_cancel_slot_generations(
+ slot, force = request.force_cancel_active, cancel = cancel, action = "Loading a model"
+ )
+
+
+async def _retire_kept_models(request: LoadRequest) -> None:
+ """vLLM and SGLang reserve the GPU, so a load of either takes the kept models down too: gated
+ before any chat is cancelled, torn down once nothing can still reject the load."""
+ _gate_kept_models(request)
+ await asyncio.to_thread(model_slots.unload_extra_models, strict = True)
+
+
+def _unload_primary() -> None:
+ llama = get_llama_cpp_backend()
+ if llama.is_active:
+ llama.unload_model()
+ backend = _peek_inference_backend()
+ name = getattr(backend, "active_model_name", None)
+ if name:
+ backend.unload_model(name)
+
+
+def _primary_model_label() -> Optional[str]:
+ llama = get_llama_cpp_backend()
+ if getattr(llama, "is_loaded", False):
+ return _llama_public_model_id(llama)
+ return getattr(_peek_inference_backend(), "active_model_name", None)
+
+
+async def _select_load_slot(request: LoadRequest) -> Optional[_ExtraSlot]:
+ """The extra slot already serving the model, or a new one for ``alongside``. None is the primary."""
+ from core.inference.npu_backend import is_npu_model_path
+
+ # The NPU backend is one per process and replaces the primary, so it never takes a slot. vLLM
+ # and SGLang reserve a fixed share of the GPU up front, so neither shares it with a kept model.
+ if is_npu_model_path(request.model_path) or request.engine != "auto":
+ routed_slot.set(None)
+ return None
+ requested = _model_key(request)
+ slot = await _route_to_extra_slot(requested)
+ if slot is None and request.gguf_variant:
+ # Another quant replaces its repo in place: requests name the repo, so a copy beside it is unreachable.
+ slot = await _route_to_extra_slot(request.model_path)
+ if slot is not None or await asyncio.to_thread(_loaded_satisfies, request.model_path):
+ return slot
+ if slot is not None:
+ return slot
+ from utils.multi_model_settings import get_multi_model_enabled
+
+ # Off unless turned on in Settings. Off, a kept model left busy when it was turned off goes
+ # at the first load after it is idle.
+ if not await asyncio.to_thread(get_multi_model_enabled):
+ if model_slots.slots:
+ await asyncio.to_thread(model_slots.unload_idle)
+ return None
+ if not request.alongside:
+ return None
+ orchestrator = _peek_inference_backend()
+ if getattr(orchestrator, "_managed_engine", None) is not None:
+ return None
+ occupied = _llama_cpp_backend.is_active or getattr(orchestrator, "active_model_name", None)
+ if not occupied or await asyncio.to_thread(_loaded_satisfies, requested):
+ return None
+ slot = _ExtraSlot(
+ await asyncio.to_thread(LlamaCppBackend),
+ await asyncio.to_thread(InferenceOrchestrator),
+ current_account_id(),
+ )
+ slot.llama._owns_pidfile = False
+ register_serving_backend(slot.llama)
+ routed_slot.set(slot)
+ return slot
def _restore_alias_if_failed_load_left_the_prior_model(
@@ -17368,9 +17663,11 @@ def _npu_load_response(resident, status: str) -> LoadResponse:
async def _unload_npu_before_local_load() -> None:
- """Unload the NPU model before loading another local model."""
+ """Unload the NPU model before loading another local model into the primary's seat."""
from core.inference.npu_backend import peek_npu_backend
+ if routed_slot.get() is not None:
+ return
npu = peek_npu_backend()
if npu is not None and npu.is_loaded:
logger.info("Unloading the NPU model before loading a local model")
@@ -17500,7 +17797,7 @@ async def _load_model_impl(
_cached_before_this_load = _repo_is_in_the_hub_cache(request.model_path)
_cache_footprint_before = _hub_cache_footprint(request.model_path)
_lora_base_before_this_load = _lora_base_already_in_the_hub_cache(request.model_path)
- from core.inference.llama_cpp import LlamaServerNotFoundError
+ from core.inference.llama_cpp import GpuMemoryShortError, LlamaServerNotFoundError
def _raise_if_scoped_load_cancelled() -> None:
if load_cancel_event is not None and load_cancel_event.is_set():
@@ -17683,7 +17980,6 @@ async def _load_model_impl(
from core.inference.gpu_arbiter import (
acquire_for_request,
current_owner,
- release,
CHAT,
DIFFUSION,
VIDEO,
@@ -17692,6 +17988,12 @@ async def _load_model_impl(
# ── Already-loaded check: skip reload if the exact model is active ──
backend = await asyncio.to_thread(get_inference_backend)
llama_backend = get_llama_cpp_backend()
+ replacing = routed_slot.get() is None
+ serving = bool(
+ getattr(llama_backend, "is_active", getattr(llama_backend, "is_loaded", False))
+ or getattr(backend, "active_model_name", None)
+ or (replacing and _resident_npu_model() is not None)
+ )
# Resolve once so dedupe, admission and launch use the same slot count.
_n_parallel = _resolve_parallel_slots(request, fastapi_request)
@@ -17729,7 +18031,8 @@ async def _load_model_impl(
_set_preview_resident(None)
if ollama_advertised_id:
llama_backend._openai_advertised_id = ollama_advertised_id
- account_access.join_resident("chat")
+ if replacing:
+ account_access.join_resident("chat")
return _gguf_load_response(
llama_backend,
"already_loaded",
@@ -17805,7 +18108,8 @@ async def _load_model_impl(
# Owns no GPU, so the arbiter would cancel a generation for nothing.
if not _resident_audio_holds_no_gpu(backend):
await asyncio.to_thread(acquire_for_request, CHAT)
- account_access.join_resident("chat")
+ if replacing:
+ account_access.join_resident("chat")
return LoadResponse(
engine = request.engine,
engine_parallelism = request.engine_parallelism,
@@ -17977,6 +18281,8 @@ async def _load_model_impl(
)
if speech_codec_path is not None:
gguf_intent = replace(gguf_intent, audio_codec_path = speech_codec_path)
+ if not replacing and not request.force_alongside:
+ gguf_intent = replace(gguf_intent, refuse_partial_gpu_fit = True)
same_loaded_model = llama_backend.matches_load_source(gguf_intent)
_loaded_companion_roots = tuple(
getattr(llama_backend, "_openai_gguf_companion_roots", ()) or ()
@@ -18022,9 +18328,11 @@ async def _load_model_impl(
# Config-resolved dedupe must run first: a duplicate must not refuse/cancel active chats.
# Refusal is non-destructive; defer forced cancellation past every remaining rejection.
account_access.require_idle_other_accounts()
- if on_reload_confirmed is not None:
+ if serving and on_reload_confirmed is not None:
on_reload_confirmed(cancel = False)
- cancel_pending = on_reload_confirmed is not None and bool(request.force_cancel_active)
+ cancel_pending = (
+ serving and on_reload_confirmed is not None and bool(request.force_cancel_active)
+ )
if not config.is_gguf and _mlx_distributed_launch_detected():
raise HTTPException(
@@ -18222,6 +18530,8 @@ async def _load_model_impl(
handoff_kwargs["expected_current"] = expected_native_owner
if not allow_gpu_owner_eviction:
handoff_kwargs["allow_evict"] = False
+ if not replacing:
+ handoff_kwargs["alongside"] = True
await asyncio.to_thread(
acquire_for_request,
CHAT,
@@ -18285,10 +18595,11 @@ async def _load_model_impl(
# Drain active generations first (the lifecycle gate blocks new starts); a forced swap
# excludes the ones it is about to cancel rather than waiting them out.
- await _wait_for_model_switch_idle(
- current_request_counted = current_request_counted,
- cancel_pending = cancel_pending,
- )
+ if replacing and serving:
+ await _wait_for_model_switch_idle(
+ current_request_counted = current_request_counted,
+ cancel_pending = cancel_pending,
+ )
# Decisive recheck, and the last thing that can reject this load, so it runs BEFORE the
# cancel: rejecting after would stop every chat for nothing.
_raise_if_sidecar_swap_in_progress()
@@ -18296,7 +18607,7 @@ async def _load_model_impl(
# Point of no return for the GGUF path: nothing left can reject this load, so stop the
# chats the swap interrupts (or refuse, if the caller never opted in).
_raise_if_scoped_load_cancelled()
- if on_reload_confirmed is not None:
+ if serving and on_reload_confirmed is not None:
on_reload_confirmed(cancel = True)
# Let the cancelled generations unwind before the teardown; no check follows, so this cannot
@@ -18411,9 +18722,9 @@ async def _load_model_impl(
"so the load was cancelled. Unload that model, then try again."
),
)
- if not chat_load_needs_gpu:
+ if replacing and not chat_load_needs_gpu:
# Drop the stale CHAT claim after any zero-VRAM load.
- await asyncio.to_thread(release, CHAT)
+ await asyncio.to_thread(_release_chat_for_zero_vram_primary)
logger.info(
f"Loaded GGUF model via llama-server: {model_log_label if native_grant_backed else config.identifier}"
@@ -18428,7 +18739,8 @@ async def _load_model_impl(
llama_backend._openai_gguf_companion_roots = tuple(request._gguf_companion_roots)
llama_backend._openai_gguf_companion_state = gguf_companion_state
- await asyncio.to_thread(note_model_loaded, llama_backend)
+ if replacing:
+ await asyncio.to_thread(note_model_loaded, llama_backend)
# None elsewhere: only an Ollama load has an identifier no client should be handed.
llama_backend._openai_advertised_id = ollama_advertised_id
@@ -18443,11 +18755,12 @@ async def _load_model_impl(
llama_backend._is_local_model = bool(native_grant_backed or config.is_local)
if _gguf_is_audio:
logger.info(f"GGUF model detected as audio: audio_type={_gguf_audio}")
- account_access.publish_resident(
- "chat",
- request.model_path,
- model_log_label if native_grant_backed else public_model_identifier,
- )
+ if replacing:
+ account_access.publish_resident(
+ "chat",
+ request.model_path,
+ model_log_label if native_grant_backed else public_model_identifier,
+ )
return _gguf_load_response(
llama_backend,
@@ -18465,16 +18778,24 @@ async def _load_model_impl(
_raise_if_sidecar_swap_in_progress()
llama_backend = get_llama_cpp_backend()
- await _wait_for_model_switch_idle(
- current_request_counted = current_request_counted,
- cancel_pending = cancel_pending,
- )
+ if replacing and serving:
+ await _wait_for_model_switch_idle(
+ current_request_counted = current_request_counted,
+ cancel_pending = cancel_pending,
+ )
_raise_if_sidecar_swap_in_progress()
# Point of no return for the Unsloth path: cancel only once nothing can still reject the load.
_raise_if_scoped_load_cancelled()
- if on_reload_confirmed is not None:
+ retire_kept = (
+ request.engine != "auto" and replacing and bool(model_slots.slots or model_slots.stuck)
+ )
+ if retire_kept:
+ _gate_kept_models(request, cancel = False)
+ if serving and on_reload_confirmed is not None:
on_reload_confirmed(cancel = True)
+ if retire_kept:
+ await _retire_kept_models(request)
# Let the cancelled generations unwind before the teardown; no check follows. Bounded like GGUF.
if cancel_pending:
@@ -18525,7 +18846,9 @@ async def _load_model_impl(
# this very load) and not before it (until the previous worker exits, this
# claim is all that stops a second pipeline allocating over a resident model).
# load_model fires it in between; the post-load release covers a re-taken claim.
- _release_chat_after_teardown = (lambda: release(CHAT)) if not chat_load_needs_gpu else None
+ _release_chat_after_teardown = (
+ _release_chat_for_zero_vram_primary if replacing and not chat_load_needs_gpu else None
+ )
anonymous_hf_kw = {"anonymous_hf_access": True} if anonymous_hf_access else {}
speech_codec_kw = (
{"audio_codec_path": speech_codec_path} if speech_codec_path is not None else {}
@@ -18605,10 +18928,10 @@ async def _load_model_impl(
"so the load was cancelled. Unload that model, then try again."
),
)
- if not chat_load_needs_gpu:
+ if replacing and not chat_load_needs_gpu:
# This load replaced whatever held CHAT; leaving the claim makes the next
# Images/Video acquire evict a model that never used the GPU.
- await asyncio.to_thread(release, CHAT)
+ await asyncio.to_thread(_release_chat_for_zero_vram_primary)
# Stamped here, not in backend.load_model: that entry is built in the load
# subprocess and only a fixed model_info mirror crosses back, so it would
@@ -18636,12 +18959,13 @@ async def _load_model_impl(
# poll clears it (and never, while idle-unload is off).
from core.inference.llama_keepwarm import note_model_loaded
- note_model_loaded()
- account_access.publish_resident(
- "chat",
- request.model_path,
- model_log_label if native_grant_backed else config.identifier,
- )
+ if replacing:
+ note_model_loaded()
+ account_access.publish_resident(
+ "chat",
+ request.model_path,
+ model_log_label if native_grant_backed else config.identifier,
+ )
# Load inference configuration parameters
inference_config = load_inference_config(config.identifier)
@@ -18750,6 +19074,8 @@ async def _load_model_impl(
logger.warning("Rejected inference GPU selection: %s", e)
# User-facing validation (e.g. "Invalid gpu_ids [99]"): redact paths, keep detail.
raise HTTPException(status_code = 400, detail = redacted_msg)
+ except GpuMemoryShortError:
+ raise # load_model_gated makes room and retries
except LlamaServerNotFoundError as e:
# Missing GGUF runtime: 400 with the install message, not a generic 500.
logger.warning("GGUF runtime missing while loading '%s': %s", model_log_label, e)
@@ -19759,6 +20085,7 @@ async def install_latest_transformers_route(
# requests stay blocked in the middleware, so neither is subtracted here.
from core.inference.llama_keepwarm import (
inference_lifecycle_gate,
+ model_load_gate,
note_model_unloaded,
other_inference_request_count,
)
@@ -19827,6 +20154,7 @@ async def install_latest_transformers_route(
stopped = backend._shutdown_subprocess()
if not stopped or worker_alive():
raise RuntimeError("Inference worker still alive before the transformers swap")
+ model_slots.stop_orchestrator_workers()
def _run_install() -> dict:
# Owns the reservation from here: releasing in the thread, not the route,
@@ -19849,7 +20177,7 @@ async def install_latest_transformers_route(
async def _gated_install() -> dict:
# Held by THIS task, not the request coroutine: a cancelled POST unwinding an
# `async with` here would drop the only guard /load honors mid-install.
- async with inference_lifecycle_gate():
+ async with model_load_gate(), inference_lifecycle_gate():
_active_now = (
getattr(backend, "active_model_name", None),
getattr(backend, "load_generation", 0),
@@ -20287,22 +20615,53 @@ async def _unload_model_impl(request: UnloadRequest, current_subject: str):
Unload a model from memory.
Routes to the correct backend (llama-server for GGUF, Unsloth otherwise).
"""
- if (
- request.cancel_load_request_id is None
- and account_access.managed_account()
- and account_access.release_shared_resident("chat")
- ):
- # Other accounts still share the model: only this account's share ends, nothing is unloaded.
- return UnloadResponse(status = "unloaded", model = request.model_path)
- account_access.require_resident_control(
- "chat", _loaded_slot_ident() if account_access.managed_account() else None
+ extra = (
+ await _route_to_extra_slot(request.model_path)
+ if request.cancel_load_request_id is None
+ else None
)
+ if extra is not None:
+ from core.inference.llama_keepwarm import inference_lifecycle_gate, model_load_gate
+
+ # Load gate first, as a load takes them: a reload of this model holds its ref while it
+ # waits on the lifecycle gate, so taking that alone would wait on a ref that cannot drain.
+ async with model_load_gate(), inference_lifecycle_gate():
+ _raise_or_cancel_slot_generations(extra, force = request.force_cancel_active)
+ # Past this unload's own ref, a ref is a request already routed here: let it start
+ # or finish, then gate whatever it started, as above.
+ deadline = time.monotonic() + _POST_CANCEL_DRAIN_TIMEOUT_S
+ while (extra.generations or extra.refs > 1) and time.monotonic() < deadline:
+ await asyncio.sleep(0.05)
+ _raise_or_cancel_slot_generations(extra, force = request.force_cancel_active)
+ await asyncio.to_thread(model_slots.drop, extra)
+ await asyncio.to_thread(release_chat_after_kept_models)
+ api_monitor.record_lifecycle(
+ event = "unload", model = _lifecycle_model_label(request.model_path), reason = "manual"
+ )
+ return UnloadResponse(status = "unloaded", model = request.model_path)
+ filling = model_slots.visible_loading()
+ if filling and _names_the_loading_model(filling[1], request.model_path):
+ routed_slot.set(filling[0])
+ on_primary = routed_slot.get() is None
+ if on_primary:
+ if (
+ request.cancel_load_request_id is None
+ and account_access.managed_account()
+ and account_access.release_shared_resident("chat")
+ ):
+ return UnloadResponse(status = "unloaded", model = request.model_path)
+ account_access.require_resident_control(
+ "chat", _loaded_slot_ident() if account_access.managed_account() else None
+ )
# A deliberate unload means "stay unloaded": drop any idle reload stash so the
# next /v1 request can't resurrect this model. The idle loop unloads via the
# backend directly (not this route), so clearing here never fights keep-warm.
from core.inference.llama_keepwarm import inference_lifecycle_gate, note_model_unloaded
from core.inference.npu_backend import MODEL_PREFIX, is_npu_model_path, peek_npu_backend
+ if not on_primary:
+ note_model_unloaded = lambda: None
+
try:
if is_npu_model_path(request.model_path):
if account_access.managed_account():
@@ -20426,6 +20785,8 @@ async def _unload_model_impl(request: UnloadRequest, current_subject: str):
note_model_unloaded()
logger.info(f"Cancelled in-flight GGUF load: {request.model_path}")
return UnloadResponse(status = "unloaded", model = request.model_path)
+ if not on_primary:
+ return UnloadResponse(status = "unloaded", model = request.model_path)
# Same gate as /load: refusal only, so a non-forced unload fails fast before queueing on the
# lifecycle gate. Skipped when no teardown branch can fire, or a request naming a model
@@ -21059,6 +21420,7 @@ async def get_llama_flags(
async def inference_status(
current_subject: str = Depends(get_current_subject),
via_api_key: bool = Depends(authenticated_via_api_key),
+ model: Optional[str] = None,
):
"""`GET /api/inference/status`, redacted the way the image and video status routes are.
@@ -21072,25 +21434,66 @@ async def inference_status(
"""
from hub.utils.host_paths import redact_host_paths
return redact_host_paths(
- await get_status(current_subject = current_subject),
+ await get_status(current_subject = current_subject, model = model),
via_api_key = via_api_key,
)
-async def get_status(current_subject: str):
+async def get_status(current_subject: str, model: Optional[str] = None):
+ """Status of the slot serving ``model`` (the primary one by default), listing every loaded model."""
+ slot = await _route_to_extra_slot(model)
+ response = await _slot_status(current_subject)
+ if isinstance(response, InferenceStatusResponse):
+ active = response.active_model
+ response.serving = [active] if active else []
+ response.serving_checkpoints = [response.model_identifier or active] if active else []
+ for other in (None, *model_slots.visible()):
+ if other is not slot:
+ entries, checkpoints = await asyncio.to_thread(
+ model_slots.in_slot, other, _slot_entries_and_checkpoints
+ )
+ response.loaded += [e["id"] for e in entries if e["id"] not in response.loaded]
+ for e in entries:
+ # By checkpoint: two local files can share a label.
+ checkpoint = checkpoints.get(e["id"]) or e["id"]
+ if checkpoint not in response.serving_checkpoints:
+ response.serving.append(e["id"])
+ response.serving_checkpoints.append(checkpoint)
+ return response
+
+
+def _slot_entries_and_checkpoints() -> "tuple[list[dict], dict[str, str]]":
+ """This slot's /v1/models entries, and the checkpoint each local one is loaded from."""
+ checkpoints: dict[str, str] = {}
+ llama = get_llama_cpp_backend()
+ if getattr(llama, "is_loaded", False):
+ public = _llama_public_model_id(llama)
+ if public:
+ checkpoints[public] = _llama_status_model_ids(llama)[1] or public
+ backend = _peek_inference_backend()
+ name = getattr(backend, "active_model_name", None)
+ if name:
+ checkpoints[_orchestrator_public_model_id(backend) or name] = name
+ return _slot_model_objects(), checkpoints
+
+
+async def _slot_status(current_subject: str):
"""
Get current inference backend status.
Reports whichever backend (Unsloth or llama-server) is active.
"""
+ on_primary = routed_slot.get() is None
backend = _peek_inference_backend()
if getattr(backend, "_managed_engine", None) is not None:
await asyncio.to_thread(reap_dead_managed_engine, backend)
- if account_access.resident_hidden("chat"):
+ if on_primary and account_access.resident_hidden("chat"):
return account_access.hidden_chat_status_response()
try:
llama_backend = get_llama_cpp_backend()
- if account_access.managed_account() and account_access.resident_hidden(
- "chat", _loaded_slot_ident()
+ if (
+ on_primary
+ and account_access.managed_account()
+ and account_access.resident_hidden("chat", _loaded_slot_ident())
):
return account_access.hidden_chat_status_response()
@@ -21130,7 +21533,7 @@ async def get_status(current_subject: str):
from core.inference.npu_backend import peek_npu_backend
_npu = peek_npu_backend()
- _npu_resident = _npu.resident() if _npu is not None else None
+ _npu_resident = _npu.resident() if _npu is not None and routed_slot.get() is None else None
if _npu_resident is not None:
_npu_model = _npu_resident.model
return InferenceStatusResponse(
@@ -21371,14 +21774,15 @@ async def get_load_progress(current_subject: str = Depends(get_current_subject))
Returns an empty payload (``phase=null, bytes=0``) when no load is in
flight. The frontend should stop polling once ``phase`` becomes ``ready``.
"""
- if account_access.resident_hidden("chat"):
+ loading = model_slots.visible_loading()
+ if loading is None and account_access.resident_hidden("chat"):
return account_access.hidden_resident_response()
try:
- backend = _peek_inference_backend()
+ backend = loading[0].orchestrator if loading else _peek_inference_backend()
managed = getattr(backend, "_managed_engine", None)
if managed is not None and backend.loading_models:
return LoadProgressResponse(phase = managed.phase)
- llama_backend = get_llama_cpp_backend()
+ llama_backend = loading[0].llama if loading else get_llama_cpp_backend()
progress = llama_backend.load_progress()
if progress is None:
return LoadProgressResponse()
@@ -27630,7 +28034,7 @@ async def produce_openai_chat_completions(
from core.inference.npu_backend import peek_npu_backend
_npu = peek_npu_backend()
- if _npu is not None and _npu.is_loaded:
+ if _npu is not None and _npu.is_loaded and routed_slot.get() is None:
_refuse_unused_mcp_image(_mcp_image)
return await _npu_chat_completions(payload, request, current_subject)
@@ -32420,14 +32824,24 @@ _OWNED_BY = "unsloth-studio"
def _openai_model_objects() -> list[dict]:
+ return [
+ entry
+ for slot in (None, *model_slots.visible())
+ for entry in model_slots.in_slot(slot, _slot_model_objects)
+ ]
+
+
+def _slot_model_objects() -> list[dict]:
"""The model objects GET /v1/models exposes, one per loaded local backend still answering to
the id it is advertised under.
Shared by the LIST and RETRIEVE handlers so both report the same ids and
field shape.
"""
- if account_access.managed_account() and account_access.resident_hidden(
- "chat", _loaded_slot_ident()
+ if (
+ routed_slot.get() is None
+ and account_access.managed_account()
+ and account_access.resident_hidden("chat", _loaded_slot_ident())
):
return []
models: list[dict] = []
@@ -32436,7 +32850,7 @@ def _openai_model_objects() -> list[dict]:
from core.inference.npu_backend import peek_npu_backend
_npu = peek_npu_backend()
- _npu_resident = _npu.resident() if _npu is not None else None
+ _npu_resident = _npu.resident() if _npu is not None and routed_slot.get() is None else None
if _npu_resident is not None:
_npu_model = _npu_resident.model
_npu_entry = {
@@ -33374,8 +33788,6 @@ async def openai_completions(request: Request, current_subject: str = Depends(ge
Proxies to the running llama-server's ``/v1/completions``. Only available
when a GGUF model is loaded.
"""
- llama_backend = get_llama_cpp_backend()
-
# Reject a request with no prompt before any automatic load so an invalid request never
# swaps or reloads the resident model (as chat/embeddings already validate before
# switching). Gate on every automatic-load trigger, and on a preview-owned slot the switch
@@ -33404,6 +33816,7 @@ async def openai_completions(request: Request, current_subject: str = Depends(ge
# Opt-in: load the requested local GGUF before the loaded-state check.
body = await _auto_switch_from_request_body(request, current_subject, gguf_only = True)
+ llama_backend = get_llama_cpp_backend()
if not llama_backend.is_loaded:
_status, _detail = await _no_model_loaded_error(
"No GGUF model loaded. Load a GGUF model first.",
@@ -34136,6 +34549,13 @@ def _embeddings_input_present(body: dict) -> bool:
async def openai_embeddings(request: Request, current_subject: str = Depends(get_current_subject)):
"""OpenAI-compatible embeddings: the resident embedding GGUF when one is loaded,
else Studio's configured embedding model."""
+ if model_slots.slots:
+ try:
+ await _route_to_extra_slot(_raw_body_model(await request.json()))
+ except (json.JSONDecodeError, ValueError):
+ pass
+ else:
+ routed_slot.set(None)
llama_backend = get_llama_cpp_backend()
# Reject a request with no input before any automatic load so an invalid request never
# swaps or reloads the resident model (as chat/responses/messages already validate before
@@ -34180,6 +34600,7 @@ async def openai_embeddings(request: Request, current_subject: str = Depends(get
if default_body is not None and not await asyncio.to_thread(_stashed_gguf_embeds):
return await _studio_embeddings(request, default_body, current_subject)
body = await _auto_switch_from_request_body(request, current_subject, gguf_only = True)
+ llama_backend = get_llama_cpp_backend()
if not llama_backend.is_loaded:
# With the slot empty _reject_unservable_model defers to _no_model_loaded_error, so
# without this the fallback would answer a decisive repo:QUANT this server does not
@@ -37516,7 +37937,7 @@ async def _mlx_count_chat_tokens(payload, request = None) -> Optional[JSONRespon
# Re-checked immediately before the only work that takes the orchestrator's lock:
# everything since the endpoint's entry check awaits, so a chat can have started in
# the gap and would then wait on this count. The GGUF path re-checks for this reason.
- if active_generations.count() > 0:
+ if model_slots.routed_generation_count() > 0:
raise HTTPException(
status_code = 503,
detail = "Cannot count tokens while a generation is in progress.",
@@ -37578,7 +37999,8 @@ async def chat_count_tokens(
# registers external-provider runs too, so those decline a count they could have served;
# narrowing it means trusting a kind/model field to decide whether to work next to a decode, and
# being wrong there costs inference time while over-refusing only costs a redraw.
- if active_generations.count() > 0:
+ await _route_to_extra_slot(payload.model)
+ if model_slots.routed_generation_count() > 0:
raise HTTPException(
status_code = 503,
detail = "Cannot count tokens while a generation is in progress.",
@@ -37642,7 +38064,7 @@ async def chat_count_tokens(
from core.inference.npu_backend import peek_npu_backend
_npu = peek_npu_backend()
- if _npu is not None and _npu.is_loaded:
+ if _npu is not None and _npu.is_loaded and routed_slot.get() is None:
# FastFlowLM has no tokenizer endpoint; each reply's usage still reports the prompt size.
raise HTTPException(
status_code = 503,
@@ -37818,7 +38240,7 @@ async def chat_count_tokens(
# Re-checked immediately before the only work that reaches llama-server, because everything
# between here and the entry check awaits, so a run can have started in the gap.
- if active_generations.count() > 0:
+ if model_slots.routed_generation_count() > 0:
raise HTTPException(
status_code = 503,
detail = "Cannot count tokens while a generation is in progress.",
@@ -37836,7 +38258,7 @@ async def chat_count_tokens(
chat_template_kwargs = _template_kwargs,
# Polled between /apply-template and /tokenize: admission and the work are separate
# steps, so a run starting in between is caught here and the second round trip is not.
- should_abort = lambda: active_generations.count() > 0,
+ should_abort = lambda: model_slots.routed_generation_count() > 0,
)
except CountAborted:
raise HTTPException(
@@ -38103,6 +38525,7 @@ async def anthropic_messages(
JSON).
"""
_admit_tool_access(payload)
+ await _route_to_extra_slot(_switch_model_for_payload(payload))
llama_backend = get_llama_cpp_backend()
# Default-off parity: with no automatic load possible and nothing loaded, 503
@@ -38249,6 +38672,7 @@ async def anthropic_messages(
else None
),
)
+ llama_backend = get_llama_cpp_backend()
if not llama_backend.is_loaded:
_status, _detail = await _no_model_loaded_error(
"No GGUF model loaded. Load a GGUF model first.",
diff --git a/studio/backend/routes/models.py b/studio/backend/routes/models.py
index e4c8d01e2db..58af39bf059 100644
--- a/studio/backend/routes/models.py
+++ b/studio/backend/routes/models.py
@@ -2015,39 +2015,34 @@ async def list_models(current_subject: str = Depends(get_current_subject)):
if await asyncio.to_thread(account_access.model_visible, m)
]
- loaded_models = []
- hide_resident = account_access.resident_hidden(
- "chat", getattr(inference_backend, "active_model_name", None)
- )
- for model_name, model_data in inference_backend.models.items():
- if hide_resident:
- continue
- _is_vision = model_data.get("is_vision", False)
- _audio_type = model_data.get("audio_type")
- model_info = ModelDetails(
- id = model_name,
- name = display_model_name(model_name),
- is_vision = _is_vision,
- is_lora = model_data.get("is_lora", False),
- is_mlx = model_data.get("is_mlx", False),
- is_audio = model_data.get("is_audio", False),
- audio_type = _audio_type,
- has_audio_input = model_data.get("has_audio_input", False),
- model_type = derive_model_type(_is_vision, _audio_type),
- )
- loaded_models.append(model_info)
-
- # Active GGUF model (llama-server), labelled from the display id /api/inference/status publishes; the
- # id stays raw for agents-tab's path filter.
+ from core.inference import model_slots
from routes.inference import _llama_status_model_ids, get_llama_cpp_backend
- llama_backend = get_llama_cpp_backend()
- hide_resident = hide_resident or account_access.resident_hidden(
- "chat", getattr(llama_backend, "model_identifier", None)
- )
- if not hide_resident and llama_backend.is_loaded and llama_backend.model_identifier:
+ def _orchestrator_models(orchestrator) -> list[ModelDetails]:
+ entries = []
+ for model_name, model_data in orchestrator.models.items():
+ _is_vision = model_data.get("is_vision", False)
+ _audio_type = model_data.get("audio_type")
+ entries.append(
+ ModelDetails(
+ id = model_name,
+ name = display_model_name(model_name),
+ is_vision = _is_vision,
+ is_lora = model_data.get("is_lora", False),
+ is_mlx = model_data.get("is_mlx", False),
+ is_audio = model_data.get("is_audio", False),
+ audio_type = _audio_type,
+ has_audio_input = model_data.get("has_audio_input", False),
+ model_type = derive_model_type(_is_vision, _audio_type),
+ )
+ )
+ return entries
+
+ def _gguf_model(llama_backend) -> list[ModelDetails]:
+ if not (llama_backend.is_loaded and llama_backend.model_identifier):
+ return []
display_id, _reported_identifier = _llama_status_model_ids(llama_backend)
- loaded_models.append(
+ return [
ModelDetails(
id = llama_backend.model_identifier,
name = display_model_name(display_id or llama_backend.model_identifier),
@@ -2056,7 +2051,25 @@ async def list_models(current_subject: str = Depends(get_current_subject)):
is_audio = getattr(llama_backend, "_is_audio", False),
audio_type = getattr(llama_backend, "_audio_type", None),
)
- )
+ ]
+
+ loaded_models = []
+ hide_resident = account_access.resident_hidden(
+ "chat", getattr(inference_backend, "active_model_name", None)
+ )
+ if not hide_resident:
+ loaded_models += _orchestrator_models(inference_backend)
+
+ llama_backend = get_llama_cpp_backend()
+ hide_resident = hide_resident or account_access.resident_hidden(
+ "chat", getattr(llama_backend, "model_identifier", None)
+ )
+ if not hide_resident:
+ loaded_models += _gguf_model(llama_backend)
+
+ for slot in model_slots.visible():
+ loaded_models += _orchestrator_models(slot.orchestrator)
+ loaded_models += model_slots.in_slot(slot, lambda: _gguf_model(slot.llama))
all_models = []
seen_ids = set()
@@ -2755,21 +2768,29 @@ async def discard_remote_code_download(
try:
from hub.services.models.deletion import _loaded_id_matches_repo
+ from core.inference import model_slots
from routes.inference import get_llama_cpp_backend
- llama_backend = get_llama_cpp_backend()
- if llama_backend.is_loaded and llama_backend.model_identifier:
- if _loaded_id_matches_repo(llama_backend.model_identifier, model_name):
- return {"deleted": False, "reason": "loaded"}
+ for llama_backend in (
+ get_llama_cpp_backend(),
+ *(slot.llama for slot in model_slots.resident()),
+ ):
+ if llama_backend.is_loaded and llama_backend.model_identifier:
+ if _loaded_id_matches_repo(llama_backend.model_identifier, model_name):
+ return {"deleted": False, "reason": "loaded"}
except Exception:
pass
try:
# Peek, not construct: no orchestrator means no active model, and building one hits get_device().
from core.inference.orchestrator import peek_inference_backend
- inference_backend = peek_inference_backend()
- if inference_backend is not None and inference_backend.active_model_name:
- if _loaded_id_matches_repo(inference_backend.active_model_name, model_name):
- return {"deleted": False, "reason": "loaded"}
+ from core.inference import model_slots
+ for inference_backend in (
+ peek_inference_backend(),
+ *(slot.orchestrator for slot in model_slots.resident()),
+ ):
+ if inference_backend is not None and inference_backend.active_model_name:
+ if _loaded_id_matches_repo(inference_backend.active_model_name, model_name):
+ return {"deleted": False, "reason": "loaded"}
except Exception:
pass
@@ -3298,60 +3319,46 @@ async def delete_finetuned_model(
) from e
try:
+ from core.inference import model_slots
from routes.inference import get_llama_cpp_backend
- llama_backend = get_llama_cpp_backend()
- if (
- llama_backend.is_active
- and not llama_backend.is_loaded
- and llama_backend.model_identifier
- and _loaded_model_matches_deleted_path(
- llama_backend.model_identifier,
- target_path,
- )
- and (
- not gguf_variant
- or not llama_backend.hf_variant
- # Alias-aware: the delete below accepts a bare quant for a qualified key, so a
- # literal comparison here would wave through the very spelling it then deletes.
- or _variant_names_same_checkpoint(llama_backend.hf_variant, gguf_variant)
- )
- ):
+ kept = model_slots.resident()
+ filling = model_slots.filling_model()
+ if filling and _loaded_model_matches_deleted_path(filling, target_path):
raise HTTPException(
status_code = 409,
detail = "Cannot delete a model while it is loading",
)
- if (
- llama_backend.is_loaded
- and llama_backend.model_identifier
- and _loaded_model_matches_deleted_path(
- llama_backend.model_identifier,
- target_path,
- )
- and (
- not gguf_variant
- or not llama_backend.hf_variant
- or _variant_names_same_checkpoint(llama_backend.hf_variant, gguf_variant)
- )
- ):
- raise HTTPException(
- status_code = 400,
- detail = "Unload the model before deleting",
- )
- except HTTPException:
- raise
- except Exception as e:
- logger.warning("Could not check llama.cpp loaded model before delete: %s", e)
- raise HTTPException(
- status_code = 503,
- detail = "Could not verify model load status before deleting",
- ) from e
-
- try:
+ for llama_backend in (get_llama_cpp_backend(), *(slot.llama for slot in kept)):
+ if (
+ (llama_backend.is_active or llama_backend.is_loaded)
+ and llama_backend.model_identifier
+ and _loaded_model_matches_deleted_path(
+ llama_backend.model_identifier,
+ target_path,
+ )
+ and (
+ not gguf_variant
+ or not llama_backend.hf_variant
+ # Alias-aware: a literal compare would pass the bare-quant spelling the delete accepts.
+ or _variant_names_same_checkpoint(llama_backend.hf_variant, gguf_variant)
+ )
+ ):
+ if llama_backend.is_loaded:
+ raise HTTPException(
+ status_code = 400,
+ detail = "Unload the model before deleting",
+ )
+ raise HTTPException(
+ status_code = 409,
+ detail = "Cannot delete a model while it is loading",
+ )
# Peek: building an orchestrator to learn there is none reaches get_device() (a torch import).
from core.inference.orchestrator import peek_inference_backend
- inference_backend = peek_inference_backend()
- if inference_backend is not None:
+
+ for inference_backend in (peek_inference_backend(), *(slot.orchestrator for slot in kept)):
+ if inference_backend is None:
+ continue
loading_models = getattr(inference_backend, "loading_models", set())
if any(
_loading_model_matches_deleted_path(loading_model, target_path)
@@ -3361,19 +3368,18 @@ async def delete_finetuned_model(
status_code = 409,
detail = "Cannot delete a model while it is loading",
)
- if inference_backend.active_model_name:
- if _loaded_model_matches_deleted_path(
- inference_backend.active_model_name,
- target_path,
- ):
- raise HTTPException(
- status_code = 400,
- detail = "Unload the model before deleting",
- )
+ if inference_backend.active_model_name and _loaded_model_matches_deleted_path(
+ inference_backend.active_model_name,
+ target_path,
+ ):
+ raise HTTPException(
+ status_code = 400,
+ detail = "Unload the model before deleting",
+ )
except HTTPException:
raise
except Exception as e:
- logger.warning("Could not check inference backend loaded model before delete: %s", e)
+ logger.warning("Could not check the loaded models before delete: %s", e)
raise HTTPException(
status_code = 503,
detail = "Could not verify model load status before deleting",
diff --git a/studio/backend/routes/settings.py b/studio/backend/routes/settings.py
index 86d7a74a313..592f52bab96 100644
--- a/studio/backend/routes/settings.py
+++ b/studio/backend/routes/settings.py
@@ -14,7 +14,7 @@ from contextvars import ContextVar
from typing import Any, Literal, Optional, get_args
from urllib.parse import unquote, urlsplit
-from fastapi import APIRouter, Depends, Header, HTTPException, Request
+from fastapi import APIRouter, BackgroundTasks, Depends, Header, HTTPException, Request
from fastapi.responses import StreamingResponse
from starlette.background import BackgroundTask
from pydantic import (
@@ -94,6 +94,11 @@ from utils.hub_settings import (
from picker.schemas import MAX_CHAT_TEMPLATE_BYTES, chat_template_byte_length
from utils.reasoning_budget import validate_reasoning_budget_message
from utils.coding_agents import CODING_AGENTS, detect_installed_coding_agents
+from utils.multi_model_settings import (
+ DEFAULT_MULTI_MODEL_ENABLED,
+ get_multi_model_enabled,
+ set_multi_model_enabled,
+)
from utils.model_memory_settings import (
DEFAULT_KEEP_RESIDENT,
DEFAULT_NO_RAM_RESERVE,
@@ -545,6 +550,15 @@ def delete_custom_generation_preset(
return {"deleted": True}
+class MultiModelPayload(BaseModel):
+ enabled: StrictBool
+
+
+class MultiModelResponse(BaseModel):
+ enabled: bool
+ default_enabled: bool = DEFAULT_MULTI_MODEL_ENABLED
+
+
class UploadLimitPayload(BaseModel):
max_upload_size_mb: int = Field(..., ge = MIN_UPLOAD_LIMIT_MB, le = MAX_UPLOAD_LIMIT_MB)
@@ -1352,6 +1366,48 @@ def update_llama_cpp_path(
return _llama_cpp_path_response()
+@_shared_settings_router.get("/multi-model", response_model = MultiModelResponse)
+def get_multi_model(current_subject: str = Depends(get_current_subject)) -> MultiModelResponse:
+ return MultiModelResponse(enabled = get_multi_model_enabled())
+
+
+@_owner_settings_router.put("/multi-model", response_model = MultiModelResponse)
+def update_multi_model(
+ payload: MultiModelPayload,
+ background_tasks: BackgroundTasks,
+ current_subject: str = Depends(get_current_subject),
+) -> MultiModelResponse:
+ """Keep the loaded models when another loads. Takes effect on the next load."""
+ try:
+ enabled = set_multi_model_enabled(payload.enabled)
+ except Exception as exc:
+ raise log_and_http_error(
+ exc,
+ 500,
+ safe_error_detail(exc, fallback = "Could not save the multiple models setting."),
+ event = "settings.update_multi_model_failed",
+ log = logger,
+ ) from exc
+ logger.info("settings.multi_model_updated subject=%s enabled=%s", current_subject, enabled)
+ if not enabled:
+ # Back to one model: the idle kept ones go after the reply (a teardown can take minutes),
+ # a busy one once it is ejected. One that fails to unload stays tracked as stuck.
+ background_tasks.add_task(_unload_idle_models)
+ return MultiModelResponse(enabled = enabled)
+
+
+def _unload_idle_models() -> None:
+ from core.inference import model_slots
+ from routes.inference import release_chat_after_kept_models
+
+ try:
+ model_slots.unload_idle()
+ except Exception:
+ logger.warning("settings.multi_model_unload_idle_failed", exc_info = True)
+ # The primary's own unload kept CHAT while these were loaded.
+ release_chat_after_kept_models()
+
+
@_shared_settings_router.get("/upload-limit", response_model = UploadLimitResponse)
def get_upload_limit(current_subject: str = Depends(get_current_subject)) -> UploadLimitResponse:
return _upload_limit_response(get_upload_limit_mb())
diff --git a/studio/backend/routes/training_vram.py b/studio/backend/routes/training_vram.py
index 977b17e1ac6..9b0d532cd41 100644
--- a/studio/backend/routes/training_vram.py
+++ b/studio/backend/routes/training_vram.py
@@ -69,6 +69,34 @@ def summarize_resident_chat() -> Dict[str, Any]:
except Exception as e:
logger.warning("Could not inspect GGUF backend: %s", e)
+ try:
+ from core.inference import model_slots
+
+ filling_slot = model_slots.loading
+ # A stuck slot's server would not stop, so it may still hold VRAM whatever it reports.
+ stuck = list(model_slots.stuck)
+ for slot in [*model_slots.slots, *stuck]:
+ pending = next(iter(getattr(slot.orchestrator, "loading_models", ()) or ()), None)
+ filling = filling_slot is not None and filling_slot[0] is slot
+ name = (
+ (slot in stuck and (slot.llama.model_identifier or "gguf"))
+ or slot.orchestrator.active_model_name
+ or pending
+ or (
+ slot.llama.is_active
+ and getattr(slot.llama, "_gpu_offload_active", None) is not False
+ and (slot.llama.model_identifier or "gguf")
+ )
+ or (filling and filling_slot[1])
+ )
+ if name:
+ gguf_name = gguf_name or name
+ # Only a still-loading slot is unsizable; a loaded one is in the free VRAM can_keep reads.
+ if pending or filling or (slot.llama.is_active and not slot.llama.is_loaded):
+ loading = True
+ except Exception as e:
+ logger.warning("Could not inspect models loaded alongside: %s", e)
+
return {
"hf": hf_name,
"gguf": gguf_name,
@@ -480,9 +508,28 @@ def free_chat_models_for_training(reason: str) -> List[str]:
except Exception as e:
logger.warning("Could not unload GGUF chat model: %s", e)
+ freed += free_kept_models_for_training(reason)
return freed
+def free_kept_models_for_training(reason: str) -> List[str]:
+ from core.inference import model_slots
+
+ kept = [
+ slot.orchestrator.active_model_name or slot.llama.model_identifier or "gguf"
+ for slot in [*model_slots.slots, *model_slots.stuck]
+ ]
+ if kept:
+ logger.info("Unloading %d model(s) kept alongside for training (%s)", len(kept), reason)
+ try:
+ model_slots.unload_extra_models(strict = True)
+ except RuntimeError as exc:
+ raise ManagedEngineStillRunning(
+ "A model kept alongside could not be stopped. Retry before starting training."
+ ) from exc
+ return [f"kept:{name}" for name in kept]
+
+
def _stt_sidecar_holds_no_vram(sidecar) -> bool:
"""True only when the resident dictation model is provably in CPU RAM. Conservative on purpose: anything
unreadable answers False and the sidecar is freed as before. Skipping one that does hold VRAM would starve
@@ -671,6 +718,16 @@ def coordinate_models_for_training(
logger.info("Keeping chat model loaded after freeing STT: %s", resident_chat)
return freed
+ from core.inference import model_slots
+
+ # Models kept alongside go before the one in use.
+ if model_slots.slots or model_slots.stuck:
+ freed += free_kept_models_for_training(reason = "insufficient training memory")
+ keep, _info = can_keep()
+ if keep:
+ logger.info("Keeping the active chat model loaded after freeing the others")
+ return freed
+
freed += free_chat_models_for_training(
reason = "insufficient VRAM to run training alongside chat",
)
diff --git a/studio/backend/run.py b/studio/backend/run.py
index 6f6f9b7df92..7cceb25d1dd 100644
--- a/studio/backend/run.py
+++ b/studio/backend/run.py
@@ -1524,6 +1524,7 @@ def _graceful_shutdown(server = None):
logger.warning("Error shutting down training subprocess: %s", e)
try:
+ from core.inference.model_slots import unload_extra_models
from routes.inference import _llama_cpp_backend, cancel_pending_loads
# Before the kill: a load still in the lifecycle gate or in preflight is not yet
@@ -1539,6 +1540,7 @@ def _graceful_shutdown(server = None):
# teardown = True: an app-level stop, not the retry ladder reaping a child it
# is about to replace. Only the former may end an in-flight health wait.
_llama_cpp_backend._kill_process(teardown = True)
+ unload_extra_models()
except Exception as e:
logger.warning("Error shutting down llama-server: %s", e)
diff --git a/studio/backend/state/active_generations.py b/studio/backend/state/active_generations.py
index a97312a1da4..495fa866532 100644
--- a/studio/backend/state/active_generations.py
+++ b/studio/backend/state/active_generations.py
@@ -21,7 +21,7 @@ from __future__ import annotations
import threading
import time
import uuid
-from typing import Any, Optional
+from typing import Any, Collection, Optional
from utils.account_context import current_account_id
@@ -115,12 +115,29 @@ class ActiveGeneration:
return False
-def snapshot(account_id: Optional[str] = None) -> list[dict[str, Any]]:
- """In-flight generations, newest last; ``account_id`` None (all) is shutdown/arbiter only."""
+def _matching(
+ account_id: Optional[str],
+ exclude: Collection[threading.Event],
+ only: Optional[Collection[threading.Event]] = None,
+) -> list[dict[str, Any]]:
+ return [
+ e
+ for e in _ACTIVE.values()
+ if (account_id is None or e["account_id"] == account_id)
+ and e["event"] not in exclude
+ and (only is None or e["event"] in only)
+ ]
+
+
+def snapshot(
+ account_id: Optional[str] = None,
+ exclude: Collection[threading.Event] = (),
+ only: Optional[Collection[threading.Event]] = None,
+) -> list[dict[str, Any]]:
+ """In-flight generations, newest last; ``account_id`` None (all) is shutdown/arbiter only.
+ ``exclude`` leaves out the runs holding those cancel events."""
with _LOCK:
- entries = [
- e for e in _ACTIVE.values() if account_id is None or e["account_id"] == account_id
- ]
+ entries = _matching(account_id, exclude, only)
entries.sort(key = lambda e: e["started_at"])
return [
{
@@ -136,25 +153,29 @@ def snapshot(account_id: Optional[str] = None) -> list[dict[str, Any]]:
]
-def active_thread_ids(account_id: Optional[str] = None) -> list[str]:
+def active_thread_ids(
+ account_id: Optional[str] = None,
+ exclude: Collection[threading.Event] = (),
+ only: Optional[Collection[threading.Event]] = None,
+) -> list[str]:
"""Distinct conversation ids with a generation in flight, in start order.
A first turn that races persistence has no thread id yet: count() sees it,
this cannot name it.
"""
seen: list[str] = []
- for e in snapshot(account_id):
+ for e in snapshot(account_id, exclude, only):
tid = e["thread_id"]
if tid and tid not in seen:
seen.append(tid)
return seen
-def count(account_id: Optional[str] = None) -> int:
+def count(account_id: Optional[str] = None, exclude: Collection[threading.Event] = ()) -> int:
with _LOCK:
- if account_id is None:
+ if account_id is None and not exclude:
return len(_ACTIVE)
- return sum(1 for e in _ACTIVE.values() if e["account_id"] == account_id)
+ return len(_matching(account_id, exclude))
def foreign_count(account_id: str) -> int:
@@ -163,15 +184,11 @@ def foreign_count(account_id: str) -> int:
return sum(1 for e in _ACTIVE.values() if e["account_id"] != account_id)
-def cancel_all(account_id: Optional[str] = None) -> int:
+def cancel_all(account_id: Optional[str] = None, exclude: Collection[threading.Event] = ()) -> int:
"""Signal in-flight generations to stop, returning the count. Request-driven callers must pass
``account_id``; None means everyone and is for shutdown only."""
with _LOCK:
- events = [
- e["event"]
- for e in _ACTIVE.values()
- if account_id is None or e["account_id"] == account_id
- ]
+ events = [e["event"] for e in _matching(account_id, exclude)]
for ev in events:
try:
ev.set()
diff --git a/studio/backend/tests/test_audio_device_guards.py b/studio/backend/tests/test_audio_device_guards.py
index 23471905e49..c4f5c40f77a 100644
--- a/studio/backend/tests/test_audio_device_guards.py
+++ b/studio/backend/tests/test_audio_device_guards.py
@@ -350,16 +350,21 @@ def test_a_zero_gpu_standard_load_drops_the_stale_chat_claim():
during-load release is the third, handed to load_model as a callback so it
fires once the previous worker is gone rather than before it."""
src = _inference_source()
- assert src.count("await asyncio.to_thread(release, CHAT)") == 2
- assert src.count("(lambda: release(CHAT)) if not chat_load_needs_gpu else None") == 1
+ assert src.count("await asyncio.to_thread(_release_chat_for_zero_vram_primary)") == 2
+ assert (
+ src.count(
+ "_release_chat_for_zero_vram_primary if replacing and not chat_load_needs_gpu else None"
+ )
+ == 1
+ )
def test_the_release_is_gated_on_the_same_flag_as_the_409():
"""Two `if not` sites plus the callback's own inline gate, which spells the
flag the same way."""
src = _inference_source()
- assert src.count("if not chat_load_needs_gpu:") == 2
- assert src.count("if not chat_load_needs_gpu else None") == 1
+ assert src.count("if replacing and not chat_load_needs_gpu:") == 2
+ assert src.count("if replacing and not chat_load_needs_gpu else None") == 1
def test_every_http_device_field_pins_the_three_canonical_values():
@@ -443,7 +448,7 @@ def test_the_stale_chat_claim_is_dropped_before_the_load_not_only_after():
src = inspect.getsource(ri._load_model_impl)
before_load = src[: src.index("backend.load_model,")]
- assert before_load.count("release, CHAT") >= 1
+ assert before_load.count("_release_chat_for_zero_vram_primary") >= 1
def test_the_gguf_audio_codec_follows_the_servers_own_placement():
diff --git a/studio/backend/tests/test_cache_inventory_path_boundary.py b/studio/backend/tests/test_cache_inventory_path_boundary.py
index 3081ceaae5a..b5be89f3dfa 100644
--- a/studio/backend/tests/test_cache_inventory_path_boundary.py
+++ b/studio/backend/tests/test_cache_inventory_path_boundary.py
@@ -1569,7 +1569,7 @@ def test_the_chat_status_does_not_hand_back_the_path_the_load_resolved(monkeypat
from models.inference import InferenceStatusResponse
from routes import inference as inference_routes
- async def _payload(current_subject: str):
+ async def _payload(current_subject: str, model = None):
return InferenceStatusResponse(
active_model = REPO_DIR,
model_identifier = REPO_DIR,
diff --git a/studio/backend/tests/test_gpu_memory_mode.py b/studio/backend/tests/test_gpu_memory_mode.py
index 940abc3b0ff..33a5be9ce7f 100644
--- a/studio/backend/tests/test_gpu_memory_mode.py
+++ b/studio/backend/tests/test_gpu_memory_mode.py
@@ -1467,8 +1467,10 @@ def test_a_cpu_only_chat_load_does_not_take_the_gpu_arbiter():
acquire = load_impl.index("if chat_load_needs_gpu:", gate)
# The stale CHAT claim is dropped only AFTER the load: this one may be replacing a GPU-backed chat model, and releasing
# earlier would let an image/video load allocate alongside the model not yet unloaded.
- release = load_impl.index("await asyncio.to_thread(release, CHAT)", acquire)
- assert load_impl.index("if not chat_load_needs_gpu:", acquire) < release
+ release = load_impl.index(
+ "await asyncio.to_thread(_release_chat_for_zero_vram_primary)", acquire
+ )
+ assert load_impl.index("if replacing and not chat_load_needs_gpu:", acquire) < release
assert load_impl.index("success = await load_with_tensor_fallback(", acquire) < release
assert "if chat_load_needs_gpu and current_owner() != CHAT:" in load_impl
# The already-loaded fast path re-asserts ownership too; same exemption.
diff --git a/studio/backend/tests/test_llama_cpp_update.py b/studio/backend/tests/test_llama_cpp_update.py
index 76fb1b8e5c1..22375b98648 100644
--- a/studio/backend/tests/test_llama_cpp_update.py
+++ b/studio/backend/tests/test_llama_cpp_update.py
@@ -16,7 +16,7 @@ import subprocess
import sys
import time
from pathlib import Path
-from types import ModuleType
+from types import ModuleType, SimpleNamespace
import pytest
@@ -1151,6 +1151,66 @@ def test_update_sets_maintenance_flag_and_unloads(monkeypatch, tmp_path):
assert backend._llama_update_in_progress is False
+def test_update_drains_and_marks_each_kept_model_before_swapping(monkeypatch, tmp_path):
+ import core.inference.model_slots as model_slots
+
+ install_dir = tmp_path / "llama.cpp"
+ binary = _write_install(install_dir, "b9493")
+ monkeypatch.setattr(upd, "_find_binary", lambda: binary)
+ monkeypatch.setattr(upd, "_installer_script", lambda: tmp_path / "install_llama_prebuilt.py")
+ monkeypatch.setattr(freshness, "_fetch_latest_release_tag", lambda repo, timeout = 5.0: "b9518")
+ _inject_backend(monkeypatch, _FakeBackend())
+ kept = _FakeBackend()
+ monkeypatch.setattr(model_slots, "slots", [SimpleNamespace(llama = kept)])
+ seen = {}
+
+ def unload_llama_slots(strict = False):
+ # A load still holding the slot's lock would have finished before this runs.
+ seen["lock_free"] = kept._serial_load_lock.acquire(blocking = False)
+ seen["flag"] = kept._llama_update_in_progress
+ return 1
+
+ monkeypatch.setattr(model_slots, "unload_llama_slots", unload_llama_slots)
+ _patch_installer_popen(monkeypatch, on_start = lambda cmd: _write_install(install_dir, "b9518"))
+ assert upd.start_update()["started"] is True
+ deadline = time.time() + 10
+ while time.time() < deadline:
+ if upd.get_update_status()["job"]["state"] in ("success", "error"):
+ break
+ time.sleep(0.05)
+ assert seen == {"lock_free": True, "flag": True}
+ # A slot the teardown kept (a non-GGUF model) is usable again once the update ends.
+ assert kept._llama_update_in_progress is False
+
+
+def test_update_stops_when_a_kept_model_would_not_unload(monkeypatch, tmp_path):
+ import core.inference.model_slots as model_slots
+
+ install_dir = tmp_path / "llama.cpp"
+ binary = _write_install(install_dir, "b9493")
+ monkeypatch.setattr(upd, "_find_binary", lambda: binary)
+ monkeypatch.setattr(upd, "_installer_script", lambda: tmp_path / "install_llama_prebuilt.py")
+ monkeypatch.setattr(freshness, "_fetch_latest_release_tag", lambda repo, timeout = 5.0: "b9518")
+ _inject_backend(monkeypatch, _FakeBackend())
+ monkeypatch.setattr(model_slots, "slots", [])
+
+ def stuck(strict = False):
+ assert strict
+ raise RuntimeError("Could not unload 1 model(s) kept alongside")
+
+ monkeypatch.setattr(model_slots, "unload_llama_slots", stuck)
+ spawned = []
+ _patch_installer_popen(monkeypatch, spawned = spawned)
+ assert upd.start_update()["started"] is True
+ deadline = time.time() + 10
+ while time.time() < deadline:
+ if upd.get_update_status()["job"]["state"] in ("success", "error"):
+ break
+ time.sleep(0.05)
+ assert upd.get_update_status()["job"]["state"] == "error"
+ assert not any("install_llama_prebuilt" in " ".join(map(str, c)) for c in spawned)
+
+
def test_update_clears_maintenance_flag_on_installer_failure(monkeypatch, tmp_path):
install_dir = tmp_path / "llama.cpp"
binary = _write_install(install_dir, "b9493")
diff --git a/studio/backend/tests/test_multi_model_serving.py b/studio/backend/tests/test_multi_model_serving.py
new file mode 100644
index 00000000000..425a3b8abb9
--- /dev/null
+++ b/studio/backend/tests/test_multi_model_serving.py
@@ -0,0 +1,1525 @@
+# SPDX-License-Identifier: AGPL-3.0-only
+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
+
+"""Models loaded alongside the primary one are routed to by request model name."""
+
+import asyncio
+import inspect
+import threading
+from types import SimpleNamespace
+
+import pytest
+from fastapi import FastAPI, HTTPException
+from fastapi.testclient import TestClient
+
+import core.inference.llama_cpp as llama_cpp
+import core.inference.llama_keepwarm as keepwarm
+import core.inference.model_slots as model_slots
+import core.inference.npu_backend as npu_backend
+import core.inference.orchestrator as orchestrator
+import routes.inference as inf
+from auth import policy
+from auth.authentication import get_current_subject
+from core.inference import gpu_arbiter
+from core.inference.llama_cpp import GpuMemoryShortError
+from hub.services.models import deletion
+from models.inference import (
+ ChatCountTokensRequest,
+ InferenceStatusResponse,
+ LoadRequest,
+ LoadResponse,
+ UnloadRequest,
+)
+from routes import training_vram
+from state import active_generations
+from utils import multi_model_settings
+from utils.account_context import AccountContext, run_as
+
+ALICE = AccountContext("a" * 32, "alice")
+BOB = AccountContext("b" * 32, "bob")
+
+
+class FakeLlama:
+ holds_no_vram = False
+
+ def __init__(
+ self,
+ identifier = None,
+ variant = None,
+ ):
+ self.model_identifier = identifier
+ self.hf_variant = variant
+ self.is_loaded = self.is_active = identifier is not None
+ self.is_vision = False
+ self.context_length = 4096
+
+ def unload_model(self):
+ self.is_loaded = self.is_active = False
+
+ def _cleanup(self):
+ pass
+
+
+class FakeOrchestrator:
+ def __init__(self, active = None):
+ self.active_model_name = active
+ self.models = {active: {}} if active else {}
+
+ def unload_model(self, name):
+ self.active_model_name = None
+
+ def _cleanup(self):
+ pass
+
+
+def backends(monkeypatch):
+ primary = FakeLlama("org/A-GGUF", "Q4_K_M")
+ extra = model_slots.ExtraSlot(FakeLlama("org/B-GGUF", "Q8_0"), FakeOrchestrator(), "owner")
+ monkeypatch.setattr(inf, "_llama_cpp_backend", primary)
+ monkeypatch.setattr(orchestrator, "_inference_backend", FakeOrchestrator())
+ monkeypatch.setattr(model_slots, "slots", [extra])
+ monkeypatch.setattr(inf, "LlamaCppBackend", FakeLlama)
+ monkeypatch.setattr(inf, "InferenceOrchestrator", FakeOrchestrator)
+ monkeypatch.setattr(inf, "_raise_if_sidecar_swap_in_progress", lambda: None)
+ monkeypatch.setattr(multi_model_settings, "get_multi_model_enabled", lambda: True)
+ monkeypatch.setattr(model_slots, "stuck", [])
+ return primary, extra
+
+
+def _routed(requested):
+ async def run():
+ slot = await inf._route_to_extra_slot(requested)
+ return slot, inf.get_llama_cpp_backend()
+
+ return asyncio.run(run())
+
+
+def test_request_model_name_picks_the_backend(backends):
+ primary, extra = backends
+ assert _routed("org/B-GGUF") == (extra, extra.llama)
+ assert _routed("org/B-GGUF:Q8_0") == (extra, extra.llama)
+ assert _routed("org/B-GGUF:Q4_K_M") == (None, primary)
+ assert _routed("org/A-GGUF") == (None, primary)
+ assert _routed(None) == (None, primary)
+ assert extra.last_used > 0.0
+ # The primary wins a bare name both serve.
+ model_slots.slots.append(
+ model_slots.ExtraSlot(FakeLlama("org/A-GGUF", "Q8_0"), FakeOrchestrator(), "owner")
+ )
+ assert _routed("org/A-GGUF") == (None, primary)
+ assert _routed("org/A-GGUF:Q8_0")[0] is model_slots.slots[-1]
+
+
+def test_a_safetensors_slot_is_served_by_its_own_orchestrator(backends):
+ safetensors = model_slots.ExtraSlot(FakeLlama(), FakeOrchestrator("org/C"), "owner")
+ model_slots.slots.append(safetensors)
+
+ async def run():
+ slot = await inf._route_to_extra_slot("org/C")
+ return slot, inf.get_llama_cpp_backend().is_loaded, inf.get_inference_backend()
+
+ assert asyncio.run(run()) == (safetensors, False, safetensors.orchestrator)
+
+
+def test_loaded_models_lists_every_backend(backends):
+ app = FastAPI()
+ app.include_router(inf.studio_router, prefix = "/api/inference")
+ app.dependency_overrides[get_current_subject] = lambda: "test-subject"
+ with TestClient(app) as client:
+ data = client.get("/api/inference/loaded-models").json()["data"]
+ assert {entry["id"] for entry in data} == {"org/A-GGUF", "org/B-GGUF"}
+ model_slots.slots.append(model_slots.ExtraSlot(FakeLlama(), FakeOrchestrator("org/C"), "owner"))
+ with TestClient(app) as client:
+ data = client.get("/api/inference/loaded-models").json()["data"]
+ assert [entry["id"] for entry in data] == ["org/A-GGUF", "org/B-GGUF", "org/C"]
+
+
+def _selected(request):
+ async def run():
+ slot = await inf._select_load_slot(request)
+ assert inf.routed_slot.get() is slot
+ return slot
+
+ return asyncio.run(run())
+
+
+def test_a_load_picks_its_slot(backends):
+ primary, extra = backends
+ alongside = LoadRequest(model_path = "org/C-GGUF", alongside = True)
+ assert _selected(alongside) not in (None, extra)
+ assert _selected(LoadRequest(model_path = "org/C-GGUF")) is None
+ served = LoadRequest(model_path = "org/B-GGUF", gguf_variant = "Q8_0")
+ assert _selected(served) is extra
+ # The NPU backend is one per process: it takes the primary's seat even alongside.
+ npu = LoadRequest(model_path = "lemonade:qwen3-0.6b-FLM", alongside = True)
+ assert _selected(npu) is None
+ assert _selected(alongside.model_copy(update = {"engine": "vllm"})) is None
+ primary.unload_model()
+ assert _selected(alongside) is None
+
+
+def test_alongside_is_off_until_settings_turns_it_on(backends, monkeypatch):
+ _, extra = backends
+ monkeypatch.setattr(multi_model_settings, "get_multi_model_enabled", lambda: False)
+ # A model already kept alongside still reloads in its own slot.
+ served = LoadRequest(model_path = "org/B-GGUF", gguf_variant = "Q8_0", alongside = True)
+ assert _selected(served) is extra
+ # Any other load goes to the primary, and retires a kept model once nothing uses it.
+ extra.refs += 1
+ assert _selected(LoadRequest(model_path = "org/C-GGUF", alongside = True)) is None
+ assert model_slots.slots == [extra]
+ extra.refs -= 1
+ assert _selected(LoadRequest(model_path = "org/D-GGUF")) is None
+ assert model_slots.slots == [] and not extra.llama.is_active
+
+
+def test_a_kept_model_with_no_gpu_layers_does_not_hold_the_gpu(backends):
+ _, extra = backends
+ assert model_slots.holds_vram()
+ extra.llama._gpu_offload_active = False
+ assert not model_slots.holds_vram() and model_slots.busy()
+
+
+def test_the_multi_model_setting_defaults_off(monkeypatch):
+ stored = {}
+ import storage.studio_db as studio_db
+
+ monkeypatch.setattr(
+ studio_db, "get_app_setting", lambda key, fallback: stored.get(key, fallback)
+ )
+ monkeypatch.setattr(studio_db, "upsert_app_settings", stored.update)
+ assert multi_model_settings.get_multi_model_enabled() is False
+ assert multi_model_settings.set_multi_model_enabled(True) is True
+ assert multi_model_settings.get_multi_model_enabled() is True
+ with pytest.raises(ValueError):
+ multi_model_settings.set_multi_model_enabled("yes")
+
+
+def test_turning_the_setting_off_saves_first_and_unloads_after_the_reply(backends, monkeypatch):
+ from fastapi import BackgroundTasks
+ import routes.settings as settings_routes
+
+ _, extra = backends
+ monkeypatch.setattr(settings_routes, "set_multi_model_enabled", lambda value: value)
+
+ def fail():
+ raise RuntimeError("llama-server would not exit")
+
+ monkeypatch.setattr(extra.llama, "unload_model", fail)
+ tasks = BackgroundTasks()
+ response = settings_routes.update_multi_model(
+ settings_routes.MultiModelPayload(enabled = False), tasks, "s"
+ )
+ # The saved value comes back before any teardown runs.
+ assert response.enabled is False and model_slots.slots == [extra]
+ asyncio.run(tasks())
+ # A teardown that fails is logged, not raised, and the slot stays counted.
+ assert model_slots.slots == [] and model_slots.stuck == [extra] and model_slots.holds_vram()
+
+
+def test_unload_drops_only_the_named_extra_slot(backends, monkeypatch):
+ primary, extra = backends
+ _hold_chat_claim(monkeypatch)
+ released, release = [], inf.release_chat_gpu_claim
+ monkeypatch.setattr(inf, "release_chat_gpu_claim", lambda: released.append(release()))
+ response = asyncio.run(inf._unload_model_impl(UnloadRequest(model_path = "org/B-GGUF"), "s"))
+ assert response.status == "unloaded"
+ assert primary.is_loaded and not extra.llama.is_loaded and model_slots.slots == []
+ # Tried, but the claim the primary holds stays.
+ assert released == [False] and gpu_arbiter.current_owner() == gpu_arbiter.CHAT
+
+
+def test_a_failed_llama_unload_still_cleans_the_orchestrator(backends):
+ _, extra = backends
+ cleaned = []
+ extra.llama.unload_model = lambda: 1 / 0
+ extra.orchestrator._cleanup = lambda: cleaned.append(True)
+ with pytest.raises(ZeroDivisionError):
+ model_slots.drop(extra)
+ assert model_slots.slots == [] and cleaned == [True]
+
+
+def test_a_managed_account_sees_only_its_own_slots(backends, monkeypatch):
+ primary, extra = backends
+ monkeypatch.setattr(inf.account_access, "managed_account", lambda: True)
+ monkeypatch.setattr(model_slots, "current_account_id", lambda: "someone-else")
+ assert _routed("org/B-GGUF") == (None, primary)
+ filling = model_slots.ExtraSlot(FakeLlama(), FakeOrchestrator(), "owner")
+ monkeypatch.setattr(model_slots, "loading", (filling, "org/D-GGUF"))
+ model_slots.slots.append(filling)
+ assert model_slots.visible_loading() is None
+ monkeypatch.setattr(model_slots, "current_account_id", lambda: "owner")
+ assert _routed("org/B-GGUF") == (extra, extra.llama)
+
+
+def test_idle_unload_spares_pinned_and_filling_slots(backends, monkeypatch):
+ _, extra = backends
+ assert model_slots.unload_extra_models(keep = lambda llama: True) == 0
+ monkeypatch.setattr(model_slots, "loading", (extra, "org/B-GGUF"))
+ assert model_slots.unload_extra_models(keep = lambda llama: False, spare_filling = True) == 0
+ assert model_slots.slots == [extra] and extra.llama.is_active
+ assert model_slots.unload_extra_models(keep = lambda llama: False) == 1
+ assert model_slots.slots == []
+
+
+def test_status_describes_the_named_slot_and_lists_the_rest(backends, monkeypatch):
+ async def slot_status(subject):
+ return InferenceStatusResponse(active_model = inf.get_llama_cpp_backend().model_identifier)
+
+ monkeypatch.setattr(inf, "_slot_status", slot_status)
+ named = asyncio.run(inf.get_status("s", model = "org/B-GGUF"))
+ assert (named.active_model, named.loaded) == ("org/B-GGUF", ["org/A-GGUF"])
+ primary = asyncio.run(inf.get_status("s"))
+ assert (primary.active_model, primary.loaded) == ("org/A-GGUF", ["org/B-GGUF"])
+
+ async def held_behind(subject):
+ return InferenceStatusResponse(
+ active_model = "org/A-GGUF", loaded = ["org/A-GGUF", "org/held-hf"]
+ )
+
+ # serving leaves out a model only held in memory behind the active one.
+ monkeypatch.setattr(inf, "_slot_status", held_behind)
+ status = asyncio.run(inf.get_status("s"))
+ assert status.serving == ["org/A-GGUF", "org/B-GGUF"] and "org/held-hf" in status.loaded
+ monkeypatch.setattr(model_slots, "slots", [])
+ assert asyncio.run(inf.get_status("s")).serving == ["org/A-GGUF"]
+
+
+def test_status_pairs_each_serving_model_with_the_checkpoint_to_select_it_by(backends, monkeypatch):
+ # A local model is listed under its label but selected, loaded and unloaded by its path.
+ _, extra = backends
+ local, twin = "/home/alice/models/B-local.gguf", "/home/alice/other/B-local.gguf"
+ extra.llama = FakeLlama(local, "Q8_0")
+ model_slots.slots.append(
+ model_slots.ExtraSlot(FakeLlama(twin, "Q8_0"), FakeOrchestrator(), "owner")
+ )
+ public = inf._llama_public_model_id
+ monkeypatch.setattr(
+ inf,
+ "_llama_public_model_id",
+ lambda llama: (
+ "B-local" if str(llama.model_identifier).endswith("B-local.gguf") else public(llama)
+ ),
+ )
+
+ async def slot_status(subject):
+ return InferenceStatusResponse(active_model = "org/A-GGUF", model_identifier = "org/A-GGUF")
+
+ monkeypatch.setattr(inf, "_slot_status", slot_status)
+ status = asyncio.run(inf.get_status("s"))
+ # Two files sharing a label both stay listed, each with its own path.
+ assert status.serving == ["org/A-GGUF", "B-local", "B-local"]
+ assert status.serving_checkpoints == ["org/A-GGUF", local, twin]
+
+
+def test_stop_loading_reaches_the_slot_being_filled(backends, monkeypatch):
+ primary, _ = backends
+ filling = model_slots.ExtraSlot(FakeLlama(), FakeOrchestrator(), "owner")
+ filling.llama.model_identifier = "org/D-GGUF"
+ filling.llama.is_active = True
+ model_slots.slots.append(filling)
+ monkeypatch.setattr(model_slots, "loading", (filling, "org/D-GGUF"))
+ response = asyncio.run(inf._unload_model_impl(UnloadRequest(model_path = "org/D-GGUF"), "s"))
+ assert response.status == "unloaded"
+ assert primary.is_loaded and not filling.llama.is_active
+
+
+def test_a_slot_evicted_while_it_loads_is_torn_down(backends, monkeypatch):
+ spawned = []
+
+ async def load_after_eviction(request, *args, **kwargs):
+ slot = model_slots.slots[-1]
+ model_slots.unload_extra_models()
+ slot.llama.is_loaded = slot.llama.is_active = True
+ spawned.append(slot)
+ return "loaded"
+
+ monkeypatch.setattr(inf, "_run_tracked_load_model_impl", load_after_eviction)
+ monkeypatch.setattr(inf, "release_chat_gpu_claim", lambda: None)
+ request = LoadRequest(model_path = "org/C-GGUF", alongside = True)
+ assert asyncio.run(inf.load_model_gated(request, None, "s")) == "loaded"
+ assert model_slots.slots == [] and not spawned[0].llama.is_active
+
+
+def test_only_a_new_slot_skips_the_running_chat_check(backends, monkeypatch):
+ seen = []
+
+ async def load(request, *args, on_reload_confirmed, **kwargs):
+ seen.append(on_reload_confirmed is None)
+ llama = inf.get_llama_cpp_backend()
+ llama.model_identifier = request.model_path
+ llama.is_loaded = llama.is_active = True
+ return "loaded"
+
+ monkeypatch.setattr(inf, "_run_tracked_load_model_impl", load)
+ for request in (
+ LoadRequest(model_path = "org/C-GGUF", alongside = True),
+ LoadRequest(model_path = "org/B-GGUF", gguf_variant = "Q8_0", force_reload = True),
+ LoadRequest(model_path = "org/D-GGUF"),
+ ):
+ asyncio.run(inf.load_model_gated(request, None, "s"))
+ assert seen == [True, False, False]
+
+
+def test_loading_alongside_leaves_the_primary_with_the_account_that_loaded_it(monkeypatch):
+ monkeypatch.setattr(gpu_arbiter, "_owner", None)
+ monkeypatch.setattr(gpu_arbiter, "_owner_account", None)
+ gpu_arbiter.acquire_for(gpu_arbiter.CHAT, lambda: None, account_id = ALICE.account_id)
+ gpu_arbiter.acquire_for(
+ gpu_arbiter.CHAT, lambda: None, account_id = BOB.account_id, alongside = True
+ )
+ assert gpu_arbiter.owner_account() == ALICE.account_id
+ gpu_arbiter.acquire_for(gpu_arbiter.CHAT, lambda: None, account_id = BOB.account_id)
+ assert gpu_arbiter.owner_account() == BOB.account_id
+
+
+def test_an_account_lists_its_own_slot_but_not_a_foreign_primary(backends, monkeypatch):
+ primary, extra = backends
+ monkeypatch.setattr(policy, "installation_is_multi_user", lambda: True)
+ monkeypatch.setattr(gpu_arbiter, "_owner", gpu_arbiter.CHAT)
+ monkeypatch.setattr(gpu_arbiter, "_owner_account", ALICE.account_id)
+ extra.account = BOB.account_id
+
+ def listed():
+ return [entry["id"] for entry in inf._openai_model_objects()]
+
+ assert run_as(BOB, listed) == ["org/B-GGUF"]
+ assert run_as(ALICE, listed) == ["org/A-GGUF"]
+
+
+def _slot(
+ model,
+ variant = None,
+ last_used = 0.0,
+):
+ return model_slots.ExtraSlot(
+ FakeLlama(model, variant),
+ FakeOrchestrator(),
+ "owner",
+ LoadRequest(model_path = model, gguf_variant = variant, alongside = True),
+ last_used,
+ )
+
+
+def _gated_load_fakes(
+ monkeypatch,
+ short_fits,
+ response = "loaded",
+):
+ monkeypatch.setattr(inf, "release_chat_gpu_claim", lambda: None)
+
+ async def load(request, *args, **kwargs):
+ if short_fits and not request.force_alongside:
+ kind = short_fits.pop()
+ raise GpuMemoryShortError(
+ "needs 13 GB, 8 GB free", capped = kind == "capped", short_mib = 5000
+ )
+ llama = inf.get_llama_cpp_backend()
+ llama.model_identifier, llama.hf_variant = request.model_path, request.gguf_variant
+ llama.is_loaded = llama.is_active = True
+ return response
+
+ monkeypatch.setattr(inf, "_run_tracked_load_model_impl", load)
+
+
+def test_a_short_fit_evicts_the_least_recently_used_slot_and_names_it(backends, monkeypatch):
+ _, extra = backends
+ extra.request, extra.last_used = LoadRequest(model_path = "org/B-GGUF", gguf_variant = "Q8_0"), 5.0
+ older = _slot("org/D-GGUF", "Q4_K_M", last_used = 1.0)
+ model_slots.slots.append(older)
+ loaded = LoadResponse.model_construct(status = "loaded", model = "org/C-GGUF", display_name = "C")
+ _gated_load_fakes(monkeypatch, short_fits = [1], response = loaded)
+ request = LoadRequest(model_path = "org/C-GGUF", alongside = True)
+ assert asyncio.run(inf.load_model_gated(request, None, "s")).evicted == ["org/D-GGUF:Q4_K_M"]
+ assert [s.llama.model_identifier for s in model_slots.slots] == ["org/B-GGUF", "org/C-GGUF"]
+ assert not older.llama.is_active
+ assert model_slots.slots[-1].request.model_path == "org/C-GGUF"
+
+
+def test_eviction_takes_as_many_lru_slots_as_the_shortfall_needs(backends, monkeypatch):
+ _, extra = backends
+ extra.request, extra.last_used = LoadRequest(model_path = "org/B-GGUF", gguf_variant = "Q8_0"), 3.0
+ extra.llama._planned_vram_mib = {0: 9000}
+ small_old = _slot("org/D-GGUF", last_used = 1.0)
+ small_old.llama._planned_vram_mib = {0: 500}
+ mid = _slot("org/E-GGUF", last_used = 2.0)
+ mid.llama._planned_vram_mib = {0: 6000}
+ model_slots.slots += [small_old, mid]
+ assert model_slots.eviction_victims(None, 5000) == [small_old, mid]
+ _gated_load_fakes(monkeypatch, short_fits = [1])
+ request = LoadRequest(model_path = "org/C-GGUF", alongside = True)
+ assert asyncio.run(inf.load_model_gated(request, None, "s")) == "loaded"
+ assert [s.llama.model_identifier for s in model_slots.slots] == ["org/B-GGUF", "org/C-GGUF"]
+
+
+def test_a_model_that_fits_nowhere_beside_the_others_replaces_the_active_one(backends, monkeypatch):
+ primary, extra = backends
+ extra.request = LoadRequest(model_path = "org/B-GGUF", gguf_variant = "Q8_0")
+ loaded = LoadResponse.model_construct(status = "loaded", model = "org/C-GGUF", display_name = "C")
+ _gated_load_fakes(monkeypatch, short_fits = [1, 1], response = loaded)
+ request = LoadRequest(model_path = "org/C-GGUF", alongside = True)
+ response = asyncio.run(inf.load_model_gated(request, None, "s"))
+ assert response.evicted == ["org/B-GGUF:Q8_0", "org/A-GGUF"]
+ assert model_slots.slots == [] and not extra.llama.is_active
+ assert primary.model_identifier == "org/C-GGUF" and primary.is_loaded
+
+
+def test_a_capped_context_is_taken_only_once_nothing_is_left_to_evict(backends, monkeypatch):
+ _, extra = backends
+ extra.request = LoadRequest(model_path = "org/B-GGUF", gguf_variant = "Q8_0")
+ _gated_load_fakes(monkeypatch, short_fits = ["capped", "capped"])
+ request = LoadRequest(model_path = "org/C-GGUF", alongside = True)
+ assert asyncio.run(inf.load_model_gated(request, None, "s")) == "loaded"
+ assert not extra.llama.is_active
+ assert [s.llama.model_identifier for s in model_slots.slots] == ["org/C-GGUF"]
+
+
+def test_a_slot_load_drops_no_claim_and_a_load_that_tears_nothing_down_stops_no_chat():
+ source = inspect.getsource(inf._load_model_impl)
+ assert source.count("if replacing and not chat_load_needs_gpu:") == 2
+ assert "if not chat_load_needs_gpu:" not in source
+ assert source.count("if serving and on_reload_confirmed is not None:") == 3
+ assert source.count("if replacing and serving:") == 2
+ assert "if on_reload_confirmed is not None:" not in source
+
+
+def test_a_load_prices_the_vram_the_other_servers_hold():
+ from core.inference.llama_cpp import LlamaCppBackend
+
+ a, b, c, stray = (LlamaCppBackend(manages_processes = False) for _ in range(4))
+ for backend in (a, b, c):
+ llama_cpp.register_serving_backend(backend)
+ try:
+ a._process = b._process = stray._process = object()
+ a._planned_vram_mib = {0: 12000}
+ b._planned_vram_mib = {0: 1000, 1: 500}
+ stray._planned_vram_mib = {0: 7000}
+ assert c._other_planned_vram_mib() == {0: 13000, 1: 500}
+ assert a._other_planned_vram_mib() == {0: 1000, 1: 500}
+ assert stray._other_planned_vram_mib() == {}
+ a._kill_process()
+ assert a._planned_vram_mib == {} and c._other_planned_vram_mib() == {0: 1000, 1: 500}
+ b._process = None
+ assert c._other_planned_vram_mib() == {}
+ finally:
+ for backend in (a, b, c):
+ llama_cpp.unregister_serving_backend(backend)
+
+
+def _hold_chat_claim(monkeypatch):
+ monkeypatch.setattr(gpu_arbiter, "_owner", gpu_arbiter.CHAT)
+ monkeypatch.setattr(llama_cpp, "chat_load_active", lambda: False)
+
+
+def test_a_failed_slot_load_drops_the_slot_but_keeps_the_primarys_claim(backends, monkeypatch):
+ primary, extra = backends
+ _hold_chat_claim(monkeypatch)
+ released = []
+ release = inf.release_chat_gpu_claim
+ monkeypatch.setattr(inf, "release_chat_gpu_claim", lambda: released.append(release()))
+
+ async def failing_load(*args, **kwargs):
+ raise RuntimeError("no such repo")
+
+ monkeypatch.setattr(inf, "_run_tracked_load_model_impl", failing_load)
+ request = LoadRequest(model_path = "org/missing-GGUF", alongside = True)
+ with pytest.raises(RuntimeError):
+ asyncio.run(inf.load_model_gated(request, None, "s"))
+ assert model_slots.slots == [extra] and model_slots.loading is None and released == [False]
+ assert primary.is_active and gpu_arbiter.current_owner() == gpu_arbiter.CHAT
+
+
+def test_a_failed_slot_load_ends_its_attempt_even_when_the_slot_will_not_stop(
+ backends, monkeypatch
+):
+ async def failing_load(*args, **kwargs):
+ raise RuntimeError("no such repo")
+
+ def refuse(slot):
+ raise RuntimeError("llama-server ignored SIGKILL")
+
+ monkeypatch.setattr(inf, "_run_tracked_load_model_impl", failing_load)
+ monkeypatch.setattr(model_slots, "drop", refuse)
+ request = LoadRequest(model_path = "org/missing-GGUF", alongside = True, load_request_id = "r1")
+ with pytest.raises(RuntimeError):
+ asyncio.run(inf.load_model_gated(request, None, "s"))
+ # Not left loading in /status, and the same request id can be retried.
+ assert inf._pending_load_attempts == {}
+ assert not any(key[1] == "r1" for key in inf._scoped_load_attempts)
+
+
+def test_a_generation_is_tracked_on_the_slot_serving_it(backends):
+ _, extra = backends
+ event = threading.Event()
+
+ async def run():
+ await inf._route_to_extra_slot("org/B-GGUF")
+ with inf._TrackedCancel(event, "k"):
+ inside = set(extra.generations)
+ return inside
+
+ assert asyncio.run(run()) == {event} and extra.generations == set()
+
+
+def test_a_slot_still_generating_is_never_evicted(backends):
+ _, extra = backends
+ older = _slot("org/D-GGUF", last_used = 1.0)
+ model_slots.slots.append(older)
+ older.generations.add(threading.Event())
+ assert model_slots.eviction_victims(None, 5000) == [extra]
+ extra.generations.add(threading.Event())
+ assert model_slots.eviction_victims(None, 5000) == []
+
+
+def test_unloading_a_generating_slot_is_refused_unless_forced(backends, monkeypatch):
+ _, extra = backends
+ monkeypatch.setattr(inf, "release_chat_gpu_claim", lambda: True)
+ event = threading.Event()
+ extra.generations.add(event)
+ with (
+ active_generations.ActiveGeneration(event, thread_id = "t1"),
+ pytest.raises(HTTPException) as excinfo,
+ ):
+ asyncio.run(inf._unload_model_impl(UnloadRequest(model_path = "org/B-GGUF"), "s"))
+ assert excinfo.value.status_code == 409 and excinfo.value.detail["thread_ids"] == ["t1"]
+ assert extra.llama.is_loaded and not event.is_set()
+
+ def finish_on_cancel():
+ event.wait(5)
+ extra.generations.discard(event)
+
+ threading.Thread(target = finish_on_cancel, daemon = True).start()
+ forced = UnloadRequest(model_path = "org/B-GGUF", force_cancel_active = True)
+ asyncio.run(inf._unload_model_impl(forced, "s"))
+ assert event.is_set() and not extra.llama.is_loaded and model_slots.slots == []
+
+
+def test_unloading_a_slot_waits_for_a_request_already_routed_to_it(backends, monkeypatch):
+ _, extra = backends
+ monkeypatch.setattr(inf, "release_chat_gpu_claim", lambda: True)
+ event = threading.Event()
+
+ async def run(starts_generating):
+ extra.refs += 1 # another request routed here, not yet generating
+ seen = {}
+
+ async def routed_request():
+ await asyncio.sleep(0.2)
+ seen["loaded_while_routed"] = extra.llama.is_loaded
+ if starts_generating:
+ extra.generations.add(event)
+ extra.refs -= 1
+
+ task = asyncio.create_task(routed_request())
+ try:
+ await inf._unload_model_impl(UnloadRequest(model_path = "org/B-GGUF"), "s")
+ finally:
+ await task
+ return seen
+
+ with active_generations.ActiveGeneration(event, thread_id = "t1"):
+ with pytest.raises(HTTPException) as excinfo:
+ asyncio.run(run(starts_generating = True))
+ assert excinfo.value.status_code == 409 and extra.llama.is_loaded
+ extra.generations.discard(event)
+ assert asyncio.run(run(starts_generating = False)) == {"loaded_while_routed": True}
+ assert model_slots.slots == [] and not extra.llama.is_loaded
+
+
+def test_unloading_a_slot_waits_for_a_reload_of_it_queued_on_the_gate(backends, monkeypatch):
+ from core.inference.llama_keepwarm import inference_lifecycle_gate, model_load_gate
+
+ _, extra = backends
+ monkeypatch.setattr(inf, "release_chat_after_kept_models", lambda: True)
+ monkeypatch.setattr(inf, "_POST_CANCEL_DRAIN_TIMEOUT_S", 0.5)
+ seen = {}
+
+ async def reload():
+ # As load_model_gated: the load gate, a ref on the slot, then the lifecycle gate.
+ async with model_load_gate():
+ extra.refs += 1
+ await asyncio.sleep(0.3)
+ async with inference_lifecycle_gate():
+ seen["still_kept"] = extra in model_slots.slots
+ extra.refs -= 1
+
+ async def run():
+ task = asyncio.create_task(reload())
+ await asyncio.sleep(0.1)
+ await inf._unload_model_impl(UnloadRequest(model_path = "org/B-GGUF"), "s")
+ await task
+
+ asyncio.run(run())
+ # The reload finished on a slot still kept, and the unload ran after it.
+ assert seen == {"still_kept": True}
+ assert model_slots.slots == [] and not extra.llama.is_loaded
+
+
+def test_a_slot_server_leaves_the_primary_pidfile_alone(monkeypatch):
+ import utils.process_lifetime as process_lifetime
+ from core.inference.llama_cpp import LlamaCppBackend
+
+ written, cleared, adopted = [], [], []
+ monkeypatch.setattr(
+ LlamaCppBackend, "_record_server_pid", classmethod(lambda cls, pid: written.append(pid))
+ )
+ monkeypatch.setattr(
+ LlamaCppBackend, "_clear_server_pid", classmethod(lambda cls: cleared.append(True))
+ )
+ monkeypatch.setattr(process_lifetime, "adopt_pid", adopted.append)
+ slot = LlamaCppBackend(manages_processes = False)
+ slot._owns_pidfile = False
+ slot._note_server_pid(4242)
+ slot._process = object()
+ slot._kill_process()
+ assert written == [] and cleared == [] and adopted == [4242]
+ primary = LlamaCppBackend(manages_processes = False)
+ primary._note_server_pid(4343)
+ assert written == [4343]
+
+
+def test_a_primary_swap_neither_refuses_on_nor_stops_another_models_chats(backends):
+ _, extra = backends
+ on_slot, on_primary = threading.Event(), threading.Event()
+ extra.generations.add(on_slot)
+ with active_generations.ActiveGeneration(on_slot, thread_id = "slot-chat"):
+ assert inf._raise_or_cancel_active_generations(force = False, action = "Loading a model") == 0
+ with active_generations.ActiveGeneration(on_primary, thread_id = "primary-chat"):
+ with pytest.raises(HTTPException) as excinfo:
+ inf._raise_or_cancel_active_generations(force = False, action = "Loading a model")
+ assert excinfo.value.detail["running"] == 1
+ assert excinfo.value.detail["thread_ids"] == ["primary-chat"]
+ assert (
+ inf._raise_or_cancel_active_generations(force = True, action = "Loading a model") == 1
+ )
+ assert on_primary.is_set() and not on_slot.is_set()
+
+
+def test_reloading_a_kept_model_stops_only_its_own_chats(backends, monkeypatch):
+ _, extra = backends
+ monkeypatch.setattr(inf, "release_chat_gpu_claim", lambda: None)
+
+ async def load(request, *args, on_reload_confirmed, **kwargs):
+ assert inf.get_llama_cpp_backend() is extra.llama
+ on_reload_confirmed(cancel = False)
+ on_reload_confirmed(cancel = True)
+ return "loaded"
+
+ monkeypatch.setattr(inf, "_run_tracked_load_model_impl", load)
+ reload = LoadRequest(model_path = "org/B-GGUF", gguf_variant = "Q8_0", force_reload = True)
+ on_primary = threading.Event()
+ with active_generations.ActiveGeneration(on_primary, thread_id = "chat-on-A"):
+ assert asyncio.run(inf.load_model_gated(reload, None, "s")) == "loaded"
+ on_slot = threading.Event()
+ extra.generations.add(on_slot)
+ forced = reload.model_copy(update = {"force_cancel_active": True})
+ assert asyncio.run(inf.load_model_gated(forced, None, "s")) == "loaded"
+ assert on_slot.is_set() and not on_primary.is_set()
+
+
+def test_training_sizes_and_frees_the_models_kept_alongside(backends, monkeypatch):
+ primary, extra = backends
+ primary.unload_model()
+ # A loaded one is in the free VRAM training reads; only a still-loading one is unsizable.
+ summary = training_vram.summarize_resident_chat()
+ assert summary["any"] and not summary["loading"]
+ monkeypatch.setattr(model_slots, "loading", (extra, "org/D-GGUF"))
+ assert training_vram.summarize_resident_chat()["loading"]
+ monkeypatch.setattr(model_slots, "loading", None)
+ extra.orchestrator.loading_models = {"org/C"}
+ assert training_vram.summarize_resident_chat()["loading"]
+ extra.orchestrator.loading_models = set()
+ assert training_vram.free_chat_models_for_training("test") == ["kept:org/B-GGUF"]
+ assert model_slots.slots == [] and not extra.llama.is_active
+
+
+def test_training_frees_the_kept_models_before_the_active_one(backends, monkeypatch):
+ primary, extra = backends
+ verdicts = [False, True]
+ monkeypatch.setattr(training_vram, "summarize_resident_stt", lambda: {"any": False})
+ freed = training_vram.coordinate_models_for_training(lambda: (verdicts.pop(0), {}))
+ assert freed == ["kept:org/B-GGUF"] and verdicts == []
+ assert model_slots.slots == [] and primary.is_active
+
+
+def test_another_quant_of_a_loaded_model_replaces_it_in_place(backends):
+ _, extra = backends
+ primary_quant = LoadRequest(model_path = "org/A-GGUF", gguf_variant = "Q8_0", alongside = True)
+ assert _selected(primary_quant) is None
+ slot_quant = LoadRequest(model_path = "org/B-GGUF", gguf_variant = "Q4_K_M", alongside = True)
+ assert _selected(slot_quant) is extra
+ assert _selected(slot_quant.model_copy(update = {"alongside": False})) is extra
+
+
+def test_an_alongside_load_counts_as_activity(backends, monkeypatch):
+ stamped = []
+ monkeypatch.setattr(keepwarm, "_note_activity", lambda: stamped.append(True))
+ _gated_load_fakes(monkeypatch, short_fits = [])
+ request = LoadRequest(model_path = "org/C-GGUF", alongside = True)
+ assert asyncio.run(inf.load_model_gated(request, None, "s")) == "loaded"
+ assert stamped == [True]
+
+
+def test_a_chat_starting_on_a_victim_during_eviction_spares_it(backends, monkeypatch):
+ _, extra = backends
+ extra.last_used = 9.0
+ older = _slot("org/D-GGUF", last_used = 1.0)
+ older.llama._planned_vram_mib = {0: 3000}
+ mid = _slot("org/E-GGUF", last_used = 2.0)
+ mid.llama._planned_vram_mib = {0: 3000}
+ model_slots.slots += [older, mid]
+ unload = older.llama.unload_model
+
+ def chat_starts_on_mid():
+ mid.generations.add(threading.Event())
+ unload()
+
+ older.llama.unload_model = chat_starts_on_mid
+ _gated_load_fakes(monkeypatch, short_fits = [1])
+ request = LoadRequest(model_path = "org/C-GGUF", alongside = True)
+ assert asyncio.run(inf.load_model_gated(request, None, "s")) == "loaded"
+ assert mid in model_slots.slots and mid.llama.is_active
+
+
+def test_a_routed_request_holds_its_slot_until_it_ends(backends):
+ _, extra = backends
+ routed, ended = asyncio.Event(), asyncio.Event()
+ seen = {}
+
+ async def request():
+ await inf._route_to_extra_slot("org/B-GGUF")
+ routed.set()
+ await ended.wait()
+
+ async def main():
+ task = asyncio.create_task(request())
+ # Routing hops through a thread: wait for it, not a fixed number of loop turns.
+ await routed.wait()
+ seen["refs"] = extra.refs
+ seen["victims"] = model_slots.eviction_victims(None, 5000)
+ seen["claimed"] = model_slots.claim_victim(extra)
+ ended.set()
+ await task
+
+ asyncio.run(main())
+ assert seen == {"refs": 1, "victims": [], "claimed": False}
+ assert extra.refs == 0 and model_slots.eviction_victims(None, 5000) == [extra]
+
+
+def test_a_chat_run_holds_its_slot_after_its_post_has_answered(backends):
+ # A durable chat run's task starts inside POST /chat-runs, which answers 202 before the run routes
+ # and preprocesses; the slot must stay held through that window, not end with the POST.
+
+ _, extra = backends
+ seen = {}
+
+ async def run():
+ await asyncio.sleep(0)
+ await inf._route_to_extra_slot("org/B-GGUF")
+ await asyncio.sleep(0)
+ seen["refs"] = extra.refs
+ seen["victims"] = model_slots.eviction_victims(None, 5000)
+
+ async def post_chat_run():
+ keepwarm.set_current_response_scope({})
+ task = asyncio.create_task(run())
+ keepwarm.set_current_response_scope(None)
+ await task
+
+ asyncio.run(post_chat_run())
+ assert seen == {"refs": 1, "victims": []}
+ assert extra.refs == 0
+
+
+def test_a_slot_evicted_while_a_request_routes_is_not_served(backends, monkeypatch):
+ _, extra = backends
+ probe = model_slots.serving_slot
+
+ def evicted_mid_probe(requested, slots, satisfies):
+ found = probe(requested, slots, satisfies)
+ assert model_slots.claim_victim(extra)
+ return found
+
+ monkeypatch.setattr(model_slots, "serving_slot", evicted_mid_probe)
+ assert _routed("org/B-GGUF") == (None, backends[0])
+
+
+def test_eviction_skips_other_gpus_and_victims_that_cannot_make_room(backends):
+ _, extra = backends
+ extra.llama._planned_vram_mib = {1: 9000}
+ other = _slot("org/D-GGUF", last_used = 5.0)
+ other.llama._planned_vram_mib = {0: 6000}
+ model_slots.slots.append(other)
+ assert model_slots.eviction_victims(None, 5000, (0,)) == [other]
+ assert model_slots.eviction_victims(None, 5000) == [extra]
+ # Victims that together cannot make room are left loaded.
+ assert model_slots.eviction_victims(None, 20000) == []
+
+
+def test_a_token_count_waits_only_on_its_own_models_chats(backends):
+ _, extra = backends
+
+ async def count_on(model):
+ await inf._route_to_extra_slot(model)
+ return model_slots.routed_generation_count()
+
+ on_primary = threading.Event()
+ with active_generations.ActiveGeneration(on_primary, thread_id = "chat-on-A"):
+ assert asyncio.run(count_on("org/B-GGUF")) == 0
+ assert asyncio.run(count_on("org/A-GGUF")) == 1
+ on_slot = threading.Event()
+ with active_generations.ActiveGeneration(on_slot, thread_id = "chat-on-B"):
+ extra.generations.add(on_slot)
+ assert asyncio.run(count_on("org/B-GGUF")) == 1
+ assert asyncio.run(count_on("org/A-GGUF")) == 0
+
+
+def test_counting_for_an_idle_kept_model_ignores_the_primarys_chat(backends):
+ payload = ChatCountTokensRequest(
+ model = "org/B-GGUF",
+ messages = [{"role": "user", "content": "hi"}],
+ )
+ with active_generations.ActiveGeneration(threading.Event(), thread_id = "chat-on-A"):
+ try:
+ asyncio.run(inf.chat_count_tokens(payload, "s", None))
+ except HTTPException as exc:
+ assert exc.detail != "Cannot count tokens while a generation is in progress."
+ except Exception:
+ pass
+
+
+def test_a_model_still_loading_alongside_keeps_the_chat_claim(backends, monkeypatch):
+ primary, extra = backends
+ _hold_chat_claim(monkeypatch)
+ primary.unload_model()
+ extra.llama.unload_model()
+ extra.orchestrator.loading_models = {"org/C"}
+ inf.release_chat_gpu_claim()
+ assert gpu_arbiter.current_owner() == gpu_arbiter.CHAT
+ extra.orchestrator.loading_models = set()
+ monkeypatch.setattr(model_slots, "loading", (extra, "org/C"))
+ inf.release_chat_gpu_claim()
+ assert gpu_arbiter.current_owner() == gpu_arbiter.CHAT
+ monkeypatch.setattr(model_slots, "loading", None)
+ inf.release_chat_gpu_claim()
+ assert gpu_arbiter.current_owner() != gpu_arbiter.CHAT
+
+
+def test_a_llama_update_stops_every_llama_slot_and_only_those(backends, monkeypatch):
+ _, extra = backends
+ safetensors = model_slots.ExtraSlot(FakeLlama(), FakeOrchestrator("org/C"), "owner")
+ starting = model_slots.ExtraSlot(FakeLlama(), FakeOrchestrator(), "owner")
+ model_slots.slots += [safetensors, starting]
+ monkeypatch.setattr(model_slots, "loading", (starting, "org/D-GGUF"))
+ assert model_slots.unload_llama_slots() == 2
+ assert model_slots.slots == [safetensors]
+ loading = model_slots.ExtraSlot(FakeLlama(), FakeOrchestrator(), "owner")
+ loading.orchestrator.loading_models = {"org/E"}
+ model_slots.slots.append(loading)
+ monkeypatch.setattr(model_slots, "loading", (loading, "org/E"))
+ assert model_slots.unload_llama_slots() == 0
+
+
+def test_deleting_a_model_another_slot_serves_or_fills_is_refused(backends, monkeypatch):
+ _, extra = backends
+ assert deletion._llama_cpp_blocks_delete("org/B-GGUF", None)
+ assert deletion._llama_cpp_blocks_delete("org/B-GGUF", "Q8_0")
+ assert not deletion._llama_cpp_blocks_delete("org/B-GGUF", "Q4_K_M")
+ assert not deletion._llama_cpp_blocks_delete("org/Z-GGUF", None)
+ model_slots.slots.append(model_slots.ExtraSlot(FakeLlama(), FakeOrchestrator("org/C"), "owner"))
+ assert deletion._inference_backend_blocks_delete("org/C")
+ assert not deletion._inference_backend_blocks_delete("org/Z")
+ starting = model_slots.ExtraSlot(FakeLlama(), FakeOrchestrator(), "owner")
+ model_slots.slots.append(starting)
+ monkeypatch.setattr(model_slots, "loading", (starting, "org/D-GGUF"))
+ assert deletion._llama_cpp_blocks_delete("org/D-GGUF", "Q4_K_M")
+ monkeypatch.setattr(model_slots, "loading", None)
+ assert not deletion._llama_cpp_blocks_delete("org/D-GGUF", "Q4_K_M")
+ starting.orchestrator.loading_models = {"org/E"}
+ assert deletion._inference_backend_blocks_delete("org/E")
+
+
+def test_clearing_the_cache_waits_for_models_kept_alongside(backends, monkeypatch):
+ primary, extra = backends
+ monkeypatch.setattr(llama_cpp, "chat_load_active", lambda: False)
+ primary.unload_model()
+ assert deletion.any_model_load_blocks_cache_clear() == (
+ "Unload the model before clearing the model cache"
+ )
+ extra.llama.unload_model()
+ monkeypatch.setattr(model_slots, "loading", (extra, "org/B-GGUF"))
+ assert "load" in deletion.any_model_load_blocks_cache_clear()
+
+
+def test_the_model_list_marks_kept_models_loaded(backends, monkeypatch):
+ import routes.models as models_routes
+
+ _, extra = backends
+ fake = FakeOrchestrator()
+ fake.default_models = []
+ monkeypatch.setattr(models_routes, "get_inference_backend", lambda: fake)
+ model_slots.slots.append(model_slots.ExtraSlot(FakeLlama(), FakeOrchestrator("org/C"), "owner"))
+ app = FastAPI()
+ app.include_router(models_routes.router, prefix = "/api/models")
+ app.dependency_overrides[get_current_subject] = lambda: "test-subject"
+ with TestClient(app) as client:
+ ids = {m["id"] for m in client.get("/api/models/list").json()["models"]}
+ assert {"org/A-GGUF", "org/B-GGUF", "org/C"} <= ids
+
+
+def test_a_model_that_does_not_fit_says_so_without_api_flags():
+ from core.inference.llama_cpp import _gpu_short_message
+
+ spill = _gpu_short_message(13.2, 8.4, 32768, 32768, False)
+ assert "13.2 GB" in spill and "8.4 GB" in spill and "Keep other models loaded" in spill
+ capped = _gpu_short_message(13.2, 8.4, 32768, 8192, True)
+ assert "8192 context" in capped
+ assert "force_alongside" not in spill + capped
+
+
+def test_a_slot_load_during_a_llama_update_is_refused(backends, monkeypatch):
+ primary, extra = backends
+ _gated_load_fakes(monkeypatch, short_fits = [])
+ seen = []
+ load = inf._run_tracked_load_model_impl
+
+ async def noting_update(request, *args, **kwargs):
+ seen.append(inf.get_llama_cpp_backend()._llama_update_in_progress)
+ return await load(request, *args, **kwargs)
+
+ monkeypatch.setattr(inf, "_run_tracked_load_model_impl", noting_update)
+ reload = LoadRequest(model_path = "org/B-GGUF", gguf_variant = "Q8_0", force_reload = True)
+ primary._llama_update_in_progress = True
+ for request in (LoadRequest(model_path = "org/C-GGUF", alongside = True), reload):
+ asyncio.run(inf.load_model_gated(request, None, "s"))
+ primary._llama_update_in_progress = False
+ asyncio.run(inf.load_model_gated(reload, None, "s"))
+ assert seen == [True, True, False]
+
+
+def test_a_resident_npu_model_keeps_to_the_primarys_seat(backends, monkeypatch):
+ primary, extra = backends
+ primary.unload_model()
+ npu_model = SimpleNamespace(model_path = "lemonade:qwen3-0.6b-FLM", id = "qwen3-0.6b-FLM")
+ npu = SimpleNamespace(is_loaded = True, loaded_model = npu_model, resident = lambda: None)
+ monkeypatch.setattr(npu_backend, "peek_npu_backend", lambda: npu)
+
+ async def route(model):
+ slot = await inf._route_to_extra_slot(model)
+ return slot, inf._resident_npu_model(), inf._loaded_slot_ident()
+
+ assert asyncio.run(route("org/B-GGUF")) == (extra, None, "org/B-GGUF")
+ assert asyncio.run(route("qwen3-0.6b-FLM")) == (None, npu_model, npu_model.model_path)
+
+
+def test_the_loaded_models_list_names_the_npu_model_once(backends, monkeypatch):
+ primary, _ = backends
+ primary.unload_model()
+ model_slots.slots.append(
+ model_slots.ExtraSlot(FakeLlama("org/D-GGUF"), FakeOrchestrator(), "owner")
+ )
+ npu_model = SimpleNamespace(
+ model_path = "lemonade:qwen3-0.6b-FLM",
+ id = "qwen3-0.6b-FLM",
+ vision = False,
+ max_context_length = 4096,
+ reasoning = False,
+ tools = False,
+ )
+ resident = SimpleNamespace(model = npu_model, context_length = 4096)
+ npu = SimpleNamespace(is_loaded = True, loaded_model = npu_model, resident = lambda: resident)
+ monkeypatch.setattr(npu_backend, "peek_npu_backend", lambda: npu)
+ app = FastAPI()
+ app.include_router(inf.studio_router, prefix = "/api/inference")
+ app.dependency_overrides[get_current_subject] = lambda: "test-subject"
+ with TestClient(app) as client:
+ ids = [entry["id"] for entry in client.get("/api/inference/loaded-models").json()["data"]]
+ assert ids.count("lemonade:qwen3-0.6b-FLM") == 1 and "org/B-GGUF" in ids and "org/D-GGUF" in ids
+
+
+def test_a_load_over_a_streaming_npu_model_asks_before_stopping_it(monkeypatch, tmp_path):
+ import struct
+
+ npu = SimpleNamespace(
+ is_loaded = True,
+ loaded_model = SimpleNamespace(model_path = "lemonade:qwen3-0.6b-FLM", id = "qwen3-0.6b-FLM"),
+ resident = lambda: None,
+ unload = lambda: None,
+ )
+ monkeypatch.setattr(orchestrator, "_inference_backend", FakeOrchestrator())
+ monkeypatch.setattr(npu_backend, "peek_npu_backend", lambda: npu)
+ monkeypatch.setattr(inf, "_raise_if_sidecar_swap_in_progress", lambda: None)
+ torn_down = []
+
+ async def _teardown():
+ torn_down.append(True)
+ raise RuntimeError("NPU unloaded while its chat is still streaming")
+
+ monkeypatch.setattr(inf, "_unload_npu_before_local_load", _teardown)
+ asked = []
+
+ def _gate(*, cancel):
+ asked.append(cancel)
+ if active_generations.count() and not cancel:
+ raise LookupError("refused")
+ return 0
+
+ def _s(x):
+ return struct.pack("<Q", len(x)) + x.encode()
+
+ gguf = tmp_path / "tiny.gguf"
+ gguf.write_bytes(
+ b"GGUF"
+ + struct.pack("<IQQ", 3, 0, 1)
+ + _s("general.architecture")
+ + struct.pack("<I", 8)
+ + _s("llama")
+ )
+
+ async def _load():
+ with active_generations.ActiveGeneration(
+ threading.Event(), thread_id = "t1", run_id = "r1", model = "npu", kind = "chat"
+ ):
+ with pytest.raises(Exception) as exc:
+ await inf._load_model_impl(
+ LoadRequest(model_path = str(gguf)), None, "s", on_reload_confirmed = _gate
+ )
+ return exc
+
+ exc = asyncio.run(_load())
+ assert asked == [False] and not torn_down, exc.value
+
+
+def test_active_generations_for_a_model_lists_only_its_chats(backends):
+ _, extra = backends
+ app = FastAPI()
+ app.include_router(inf.studio_router, prefix = "/api/inference")
+ app.dependency_overrides[get_current_subject] = lambda: "test-subject"
+ on_a, on_b = threading.Event(), threading.Event()
+ extra.generations.add(on_b)
+ with (
+ active_generations.ActiveGeneration(on_a, thread_id = "chat-on-A"),
+ active_generations.ActiveGeneration(on_b, thread_id = "chat-on-B"),
+ TestClient(app) as client,
+ ):
+ get = lambda q: client.get("/api/inference/active-generations" + q).json()["thread_ids"]
+ assert get("?model=org/B-GGUF") == ["chat-on-B"]
+ assert get("?model=org/A-GGUF") == ["chat-on-A"]
+ assert sorted(get("")) == ["chat-on-A", "chat-on-B"]
+ # A name nothing serves (a row just evicted): its unload stops nothing.
+ assert get("?model=org/gone-GGUF") == []
+ extra.llama.effective_parallel_slots = 4
+ slots = client.get("/api/inference/active-generations?model=org/B-GGUF").json()
+ assert slots["parallel_slots"] == 4
+
+
+def test_an_integrated_gpu_keeps_its_free_memory_next_to_a_loaded_model():
+ from core.inference.llama_cpp import _net_of_held_vram
+
+ held = {0: 900}
+ # Discrete: capped at total less what the other model planned.
+ assert _net_of_held_vram([(0, 20_000, 24_000)], held) == [(0, 20_000, 24_000)]
+ assert _net_of_held_vram([(0, 23_500, 24_000)], held) == [(0, 23_100, 24_000)]
+ # Integrated (total 0, shared RAM): the free reading is left alone, not zeroed.
+ assert _net_of_held_vram([(0, 60_000, 0)], held) == [(0, 60_000, 0)]
+ assert _net_of_held_vram([(0, 60_000, 0)], {}) == [(0, 60_000, 0)]
+
+
+def test_a_model_that_fits_one_card_takes_one_no_other_model_runs_on():
+ from core.inference.llama_cpp import LlamaCppBackend
+
+ gpus = [(0, 170_000), (1, 150_000), (2, 20_000)]
+ total = {0: 180_000, 1: 180_000, 2: 180_000}
+ pick = lambda **kw: LlamaCppBackend._select_gpus(
+ 10 * 1024**3, gpus, usable_fraction = 0.9, total_by_idx = total, **kw
+ )
+ # Alone, the most free card, as before.
+ assert pick() == ([0], False)
+ # Card 0 serves another model: the next card that holds it alone.
+ assert pick(shared = frozenset({0})) == ([1], False)
+ # Every card that could hold it is shared: still the most free one, never a split.
+ assert pick(shared = frozenset({0, 1})) == ([0], False)
+ assert LlamaCppBackend._select_gpus_split_aware(
+ 10 * 1024**3, gpus, usable_fraction = 0.9, total_by_idx = total, shared = frozenset({0})
+ ) == ([1], False)
+
+
+def test_a_public_preview_never_reaches_a_model_kept_alongside(backends):
+ primary, extra = backends
+ preview = SimpleNamespace(scope = {}, headers = {}, state = SimpleNamespace())
+ inf.disable_openai_auto_switch_for_request(preview.scope)
+
+ async def chat(model):
+ await inf._maybe_auto_switch_model(model, preview, "s")
+ return inf.get_llama_cpp_backend()
+
+ assert asyncio.run(chat("org/B-GGUF")) is primary
+
+
+def test_a_transformers_install_stops_the_workers_of_models_kept_alongside(backends, monkeypatch):
+ import core.export as export
+ import core.training as training
+ import utils.transformers_latest as transformers_latest
+ from models.inference import InstallLatestTransformersRequest
+
+ kept = model_slots.ExtraSlot(FakeLlama(), FakeOrchestrator("org/C"), "owner")
+ kept.orchestrator.is_worker_alive = lambda: kept.orchestrator.active_model_name is not None
+ kept.orchestrator._cleanup = lambda: setattr(kept.orchestrator, "active_model_name", None)
+ model_slots.slots.append(kept)
+ idle = SimpleNamespace(
+ is_training_active = lambda: False,
+ is_export_active = lambda: False,
+ current_checkpoint = None,
+ cleanup_memory = lambda: None,
+ is_worker_alive = lambda: False,
+ )
+ monkeypatch.setattr(training, "get_training_backend", lambda: idle)
+ monkeypatch.setattr(export, "get_export_backend", lambda: idle)
+ monkeypatch.setattr(keepwarm, "other_inference_request_count", lambda **kwargs: 0)
+ alive_at_swap = []
+
+ def install(version, before_swap, *args):
+ before_swap()
+ alive_at_swap.append(kept.orchestrator.is_worker_alive())
+ return {"success": False, "message": "stop here"}
+
+ monkeypatch.setattr(transformers_latest, "install_latest_transformers", install)
+ with pytest.raises(HTTPException):
+ asyncio.run(
+ inf.install_latest_transformers_route(
+ InstallLatestTransformersRequest(version = "9.9.9"), "s"
+ )
+ )
+ assert alive_at_swap == [False] and kept not in model_slots.slots
+
+
+def test_a_zero_vram_primary_keeps_the_chat_claim_a_kept_model_holds(backends, monkeypatch):
+ _, extra = backends
+ _hold_chat_claim(monkeypatch)
+ source = inspect.getsource(inf._load_model_impl)
+ assert "release, CHAT)" not in source and "release(CHAT)" not in source
+ assert source.count("_release_chat_for_zero_vram_primary") == 3
+ inf._release_chat_for_zero_vram_primary()
+ assert gpu_arbiter.current_owner() == gpu_arbiter.CHAT
+ model_slots.drop(extra)
+ inf._release_chat_for_zero_vram_primary()
+ assert gpu_arbiter.current_owner() is None
+
+
[email protected]("primary_on_gpu", [False, True])
+def test_ejecting_the_last_kept_model_beside_a_cpu_primary_drops_the_claim(
+ backends, monkeypatch, primary_on_gpu
+):
+ primary, extra = backends
+ _hold_chat_claim(monkeypatch)
+ primary.holds_no_vram = not primary_on_gpu
+ response = asyncio.run(inf._unload_model_impl(UnloadRequest(model_path = "org/B-GGUF"), "s"))
+ assert response.status == "unloaded" and primary.is_loaded
+ expected = gpu_arbiter.CHAT if primary_on_gpu else None
+ assert gpu_arbiter.current_owner() == expected
+
+
+def test_turning_the_setting_off_drops_the_claim_its_kept_models_held(backends, monkeypatch):
+ from fastapi import BackgroundTasks
+ import routes.settings as settings_routes
+
+ primary, extra = backends
+ _hold_chat_claim(monkeypatch)
+ # The primary was ejected earlier and kept CHAT for the model still loaded beside it.
+ primary.unload_model()
+ monkeypatch.setattr(settings_routes, "set_multi_model_enabled", lambda value: value)
+ tasks = BackgroundTasks()
+ settings_routes.update_multi_model(settings_routes.MultiModelPayload(enabled = False), tasks, "s")
+ assert gpu_arbiter.current_owner() == gpu_arbiter.CHAT
+ asyncio.run(tasks())
+ assert model_slots.slots == [] and gpu_arbiter.current_owner() is None
+
+
+def test_a_trained_model_still_filling_a_slot_cannot_be_deleted(backends, monkeypatch, tmp_path):
+ import routes.models as models_routes
+
+ run = tmp_path / "run1"
+ run.mkdir()
+ monkeypatch.setattr(models_routes, "outputs_root", lambda: tmp_path)
+ starting = model_slots.ExtraSlot(FakeLlama(), FakeOrchestrator(), "owner")
+ model_slots.slots.append(starting)
+ monkeypatch.setattr(model_slots, "loading", (starting, str(run)))
+ with pytest.raises(HTTPException) as excinfo:
+ asyncio.run(
+ models_routes.delete_finetuned_model(
+ model_path = str(run), source = "training", current_subject = "s"
+ )
+ )
+ assert excinfo.value.status_code == 409 and run.exists()
+
+
+def test_reloading_a_kept_model_leaves_the_primarys_npu_model_loaded(
+ backends, monkeypatch, tmp_path
+):
+ import struct
+
+ primary, extra = backends
+ primary.unload_model()
+ unloaded = []
+ npu = SimpleNamespace(
+ is_loaded = True,
+ loaded_model = SimpleNamespace(model_path = "lemonade:qwen3-0.6b-FLM", id = "qwen3-0.6b-FLM"),
+ resident = lambda: None,
+ unload = lambda: unloaded.append(True),
+ )
+ monkeypatch.setattr(npu_backend, "peek_npu_backend", lambda: npu)
+
+ def _s(x):
+ return struct.pack("<Q", len(x)) + x.encode()
+
+ gguf = tmp_path / "tiny.gguf"
+ gguf.write_bytes(
+ b"GGUF"
+ + struct.pack("<IQQ", 3, 0, 1)
+ + _s("general.architecture")
+ + struct.pack("<I", 8)
+ + _s("llama")
+ )
+
+ from core.inference.llama_cpp import LlamaCppBackend
+
+ extra.llama = LlamaCppBackend(manages_processes = False)
+ monkeypatch.setattr(inf, "_llama_cpp_backend", LlamaCppBackend(manages_processes = False))
+ monkeypatch.setattr(inf, "LlamaCppBackend", LlamaCppBackend)
+
+ async def reload_kept():
+ inf.routed_slot.set(extra)
+ with pytest.raises(Exception):
+ await inf._load_model_impl(
+ LoadRequest(model_path = str(gguf), force_reload = True),
+ None,
+ "s",
+ on_reload_confirmed = lambda *, cancel: 0,
+ )
+
+ asyncio.run(reload_kept())
+ assert unloaded == []
+
+
+def test_an_api_key_status_does_not_leak_a_local_path_through_serving():
+ from hub.utils.host_paths import redact_host_paths
+
+ local = "/home/alice/models/my-finetune"
+ status = InferenceStatusResponse(
+ active_model = local, loaded = [local], serving = [local, "org/B-GGUF"]
+ )
+ redacted = redact_host_paths(status, via_api_key = True)
+ assert local not in redacted["active_model"] and local not in redacted["loaded"]
+ assert local not in redacted["serving"] and "org/B-GGUF" in redacted["serving"]
+ assert redact_host_paths(status, via_api_key = False).serving == [local, "org/B-GGUF"]
+
+
+def test_a_managed_engine_load_never_lands_in_a_kept_slot(backends):
+ _, extra = backends
+ kept = LoadRequest(model_path = "org/B-GGUF", gguf_variant = "Q8_0", engine = "vllm")
+ assert _selected(kept) is None
+ assert _selected(kept.model_copy(update = {"alongside": True})) is None
+
+
+def test_a_managed_engine_load_refuses_while_a_kept_model_generates(backends):
+ _, extra = backends
+ generating = threading.Event()
+ extra.generations.add(generating)
+ request = LoadRequest(model_path = "org/C", engine = "vllm")
+ with pytest.raises(HTTPException) as excinfo:
+ inf._gate_kept_models(request, cancel = False)
+ assert excinfo.value.status_code == 409 and not generating.is_set()
+ forced = request.model_copy(update = {"force_cancel_active": True})
+ inf._gate_kept_models(forced, cancel = False)
+ assert not generating.is_set() and model_slots.slots == [extra]
+ asyncio.run(inf._retire_kept_models(forced))
+ assert generating.is_set() and model_slots.slots == [] and not extra.llama.is_active
+
+
+def test_kept_models_go_only_once_nothing_can_reject_a_managed_load():
+ source = inspect.getsource(inf._load_model_impl)
+ gate = source.index("_gate_kept_models(request, cancel = False)")
+ primary_cancel = source.index("on_reload_confirmed(cancel = True)", gate)
+ retire = source.index("await _retire_kept_models(request)", primary_cancel)
+ assert source.index("_reject_unsupported_managed_kind(request, config)") < gate
+ assert source.index("account_access.require_idle_other_accounts()") < gate < retire
+
+
+def test_a_kept_model_that_would_not_unload_is_retried_and_still_priced(backends, monkeypatch):
+ _, extra = backends
+ registered = []
+ monkeypatch.setattr(llama_cpp, "unregister_serving_backend", registered.append)
+ real = extra.llama.unload_model
+ monkeypatch.setattr(
+ extra.llama, "unload_model", lambda: (_ for _ in ()).throw(RuntimeError("stuck"))
+ )
+ with pytest.raises(RuntimeError):
+ model_slots.unload_extra_models(strict = True)
+ assert model_slots.slots == [] and model_slots.stuck == [extra] and registered == []
+ assert model_slots.holds_vram()
+ monkeypatch.setattr(extra.llama, "unload_model", real)
+ assert model_slots.unload_extra_models(strict = True) == 0
+ assert model_slots.stuck == [] and registered == [extra.llama]
+
+
+def test_a_gpu_handoff_stops_on_a_kept_model_that_would_not_unload(backends, monkeypatch):
+ _, extra = backends
+
+ def stuck():
+ raise RuntimeError("llama-server ignored SIGKILL")
+
+ monkeypatch.setattr(extra.llama, "unload_model", stuck)
+ assert model_slots.unload_extra_models() == 1
+ model_slots.slots.append(extra)
+ with pytest.raises(RuntimeError, match = "kept alongside"):
+ model_slots.unload_extra_models(strict = True)
+
+
+def test_a_non_gguf_load_out_of_memory_makes_room_then_replaces(backends, monkeypatch):
+ primary, extra = backends
+ extra.request = LoadRequest(model_path = "org/B-GGUF", gguf_variant = "Q8_0")
+ monkeypatch.setattr(inf, "release_chat_gpu_claim", lambda: None)
+ failures = [HTTPException(500, "Failed to load model: CUDA out of memory")] * 2
+ seen = []
+
+ async def load(request, *args, **kwargs):
+ seen.append((request.model_path, request.alongside))
+ if request.alongside and failures:
+ raise failures.pop()
+ primary.model_identifier, primary.is_loaded = request.model_path, True
+ return LoadResponse.model_construct(status = "loaded", model = request.model_path)
+
+ monkeypatch.setattr(inf, "_run_tracked_load_model_impl", load)
+ response = asyncio.run(
+ inf.load_model_gated(LoadRequest(model_path = "org/C", alongside = True), None, "s")
+ )
+ assert response.evicted == ["org/B-GGUF:Q8_0", "org/A-GGUF"] and model_slots.slots == []
+ assert seen == [("org/C", True), ("org/C", True), ("org/C", False)]
+ other = HTTPException(500, "Failed to load model: no such repo")
+ failures[:] = [other]
+ with pytest.raises(HTTPException):
+ asyncio.run(
+ inf.load_model_gated(LoadRequest(model_path = "org/D", alongside = True), None, "s")
+ )
+
+
+def test_turning_the_setting_off_unloads_only_idle_kept_models(backends):
+ _, extra = backends
+ busy = _slot("org/D-GGUF")
+ busy.generations.add(threading.Event())
+ model_slots.slots.append(busy)
+ assert model_slots.unload_idle() == 1
+ assert model_slots.slots == [busy] and not extra.llama.is_active
+
+
+def test_the_replace_fallback_refuses_while_the_active_model_generates(backends, monkeypatch):
+ primary, extra = backends
+ extra.request = LoadRequest(model_path = "org/B-GGUF", gguf_variant = "Q8_0")
+ _gated_load_fakes(monkeypatch, short_fits = [1, 1])
+ seen = []
+ monkeypatch.setattr(
+ inf,
+ "_raise_or_cancel_active_generations",
+ lambda **kw: seen.append(kw) or (_ for _ in ()).throw(HTTPException(409, "chats running")),
+ )
+ with pytest.raises(HTTPException) as excinfo:
+ asyncio.run(
+ inf.load_model_gated(LoadRequest(model_path = "org/C-GGUF", alongside = True), None, "s")
+ )
+ assert excinfo.value.status_code == 409 and seen[0]["cancel"] is False
+ assert primary.model_identifier == "org/A-GGUF" and primary.is_loaded
+ assert model_slots.slots == []
+
+
+def test_training_stops_on_a_kept_model_that_would_not_unload(backends, monkeypatch):
+ _, extra = backends
+
+ def stuck():
+ raise RuntimeError("llama-server ignored SIGKILL")
+
+ monkeypatch.setattr(extra.llama, "unload_model", stuck)
+ with pytest.raises(training_vram.ManagedEngineStillRunning):
+ training_vram.free_kept_models_for_training("test")
+
+
+def test_training_sees_and_retries_a_kept_model_that_failed_to_unload(backends, monkeypatch):
+ primary, extra = backends
+ primary.unload_model()
+
+ def refuse():
+ raise RuntimeError("llama-server ignored SIGKILL")
+
+ monkeypatch.setattr(extra.llama, "unload_model", refuse)
+ with pytest.raises(RuntimeError):
+ model_slots.drop(extra)
+ assert model_slots.slots == [] and model_slots.stuck == [extra]
+ _hold_chat_claim(monkeypatch)
+ inf._release_chat_for_zero_vram_primary()
+ assert gpu_arbiter.current_owner() == gpu_arbiter.CHAT
+ # Out of routing, but its server may still hold VRAM: training counts it and stops on it.
+ assert training_vram.summarize_resident_chat()["gguf"] == "org/B-GGUF"
+ monkeypatch.setattr(training_vram, "summarize_resident_stt", lambda: {"any": False})
+ with pytest.raises(training_vram.ManagedEngineStillRunning):
+ training_vram.coordinate_models_for_training(lambda: (False, {}))
+ # Once it stops, the retry clears it.
+ monkeypatch.delattr(extra.llama, "unload_model")
+ assert training_vram.free_kept_models_for_training("test") == ["kept:org/B-GGUF"]
+ assert model_slots.stuck == [] and not extra.llama.is_active
+
+
+def test_a_kept_worker_that_outlives_its_kill_stays_tracked(backends, monkeypatch):
+ _, extra = backends
+ extra.orchestrator.active_model_name = "org/S"
+ alive = [True]
+ extra.orchestrator.is_worker_alive = lambda: alive[0]
+ extra.orchestrator._cleanup = lambda: setattr(extra.orchestrator, "active_model_name", None)
+ with pytest.raises(RuntimeError):
+ model_slots.unload_extra_models(strict = True)
+ # Still priced for VRAM and retried, though _cleanup cleared its model name.
+ assert model_slots.slots == [] and model_slots.stuck == [extra] and model_slots.holds_vram()
+ alive[0] = False
+ model_slots.unload_extra_models(strict = True)
+ assert model_slots.stuck == [] and not model_slots.holds_vram()
+
+
+def test_a_stuck_server_still_blocks_deleting_its_files(backends, monkeypatch):
+ _, extra = backends
+
+ def refuse():
+ raise RuntimeError("llama-server ignored SIGKILL")
+
+ monkeypatch.setattr(extra.llama, "unload_model", refuse)
+ with pytest.raises(RuntimeError):
+ model_slots.drop(extra)
+ assert model_slots.stuck == [extra]
+ assert deletion._llama_cpp_blocks_delete("org/B-GGUF", None)
+
+
+def test_a_partly_offloaded_model_plans_only_the_vram_its_cards_had():
+ from core.inference.llama_cpp import _gpu_plan_mib
+
+ gib = 1024**3
+ assert _gpu_plan_mib(20 * gib, [0, 1], [(0, 30_000), (1, 30_000)]) == {0: 10240, 1: 10240}
+ assert _gpu_plan_mib(20 * gib, [0], [(0, 6_000)]) == {0: 6_000}
+ assert _gpu_plan_mib(4 * gib, [2], []) == {2: 4096}
+
+
+def test_the_replace_fallback_frees_the_active_model_before_placing_the_new_one(
+ backends, monkeypatch
+):
+ primary, extra = backends
+ _gated_load_fakes(monkeypatch, short_fits = [1, 1])
+ seen = []
+ real = inf._run_tracked_load_model_impl
+
+ async def load(request, *args, **kwargs):
+ if not request.alongside:
+ seen.append(primary.is_active)
+ return await real(request, *args, **kwargs)
+
+ monkeypatch.setattr(inf, "_run_tracked_load_model_impl", load)
+ asyncio.run(
+ inf.load_model_gated(LoadRequest(model_path = "org/C-GGUF", alongside = True), None, "s")
+ )
+ assert seen == [False] and primary.model_identifier == "org/C-GGUF"
diff --git a/studio/backend/tests/test_orchestrator_unload_cancel.py b/studio/backend/tests/test_orchestrator_unload_cancel.py
index 2bf4d2ebd9a..8b89035b37d 100644
--- a/studio/backend/tests/test_orchestrator_unload_cancel.py
+++ b/studio/backend/tests/test_orchestrator_unload_cancel.py
@@ -3540,7 +3540,7 @@ def test_a_llama_server_spawned_as_shutdown_began_is_reaped():
)
body = textwrap.dedent(ast.get_source_segment(src, fn) or "")
- publish = body.index("self._record_server_pid(")
+ publish = body.index("self._note_server_pid(")
recheck = body.index("_spawn_is_stale", publish)
kill = body.index("self._kill_process()", publish)
assert publish < recheck < kill, (
@@ -3631,7 +3631,7 @@ def test_the_primary_llama_launch_rechecks_after_recording_the_pid():
record = [
n.lineno
for n in ast.walk(fn)
- if isinstance(n, ast.Call) and getattr(n.func, "attr", None) == "_record_server_pid"
+ if isinstance(n, ast.Call) and getattr(n.func, "attr", None) == "_note_server_pid"
]
stale = [
n.lineno
diff --git a/studio/backend/utils/llama_cpp_update.py b/studio/backend/utils/llama_cpp_update.py
index f6835e79a15..a147b37ed5d 100644
--- a/studio/backend/utils/llama_cpp_update.py
+++ b/studio/backend/utils/llama_cpp_update.py
@@ -638,6 +638,7 @@ def _run_llama_phase(
) -> dict:
"""The llama phase of a chained update: put the backend into a maintenance state, run the installer for the latest prebuilt, then refresh caches so the next load uses the new build. Returns {to_tag, reload_required, message}; raises on failure. pin_release_tag pins the installer to that exact published release instead of letting it re-resolve "latest" (see start_update)."""
backend = None
+ marked = []
model_was_active = False
mtmd_guard = ExitStack()
# The installer exits 0 for a transient failure it answered by keeping the tree, so success no longer implies a new release. Read as the post-install check reads it.
@@ -662,6 +663,20 @@ def _run_llama_phase(
backend.unload_model()
except Exception as exc:
logger.debug("llama update: load coordination failed", error = str(exc))
+ from core.inference import model_slots
+
+ # Each kept model's in-flight load drains under its own lock, as the primary's did above.
+ for slot in list(model_slots.slots):
+ with slot.llama._serial_load_lock:
+ slot.llama._llama_update_in_progress = True
+ marked.append(slot.llama)
+ try:
+ if model_slots.unload_llama_slots(strict = True):
+ model_was_active = True
+ except RuntimeError:
+ # A kept server that survived would run from, or lock, the tree being replaced.
+ model_was_active = True
+ raise
# The mtmd dictation sidecar serves Qwen3-ASR from this same llama-server out of this same tree, so a live one locks the exe on Windows and a concurrent load would start against a half-swapped install.
model_was_active = _block_mtmd_sidecar(mtmd_guard) or model_was_active
@@ -829,11 +844,14 @@ def _run_llama_phase(
raise
finally:
mtmd_guard.close()
- if backend is not None:
- try:
- backend._llama_update_in_progress = False
- except Exception: # pragma: no cover - defensive
- pass
+ # A kept slot serving a non-GGUF model stays loaded through the update; its llama backend
+ # must not stay refused once the update is done.
+ for llama in (backend, *marked):
+ if llama is not None:
+ try:
+ llama._llama_update_in_progress = False
+ except Exception: # pragma: no cover - defensive
+ pass
def _block_mtmd_sidecar(stack: ExitStack) -> bool:
diff --git a/studio/backend/utils/multi_model_settings.py b/studio/backend/utils/multi_model_settings.py
new file mode 100644
index 00000000000..d536f04fa51
--- /dev/null
+++ b/studio/backend/utils/multi_model_settings.py
@@ -0,0 +1,30 @@
+# SPDX-License-Identifier: AGPL-3.0-only
+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
+
+"""Persisted switch for keeping other models loaded when another one loads."""
+
+from __future__ import annotations
+
+from typing import Any
+
+MULTI_MODEL_SETTING_KEY = "keep_multiple_models_loaded"
+DEFAULT_MULTI_MODEL_ENABLED = False
+
+
+def get_multi_model_enabled() -> bool:
+ try:
+ from storage.studio_db import get_app_setting
+ stored = get_app_setting(MULTI_MODEL_SETTING_KEY, None)
+ except Exception:
+ stored = None
+ return stored if isinstance(stored, bool) else DEFAULT_MULTI_MODEL_ENABLED
+
+
+def set_multi_model_enabled(value: Any) -> bool:
+ if not isinstance(value, bool):
+ raise ValueError("Keep multiple models loaded must be true or false.")
+
+ from storage.studio_db import upsert_app_settings
+
+ upsert_app_settings({MULTI_MODEL_SETTING_KEY: value})
+ return value
diff --git a/studio/frontend/src/features/chat/api/chat-api.ts b/studio/frontend/src/features/chat/api/chat-api.ts
index 0b97eff95a1..eaece2b1500 100644
--- a/studio/frontend/src/features/chat/api/chat-api.ts
+++ b/studio/frontend/src/features/chat/api/chat-api.ts
@@ -233,8 +233,10 @@ export async function listLoras(
export async function getInferenceStatus(
signal?: AbortSignal,
+ model?: string,
): Promise<InferenceStatusResponse> {
- const response = await authFetch("/api/inference/status", { signal });
+ const query = model ? `?${new URLSearchParams({ model }).toString()}` : "";
+ const response = await authFetch(`/api/inference/status${query}`, { signal });
return parseJsonOrThrow<InferenceStatusResponse>(response);
}
@@ -273,8 +275,11 @@ export interface ActiveGenerationsResponse {
/** Chats generating on the backend right now. Authoritative where `runningByThreadId` is not: that
* map is per-tab, empty after a reload and blind to a second tab, and /load 409s on these. */
-export async function getActiveGenerations(): Promise<ActiveGenerationsResponse> {
- const response = await authFetch("/api/inference/active-generations");
+export async function getActiveGenerations(
+ model?: string,
+): Promise<ActiveGenerationsResponse> {
+ const query = model ? `?model=${encodeURIComponent(model)}` : "";
+ const response = await authFetch(`/api/inference/active-generations${query}`);
return parseJsonOrThrow<ActiveGenerationsResponse>(response);
}
diff --git a/studio/frontend/src/features/chat/chat-page.tsx b/studio/frontend/src/features/chat/chat-page.tsx
index 723b06d28bc..db3b6e7a77e 100644
--- a/studio/frontend/src/features/chat/chat-page.tsx
+++ b/studio/frontend/src/features/chat/chat-page.tsx
@@ -2359,6 +2359,7 @@ export function ChatPage({
const residentCheckpoint = useChatRuntimeStore(
(state) => state.residentCheckpoint,
);
+ const loadedCount = useChatRuntimeStore((state) => state.loadedModels.length);
const loadedContextLength = useChatRuntimeStore(
(state) => state.loadedContextLength,
);
@@ -2473,6 +2474,7 @@ export function ChatPage({
selectModel,
loadNpuModel,
ejectModel,
+ ejectAllModels,
cancelLoading,
loadingModel,
loadProgress,
@@ -3607,13 +3609,16 @@ export function ChatPage({
handleCheckpointChange,
],
);
- const handleEject = useCallback(() => {
- void (async () => {
- if (await ejectModel()) {
- resetArtifacts();
- }
- })();
- }, [ejectModel, resetArtifacts]);
+ const handleEject = useCallback(
+ (modelId?: string) => {
+ const ejectedSelected = !modelId || modelId === inferenceParams.checkpoint;
+ void ejectModel(modelId).then((ok) => ok && ejectedSelected && resetArtifacts());
+ },
+ [ejectModel, inferenceParams.checkpoint, resetArtifacts],
+ );
+ const handleEjectAll = useCallback(() => {
+ void ejectAllModels().then((ok) => ok && resetArtifacts());
+ }, [ejectAllModels, resetArtifacts]);
// Pins the picker open so a stray click cannot dismiss the step under it. Tour steps only: the
// effect below shuts anything left pinned once the tour is gone.
@@ -4233,6 +4238,8 @@ export function ChatPage({
onConfigRequestAdopted={handleModelConfigRequestAdopted}
onValueChange={handleCheckpointChange}
onEject={handleEject}
+ onEjectAll={handleEjectAll}
+ loadedCount={loadedCount}
onFoldersChange={refreshLocalModels}
onModelsChange={refreshModelLists}
deleteDisabled={modelOperationInProgress}
diff --git a/studio/frontend/src/features/chat/hooks/use-chat-model-runtime.ts b/studio/frontend/src/features/chat/hooks/use-chat-model-runtime.ts
index c9692975aa7..f6401e24c08 100644
--- a/studio/frontend/src/features/chat/hooks/use-chat-model-runtime.ts
+++ b/studio/frontend/src/features/chat/hooks/use-chat-model-runtime.ts
@@ -47,7 +47,11 @@ import {
} from "@/features/settings/lib/view-logs-action";
import { loadModelMemorySettings } from "@/features/settings/api/model-memory";
import { loadVramBudgetSettings } from "@/features/settings/api/vram-budget";
-import { loadOpenAIAutoSwitchSettings } from "@/features/settings";
+import {
+ listOpenAIModels,
+ loadMultiModelEnabled,
+ loadOpenAIAutoSwitchSettings,
+} from "@/features/settings";
import {
confirmTransformersUpgradeIfNeeded,
useTransformersUpgradeDialogStore,
@@ -86,9 +90,13 @@ import {
ownsModelLoadRun,
releaseOwnedModelLoadRun,
} from "../utils/model-load-run";
-import { confirmStopRunningChatsIfNeeded } from "../utils/confirm-stop-running-chats";
+import {
+ confirmStopRunningChatsIfNeeded,
+ type StopRunningChatsDecision,
+} from "../utils/confirm-stop-running-chats";
import {
requestLocalPromptQueueStop,
+ requestPromptQueueStop,
notifyLocalPromptQueueLoadFailed,
} from "../utils/prompt-queue-boundary";
import { cancelPreStreamRunReservations } from "../utils/pre-stream-run-reservation";
@@ -591,6 +599,69 @@ async function readIdleUnloadArmed(): Promise<boolean> {
return lastIdleUnloadArmed;
}
+// A lookup started before a newer status must not write the quant that status replaced.
+let quantLookupGeneration = 0;
+
+function publishLoadedModels(
+ ids: string[],
+ statusId?: string | null,
+ statusQuant?: string | null,
+ checkpoints?: string[],
+): void {
+ // The status names the quant of the model it describes; only the others need /v1/models.
+ const current = useChatRuntimeStore.getState().loadedModels;
+ const known = new Map(current.map((m) => [m.checkpoint ?? m.id, m]));
+ const next = ids.map((id, i) => {
+ const checkpoint = checkpoints?.[i] || id;
+ const prev = known.get(checkpoint);
+ const quant = id === statusId ? (statusQuant ?? null) : prev?.quant;
+ return prev && prev.quant === quant && prev.checkpoint === checkpoint
+ ? prev
+ : { id, quant, checkpoint };
+ });
+ if (next.length !== current.length || next.some((m, i) => m !== current[i])) {
+ useChatRuntimeStore.setState({ loadedModels: next });
+ }
+ if (next.every((m) => m.quant !== undefined)) return;
+ const lookup = ++quantLookupGeneration;
+ void listOpenAIModels().then(
+ (models) => {
+ if (lookup !== quantLookupGeneration) return;
+ const details = new Map(models.filter((m) => m.loaded).map((m) => [m.id, m]));
+ const { loadedModels } = useChatRuntimeStore.getState();
+ if (!loadedModels.some((m) => m.quant === undefined && details.has(m.id))) return;
+ useChatRuntimeStore.setState({
+ loadedModels: loadedModels.map((m) =>
+ m.quant !== undefined || !details.has(m.id)
+ ? m
+ : { ...m, quant: details.get(m.id)?.quant ?? null },
+ ),
+ });
+ },
+ () => {},
+ );
+}
+
+function stopQueuedRuns(decision: StopRunningChatsDecision, scoped: boolean): void {
+ if (scoped) {
+ requestPromptQueueStop(decision.promptQueueThreadIds);
+ return;
+ }
+ cancelPreStreamRunReservations(decision.preStreamRunTokens);
+ requestLocalPromptQueueStop(decision.promptQueueThreadIds);
+}
+
+function unloadKeptModel(keptId: string): Promise<boolean> {
+ return confirmStopRunningChatsIfNeeded("Unloading this model", "unload", keptId).then(
+ async (decision) => {
+ if (!decision.proceed) return false;
+ stopQueuedRuns(decision, true);
+ await unloadModel({ model_path: keptId, force_cancel_active: decision.forceCancelActive });
+ return true;
+ },
+ );
+}
+
async function syncInferenceStatusToStore(options?: {
signal?: AbortSignal;
includeLoras?: boolean;
@@ -613,7 +684,7 @@ async function syncInferenceStatusToStore(options?: {
const selectedAtStart = useChatRuntimeStore.getState().params.checkpoint;
const [listRes, statusRes, , idleUnloadArmed] = await Promise.all([
listModels(),
- getInferenceStatus(signal),
+ getInferenceStatus(signal, selectedAtStart),
// Settled from this request alone. Read out of the aggregate below, a sibling
// rejection discarded a good list yet still marked the inventory settled, so a
// resident LoRA classified as a base model and pinned a new pair generalized.
@@ -641,6 +712,12 @@ async function syncInferenceStatusToStore(options?: {
if (signal?.aborted || superseded()) return;
setModels(listRes.models.map(toChatModelRow));
+ publishLoadedModels(
+ statusRes.serving ?? [],
+ statusRes.active_model,
+ statusRes.gguf_variant,
+ statusRes.serving_checkpoints,
+ );
const statusLoading = (statusRes.loading?.length ?? 0) > 0;
// A replacement names the outgoing model active and the incoming one loading. Adopting
@@ -816,6 +893,22 @@ export function useChatModelRuntime() {
);
const clearCheckpoint = useChatRuntimeStore((state) => state.clearCheckpoint);
+ // Settings owns "Keep multiple models loaded"; until it answers, loads replace as before.
+ useEffect(() => {
+ let cancelled = false;
+ loadMultiModelEnabled("Failed to read the multiple models setting").then(
+ (enabled) => {
+ if (!cancelled) {
+ useChatRuntimeStore.getState().setKeepModelsLoaded(enabled);
+ }
+ },
+ () => undefined,
+ );
+ return () => {
+ cancelled = true;
+ };
+ }, []);
+
const [loadingModel, setLoadingModel] = useState<{
id: string;
displayName: string;
@@ -999,7 +1092,10 @@ export function useChatModelRuntime() {
// /unload then leaves the resident model untouched, so derive the UI
// checkpoint from the backend rather than clearing it optimistically.
if (!preserveCheckpoint) {
- clearCheckpoint();
+ // Kept models stay loaded, unless this run already unloaded the selected one (a reload).
+ if (!useChatRuntimeStore.getState().keepModelsLoaded || run.residentModelUnloaded) {
+ clearCheckpoint();
+ }
await refresh();
}
return true;
@@ -1038,7 +1134,7 @@ export function useChatModelRuntime() {
await run.settledPromise;
if (run.loadAttemptPath && run.forceCancelActive && !run.residentModelUnloaded) {
try {
- const status = await getInferenceStatus();
+ const status = await getInferenceStatus(undefined, run.rollbackCheckpoint ?? undefined);
if ((status.loading?.length ?? 0) === 0) {
run.residentModelUnloaded = !residentModelMatchesPick(status, {
id: run.rollbackCheckpoint ?? "",
@@ -1255,7 +1351,10 @@ export function useChatModelRuntime() {
}
if (activeRunBeforeCredentials.residentModelUnloaded) {
try {
- const status = await getInferenceStatus();
+ const status = await getInferenceStatus(
+ undefined,
+ activeRunBeforeCredentials.rollbackCheckpoint ?? undefined,
+ );
if (modelSelectionIntentEpoch !== loadIntentId) return;
if (
(status.loading?.length ?? 0) === 0 &&
@@ -1320,7 +1419,10 @@ export function useChatModelRuntime() {
// rollback checkpoint. Confirm that resident before snapshotting its effective config;
// a failed compensation leaves the same checkpoint in the store but no resident.
try {
- const status = await getInferenceStatus();
+ const status = await getInferenceStatus(
+ undefined,
+ activeRunBeforeCredentials.rollbackCheckpoint ?? undefined,
+ );
if (
(status.loading?.length ?? 0) === 0 &&
residentModelMatchesPick(status, {
@@ -1457,7 +1559,9 @@ export function useChatModelRuntime() {
const pendingConfig =
typeof selection !== "string" ? selection.config : undefined;
if (!forceReload && !nativePathToken) {
- const residentStatus = await getInferenceStatus().catch(() => null);
+ const readPickStatus = () =>
+ getInferenceStatus(undefined, modelId).catch(() => null);
+ const residentStatus = await readPickStatus();
// Warm before reconciling the remembered GPU pick below: load-on-selection can run before any
// GPU hook mounted, and a cold cache passes the pick through unvalidated.
if (residentStatus && pendingConfig?.selectedGpuIds !== undefined) {
@@ -1584,7 +1688,7 @@ export function useChatModelRuntime() {
// Read again, and judge again: a status fetched before those awaits describes the model that was
// resident then, and adopting it would leave the picker naming this model while prompts went
// to another. A failed read falls through to /load.
- const confirmedStatus = await getInferenceStatus().catch(() => null);
+ const confirmedStatus = await readPickStatus();
if (confirmedStatus && adoptable(confirmedStatus)) {
// Same window as the confirm below: a rival load may have started during that GET, and it owns
// the resident model now.
@@ -1707,10 +1811,28 @@ export function useChatModelRuntime() {
let stopDecision: Awaited<
ReturnType<typeof confirmStopRunningChatsIfNeeded>
>;
+ const { keepModelsLoaded, loadedModels: loadedNow, params: paramsNow } =
+ useChatRuntimeStore.getState();
+ // vLLM and SGLang always replace the loaded models, so they keep the running-chat prompt.
+ const keepsOthers =
+ keepModelsLoaded && !forceReload && (paramsNow.engine ?? "auto") === "auto";
+ const switchingNote = keepsOthers ? "Keeping the loaded models." : "Switching models.";
+ // Reloading one of several touches only its own slot, so only its chats stop.
+ const touchesOnlySelected =
+ forceReload && !isExternalModelId(paramsNow.checkpoint) && loadedNow.length > 1;
try {
- stopDecision = await confirmStopRunningChatsIfNeeded(
- forceReload ? "Applying these settings" : "Loading a different model",
- );
+ stopDecision = keepsOthers
+ ? {
+ proceed: true,
+ forceCancelActive: false,
+ promptQueueThreadIds: [],
+ preStreamRunTokens: [],
+ }
+ : await confirmStopRunningChatsIfNeeded(
+ forceReload ? "Applying these settings" : "Loading a different model",
+ "reload",
+ touchesOnlySelected ? (paramsNow.checkpoint ?? undefined) : undefined,
+ );
} catch (error) {
releasePreflightLifecycleLease();
throw error;
@@ -1796,7 +1918,7 @@ export function useChatModelRuntime() {
previousModel?.isLora ?? (previousLora?.exportType === "lora");
const isCachedLora = isLora && isLocal;
let loadingDescription = [
- currentCheckpoint ? "Switching models." : null,
+ currentCheckpoint ? switchingNote : null,
extraLoadingDescription ?? null,
isDownloaded ? "Loading cached model into memory." : null,
!isDownloaded && isCachedLora ? "Loading trained model into memory." : null,
@@ -2286,7 +2408,7 @@ export function useChatModelRuntime() {
: [validation.mlx_loads_base_model];
downloadComplete = false;
loadingDescription = [
- currentCheckpoint ? "Switching models." : null,
+ currentCheckpoint ? switchingNote : null,
extraLoadingDescription ?? null,
mlxBaseDescription,
]
@@ -2356,13 +2478,14 @@ export function useChatModelRuntime() {
? (await consumeNativePathToken(nativePathToken, "load-model")).nativePathLease
: undefined;
- cancelPreStreamRunReservations(stopDecision.preStreamRunTokens);
- requestLocalPromptQueueStop(stopDecision.promptQueueThreadIds);
- if (currentCheckpoint) {
+ stopQueuedRuns(stopDecision, keepsOthers || touchesOnlySelected);
+ // A settings reload is in place, so a failed one must roll back even with others kept.
+ if (currentCheckpoint && !keepsOthers) {
// With chats generating, skip this preliminary unload: it cancels them ahead of /load's
// preflight, so a rejected target truncates replies for a model that never loads. Idle,
// unload first and free VRAM early.
- if (!forceCancelActive) {
+ // A reload of one of several stays in its own slot: /load replaces it there.
+ if (!forceCancelActive && !touchesOnlySelected) {
if (loadLlamaCppConfig?.mode !== "custom") {
await unloadModel({ model_path: currentCheckpoint });
// Custom mode leaves replacement to /load, which preflights before evicting.
@@ -2498,7 +2621,9 @@ export function useChatModelRuntime() {
const effectiveChatTemplateOverride =
loadChatTemplateOverride?.trim() ? loadChatTemplateOverride : null;
// Invalidate factories started before the final loading boundary.
- requestLocalPromptQueueStop();
+ if (!keepsOthers && !touchesOnlySelected) {
+ requestLocalPromptQueueStop();
+ }
if (lifecycleLease !== null) {
chatModelLifecycleGate.markLoading(lifecycleLease);
}
@@ -2564,10 +2689,20 @@ export function useChatModelRuntime() {
force_cancel_active: forceCancelActive,
force_reload: forceReload,
+ alongside: keepModelsLoaded || touchesOnlySelected,
});
cpuFallbackReason = loadResponse.cpu_fallback_reason ?? null;
mmprojFallbackReason = loadResponse.mmproj_fallback_reason ?? null;
offloadCounts = offloadCountsFrom(loadResponse);
+ if (loadResponse.evicted?.length) {
+ toast.info(
+ `Unloaded ${loadResponse.evicted.join(", ")} to make room`,
+ {
+ description:
+ "Select it again to load it back.",
+ },
+ );
+ }
// If cancelled while loading, do not show the model as active: it is being unloaded.
if (abortCtrl.signal.aborted) throw new Error("Cancelled");
@@ -2952,6 +3087,7 @@ export function useChatModelRuntime() {
// unloaded the live server.
cpu_fallback: rollbackState.loadedCpuFallback,
n_cpu_moe: rollbackState.loadedNCpuMoe ?? 0,
+ alongside: keepModelsLoaded || touchesOnlySelected,
tensor_split: rollbackState.loadedSplitRatio ?? undefined,
gpu_ids: rollbackState.loadedGpuIds ?? undefined,
// The failed swap already unloaded the server those runs used.
@@ -3148,7 +3284,7 @@ export function useChatModelRuntime() {
const watchForCacheMiss =
!managedLoad && isDownloaded && !isLocal && nativePathToken == null && !isOllamaModelId(modelId);
const cacheMissDescription = [
- currentCheckpoint ? "Switching models." : null,
+ currentCheckpoint ? switchingNote : null,
extraLoadingDescription ?? null,
CACHE_MISS_DOWNLOAD_DESCRIPTION,
]
@@ -3661,7 +3797,23 @@ export function useChatModelRuntime() {
[resetLoadingUiForRun, setLastModelLoadError, setModelsError],
);
- const ejectModel = useCallback(async (): Promise<boolean> => {
+ const ejectModel = useCallback(async (
+ modelId?: string,
+ confirmed?: StopRunningChatsDecision,
+ ): Promise<boolean> => {
+ if (modelId && modelId !== params.checkpoint) {
+ try {
+ if (!(await unloadKeptModel(modelId))) return false;
+ await refresh();
+ toast.success("Model unloaded", { duration: 1200 });
+ return true;
+ } catch (err) {
+ toast.error(
+ err instanceof Error ? err.message : "Failed to unload model",
+ );
+ return false;
+ }
+ }
if (!params.checkpoint) {
return false;
}
@@ -3690,21 +3842,29 @@ export function useChatModelRuntime() {
return false;
}
// Ejecting tears down llama-server, so every chat stops. Same prompt, but it leaves no model
- // loaded, so it must not be worded as a reload.
- const stopDecision = await confirmStopRunningChatsIfNeeded(
- "Unloading the model",
- "unload",
- );
+ // loaded, so it must not be worded as a reload. With several loaded only this one's chats stop.
+ const scope =
+ !confirmed && useChatRuntimeStore.getState().loadedModels.length > 1
+ ? params.checkpoint
+ : undefined;
+ const stopDecision =
+ confirmed ??
+ (await confirmStopRunningChatsIfNeeded(
+ "Unloading the model",
+ "unload",
+ scope,
+ ));
if (!stopDecision.proceed) return false;
async function performUnload(): Promise<void> {
- cancelPreStreamRunReservations(stopDecision.preStreamRunTokens);
- requestLocalPromptQueueStop(stopDecision.promptQueueThreadIds);
+ stopQueuedRuns(stopDecision, Boolean(scope));
await unloadModel({
model_path: params.checkpoint,
force_cancel_active: stopDecision.forceCancelActive,
});
- requestLocalPromptQueueStop();
+ if (!scope) {
+ requestLocalPromptQueueStop();
+ }
clearCheckpoint();
await refresh();
}
@@ -3731,11 +3891,48 @@ export function useChatModelRuntime() {
}
}, [clearCheckpoint, params.checkpoint, refresh, setModelsError]);
+ const ejectAllModels = useCallback(async (): Promise<boolean> => {
+ const others = useChatRuntimeStore
+ .getState()
+ .loadedModels.map((m) => m.checkpoint ?? m.id)
+ .filter((id) => id !== params.checkpoint);
+ const selectedLocal =
+ Boolean(params.checkpoint) && !isExternalModelId(params.checkpoint);
+ const decision = await confirmStopRunningChatsIfNeeded(
+ "Unloading every model",
+ "unload",
+ );
+ if (!decision.proceed) return false;
+ // Before any unload, so a queued send cannot hold a kept model or load one back.
+ stopQueuedRuns(decision, false);
+ // Others first: the selected model's eject refreshes, which would adopt one still loaded.
+ const results = await Promise.allSettled(
+ others.map((id) =>
+ unloadModel({ model_path: id, force_cancel_active: decision.forceCancelActive }),
+ ),
+ );
+ if (selectedLocal && !(await ejectModel(undefined, decision))) return false;
+ await refresh();
+ const failed = results.find(
+ (result): result is PromiseRejectedResult => result.status === "rejected",
+ );
+ if (failed) {
+ setModelsError(
+ failed.reason instanceof Error
+ ? failed.reason.message
+ : "Failed to unload every model",
+ );
+ return false;
+ }
+ return true;
+ }, [ejectModel, params.checkpoint, refresh, setModelsError]);
+
return {
refresh,
selectModel,
loadNpuModel,
ejectModel,
+ ejectAllModels,
cancelLoading,
cancelLoadingForReplacement,
invalidatePendingModelSelection,
diff --git a/studio/frontend/src/features/chat/shared-composer.tsx b/studio/frontend/src/features/chat/shared-composer.tsx
index 9a916cc0ad3..b423b7193fe 100644
--- a/studio/frontend/src/features/chat/shared-composer.tsx
+++ b/studio/frontend/src/features/chat/shared-composer.tsx
@@ -1819,6 +1819,7 @@ export function SharedComposer({
const loadRequestId = crypto.randomUUID();
const resp = await loadModel({
model_path: sel.id,
+ alongside: useChatRuntimeStore.getState().keepModelsLoaded,
load_request_id: loadRequestId,
hf_token: useChatRuntimeStore.getState().hfToken || null,
max_seq_length: compareMaxSeqLength,
diff --git a/studio/frontend/src/features/chat/stores/chat-runtime-store.ts b/studio/frontend/src/features/chat/stores/chat-runtime-store.ts
index fec2a40cd44..8ae72e35bc3 100644
--- a/studio/frontend/src/features/chat/stores/chat-runtime-store.ts
+++ b/studio/frontend/src/features/chat/stores/chat-runtime-store.ts
@@ -2307,6 +2307,8 @@ type ToolStatusEntry = {
owner?: () => void;
};
+export type LoadedModelSummary = { id: string; quant?: string | null; checkpoint?: string };
+
type ChatRuntimeStore = {
settingsHydrated: boolean;
/** The open chat's settings were asked for but have not arrived, so the store shows the
@@ -2343,6 +2345,7 @@ type ChatRuntimeStore = {
/** What /api/inference/status says is resident, as opposed to what the picker selected.
* undefined until the first read, so the header does not flash "not loaded". */
residentCheckpoint: string | null | undefined;
+ loadedModels: LoadedModelSummary[];
activeModelIsLocal: boolean;
loadedContextLength: number | null;
maxContextLength: number | null;
@@ -2389,6 +2392,8 @@ type ChatRuntimeStore = {
/** Whether the provider exposes server-side web_fetch (Anthropic `web_fetch_*`). Gates the
* composer's Fetch pill, independent of Search. */
supportsBuiltinWebFetch: boolean;
+ /** Mirrors the backend Settings switch "Keep multiple models loaded". */
+ keepModelsLoaded: boolean;
toolsEnabled: boolean;
/** Persisted Code preference. Use codeToolsOn() for the effective value. */
codeToolsEnabled: boolean;
@@ -2692,6 +2697,7 @@ type ChatRuntimeStore = {
setReasoningEffort: (effort: ReasoningEffort) => void;
setPreserveThinking: (value: boolean) => void;
setToolsEnabled: (enabled: boolean, options?: { persist?: boolean }) => void;
+ setKeepModelsLoaded: (keep: boolean) => void;
setCodeToolsEnabled: (enabled: boolean) => void;
setImageToolsEnabled: (enabled: boolean) => void;
setDeepResearchEnabled: (enabled: boolean) => void;
@@ -4175,6 +4181,7 @@ export const useChatRuntimeStore = create<ChatRuntimeStore>((set, get) => ({
lastModelLoadError: null,
activeGgufVariant: null,
residentCheckpoint: undefined,
+ loadedModels: [],
activeModelIsLocal: false,
loadedContextLength: null,
maxContextLength: null,
@@ -4205,6 +4212,7 @@ export const useChatRuntimeStore = create<ChatRuntimeStore>((set, get) => ({
supportsBuiltinImageGeneration: false,
supportsBuiltinWebFetch: false,
toolsEnabled: loadBool(CHAT_TOOLS_ENABLED_KEY, false),
+ keepModelsLoaded: false,
codeToolsEnabled: loadBool(CHAT_CODE_TOOLS_ENABLED_KEY, false),
codeToolsDeclinedUnderFullAccess: false,
imageToolsEnabled: loadBool(CHAT_IMAGE_TOOLS_ENABLED_KEY, false),
@@ -5392,6 +5400,7 @@ export const useChatRuntimeStore = create<ChatRuntimeStore>((set, get) => ({
queuedSettingsEpoch: state.queuedSettingsEpoch + 1,
};
}),
+ setKeepModelsLoaded: (keepModelsLoaded) => set({ keepModelsLoaded }),
setCodeToolsEnabled: (codeToolsEnabled) =>
set((state) => {
saveBool(CHAT_CODE_TOOLS_ENABLED_KEY, codeToolsEnabled);
diff --git a/studio/frontend/src/features/chat/types/api.ts b/studio/frontend/src/features/chat/types/api.ts
index b193909ee89..320fd7e319d 100644
--- a/studio/frontend/src/features/chat/types/api.ts
+++ b/studio/frontend/src/features/chat/types/api.ts
@@ -59,6 +59,7 @@ export interface LoadModelRequest {
/** Start a fresh runtime even when the active settings already match. */
force_reload?: boolean;
+ alongside?: boolean;
/** Stop any chats still generating instead of getting a 409: a load replaces the single
* llama-server they all decode on. Set only after the user confirms. */
force_cancel_active?: boolean;
@@ -233,6 +234,7 @@ export interface LoadModelResponse {
engine_precision?: "auto" | "bf16" | "fp16" | "int4" | "int8" | "fp8";
engine?: "auto" | "vllm" | "sglang";
is_mlx?: boolean;
+ evicted?: string[];
is_npu?: boolean;
status: string;
model: string;
@@ -380,6 +382,10 @@ export interface InferenceStatusResponse {
has_video_input?: boolean;
loading: string[];
loaded: string[];
+ /** The models answering requests; `loaded` also names one only held behind the active model. */
+ serving?: string[];
+ /** Per `serving` entry, the id to select, load and unload it by: a local model's path. */
+ serving_checkpoints?: string[];
inference?: {
temperature?: number;
top_p?: number;
diff --git a/studio/frontend/src/features/chat/utils/confirm-stop-running-chats.ts b/studio/frontend/src/features/chat/utils/confirm-stop-running-chats.ts
index c43e6382b4b..3817724062d 100644
--- a/studio/frontend/src/features/chat/utils/confirm-stop-running-chats.ts
+++ b/studio/frontend/src/features/chat/utils/confirm-stop-running-chats.ts
@@ -40,15 +40,19 @@ export function getLocalPromptQueueThreadIds(): string[] {
export async function confirmStopRunningChatsIfNeeded(
action = "Loading a different model",
effect: StopRunningChatsEffect = "reload",
+ /** Only this model's chats, from the backend: a tab cannot tell which model a local run is on. */
+ model?: string,
): Promise<StopRunningChatsDecision> {
// Local runs only: an external-provider chat is not stopped by the swap, so counting it would
// block a safe load behind a dialog. The backend excludes them for the same reason.
const { runningByThreadId, localRunByThreadId } =
useChatRuntimeStore.getState();
- let running = Object.entries(runningByThreadId)
- .filter(([threadId, on]) => on && localRunByThreadId[threadId])
- .map(([threadId]) => threadId);
- const preStreamRuns = listLocalPreStreamRunReservations();
+ let running = model
+ ? []
+ : Object.entries(runningByThreadId)
+ .filter(([threadId, on]) => on && localRunByThreadId[threadId])
+ .map(([threadId]) => threadId);
+ const preStreamRuns = model ? [] : listLocalPreStreamRunReservations();
const preStreamRunTokens = preStreamRuns.map(({ token }) => token);
let unnamedPreStreamRuns = 0;
const runningIds = new Set(running);
@@ -65,7 +69,7 @@ export async function confirmStopRunningChatsIfNeeded(
unnamedPreStreamRuns += 1;
}
}
- const promptQueueThreadIds = getLocalPromptQueueThreadIds();
+ let promptQueueThreadIds = model ? [] : getLocalPromptQueueThreadIds();
const promptQueuesByThreadId = usePromptQueueUI.getState().byThreadId;
const aliasesByQueuedRun = new Map<string, string[]>();
for (const threadId of promptQueueThreadIds) {
@@ -94,13 +98,16 @@ export async function confirmStopRunningChatsIfNeeded(
// reload and blind to a second tab, while force_cancel_active cancels every backend run. The
// union stays local-only, since external-provider runs are never in it.
try {
- const active = await getActiveGenerations();
+ const active = await getActiveGenerations(model);
const entries = active.active ?? [];
const merged = new Set(running);
for (const threadId of active.thread_ids ?? []) {
merged.add(threadId);
}
running = [...merged];
+ if (model) {
+ promptQueueThreadIds = running;
+ }
// Count conversations, not handles: one chat holds several at once while a tool continuation
// registers its next leg before the previous unwinds, and active.count counts those
// separately. A first turn started before its id was persisted has no id to merge, so add
diff --git a/studio/frontend/src/features/hub/hub-page.tsx b/studio/frontend/src/features/hub/hub-page.tsx
index b1fe71c780b..96c654e59a0 100644
--- a/studio/frontend/src/features/hub/hub-page.tsx
+++ b/studio/frontend/src/features/hub/hub-page.tsx
@@ -472,7 +472,14 @@ export function ModelsPage() {
} = {},
): Promise<void> => {
const seq = ++residentStatusSeq.current;
- const read = Promise.all([getInferenceStatus(), readIdleUnloadArmed()])
+ const selected = useChatRuntimeStore.getState().params.checkpoint;
+ const read = Promise.all([
+ getInferenceStatus(
+ undefined,
+ selected && !isExternalModelId(selected) ? selected : undefined,
+ ),
+ readIdleUnloadArmed(),
+ ])
.then(([status, idleUnloadArmed]) => {
if (seq !== residentStatusSeq.current)
return supersedingRefresh(residentStatusSupersession.current, seq);
diff --git a/studio/frontend/src/features/model-picker/components/model-selector.tsx b/studio/frontend/src/features/model-picker/components/model-selector.tsx
index bf59bca0b2c..9f2aee1acc6 100644
--- a/studio/frontend/src/features/model-picker/components/model-selector.tsx
+++ b/studio/frontend/src/features/model-picker/components/model-selector.tsx
@@ -112,7 +112,9 @@ interface ModelSelectorProps {
onValueChange?: (value: string, meta: ModelSelectorChangeMeta) => void;
/** Optional task-specific resolver for companion assets a GGUF row alone cannot describe. */
resolveDownloadFootprint?: ModelDownloadFootprintResolver;
- onEject?: () => void;
+ onEject?: (modelId?: string) => void;
+ onEjectAll?: () => void;
+ loadedCount?: number;
onFoldersChange?: () => void;
onModelsChange?: (deletedModel?: DeletedModelRef) => void;
deleteDisabled?: boolean;
@@ -157,9 +159,11 @@ function ModelSelectorTrigger({
triggerLabelClassName,
dataTour,
onEject,
+ loadedCount = 0,
// Task pages name what they pick ("Select image model"), so the choice reads as separate from the chat model.
placeholder = "Select model",
}: {
+ loadedCount?: number;
currentModel?: ModelOption;
isLoaded: boolean;
showCloudIndicator?: boolean;
@@ -171,6 +175,11 @@ function ModelSelectorTrigger({
onEject?: () => void;
placeholder?: string;
}) {
+ const severalLoaded = loadedCount > 1;
+ const triggerTitle = severalLoaded
+ ? `${loadedCount} models loaded`
+ : (currentModel?.name ?? placeholder);
+ const subtitle = severalLoaded ? currentModel?.name : currentModel?.description;
return (
<PopoverTrigger asChild={true}>
<button
@@ -241,7 +250,7 @@ function ModelSelectorTrigger({
triggerLabelClassName,
)}
>
- <span className="min-w-0 truncate">{currentModel?.name ?? placeholder}</span>
+ <span className="min-w-0 truncate">{triggerTitle}</span>
{showCloudIndicator ? (
<HugeiconsIcon
icon={CloudIcon}
@@ -250,14 +259,14 @@ function ModelSelectorTrigger({
/>
) : null}
</span>
- {currentModel?.description && (
+ {subtitle && (
<span
className={cn(
"min-w-0 shrink-[1000] truncate text-xs leading-tight text-muted-foreground",
!showCloudIndicator && GAP_BEFORE,
)}
>
- {currentModel.description}
+ {subtitle}
</span>
)}
</span>
@@ -265,10 +274,10 @@ function ModelSelectorTrigger({
<span
className={cn(
"min-w-0 shrink-[1000000] truncate whitespace-nowrap text-xs leading-tight text-muted-foreground",
- !currentModel.description && !showCloudIndicator && GAP_BEFORE,
+ !subtitle && !showCloudIndicator && GAP_BEFORE,
)}
>
- {currentModel.description ? " - " : ""}
+ {subtitle ? " - " : ""}
{currentModel.descriptionSuffix}
</span>
)}
@@ -348,6 +357,7 @@ function ModelSelectorContent({
onSelect,
resolveDownloadFootprint,
onEject,
+ onEjectAll,
onFoldersChange,
onBrowseHub,
onConfigureConnection,
@@ -377,7 +387,8 @@ function ModelSelectorContent({
onConfigRequestAdopted?: (requestId: string) => void;
onSelect: (id: string, meta: ModelSelectorChangeMeta) => void;
resolveDownloadFootprint?: ModelDownloadFootprintResolver;
- onEject?: () => void;
+ onEject?: (modelId?: string) => void;
+ onEjectAll?: () => void;
onFoldersChange?: () => void;
onBrowseHub?: () => void;
onConfigureConnection?: (providerId: string) => void;
@@ -660,6 +671,7 @@ function ModelSelectorContent({
onConfigure={openConfigPage}
deleteDisabled={deleteDisabled}
onEject={hasSelection && onEject ? onEject : undefined}
+ onEjectAll={onEjectAll}
task={task}
catalog={catalog}
communityModelPolicy={communityModelPolicy}
@@ -709,6 +721,7 @@ export function ModelSelector({
onValueChange,
resolveDownloadFootprint,
onEject,
+ onEjectAll,
onFoldersChange,
onModelsChange,
deleteDisabled,
@@ -729,6 +742,7 @@ export function ModelSelector({
hubCapability,
placeholder,
loaded,
+ loadedCount,
}: ModelSelectorProps) {
const [uncontrolledOpen, setUncontrolledOpen] = useState(false);
const open = controlledOpen ?? uncontrolledOpen;
@@ -849,8 +863,13 @@ export function ModelSelector({
setOpen(false);
}
- function handleEject() {
- onEject?.();
+ function handleEject(modelId?: string) {
+ onEject?.(modelId);
+ if (!modelId) setOpen(false);
+ }
+
+ function handleEjectAll() {
+ onEjectAll?.();
setOpen(false);
}
@@ -880,7 +899,8 @@ export function ModelSelector({
className={className}
triggerLabelClassName={triggerLabelClassName}
dataTour={triggerDataTour}
- onEject={onEject ? handleEject : undefined}
+ onEject={onEject ? () => handleEject() : undefined}
+ loadedCount={loadedCount}
placeholder={placeholder}
/>
<ModelSelectorContent
@@ -902,6 +922,7 @@ export function ModelSelector({
onSelect={handleSelect}
resolveDownloadFootprint={resolveDownloadFootprint}
onEject={onEject ? handleEject : undefined}
+ onEjectAll={onEjectAll ? handleEjectAll : undefined}
onFoldersChange={onFoldersChange}
// Curated task pickers show it only with a Hub filter; community-enabled ones always do.
onBrowseHub={
diff --git a/studio/frontend/src/features/model-picker/components/model-selector/pickers.tsx b/studio/frontend/src/features/model-picker/components/model-selector/pickers.tsx
index 59714221487..d101cad5eed 100644
--- a/studio/frontend/src/features/model-picker/components/model-selector/pickers.tsx
+++ b/studio/frontend/src/features/model-picker/components/model-selector/pickers.tsx
@@ -1805,8 +1805,10 @@ function GgufVariantExpander({
diffusionLoad = false,
hostPooledMemory = false,
gpuCount,
+ loadedQuants,
}: {
repoId: string;
+ loadedQuants?: readonly string[];
pipelineTag?: string | null;
/** True on Images / Video, where a GGUF is placed by the diffusion backend rather than
* llama-server, so the llama.cpp budget does not apply. Audio is task-scoped but not this. */
@@ -2351,6 +2353,10 @@ function GgufVariantExpander({
<span className="ml-1.5 text-ui-9 font-sans font-medium text-amber-700 dark:text-amber-300">
incomplete
</span>
+ ) : loadedQuants?.some((q) => ggufVariantsMatchForPicker(q, v.quant)) ? (
+ <span className="ml-1.5 text-ui-9 font-sans font-medium text-green-600/90 dark:text-green-400/80">
+ loaded
+ </span>
) : v.downloaded ? (
<>
<span className="ml-1.5 text-ui-9 font-sans font-medium text-green-600/90 dark:text-green-400/80">
@@ -2915,6 +2921,7 @@ export function HubModelPicker({
section = "downloaded",
sectionToggle,
onEject,
+ onEjectAll,
task,
catalog,
communityModelPolicy = "none",
@@ -2944,7 +2951,8 @@ export function HubModelPicker({
/** Section shown when not searching. Search spans all sections. */
section?: "downloaded" | "recommended" | "connected";
sectionToggle?: ReactNode;
- onEject?: () => void;
+ onEject?: (modelId?: string) => void;
+ onEjectAll?: () => void;
/** Restrict results to a pipeline task; undefined = all tasks (the chat default). */
task?: HfTaskFilter;
/** Curated catalog for a task-scoped picker: one canonical row per model, formats as the second level. */
@@ -2971,6 +2979,8 @@ export function HubModelPicker({
// resident. Same predicate as the header tick.
const selectedCheckpoint = useChatRuntimeStore((s) => s.params.checkpoint);
const residentCheckpoint = useChatRuntimeStore((s) => s.residentCheckpoint);
+ const loadedModels = useChatRuntimeStore((s) => s.loadedModels);
+ const isChatPicker = task === undefined;
const chatLoadedModelId = chatModelLoaded({
checkpoint: selectedCheckpoint,
isExternalModel: isExternalModelId(selectedCheckpoint),
@@ -2989,6 +2999,32 @@ export function HubModelPicker({
const hfToken = useHfTokenStore((s) => s.token);
const [query, setQuery] = useState("");
const debouncedQuery = useDebouncedValue(query);
+ const loadedIdSet = useMemo(
+ () => new Set(isChatPicker ? loadedModels.map((m) => m.id.toLowerCase()) : []),
+ [isChatPicker, loadedModels],
+ );
+ const isKeptLoaded = (repoId: string) =>
+ loadedIdSet.has(repoId.toLowerCase());
+ const loadedQuantsFor = (repoId: string): string[] => {
+ const quants = isChatPicker
+ ? loadedModels
+ .filter((m) => m.quant && modelIdsMatchForPicker(m.id, repoId))
+ .map((m) => m.quant as string)
+ : [];
+ if (activeGgufVariant && modelIdsMatchForPicker(loadedModelId, repoId)) {
+ quants.push(activeGgufVariant);
+ }
+ return [...new Set(quants)];
+ };
+ const loadedRows = useMemo(() => {
+ if (!isChatPicker || section !== "recommended") return [];
+ const q = debouncedQuery.trim().toLowerCase();
+ return loadedModels.filter((m) => m.id.toLowerCase().includes(q));
+ }, [isChatPicker, section, debouncedQuery, loadedModels]);
+ const loadedRowIds = useMemo(
+ () => new Set(loadedRows.map((m) => m.id.toLowerCase())),
+ [loadedRows],
+ );
// Shared Hub search stack so the picker and Hub run one implementation. Scoped to unsloth like the old listing.
const online = useOnlineStatus();
const { phase: hubPhase } = useHubAvailability();
@@ -4904,41 +4940,52 @@ export function HubModelPicker({
);
// Split downloaded models so non-Unsloth repos get their own "Other models" section above Fine-tuned.
+ // Kept-loaded models first: a stable partition, so the splits below keep that order.
+ const [keptFirstCachedGguf, keptFirstCachedModelRows] = useMemo(() => {
+ const first = <T extends { repo_id: string }>(rows: T[]): T[] =>
+ loadedIdSet.size === 0
+ ? rows
+ : [
+ ...rows.filter((r) => loadedIdSet.has(r.repo_id.toLowerCase())),
+ ...rows.filter((r) => !loadedIdSet.has(r.repo_id.toLowerCase())),
+ ];
+ return [first(visibleCachedGguf), first(visibleCachedModelRows)] as const;
+ }, [visibleCachedGguf, visibleCachedModelRows, loadedIdSet]);
const unslothCachedGguf = useMemo(
() =>
- visibleCachedGguf.filter(
+ keptFirstCachedGguf.filter(
(c) =>
isUnslothPublisherRepoId(c.repo_id) &&
!pinnedSoleQuantRepoIds.has(c.repo_id),
),
- [visibleCachedGguf, pinnedSoleQuantRepoIds],
+ [keptFirstCachedGguf, pinnedSoleQuantRepoIds],
);
const otherCachedGguf = useMemo(
() =>
- visibleCachedGguf.filter(
+ keptFirstCachedGguf.filter(
(c) =>
!isUnslothPublisherRepoId(c.repo_id) &&
!pinnedSoleQuantRepoIds.has(c.repo_id),
),
- [visibleCachedGguf, pinnedSoleQuantRepoIds],
+ [keptFirstCachedGguf, pinnedSoleQuantRepoIds],
);
const unslothCachedModelRows = useMemo(
() =>
- visibleCachedModelRows.filter(
+ keptFirstCachedModelRows.filter(
(c) =>
isUnslothPublisherRepoId(c.repo_id) &&
!pinnedSet.has(pinKey(c.repo_id)),
),
- [visibleCachedModelRows, pinnedSet],
+ [keptFirstCachedModelRows, pinnedSet],
);
const otherCachedModelRows = useMemo(
() =>
- visibleCachedModelRows.filter(
+ keptFirstCachedModelRows.filter(
(c) =>
!isUnslothPublisherRepoId(c.repo_id) &&
!pinnedSet.has(pinKey(c.repo_id)),
),
- [visibleCachedModelRows, pinnedSet],
+ [keptFirstCachedModelRows, pinnedSet],
);
// Param counts come straight off the unsloth listings the picker already loaded, so the VRAM
@@ -5243,7 +5290,9 @@ export function HubModelPicker({
];
}, [connectedMatches, pinnedConnectedSet, connectedSort]);
const hubOptionKeys = useMemo(() => {
- const keys: string[] = [];
+ const keys: string[] = loadedRows.map((m) =>
+ makeModelOptionKey("loaded", m.checkpoint ?? m.id),
+ );
// The tab lists nothing else, so these are the whole roving order, in drawn order.
if (section === "connected") {
@@ -5380,12 +5429,16 @@ export function HubModelPicker({
if (section === "recommended") {
keys.push(
- ...recommendedRows.map((r) => makeModelOptionKey("recommended", r.id)),
+ ...recommendedRows
+ .filter((r) => !loadedRowIds.has(r.id.toLowerCase()))
+ .map((r) => makeModelOptionKey("recommended", r.id)),
);
}
return keys;
}, [
+ loadedRows,
+ loadedRowIds,
cachedReady,
chatOnly,
sortedCustomFolderModels,
@@ -5752,6 +5805,9 @@ export function HubModelPicker({
);
const showConnected = section === "connected";
+ const severalLoaded = loadedModels.length > 1;
+ const ejectsAll = Boolean(onEjectAll) && severalLoaded;
+ const ejectsKept = Boolean(onEject) && severalLoaded;
// The Connected layout uses a wider box, so it drops the search inset to keep Search Hub on the last dropdown's edge.
const hasConnected = externalModels.length > 0;
// The Other models section and its shortcut only show with non-Unsloth downloads.
@@ -5811,6 +5867,90 @@ export function HubModelPicker({
);
};
+ const ejectMenuItems = (modelId: string) =>
+ ejectsKept && isKeptLoaded(modelId)
+ ? [
+ {
+ key: "eject",
+ label: "Eject",
+ icon: (
+ <HugeiconsIcon
+ icon={RemoveCircleIcon}
+ strokeWidth={1.75}
+ className="size-icon"
+ />
+ ),
+ onSelect: () => onEject?.(modelId),
+ },
+ ]
+ : undefined;
+
+ const renderLoadedRow = (entry: (typeof loadedModels)[number]) => {
+ // Select, eject and key by the checkpoint: a local model is labelled by name but loaded by
+ // path, and two files can share a name.
+ const checkpoint = entry.checkpoint ?? entry.id;
+ const optionKey = makeModelOptionKey("loaded", checkpoint);
+ const isSelected = modelIdsMatchForPicker(value, checkpoint);
+ const quant = entry.quant ?? (isSelected ? activeGgufVariant : null);
+ return (
+ <div
+ key={optionKey}
+ className={downloadedRowShellClassName(isSelected)}
+ >
+ <div className="min-w-0 flex-1">
+ <ModelRow
+ label={entry.id}
+ tooltipText={entry.id}
+ hideOwner={isUnslothOwned(entry.id)}
+ alignMeta="hub"
+ meta={quant ? "GGUF" : extractParamLabel(entry.id)}
+ tags={quant ? [ggufQuantChipLabel(quant)] : undefined}
+ selected={isSelected}
+ loaded={true}
+ optionProps={hubModelList.getOptionProps(optionKey, isSelected)}
+ onClick={() =>
+ onSelect(checkpoint, {
+ source: /^(?:[a-zA-Z]:[\\/]|[\\/]|~)/.test(checkpoint) ? "local" : "hub",
+ isLora: false,
+ ggufVariant: quant ?? undefined,
+ isDownloaded: true,
+ isGguf: Boolean(quant),
+ })
+ }
+ vramStatus={null}
+ className={downloadedRowButtonClassName}
+ />
+ </div>
+ <span className={ROW_ACTIONS_CLASS}>
+ {ejectsKept ? (
+ <Tooltip delayDuration={0}>
+ <TooltipTrigger asChild={true}>
+ <button
+ type="button"
+ onClick={(e) => {
+ e.stopPropagation();
+ onEject?.(checkpoint);
+ }}
+ aria-label={`Eject ${entry.id}`}
+ className="flex size-5 shrink-0 items-center justify-center rounded-md text-muted-foreground/60 transition-colors hover:bg-[rgb(0_0_0_/_calc(0.05*var(--contrast-wash-gain,1)))] hover:text-red-500 dark:hover:bg-[rgb(255_255_255_/_calc(0.1*var(--contrast-wash-gain,1)))]"
+ >
+ <HugeiconsIcon
+ icon={RemoveCircleIcon}
+ strokeWidth={1.75}
+ className="size-3.5"
+ />
+ </button>
+ </TooltipTrigger>
+ <TooltipContent side="top" className="tooltip-compact">
+ Eject
+ </TooltipContent>
+ </Tooltip>
+ ) : null}
+ </span>
+ </div>
+ );
+ };
+
// One connected model, through ModelRow like every On Device row, so the badges and the hover
// gutter are the same components rather than a second set that drifts.
const renderConnectedModelRow = (
@@ -6164,7 +6304,7 @@ export function HubModelPicker({
gpuGb={expanderGpuGb}
showVision={c.has_vision || sole.hasVision}
selected={isSelected}
- loaded={rowState.loaded}
+ loaded={rowState.loaded || isKeptLoaded(c.repo_id)}
alignMeta="device"
optionProps={hubModelList.getOptionProps(optionKey, isSelected)}
onClick={() => onSelect(c.repo_id, selectMeta)}
@@ -6182,6 +6322,7 @@ export function HubModelPicker({
<ModelRowMenu
ariaLabel={`More options for ${c.repo_id} ${variant.quant}`}
cachePath={{ repoId: c.repo_id, variant: variant.quant }}
+ items={ejectMenuItems(c.repo_id)}
pin={{
pinned: isPinned,
pinLabel: "Pin",
@@ -6240,6 +6381,7 @@ export function HubModelPicker({
const isSelected = value === c.repo_id;
const soleQuant = soleQuants.quants.get(c.repo_id);
if (soleQuant) return renderSoleQuantGgufRow(c, soleQuant);
+ const loadedQuants = loadedQuantsFor(c.repo_id);
// Auto-expansion waits for the probe: expanding every row first would mount an expander, and
// its remote listing, for repos about to collapse.
const expanderOpen = shouldMountVariantExpander({
@@ -6257,17 +6399,21 @@ export function HubModelPicker({
label={c.repo_id}
tooltipText={localPathTooltip(c.repo_id, c.cache_path)}
meta="GGUF"
+ quantChip={loadedQuants.map(ggufQuantChipLabel).join(", ") || undefined}
showVision={c.has_vision ?? visionByRepo[c.repo_id]}
alignMeta="device"
partial={isPartialRepo}
partialResumable={c.partial_resumable}
selected={isSelected}
- loaded={isRuntimeLoadedModel(
- loadedModelId,
- activeGgufVariant,
- c.repo_id,
- "required",
- )}
+ loaded={
+ isKeptLoaded(c.repo_id) ||
+ isRuntimeLoadedModel(
+ loadedModelId,
+ activeGgufVariant,
+ c.repo_id,
+ "required",
+ )
+ }
optionProps={hubModelList.getOptionProps(optionKey, isSelected)}
onClick={() => toggleGgufExpanded(c.repo_id, expanderOpen)}
onArrowDownIntoChildren={
@@ -6335,6 +6481,7 @@ export function HubModelPicker({
hostPooledMemory={gpu.loadDeviceSharesHostMemory}
gpuCount={expanderGpuCount}
repoId={c.repo_id}
+ loadedQuants={loadedQuants}
pipelineTag={c.task ?? null}
loadId={c.load_id}
cachePath={c.cache_path}
@@ -6394,12 +6541,15 @@ export function HubModelPicker({
alignMeta="device"
partial={isPartial}
partialResumable={c.partial_resumable}
- loaded={isRuntimeLoadedModel(
- loadedModelId,
- activeGgufVariant,
- c.repo_id,
- "none",
- )}
+ loaded={
+ isKeptLoaded(c.repo_id) ||
+ isRuntimeLoadedModel(
+ loadedModelId,
+ activeGgufVariant,
+ c.repo_id,
+ "none",
+ )
+ }
optionProps={hubModelList.getOptionProps(optionKey, isSelected)}
onClick={() =>
onSelect(c.repo_id, {
@@ -6445,6 +6595,7 @@ export function HubModelPicker({
<ModelRowMenu
ariaLabel={`More options for ${c.repo_id}`}
cachePath={{ repoId: c.repo_id }}
+ items={ejectMenuItems(c.repo_id)}
pin={{
pinned: pinnedSet.has(pinKey(c.repo_id)),
pinLabel: "Pin",
@@ -6767,6 +6918,12 @@ export function HubModelPicker({
onEject ? "pb-[calc(60px*var(--ui-space-scale,1))]" : "pb-4",
)}
>
+ {loadedRows.length > 0 ? (
+ <div className="pb-1.5">
+ {loadedRows.map(renderLoadedRow)}
+ <div className="mx-2.5 mt-1.5 border-t border-border/50" />
+ </div>
+ ) : null}
{showConnected ? (
connectedMatches.length === 0 ? (
<div className="px-2.5 py-2 text-xs leading-relaxed text-muted-foreground">
@@ -7445,6 +7602,7 @@ export function HubModelPicker({
hostPooledMemory={gpu.loadDeviceSharesHostMemory}
gpuCount={expanderGpuCount}
repoId={m.id}
+ loadedQuants={loadedQuantsFor(m.id)}
onDevice={true}
onSelect={onSelect}
resolveDownloadFootprint={resolveDownloadFootprint}
@@ -7583,6 +7741,7 @@ export function HubModelPicker({
hostPooledMemory={gpu.loadDeviceSharesHostMemory}
gpuCount={expanderGpuCount}
repoId={m.id}
+ loadedQuants={loadedQuantsFor(m.id)}
onDevice={true}
onSelect={onSelect}
resolveDownloadFootprint={resolveDownloadFootprint}
@@ -7710,6 +7869,7 @@ export function HubModelPicker({
hostPooledMemory={gpu.loadDeviceSharesHostMemory}
gpuCount={expanderGpuCount}
repoId={m.id}
+ loadedQuants={loadedQuantsFor(m.id)}
onDevice={true}
onSelect={onSelect}
resolveDownloadFootprint={resolveDownloadFootprint}
@@ -7784,6 +7944,7 @@ export function HubModelPicker({
) : (
recommendedRows.map((r) => {
const id = r.id;
+ if (loadedRowIds.has(id.toLowerCase())) return null;
const info = recommendedMeta.get(id);
const isG = isKnownGgufRepo(id);
const optionKey = makeModelOptionKey("recommended", id);
@@ -7847,6 +8008,7 @@ export function HubModelPicker({
hostPooledMemory={gpu.loadDeviceSharesHostMemory}
gpuCount={expanderGpuCount}
repoId={id}
+ loadedQuants={loadedQuantsFor(id)}
pipelineTag={pipelineTagById.get(id) ?? null}
onSelect={onSelect}
resolveDownloadFootprint={resolveDownloadFootprint}
@@ -7979,6 +8141,7 @@ export function HubModelPicker({
hostPooledMemory={gpu.loadDeviceSharesHostMemory}
gpuCount={expanderGpuCount}
repoId={id}
+ loadedQuants={loadedQuantsFor(id)}
pipelineTag={pipelineTagById.get(id) ?? null}
onSelect={onSelect}
resolveDownloadFootprint={resolveDownloadFootprint}
@@ -8106,6 +8269,7 @@ export function HubModelPicker({
hostPooledMemory={gpu.loadDeviceSharesHostMemory}
gpuCount={expanderGpuCount}
repoId={id}
+ loadedQuants={loadedQuantsFor(id)}
pipelineTag={pipelineTagById.get(id) ?? null}
onSelect={onSelect}
resolveDownloadFootprint={resolveDownloadFootprint}
@@ -8155,12 +8319,12 @@ export function HubModelPicker({
<div className="pointer-events-none absolute inset-x-0 bottom-0 flex justify-end pr-3.5 pb-[calc(19px*var(--ui-space-scale,1))]">
<button
type="button"
- onClick={onEject}
+ onClick={() => (ejectsAll ? onEjectAll?.() : onEject())}
className="pointer-events-auto inline-flex items-center justify-center gap-2 rounded-md bg-popover px-3 py-2 text-ui-13 font-medium text-destructive shadow-[0_2px_8px_-2px_rgba(0,0,0,0.16)] transition-colors hover:bg-[color-mix(in_srgb,var(--destructive)_12%,var(--popover))] dark:bg-[color-mix(in_srgb,var(--foreground)_10%,var(--sidebar))] dark:shadow-none dark:hover:bg-[color-mix(in_srgb,var(--destructive)_22%,var(--sidebar))]"
- title="Eject model"
+ title={ejectsAll ? "Eject all models" : "Eject model"}
>
<HugeiconsIcon icon={RemoveCircleIcon} className="size-3.5" />
- Eject model
+ {ejectsAll ? "Eject all" : "Eject model"}
</button>
</div>
) : null}
@@ -8429,6 +8593,11 @@ function FineTunedRows({
{expandedGguf === adapter.id && (
<GgufVariantExpander
repoId={adapter.id}
+ loadedQuants={
+ activeGgufVariant && modelIdsMatchForPicker(loadedModelId, adapter.id)
+ ? [activeGgufVariant]
+ : undefined
+ }
onSelect={onSelect}
onConfigure={onConfigure}
parentOptionKey={optionKey}
diff --git a/studio/frontend/src/features/settings/api/multi-model.ts b/studio/frontend/src/features/settings/api/multi-model.ts
new file mode 100644
index 00000000000..746ae6704c1
--- /dev/null
+++ b/studio/frontend/src/features/settings/api/multi-model.ts
@@ -0,0 +1,30 @@
+// SPDX-License-Identifier: AGPL-3.0-only
+// Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
+
+import { authFetch } from "@/features/auth";
+import { readFastApiError } from "@/lib/format-fastapi-error";
+
+export async function loadMultiModelEnabled(
+ fallbackMessage: string,
+): Promise<boolean> {
+ const res = await authFetch("/api/settings/multi-model");
+ if (!res.ok) {
+ throw new Error(await readFastApiError(res, fallbackMessage));
+ }
+ return Boolean((await res.json()).enabled);
+}
+
+export async function updateMultiModelEnabled(
+ enabled: boolean,
+ fallbackMessage: string,
+): Promise<boolean> {
+ const res = await authFetch("/api/settings/multi-model", {
+ method: "PUT",
+ headers: { "Content-Type": "application/json" },
+ body: JSON.stringify({ enabled }),
+ });
+ if (!res.ok) {
+ throw new Error(await readFastApiError(res, fallbackMessage));
+ }
+ return Boolean((await res.json()).enabled);
+}
diff --git a/studio/frontend/src/features/settings/components/model-memory-section.tsx b/studio/frontend/src/features/settings/components/model-memory-section.tsx
index 1c00cd09580..3bef4a9c212 100644
--- a/studio/frontend/src/features/settings/components/model-memory-section.tsx
+++ b/studio/frontend/src/features/settings/components/model-memory-section.tsx
@@ -2,6 +2,7 @@
// Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
import { Switch } from "@/components/ui/switch";
+import { useChatRuntimeStore } from "@/features/chat";
import { formatBytes } from "@/features/hub/lib/format";
import { useT } from "@/i18n";
import { subscribeModelLifecycle } from "@/lib/model-lifecycle-events";
@@ -11,6 +12,10 @@ import {
loadModelMemorySettings,
updateModelMemorySettings,
} from "../api/model-memory";
+import {
+ loadMultiModelEnabled,
+ updateMultiModelEnabled,
+} from "../api/multi-model";
import { SettingsRow } from "./settings-row";
import { SettingsSection } from "./settings-section";
@@ -61,6 +66,56 @@ export function ModelMemorySection() {
const [isSaving, setIsSaving] = useState(false);
// Bumped by every refresh and save: a read that resolves after either no longer describes the panel.
const latestRef = useRef(0);
+ const multiModel = useChatRuntimeStore((s) => s.keepModelsLoaded);
+ const [multiModelRead, setMultiModelRead] = useState(false);
+ const [isSavingMultiModel, setIsSavingMultiModel] = useState(false);
+
+ useEffect(() => {
+ let cancelled = false;
+ loadMultiModelEnabled(t("settings.resources.modelMemory.loadError")).then(
+ (enabled) => {
+ if (!cancelled) {
+ useChatRuntimeStore.getState().setKeepModelsLoaded(enabled);
+ setMultiModelRead(true);
+ }
+ },
+ (loadError) => {
+ if (!cancelled) {
+ setError(
+ loadError instanceof Error
+ ? loadError.message
+ : t("settings.resources.modelMemory.loadError"),
+ );
+ }
+ },
+ );
+ return () => {
+ cancelled = true;
+ };
+ }, [t]);
+
+ const persistMultiModel = async (enabled: boolean) => {
+ setIsSavingMultiModel(true);
+ setError(null);
+ try {
+ useChatRuntimeStore
+ .getState()
+ .setKeepModelsLoaded(
+ await updateMultiModelEnabled(
+ enabled,
+ t("settings.resources.modelMemory.saveError"),
+ ),
+ );
+ } catch (saveError) {
+ setError(
+ saveError instanceof Error
+ ? saveError.message
+ : t("settings.resources.modelMemory.saveError"),
+ );
+ } finally {
+ setIsSavingMultiModel(false);
+ }
+ };
useEffect(() => {
let cancelled = false;
@@ -146,6 +201,18 @@ export function ModelMemorySection() {
onCheckedChange={(noRamReserve) => void persist({ noRamReserve })}
/>
</SettingsRow>
+ <SettingsRow
+ label={t("settings.resources.modelMemory.multiModel")}
+ description={t("settings.resources.modelMemory.multiModelDescription")}
+ hint={t("settings.resources.modelMemory.multiModelHint")}
+ >
+ <Switch
+ aria-label={t("settings.resources.modelMemory.multiModel")}
+ checked={multiModel}
+ disabled={!multiModelRead || isSavingMultiModel}
+ onCheckedChange={(enabled) => void persistMultiModel(enabled)}
+ />
+ </SettingsRow>
{error ? (
<p className="pb-3 text-xs text-destructive">{error}</p>
) : (
diff --git a/studio/frontend/src/features/settings/index.ts b/studio/frontend/src/features/settings/index.ts
index effde1179f8..df2986a6734 100644
--- a/studio/frontend/src/features/settings/index.ts
+++ b/studio/frontend/src/features/settings/index.ts
@@ -11,7 +11,9 @@ export {
} from "./api/download-transport";
export { loadEmbeddingModelSettings } from "./api/embedding-model";
export { updateHubSource } from "./api/hub-settings";
+export { loadMultiModelEnabled } from "./api/multi-model";
export { loadOpenAIAutoSwitchSettings } from "./api/openai-auto-switch";
+export { listOpenAIModels } from "./api/openai-models";
export {
loadSystemOneSettings,
subscribeSystemOneSettings,
diff --git a/studio/frontend/src/i18n/locales/ar.ts b/studio/frontend/src/i18n/locales/ar.ts
index 275fdf819da..9d709f0ef8b 100644
--- a/studio/frontend/src/i18n/locales/ar.ts
+++ b/studio/frontend/src/i18n/locales/ar.ts
@@ -1616,6 +1616,9 @@ export const ar = {
noRamReserve: "عدم حجز ذاكرة النظام للنموذج",
noRamReserveDescription: "يقلل ذاكرة النظام المحجوزة لأوزان النموذج.",
noRamReserveHint: "يتخطى التحميل المعتمد على تعيين الذاكرة في إصدارات ويندوز المدعومة عندما يكون النموذج محمّلاً بالكامل على كرت الرسوميات، فلا تبقى صفحاته مقيمة. وإلا يُبقي التحميل المعتمد على تعيين الذاكرة. قد تستهلك مخازن المعالج المطلوبة ذاكرة النظام. ويحذف ‎--no-mmap‎ و‎--mlock‎.",
+ multiModel: "إبقاء عدة نماذج محمّلة",
+ multiModelDescription: "عند تحميل نموذج تبقى النماذج الأخرى في الذاكرة.",
+ multiModelHint: "يجيب كل نموذج محمّل عن الطلبات التي تذكره بالاسم. مع عدة كروت رسوميات، يوضع النموذج الجديد على كرت لا يستخدمه نموذج آخر إن كان فيه متسع. وإن لم يتسع بجانب النماذج الأخرى، تُزال أولًا النماذج غير المستخدمة، ثم يحل محل النموذج النشط. ويزيل التدريب النماذج الإضافية قبل النموذج النشط.",
mlockVetoed: "يبقى ‎--mlock‎ معطَّلاً: تثبيت النموذج سيحجز ذاكرة تكفي حجمه كاملاً. ولا يزال التفريغ التلقائي عند الخمول معطَّلاً.",
mlockNotApplicable: "بالكامل على وحدة GPU: لا شيء في ذاكرة النظام لقفله. يبقى التفريغ التلقائي عند الخمول معطَّلاً.",
memlockCapped: "يحدّ هذا النظام الذاكرة المقفلة عند {limit}. لن يُثبَّت نموذج أكبر بالكامل؛ ارفع الحد باستخدام ulimit -l.",
diff --git a/studio/frontend/src/i18n/locales/de.ts b/studio/frontend/src/i18n/locales/de.ts
index edae84c5fd3..e8e7be2d316 100644
--- a/studio/frontend/src/i18n/locales/de.ts
+++ b/studio/frontend/src/i18n/locales/de.ts
@@ -1645,6 +1645,9 @@ export const de = {
noRamReserve: "Keinen System-RAM für das Modell reservieren",
noRamReserveDescription: "Weniger RAM für die Modellgewichte belegen.",
noRamReserveHint: "Überspringt auf unterstützten Windows-Builds das speicherabgebildete Laden, wenn das Modell vollständig auf die GPU ausgelagert ist, sodass dessen Seiten nicht im Speicher gehalten werden. Andernfalls bleibt das speicherabgebildete Laden erhalten. Erforderliche CPU-Puffer können weiterhin RAM belegen. Entfernt --no-mmap und --mlock.",
+ multiModel: "Mehrere Modelle geladen lassen",
+ multiModelDescription: "Beim Laden eines Modells bleiben die anderen im Speicher.",
+ multiModelHint: "Jedes geladene Modell beantwortet die Anfragen, die es nennen. Bei mehreren GPUs kommt ein neues Modell auf eine GPU, die kein anderes Modell nutzt, sofern dort Platz ist. Passt es nicht neben die anderen, werden zuerst die nicht genutzten Modelle entladen, danach ersetzt es das aktive Modell. Training entlädt die zusätzlichen Modelle vor dem aktiven.",
mlockVetoed: "--mlock bleibt aus: das Fixieren des Modells würde RAM für das gesamte Modell reservieren. Das automatische Entladen im Leerlauf bleibt deaktiviert.",
mlockNotApplicable: "Vollständig auf der GPU: nichts im System-RAM zu sperren. Automatisches Entladen im Leerlauf bleibt aus.",
memlockCapped: "Dieses System begrenzt gesperrten Speicher auf {limit}. Ein größeres Modell wird nicht vollständig fixiert; erhöhen Sie das Limit mit ulimit -l.",
diff --git a/studio/frontend/src/i18n/locales/en.ts b/studio/frontend/src/i18n/locales/en.ts
index fc04139ba9a..b65cea3d9a1 100644
--- a/studio/frontend/src/i18n/locales/en.ts
+++ b/studio/frontend/src/i18n/locales/en.ts
@@ -1656,6 +1656,9 @@ export const en = {
noRamReserve: "Don't reserve system RAM for the model",
noRamReserveDescription: "Reduce host RAM held for model weights.",
noRamReserveHint: "Skips memory-mapped loading on supported Windows builds when the model is fully offloaded to the GPU, so its pages are not held resident. Otherwise keeps memory-mapped loading. Required CPU buffers can still use RAM. Drops --no-mmap and --mlock.",
+ multiModel: "Keep multiple models loaded",
+ multiModelDescription: "Loading a model keeps the others in memory.",
+ multiModelHint: "Each loaded model answers the requests that name it. With several GPUs, a new model goes to a GPU no other model uses when one has room. When it does not fit beside the others, the models not in use are unloaded first, then it replaces the active model. Training unloads the extra models before the active one.",
mlockVetoed: "--mlock stays off: pinning the model in place would reserve RAM for all of it. Idle auto-unload is still disabled.",
mlockNotApplicable: "Fully on the GPU: nothing in system RAM to lock. Idle auto-unload stays off.",
memlockCapped: "This system caps locked memory at {limit}. A larger model will not be fully pinned; raise the limit with ulimit -l.",
diff --git a/studio/frontend/src/i18n/locales/es.ts b/studio/frontend/src/i18n/locales/es.ts
index 15002e93d5b..521815ce072 100644
--- a/studio/frontend/src/i18n/locales/es.ts
+++ b/studio/frontend/src/i18n/locales/es.ts
@@ -1640,6 +1640,9 @@ export const es = {
noRamReserve: "No reservar RAM del sistema para el modelo",
noRamReserveDescription: "Reduce la RAM ocupada por los pesos del modelo.",
noRamReserveHint: "Omite la carga mapeada en memoria en las compilaciones de Windows compatibles cuando el modelo está totalmente descargado en la GPU, de modo que sus páginas no quedan residentes. En caso contrario conserva la carga mapeada en memoria. Los búferes de CPU necesarios pueden seguir usando RAM. Elimina --no-mmap y --mlock.",
+ multiModel: "Mantener varios modelos cargados",
+ multiModelDescription: "Al cargar un modelo, los demás permanecen en memoria.",
+ multiModelHint: "Cada modelo cargado responde a las solicitudes que lo nombran. Con varias GPU, un modelo nuevo va a una GPU que ningún otro modelo usa cuando hay espacio. Si no cabe junto a los demás, primero se descargan los modelos que no se están usando y después reemplaza al modelo activo. El entrenamiento descarga los modelos adicionales antes que el activo.",
mlockVetoed: "--mlock permanece desactivado: fijar el modelo reservaría RAM para todo él. La descarga automática por inactividad sigue desactivada.",
mlockNotApplicable: "Totalmente en la GPU: nada que bloquear en la RAM del sistema. La descarga automática por inactividad sigue desactivada.",
memlockCapped: "Este sistema limita la memoria bloqueada a {limit}. Un modelo mayor no quedará fijado por completo; aumenta el límite con ulimit -l.",
diff --git a/studio/frontend/src/i18n/locales/fr.ts b/studio/frontend/src/i18n/locales/fr.ts
index 715685fec69..80bacfa0d2c 100644
--- a/studio/frontend/src/i18n/locales/fr.ts
+++ b/studio/frontend/src/i18n/locales/fr.ts
@@ -1644,6 +1644,9 @@ export const fr = {
noRamReserve: "Ne pas réserver de RAM système pour le modèle",
noRamReserveDescription: "Réduit la RAM occupée par les poids du modèle.",
noRamReserveHint: "Ignore le chargement mappé en mémoire sur les versions Windows compatibles lorsque le modèle est entièrement déchargé sur le GPU, afin que ses pages ne restent pas résidentes. Sinon, conserve le chargement mappé en mémoire. Les tampons CPU nécessaires peuvent toujours occuper de la RAM. Supprime --no-mmap et --mlock.",
+ multiModel: "Garder plusieurs modèles chargés",
+ multiModelDescription: "Charger un modèle garde les autres en mémoire.",
+ multiModelHint: "Chaque modèle chargé répond aux requêtes qui le nomment. Avec plusieurs GPU, un nouveau modèle va sur un GPU qu'aucun autre modèle n'utilise lorsqu'il y a de la place. S'il ne tient pas à côté des autres, les modèles inutilisés sont d'abord déchargés, puis il remplace le modèle actif. L'entraînement décharge les modèles supplémentaires avant le modèle actif.",
mlockVetoed: "--mlock reste désactivé : épingler le modèle réserverait de la RAM pour l'intégralité de celui-ci. Le déchargement automatique en veille reste désactivé.",
mlockNotApplicable: "Entièrement sur le GPU : rien à verrouiller en RAM système. Le déchargement automatique en veille reste désactivé.",
memlockCapped: "Ce système limite la mémoire verrouillée à {limit}. Un modèle plus grand ne sera pas entièrement épinglé ; augmentez la limite avec ulimit -l.",
diff --git a/studio/frontend/src/i18n/locales/hi.ts b/studio/frontend/src/i18n/locales/hi.ts
index 2ad6bf118fc..020599cbe5c 100644
--- a/studio/frontend/src/i18n/locales/hi.ts
+++ b/studio/frontend/src/i18n/locales/hi.ts
@@ -1619,6 +1619,9 @@ export const hi = {
noRamReserve: "मॉडल के लिए सिस्टम RAM आरक्षित न करें",
noRamReserveDescription: "मॉडल वेट्स के लिए रखी गई RAM घटाता है।",
noRamReserveHint: "जब मॉडल पूरी तरह GPU पर ऑफ़लोड हो, तो समर्थित Windows बिल्ड पर मेमोरी-मैप्ड लोडिंग छोड़ देता है, ताकि उसके पेज रेज़िडेंट न रहें। अन्यथा मेमोरी-मैप्ड लोडिंग बनाए रखता है। आवश्यक CPU बफ़र अब भी RAM ले सकते हैं। --no-mmap तथा --mlock हटा देता है।",
+ multiModel: "कई मॉडल लोड रखें",
+ multiModelDescription: "मॉडल लोड करने पर बाकी मॉडल मेमोरी में बने रहते हैं।",
+ multiModelHint: "हर लोड किया गया मॉडल उन अनुरोधों का जवाब देता है जो उसका नाम लेते हैं। कई GPU होने पर, जगह हो तो नया मॉडल ऐसे GPU पर जाता है जिसे कोई दूसरा मॉडल इस्तेमाल नहीं कर रहा। अगर यह बाकी मॉडलों के साथ नहीं समाता, तो पहले इस्तेमाल में न रहे मॉडल अनलोड होते हैं, फिर यह सक्रिय मॉडल की जगह लेता है। ट्रेनिंग सक्रिय मॉडल से पहले अतिरिक्त मॉडलों को अनलोड करती है।",
mlockVetoed: "--mlock बंद ही रहता है: मॉडल को पिन करने से उसकी पूरी मात्रा के लिए RAM आरक्षित हो जाएगी। निष्क्रिय होने पर स्वतः अनलोड अब भी अक्षम है।",
mlockNotApplicable: "पूरी तरह GPU पर: सिस्टम RAM में लॉक करने के लिए कुछ नहीं। निष्क्रिय होने पर स्वतः अनलोड बंद रहता है।",
memlockCapped: "यह सिस्टम लॉक की गई मेमोरी को {limit} तक सीमित करता है। इससे बड़ा मॉडल पूरी तरह पिन नहीं होगा; ulimit -l से सीमा बढ़ाएँ।",
diff --git a/studio/frontend/src/i18n/locales/it.ts b/studio/frontend/src/i18n/locales/it.ts
index d8b374b4f95..ab5e705e484 100644
--- a/studio/frontend/src/i18n/locales/it.ts
+++ b/studio/frontend/src/i18n/locales/it.ts
@@ -1608,6 +1608,9 @@ export const it = {
noRamReserve: "Non riservare RAM di sistema per il modello",
noRamReserveDescription: "Riduce la RAM occupata dai pesi del modello.",
noRamReserveHint: "Salta il caricamento mappato in memoria sulle build Windows supportate quando il modello è interamente scaricato sulla GPU, così le sue pagine non restano residenti. Altrimenti mantiene il caricamento mappato in memoria. I buffer CPU necessari possono comunque occupare RAM. Rimuove --no-mmap e --mlock.",
+ multiModel: "Mantieni più modelli caricati",
+ multiModelDescription: "Caricare un modello mantiene gli altri in memoria.",
+ multiModelHint: "Ogni modello caricato risponde alle richieste che lo nominano. Con più GPU, un nuovo modello va su una GPU che nessun altro modello usa, se c'è spazio. Se non entra accanto agli altri, vengono prima scaricati i modelli non in uso, poi sostituisce il modello attivo. L'addestramento scarica i modelli aggiuntivi prima di quello attivo.",
mlockVetoed: "--mlock resta disattivato: bloccare il modello riserverebbe RAM per l'intero modello. Lo scaricamento automatico in inattività resta disattivato.",
mlockNotApplicable: "Interamente sulla GPU: nulla da bloccare nella RAM di sistema. Lo scaricamento automatico in inattività resta disattivato.",
memlockCapped: "Questo sistema limita la memoria bloccata a {limit}. Un modello più grande non verrà bloccato del tutto; aumenta il limite con ulimit -l.",
diff --git a/studio/frontend/src/i18n/locales/ja.ts b/studio/frontend/src/i18n/locales/ja.ts
index b6700b8250d..ca28e4e8ad0 100644
--- a/studio/frontend/src/i18n/locales/ja.ts
+++ b/studio/frontend/src/i18n/locales/ja.ts
@@ -1597,6 +1597,9 @@ export const ja = {
noRamReserve: "モデル用にシステム RAM を確保しない",
noRamReserveDescription: "モデルの重みが占有する RAM を減らします。",
noRamReserveHint: "モデルが GPU に完全にオフロードされている場合、対応する Windows ビルドではメモリマップ読み込みを行わず、ファイルのページが常駐しないようにします。それ以外ではメモリマップ読み込みを維持します。必要な CPU バッファは RAM を使用することがあります。--no-mmap と --mlock を除去します。",
+ multiModel: "複数のモデルを読み込んだままにする",
+ multiModelDescription: "モデルを読み込んでも、他のモデルはメモリに残ります。",
+ multiModelHint: "読み込まれた各モデルは、そのモデルを指定したリクエストに応答します。複数の GPU がある場合、空きがあれば新しいモデルは他のモデルが使っていない GPU に配置されます。他のモデルと並べて収まらない場合は、使用中でないモデルを先にアンロードし、それでも足りなければアクティブなモデルを置き換えます。トレーニングはアクティブなモデルより先に追加のモデルをアンロードします。",
mlockVetoed: "--mlock は無効のままです。モデルを固定するとモデル全体分の RAM を確保することになります。アイドル時の自動アンロードは引き続き無効です。",
mlockNotApplicable: "すべて GPU 上にあるため、システム RAM に固定するものはありません。アイドル時の自動アンロードは無効のままです。",
memlockCapped: "このシステムはロック可能なメモリを {limit} に制限しています。これより大きいモデルは完全には固定されません。ulimit -l で上限を引き上げてください。",
diff --git a/studio/frontend/src/i18n/locales/ko.ts b/studio/frontend/src/i18n/locales/ko.ts
index 684d52d6c6b..52952d7a5c1 100644
--- a/studio/frontend/src/i18n/locales/ko.ts
+++ b/studio/frontend/src/i18n/locales/ko.ts
@@ -1612,6 +1612,9 @@ export const ko = {
noRamReserve: "모델을 위해 시스템 RAM을 예약하지 않음",
noRamReserveDescription: "모델 가중치가 차지하는 RAM을 줄입니다.",
noRamReserveHint: "모델이 GPU에 완전히 오프로드된 경우 지원되는 Windows 빌드에서 메모리 매핑 로딩을 건너뛰어 파일 페이지가 상주하지 않도록 합니다. 그렇지 않으면 메모리 매핑 로딩을 유지합니다. 필요한 CPU 버퍼는 여전히 RAM을 사용할 수 있습니다. --no-mmap과 --mlock을 제거합니다.",
+ multiModel: "여러 모델을 로드된 상태로 유지",
+ multiModelDescription: "모델을 로드해도 다른 모델이 메모리에 남아 있습니다.",
+ multiModelHint: "로드된 각 모델은 해당 모델을 지정한 요청에 응답합니다. GPU가 여러 개이면 여유가 있을 때 새 모델은 다른 모델이 사용하지 않는 GPU에 배치됩니다. 다른 모델과 함께 들어가지 않으면 사용하지 않는 모델을 먼저 언로드하고, 그래도 부족하면 활성 모델을 대체합니다. 학습은 활성 모델보다 추가 모델을 먼저 언로드합니다.",
mlockVetoed: "--mlock은 꺼진 상태로 유지됩니다. 모델을 고정하면 모델 전체 크기만큼 RAM을 예약하게 됩니다. 유휴 시 자동 언로드는 계속 비활성화됩니다.",
mlockNotApplicable: "모두 GPU에 있어 시스템 RAM에 고정할 것이 없습니다. 유휴 시 자동 언로드는 계속 꺼져 있습니다.",
memlockCapped: "이 시스템은 잠금 메모리를 {limit}로 제한합니다. 더 큰 모델은 완전히 고정되지 않습니다. ulimit -l로 한도를 늘리세요.",
diff --git a/studio/frontend/src/i18n/locales/pt-br.ts b/studio/frontend/src/i18n/locales/pt-br.ts
index 500eb4b4847..505c317a2c0 100644
--- a/studio/frontend/src/i18n/locales/pt-br.ts
+++ b/studio/frontend/src/i18n/locales/pt-br.ts
@@ -1628,6 +1628,9 @@ export const ptBR = {
noRamReserve: "Não reservar RAM do sistema para o modelo",
noRamReserveDescription: "Reduz a RAM ocupada pelos pesos do modelo.",
noRamReserveHint: "Ignora o carregamento mapeado em memória nas versões compatíveis do Windows quando o modelo está totalmente descarregado na GPU, para que suas páginas não fiquem residentes. Caso contrário, mantém o carregamento mapeado em memória. Buffers de CPU necessários ainda podem usar RAM. Remove --no-mmap e --mlock.",
+ multiModel: "Manter vários modelos carregados",
+ multiModelDescription: "Carregar um modelo mantém os outros na memória.",
+ multiModelHint: "Cada modelo carregado responde às solicitações que o nomeiam. Com várias GPUs, um novo modelo vai para uma GPU que nenhum outro modelo usa quando há espaço. Se não couber ao lado dos outros, os modelos fora de uso são descarregados primeiro e depois ele substitui o modelo ativo. O treinamento descarrega os modelos extras antes do ativo.",
mlockVetoed: "--mlock continua desativado: fixar o modelo reservaria RAM para todo ele. A descarga automática por inatividade continua desativada.",
mlockNotApplicable: "Totalmente na GPU: nada para bloquear na RAM do sistema. A descarga automática por inatividade continua desativada.",
memlockCapped: "Este sistema limita a memória bloqueada a {limit}. Um modelo maior não será totalmente fixado; aumente o limite com ulimit -l.",
diff --git a/studio/frontend/src/i18n/locales/ru.ts b/studio/frontend/src/i18n/locales/ru.ts
index 34f22e9374c..c807f7c13d2 100644
--- a/studio/frontend/src/i18n/locales/ru.ts
+++ b/studio/frontend/src/i18n/locales/ru.ts
@@ -1623,6 +1623,9 @@ export const ru = {
noRamReserve: "Не резервировать системную ОЗУ под модель",
noRamReserveDescription: "Уменьшает объём ОЗУ, занятый весами модели.",
noRamReserveHint: "Пропускает загрузку через отображение файла в память в поддерживаемых сборках Windows, когда модель полностью выгружена на видеокарту, чтобы её страницы не оставались в памяти. В остальных случаях сохраняет загрузку через отображение файла в память. Необходимые буферы ЦП всё равно могут занимать ОЗУ. Убирает --no-mmap и --mlock.",
+ multiModel: "Держать несколько моделей загруженными",
+ multiModelDescription: "При загрузке модели остальные остаются в памяти.",
+ multiModelHint: "Каждая загруженная модель отвечает на запросы, в которых она указана. При нескольких GPU новая модель размещается на GPU, который не занят другой моделью, если там есть место. Если она не помещается рядом с остальными, сначала выгружаются неиспользуемые модели, затем она заменяет активную. Обучение выгружает дополнительные модели раньше активной.",
mlockVetoed: "--mlock остаётся выключенным: закрепление модели зарезервировало бы ОЗУ под неё целиком. Автовыгрузка по простою по-прежнему отключена.",
mlockNotApplicable: "Полностью на GPU: в системной памяти нечего закреплять. Автовыгрузка по простою остаётся выключенной.",
memlockCapped: "Система ограничивает блокируемую память значением {limit}. Модель большего размера не будет закреплена полностью; увеличьте лимит через ulimit -l.",
diff --git a/studio/frontend/src/i18n/locales/zh-CN.ts b/studio/frontend/src/i18n/locales/zh-CN.ts
index 156cb303a51..77a679c1450 100644
--- a/studio/frontend/src/i18n/locales/zh-CN.ts
+++ b/studio/frontend/src/i18n/locales/zh-CN.ts
@@ -1587,6 +1587,9 @@ export const zhCN = {
noRamReserve: "不为模型预留系统内存",
noRamReserveDescription: "减少模型权重占用的内存。",
noRamReserveHint: "当模型完全卸载到 GPU 时,在受支持的 Windows 版本上跳过内存映射加载,使其页面不再常驻;否则保持内存映射加载。必要的 CPU 缓冲区仍可能占用内存。移除 --no-mmap 和 --mlock。",
+ multiModel: "保持多个模型加载",
+ multiModelDescription: "加载模型时,其他模型仍保留在内存中。",
+ multiModelHint: "每个已加载的模型都会响应指定它的请求。在多 GPU 机器上,如果有空间,新模型会放到没有其他模型使用的 GPU 上。如果无法与其他模型同时容纳,会先卸载未在使用的模型,然后替换当前模型。训练会先卸载额外的模型,再卸载当前模型。",
mlockVetoed: "--mlock 保持关闭:锁定模型会为其全部内容预留内存。空闲自动卸载仍处于禁用状态。",
mlockNotApplicable: "已完全位于 GPU 上:系统内存中没有需要锁定的内容。空闲自动卸载保持关闭。",
memlockCapped: "本系统将锁定内存限制为 {limit}。更大的模型无法被完全锁定;请使用 ulimit -l 提高上限。",
diff --git a/studio/frontend/tests/chat-model-load-replacement.test.ts b/studio/frontend/tests/chat-model-load-replacement.test.ts
index 0060c122aea..1346212a300 100644
--- a/studio/frontend/tests/chat-model-load-replacement.test.ts
+++ b/studio/frontend/tests/chat-model-load-replacement.test.ts
@@ -42,7 +42,7 @@ const blocks = {
"if (pendingReplacementRollback?.config)",
),
rollbackClear: section(runtime, "function restoreRollbackConfigForClear(", "const approvedRemoteCodeFingerprints"),
- preliminaryUnload: section(runtime, "if (!forceCancelActive) {", "// Set either way: /load can still leave no model resident"),
+ preliminaryUnload: section(runtime, "if (!forceCancelActive && !touchesOnlySelected) {", "// Set either way: /load can still leave no model resident"),
preflightLease: section(runtime, "// Hold the lifecycle lease through confirmation and loading.", "if (lifecycleLease === null) {"),
loadGate: section(runtime, "async function performLoad(): Promise<void> {", "const pendingLoadConfig ="),
stagedMetadata: section(runtime, "if (isGguf && isDiffusion === undefined) {", "const targetIsDiffusion = isDiffusion === true;"),
@@ -212,7 +212,7 @@ const contracts: Contract[] = [
match: [["discardExternal", /pendingExternalReplacement = null;\s*pendingReplacementRollback = null;/]] },
{ name: "forced cancellation reconciles resident status before preserving rollback",
match: [["forcedCancel", /run\.forceCancelActive/],
- ["forcedCancel", /await getInferenceStatus\(\)/], ["forcedCancel", /residentModelMatchesPick\(status/],
+ ["forcedCancel", /await getInferenceStatus\(undefined, run\.rollbackCheckpoint \?\? undefined\)/], ["forcedCancel", /residentModelMatchesPick\(status/],
["forcedCancel", /status\.loading\?\.length/]] },
{ name: "approved Hub credentials inherit a settled run rollback before replacement",
match: [["selection", /!activeRunBeforeCredentials\.loadAttemptPath/],
@@ -228,10 +228,10 @@ const contracts: Contract[] = [
{ name: "declining Hub credentials reconciles an unloaded predecessor after it settles",
match: [["credentialDecline", /activeRunBeforeCredentials\.settledPromise\.then\(async\s*\(\)\s*=>/],
["credentialDecline", /if \(activeRunBeforeCredentials\.residentModelUnloaded\)/],
- ["credentialDecline", /await getInferenceStatus\(\)/], ["credentialDecline", /status\.loading\?\.length[\s\S]*?!status\.active_model[\s\S]*?clearCheckpoint\(\)/]] },
+ ["credentialDecline", /await getInferenceStatus\(\s*undefined,\s*activeRunBeforeCredentials\.rollbackCheckpoint \?\? undefined,\s*\)/], ["credentialDecline", /status\.loading\?\.length[\s\S]*?!status\.active_model[\s\S]*?clearCheckpoint\(\)/]] },
{ name: "approved Hub credentials resnapshot a successfully compensated resident",
match: [["successfulCredentialRefresh", /activeRunBeforeCredentials\.residentModelUnloaded/],
- ["successfulCredentialRefresh", /await getInferenceStatus\(\)/],
+ ["successfulCredentialRefresh", /await getInferenceStatus\(\s*undefined,\s*activeRunBeforeCredentials\.rollbackCheckpoint \?\? undefined,\s*\)/],
["successfulCredentialRefresh", /residentModelMatchesPick\(status/], ["successfulCredentialRefresh", /previousConfigForReplacement = currentRuntimePerModelConfig\(\{\s*includeMaxSeqLength: true/]] },
];
for (const { name, match = [], absent = [], order = [] } of contracts) {
diff --git a/studio/frontend/tests/hub-runtime-boundary.test.ts b/studio/frontend/tests/hub-runtime-boundary.test.ts
index 3bd66fed69b..76946f0a026 100644
--- a/studio/frontend/tests/hub-runtime-boundary.test.ts
+++ b/studio/frontend/tests/hub-runtime-boundary.test.ts
@@ -87,11 +87,11 @@ test("residency remains a cache mutation safety input", () => {
"utf8",
);
- assert.match(hubPage, /getInferenceStatus\(\)/);
+ assert.match(hubPage, /getInferenceStatus\(\s*undefined,/);
assert.match(hubPage, /adoptResidentModelStatus\(/);
assert.match(
hubPage,
- /Promise\.all\(\[getInferenceStatus\(\), readIdleUnloadArmed\(\)\]\)\s*\.then\(\(\[status, idleUnloadArmed\]\) => \{/,
+ /Promise\.all\(\[\s*getInferenceStatus\([\s\S]*?\),\s*readIdleUnloadArmed\(\),\s*\]\)\s*\.then\(\(\[status, idleUnloadArmed\]\) => \{/,
);
assert.match(
hubPage,
diff --git a/studio/frontend/tests/loaded-gguf-quant-marker.test.ts b/studio/frontend/tests/loaded-gguf-quant-marker.test.ts
new file mode 100644
index 00000000000..162c02d84c9
--- /dev/null
+++ b/studio/frontend/tests/loaded-gguf-quant-marker.test.ts
@@ -0,0 +1,37 @@
+// SPDX-License-Identifier: AGPL-3.0-only
+// Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
+
+import assert from "node:assert/strict";
+import test from "node:test";
+
+import { readSrc } from "./helpers/kit.ts";
+
+const picker = readSrc("features/model-picker/components/model-selector/pickers.tsx");
+
+test("every quant list and cached GGUF row marks the quant each loaded model runs", () => {
+ const expanders = picker.match(/<GgufVariantExpander\b/g)?.length ?? 0;
+ assert.equal(picker.match(/loadedQuants=\{/g)?.length ?? 0, expanders);
+ assert.match(
+ picker,
+ /loadedQuants\?\.some\(\(q\) => ggufVariantsMatchForPicker\(q, v\.quant\)\) \? \(\s*<span[^>]*>\s*loaded\s*<\/span>/,
+ );
+ // Every loaded model, not just the one chat uses.
+ assert.match(picker, /loadedModels\s*\.filter\(\(m\) => m\.quant && modelIdsMatchForPicker\(m\.id, repoId\)\)/);
+ assert.match(
+ picker,
+ /if \(activeGgufVariant && modelIdsMatchForPicker\(loadedModelId, repoId\)\) \{\s*quants\.push\(activeGgufVariant\);/,
+ );
+ assert.match(picker, /meta="GGUF"\s*quantChip=\{loadedQuants\.map\(ggufQuantChipLabel\)\.join\(", "\) \|\| undefined\}/);
+});
+
+test("a loaded row is selected, picked and ejected by its checkpoint, not its label", () => {
+ const runtime = readSrc("features/chat/hooks/use-chat-model-runtime.ts");
+ assert.match(runtime, /const checkpoint = checkpoints\?\.\[i\] \|\| id;/);
+ assert.match(runtime, /statusRes\.serving_checkpoints,\s*\);/);
+ assert.match(runtime, /\.loadedModels\.map\(\(m\) => m\.checkpoint \?\? m\.id\)/);
+ assert.match(picker, /const checkpoint = entry\.checkpoint \?\? entry\.id;\s*const optionKey = makeModelOptionKey\("loaded", checkpoint\);\s*const isSelected = modelIdsMatchForPicker\(value, checkpoint\);/);
+ // Two files sharing a label are two rows, keyed by path.
+ assert.match(runtime, /const known = new Map\(current\.map\(\(m\) => \[m\.checkpoint \?\? m\.id, m\]\)\);/);
+ assert.match(picker, /onSelect\(checkpoint, \{/);
+ assert.match(picker, /onEject\?\.\(checkpoint\);/);
+});
diff --git a/studio/frontend/tests/resident-config-match.test.ts b/studio/frontend/tests/resident-config-match.test.ts
index b5793a6617b..9724c9ab4d4 100644
--- a/studio/frontend/tests/resident-config-match.test.ts
+++ b/studio/frontend/tests/resident-config-match.test.ts
@@ -1686,7 +1686,7 @@ test("the shortcut re-reads and re-judges the status before adopting", () => {
"the residency verdict is no longer callable against a second status",
);
const decision = USE_CHAT_MODEL_RUNTIME.search(
- /const confirmedStatus = await getInferenceStatus\(\)/,
+ /const confirmedStatus = await readPickStatus\(\)/,
);
assert.ok(
decision > 0,
@@ -1714,8 +1714,9 @@ test("the shortcut re-reads and re-judges the status before adopting", () => {
);
// A failed re-read must not adopt either: falling out of the block reaches /load.
assert.ok(
- USE_CHAT_MODEL_RUNTIME.indexOf("await getInferenceStatus().catch(() => null)", decision) ===
- decision + "const confirmedStatus = ".length,
+ USE_CHAT_MODEL_RUNTIME.includes(
+ "getInferenceStatus(undefined, modelId).catch(() => null);",
+ ),
"the re-read no longer tolerates a failed status",
);
});
@@ -1966,3 +1967,61 @@ test("legacy status without reasoning request echoes keeps its comparison", () =
reasoning_budget_message: "Conclude now.",
}, { ...BLANK, reasoningBudget: 32, reasoningBudgetMessage: "Conclude now." }), true);
});
+
+test("a pick asks the status about its own model and keeps or replaces the others per the box", () => {
+ const CONFIRM = readSrc("features/chat/utils/confirm-stop-running-chats.ts");
+ assert.equal(USE_CHAT_MODEL_RUNTIME.match(/await readPickStatus\(\)/g)?.length, 2);
+ assert.match(
+ USE_CHAT_MODEL_RUNTIME,
+ /const keepsOthers =\s*keepModelsLoaded && !forceReload && \(paramsNow\.engine \?\? "auto"\) === "auto";[\s\S]*?const touchesOnlySelected =\s*forceReload && !isExternalModelId\(paramsNow\.checkpoint\) && loadedNow\.length > 1;/,
+ );
+ assert.match(USE_CHAT_MODEL_RUNTIME, /touchesOnlySelected \? \(paramsNow\.checkpoint \?\? undefined\) : undefined,/);
+ assert.match(USE_CHAT_MODEL_RUNTIME, /stopQueuedRuns\(stopDecision, keepsOthers \|\| touchesOnlySelected\);/);
+ assert.match(USE_CHAT_MODEL_RUNTIME, /if \(!keepsOthers && !touchesOnlySelected\) \{\s*requestLocalPromptQueueStop\(\);/);
+ assert.match(USE_CHAT_MODEL_RUNTIME, /if \(currentCheckpoint && !keepsOthers\)/);
+ assert.match(USE_CHAT_MODEL_RUNTIME, /if \(!forceCancelActive && !touchesOnlySelected\) \{/);
+ assert.equal(
+ USE_CHAT_MODEL_RUNTIME.match(/alongside: keepModelsLoaded \|\| touchesOnlySelected,/g)?.length,
+ 2,
+ );
+ assert.match(CONFIRM, /let running = model\s*\?\s*\[\]/);
+ assert.match(CONFIRM, /await getActiveGenerations\(model\)/);
+});
+
+test("ejects stop only the ejected model's chats; eject all asks once and unloads the others first", () => {
+ assert.match(
+ USE_CHAT_MODEL_RUNTIME,
+ /function stopQueuedRuns\(decision: StopRunningChatsDecision, scoped: boolean\): void \{\s*if \(scoped\) \{\s*requestPromptQueueStop\(decision\.promptQueueThreadIds\);\s*return;\s*\}\s*cancelPreStreamRunReservations\(decision\.preStreamRunTokens\);\s*requestLocalPromptQueueStop\(decision\.promptQueueThreadIds\);/,
+ );
+ assert.match(
+ USE_CHAT_MODEL_RUNTIME,
+ /confirmStopRunningChatsIfNeeded\("Unloading this model", "unload", keptId\)[\s\S]{0,80}?if \(!decision\.proceed\) return false;\s*stopQueuedRuns\(decision, true\);/,
+ );
+ assert.match(
+ USE_CHAT_MODEL_RUNTIME,
+ /const scope =\s*!confirmed && useChatRuntimeStore\.getState\(\)\.loadedModels\.length > 1\s*\?\s*params\.checkpoint\s*:\s*undefined;\s*const stopDecision =\s*confirmed \?\?\s*\(await confirmStopRunningChatsIfNeeded\(\s*"Unloading the model",\s*"unload",\s*scope,\s*\)\);/,
+ );
+ assert.match(USE_CHAT_MODEL_RUNTIME, /stopQueuedRuns\(stopDecision, Boolean\(scope\)\);/);
+ assert.match(
+ USE_CHAT_MODEL_RUNTIME,
+ /"Unloading every model",\s*"unload",\s*\);\s*if \(!decision\.proceed\) return false;\s*\/\/ Before any unload[^\n]*\n\s*stopQueuedRuns\(decision, false\);\s*\/\/ Others first/,
+ );
+ assert.match(
+ USE_CHAT_MODEL_RUNTIME,
+ /others\.map\(\(id\) =>\s*unloadModel\(\{ model_path: id, force_cancel_active: decision\.forceCancelActive \}\),\s*\),\s*\);\s*if \(selectedLocal && !\(await ejectModel\(undefined, decision\)\)\) return false;\s*await refresh\(\);/,
+ );
+});
+
+test("cancelling a load clears the selection unless kept models stay loaded and this run unloaded none", () => {
+ assert.match(
+ USE_CHAT_MODEL_RUNTIME,
+ /if \(!preserveCheckpoint\) \{[\s\S]{0,160}?if \(!useChatRuntimeStore\.getState\(\)\.keepModelsLoaded \|\| run\.residentModelUnloaded\) \{\s*clearCheckpoint\(\);\s*\}\s*await refresh\(\);/,
+ );
+});
+
+test("reloading one of several stays in its own slot; a new pick with the setting off replaces as before", () => {
+ // No preliminary unload for the reload: /load finds the model's slot and replaces it there, so
+ // the server's setting gate never sends it to the primary seat.
+ assert.doesNotMatch(USE_CHAT_MODEL_RUNTIME, /replacesOneOfSeveral/);
+ assert.match(USE_CHAT_MODEL_RUNTIME, /const touchesOnlySelected =\s*forceReload &&/);
+});
diff --git a/tests/studio/test_cancel_atomicity.py b/tests/studio/test_cancel_atomicity.py
index 361b8208933..b4278a4fd6a 100644
--- a/tests/studio/test_cancel_atomicity.py
+++ b/tests/studio/test_cancel_atomicity.py
@@ -104,6 +104,7 @@ _WANTED = {
"_prune_pending",
"_remember_pending_cancel",
"_TrackedCancel",
+ "_slot_lock",
"_cancel_by_keys",
"_cancel_by_cancel_id_or_stash",
}
@@ -168,6 +169,7 @@ def _registry_source():
def _load_registry_module():
mod = {"active_generations": _load_active_generations()}
exec(
+ 'import contextvars\nrouted_slot = contextvars.ContextVar("routed_slot", default = None)\n'
"import threading, time\nfrom typing import Optional\n_account_cancel_key = lambda key: key\n"
+ _registry_source(),
mod,
diff --git a/tests/studio/test_chat_mount_cli_load_adoption.py b/tests/studio/test_chat_mount_cli_load_adoption.py
index ffb05479d43..82d57577a74 100644
--- a/tests/studio/test_chat_mount_cli_load_adoption.py
+++ b/tests/studio/test_chat_mount_cli_load_adoption.py
@@ -318,6 +318,7 @@ def _run(script_body: str) -> dict:
ggufContextLength: null,
models: [],
loras: [],
+ loadedModels: [],
...over,
};
}
diff --git a/tests/studio/test_model_picker_contracts.py b/tests/studio/test_model_picker_contracts.py
index 35f745751fd..4c769b3180c 100644
--- a/tests/studio/test_model_picker_contracts.py
+++ b/tests/studio/test_model_picker_contracts.py
@@ -2544,7 +2544,7 @@ def test_adopting_a_resident_model_reseeds_the_slot_and_batch_controls():
# And the rollback that makes the reseed necessary is still ordered before the
# hydration it protects, in the adopt path.
runtime = _read("features/chat/hooks/use-chat-model-runtime.ts")
- adopt = runtime[runtime.index("const confirmedStatus = await getInferenceStatus()") :]
+ adopt = runtime[runtime.index("const confirmedStatus = await readPickStatus()") :]
adopt = adopt[: adopt.index("void refreshContextUsage(")]
assert (
adopt.index("restorePreviousConfig();")
@@ -2951,7 +2951,7 @@ def test_adoption_takes_its_own_pin_before_moving_the_checkpoint():
loadable. The ordering requirement is unchanged and is what this still pins.
"""
src = _read("features/chat/hooks/use-chat-model-runtime.ts")
- branch = src[src.index("const confirmedStatus = await getInferenceStatus()") :]
+ branch = src[src.index("const confirmedStatus = await readPickStatus()") :]
branch = branch[: branch.index("void refreshContextUsage(")]
assert "activeLoadId: loadPath === modelId ? null : loadPath," in branch
# Landing before the checkpoint moves, so nothing reads the pair half updated.
diff --git a/tests/studio/test_new_chat_context_recount.py b/tests/studio/test_new_chat_context_recount.py
index 1d65c3e3679..faae947a495 100644
--- a/tests/studio/test_new_chat_context_recount.py
+++ b/tests/studio/test_new_chat_context_recount.py
@@ -147,7 +147,7 @@ def _resident_fast_path() -> str:
"""
return slice_between(
read(RUNTIME),
- " const confirmedStatus = await getInferenceStatus().catch(() => null);",
+ " const confirmedStatus = await readPickStatus();",
# The lease claim, which the tail must not run into: the pick here never starts a load.
# `lifecycleLease` sits inside a bounded retry loop, so its own declaration is the first
# line that is about the load this tail does not start rather than about the adoption.
@@ -523,6 +523,7 @@ export async function adoptResidentModel(props: any): Promise<void> {
let pendingReplacementRollback: any = null;
const restorePreviousConfig = (): void => {};
const getInferenceStatus = async (): Promise<any> => props.residentStatus;
+ const readPickStatus = async (): Promise<any> => props.residentStatus;
const reconcilePersistedGpuIds = (ids: any): any => ids;
const sameGpuSelection = (a: any, b: any): boolean =>
JSON.stringify(a ?? null) === JSON.stringify(b ?? null);
diff --git a/tests/studio/test_stream_cancel_registration_timing.py b/tests/studio/test_stream_cancel_registration_timing.py
index 14cea4861c7..b8681469a69 100644
--- a/tests/studio/test_stream_cancel_registration_timing.py
+++ b/tests/studio/test_stream_cancel_registration_timing.py
@@ -272,6 +272,7 @@ _WANTED = {
"_PENDING_CANCEL_TTL_S",
"_prune_pending",
"_TrackedCancel",
+ "_slot_lock",
"_cancel_by_keys",
"_cancel_by_cancel_id_or_stash",
}
@@ -336,7 +337,9 @@ def _registry_source():
def _load_registry_module():
mod = {"active_generations": _load_active_generations()}
exec(
- "import threading, time\n_account_cancel_key = lambda key: key\n" + _registry_source(), mod
+ 'import contextvars\nrouted_slot = contextvars.ContextVar("routed_slot", default = None)\n'
+ "import threading, time\n_account_cancel_key = lambda key: key\n" + _registry_source(),
+ mod,
)
return mod
diff --git a/tests/studio/test_studio_text_descender_clipping.py b/tests/studio/test_studio_text_descender_clipping.py
index 9cad04e87de..384e019332a 100644
--- a/tests/studio/test_studio_text_descender_clipping.py
+++ b/tests/studio/test_studio_text_descender_clipping.py
@@ -30,7 +30,7 @@ def _read(path: Path) -> str:
# The trigger's name label. Since #11757 the label stays whole and the model name truncates in its
# own span inside it, inheriting the label's line height.
TRIGGER_LABEL_CLASSES = frozenset({"font-heading", "text-ui-16", "whitespace-nowrap"})
-TRIGGER_NAME_SPAN = '<span className="min-w-0 truncate">{currentModel?.name ?? placeholder}</span>'
+TRIGGER_NAME_SPAN = '<span className="min-w-0 truncate">{triggerTitle}</span>'
SIDEBAR_ACCOUNT_CLASSES = frozenset(