Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

Studio: reload an idle-unloaded GGUF with its original launch settings (#16927)

Nilay847a8ec25d7e
diff --git a/studio/backend/core/inference/llama_keepwarm.py b/studio/backend/core/inference/llama_keepwarm.py
index 633411cf687..248bcdad2e0 100644
--- a/studio/backend/core/inference/llama_keepwarm.py
+++ b/studio/backend/core/inference/llama_keepwarm.py
@@ -49,6 +49,7 @@ _unattended: contextvars.ContextVar[bool] = contextvars.ContextVar(
# can reload it (set on unload, cleared on reload). The quant lets the reload restore the exact freed variant.
_last_unloaded_model = None
_last_unloaded_mlx = False
+_last_unloaded_intent = None
# Slot KV manifest saved by the idle unload; whoever pops it owns deleting its files.
_kv_resume = None
# Slot saves of manually unloaded GGUFs, oldest first, until that GGUF loads again.
@@ -557,12 +558,22 @@ def stash_is_mlx(identity) -> bool:
return _last_unloaded_mlx and _last_unloaded_model is identity
-def _set_last_unloaded(value, mlx: bool = False) -> None:
- global _last_unloaded_model, _last_unloaded_mlx, _kv_resume
+def get_last_unloaded_intent(identity):
+ with _lock:
+ return _last_unloaded_intent if _last_unloaded_model is identity else None
+
+
+def _set_last_unloaded(
+ value,
+ mlx: bool = False,
+ intent = None,
+) -> None:
+ global _last_unloaded_model, _last_unloaded_mlx, _last_unloaded_intent, _kv_resume
stale = None
with _lock:
_last_unloaded_model = value
_last_unloaded_mlx = mlx and value is not None
+ _last_unloaded_intent = intent if value is not None else None
if value is None and _kv_resume is not None:
stale, _kv_resume = _kv_resume, None
if stale:
@@ -892,9 +903,20 @@ def loaded_identity():
return _loaded_identity(get_llama_cpp_backend())
-def stash_for_reload(identity) -> None:
+def launch_to_replay(backend = None):
+ if backend is None:
+ from routes.inference import get_llama_cpp_backend
+ backend = get_llama_cpp_backend()
+ launch = getattr(backend, "replay_launch", None)
+ if not backend.is_loaded or launch is None:
+ return None
+ intent = getattr(backend, "last_load_intent", None)
+ return {"kwargs": launch, "cpu_fallback": bool(getattr(intent, "cpu_fallback", False))}
+
+
+def stash_for_reload(identity, launch = None) -> None:
"""Let the next reload-only request bring ``identity`` back, as after an idle unload."""
- _set_last_unloaded(identity)
+ _set_last_unloaded(identity, intent = launch)
def _note_idle_unload_event(freed, reason: str = "idle") -> None:
@@ -997,6 +1019,7 @@ async def idle_unload_loop(poll_seconds: float = 15.0) -> None:
if manifest:
_delete_resume_files(manifest)
continue
+ launch = launch_to_replay(backend) if mlx is None else None
try:
if mlx is None:
await asyncio.to_thread(backend.unload_model)
@@ -1006,7 +1029,9 @@ async def idle_unload_loop(poll_seconds: float = 15.0) -> None:
if manifest:
_delete_resume_files(manifest)
raise
- _set_last_unloaded(freed, mlx = mlx is not None) # let an alias request reload it
+ _set_last_unloaded(
+ freed, mlx = mlx is not None, intent = launch
+ ) # let an alias request reload it
if manifest and freed:
_set_kv_resume({"identity": freed, "origin": "idle", **manifest})
logger.info("Idle auto-unload: saved chat KV for restore on reload")
diff --git a/studio/backend/core/scheduled_tasks/scheduler.py b/studio/backend/core/scheduled_tasks/scheduler.py
index 0e42b0f65f7..93a71da95c0 100644
--- a/studio/backend/core/scheduled_tasks/scheduler.py
+++ b/studio/backend/core/scheduled_tasks/scheduler.py
@@ -82,6 +82,7 @@ class _Swap:
attended: Optional[float] = None
# A Transformers or MLX model in the primary has no GGUF identity: it is loaded back by name.
previous_model: Optional[str] = None
+ previous_launch: Optional[dict] = None
@dataclass
@@ -357,7 +358,12 @@ def _model_fields(model: Optional[str], may_load: bool) -> tuple[dict, Optional[
async def _plan_swap(task: dict, model: str, unattended: bool, carried: Optional[_Swap]) -> _Swap:
"""What a load must put back; an unattended run swaps only a restorable GGUF while Studio is idle."""
- from core.inference.llama_keepwarm import attended_stamp, idle_for, loaded_identity
+ from core.inference.llama_keepwarm import (
+ attended_stamp,
+ idle_for,
+ launch_to_replay,
+ loaded_identity,
+ )
from routes.inference import (
_loaded_slot_ident,
_resident_npu_model,
@@ -384,6 +390,7 @@ async def _plan_swap(task: dict, model: str, unattended: bool, carried: Optional
keep_carried = carried is not None and carried.model and _model_is_loaded(carried.model)
previous = carried.previous if keep_carried else loaded_identity()
previous_model = carried.previous_model if keep_carried else None
+ previous_launch = carried.previous_launch if keep_carried else launch_to_replay()
if not keep_carried and previous is None and _resident_npu_model() is None:
previous_model = getattr(
await asyncio.to_thread(get_inference_backend), "active_model_name", None
@@ -396,6 +403,7 @@ async def _plan_swap(task: dict, model: str, unattended: bool, carried: Optional
task_id = task["id"],
attended = attended_stamp(),
previous_model = previous_model,
+ previous_launch = previous_launch,
)
@@ -457,7 +465,7 @@ async def _restore_after_swap(swap: _Swap, app: Any) -> None:
if swap.previous is None and not swap.previous_model:
return
if swap.previous is not None:
- stash_for_reload(swap.previous)
+ stash_for_reload(swap.previous, swap.previous_launch)
# Named for a non-GGUF model, which must not be held to what is loaded now.
request = _producer_request(
app,
diff --git a/studio/backend/routes/inference.py b/studio/backend/routes/inference.py
index 68156c6999a..49a4b79b205 100644
--- a/studio/backend/routes/inference.py
+++ b/studio/backend/routes/inference.py
@@ -8115,28 +8115,115 @@ def _owner_chosen_launch(
)
sources.append(override.get("llama_extra_args"))
intent = getattr(get_llama_cpp_backend(), "last_load_intent", None)
- if (
- intent is not None
- # A snapshot path and its repo id are one model; other paths compare as before.
- and _same_loaded_identifier(
- _snapshot_repo_or_self(
- getattr(intent, "model_identifier", None), require_cache = True
- ),
- _snapshot_repo_or_self(identifier, require_cache = True),
- )
- # No variant named: the load resolves the same quant the resident one did.
- and (
- variant is None
- or (getattr(intent, "hf_variant", None) or "").casefold() == variant.casefold()
- )
+ if intent is not None and _same_launch_target(
+ getattr(intent, "model_identifier", None),
+ getattr(intent, "hf_variant", None),
+ identifier,
+ variant,
):
sources.append(getattr(intent, "extra_args", None))
+ # The idle-freed launch passed this check when it loaded, as the resident one did.
+ from core.inference.llama_keepwarm import get_last_unloaded_intent, get_last_unloaded_model
+
+ last = get_last_unloaded_model()
+ launch = get_last_unloaded_intent(last) if last else None
+ if launch and _same_launch_target(last[0], last[1], identifier, variant):
+ sources.append(launch["kwargs"].get("llama_extra_args"))
pairs: set[tuple[str, str]] = set()
for args in sources:
pairs.update(owner_only_path_args(args))
return pairs
+def _same_launch_target(loaded_id, loaded_variant, identifier, variant) -> bool:
+ # A snapshot path and its repo id are one model; no variant named resolves the same quant.
+ return _same_loaded_identifier(
+ _snapshot_repo_or_self(loaded_id, require_cache = True),
+ _snapshot_repo_or_self(identifier, require_cache = True),
+ ) and (variant is None or (loaded_variant or "").casefold() == variant.casefold())
+
+
+_REPLAY_LOAD_FIELDS = frozenset(
+ {
+ "max_seq_length",
+ "max_seq_length_auto_derived",
+ "chat_template_override",
+ "cache_type_kv",
+ "gpu_ids",
+ "speculative_type",
+ "spec_draft_n_max",
+ "n_parallel",
+ "n_batch",
+ "n_ubatch",
+ "load_mode",
+ "spec_draft_cache_type",
+ "ctx_checkpoints",
+ "cache_ram",
+ "tensor_parallel",
+ "disable_vision",
+ "gpu_memory_mode",
+ "gpu_layers",
+ "n_cpu_moe",
+ "tensor_split",
+ "llama_extra_args",
+ "reasoning_budget",
+ "reasoning_budget_message",
+ }
+)
+_PLACEMENT_FIELDS = (
+ "gpu_memory_mode",
+ "gpu_layers",
+ "n_cpu_moe",
+ "gpu_ids",
+ "tensor_split",
+ "tensor_parallel",
+)
+
+
+def _note_replay_launch(llama_backend, request: LoadRequest) -> None:
+ from utils.openai_auto_switch_settings import (
+ model_override_load_kwargs,
+ resolve_override_for_load,
+ )
+
+ # Saved settings are re-read on reload, so a reset one must not come back from here.
+ _, override = resolve_override_for_load(
+ request.model_path,
+ request._override_alias_id
+ or request._advertised_id
+ or _snapshot_repo_or_self(request.model_path, require_cache = True),
+ request.gguf_variant,
+ )
+ owned = model_override_load_kwargs(override, is_gguf = True)
+ fields = (request.model_fields_set & _REPLAY_LOAD_FIELDS) - owned.keys()
+ llama_backend.replay_launch = request.model_dump(include = fields)
+
+
+def _replay_load_kwargs(launch, override_kwargs) -> dict[str, Any]:
+ kwargs = dict(launch["kwargs"])
+ # A CPU fallback is per launch: reload with automatic placement.
+ if launch.get("cpu_fallback"):
+ for name in _PLACEMENT_FIELDS:
+ kwargs.pop(name, None)
+ if "gpu_ids" in override_kwargs:
+ kwargs.pop("tensor_split", None)
+ if "max_seq_length" in override_kwargs:
+ kwargs.pop("max_seq_length_auto_derived", None)
+ if "gpu_memory_mode" in override_kwargs:
+ kwargs.pop("gpu_layers", None)
+ if kwargs.get("llama_extra_args") and "llama_extra_args" not in override_kwargs:
+ from utils.openai_auto_switch_settings import strip_args_shadowed_by
+ kwargs["llama_extra_args"] = strip_args_shadowed_by(
+ kwargs["llama_extra_args"],
+ override_kwargs,
+ strip_offload = override_kwargs.get("gpu_memory_mode") == "manual",
+ strip_tensor_split = "gpu_ids" in override_kwargs,
+ strip_load_mode = "load_mode" in override_kwargs,
+ strip_load_mode_aliases = "load_mode" in override_kwargs,
+ )
+ return kwargs
+
+
def _snapshot_repo_or_self(model_id, require_cache = False):
from core.inference.model_ids import hf_cache_repo_id
@@ -11177,6 +11264,7 @@ async def _maybe_auto_switch_model(
)
from utils.security.remote_code_approvals import approved_local_fingerprint
from core.inference.llama_keepwarm import (
+ get_last_unloaded_intent,
get_last_unloaded_model,
inference_lifecycle_gate,
model_load_gate,
@@ -11319,6 +11407,7 @@ async def _maybe_auto_switch_model(
repo_level_companion_resolution = False
stashed_gguf_companion_roots: tuple[str, ...] = ()
stashed_mlx = False
+ stashed_intent = None
if auto_switch_on and not reload_only:
# audio.cpp GGUFs load via the audio worker, not llama-server; the HF cache resolves them.
resolved = await asyncio.to_thread(_audio_cpp_switch_target, requested_model)
@@ -11376,6 +11465,7 @@ async def _maybe_auto_switch_model(
target_id, variant = last
override_id = target_id
stashed_mlx = stash_is_mlx(last)
+ stashed_intent = get_last_unloaded_intent(last)
# Held-back keyless callers may restore only the model they named; check before loading.
if keyless_held_back and not reload_only:
requested_base, requested_variant = split_model_ref(requested_model)
@@ -11397,6 +11487,10 @@ async def _maybe_auto_switch_model(
requested_base,
(override_id, public_model_id(override_id)),
)
+ # A request naming the idle-freed model reloads it as it was launched too.
+ last = get_last_unloaded_model()
+ if last and _same_launch_target(last[0], last[1], target_id, variant):
+ stashed_intent = get_last_unloaded_intent(last)
# Not inferred from the quant; audio.cpp GGUFs are read only by audiocpp_server.
from core.inference.audio_cpp_models import looks_like_audio_cpp
@@ -11773,9 +11867,12 @@ async def _maybe_auto_switch_model(
),
)
load_kwargs = {"model_path": target_id, "gguf_variant": variant}
- load_kwargs.update(
- model_override_load_kwargs(override, is_gguf = target_is_gguf)
+ override_kwargs = model_override_load_kwargs(
+ override, is_gguf = target_is_gguf
)
+ if stashed_intent is not None and target_is_gguf:
+ load_kwargs.update(_replay_load_kwargs(stashed_intent, override_kwargs))
+ load_kwargs.update(override_kwargs)
if caller_hf_token:
load_kwargs["hf_token"] = caller_hf_token
if remote_code_fingerprint:
@@ -11787,7 +11884,11 @@ async def _maybe_auto_switch_model(
load_kwargs["alongside"] = True
saved_gpu_ids = load_kwargs.get("gpu_ids")
if saved_gpu_ids and not await _override_gpu_ids_still_resolve(
- saved_gpu_ids, stored_gpu_index_kind(override)
+ saved_gpu_ids,
+ # A replayed pin was sent in this process's index space.
+ stored_gpu_index_kind(override)
+ if "gpu_ids" in override_kwargs
+ else None,
):
# Stale pin (GPU removed, another host, or a backend change
# renumbering these ids): drop it rather than 400 the load.
@@ -15533,7 +15634,7 @@ async def _validate_reasoning_budget_preflight(
async def _override_gpu_ids_still_resolve(
- gpu_ids: List[int], stored_index_kind: str = "physical"
+ gpu_ids: List[int], stored_index_kind: Optional[str] = "physical"
) -> bool:
"""Checks the pin's stored index space first: Vulkan ordinals and CUDA ids name different devices."""
try:
@@ -15553,7 +15654,7 @@ async def _override_gpu_ids_still_resolve(
from utils.hardware.hardware import current_gpu_index_kind
current_kind = await asyncio.to_thread(current_gpu_index_kind, is_vulkan)
- if stored_index_kind != current_kind:
+ if stored_index_kind is not None and stored_index_kind != current_kind:
return False
if device == DeviceType.XPU and not is_vulkan:
# Rejected outright on XPU.
@@ -19657,6 +19758,7 @@ async def _load_model_impl(
llama_backend._openai_gguf_companion_roots = tuple(request._gguf_companion_roots)
llama_backend._openai_gguf_companion_state = gguf_companion_state
+ _note_replay_launch(llama_backend, request)
if replacing:
await asyncio.to_thread(note_model_loaded, llama_backend)
# Else None, or the catalog id a /models/load named: only those loads have an alias to keep.
diff --git a/studio/backend/tests/test_openai_auto_switch.py b/studio/backend/tests/test_openai_auto_switch.py
index 8f6b6b87647..5a3da4cbbd2 100644
--- a/studio/backend/tests/test_openai_auto_switch.py
+++ b/studio/backend/tests/test_openai_auto_switch.py
@@ -76,6 +76,7 @@ def _reset_keepwarm():
kw._last_active = time.monotonic() - 3600
kw._last_unloaded_model = None
kw._last_unloaded_mlx = False
+ kw._last_unloaded_intent = None
kw._kv_resume = None
@@ -3079,6 +3080,236 @@ def test_idle_alias_reload_preserves_override_via_advertised_id(monkeypatch):
assert rec.calls[0].max_seq_length == 8192
+_CUSTOM_LAUNCH = {
+ "max_seq_length": 8192,
+ "cache_type_kv": "q8_0",
+ "cache_ram": 1024,
+ "speculative_type": "off",
+ "gpu_memory_mode": "manual",
+ "gpu_layers": 0,
+ "llama_extra_args": ["--no-warmup"],
+ "n_parallel": 2,
+}
+
+
+class _LaunchRecorder(_LoadRecorder):
+ async def __call__(self, request, *args, **kwargs):
+ await super().__call__(request, *args, **kwargs)
+ inference_route._note_replay_launch(self.backend, request)
+
+
+def _idle_unload(monkeypatch, backend):
+ monkeypatch.setattr(settings, "get_auto_unload_idle_seconds", lambda: 0.005)
+ _drive_idle_loop(kw, run_for = 0.05, until = lambda: kw.get_last_unloaded_model() is not None)
+ assert not backend.is_loaded
+ monkeypatch.setattr(settings, "get_auto_unload_idle_seconds", lambda: 600)
+
+
+def _idle_unload_a_custom_launch(
+ monkeypatch,
+ launch = None,
+ overrides = None,
+ enabled = False,
+ resolves_to = None,
+ cpu_fallback = False,
+):
+ monkeypatch.setattr(settings, "idle_unload_is_configured", lambda: True)
+ _reset_keepwarm()
+ overrides = {} if overrides is None else overrides
+ monkeypatch.setattr(settings, "get_model_override", lambda mid: overrides.get(mid, {}))
+ backend = _FakeBackend("/cache/snap/A", hf_variant = "Q4_K_M", advertised_id = "org/A-GGUF")
+ backend.last_load_intent = SimpleNamespace(cpu_fallback = cpu_fallback)
+ inference_route._note_replay_launch(
+ backend,
+ LoadRequest(
+ model_path = "/cache/snap/A", gguf_variant = "Q4_K_M", **(launch or _CUSTOM_LAUNCH)
+ ),
+ )
+
+ def _unload():
+ backend.is_loaded = False
+
+ backend.unload_model = _unload
+ rec = _LaunchRecorder(backend)
+ _wire(monkeypatch, enabled = enabled, resolves_to = resolves_to, backend = backend, recorder = rec)
+ _idle_unload(monkeypatch, backend)
+ return rec
+
+
+def _reload_unnamed():
+ asyncio.run(
+ inference_route._auto_switch_from_request_body(
+ _json_body_request({"prompt": "hi"}), "tester"
+ )
+ )
+
+
+def test_idle_reload_keeps_the_launch_settings_it_was_loaded_with(monkeypatch):
+ rec = _idle_unload_a_custom_launch(monkeypatch)
+ _reload_unnamed()
+ (reload,) = rec.calls
+ assert (reload.model_path, reload.gguf_variant) == ("/cache/snap/A", "Q4_K_M")
+ assert reload.max_seq_length == 8192
+ assert reload.cache_type_kv == "q8_0"
+ assert reload.cache_ram == 1024
+ assert reload.speculative_type == "off"
+ assert (reload.gpu_memory_mode, reload.gpu_layers) == ("manual", 0)
+ assert reload.llama_extra_args == ["--no-warmup"]
+ assert reload.n_parallel == 2
+
+
+def test_idle_reload_lets_a_saved_override_win_over_the_freed_launch(monkeypatch):
+ rec = _idle_unload_a_custom_launch(
+ monkeypatch, overrides = {"org/A-GGUF": {"max_seq_length": 4096}}
+ )
+ _reload_unnamed()
+ (reload,) = rec.calls
+ assert reload.max_seq_length == 4096
+ assert reload.cache_type_kv == "q8_0"
+
+
+def test_idle_reload_strips_a_freed_flag_the_saved_context_shadows(monkeypatch):
+ launch = {**_CUSTOM_LAUNCH, "llama_extra_args": ["--ctx-size", "8192", "--no-warmup"]}
+ rec = _idle_unload_a_custom_launch(monkeypatch, launch = launch)
+ saved = {"org/A-GGUF": {"max_seq_length": 4096}}
+ monkeypatch.setattr(settings, "get_model_override", lambda mid: saved.get(mid, {}))
+ _reload_unnamed()
+ (reload,) = rec.calls
+ assert reload.max_seq_length == 4096
+ assert reload.llama_extra_args == ["--no-warmup"]
+
+
+def test_idle_reload_uses_saved_auto_layers_over_a_freed_cpu_only_launch(monkeypatch):
+ rec = _idle_unload_a_custom_launch(monkeypatch)
+ saved = {"org/A-GGUF": {"gpu_memory_mode": "manual"}}
+ monkeypatch.setattr(settings, "get_model_override", lambda mid: saved.get(mid, {}))
+ _reload_unnamed()
+ (reload,) = rec.calls
+ assert (reload.gpu_memory_mode, reload.gpu_layers) == ("manual", -1)
+
+
+def test_idle_reload_strips_a_freed_split_flag_the_saved_pin_replaces(monkeypatch):
+ async def _resolves(ids, kind = "physical"):
+ return True
+
+ monkeypatch.setattr(inference_route, "_override_gpu_ids_still_resolve", _resolves)
+ launch = {**_CUSTOM_LAUNCH, "llama_extra_args": ["--tensor-split", "2,1", "--no-warmup"]}
+ rec = _idle_unload_a_custom_launch(monkeypatch, launch = launch)
+ saved = {"org/A-GGUF": {"gpu_memory_mode": "manual", "gpu_layers": 99, "gpu_ids": [0, 1]}}
+ monkeypatch.setattr(settings, "get_model_override", lambda mid: saved.get(mid, {}))
+ _reload_unnamed()
+ (reload,) = rec.calls
+ assert reload.llama_extra_args == ["--no-warmup"]
+
+
+def test_hub_snapshot_load_does_not_record_its_saved_settings_as_launch(monkeypatch):
+ path = "/hf/models--org--A-GGUF/snapshots/abc"
+ monkeypatch.setattr(inference_route, "_inside_hf_cache", lambda p: True)
+ saved = {"org/A-GGUF": {"kv_cache_dtype": "q8_0"}}
+ monkeypatch.setattr(settings, "get_model_override", lambda mid: saved.get(mid, {}))
+ backend = SimpleNamespace()
+ inference_route._note_replay_launch(
+ backend,
+ LoadRequest(
+ model_path = path, gguf_variant = "Q4_K_M", max_seq_length = 8192, cache_type_kv = "q8_0"
+ ),
+ )
+ assert backend.replay_launch == {"max_seq_length": 8192}
+
+
+def test_idle_reload_replays_only_fields_the_launch_set(monkeypatch):
+ rec = _idle_unload_a_custom_launch(monkeypatch, launch = {"max_seq_length": 8192})
+ _reload_unnamed()
+ (reload,) = rec.calls
+ assert reload.max_seq_length == 8192
+ assert reload.model_fields_set == {"model_path", "gguf_variant", "max_seq_length"}
+
+
+def test_named_request_reloads_the_idle_freed_model_with_its_launch_settings(monkeypatch):
+ rec = _idle_unload_a_custom_launch(
+ monkeypatch, enabled = True, resolves_to = ("/cache/snap/A", "Q4_K_M", "org/A-GGUF")
+ )
+ _run_hook("org/A-GGUF:Q4_K_M")
+ (reload,) = rec.calls
+ assert reload.model_path == "/cache/snap/A"
+ assert reload.max_seq_length == 8192
+ assert reload.cache_type_kv == "q8_0"
+
+
+def test_named_request_for_another_model_does_not_take_the_freed_launch(monkeypatch):
+ rec = _idle_unload_a_custom_launch(
+ monkeypatch, enabled = True, resolves_to = ("/cache/snap/B", "Q4_K_M", "org/B-GGUF")
+ )
+ _run_hook("org/B-GGUF:Q4_K_M")
+ (reload,) = rec.calls
+ assert reload.model_path == "/cache/snap/B"
+ assert reload.max_seq_length == 0
+ assert reload.cache_type_kv is None
+
+
+def test_idle_reload_replays_gpu_placement_and_drops_a_stale_pin(monkeypatch):
+ kinds = []
+ usable = [True]
+
+ async def _resolves(ids, kind = "physical"):
+ kinds.append(kind)
+ return usable[0]
+
+ monkeypatch.setattr(inference_route, "_override_gpu_ids_still_resolve", _resolves)
+ launch = {"max_seq_length": 8192, "gpu_ids": [0, 1], "tensor_split": [2.0, 1.0]}
+ rec = _idle_unload_a_custom_launch(monkeypatch, launch = launch)
+ _reload_unnamed()
+ assert (rec.calls[0].gpu_ids, rec.calls[0].tensor_split) == ([0, 1], [2.0, 1.0])
+ assert kinds == [None]
+ _idle_unload(monkeypatch, rec.backend)
+ usable[0] = False
+ _reload_unnamed()
+ assert (rec.calls[1].gpu_ids, rec.calls[1].tensor_split) == (None, None)
+ assert rec.calls[1].max_seq_length == 8192
+
+
+def test_idle_reload_after_a_cpu_fallback_uses_automatic_placement(monkeypatch):
+ rec = _idle_unload_a_custom_launch(monkeypatch, cpu_fallback = True)
+ _reload_unnamed()
+ (reload,) = rec.calls
+ assert (reload.gpu_memory_mode, reload.gpu_layers) == ("auto", -1)
+ assert "gpu_layers" not in reload.model_fields_set
+ assert reload.cache_type_kv == "q8_0"
+
+
+def test_a_reset_saved_setting_does_not_come_back_from_the_freed_launch(monkeypatch):
+ overrides = {"org/A-GGUF": {"kv_cache_dtype": "q8_0"}}
+ rec = _idle_unload_a_custom_launch(
+ monkeypatch, launch = {"max_seq_length": 8192}, overrides = overrides
+ )
+ _reload_unnamed()
+ assert rec.calls[0].cache_type_kv == "q8_0"
+ overrides.clear()
+ _idle_unload(monkeypatch, rec.backend)
+ _reload_unnamed()
+ assert rec.calls[1].cache_type_kv is None
+ assert rec.calls[1].max_seq_length == 8192
+
+
+def test_managed_caller_may_reload_the_owners_idle_freed_projector(monkeypatch):
+ _idle_unload_a_custom_launch(
+ monkeypatch, launch = {"llama_extra_args": ["--mmproj", "/owner/p.gguf"]}
+ )
+ monkeypatch.setattr(inference_route.account_access, "managed_account", lambda: True)
+ inference_route._refuse_managed_custom_projector(
+ ["--mmproj", "/owner/p.gguf"], "/cache/snap/A", "org/A-GGUF", "Q4_K_M"
+ )
+ with pytest.raises(HTTPException) as err:
+ inference_route._refuse_managed_custom_projector(
+ ["--mmproj", "/other/p.gguf"], "/cache/snap/A", "org/A-GGUF", "Q4_K_M"
+ )
+ assert err.value.status_code == 403
+ with pytest.raises(HTTPException):
+ inference_route._refuse_managed_custom_projector(
+ ["--mmproj", "/owner/p.gguf"], "/cache/snap/B", "org/B-GGUF", "Q4_K_M"
+ )
+
+
def test_load_route_holds_lifecycle_gate(monkeypatch):
# The /load route must hold inference_lifecycle_gate so idle-unload cannot fire mid-load.
diff --git a/studio/backend/tests/test_scheduled_tasks.py b/studio/backend/tests/test_scheduled_tasks.py
index 845a847c349..2aa580c531e 100644
--- a/studio/backend/tests/test_scheduled_tasks.py
+++ b/studio/backend/tests/test_scheduled_tasks.py
@@ -1630,11 +1630,18 @@ class SwapWorld:
from utils import openai_auto_switch_settings
self.loaded = earlier
- self.unloaded, self.stashed, self.reloads = [], [], []
+ self.unloaded, self.stashed, self.reloads, self.launches = [], [], [], []
+ self.launch = None
self.idle = idle
monkeypatch.setattr(llama_keepwarm, "idle_for", lambda seconds: self.idle)
monkeypatch.setattr(llama_keepwarm, "loaded_identity", lambda: self.loaded)
- monkeypatch.setattr(llama_keepwarm, "stash_for_reload", self.stashed.append)
+ monkeypatch.setattr(llama_keepwarm, "launch_to_replay", lambda: self.launch)
+
+ def stash(identity, launch = None):
+ self.stashed.append(identity)
+ self.launches.append(launch)
+
+ monkeypatch.setattr(llama_keepwarm, "stash_for_reload", stash)
monkeypatch.setattr(
inference_routes,
"_loaded_satisfies",
@@ -1711,6 +1718,14 @@ def test_a_scheduled_run_while_away_switches_back_to_the_earlier_model(monkeypat
assert world.loaded == SwapWorld.EARLIER
+def test_a_scheduled_run_restores_the_earlier_model_with_its_launch_settings(monkeypatch):
+ world = SwapWorld(monkeypatch)
+ world.launch = {"kwargs": {"max_seq_length": 8192, "cache_type_kv": "q8_0"}}
+ world.run(monkeypatch, {"id": "run-1", "source": "schedule"})
+ assert world.stashed == [SwapWorld.EARLIER]
+ assert world.launches == [world.launch]
+
+
def test_a_scheduled_run_stays_off_a_model_the_user_loaded_while_it_waited(monkeypatch):
# The user's load holds the lifecycle gate, so admission waits it out; the load landing stamps attendance.
world = SwapWorld(monkeypatch)
diff --git a/studio/backend/utils/openai_auto_switch_settings.py b/studio/backend/utils/openai_auto_switch_settings.py
index 77ec325e8ab..b218c464087 100644
--- a/studio/backend/utils/openai_auto_switch_settings.py
+++ b/studio/backend/utils/openai_auto_switch_settings.py
@@ -742,34 +742,36 @@ def model_override_load_kwargs(override: dict[str, Any], *, is_gguf: bool) -> di
if kwargs.get("llama_extra_args"):
# One entry can hold a pass-through flag AND the field it shadows, and llama.cpp's last-wins parse would hand the load the stale flag, so the /load stripper (_resolve_inherited_extra_args) is imported, not mirrored. The settings page has no control for flags, so a save carries the stored ones over (routes/settings.py); the allow-list this module stays out of is validate_extra_args.
- from core.inference.llama_server_args import (
- matches_explicit_ctx_override,
- strip_shadowing_flags,
- )
+ kwargs["llama_extra_args"] = strip_args_shadowed_by(kwargs["llama_extra_args"], kwargs)
+ return kwargs
- # Context's load-time value is a VRAM-fit target. A MATCHING -c/--ctx-size is the user's opt-in to exceed the safe threshold and survives, while stale and malformed flags are still stripped; /props then publishes what was really allocated.
- matching_explicit_ctx = matches_explicit_ctx_override(
- kwargs["llama_extra_args"], kwargs.get("max_seq_length")
- )
- kwargs["llama_extra_args"] = strip_shadowing_flags(
- kwargs["llama_extra_args"],
- # Only the groups this override actually supplies, as the route gates on its request's set fields: a flag with no first-class field behind it is the user's only way to set that knob and still passes through.
- strip_context = "max_seq_length" in kwargs and not matching_explicit_ctx,
- strip_cache = "cache_type_kv" in kwargs,
- strip_spec = "speculative_type" in kwargs or "spec_draft_n_max" in kwargs,
- strip_template = "chat_template_override" in kwargs,
- strip_reasoning_budget = "reasoning_budget" in kwargs,
- strip_reasoning_budget_message = "reasoning_budget_message" in kwargs,
- # Sent only when on, so it is always the Tensor Parallelism toggle overriding the flag; an override that leaves the toggle off keeps a row/none/layer split mode.
- strip_split_mode = bool(kwargs.get("tensor_parallel")),
- strip_batch = "n_batch" in kwargs,
- strip_ubatch = "n_ubatch" in kwargs,
- strip_ctx_checkpoints = "ctx_checkpoints" in kwargs,
- strip_cache_ram = "cache_ram" in kwargs,
- strip_spec_draft_cache = "spec_draft_cache_type" in kwargs,
- )
- return kwargs
+def strip_args_shadowed_by(args, fields: dict[str, Any], **extra_strips) -> list[str]:
+ from core.inference.llama_server_args import (
+ matches_explicit_ctx_override,
+ strip_shadowing_flags,
+ )
+
+ # Context's load-time value is a VRAM-fit target. A MATCHING -c/--ctx-size is the user's opt-in to exceed the safe threshold and survives, while stale and malformed flags are still stripped; /props then publishes what was really allocated.
+ matching_explicit_ctx = matches_explicit_ctx_override(args, fields.get("max_seq_length"))
+ return strip_shadowing_flags(
+ args,
+ # Only the groups these fields actually supply, as the route gates on its request's set fields: a flag with no first-class field behind it is the user's only way to set that knob and still passes through.
+ strip_context = "max_seq_length" in fields and not matching_explicit_ctx,
+ strip_cache = "cache_type_kv" in fields,
+ strip_spec = "speculative_type" in fields or "spec_draft_n_max" in fields,
+ strip_template = "chat_template_override" in fields,
+ strip_reasoning_budget = "reasoning_budget" in fields,
+ strip_reasoning_budget_message = "reasoning_budget_message" in fields,
+ # Sent only when on, so it is always the Tensor Parallelism toggle overriding the flag; an override that leaves the toggle off keeps a row/none/layer split mode.
+ strip_split_mode = bool(fields.get("tensor_parallel")),
+ strip_batch = "n_batch" in fields,
+ strip_ubatch = "n_ubatch" in fields,
+ strip_ctx_checkpoints = "ctx_checkpoints" in fields,
+ strip_cache_ram = "cache_ram" in fields,
+ strip_spec_draft_cache = "spec_draft_cache_type" in fields,
+ **extra_strips,
+ )
def _looks_like_filesystem_path(model_id: str) -> bool: