Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
Preserve runtime settings when unsloth start names the loaded model (#15026)
Nilayc3280b2e0f82
diff --git a/unsloth_cli/commands/start.py b/unsloth_cli/commands/start.pyindex 96963ee92f7..99845852857 100644--- a/unsloth_cli/commands/start.py+++ b/unsloth_cli/commands/start.py@@ -2268,6 +2268,7 @@ def _resolve_model(status_snapshot = None# Whether the inferred settings can restart the resident. Computed once: the preload gate, the warning, the consent refusal and force_reload must all agree, and asking twice against a snapshot taken at different times is how they drift apart.inferred_differs = False+ same_checkpoint = Trueif requested is None and load_has_overrides and infer_resident:status_snapshot = _inference_status(base, key)requested, attach_public_id = _resident_load_target(models, status_snapshot, allow_casefold)@@ -2285,6 +2286,25 @@ def _resolve_model(match = None if requested and load_has_overrides else loaded_matchif requested and match is None:load_requested = True+ if attach_public_id is None and loaded_match is not None:+ # A full load resets omitted settings, so read the named resident's own slot to carry them.+ status_snapshot = _inference_status(base, key, loaded_match["id"])+ # The listing shows a local checkpoint by basename, so ./foo matches a resident old/foo: inherit only from the checkpoint this load resolves to.+ wanted_path = os.path.abspath(os.path.expanduser(requested))+ # A snapshot naming another model (replaced since the listing, or a server ignoring ?model=) describes nothing of this one.+ same_checkpoint = _model_id_matches(+ status_snapshot.get("active_model"),+ loaded_match["id"],+ allow_casefold = allow_casefold,+ ) and (+ _is_hub_model_id(requested)+ or any(+ os.path.abspath(os.path.expanduser(str(status_snapshot.get(field))))+ == wanted_path+ for field in ("model_identifier", "gguf_path", "model_path")+ if status_snapshot.get(field)+ )+ )# Only here is an evicting load certain: the gate must not reject a request the resident model already satisfies (a path-loaded GGUF shown as a bare basename can collide with a non-GGUF unsloth/<name>).# A load of a model already served replaces it in its own slot, which may not be the primary.active = loaded_match or next((m for m in models if m.get("loaded") is not False), None)@@ -2434,12 +2454,8 @@ def _resolve_model(payload["gguf_variant"] = resident_variantelif loaded_match is not None and not str(requested).lower().endswith(".gguf"):# Same for a named model already served, read from the slot serving it.- try:- status = _model_status(base, key, loaded_match["id"])- except Exception:- status = {}- if status.get("is_gguf") and status.get("gguf_variant"):- payload["gguf_variant"] = status["gguf_variant"]+ if status_snapshot.get("is_gguf") and status_snapshot.get("gguf_variant"):+ payload["gguf_variant"] = status_snapshot["gguf_variant"]if "max_seq_length" in overrides:payload["max_seq_length"] = load.max_seq_lengthif "load_in_4bit" in overrides:@@ -2450,7 +2466,9 @@ def _resolve_model(payload["gpu_memory_mode"] = load.gpu_memory_mode# -1 means "pick the layers", which is right when the user is switching INTO manual, but on an inferred attach to a resident already in manual it would throw away the layer count it was pinned to. Leave it for the round-trip.already_manual = (- attach_public_id is not None and status_snapshot.get("gpu_memory_mode") == "manual"+ same_checkpoint+ and bool(status_snapshot)+ and status_snapshot.get("gpu_memory_mode") == "manual")if load.gpu_memory_mode == "manual" and not already_manual:payload["gpu_layers"] = -1@@ -2464,11 +2482,22 @@ def _resolve_model(f"'{attach_public_id}' was loaded with trust_remote_code, which an attach ""cannot re-authorize. Re-run with --model naming it to apply these settings.")- if attach_public_id is not None:- # An inferred reload is a full load, not a PATCH: _gguf_request_intent copies every defaulted LoadRequest field into the new intent, so a knob we leave out is reset rather than kept. Carry the resident's own values for the ones the user did not name, or changing the context alone would drop their KV dtype, slot count, batch sizes and GPU placement.- payload.update(_resident_runtime_payload(status_snapshot, payload))+ if attach_public_id is not None or loaded_match is not None:+ # A resident reload is a full load, not a PATCH: carry the knobs the user did not name, or changing the context alone drops their KV dtype, slot count, batch sizes and GPU placement.+ carried = _resident_runtime_payload(status_snapshot, payload)+ if attach_public_id is None:+ # Omitted extras are inherited server-side, shadow-stripped against the fields sent: a copied list would beat --context-length, and a carried field would strip an extra that won. So extras mean no carry.+ if not same_checkpoint or status_snapshot.get("requested_llama_extra_args"):+ carried = {}+ carried.pop("llama_extra_args", None)+ # Outside manual, status reports the planner's emitted ratio, not the request (LlamaCppBackend.tensor_split); sending it pins an Auto split and reloads.+ if {**payload, **carried}.get("gpu_memory_mode") != "manual":+ carried.pop("tensor_split", None)+ # Status cannot say whether the slot count was the default, and a sent count stops Auto cutting slots to keep a drafter (n_parallel_from_default).+ carried.pop("n_parallel", None)+ payload.update(carried)# The server cannot tell an explicit `--context-length 0` reset from the 0 that every UI load sends, so it treats 0 as "no preference" and would answer already_loaded. Say outright that this one is a reload, but only when status PROVED a difference: on an older server _load_settings_differ cannot tell, and forcing there would evict on every attach.- if status_snapshot and inferred_differs:+ if attach_public_id is not None and status_snapshot and inferred_differs:payload["force_reload"] = Truetry:loaded = _load_model_with_progress(base, key, requested, load, payload)diff --git a/unsloth_cli/tests/test_start_named_runtime.py b/unsloth_cli/tests/test_start_named_runtime.pynew file mode 100644index 00000000000..00122ea0c99--- /dev/null+++ b/unsloth_cli/tests/test_start_named_runtime.py@@ -0,0 +1,203 @@+# SPDX-License-Identifier: AGPL-3.0-only+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0++"""A named resident reload changes only the load settings the caller supplied."""++import sys+from pathlib import Path++sys.path.insert(0, str(Path(__file__).resolve().parents[2]))++import pytest++import unsloth_cli.commands.start as start++MODEL = "owner/model-GGUF"+BASE = "http://127.0.0.1:8888"++[email protected]("kept", [False, True])[email protected]("manual", [False, True])+def test_named_resident_context_change_keeps_other_runtime_settings(monkeypatch, kept, manual):+ models = [{"id": MODEL, "loaded": True}]+ if kept:+ models.insert(0, {"id": "owner/primary-GGUF", "loaded": True})+ status = {+ "active_model": MODEL,+ "is_gguf": True,+ "gguf_variant": "Q8_0",+ "requested_context_length": 4096,+ "requested_n_batch": 128,+ "requested_n_ubatch": 64,+ "cache_type_kv": "q8_0",+ "gpu_memory_mode": "manual",+ "gpu_layers": 7,+ }+ payloads = []+ monkeypatch.setattr(start, "_loaded_models", lambda *a: models)++ def model_status(+ base,+ key,+ model_id = None,+ ):+ assert model_id == MODEL, "must read the requested slot, not the primary"+ return status++ monkeypatch.setattr(start, "_model_status", model_status)+ monkeypatch.setattr(+ start,+ "_load_model_with_progress",+ lambda base, key, model, load, payload: payloads.append(payload)+ or {"status": "loaded", "model": model},+ )+ options = start.LoadOptions(max_seq_length = 8192, gpu_memory_mode = "manual" if manual else None)+ start._resolve_model(BASE, "key", MODEL, options)+ assert payloads == [+ {+ "model_path": MODEL,+ "gguf_variant": "Q8_0",+ "max_seq_length": 8192,+ "n_batch": 128,+ "n_ubatch": 64,+ "cache_type_kv": "q8_0",+ "gpu_memory_mode": "manual",+ "gpu_layers": 7,+ }+ ]++[email protected]("requested", ["owner/new-GGUF", "/new/model.gguf"])+def test_new_model_never_inherits_previous_resident_settings(monkeypatch, requested):+ models = [{"id": MODEL, "loaded": True}]+ monkeypatch.setattr(start, "_loaded_models", lambda *a: models)+ monkeypatch.setattr(start, "_model_status", lambda *a: pytest.fail("unrelated resident status"))+ payloads = []+ monkeypatch.setattr(+ start,+ "_load_model_with_progress",+ lambda base, key, model, load, payload: payloads.append(payload)+ or {"status": "loaded", "model": MODEL},+ )+ start._resolve_model(BASE, "key", requested, start.LoadOptions(max_seq_length = 8192))+ assert payloads == [{"model_path": requested, "max_seq_length": 8192}]+++def test_named_resident_explicit_automatic_context_wins(monkeypatch):+ monkeypatch.setattr(start, "_loaded_models", lambda *a: [{"id": MODEL, "loaded": True}])+ monkeypatch.setattr(+ start,+ "_model_status",+ lambda *a: {+ "active_model": MODEL,+ "requested_context_length": 4096,+ "requested_n_batch": 128,+ },+ )+ payloads = []+ monkeypatch.setattr(+ start,+ "_load_model_with_progress",+ lambda base, key, model, load, payload: payloads.append(payload)+ or {"status": "loaded", "model": model},+ )+ start._resolve_model(+ BASE, "key", MODEL, start.LoadOptions(supplied = frozenset({"max_seq_length"}))+ )+ assert payloads == [{"model_path": MODEL, "max_seq_length": 0, "n_batch": 128}]+++def _named_reload_payload(monkeypatch, status, options):+ status = {"active_model": MODEL, **status}+ monkeypatch.setattr(start, "_loaded_models", lambda *a: [{"id": MODEL, "loaded": True}])+ monkeypatch.setattr(start, "_model_status", lambda *a: status)+ payloads = []+ monkeypatch.setattr(+ start,+ "_load_model_with_progress",+ lambda base, key, model, load, payload: payloads.append(payload)+ or {"status": "loaded", "model": model},+ )+ start._resolve_model(BASE, "key", MODEL, options)+ return payloads[0]+++ "options, sent",+ [+ (start.LoadOptions(max_seq_length = 32768), {"max_seq_length": 32768}),+ (+ start.LoadOptions(gpu_memory_mode = "manual"),+ {"gpu_memory_mode": "manual", "gpu_layers": -1},+ ),+ ],+)+def test_named_resident_with_extras_left_to_server_inheritance(monkeypatch, options, sent):+ status = {+ "is_gguf": True,+ "gguf_variant": "Q8_0",+ "requested_context_length": 0,+ "requested_llama_extra_args": ["--ctx-size", "8192", "--top-k", "20"],+ "requested_n_batch": 128,+ }+ payload = _named_reload_payload(monkeypatch, status, options)+ assert payload == {"model_path": MODEL, "gguf_variant": "Q8_0", **sent}++[email protected]("effective", [2, 4])+def test_named_resident_slot_count_stays_defaulted(monkeypatch, effective):+ status = {+ "is_gguf": True,+ "gguf_variant": "Q4_K_M",+ "requested_parallel_slots": 4,+ "parallel_slots": effective,+ }+ payload = _named_reload_payload(monkeypatch, status, start.LoadOptions(gguf_variant = "Q4_K_M"))+ assert "n_parallel" not in payload++[email protected]("mode, carried", [("auto", False), ("manual", True)])+def test_named_resident_auto_tensor_split_not_pinned(monkeypatch, mode, carried):+ status = {+ "is_gguf": True,+ "gguf_variant": "Q8_0",+ "gpu_memory_mode": mode,+ "tensor_parallel": True,+ "tensor_split": [0.6, 0.4],+ }+ payload = _named_reload_payload(+ monkeypatch, status, start.LoadOptions(gguf_variant = "Q8_0", tensor_parallel = True)+ )+ assert ("tensor_split" in payload) is carried++[email protected]("resident, inherits", [("/old/foo", False), (None, True)])+def test_named_local_checkpoint_inherits_only_from_itself(+ monkeypatch, tmp_path, resident, inherits+):+ monkeypatch.chdir(tmp_path)+ (tmp_path / "foo").mkdir()+ status = {+ "active_model": "foo",+ "model_identifier": resident or str(tmp_path / "foo"),+ "load_in_4bit": False,+ "requested_gpu_ids": [1],+ }+ monkeypatch.setattr(start, "_loaded_models", lambda *a: [{"id": "foo", "loaded": True}])+ monkeypatch.setattr(start, "_model_status", lambda *a: status)+ payloads = []+ monkeypatch.setattr(+ start,+ "_load_model_with_progress",+ lambda base, key, model, load, payload: payloads.append(payload)+ or {"status": "loaded", "model": model},+ )+ start._resolve_model(BASE, "key", "foo", start.LoadOptions(max_seq_length = 8192))+ carried = {"load_in_4bit": False, "gpu_ids": [1]} if inherits else {}+ assert payloads == [{"model_path": "foo", "max_seq_length": 8192, **carried}]+++def test_named_resident_status_for_another_model_is_not_inherited(monkeypatch):+ status = {"active_model": "owner/other-GGUF", "load_in_4bit": False, "requested_gpu_ids": [1]}+ payload = _named_reload_payload(monkeypatch, status, start.LoadOptions(max_seq_length = 8192))+ assert payload == {"model_path": MODEL, "max_seq_length": 8192}