Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
Apply the context length and quant settings when attaching to a running server (#10093)
Nilay9aeefd55e617
diff --git a/studio/backend/models/inference.py b/studio/backend/models/inference.pyindex 537c249a400..3cd90687a60 100644--- a/studio/backend/models/inference.py+++ b/studio/backend/models/inference.py@@ -1464,9 +1464,19 @@ class InferenceStatusResponse(_InferenceRuntimeFields):requested_context_length: Optional[int] = Field(None,description = (- "The n_ctx the active GGUF load was invoked with (0 = Auto). Lets the "- "UI re-seed a Manual + Auto-layers context pin on hydration, where "- "context_length only exposes the resolved value. None for non-GGUF."+ "The max_seq_length the active load was invoked with (0 = Auto). Lets the "+ "UI re-seed a Manual + Auto-layers context pin on hydration, and lets a CLI "+ "attach tell a no-op from a reload, where context_length only exposes the "+ "resolved value. None when the running model predates this field."+ ),+ )+ load_in_4bit: Optional[bool] = Field(+ None,+ description = (+ "The 4-bit setting the active non-GGUF load was REQUESTED with, not the "+ "value it resolved to: LoRA and the latest-transformers tier rewrite it, so "+ "only the requested value can be compared against a later request. None for "+ "GGUF, which does not use it."),)llama_cpp_supports_mtp: bool = Field(diff --git a/studio/backend/routes/inference.py b/studio/backend/routes/inference.pyindex d6829143554..4d97ed006f2 100644--- a/studio/backend/routes/inference.py+++ b/studio/backend/routes/inference.py@@ -12862,6 +12862,31 @@ def _mlx_runtime_settings_match(backend, request) -> bool:) == (request.chat_template_override or None)+def _non_gguf_runtime_settings_match(backend, request) -> bool:+ """Whether the resident non-GGUF model already runs the request's load settings.++ An unrecorded resident value counts as a MATCH: every UI call site ships+ max_seq_length and load_in_4bit on every load, so otherwise every pick would reload.+ max_seq_length 0 means "model default", so a `--context-length 0` reset is honoured+ on GGUF but not here.+ """+ if getattr(request, "force_reload", False):+ return False+ fields_set = getattr(request, "model_fields_set", set()) or set()+ entry = backend.models.get(backend.active_model_name, {}) or {}+ if "max_seq_length" in fields_set and int(request.max_seq_length or 0) > 0:+ resident = entry.get("max_seq_length_requested")+ if resident is not None and int(resident) != int(request.max_seq_length):+ return False+ if "load_in_4bit" in fields_set:+ # As REQUESTED, not as resolved: _effective_load_in_4bit rewrites it for LoRA and+ # the latest-transformers tier, so raw-vs-resolved would never match.+ resident = entry.get("load_in_4bit_requested")+ if resident is not None and bool(resident) != bool(request.load_in_4bit):+ return False+ return True++class _ScopedLoadAttempt(NamedTuple):token: strrequest_id: Optional[str]@@ -13303,9 +13328,11 @@ async def _load_model_impl(await asyncio.to_thread(acquire_for, CHAT)return reusedif not (request.gguf_variant or is_direct_gguf_request):- if _same_loaded_identifier(- backend.active_model_name, model_identifier- ) and _mlx_runtime_settings_match(backend, request):+ if (+ _same_loaded_identifier(backend.active_model_name, model_identifier)+ and _mlx_runtime_settings_match(backend, request)+ and _non_gguf_runtime_settings_match(backend, request)+ ):api_monitor.discard(_load_event) # nothing loaded, no monitor rowlogger.info(f"Model already loaded (Unsloth): {model_log_label}, skipping reload")# A no-op Unsloth load of a preview-owned checkpoint still claims it.@@ -14011,6 +14038,20 @@ async def _load_model_impl(),)+ # Stamped here, not in backend.load_model: that entry is built in the load+ # subprocess and only a fixed model_info mirror crosses back, so it would+ # never be read.+ _resident_entry = backend.models.get(backend.active_model_name)+ if isinstance(_resident_entry, dict):+ _resident_entry["max_seq_length_requested"] = int(request.max_seq_length or 0)+ _resident_entry["load_in_4bit_requested"] = bool(request.load_in_4bit)+ # Placement too, or a reload for some other knob would fall back to automatic+ # selection and could land the model on a different GPU. The parent-side+ # orchestrator entry does not keep it, so status has nothing else to report.+ _resident_entry["gpu_ids_requested"] = (+ list(request.gpu_ids) if request.gpu_ids else None+ )+logger.info(f"Loaded model: {model_log_label if native_grant_backed else config.identifier}")@@ -16188,6 +16229,9 @@ async def get_status(current_subject: str = Depends(get_current_subject)):preserve_thinking_default = _sf_flags.get("preserve_thinking_default", False),supports_tools = _sf_flags["supports_tools"],context_length = _positive_int_or_none(model_info.get("context_length")),+ requested_context_length = model_info.get("max_seq_length_requested"),+ load_in_4bit = model_info.get("load_in_4bit_requested"),+ requested_gpu_ids = model_info.get("gpu_ids_requested"),chat_template = chat_template,llama_cpp_supports_mtp = _supports_mtp,llama_cpp_prebuilt_stale = _stale,diff --git a/studio/backend/tests/test_non_gguf_reload_settings.py b/studio/backend/tests/test_non_gguf_reload_settings.pynew file mode 100644index 00000000000..549684ae6de--- /dev/null+++ b/studio/backend/tests/test_non_gguf_reload_settings.py@@ -0,0 +1,150 @@+# SPDX-License-Identifier: AGPL-3.0-only+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0++"""Reload gating and status reporting for a resident non-GGUF model."""++from __future__ import annotations++import logging+import sys+import types as _types+from pathlib import Path++import pytest++_BACKEND = Path(__file__).resolve().parents[1]+if str(_BACKEND) not in sys.path:+ sys.path.insert(0, str(_BACKEND))++# Stub the optional deps routes/__init__ pulls in, so this module imports standalone.+_loggers_stub = _types.ModuleType("loggers")+_loggers_stub.get_logger = lambda name: logging.getLogger(name)+sys.modules.setdefault("loggers", _loggers_stub)++_structlog_stub = _types.ModuleType("structlog")+_structlog_stub.get_logger = lambda *_a, **_k: logging.getLogger("structlog_stub")+sys.modules.setdefault("structlog", _structlog_stub)+if not hasattr(sys.modules["structlog"], "get_logger"):+ sys.modules["structlog"].get_logger = _structlog_stub.get_logger++import routes.inference as inference_route # noqa: E402+++RESIDENT = "unsloth/Qwen3-8B"+++class _Backend:+ def __init__(self, entry):+ self.active_model_name = RESIDENT+ self.models = {RESIDENT: entry}+++class _Request:+ """model_fields_set is what pydantic records."""+ def __init__(self, **fields):+ self.model_fields_set = set(fields)+ self.force_reload = fields.pop("force_reload", False)+ self.max_seq_length = fields.pop("max_seq_length", 0)+ self.load_in_4bit = fields.pop("load_in_4bit", True)+ self.tensor_parallel = fields.pop("tensor_parallel", False)+ self.gpu_memory_mode = fields.pop("gpu_memory_mode", None)+++def _loaded(max_seq_length = 4096, load_in_4bit = True):+ return _Backend(+ {+ "max_seq_length_requested": max_seq_length,+ "load_in_4bit_requested": load_in_4bit,+ }+ )+++def test_matching_explicit_settings_are_reused():+ backend = _loaded(max_seq_length = 4096, load_in_4bit = True)+ request = _Request(max_seq_length = 4096, load_in_4bit = True)+ assert inference_route._non_gguf_runtime_settings_match(backend, request)+++def test_changed_context_forces_a_reload():+ backend = _loaded(max_seq_length = 4096)+ request = _Request(max_seq_length = 32768)+ assert not inference_route._non_gguf_runtime_settings_match(backend, request)+++def test_changed_precision_forces_a_reload():+ backend = _loaded(load_in_4bit = True)+ request = _Request(load_in_4bit = False)+ assert not inference_route._non_gguf_runtime_settings_match(backend, request)+++def test_omitted_settings_keep_the_legacy_reuse():+ """A caller that sends only model_path still reuses."""+ backend = _loaded(max_seq_length = 4096, load_in_4bit = True)+ assert inference_route._non_gguf_runtime_settings_match(backend, _Request())+++def test_zero_context_expresses_no_preference():+ """max_seq_length 0 never forces a reload."""+ assert inference_route._non_gguf_runtime_settings_match(+ _loaded(max_seq_length = 2048), _Request(max_seq_length = 0)+ )+++def test_unrecorded_resident_settings_are_reused_not_reloaded():+ """An unrecorded resident value is not a mismatch."""+ backend = _Backend({})+ assert inference_route._non_gguf_runtime_settings_match(+ backend, _Request(max_seq_length = 32768, load_in_4bit = False)+ )+++def test_force_reload_is_honored():+ """force_reload defeats the match."""+ backend = _loaded(max_seq_length = 4096)+ request = _Request(force_reload = True, max_seq_length = 4096)+ assert not inference_route._non_gguf_runtime_settings_match(backend, request)+++ "field, value",+ [("tensor_parallel", True), ("gpu_memory_mode", "auto"), ("gpu_memory_mode", "manual")],+)+def test_gguf_only_knobs_never_block_reuse(field, value):+ """The chat UI sends gpu_memory_mode ungated and keeps tensor_parallel across a+ model switch, so neither carries user intent for a transformers load."""+ assert inference_route._non_gguf_runtime_settings_match(_loaded(), _Request(**{field: value}))+++class TestNonGgufStatusReportsWhatTheLoadAskedFor:+ """Placement is not kept on the parent-side orchestrator entry at all, so anything+ the route does not stamp is simply unavailable to a client."""++ STAMPED = ("max_seq_length_requested", "load_in_4bit_requested", "gpu_ids_requested")++ def _stamp_block(self):+ import inspect+ import routes.inference as ri++ src = inspect.getsource(ri._load_model_impl)+ start = src.index("_resident_entry = backend.models.get")+ return src[start : start + 900]++ @pytest.mark.parametrize("field", STAMPED)+ def test_the_route_stamps_it_after_a_successful_load(self, field):+ assert field in self._stamp_block(), f"{field} is never recorded on the resident"++ def test_the_non_gguf_status_branch_publishes_them(self):+ import inspect+ import routes.inference as ri++ src = inspect.getsource(ri.get_status)+ # The GGUF branch returns first, so the last occurrence is the non-GGUF return.+ non_gguf = src[src.rindex("Non-GGUF: classify from the loaded template") :]+ for wire, stamped in (+ ("requested_context_length", "max_seq_length_requested"),+ ("load_in_4bit", "load_in_4bit_requested"),+ ("requested_gpu_ids", "gpu_ids_requested"),+ ):+ assert (+ f'{wire} = model_info.get("{stamped}")' in non_gguf+ ), f"non-GGUF status does not publish {wire}"diff --git a/studio/frontend/src/features/chat/lib/resident-config-match.ts b/studio/frontend/src/features/chat/lib/resident-config-match.tsindex b393b6ef163..51f4eba2592 100644--- a/studio/frontend/src/features/chat/lib/resident-config-match.ts+++ b/studio/frontend/src/features/chat/lib/resident-config-match.ts@@ -286,8 +286,8 @@ const SETTING_CHECKS: SettingCheck[] = [// cross-model GGUF pick and as the resident context when re-picking the same one.// Reading null as "no opinion" against a status echoing either number was a reload.pinned: () => true,- // A GGUF invocation field: a safetensors or MLX status never sets it, and the general- // non-GGUF rule below is what keeps this from reading that absence as 0.+ // A safetensors or MLX status reports this too, so the general non-GGUF rule below+ // is what keeps this check off those models.agrees: (c, s, standing) =>standing.resolveContextLength(c.customContextLength ?? null) ===(s.requested_context_length ?? 0),diff --git a/studio/frontend/src/features/chat/lib/resolve-ctx-pin-seed.ts b/studio/frontend/src/features/chat/lib/resolve-ctx-pin-seed.tsindex 58c9b8f18ad..4eca65b9922 100644--- a/studio/frontend/src/features/chat/lib/resolve-ctx-pin-seed.ts+++ b/studio/frontend/src/features/chat/lib/resolve-ctx-pin-seed.ts@@ -47,7 +47,8 @@ const CLEAR: CtxPinSeed = {export function resolveCtxPinSeed(options: {/** ``status.requested_context_length``; undefined on a backend that omits the field. */incoming: number | null | undefined;- /** ``status.is_gguf``: only a GGUF load carries an n_ctx. */+ /** ``status.is_gguf``. A non-GGUF status reports ``incoming`` too, so this flag, not+ * its presence, is what keeps a non-GGUF load out of the pin. */isGguf: boolean;/** No load of this tab's own is in flight (``!modelLoading``). */seedLoadParams: boolean;diff --git a/tests/studio/test_cli_attach_load_knobs.py b/tests/studio/test_cli_attach_load_knobs.pynew file mode 100644index 00000000000..9182f7c7507--- /dev/null+++ b/tests/studio/test_cli_attach_load_knobs.py@@ -0,0 +1,1114 @@+# SPDX-License-Identifier: AGPL-3.0-only+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0+++from __future__ import annotations++import pytest+import typer++import unsloth_cli.commands.start as start_cli+++BASE = "http://127.0.0.1:8888"+KEY = "k"+RESIDENT = {"id": "unsloth/Qwen3-8B", "loaded": True}+++class FakeServer:+ """Serves /v1/models and /api/inference/status; records every load payload."""+ def __init__(self, models, status):+ self.models = models+ self.status = status+ self.loads = []+ self.requests = []++ def http_json(+ self,+ method,+ url,+ token,+ payload = None,+ timeout = 30,+ error = None,+ ):+ self.requests.append((method, url))+ if url.endswith("/v1/models"):+ return {"data": [dict(m) for m in self.models]}+ if url.endswith("/api/inference/status"):+ return dict(self.status)+ raise AssertionError(f"unexpected request: {method} {url}")++ def load(self, base, key, requested, load, payload):+ self.loads.append(payload)+ public = start_cli._public_model_id(requested) or requested+ if not any(m.get("id") == public for m in self.models):+ self.models.append({"id": public, "loaded": True})+ return {"status": "already_loaded", "model": requested}++ def install(self, monkeypatch):+ monkeypatch.setattr(start_cli, "_http_json", self.http_json)+ monkeypatch.setattr(start_cli, "_load_model_with_progress", self.load)+ return self+++def loads(monkeypatch):+ calls = []++ def _load(base, key, requested, load, payload):+ calls.append(payload)+ return {"status": "already_loaded", "model": requested}++ monkeypatch.setattr(start_cli, "_loaded_models", lambda base, key: [dict(RESIDENT)])+ monkeypatch.setattr(start_cli, "_load_model_with_progress", _load)+ monkeypatch.setattr(start_cli, "_http_json", lambda *a, **k: {})+ return calls+++def sent(server, index = 0):+ """The load payload without force_reload, which the tests that care assert alone."""+ return {k: v for k, v in server.loads[index].items() if k != "force_reload"}+++def test_context_length_without_model_reloads_the_resident_model(loads):+ entry = start_cli._resolve_model(+ BASE,+ KEY,+ None,+ start_cli.LoadOptions(max_seq_length = 32768),+ )+ assert loads == [{"model_path": RESIDENT["id"], "max_seq_length": 32768}]+ assert entry["id"] == RESIDENT["id"]+++def test_gguf_variant_without_model_is_forwarded(loads):+ start_cli._resolve_model(+ BASE,+ KEY,+ None,+ start_cli.LoadOptions(gguf_variant = "UD-Q8_K_XL", max_seq_length = 32768),+ )+ assert loads == [+ {+ "model_path": RESIDENT["id"],+ "gguf_variant": "UD-Q8_K_XL",+ "max_seq_length": 32768,+ }+ ]+++def test_bare_attach_does_not_load(loads):+ entry = start_cli._resolve_model(BASE, KEY, None, start_cli.LoadOptions())+ assert loads == []+ assert entry["id"] == RESIDENT["id"]+++def test_bare_attach_does_not_query_status(monkeypatch):+ """A bare attach must not even ask for status."""+ server = FakeServer(+ [dict(RESIDENT)],+ {"is_gguf": False, "active_model": RESIDENT["id"], "model_identifier": RESIDENT["id"]},+ ).install(monkeypatch)++ entry = start_cli._resolve_model(BASE, KEY, None, start_cli.LoadOptions())++ assert server.loads == []+ assert entry["id"] == RESIDENT["id"]+ assert not any(url.endswith("/api/inference/status") for _, url in server.requests)+++def test_path_loaded_resident_is_reloaded_by_its_real_path(monkeypatch):+ """The load carries the identifier from status, not the advertised basename."""+ path = "/srv/models/Foo-Q4_K_M.gguf"+ server = FakeServer(+ [{"id": "Foo-Q4_K_M", "loaded": True}],+ {+ "is_gguf": True,+ "active_model": "Foo-Q4_K_M",+ "model_identifier": path,+ "gguf_variant": "Q4_K_M",+ "requested_context_length": 4096,+ },+ ).install(monkeypatch)++ entry = start_cli._resolve_model(BASE, KEY, None, start_cli.LoadOptions(max_seq_length = 32768))++ assert sent(server) == {"model_path": path, "max_seq_length": 32768}+ assert entry["id"] == "Foo-Q4_K_M"+++def test_inferred_target_still_runs_the_preload_check(monkeypatch):+ """preload_check still runs on an inferred target."""+ server = FakeServer(+ [dict(RESIDENT)],+ {"is_gguf": False, "active_model": RESIDENT["id"], "model_identifier": RESIDENT["id"]},+ ).install(monkeypatch)++ def gate(+ base,+ key,+ model,+ variant = None,+ ):+ raise typer.Exit(code = 1)++ with pytest.raises(typer.Exit):+ start_cli._resolve_model(+ BASE,+ KEY,+ None,+ start_cli.LoadOptions(max_seq_length = 4096),+ preload_check = gate,+ )++ assert server.loads == []+++def test_inferred_reload_warns_that_it_unloads_for_every_session(monkeypatch, capsys):+ """A settings change is announced as an unload."""+ server = FakeServer(+ [dict(RESIDENT)],+ {+ "is_gguf": True,+ "active_model": RESIDENT["id"],+ "model_identifier": RESIDENT["id"],+ "gguf_variant": "Q4_K_M",+ "requested_context_length": 4096,+ },+ ).install(monkeypatch)++ start_cli._resolve_model(BASE, KEY, None, start_cli.LoadOptions(max_seq_length = 32768))++ assert len(server.loads) == 1+ assert "unloads the current model for every attached session" in capsys.readouterr().out+++def test_active_model_decides_the_resident_not_list_order(monkeypatch):+ """active_model, not catalog order, names the resident."""+ server = FakeServer(+ [+ {"id": "unsloth/whisper-large", "loaded": True},+ {"id": RESIDENT["id"], "loaded": True},+ ],+ {"is_gguf": False, "active_model": RESIDENT["id"], "model_identifier": RESIDENT["id"]},+ ).install(monkeypatch)++ start_cli._resolve_model(BASE, KEY, None, start_cli.LoadOptions(max_seq_length = 32768))++ assert sent(server) == {"model_path": RESIDENT["id"], "max_seq_length": 32768}+++def test_unreloadable_resident_fails_before_loading(monkeypatch):+ """A redacted model_identifier is refused, not guessed from the basename."""+ server = FakeServer(+ [{"id": "Foo-Q4_K_M", "loaded": True}],+ {"is_gguf": True, "active_model": "Foo-Q4_K_M", "model_identifier": None},+ ).install(monkeypatch)++ with pytest.raises(typer.Exit):+ start_cli._resolve_model(BASE, KEY, None, start_cli.LoadOptions(max_seq_length = 32768))++ assert server.loads == []+++def test_explicit_flags_matching_defaults_still_reload(monkeypatch):+ """A flag typed at its default value is still sent."""+ server = FakeServer(+ [dict(RESIDENT)],+ {+ "is_gguf": True,+ "active_model": RESIDENT["id"],+ "model_identifier": RESIDENT["id"],+ "requested_context_length": 32768,+ "tensor_parallel": True,+ },+ ).install(monkeypatch)++ start_cli._resolve_model(+ BASE,+ KEY,+ None,+ start_cli.LoadOptions(+ max_seq_length = 0,+ tensor_parallel = False,+ supplied = frozenset({"max_seq_length", "tensor_parallel"}),+ ),+ )++ assert sent(server) == {+ "model_path": RESIDENT["id"],+ "max_seq_length": 0,+ "tensor_parallel": False,+ }+++def test_inferred_attach_pins_the_resident_quant(monkeypatch):+ """A repo-id GGUF is re-sent with the quant it is running."""+ server = FakeServer(+ [{"id": "unsloth/Qwen3-30B-A3B-GGUF", "loaded": True}],+ {+ "is_gguf": True,+ "active_model": "unsloth/Qwen3-30B-A3B-GGUF",+ "model_identifier": "unsloth/Qwen3-30B-A3B-GGUF",+ "gguf_variant": "Q8_0",+ "requested_context_length": 0,+ },+ ).install(monkeypatch)++ start_cli._resolve_model(+ BASE,+ KEY,+ None,+ start_cli.LoadOptions(max_seq_length = 8192, supplied = frozenset({"max_seq_length"})),+ )++ assert sent(server) == {+ "model_path": "unsloth/Qwen3-30B-A3B-GGUF",+ "gguf_variant": "Q8_0",+ "max_seq_length": 8192,+ }+++def test_inferred_attach_at_the_default_context_does_not_reresolve_the_quant(monkeypatch):+ """The --context-length 0 reset still names the running quant."""+ server = FakeServer(+ [{"id": "unsloth/Qwen3-30B-A3B-GGUF", "loaded": True}],+ {+ "is_gguf": True,+ "active_model": "unsloth/Qwen3-30B-A3B-GGUF",+ "model_identifier": "unsloth/Qwen3-30B-A3B-GGUF",+ "gguf_variant": "Q8_0",+ "requested_context_length": 0,+ },+ ).install(monkeypatch)++ start_cli._resolve_model(+ BASE,+ KEY,+ None,+ start_cli.LoadOptions(max_seq_length = 0, supplied = frozenset({"max_seq_length"})),+ )++ assert sent(server) == {+ "model_path": "unsloth/Qwen3-30B-A3B-GGUF",+ "gguf_variant": "Q8_0",+ "max_seq_length": 0,+ }+++def test_inferred_attach_does_not_pin_a_variant_onto_a_direct_gguf_file(monkeypatch):+ """A direct .gguf path is sent with no variant."""+ path = "/srv/models/Foo-Q4_K_M.gguf"+ server = FakeServer(+ [{"id": "Foo-Q4_K_M", "loaded": True}],+ {+ "is_gguf": True,+ "active_model": "Foo-Q4_K_M",+ "model_identifier": path,+ "gguf_variant": "Q4_K_M",+ "requested_context_length": 4096,+ },+ ).install(monkeypatch)++ start_cli._resolve_model(BASE, KEY, None, start_cli.LoadOptions(max_seq_length = 32768))++ assert sent(server) == {"model_path": path, "max_seq_length": 32768}+++def test_inferred_attach_to_a_non_gguf_resident_sends_no_variant(monkeypatch):+ """A non-GGUF resident is sent no variant."""+ server = FakeServer(+ [dict(RESIDENT)],+ {+ "is_gguf": False,+ "active_model": RESIDENT["id"],+ "model_identifier": RESIDENT["id"],+ "requested_context_length": 4096,+ },+ ).install(monkeypatch)++ start_cli._resolve_model(BASE, KEY, None, start_cli.LoadOptions(max_seq_length = 32768))++ assert sent(server) == {"model_path": RESIDENT["id"], "max_seq_length": 32768}+++def test_hf_cache_resident_matches_the_advertised_repo_id(monkeypatch):+ """A cache-path resident resolves to the repo id the server advertises."""+ cache_path = (+ "/home/u/.cache/huggingface/hub/models--unsloth--Qwen3-8B-GGUF"+ "/snapshots/abc123/qwen3-8b-Q4_K_M.gguf"+ )+ server = FakeServer(+ [{"id": "unsloth/Qwen3-8B-GGUF", "loaded": True}],+ {+ "is_gguf": True,+ "active_model": "unsloth/Qwen3-8B-GGUF",+ "model_identifier": cache_path,+ "requested_context_length": 4096,+ },+ ).install(monkeypatch)++ entry = start_cli._resolve_model(BASE, KEY, None, start_cli.LoadOptions(max_seq_length = 32768))++ assert sent(server) == {"model_path": cache_path, "max_seq_length": 32768}+ assert entry["id"] == "unsloth/Qwen3-8B-GGUF"+++def test_status_names_the_resident_even_when_the_catalog_lags(monkeypatch):+ """A status id absent from /v1/models still names the resident."""+ server = FakeServer(+ [{"id": "unsloth/whisper-large", "loaded": True}],+ {+ "is_gguf": False,+ "active_model": RESIDENT["id"],+ "model_identifier": RESIDENT["id"],+ },+ ).install(monkeypatch)++ entry = start_cli._resolve_model(BASE, KEY, None, start_cli.LoadOptions(max_seq_length = 32768))++ assert sent(server) == {"model_path": RESIDENT["id"], "max_seq_length": 32768}+ assert entry["id"] == RESIDENT["id"]+++def test_a_freshly_started_server_is_not_reloaded(monkeypatch):+ """An auto-started server passes infer_resident False."""+ server = FakeServer(+ [dict(RESIDENT)],+ {"is_gguf": False, "active_model": RESIDENT["id"], "model_identifier": RESIDENT["id"]},+ ).install(monkeypatch)++ entry = start_cli._resolve_model(+ BASE,+ KEY,+ None,+ start_cli.LoadOptions(max_seq_length = 32768),+ infer_resident = False,+ )++ assert server.loads == []+ assert entry["id"] == RESIDENT["id"]+++def test_omitted_default_flags_are_not_forwarded(monkeypatch):+ """A bare --model load sends only model_path."""+ server = FakeServer(+ [dict(RESIDENT)],+ {"is_gguf": False, "active_model": RESIDENT["id"], "model_identifier": RESIDENT["id"]},+ ).install(monkeypatch)++ start_cli._resolve_model(BASE, KEY, "unsloth/Qwen3-14B", start_cli.LoadOptions())++ assert sent(server) == {"model_path": "unsloth/Qwen3-14B"}+++class TestExplicitFlagsThroughTheRealCli:+ """`supplied` tracking through the real Typer and Click stack."""+ @staticmethod+ def _load_for(argv):+ from typer.testing import CliRunner++ captured = {}+ real_connect = start_cli._connect++ def fake_connect(api_key, model, load, *a, **k):+ captured["load"] = load+ raise SystemExit(0)++ start_cli._connect = fake_connect+ try:+ CliRunner().invoke(start_cli.start_app, argv)+ finally:+ start_cli._connect = real_connect+ return captured.get("load")++ @pytest.mark.parametrize(+ "flag, expected",+ [+ (["--context-length", "0"], "max_seq_length"),+ (["--load-in-4bit"], "load_in_4bit"),+ (["--no-tensor-parallel"], "tensor_parallel"),+ (["--gpu-memory-mode", "auto"], "gpu_memory_mode"),+ ],+ )+ def test_flags_equal_to_their_default_are_recorded(self, flag, expected):+ load = self._load_for(["codex", "--no-launch", *flag])+ assert load is not None, "the command never reached _connect"+ assert expected in load.supplied+ assert expected in load.overrides()++ def test_a_bare_invocation_records_nothing(self):+ load = self._load_for(["codex", "--no-launch"])+ assert load is not None+ assert load.supplied == frozenset()+ assert load.overrides() == frozenset()++ @pytest.mark.parametrize("command", ["codex", "claude", "opencode", "hermes", "pi", "openclaw"])+ def test_every_agent_command_tracks_flags_identically(self, command):+ load = self._load_for([command, "--no-launch", "--context-length", "0"])+ assert load is not None, f"{command} never reached _connect"+ assert "max_seq_length" in load.supplied+++def test_inferred_reload_carries_the_resident_runtime_settings(monkeypatch):+ """Knobs the user did not name are carried over from the resident."""+ server = FakeServer(+ [dict(RESIDENT)],+ {+ "is_gguf": True,+ "active_model": RESIDENT["id"],+ "model_identifier": RESIDENT["id"],+ "gguf_variant": "Q8_0",+ "requested_context_length": 4096,+ "cache_type_kv": "q8_0",+ "requested_parallel_slots": 4,+ "requested_n_batch": 1024,+ "requested_llama_extra_args": ["--foo"],+ "tensor_split": [0.5, 0.5],+ "requested_load_mode": None,+ },+ ).install(monkeypatch)++ start_cli._resolve_model(BASE, KEY, None, start_cli.LoadOptions(max_seq_length = 32768))++ sent = server.loads[0]+ assert sent["cache_type_kv"] == "q8_0"+ assert sent["n_parallel"] == 4+ assert sent["n_batch"] == 1024+ assert sent["llama_extra_args"] == ["--foo"]+ assert sent["tensor_split"] == [0.5, 0.5]+ assert "load_mode" not in sent+++def test_user_supplied_knobs_beat_the_resident_values(monkeypatch):+ server = FakeServer(+ [dict(RESIDENT)],+ {+ "is_gguf": True,+ "active_model": RESIDENT["id"],+ "model_identifier": RESIDENT["id"],+ "requested_context_length": 4096,+ "tensor_parallel": True,+ },+ ).install(monkeypatch)++ start_cli._resolve_model(+ BASE,+ KEY,+ None,+ start_cli.LoadOptions(tensor_parallel = False, supplied = frozenset({"tensor_parallel"})),+ )++ assert server.loads[0]["tensor_parallel"] is False+++def test_explicit_zero_context_forces_the_reload(monkeypatch):+ """An explicit 0 is force_reloaded."""+ server = FakeServer(+ [dict(RESIDENT)],+ {+ "is_gguf": False,+ "active_model": RESIDENT["id"],+ "model_identifier": RESIDENT["id"],+ "requested_context_length": 32768,+ },+ ).install(monkeypatch)++ start_cli._resolve_model(+ BASE,+ KEY,+ None,+ start_cli.LoadOptions(max_seq_length = 0, supplied = frozenset({"max_seq_length"})),+ )++ assert server.loads[0]["max_seq_length"] == 0+ assert server.loads[0]["force_reload"] is True+++def test_a_provable_no_op_is_not_forced(monkeypatch):+ server = FakeServer(+ [dict(RESIDENT)],+ {+ "is_gguf": False,+ "active_model": RESIDENT["id"],+ "model_identifier": RESIDENT["id"],+ "requested_context_length": 32768,+ },+ ).install(monkeypatch)++ start_cli._resolve_model(BASE, KEY, None, start_cli.LoadOptions(max_seq_length = 32768))++ assert "force_reload" not in server.loads[0]+++def test_an_older_server_is_never_force_reloaded(monkeypatch):+ """No status means no proof, so no force_reload."""+ server = FakeServer([dict(RESIDENT)], {}).install(monkeypatch)++ start_cli._resolve_model(BASE, KEY, None, start_cli.LoadOptions(max_seq_length = 32768))++ assert "force_reload" not in server.loads[0]+++def test_four_bit_flag_does_not_warn_about_a_gguf_resident(monkeypatch, capsys):+ """A null load_in_4bit on GGUF is not a difference."""+ server = FakeServer(+ [dict(RESIDENT)],+ {+ "is_gguf": True,+ "active_model": RESIDENT["id"],+ "model_identifier": RESIDENT["id"],+ "gguf_variant": "Q8_0",+ "requested_context_length": 4096,+ "load_in_4bit": None,+ },+ ).install(monkeypatch)++ start_cli._resolve_model(+ BASE,+ KEY,+ None,+ start_cli.LoadOptions(load_in_4bit = True, supplied = frozenset({"load_in_4bit"})),+ )++ assert "unloads the current model" not in capsys.readouterr().out+++def test_status_reporting_no_chat_resident_does_not_pick_a_speech_sidecar(monkeypatch):+ """A null active_model is refused, not resolved from the catalog."""+ server = FakeServer(+ [{"id": "unsloth/whisper-large-v3", "loaded": True}],+ {"is_gguf": False, "active_model": None, "model_identifier": None},+ ).install(monkeypatch)++ with pytest.raises(typer.Exit):+ start_cli._resolve_model(BASE, KEY, None, start_cli.LoadOptions(max_seq_length = 32768))++ assert server.loads == []+++def test_failed_inferred_load_names_the_inferred_resident(monkeypatch, capsys):+ """The survivor probe uses the inferred target."""+ server = FakeServer(+ [+ {"id": "unsloth/whisper-large-v3", "loaded": True},+ {"id": RESIDENT["id"], "loaded": True},+ ],+ {+ "is_gguf": True,+ "active_model": RESIDENT["id"],+ "model_identifier": RESIDENT["id"],+ "gguf_variant": "Q8_0",+ "requested_context_length": 4096,+ },+ ).install(monkeypatch)++ def boom(*a, **k):+ raise RuntimeError("load refused")++ monkeypatch.setattr(start_cli, "_load_model_with_progress", boom)+ monkeypatch.setattr(start_cli, "_model_still_loaded", lambda *a: False)++ with pytest.raises(RuntimeError):+ start_cli._resolve_model(BASE, KEY, None, start_cli.LoadOptions(max_seq_length = 32768))++ assert "Nothing was unloaded" not in capsys.readouterr().err+++def test_inferred_reload_keeps_a_full_precision_resident(monkeypatch):+ """A full-precision resident comes back at the same precision."""+ server = FakeServer(+ [dict(RESIDENT)],+ {+ "is_gguf": False,+ "active_model": RESIDENT["id"],+ "model_identifier": RESIDENT["id"],+ "requested_context_length": 4096,+ "load_in_4bit": False,+ },+ ).install(monkeypatch)++ start_cli._resolve_model(BASE, KEY, None, start_cli.LoadOptions(max_seq_length = 32768))++ assert server.loads[0]["load_in_4bit"] is False+++def test_an_explicit_precision_flag_beats_the_resident(monkeypatch):+ server = FakeServer(+ [dict(RESIDENT)],+ {+ "is_gguf": False,+ "active_model": RESIDENT["id"],+ "model_identifier": RESIDENT["id"],+ "requested_context_length": 4096,+ "load_in_4bit": False,+ },+ ).install(monkeypatch)++ start_cli._resolve_model(+ BASE,+ KEY,+ None,+ start_cli.LoadOptions(load_in_4bit = True, supplied = frozenset({"load_in_4bit"})),+ )++ assert server.loads[0]["load_in_4bit"] is True+++def test_a_gguf_resident_is_sent_no_precision_flag(monkeypatch):+ """A GGUF resident is sent no load_in_4bit."""+ server = FakeServer(+ [dict(RESIDENT)],+ {+ "is_gguf": True,+ "active_model": RESIDENT["id"],+ "model_identifier": RESIDENT["id"],+ "gguf_variant": "Q8_0",+ "requested_context_length": 4096,+ "load_in_4bit": None,+ },+ ).install(monkeypatch)++ start_cli._resolve_model(BASE, KEY, None, start_cli.LoadOptions(max_seq_length = 32768))++ assert "load_in_4bit" not in server.loads[0]+++def _gguf_status(**extra):+ base = {+ "is_gguf": True,+ "active_model": RESIDENT["id"],+ "model_identifier": RESIDENT["id"],+ "gguf_variant": "Q4_K_M",+ "requested_context_length": 8192,+ }+ base.update(extra)+ return base+++def test_changing_one_knob_keeps_the_custom_context(monkeypatch):+ """A custom context survives a change to another knob."""+ server = FakeServer([dict(RESIDENT)], _gguf_status()).install(monkeypatch)++ start_cli._resolve_model(+ BASE,+ KEY,+ None,+ start_cli.LoadOptions(tensor_parallel = True, supplied = frozenset({"tensor_parallel"})),+ )++ assert server.loads[0]["max_seq_length"] == 8192+++def test_remote_code_resident_is_refused_before_the_load(monkeypatch):+ """A trust_remote_code resident is refused before anything is evicted."""+ server = FakeServer(+ [dict(RESIDENT)],+ {+ "is_gguf": False,+ "active_model": RESIDENT["id"],+ "model_identifier": RESIDENT["id"],+ "requested_context_length": 4096,+ "requires_trust_remote_code": True,+ },+ ).install(monkeypatch)++ with pytest.raises(typer.Exit):+ start_cli._resolve_model(BASE, KEY, None, start_cli.LoadOptions(max_seq_length = 32768))++ assert server.loads == []+++def test_tensor_parallel_does_not_restart_a_non_gguf_resident(monkeypatch, capsys):+ """tensor_parallel is GGUF-only."""+ server = FakeServer(+ [dict(RESIDENT)],+ {+ "is_gguf": False,+ "active_model": RESIDENT["id"],+ "model_identifier": RESIDENT["id"],+ "requested_context_length": 4096,+ "tensor_parallel": False,+ },+ ).install(monkeypatch)++ start_cli._resolve_model(+ BASE,+ KEY,+ None,+ start_cli.LoadOptions(tensor_parallel = True, supplied = frozenset({"tensor_parallel"})),+ )++ assert "force_reload" not in server.loads[0]+ assert "unloads the current model" not in capsys.readouterr().out+++def test_a_differently_spelled_quant_still_counts_as_a_change(monkeypatch, capsys):+ """Q4KM and Q4_K_M are compared as typed."""+ server = FakeServer([dict(RESIDENT)], _gguf_status()).install(monkeypatch)++ start_cli._resolve_model(BASE, KEY, None, start_cli.LoadOptions(gguf_variant = "Q4KM"))++ assert server.loads[0]["force_reload"] is True+ assert "unloads the current model" in capsys.readouterr().out+++def test_manual_mode_keeps_a_pinned_layer_count(monkeypatch, capsys):+ """A resident already in manual keeps its pinned layer count."""+ server = FakeServer(+ [dict(RESIDENT)],+ _gguf_status(gpu_memory_mode = "manual", gpu_layers = 20),+ ).install(monkeypatch)++ start_cli._resolve_model(+ BASE,+ KEY,+ None,+ start_cli.LoadOptions(gpu_memory_mode = "manual", supplied = frozenset({"gpu_memory_mode"})),+ )++ assert server.loads[0]["gpu_layers"] == 20+ assert "force_reload" not in server.loads[0]+ assert "unloads the current model" not in capsys.readouterr().out+++def test_switching_into_manual_still_asks_for_automatic_layers(monkeypatch):+ server = FakeServer([dict(RESIDENT)], _gguf_status(gpu_memory_mode = "auto")).install(monkeypatch)++ start_cli._resolve_model(+ BASE,+ KEY,+ None,+ start_cli.LoadOptions(gpu_memory_mode = "manual", supplied = frozenset({"gpu_memory_mode"})),+ )++ assert server.loads[0]["gpu_layers"] == -1+++def test_arch_gated_tensor_request_is_not_restarted(monkeypatch):+ """Re-asking for an arch-gated tensor request is a no-op."""+ server = FakeServer(+ [dict(RESIDENT)],+ _gguf_status(tensor_parallel = False, tensor_parallel_dropped_by_arch_gate = True),+ ).install(monkeypatch)++ start_cli._resolve_model(+ BASE,+ KEY,+ None,+ start_cli.LoadOptions(tensor_parallel = True, supplied = frozenset({"tensor_parallel"})),+ )++ assert "force_reload" not in server.loads[0]+++def test_paravirtual_placement_is_not_restarted(monkeypatch):+ """Placement cannot differ on a paravirtual host."""+ server = FakeServer(+ [dict(RESIDENT)],+ _gguf_status(gpu_memory_mode = "manual", gpu_placement_paravirtual = True),+ ).install(monkeypatch)++ start_cli._resolve_model(+ BASE,+ KEY,+ None,+ start_cli.LoadOptions(gpu_memory_mode = "auto", supplied = frozenset({"gpu_memory_mode"})),+ )++ assert "force_reload" not in server.loads[0]+++def test_a_no_op_attach_to_a_custom_code_resident_is_allowed(monkeypatch):+ """The refusal exists to protect a reload; a no-op has no reload to protect."""+ server = FakeServer(+ [dict(RESIDENT)],+ {+ "is_gguf": False,+ "active_model": RESIDENT["id"],+ "model_identifier": RESIDENT["id"],+ "requested_context_length": 4096,+ "load_in_4bit": True,+ "requires_trust_remote_code": True,+ },+ ).install(monkeypatch)++ entry = start_cli._resolve_model(BASE, KEY, None, start_cli.LoadOptions(max_seq_length = 4096))++ assert "force_reload" not in server.loads[0]+ assert entry["id"] == RESIDENT["id"]+++def test_cpu_fallback_placement_is_not_restarted(monkeypatch):+ """Placement cannot differ under a CPU fallback."""+ server = FakeServer(+ [dict(RESIDENT)],+ _gguf_status(+ gpu_memory_mode = "manual",+ gpu_layers = 0,+ cpu_fallback_reason = "vulkan_startup_crash",+ ),+ ).install(monkeypatch)++ start_cli._resolve_model(+ BASE,+ KEY,+ None,+ start_cli.LoadOptions(gpu_memory_mode = "auto", supplied = frozenset({"gpu_memory_mode"})),+ )++ assert "force_reload" not in server.loads[0]+++def test_the_requested_mlx_kv_width_survives_a_reload(monkeypatch):+ """The requested MLX KV width round-trips, not the refused applied one."""+ server = FakeServer(+ [dict(RESIDENT)],+ {+ "is_gguf": False,+ "active_model": RESIDENT["id"],+ "model_identifier": RESIDENT["id"],+ "requested_context_length": 4096,+ "load_in_4bit": True,+ "mlx_kv_bits": None,+ "mlx_kv_bits_requested": 4,+ },+ ).install(monkeypatch)++ start_cli._resolve_model(BASE, KEY, None, start_cli.LoadOptions(max_seq_length = 32768))++ assert server.loads[0]["mlx_kv_bits"] == 4+++def test_a_proven_no_op_skips_the_preload_gate(monkeypatch):+ """A proven no-op skips the gate."""+ server = FakeServer([dict(RESIDENT)], _gguf_status()).install(monkeypatch)+ checked = []++ start_cli._resolve_model(+ BASE,+ KEY,+ None,+ start_cli.LoadOptions(max_seq_length = 8192),+ preload_check = lambda *a: checked.append(a),+ )++ assert checked == []+++def test_a_real_change_still_runs_the_preload_gate(monkeypatch):+ server = FakeServer([dict(RESIDENT)], _gguf_status()).install(monkeypatch)++ def gate(+ base,+ key,+ model,+ variant = None,+ ):+ raise typer.Exit(code = 1)++ with pytest.raises(typer.Exit):+ start_cli._resolve_model(+ BASE, KEY, None, start_cli.LoadOptions(max_seq_length = 32768), preload_check = gate+ )++ assert server.loads == []+++def test_a_quant_override_on_a_direct_file_is_refused(monkeypatch):+ """A different quant on a direct .gguf file is refused."""+ path = "/srv/models/Foo-Q4_K_M.gguf"+ server = FakeServer(+ [{"id": "Foo-Q4_K_M", "loaded": True}],+ {+ "is_gguf": True,+ "active_model": "Foo-Q4_K_M",+ "model_identifier": path,+ "gguf_variant": "Q4_K_M",+ "requested_context_length": 4096,+ },+ ).install(monkeypatch)++ with pytest.raises(typer.Exit):+ start_cli._resolve_model(BASE, KEY, None, start_cli.LoadOptions(gguf_variant = "UD-Q8_K_XL"))++ assert server.loads == []+++def test_explicit_tensor_disable_clears_an_arch_gated_fallback(monkeypatch, capsys):+ """Turning tensor_parallel off clears an arch-gated fallback."""+ server = FakeServer(+ [dict(RESIDENT)],+ _gguf_status(tensor_parallel = False, tensor_parallel_dropped_by_arch_gate = True),+ ).install(monkeypatch)++ start_cli._resolve_model(+ BASE,+ KEY,+ None,+ start_cli.LoadOptions(tensor_parallel = False, supplied = frozenset({"tensor_parallel"})),+ )++ assert server.loads[0]["force_reload"] is True+ assert "unloads the current model" in capsys.readouterr().out+++def test_switching_into_manual_is_still_a_change(monkeypatch, capsys):+ """auto to manual is a real change."""+ server = FakeServer([dict(RESIDENT)], _gguf_status(gpu_memory_mode = "auto")).install(monkeypatch)++ start_cli._resolve_model(+ BASE,+ KEY,+ None,+ start_cli.LoadOptions(gpu_memory_mode = "manual", supplied = frozenset({"gpu_memory_mode"})),+ )++ assert server.loads[0]["force_reload"] is True+ assert "unloads the current model" in capsys.readouterr().out+++def test_no_status_and_one_loaded_model_still_works(monkeypatch):+ """One loaded model is unambiguous without status."""+ server = FakeServer([dict(RESIDENT)], {}).install(monkeypatch)++ start_cli._resolve_model(BASE, KEY, None, start_cli.LoadOptions(max_seq_length = 32768))++ assert server.loads[0]["model_path"] == RESIDENT["id"]+++def test_no_status_and_several_loaded_models_refuses_to_guess(monkeypatch):+ """Several loaded models without status is refused."""+ server = FakeServer(+ [+ {"id": "unsloth/whisper-large-v3", "loaded": True},+ dict(RESIDENT),+ ],+ {},+ ).install(monkeypatch)++ with pytest.raises(typer.Exit):+ start_cli._resolve_model(BASE, KEY, None, start_cli.LoadOptions(max_seq_length = 32768))++ assert server.loads == []+++def test_non_gguf_gpu_selection_survives_a_reload(monkeypatch):+ """GPU placement is carried over."""+ server = FakeServer(+ [dict(RESIDENT)],+ {+ "is_gguf": False,+ "active_model": RESIDENT["id"],+ "model_identifier": RESIDENT["id"],+ "requested_context_length": 4096,+ "load_in_4bit": True,+ "requested_gpu_ids": [2, 3],+ },+ ).install(monkeypatch)++ start_cli._resolve_model(BASE, KEY, None, start_cli.LoadOptions(max_seq_length = 32768))++ assert server.loads[0]["gpu_ids"] == [2, 3]+++def _direct_gguf_server(monkeypatch, path = "/srv/models/Foo-Q4_K_M.gguf"):+ return FakeServer(+ [{"id": "Foo-Q4_K_M", "loaded": True}],+ {+ "is_gguf": True,+ "active_model": "Foo-Q4_K_M",+ "model_identifier": path,+ "gguf_variant": "Q4_K_M",+ "requested_context_length": 4096,+ },+ ).install(monkeypatch)+++def test_restating_the_running_quant_applies_the_other_overrides(monkeypatch):+ """Restating the running quant does not block the other overrides."""+ path = "/srv/models/Foo-Q4_K_M.gguf"+ server = _direct_gguf_server(monkeypatch, path)++ start_cli._resolve_model(+ BASE,+ KEY,+ None,+ start_cli.LoadOptions(gguf_variant = "Q4_K_M", max_seq_length = 32768),+ )++ sent_payload = server.loads[0]+ assert sent_payload["model_path"] == path+ assert sent_payload["max_seq_length"] == 32768+ assert "gguf_variant" not in sent_payload+++def test_a_matching_quant_is_compared_case_insensitively(monkeypatch):+ server = _direct_gguf_server(monkeypatch)++ start_cli._resolve_model(+ BASE,+ KEY,+ None,+ start_cli.LoadOptions(gguf_variant = "q4_k_m", max_seq_length = 32768),+ )++ assert server.loads[0]["max_seq_length"] == 32768+++def test_a_differing_quant_on_a_direct_file_is_still_refused(monkeypatch):+ server = _direct_gguf_server(monkeypatch)++ with pytest.raises(typer.Exit):+ start_cli._resolve_model(+ BASE,+ KEY,+ None,+ start_cli.LoadOptions(gguf_variant = "UD-Q8_K_XL", max_seq_length = 32768),+ )++ assert server.loads == []+++def test_a_diffusion_resident_is_never_an_attach_target(monkeypatch):+ """An image runtime answers with an active_model but can never serve chat."""+ server = FakeServer(+ [{"id": "unsloth/FLUX.1-dev-GGUF", "loaded": True}],+ {+ "is_gguf": True,+ "is_diffusion": True,+ "active_model": "unsloth/FLUX.1-dev-GGUF",+ "model_identifier": "unsloth/FLUX.1-dev-GGUF",+ "requested_context_length": 4096,+ },+ ).install(monkeypatch)++ with pytest.raises(typer.Exit):+ start_cli._resolve_model(BASE, KEY, None, start_cli.LoadOptions(max_seq_length = 32768))++ assert server.loads == []+++ "field",+ ["spec_probe_retry_pending", "spec_dflash_retry_pending", "spec_fallback_binary_changed"],+)+def test_a_pending_retry_is_not_a_no_op(monkeypatch, capsys, field):+ """_runtime_matches_intent reloads on an identical intent while a retry is pending."""+ server = FakeServer(+ [dict(RESIDENT)], _gguf_status(**{field: True})+ ).install(monkeypatch)++ start_cli._resolve_model(BASE, KEY, None, start_cli.LoadOptions(max_seq_length = 8192))++ assert server.loads[0]["force_reload"] is True+ assert "unloads the current model" in capsys.readouterr().out+++def test_no_pending_retry_is_still_a_no_op(monkeypatch):+ server = FakeServer([dict(RESIDENT)], _gguf_status()).install(monkeypatch)++ start_cli._resolve_model(BASE, KEY, None, start_cli.LoadOptions(max_seq_length = 8192))++ assert "force_reload" not in server.loads[0]diff --git a/unsloth_cli/commands/start.py b/unsloth_cli/commands/start.pyindex 616ff5c7789..013ce3f7a8a 100644--- a/unsloth_cli/commands/start.py+++ b/unsloth_cli/commands/start.py@@ -600,6 +600,71 @@ class LoadOptions(NamedTuple):load_in_4bit: bool = Truetensor_parallel: bool = Falsegpu_memory_mode: Optional[Literal["auto", "manual"]] = None+ # Names the user actually typed: --context-length 0 equals the declared default yet+ # is a reset the server must hear. Appended last to keep positional callers working.+ supplied: frozenset = frozenset()++ def overrides(self) -> frozenset:+ """Fields that must reach the load: typed explicitly, or differing from default."""+ differing = {+ name+ for name, default in (+ ("gguf_variant", None),+ ("max_seq_length", 0),+ ("load_in_4bit", True),+ ("tensor_parallel", False),+ ("gpu_memory_mode", None),+ )+ if getattr(self, name) != default+ }+ # Internal callers never populate `supplied`, so a non-default value counts too.+ return frozenset(differing) | frozenset(self.supplied)+++_LOAD_OPTION_PARAMS = (+ "gguf_variant",+ "max_seq_length",+ "load_in_4bit",+ "tensor_parallel",+ "gpu_memory_mode",+)+++def _supplied_load_params(ctx) -> frozenset:+ """Which load knobs Click saw on the command line.++ The context must be PASSED IN: Typer invokes callbacks with no active click context,+ so click.get_current_context() is None. Unaskable -> empty set, and `overrides()`+ falls back to comparing values.+ """+ getter = getattr(ctx, "get_parameter_source", None)+ if getter is None:+ return frozenset()+ supplied = set()+ for name in _LOAD_OPTION_PARAMS:+ try:+ source = getter(name)+ except Exception:+ continue+ # By member NAME, not identity or ordering: Typer vendors its own click, so this+ # is typer._click's ParameterSource, and click 8.3 reordered the IntEnum.+ if getattr(source, "name", None) == "COMMANDLINE":+ supplied.add(name)+ return frozenset(supplied)+++def _load_options(+ ctx, gguf_variant, max_seq_length, load_in_4bit, tensor_parallel, gpu_memory_mode+) -> LoadOptions:+ """Build LoadOptions for an agent command, recording what was typed."""+ return LoadOptions(+ gguf_variant,+ max_seq_length,+ load_in_4bit,+ tensor_parallel,+ gpu_memory_mode,+ _supplied_load_params(ctx),+ )class ServerOptions(NamedTuple):@@ -1645,12 +1710,214 @@ def _model_id_matches(return str(actual).casefold() == str(requested).casefold()+def _inference_status(base: str, key: str) -> dict:+ """Runtime state of the resident model. {} means "cannot prove anything" (older+ server), never "nothing is set"."""+ try:+ return _http_json("GET", f"{base}/api/inference/status", key)+ except Exception:+ return {}+++def _resident_load_target(models: list, status: dict, allow_casefold: bool):+ """(identifier to post, id it is advertised as) for the running model.++ /v1/models shows only the sanitized basename while _same_loaded_identifier compares+ resident paths exactly, so the load must carry the identifier status reports.+ """+ if status.get("is_diffusion"):+ # An image runtime answers with an active_model like any other, but it cannot+ # serve chat: targeting it would tear down the diffusion server and then point+ # the agent at a model that can never answer it.+ _fail(+ "Unsloth is serving an image model, which cannot serve chat, so there are no "+ "settings to apply. Re-run with --model naming the chat model to load."+ )+ active_id = status.get("active_model")+ entry = None+ if active_id:+ entry = next(+ (+ m+ for m in models+ if _model_id_matches(m.get("id"), active_id, allow_casefold = allow_casefold)+ and m.get("loaded") is not False+ ),+ None,+ )+ if entry is None and not status:+ # Only when there is no status at all (older server). A status that ANSWERED with+ # active_model null is stating there is no chat resident.+ # Catalog ORDER is not evidence either: /v1/models lists loaded speech sidecars+ # too, so the first entry can be one. Answer only when the catalog is unambiguous.+ loaded = [m for m in models if m.get("loaded") is not False]+ if len(loaded) == 1:+ entry = loaded[0]+ elif loaded:+ _fail(+ "This Unsloth cannot say which model is serving chat, and more than one is "+ "loaded. Re-run with --model naming the one these settings are for."+ )+ public_id = active_id or (entry or {}).get("id")+ if not public_id:+ if status:+ # Status answered and named no chat model. Returning empty here would drop+ # the knobs silently, which is the bug this path exists to fix.+ _fail(+ "No chat model is currently loaded, so there are no settings to apply. "+ "Re-run with --model naming the model to load."+ )+ return None, None+ identifier = status.get("model_identifier")+ if identifier:+ return identifier, public_id+ # A native path lease redacts the internal path; inventing one from the basename+ # would address the wrong file, so only a hub id can be posted back.+ if _is_hub_model_id(public_id):+ return public_id, public_id+ _fail(+ f"Unsloth is serving '{public_id}' from a local path it does not expose, so these "+ "settings cannot be applied by attaching. Re-run with --model naming that path."+ )+++# /api/inference/status field -> the /api/inference/load field that reproduces it. The+# "requested_" values are what the load was INVOKED with, which is what has to be resent;+# the bare names are the resolved ones and would pin a value the user never chose.+_RESIDENT_RUNTIME_FIELDS = {+ "cache_type_kv": "cache_type_kv",+ "chat_template_override": "chat_template_override",+ "disable_vision": "disable_vision",+ "gpu_memory_mode": "gpu_memory_mode",+ "gpu_layers": "gpu_layers",+ "n_cpu_moe": "n_cpu_moe",+ "tensor_split": "tensor_split",+ "tensor_parallel": "tensor_parallel",+ "speculative_type": "speculative_type",+ "spec_draft_n_max": "spec_draft_n_max",+ # The applied value is null when the runtime refused the request.+ "mlx_kv_bits_requested": "mlx_kv_bits",+ # LoadRequest defaults this to True, so omitting it would reload a full-precision+ # model in 4-bit. Null on GGUF, which has no such setting.+ "load_in_4bit": "load_in_4bit",+ # Same trap: max_seq_length defaults to 0, which _gguf_request_intent copies into+ # n_ctx, so changing another knob would reset a custom context to automatic.+ "requested_context_length": "max_seq_length",+ "requested_gpu_ids": "gpu_ids",+ "requested_parallel_slots": "n_parallel",+ "requested_n_batch": "n_batch",+ "requested_n_ubatch": "n_ubatch",+ "requested_load_mode": "load_mode",+ "requested_ctx_checkpoints": "ctx_checkpoints",+ "requested_cache_ram": "cache_ram",+ "requested_spec_draft_cache_type": "spec_draft_cache_type",+ "requested_llama_extra_args": "llama_extra_args",+}+++def _resident_runtime_payload(status: dict, payload: dict) -> dict:+ """The resident's own settings for knobs this load did not name.++ None means "never set" for every one of these, so it is dropped rather than sent:+ omitting a field is what lets the server inherit, while sending null would pin the+ absence. An explicit empty list is kept, since that is a real "launch with none".+ """+ if not status:+ return {}+ carried = {}+ for field, load_field in _RESIDENT_RUNTIME_FIELDS.items():+ if load_field in payload:+ continue+ value = status.get(field)+ if value is None:+ continue+ carried[load_field] = value+ return carried+++def _load_settings_differ(status: dict, load: LoadOptions, overrides: frozenset) -> bool:+ """Whether applying these settings can restart the resident. Unproven equality+ counts as a difference: a silent restart is worse than a spurious warning."""+ if not status:+ return True+ # _runtime_matches_intent rejects an identical intent while a spec probe, a DFlash+ # drafter or a changed speculative binary is waiting to be retried, so the server+ # reloads regardless of what the CLI asked for. Equality of the overrides is then no+ # proof of a no-op, and claiming one would skip the gate and the warning.+ if any(+ status.get(field)+ for field in (+ "spec_probe_retry_pending",+ "spec_dflash_retry_pending",+ "spec_fallback_binary_changed",+ )+ ):+ return True+ for name in overrides:+ if name == "gguf_variant":+ resident = status.get("gguf_variant") if status.get("is_gguf") else None+ # Casefold, not _normalized_variant, which strips separators: a mistyped Q4KM+ # would read as equal here yet still really reload on the server. The preload+ # gate below already compares this way, and the two have to agree.+ if (+ not resident+ or str(resident).strip().lower() != str(load.gguf_variant).strip().lower()+ ):+ return True+ elif name == "max_seq_length":+ # Requested, not resolved: llama.cpp clamps n_ctx at fit time.+ resident = status.get("requested_context_length")+ if resident is None or int(resident) != int(load.max_seq_length):+ return True+ elif name == "load_in_4bit":+ # GGUF has no 4-bit setting and reports null, which would read as "differs"+ # and warn about an unload the server is not going to perform.+ if status.get("is_gguf"):+ continue+ resident = status.get("load_in_4bit")+ if resident is None or bool(resident) != bool(load.load_in_4bit):+ return True+ elif name == "tensor_parallel":+ # llama.cpp only. The standard load never forwards it, so a restart would+ # apply nothing.+ if not status.get("is_gguf"):+ continue+ # The architecture gate can normalize a tensor request to layer mode and say+ # so. Asking for it AGAIN is the request already applied, not a difference,+ # and the backend dedupes exactly this state. Asking to turn it OFF is the+ # opposite: the backend keeps the tensor intent behind that fallback and does+ # not read a bare false as an explicit drop (the UI sends false routinely), so+ # only a real reload can clear it.+ if status.get("tensor_parallel_dropped_by_arch_gate"):+ if load.tensor_parallel:+ continue+ return True+ if bool(status.get("tensor_parallel")) != bool(load.tensor_parallel):+ return True+ elif name == "gpu_memory_mode":+ if not status.get("is_gguf"):+ continue+ # A paravirtual host pins every placement request to the same runtime, and a+ # CPU fallback is preserved across reloads by _preserve_cpu_fallback_intent, so+ # in both cases the raw mode cannot tell two requests apart. resident-config-+ # match.ts skips placement on exactly these two for the same reason.+ if status.get("gpu_placement_paravirtual") or status.get("cpu_fallback_reason"):+ continue+ if status.get("gpu_memory_mode") != load.gpu_memory_mode:+ return True+ # Manual to manual is a real no-op: the payload only sends the implicit+ # gpu_layers = -1 when switching INTO manual, so a resident already pinned to+ # a layer count keeps it through the round-trip and nothing changes.+ return False++def _resolve_model(base: str,key: str,requested: Optional[str],load: LoadOptions = LoadOptions(),preload_check = None,+ infer_resident: bool = True,) -> dict:models = _loaded_models(base, key)load_requested = False@@ -1663,13 +1930,22 @@ def _resolve_model(# /api/inference/load: the server's already-loaded dedup answers "already_loaded"# without reloading when the variant AND settings match, so a second session running# the same command still attaches without evicting the first.- load_has_overrides = bool(- load.gguf_variant- or load.max_seq_length- or not load.load_in_4bit- or load.tensor_parallel- or load.gpu_memory_mode is not None- )+ overrides = load.overrides()+ load_has_overrides = bool(overrides)+ # Inferred-attach path only: `requested` becomes the resident's internal identifier+ # (possibly a server path), so this is the id to show and to match on.+ attach_public_id = None+ status_snapshot = None+ # Whether the inferred settings can restart the resident. Computed once: the preload+ # gate, the warning, the consent refusal and force_reload must all agree, and asking+ # twice against a snapshot taken at different times is how they drift apart.+ inferred_differs = False+ if requested is None and load_has_overrides and infer_resident:+ status_snapshot = _inference_status(base, key)+ requested, attach_public_id = _resident_load_target(models, status_snapshot, allow_casefold)+ inferred_differs = _load_settings_differ(status_snapshot, load, overrides)+ # preload_check deliberately survives: it is the only gate before the load evicts+ # the shared model (_require_gguf_for_codex runs after _connect returns).# /v1/models also lists cached-but-unloaded catalog entries (loaded == False);# matching one would skip /api/inference/load and leave the agent pointed at a# model that is not resident, so only attach to an entry that is actually loaded.@@ -1692,17 +1968,27 @@ def _resolve_model(# resident model already satisfies (a path-loaded GGUF shown as a bare basename# can collide with a non-GGUF unsloth/<name>).active = next((m for m in models if m.get("loaded") is not False), None)+ # On the inferred path the target came from status, so catalog order can name a+ # different entry (a speech sidecar listed first). Using it would make the+ # survivor probe below report the wrong model as still serving.+ if attach_public_id is not None:+ active = next(+ (+ m+ for m in models+ if _model_id_matches(+ m.get("id"), attach_public_id, allow_casefold = allow_casefold+ )+ and m.get("loaded") is not False+ ),+ None,+ ) or {"id": attach_public_id}if preload_check is not None:# An explicit knob forces match to None so the server's disk-free dedupe can# answer already_loaded; gating it would reject a second session for the model# already serving, whose file may have moved. Only the quant is checked below:# any other run knob changes the runtime intent, a real reload nothing dedupes.- other_overrides = bool(- load.max_seq_length- or not load.load_in_4bit- or load.tensor_parallel- or load.gpu_memory_mode is not None- )+ other_overrides = bool(overrides - {"gguf_variant"})# /v1/models shows a path-loaded GGUF under its basename, so match that spelling# too, or a second session reruns the gate.wanted_ids = {requested, _public_model_id(requested)} - {None}@@ -1714,6 +2000,11 @@ def _resolve_model()for m in models)+ # A proven no-op evicts nothing, so the gate has nothing to protect, and+ # running it would reject an attach the disk-free already-loaded path can+ # still serve (a direct .gguf the server has mapped but that has since moved).+ if attach_public_id is not None and not inferred_differs:+ resident_serves_request = True# /v1/models shows only the basename, so confirm a path request against the# identifier the server loaded -- else /new/foo.gguf reads as resident because# /old/foo.gguf is.@@ -1749,7 +2040,14 @@ def _resolve_model(preload_check(base, key, requested, load.gguf_variant)active_id = active.get("id") if active else Noneannounced_switch = False- if active_id and not _model_id_matches(+ if attach_public_id is not None:+ # An inferred attach never switches model, so the comparison below would+ # misreport a switch and print the server's path.+ if inferred_differs:+ typer.echo(f"Applying new load settings to {attach_public_id}.")+ typer.echo("This unloads the current model for every attached session.")+ announced_switch = True+ elif active_id and not _model_id_matches(active_id,requested,allow_casefold = allow_casefold,@@ -1773,20 +2071,83 @@ def _resolve_model(typer.echo("This unloads the current model for every attached session.")announced_switch = True# Mirror `unsloth run`'s load knobs; keep the default payload as just- # model_path so a bare `--model` load is unchanged.+ # model_path so a bare `--model` load is unchanged. Membership decides, not+ # truthiness: a reset like --context-length 0 equals the default yet must be sent.payload = {"model_path": requested}- if load.gguf_variant:- payload["gguf_variant"] = load.gguf_variant- if load.max_seq_length:+ if "gguf_variant" in overrides and load.gguf_variant:+ direct_file = attach_public_id is not None and str(requested).lower().endswith(".gguf")+ if direct_file:+ # from_identifier consults a variant only for a DIRECTORY, so a DIFFERENT+ # quant cannot be selected: posting one would reload the very same file and+ # label it with a quant that does not describe its weights. Restating the+ # one already running asks for no change, so drop the inapplicable field+ # and let the other overrides through.+ resident_variant = status_snapshot.get("gguf_variant")+ same = (+ bool(resident_variant)+ and str(resident_variant).strip().lower()+ == str(load.gguf_variant).strip().lower()+ )+ if not same:+ _fail(+ f"'{attach_public_id}' was loaded from a single .gguf file, so "+ f"--gguf-variant {load.gguf_variant} cannot select a different quant. "+ "Re-run with --model naming the repository to switch quants."+ )+ else:+ payload["gguf_variant"] = load.gguf_variant+ elif attach_public_id is not None and status_snapshot.get("is_gguf"):+ # Re-send the running quant: a repo id carries none, so from_identifier would+ # auto-pick (_GGUF_QUANT_PREFERENCE, UD-Q4_K_XL first) and changing only the+ # context would evict a chosen Q8_0 to download a different quant. Skip a+ # .gguf path, which loads as itself -- the server gates on the same suffix.+ resident_variant = status_snapshot.get("gguf_variant")+ if resident_variant and not str(requested).lower().endswith(".gguf"):+ payload["gguf_variant"] = resident_variant+ if "max_seq_length" in overrides:payload["max_seq_length"] = load.max_seq_length- if not load.load_in_4bit:- payload["load_in_4bit"] = False- if load.tensor_parallel:- payload["tensor_parallel"] = True- if load.gpu_memory_mode is not None:+ if "load_in_4bit" in overrides:+ payload["load_in_4bit"] = load.load_in_4bit+ if "tensor_parallel" in overrides:+ payload["tensor_parallel"] = load.tensor_parallel+ if "gpu_memory_mode" in overrides and load.gpu_memory_mode is not None:payload["gpu_memory_mode"] = load.gpu_memory_mode- if load.gpu_memory_mode == "manual":+ # -1 means "pick the layers", which is right when the user is switching INTO+ # manual, but on an inferred attach to a resident already in manual it would+ # throw away the layer count it was pinned to. Leave it for the round-trip.+ already_manual = (+ attach_public_id is not None and status_snapshot.get("gpu_memory_mode") == "manual"+ )+ if load.gpu_memory_mode == "manual" and not already_manual:payload["gpu_layers"] = -1+ if (+ attach_public_id is not None+ and inferred_differs+ and status_snapshot.get("requires_trust_remote_code")+ ):+ # The reload cannot reproduce the consent: the payload has no+ # trust_remote_code and no approval fingerprint, and the standard backend+ # tears the worker down BEFORE the replacement is accepted, so a rejected+ # custom-code load leaves nothing resident. Refuse while the model is still+ # serving; naming it with --model goes through the normal consent path.+ _fail(+ f"'{attach_public_id}' was loaded with trust_remote_code, which an attach "+ "cannot re-authorize. Re-run with --model naming it to apply these settings."+ )+ if attach_public_id is not None:+ # An inferred reload is a full load, not a PATCH: _gguf_request_intent copies+ # every defaulted LoadRequest field into the new intent, so a knob we leave out+ # is reset rather than kept. Carry the resident's own values for the ones the+ # user did not name, or changing the context alone would drop their KV dtype,+ # slot count, batch sizes and GPU placement.+ payload.update(_resident_runtime_payload(status_snapshot, payload))+ # The server cannot tell an explicit `--context-length 0` reset from the 0 that+ # every UI load sends, so it treats 0 as "no preference" and would answer+ # already_loaded. Say outright that this one is a reload, but only when status+ # PROVED a difference: on an older server _load_settings_differ cannot tell,+ # and forcing there would evict on every attach.+ if status_snapshot and inferred_differs:+ payload["force_reload"] = Truetry:loaded = _load_model_with_progress(base, key, requested, load, payload)except Exception:@@ -1797,12 +2158,16 @@ def _resolve_model(typer.echo(f"Nothing was unloaded; {active_id} is still serving.", err = True)raiseif loaded.get("status") == "already_loaded":- typer.echo(f"Reusing loaded model: {_display_model_spec(requested, load.gguf_variant)}")+ # Show the public id on the inferred path; `requested` may be a server path.+ shown = attach_public_id or requested+ typer.echo(f"Reusing loaded model: {_display_model_spec(shown, load.gguf_variant)}")# Unsloth registers the model under a canonical id (resolved identifier,# casing) that /v1/models echoes but which may differ from the path we# passed; match on the id the load reports so we don't silently fall# through to models[0] and connect to a different loaded model.- wanted = {requested, _public_model_id(requested)} - {None}+ # attach_public_id: our _public_model_id only strips a basename, while the+ # server also maps an HF cache path to its repo id, so the two can disagree.+ wanted = {requested, _public_model_id(requested), attach_public_id} - {None}if isinstance(loaded, dict):wanted |= {loaded.get("model"), loaded.get("display_name")} - {None}models = _loaded_models(base, key)@@ -3868,6 +4233,9 @@ def _connect(None if server is not None else model,load,preload_check = None if server is not None else preload_check,+ # That server was started FROM these knobs, so inferring a target here would+ # reload what was just loaded.+ infer_resident = server is None,)except BaseException:_shutdown_auto_served()@@ -4535,7 +4903,9 @@ def claude(base, key, entry = _connect(api_key,model,- LoadOptions(gguf_variant, max_seq_length, load_in_4bit, tensor_parallel, gpu_memory_mode),+ _load_options(+ ctx, gguf_variant, max_seq_length, load_in_4bit, tensor_parallel, gpu_memory_mode+ ),serve = serve,launch = launch,server_options = ServerOptions(@@ -4656,7 +5026,9 @@ def codex(base, key, entry = _connect(api_key,model,- LoadOptions(gguf_variant, max_seq_length, load_in_4bit, tensor_parallel, gpu_memory_mode),+ _load_options(+ ctx, gguf_variant, max_seq_length, load_in_4bit, tensor_parallel, gpu_memory_mode+ ),serve = serve,launch = launch,preload_check = _attach_gguf_check_for_codex,@@ -4766,7 +5138,9 @@ def openclaw(base, key, entry = _connect(api_key,model,- LoadOptions(gguf_variant, max_seq_length, load_in_4bit, tensor_parallel, gpu_memory_mode),+ _load_options(+ ctx, gguf_variant, max_seq_length, load_in_4bit, tensor_parallel, gpu_memory_mode+ ),serve = serve,launch = launch,server_options = ServerOptions(@@ -4856,7 +5230,9 @@ def opencode(base, key, entry = _connect(api_key,model,- LoadOptions(gguf_variant, max_seq_length, load_in_4bit, tensor_parallel, gpu_memory_mode),+ _load_options(+ ctx, gguf_variant, max_seq_length, load_in_4bit, tensor_parallel, gpu_memory_mode+ ),serve = serve,launch = launch,server_options = ServerOptions(@@ -5041,7 +5417,9 @@ def hermes(base, key, entry = _connect(api_key,model,- LoadOptions(gguf_variant, max_seq_length, load_in_4bit, tensor_parallel, gpu_memory_mode),+ _load_options(+ ctx, gguf_variant, max_seq_length, load_in_4bit, tensor_parallel, gpu_memory_mode+ ),serve = serve,launch = launch,server_options = ServerOptions(@@ -5106,7 +5484,9 @@ def pi(base, key, entry = _connect(api_key,model,- LoadOptions(gguf_variant, max_seq_length, load_in_4bit, tensor_parallel, gpu_memory_mode),+ _load_options(+ ctx, gguf_variant, max_seq_length, load_in_4bit, tensor_parallel, gpu_memory_mode+ ),serve = serve,launch = launch,server_options = ServerOptions(