Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

Apply the context length and quant settings when attaching to a running server (#10093)

Nilay9aeefd55e617
diff --git a/studio/backend/models/inference.py b/studio/backend/models/inference.py
index 537c249a400..3cd90687a60 100644
--- a/studio/backend/models/inference.py
+++ b/studio/backend/models/inference.py
@@ -1464,9 +1464,19 @@ class InferenceStatusResponse(_InferenceRuntimeFields):
requested_context_length: Optional[int] = Field(
None,
description = (
- "The n_ctx the active GGUF load was invoked with (0 = Auto). Lets the "
- "UI re-seed a Manual + Auto-layers context pin on hydration, where "
- "context_length only exposes the resolved value. None for non-GGUF."
+ "The max_seq_length the active load was invoked with (0 = Auto). Lets the "
+ "UI re-seed a Manual + Auto-layers context pin on hydration, and lets a CLI "
+ "attach tell a no-op from a reload, where context_length only exposes the "
+ "resolved value. None when the running model predates this field."
+ ),
+ )
+ load_in_4bit: Optional[bool] = Field(
+ None,
+ description = (
+ "The 4-bit setting the active non-GGUF load was REQUESTED with, not the "
+ "value it resolved to: LoRA and the latest-transformers tier rewrite it, so "
+ "only the requested value can be compared against a later request. None for "
+ "GGUF, which does not use it."
),
)
llama_cpp_supports_mtp: bool = Field(
diff --git a/studio/backend/routes/inference.py b/studio/backend/routes/inference.py
index d6829143554..4d97ed006f2 100644
--- a/studio/backend/routes/inference.py
+++ b/studio/backend/routes/inference.py
@@ -12862,6 +12862,31 @@ def _mlx_runtime_settings_match(backend, request) -> bool:
) == (request.chat_template_override or None)
+def _non_gguf_runtime_settings_match(backend, request) -> bool:
+ """Whether the resident non-GGUF model already runs the request's load settings.
+
+ An unrecorded resident value counts as a MATCH: every UI call site ships
+ max_seq_length and load_in_4bit on every load, so otherwise every pick would reload.
+ max_seq_length 0 means "model default", so a `--context-length 0` reset is honoured
+ on GGUF but not here.
+ """
+ if getattr(request, "force_reload", False):
+ return False
+ fields_set = getattr(request, "model_fields_set", set()) or set()
+ entry = backend.models.get(backend.active_model_name, {}) or {}
+ if "max_seq_length" in fields_set and int(request.max_seq_length or 0) > 0:
+ resident = entry.get("max_seq_length_requested")
+ if resident is not None and int(resident) != int(request.max_seq_length):
+ return False
+ if "load_in_4bit" in fields_set:
+ # As REQUESTED, not as resolved: _effective_load_in_4bit rewrites it for LoRA and
+ # the latest-transformers tier, so raw-vs-resolved would never match.
+ resident = entry.get("load_in_4bit_requested")
+ if resident is not None and bool(resident) != bool(request.load_in_4bit):
+ return False
+ return True
+
+
class _ScopedLoadAttempt(NamedTuple):
token: str
request_id: Optional[str]
@@ -13303,9 +13328,11 @@ async def _load_model_impl(
await asyncio.to_thread(acquire_for, CHAT)
return reused
if not (request.gguf_variant or is_direct_gguf_request):
- if _same_loaded_identifier(
- backend.active_model_name, model_identifier
- ) and _mlx_runtime_settings_match(backend, request):
+ if (
+ _same_loaded_identifier(backend.active_model_name, model_identifier)
+ and _mlx_runtime_settings_match(backend, request)
+ and _non_gguf_runtime_settings_match(backend, request)
+ ):
api_monitor.discard(_load_event) # nothing loaded, no monitor row
logger.info(f"Model already loaded (Unsloth): {model_log_label}, skipping reload")
# A no-op Unsloth load of a preview-owned checkpoint still claims it.
@@ -14011,6 +14038,20 @@ async def _load_model_impl(
),
)
+ # Stamped here, not in backend.load_model: that entry is built in the load
+ # subprocess and only a fixed model_info mirror crosses back, so it would
+ # never be read.
+ _resident_entry = backend.models.get(backend.active_model_name)
+ if isinstance(_resident_entry, dict):
+ _resident_entry["max_seq_length_requested"] = int(request.max_seq_length or 0)
+ _resident_entry["load_in_4bit_requested"] = bool(request.load_in_4bit)
+ # Placement too, or a reload for some other knob would fall back to automatic
+ # selection and could land the model on a different GPU. The parent-side
+ # orchestrator entry does not keep it, so status has nothing else to report.
+ _resident_entry["gpu_ids_requested"] = (
+ list(request.gpu_ids) if request.gpu_ids else None
+ )
+
logger.info(
f"Loaded model: {model_log_label if native_grant_backed else config.identifier}"
)
@@ -16188,6 +16229,9 @@ async def get_status(current_subject: str = Depends(get_current_subject)):
preserve_thinking_default = _sf_flags.get("preserve_thinking_default", False),
supports_tools = _sf_flags["supports_tools"],
context_length = _positive_int_or_none(model_info.get("context_length")),
+ requested_context_length = model_info.get("max_seq_length_requested"),
+ load_in_4bit = model_info.get("load_in_4bit_requested"),
+ requested_gpu_ids = model_info.get("gpu_ids_requested"),
chat_template = chat_template,
llama_cpp_supports_mtp = _supports_mtp,
llama_cpp_prebuilt_stale = _stale,
diff --git a/studio/backend/tests/test_non_gguf_reload_settings.py b/studio/backend/tests/test_non_gguf_reload_settings.py
new file mode 100644
index 00000000000..549684ae6de
--- /dev/null
+++ b/studio/backend/tests/test_non_gguf_reload_settings.py
@@ -0,0 +1,150 @@
+# SPDX-License-Identifier: AGPL-3.0-only
+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
+
+"""Reload gating and status reporting for a resident non-GGUF model."""
+
+from __future__ import annotations
+
+import logging
+import sys
+import types as _types
+from pathlib import Path
+
+import pytest
+
+_BACKEND = Path(__file__).resolve().parents[1]
+if str(_BACKEND) not in sys.path:
+ sys.path.insert(0, str(_BACKEND))
+
+# Stub the optional deps routes/__init__ pulls in, so this module imports standalone.
+_loggers_stub = _types.ModuleType("loggers")
+_loggers_stub.get_logger = lambda name: logging.getLogger(name)
+sys.modules.setdefault("loggers", _loggers_stub)
+
+_structlog_stub = _types.ModuleType("structlog")
+_structlog_stub.get_logger = lambda *_a, **_k: logging.getLogger("structlog_stub")
+sys.modules.setdefault("structlog", _structlog_stub)
+if not hasattr(sys.modules["structlog"], "get_logger"):
+ sys.modules["structlog"].get_logger = _structlog_stub.get_logger
+
+import routes.inference as inference_route # noqa: E402
+
+
+RESIDENT = "unsloth/Qwen3-8B"
+
+
+class _Backend:
+ def __init__(self, entry):
+ self.active_model_name = RESIDENT
+ self.models = {RESIDENT: entry}
+
+
+class _Request:
+ """model_fields_set is what pydantic records."""
+ def __init__(self, **fields):
+ self.model_fields_set = set(fields)
+ self.force_reload = fields.pop("force_reload", False)
+ self.max_seq_length = fields.pop("max_seq_length", 0)
+ self.load_in_4bit = fields.pop("load_in_4bit", True)
+ self.tensor_parallel = fields.pop("tensor_parallel", False)
+ self.gpu_memory_mode = fields.pop("gpu_memory_mode", None)
+
+
+def _loaded(max_seq_length = 4096, load_in_4bit = True):
+ return _Backend(
+ {
+ "max_seq_length_requested": max_seq_length,
+ "load_in_4bit_requested": load_in_4bit,
+ }
+ )
+
+
+def test_matching_explicit_settings_are_reused():
+ backend = _loaded(max_seq_length = 4096, load_in_4bit = True)
+ request = _Request(max_seq_length = 4096, load_in_4bit = True)
+ assert inference_route._non_gguf_runtime_settings_match(backend, request)
+
+
+def test_changed_context_forces_a_reload():
+ backend = _loaded(max_seq_length = 4096)
+ request = _Request(max_seq_length = 32768)
+ assert not inference_route._non_gguf_runtime_settings_match(backend, request)
+
+
+def test_changed_precision_forces_a_reload():
+ backend = _loaded(load_in_4bit = True)
+ request = _Request(load_in_4bit = False)
+ assert not inference_route._non_gguf_runtime_settings_match(backend, request)
+
+
+def test_omitted_settings_keep_the_legacy_reuse():
+ """A caller that sends only model_path still reuses."""
+ backend = _loaded(max_seq_length = 4096, load_in_4bit = True)
+ assert inference_route._non_gguf_runtime_settings_match(backend, _Request())
+
+
+def test_zero_context_expresses_no_preference():
+ """max_seq_length 0 never forces a reload."""
+ assert inference_route._non_gguf_runtime_settings_match(
+ _loaded(max_seq_length = 2048), _Request(max_seq_length = 0)
+ )
+
+
+def test_unrecorded_resident_settings_are_reused_not_reloaded():
+ """An unrecorded resident value is not a mismatch."""
+ backend = _Backend({})
+ assert inference_route._non_gguf_runtime_settings_match(
+ backend, _Request(max_seq_length = 32768, load_in_4bit = False)
+ )
+
+
+def test_force_reload_is_honored():
+ """force_reload defeats the match."""
+ backend = _loaded(max_seq_length = 4096)
+ request = _Request(force_reload = True, max_seq_length = 4096)
+ assert not inference_route._non_gguf_runtime_settings_match(backend, request)
+
+
+ "field, value",
+ [("tensor_parallel", True), ("gpu_memory_mode", "auto"), ("gpu_memory_mode", "manual")],
+)
+def test_gguf_only_knobs_never_block_reuse(field, value):
+ """The chat UI sends gpu_memory_mode ungated and keeps tensor_parallel across a
+ model switch, so neither carries user intent for a transformers load."""
+ assert inference_route._non_gguf_runtime_settings_match(_loaded(), _Request(**{field: value}))
+
+
+class TestNonGgufStatusReportsWhatTheLoadAskedFor:
+ """Placement is not kept on the parent-side orchestrator entry at all, so anything
+ the route does not stamp is simply unavailable to a client."""
+
+ STAMPED = ("max_seq_length_requested", "load_in_4bit_requested", "gpu_ids_requested")
+
+ def _stamp_block(self):
+ import inspect
+ import routes.inference as ri
+
+ src = inspect.getsource(ri._load_model_impl)
+ start = src.index("_resident_entry = backend.models.get")
+ return src[start : start + 900]
+
+ @pytest.mark.parametrize("field", STAMPED)
+ def test_the_route_stamps_it_after_a_successful_load(self, field):
+ assert field in self._stamp_block(), f"{field} is never recorded on the resident"
+
+ def test_the_non_gguf_status_branch_publishes_them(self):
+ import inspect
+ import routes.inference as ri
+
+ src = inspect.getsource(ri.get_status)
+ # The GGUF branch returns first, so the last occurrence is the non-GGUF return.
+ non_gguf = src[src.rindex("Non-GGUF: classify from the loaded template") :]
+ for wire, stamped in (
+ ("requested_context_length", "max_seq_length_requested"),
+ ("load_in_4bit", "load_in_4bit_requested"),
+ ("requested_gpu_ids", "gpu_ids_requested"),
+ ):
+ assert (
+ f'{wire} = model_info.get("{stamped}")' in non_gguf
+ ), f"non-GGUF status does not publish {wire}"
diff --git a/studio/frontend/src/features/chat/lib/resident-config-match.ts b/studio/frontend/src/features/chat/lib/resident-config-match.ts
index b393b6ef163..51f4eba2592 100644
--- a/studio/frontend/src/features/chat/lib/resident-config-match.ts
+++ b/studio/frontend/src/features/chat/lib/resident-config-match.ts
@@ -286,8 +286,8 @@ const SETTING_CHECKS: SettingCheck[] = [
// cross-model GGUF pick and as the resident context when re-picking the same one.
// Reading null as "no opinion" against a status echoing either number was a reload.
pinned: () => true,
- // A GGUF invocation field: a safetensors or MLX status never sets it, and the general
- // non-GGUF rule below is what keeps this from reading that absence as 0.
+ // A safetensors or MLX status reports this too, so the general non-GGUF rule below
+ // is what keeps this check off those models.
agrees: (c, s, standing) =>
standing.resolveContextLength(c.customContextLength ?? null) ===
(s.requested_context_length ?? 0),
diff --git a/studio/frontend/src/features/chat/lib/resolve-ctx-pin-seed.ts b/studio/frontend/src/features/chat/lib/resolve-ctx-pin-seed.ts
index 58c9b8f18ad..4eca65b9922 100644
--- a/studio/frontend/src/features/chat/lib/resolve-ctx-pin-seed.ts
+++ b/studio/frontend/src/features/chat/lib/resolve-ctx-pin-seed.ts
@@ -47,7 +47,8 @@ const CLEAR: CtxPinSeed = {
export function resolveCtxPinSeed(options: {
/** ``status.requested_context_length``; undefined on a backend that omits the field. */
incoming: number | null | undefined;
- /** ``status.is_gguf``: only a GGUF load carries an n_ctx. */
+ /** ``status.is_gguf``. A non-GGUF status reports ``incoming`` too, so this flag, not
+ * its presence, is what keeps a non-GGUF load out of the pin. */
isGguf: boolean;
/** No load of this tab's own is in flight (``!modelLoading``). */
seedLoadParams: boolean;
diff --git a/tests/studio/test_cli_attach_load_knobs.py b/tests/studio/test_cli_attach_load_knobs.py
new file mode 100644
index 00000000000..9182f7c7507
--- /dev/null
+++ b/tests/studio/test_cli_attach_load_knobs.py
@@ -0,0 +1,1114 @@
+# SPDX-License-Identifier: AGPL-3.0-only
+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
+
+
+from __future__ import annotations
+
+import pytest
+import typer
+
+import unsloth_cli.commands.start as start_cli
+
+
+BASE = "http://127.0.0.1:8888"
+KEY = "k"
+RESIDENT = {"id": "unsloth/Qwen3-8B", "loaded": True}
+
+
+class FakeServer:
+ """Serves /v1/models and /api/inference/status; records every load payload."""
+ def __init__(self, models, status):
+ self.models = models
+ self.status = status
+ self.loads = []
+ self.requests = []
+
+ def http_json(
+ self,
+ method,
+ url,
+ token,
+ payload = None,
+ timeout = 30,
+ error = None,
+ ):
+ self.requests.append((method, url))
+ if url.endswith("/v1/models"):
+ return {"data": [dict(m) for m in self.models]}
+ if url.endswith("/api/inference/status"):
+ return dict(self.status)
+ raise AssertionError(f"unexpected request: {method} {url}")
+
+ def load(self, base, key, requested, load, payload):
+ self.loads.append(payload)
+ public = start_cli._public_model_id(requested) or requested
+ if not any(m.get("id") == public for m in self.models):
+ self.models.append({"id": public, "loaded": True})
+ return {"status": "already_loaded", "model": requested}
+
+ def install(self, monkeypatch):
+ monkeypatch.setattr(start_cli, "_http_json", self.http_json)
+ monkeypatch.setattr(start_cli, "_load_model_with_progress", self.load)
+ return self
+
+
+def loads(monkeypatch):
+ calls = []
+
+ def _load(base, key, requested, load, payload):
+ calls.append(payload)
+ return {"status": "already_loaded", "model": requested}
+
+ monkeypatch.setattr(start_cli, "_loaded_models", lambda base, key: [dict(RESIDENT)])
+ monkeypatch.setattr(start_cli, "_load_model_with_progress", _load)
+ monkeypatch.setattr(start_cli, "_http_json", lambda *a, **k: {})
+ return calls
+
+
+def sent(server, index = 0):
+ """The load payload without force_reload, which the tests that care assert alone."""
+ return {k: v for k, v in server.loads[index].items() if k != "force_reload"}
+
+
+def test_context_length_without_model_reloads_the_resident_model(loads):
+ entry = start_cli._resolve_model(
+ BASE,
+ KEY,
+ None,
+ start_cli.LoadOptions(max_seq_length = 32768),
+ )
+ assert loads == [{"model_path": RESIDENT["id"], "max_seq_length": 32768}]
+ assert entry["id"] == RESIDENT["id"]
+
+
+def test_gguf_variant_without_model_is_forwarded(loads):
+ start_cli._resolve_model(
+ BASE,
+ KEY,
+ None,
+ start_cli.LoadOptions(gguf_variant = "UD-Q8_K_XL", max_seq_length = 32768),
+ )
+ assert loads == [
+ {
+ "model_path": RESIDENT["id"],
+ "gguf_variant": "UD-Q8_K_XL",
+ "max_seq_length": 32768,
+ }
+ ]
+
+
+def test_bare_attach_does_not_load(loads):
+ entry = start_cli._resolve_model(BASE, KEY, None, start_cli.LoadOptions())
+ assert loads == []
+ assert entry["id"] == RESIDENT["id"]
+
+
+def test_bare_attach_does_not_query_status(monkeypatch):
+ """A bare attach must not even ask for status."""
+ server = FakeServer(
+ [dict(RESIDENT)],
+ {"is_gguf": False, "active_model": RESIDENT["id"], "model_identifier": RESIDENT["id"]},
+ ).install(monkeypatch)
+
+ entry = start_cli._resolve_model(BASE, KEY, None, start_cli.LoadOptions())
+
+ assert server.loads == []
+ assert entry["id"] == RESIDENT["id"]
+ assert not any(url.endswith("/api/inference/status") for _, url in server.requests)
+
+
+def test_path_loaded_resident_is_reloaded_by_its_real_path(monkeypatch):
+ """The load carries the identifier from status, not the advertised basename."""
+ path = "/srv/models/Foo-Q4_K_M.gguf"
+ server = FakeServer(
+ [{"id": "Foo-Q4_K_M", "loaded": True}],
+ {
+ "is_gguf": True,
+ "active_model": "Foo-Q4_K_M",
+ "model_identifier": path,
+ "gguf_variant": "Q4_K_M",
+ "requested_context_length": 4096,
+ },
+ ).install(monkeypatch)
+
+ entry = start_cli._resolve_model(BASE, KEY, None, start_cli.LoadOptions(max_seq_length = 32768))
+
+ assert sent(server) == {"model_path": path, "max_seq_length": 32768}
+ assert entry["id"] == "Foo-Q4_K_M"
+
+
+def test_inferred_target_still_runs_the_preload_check(monkeypatch):
+ """preload_check still runs on an inferred target."""
+ server = FakeServer(
+ [dict(RESIDENT)],
+ {"is_gguf": False, "active_model": RESIDENT["id"], "model_identifier": RESIDENT["id"]},
+ ).install(monkeypatch)
+
+ def gate(
+ base,
+ key,
+ model,
+ variant = None,
+ ):
+ raise typer.Exit(code = 1)
+
+ with pytest.raises(typer.Exit):
+ start_cli._resolve_model(
+ BASE,
+ KEY,
+ None,
+ start_cli.LoadOptions(max_seq_length = 4096),
+ preload_check = gate,
+ )
+
+ assert server.loads == []
+
+
+def test_inferred_reload_warns_that_it_unloads_for_every_session(monkeypatch, capsys):
+ """A settings change is announced as an unload."""
+ server = FakeServer(
+ [dict(RESIDENT)],
+ {
+ "is_gguf": True,
+ "active_model": RESIDENT["id"],
+ "model_identifier": RESIDENT["id"],
+ "gguf_variant": "Q4_K_M",
+ "requested_context_length": 4096,
+ },
+ ).install(monkeypatch)
+
+ start_cli._resolve_model(BASE, KEY, None, start_cli.LoadOptions(max_seq_length = 32768))
+
+ assert len(server.loads) == 1
+ assert "unloads the current model for every attached session" in capsys.readouterr().out
+
+
+def test_active_model_decides_the_resident_not_list_order(monkeypatch):
+ """active_model, not catalog order, names the resident."""
+ server = FakeServer(
+ [
+ {"id": "unsloth/whisper-large", "loaded": True},
+ {"id": RESIDENT["id"], "loaded": True},
+ ],
+ {"is_gguf": False, "active_model": RESIDENT["id"], "model_identifier": RESIDENT["id"]},
+ ).install(monkeypatch)
+
+ start_cli._resolve_model(BASE, KEY, None, start_cli.LoadOptions(max_seq_length = 32768))
+
+ assert sent(server) == {"model_path": RESIDENT["id"], "max_seq_length": 32768}
+
+
+def test_unreloadable_resident_fails_before_loading(monkeypatch):
+ """A redacted model_identifier is refused, not guessed from the basename."""
+ server = FakeServer(
+ [{"id": "Foo-Q4_K_M", "loaded": True}],
+ {"is_gguf": True, "active_model": "Foo-Q4_K_M", "model_identifier": None},
+ ).install(monkeypatch)
+
+ with pytest.raises(typer.Exit):
+ start_cli._resolve_model(BASE, KEY, None, start_cli.LoadOptions(max_seq_length = 32768))
+
+ assert server.loads == []
+
+
+def test_explicit_flags_matching_defaults_still_reload(monkeypatch):
+ """A flag typed at its default value is still sent."""
+ server = FakeServer(
+ [dict(RESIDENT)],
+ {
+ "is_gguf": True,
+ "active_model": RESIDENT["id"],
+ "model_identifier": RESIDENT["id"],
+ "requested_context_length": 32768,
+ "tensor_parallel": True,
+ },
+ ).install(monkeypatch)
+
+ start_cli._resolve_model(
+ BASE,
+ KEY,
+ None,
+ start_cli.LoadOptions(
+ max_seq_length = 0,
+ tensor_parallel = False,
+ supplied = frozenset({"max_seq_length", "tensor_parallel"}),
+ ),
+ )
+
+ assert sent(server) == {
+ "model_path": RESIDENT["id"],
+ "max_seq_length": 0,
+ "tensor_parallel": False,
+ }
+
+
+def test_inferred_attach_pins_the_resident_quant(monkeypatch):
+ """A repo-id GGUF is re-sent with the quant it is running."""
+ server = FakeServer(
+ [{"id": "unsloth/Qwen3-30B-A3B-GGUF", "loaded": True}],
+ {
+ "is_gguf": True,
+ "active_model": "unsloth/Qwen3-30B-A3B-GGUF",
+ "model_identifier": "unsloth/Qwen3-30B-A3B-GGUF",
+ "gguf_variant": "Q8_0",
+ "requested_context_length": 0,
+ },
+ ).install(monkeypatch)
+
+ start_cli._resolve_model(
+ BASE,
+ KEY,
+ None,
+ start_cli.LoadOptions(max_seq_length = 8192, supplied = frozenset({"max_seq_length"})),
+ )
+
+ assert sent(server) == {
+ "model_path": "unsloth/Qwen3-30B-A3B-GGUF",
+ "gguf_variant": "Q8_0",
+ "max_seq_length": 8192,
+ }
+
+
+def test_inferred_attach_at_the_default_context_does_not_reresolve_the_quant(monkeypatch):
+ """The --context-length 0 reset still names the running quant."""
+ server = FakeServer(
+ [{"id": "unsloth/Qwen3-30B-A3B-GGUF", "loaded": True}],
+ {
+ "is_gguf": True,
+ "active_model": "unsloth/Qwen3-30B-A3B-GGUF",
+ "model_identifier": "unsloth/Qwen3-30B-A3B-GGUF",
+ "gguf_variant": "Q8_0",
+ "requested_context_length": 0,
+ },
+ ).install(monkeypatch)
+
+ start_cli._resolve_model(
+ BASE,
+ KEY,
+ None,
+ start_cli.LoadOptions(max_seq_length = 0, supplied = frozenset({"max_seq_length"})),
+ )
+
+ assert sent(server) == {
+ "model_path": "unsloth/Qwen3-30B-A3B-GGUF",
+ "gguf_variant": "Q8_0",
+ "max_seq_length": 0,
+ }
+
+
+def test_inferred_attach_does_not_pin_a_variant_onto_a_direct_gguf_file(monkeypatch):
+ """A direct .gguf path is sent with no variant."""
+ path = "/srv/models/Foo-Q4_K_M.gguf"
+ server = FakeServer(
+ [{"id": "Foo-Q4_K_M", "loaded": True}],
+ {
+ "is_gguf": True,
+ "active_model": "Foo-Q4_K_M",
+ "model_identifier": path,
+ "gguf_variant": "Q4_K_M",
+ "requested_context_length": 4096,
+ },
+ ).install(monkeypatch)
+
+ start_cli._resolve_model(BASE, KEY, None, start_cli.LoadOptions(max_seq_length = 32768))
+
+ assert sent(server) == {"model_path": path, "max_seq_length": 32768}
+
+
+def test_inferred_attach_to_a_non_gguf_resident_sends_no_variant(monkeypatch):
+ """A non-GGUF resident is sent no variant."""
+ server = FakeServer(
+ [dict(RESIDENT)],
+ {
+ "is_gguf": False,
+ "active_model": RESIDENT["id"],
+ "model_identifier": RESIDENT["id"],
+ "requested_context_length": 4096,
+ },
+ ).install(monkeypatch)
+
+ start_cli._resolve_model(BASE, KEY, None, start_cli.LoadOptions(max_seq_length = 32768))
+
+ assert sent(server) == {"model_path": RESIDENT["id"], "max_seq_length": 32768}
+
+
+def test_hf_cache_resident_matches_the_advertised_repo_id(monkeypatch):
+ """A cache-path resident resolves to the repo id the server advertises."""
+ cache_path = (
+ "/home/u/.cache/huggingface/hub/models--unsloth--Qwen3-8B-GGUF"
+ "/snapshots/abc123/qwen3-8b-Q4_K_M.gguf"
+ )
+ server = FakeServer(
+ [{"id": "unsloth/Qwen3-8B-GGUF", "loaded": True}],
+ {
+ "is_gguf": True,
+ "active_model": "unsloth/Qwen3-8B-GGUF",
+ "model_identifier": cache_path,
+ "requested_context_length": 4096,
+ },
+ ).install(monkeypatch)
+
+ entry = start_cli._resolve_model(BASE, KEY, None, start_cli.LoadOptions(max_seq_length = 32768))
+
+ assert sent(server) == {"model_path": cache_path, "max_seq_length": 32768}
+ assert entry["id"] == "unsloth/Qwen3-8B-GGUF"
+
+
+def test_status_names_the_resident_even_when_the_catalog_lags(monkeypatch):
+ """A status id absent from /v1/models still names the resident."""
+ server = FakeServer(
+ [{"id": "unsloth/whisper-large", "loaded": True}],
+ {
+ "is_gguf": False,
+ "active_model": RESIDENT["id"],
+ "model_identifier": RESIDENT["id"],
+ },
+ ).install(monkeypatch)
+
+ entry = start_cli._resolve_model(BASE, KEY, None, start_cli.LoadOptions(max_seq_length = 32768))
+
+ assert sent(server) == {"model_path": RESIDENT["id"], "max_seq_length": 32768}
+ assert entry["id"] == RESIDENT["id"]
+
+
+def test_a_freshly_started_server_is_not_reloaded(monkeypatch):
+ """An auto-started server passes infer_resident False."""
+ server = FakeServer(
+ [dict(RESIDENT)],
+ {"is_gguf": False, "active_model": RESIDENT["id"], "model_identifier": RESIDENT["id"]},
+ ).install(monkeypatch)
+
+ entry = start_cli._resolve_model(
+ BASE,
+ KEY,
+ None,
+ start_cli.LoadOptions(max_seq_length = 32768),
+ infer_resident = False,
+ )
+
+ assert server.loads == []
+ assert entry["id"] == RESIDENT["id"]
+
+
+def test_omitted_default_flags_are_not_forwarded(monkeypatch):
+ """A bare --model load sends only model_path."""
+ server = FakeServer(
+ [dict(RESIDENT)],
+ {"is_gguf": False, "active_model": RESIDENT["id"], "model_identifier": RESIDENT["id"]},
+ ).install(monkeypatch)
+
+ start_cli._resolve_model(BASE, KEY, "unsloth/Qwen3-14B", start_cli.LoadOptions())
+
+ assert sent(server) == {"model_path": "unsloth/Qwen3-14B"}
+
+
+class TestExplicitFlagsThroughTheRealCli:
+ """`supplied` tracking through the real Typer and Click stack."""
+ @staticmethod
+ def _load_for(argv):
+ from typer.testing import CliRunner
+
+ captured = {}
+ real_connect = start_cli._connect
+
+ def fake_connect(api_key, model, load, *a, **k):
+ captured["load"] = load
+ raise SystemExit(0)
+
+ start_cli._connect = fake_connect
+ try:
+ CliRunner().invoke(start_cli.start_app, argv)
+ finally:
+ start_cli._connect = real_connect
+ return captured.get("load")
+
+ @pytest.mark.parametrize(
+ "flag, expected",
+ [
+ (["--context-length", "0"], "max_seq_length"),
+ (["--load-in-4bit"], "load_in_4bit"),
+ (["--no-tensor-parallel"], "tensor_parallel"),
+ (["--gpu-memory-mode", "auto"], "gpu_memory_mode"),
+ ],
+ )
+ def test_flags_equal_to_their_default_are_recorded(self, flag, expected):
+ load = self._load_for(["codex", "--no-launch", *flag])
+ assert load is not None, "the command never reached _connect"
+ assert expected in load.supplied
+ assert expected in load.overrides()
+
+ def test_a_bare_invocation_records_nothing(self):
+ load = self._load_for(["codex", "--no-launch"])
+ assert load is not None
+ assert load.supplied == frozenset()
+ assert load.overrides() == frozenset()
+
+ @pytest.mark.parametrize("command", ["codex", "claude", "opencode", "hermes", "pi", "openclaw"])
+ def test_every_agent_command_tracks_flags_identically(self, command):
+ load = self._load_for([command, "--no-launch", "--context-length", "0"])
+ assert load is not None, f"{command} never reached _connect"
+ assert "max_seq_length" in load.supplied
+
+
+def test_inferred_reload_carries_the_resident_runtime_settings(monkeypatch):
+ """Knobs the user did not name are carried over from the resident."""
+ server = FakeServer(
+ [dict(RESIDENT)],
+ {
+ "is_gguf": True,
+ "active_model": RESIDENT["id"],
+ "model_identifier": RESIDENT["id"],
+ "gguf_variant": "Q8_0",
+ "requested_context_length": 4096,
+ "cache_type_kv": "q8_0",
+ "requested_parallel_slots": 4,
+ "requested_n_batch": 1024,
+ "requested_llama_extra_args": ["--foo"],
+ "tensor_split": [0.5, 0.5],
+ "requested_load_mode": None,
+ },
+ ).install(monkeypatch)
+
+ start_cli._resolve_model(BASE, KEY, None, start_cli.LoadOptions(max_seq_length = 32768))
+
+ sent = server.loads[0]
+ assert sent["cache_type_kv"] == "q8_0"
+ assert sent["n_parallel"] == 4
+ assert sent["n_batch"] == 1024
+ assert sent["llama_extra_args"] == ["--foo"]
+ assert sent["tensor_split"] == [0.5, 0.5]
+ assert "load_mode" not in sent
+
+
+def test_user_supplied_knobs_beat_the_resident_values(monkeypatch):
+ server = FakeServer(
+ [dict(RESIDENT)],
+ {
+ "is_gguf": True,
+ "active_model": RESIDENT["id"],
+ "model_identifier": RESIDENT["id"],
+ "requested_context_length": 4096,
+ "tensor_parallel": True,
+ },
+ ).install(monkeypatch)
+
+ start_cli._resolve_model(
+ BASE,
+ KEY,
+ None,
+ start_cli.LoadOptions(tensor_parallel = False, supplied = frozenset({"tensor_parallel"})),
+ )
+
+ assert server.loads[0]["tensor_parallel"] is False
+
+
+def test_explicit_zero_context_forces_the_reload(monkeypatch):
+ """An explicit 0 is force_reloaded."""
+ server = FakeServer(
+ [dict(RESIDENT)],
+ {
+ "is_gguf": False,
+ "active_model": RESIDENT["id"],
+ "model_identifier": RESIDENT["id"],
+ "requested_context_length": 32768,
+ },
+ ).install(monkeypatch)
+
+ start_cli._resolve_model(
+ BASE,
+ KEY,
+ None,
+ start_cli.LoadOptions(max_seq_length = 0, supplied = frozenset({"max_seq_length"})),
+ )
+
+ assert server.loads[0]["max_seq_length"] == 0
+ assert server.loads[0]["force_reload"] is True
+
+
+def test_a_provable_no_op_is_not_forced(monkeypatch):
+ server = FakeServer(
+ [dict(RESIDENT)],
+ {
+ "is_gguf": False,
+ "active_model": RESIDENT["id"],
+ "model_identifier": RESIDENT["id"],
+ "requested_context_length": 32768,
+ },
+ ).install(monkeypatch)
+
+ start_cli._resolve_model(BASE, KEY, None, start_cli.LoadOptions(max_seq_length = 32768))
+
+ assert "force_reload" not in server.loads[0]
+
+
+def test_an_older_server_is_never_force_reloaded(monkeypatch):
+ """No status means no proof, so no force_reload."""
+ server = FakeServer([dict(RESIDENT)], {}).install(monkeypatch)
+
+ start_cli._resolve_model(BASE, KEY, None, start_cli.LoadOptions(max_seq_length = 32768))
+
+ assert "force_reload" not in server.loads[0]
+
+
+def test_four_bit_flag_does_not_warn_about_a_gguf_resident(monkeypatch, capsys):
+ """A null load_in_4bit on GGUF is not a difference."""
+ server = FakeServer(
+ [dict(RESIDENT)],
+ {
+ "is_gguf": True,
+ "active_model": RESIDENT["id"],
+ "model_identifier": RESIDENT["id"],
+ "gguf_variant": "Q8_0",
+ "requested_context_length": 4096,
+ "load_in_4bit": None,
+ },
+ ).install(monkeypatch)
+
+ start_cli._resolve_model(
+ BASE,
+ KEY,
+ None,
+ start_cli.LoadOptions(load_in_4bit = True, supplied = frozenset({"load_in_4bit"})),
+ )
+
+ assert "unloads the current model" not in capsys.readouterr().out
+
+
+def test_status_reporting_no_chat_resident_does_not_pick_a_speech_sidecar(monkeypatch):
+ """A null active_model is refused, not resolved from the catalog."""
+ server = FakeServer(
+ [{"id": "unsloth/whisper-large-v3", "loaded": True}],
+ {"is_gguf": False, "active_model": None, "model_identifier": None},
+ ).install(monkeypatch)
+
+ with pytest.raises(typer.Exit):
+ start_cli._resolve_model(BASE, KEY, None, start_cli.LoadOptions(max_seq_length = 32768))
+
+ assert server.loads == []
+
+
+def test_failed_inferred_load_names_the_inferred_resident(monkeypatch, capsys):
+ """The survivor probe uses the inferred target."""
+ server = FakeServer(
+ [
+ {"id": "unsloth/whisper-large-v3", "loaded": True},
+ {"id": RESIDENT["id"], "loaded": True},
+ ],
+ {
+ "is_gguf": True,
+ "active_model": RESIDENT["id"],
+ "model_identifier": RESIDENT["id"],
+ "gguf_variant": "Q8_0",
+ "requested_context_length": 4096,
+ },
+ ).install(monkeypatch)
+
+ def boom(*a, **k):
+ raise RuntimeError("load refused")
+
+ monkeypatch.setattr(start_cli, "_load_model_with_progress", boom)
+ monkeypatch.setattr(start_cli, "_model_still_loaded", lambda *a: False)
+
+ with pytest.raises(RuntimeError):
+ start_cli._resolve_model(BASE, KEY, None, start_cli.LoadOptions(max_seq_length = 32768))
+
+ assert "Nothing was unloaded" not in capsys.readouterr().err
+
+
+def test_inferred_reload_keeps_a_full_precision_resident(monkeypatch):
+ """A full-precision resident comes back at the same precision."""
+ server = FakeServer(
+ [dict(RESIDENT)],
+ {
+ "is_gguf": False,
+ "active_model": RESIDENT["id"],
+ "model_identifier": RESIDENT["id"],
+ "requested_context_length": 4096,
+ "load_in_4bit": False,
+ },
+ ).install(monkeypatch)
+
+ start_cli._resolve_model(BASE, KEY, None, start_cli.LoadOptions(max_seq_length = 32768))
+
+ assert server.loads[0]["load_in_4bit"] is False
+
+
+def test_an_explicit_precision_flag_beats_the_resident(monkeypatch):
+ server = FakeServer(
+ [dict(RESIDENT)],
+ {
+ "is_gguf": False,
+ "active_model": RESIDENT["id"],
+ "model_identifier": RESIDENT["id"],
+ "requested_context_length": 4096,
+ "load_in_4bit": False,
+ },
+ ).install(monkeypatch)
+
+ start_cli._resolve_model(
+ BASE,
+ KEY,
+ None,
+ start_cli.LoadOptions(load_in_4bit = True, supplied = frozenset({"load_in_4bit"})),
+ )
+
+ assert server.loads[0]["load_in_4bit"] is True
+
+
+def test_a_gguf_resident_is_sent_no_precision_flag(monkeypatch):
+ """A GGUF resident is sent no load_in_4bit."""
+ server = FakeServer(
+ [dict(RESIDENT)],
+ {
+ "is_gguf": True,
+ "active_model": RESIDENT["id"],
+ "model_identifier": RESIDENT["id"],
+ "gguf_variant": "Q8_0",
+ "requested_context_length": 4096,
+ "load_in_4bit": None,
+ },
+ ).install(monkeypatch)
+
+ start_cli._resolve_model(BASE, KEY, None, start_cli.LoadOptions(max_seq_length = 32768))
+
+ assert "load_in_4bit" not in server.loads[0]
+
+
+def _gguf_status(**extra):
+ base = {
+ "is_gguf": True,
+ "active_model": RESIDENT["id"],
+ "model_identifier": RESIDENT["id"],
+ "gguf_variant": "Q4_K_M",
+ "requested_context_length": 8192,
+ }
+ base.update(extra)
+ return base
+
+
+def test_changing_one_knob_keeps_the_custom_context(monkeypatch):
+ """A custom context survives a change to another knob."""
+ server = FakeServer([dict(RESIDENT)], _gguf_status()).install(monkeypatch)
+
+ start_cli._resolve_model(
+ BASE,
+ KEY,
+ None,
+ start_cli.LoadOptions(tensor_parallel = True, supplied = frozenset({"tensor_parallel"})),
+ )
+
+ assert server.loads[0]["max_seq_length"] == 8192
+
+
+def test_remote_code_resident_is_refused_before_the_load(monkeypatch):
+ """A trust_remote_code resident is refused before anything is evicted."""
+ server = FakeServer(
+ [dict(RESIDENT)],
+ {
+ "is_gguf": False,
+ "active_model": RESIDENT["id"],
+ "model_identifier": RESIDENT["id"],
+ "requested_context_length": 4096,
+ "requires_trust_remote_code": True,
+ },
+ ).install(monkeypatch)
+
+ with pytest.raises(typer.Exit):
+ start_cli._resolve_model(BASE, KEY, None, start_cli.LoadOptions(max_seq_length = 32768))
+
+ assert server.loads == []
+
+
+def test_tensor_parallel_does_not_restart_a_non_gguf_resident(monkeypatch, capsys):
+ """tensor_parallel is GGUF-only."""
+ server = FakeServer(
+ [dict(RESIDENT)],
+ {
+ "is_gguf": False,
+ "active_model": RESIDENT["id"],
+ "model_identifier": RESIDENT["id"],
+ "requested_context_length": 4096,
+ "tensor_parallel": False,
+ },
+ ).install(monkeypatch)
+
+ start_cli._resolve_model(
+ BASE,
+ KEY,
+ None,
+ start_cli.LoadOptions(tensor_parallel = True, supplied = frozenset({"tensor_parallel"})),
+ )
+
+ assert "force_reload" not in server.loads[0]
+ assert "unloads the current model" not in capsys.readouterr().out
+
+
+def test_a_differently_spelled_quant_still_counts_as_a_change(monkeypatch, capsys):
+ """Q4KM and Q4_K_M are compared as typed."""
+ server = FakeServer([dict(RESIDENT)], _gguf_status()).install(monkeypatch)
+
+ start_cli._resolve_model(BASE, KEY, None, start_cli.LoadOptions(gguf_variant = "Q4KM"))
+
+ assert server.loads[0]["force_reload"] is True
+ assert "unloads the current model" in capsys.readouterr().out
+
+
+def test_manual_mode_keeps_a_pinned_layer_count(monkeypatch, capsys):
+ """A resident already in manual keeps its pinned layer count."""
+ server = FakeServer(
+ [dict(RESIDENT)],
+ _gguf_status(gpu_memory_mode = "manual", gpu_layers = 20),
+ ).install(monkeypatch)
+
+ start_cli._resolve_model(
+ BASE,
+ KEY,
+ None,
+ start_cli.LoadOptions(gpu_memory_mode = "manual", supplied = frozenset({"gpu_memory_mode"})),
+ )
+
+ assert server.loads[0]["gpu_layers"] == 20
+ assert "force_reload" not in server.loads[0]
+ assert "unloads the current model" not in capsys.readouterr().out
+
+
+def test_switching_into_manual_still_asks_for_automatic_layers(monkeypatch):
+ server = FakeServer([dict(RESIDENT)], _gguf_status(gpu_memory_mode = "auto")).install(monkeypatch)
+
+ start_cli._resolve_model(
+ BASE,
+ KEY,
+ None,
+ start_cli.LoadOptions(gpu_memory_mode = "manual", supplied = frozenset({"gpu_memory_mode"})),
+ )
+
+ assert server.loads[0]["gpu_layers"] == -1
+
+
+def test_arch_gated_tensor_request_is_not_restarted(monkeypatch):
+ """Re-asking for an arch-gated tensor request is a no-op."""
+ server = FakeServer(
+ [dict(RESIDENT)],
+ _gguf_status(tensor_parallel = False, tensor_parallel_dropped_by_arch_gate = True),
+ ).install(monkeypatch)
+
+ start_cli._resolve_model(
+ BASE,
+ KEY,
+ None,
+ start_cli.LoadOptions(tensor_parallel = True, supplied = frozenset({"tensor_parallel"})),
+ )
+
+ assert "force_reload" not in server.loads[0]
+
+
+def test_paravirtual_placement_is_not_restarted(monkeypatch):
+ """Placement cannot differ on a paravirtual host."""
+ server = FakeServer(
+ [dict(RESIDENT)],
+ _gguf_status(gpu_memory_mode = "manual", gpu_placement_paravirtual = True),
+ ).install(monkeypatch)
+
+ start_cli._resolve_model(
+ BASE,
+ KEY,
+ None,
+ start_cli.LoadOptions(gpu_memory_mode = "auto", supplied = frozenset({"gpu_memory_mode"})),
+ )
+
+ assert "force_reload" not in server.loads[0]
+
+
+def test_a_no_op_attach_to_a_custom_code_resident_is_allowed(monkeypatch):
+ """The refusal exists to protect a reload; a no-op has no reload to protect."""
+ server = FakeServer(
+ [dict(RESIDENT)],
+ {
+ "is_gguf": False,
+ "active_model": RESIDENT["id"],
+ "model_identifier": RESIDENT["id"],
+ "requested_context_length": 4096,
+ "load_in_4bit": True,
+ "requires_trust_remote_code": True,
+ },
+ ).install(monkeypatch)
+
+ entry = start_cli._resolve_model(BASE, KEY, None, start_cli.LoadOptions(max_seq_length = 4096))
+
+ assert "force_reload" not in server.loads[0]
+ assert entry["id"] == RESIDENT["id"]
+
+
+def test_cpu_fallback_placement_is_not_restarted(monkeypatch):
+ """Placement cannot differ under a CPU fallback."""
+ server = FakeServer(
+ [dict(RESIDENT)],
+ _gguf_status(
+ gpu_memory_mode = "manual",
+ gpu_layers = 0,
+ cpu_fallback_reason = "vulkan_startup_crash",
+ ),
+ ).install(monkeypatch)
+
+ start_cli._resolve_model(
+ BASE,
+ KEY,
+ None,
+ start_cli.LoadOptions(gpu_memory_mode = "auto", supplied = frozenset({"gpu_memory_mode"})),
+ )
+
+ assert "force_reload" not in server.loads[0]
+
+
+def test_the_requested_mlx_kv_width_survives_a_reload(monkeypatch):
+ """The requested MLX KV width round-trips, not the refused applied one."""
+ server = FakeServer(
+ [dict(RESIDENT)],
+ {
+ "is_gguf": False,
+ "active_model": RESIDENT["id"],
+ "model_identifier": RESIDENT["id"],
+ "requested_context_length": 4096,
+ "load_in_4bit": True,
+ "mlx_kv_bits": None,
+ "mlx_kv_bits_requested": 4,
+ },
+ ).install(monkeypatch)
+
+ start_cli._resolve_model(BASE, KEY, None, start_cli.LoadOptions(max_seq_length = 32768))
+
+ assert server.loads[0]["mlx_kv_bits"] == 4
+
+
+def test_a_proven_no_op_skips_the_preload_gate(monkeypatch):
+ """A proven no-op skips the gate."""
+ server = FakeServer([dict(RESIDENT)], _gguf_status()).install(monkeypatch)
+ checked = []
+
+ start_cli._resolve_model(
+ BASE,
+ KEY,
+ None,
+ start_cli.LoadOptions(max_seq_length = 8192),
+ preload_check = lambda *a: checked.append(a),
+ )
+
+ assert checked == []
+
+
+def test_a_real_change_still_runs_the_preload_gate(monkeypatch):
+ server = FakeServer([dict(RESIDENT)], _gguf_status()).install(monkeypatch)
+
+ def gate(
+ base,
+ key,
+ model,
+ variant = None,
+ ):
+ raise typer.Exit(code = 1)
+
+ with pytest.raises(typer.Exit):
+ start_cli._resolve_model(
+ BASE, KEY, None, start_cli.LoadOptions(max_seq_length = 32768), preload_check = gate
+ )
+
+ assert server.loads == []
+
+
+def test_a_quant_override_on_a_direct_file_is_refused(monkeypatch):
+ """A different quant on a direct .gguf file is refused."""
+ path = "/srv/models/Foo-Q4_K_M.gguf"
+ server = FakeServer(
+ [{"id": "Foo-Q4_K_M", "loaded": True}],
+ {
+ "is_gguf": True,
+ "active_model": "Foo-Q4_K_M",
+ "model_identifier": path,
+ "gguf_variant": "Q4_K_M",
+ "requested_context_length": 4096,
+ },
+ ).install(monkeypatch)
+
+ with pytest.raises(typer.Exit):
+ start_cli._resolve_model(BASE, KEY, None, start_cli.LoadOptions(gguf_variant = "UD-Q8_K_XL"))
+
+ assert server.loads == []
+
+
+def test_explicit_tensor_disable_clears_an_arch_gated_fallback(monkeypatch, capsys):
+ """Turning tensor_parallel off clears an arch-gated fallback."""
+ server = FakeServer(
+ [dict(RESIDENT)],
+ _gguf_status(tensor_parallel = False, tensor_parallel_dropped_by_arch_gate = True),
+ ).install(monkeypatch)
+
+ start_cli._resolve_model(
+ BASE,
+ KEY,
+ None,
+ start_cli.LoadOptions(tensor_parallel = False, supplied = frozenset({"tensor_parallel"})),
+ )
+
+ assert server.loads[0]["force_reload"] is True
+ assert "unloads the current model" in capsys.readouterr().out
+
+
+def test_switching_into_manual_is_still_a_change(monkeypatch, capsys):
+ """auto to manual is a real change."""
+ server = FakeServer([dict(RESIDENT)], _gguf_status(gpu_memory_mode = "auto")).install(monkeypatch)
+
+ start_cli._resolve_model(
+ BASE,
+ KEY,
+ None,
+ start_cli.LoadOptions(gpu_memory_mode = "manual", supplied = frozenset({"gpu_memory_mode"})),
+ )
+
+ assert server.loads[0]["force_reload"] is True
+ assert "unloads the current model" in capsys.readouterr().out
+
+
+def test_no_status_and_one_loaded_model_still_works(monkeypatch):
+ """One loaded model is unambiguous without status."""
+ server = FakeServer([dict(RESIDENT)], {}).install(monkeypatch)
+
+ start_cli._resolve_model(BASE, KEY, None, start_cli.LoadOptions(max_seq_length = 32768))
+
+ assert server.loads[0]["model_path"] == RESIDENT["id"]
+
+
+def test_no_status_and_several_loaded_models_refuses_to_guess(monkeypatch):
+ """Several loaded models without status is refused."""
+ server = FakeServer(
+ [
+ {"id": "unsloth/whisper-large-v3", "loaded": True},
+ dict(RESIDENT),
+ ],
+ {},
+ ).install(monkeypatch)
+
+ with pytest.raises(typer.Exit):
+ start_cli._resolve_model(BASE, KEY, None, start_cli.LoadOptions(max_seq_length = 32768))
+
+ assert server.loads == []
+
+
+def test_non_gguf_gpu_selection_survives_a_reload(monkeypatch):
+ """GPU placement is carried over."""
+ server = FakeServer(
+ [dict(RESIDENT)],
+ {
+ "is_gguf": False,
+ "active_model": RESIDENT["id"],
+ "model_identifier": RESIDENT["id"],
+ "requested_context_length": 4096,
+ "load_in_4bit": True,
+ "requested_gpu_ids": [2, 3],
+ },
+ ).install(monkeypatch)
+
+ start_cli._resolve_model(BASE, KEY, None, start_cli.LoadOptions(max_seq_length = 32768))
+
+ assert server.loads[0]["gpu_ids"] == [2, 3]
+
+
+def _direct_gguf_server(monkeypatch, path = "/srv/models/Foo-Q4_K_M.gguf"):
+ return FakeServer(
+ [{"id": "Foo-Q4_K_M", "loaded": True}],
+ {
+ "is_gguf": True,
+ "active_model": "Foo-Q4_K_M",
+ "model_identifier": path,
+ "gguf_variant": "Q4_K_M",
+ "requested_context_length": 4096,
+ },
+ ).install(monkeypatch)
+
+
+def test_restating_the_running_quant_applies_the_other_overrides(monkeypatch):
+ """Restating the running quant does not block the other overrides."""
+ path = "/srv/models/Foo-Q4_K_M.gguf"
+ server = _direct_gguf_server(monkeypatch, path)
+
+ start_cli._resolve_model(
+ BASE,
+ KEY,
+ None,
+ start_cli.LoadOptions(gguf_variant = "Q4_K_M", max_seq_length = 32768),
+ )
+
+ sent_payload = server.loads[0]
+ assert sent_payload["model_path"] == path
+ assert sent_payload["max_seq_length"] == 32768
+ assert "gguf_variant" not in sent_payload
+
+
+def test_a_matching_quant_is_compared_case_insensitively(monkeypatch):
+ server = _direct_gguf_server(monkeypatch)
+
+ start_cli._resolve_model(
+ BASE,
+ KEY,
+ None,
+ start_cli.LoadOptions(gguf_variant = "q4_k_m", max_seq_length = 32768),
+ )
+
+ assert server.loads[0]["max_seq_length"] == 32768
+
+
+def test_a_differing_quant_on_a_direct_file_is_still_refused(monkeypatch):
+ server = _direct_gguf_server(monkeypatch)
+
+ with pytest.raises(typer.Exit):
+ start_cli._resolve_model(
+ BASE,
+ KEY,
+ None,
+ start_cli.LoadOptions(gguf_variant = "UD-Q8_K_XL", max_seq_length = 32768),
+ )
+
+ assert server.loads == []
+
+
+def test_a_diffusion_resident_is_never_an_attach_target(monkeypatch):
+ """An image runtime answers with an active_model but can never serve chat."""
+ server = FakeServer(
+ [{"id": "unsloth/FLUX.1-dev-GGUF", "loaded": True}],
+ {
+ "is_gguf": True,
+ "is_diffusion": True,
+ "active_model": "unsloth/FLUX.1-dev-GGUF",
+ "model_identifier": "unsloth/FLUX.1-dev-GGUF",
+ "requested_context_length": 4096,
+ },
+ ).install(monkeypatch)
+
+ with pytest.raises(typer.Exit):
+ start_cli._resolve_model(BASE, KEY, None, start_cli.LoadOptions(max_seq_length = 32768))
+
+ assert server.loads == []
+
+
+ "field",
+ ["spec_probe_retry_pending", "spec_dflash_retry_pending", "spec_fallback_binary_changed"],
+)
+def test_a_pending_retry_is_not_a_no_op(monkeypatch, capsys, field):
+ """_runtime_matches_intent reloads on an identical intent while a retry is pending."""
+ server = FakeServer(
+ [dict(RESIDENT)], _gguf_status(**{field: True})
+ ).install(monkeypatch)
+
+ start_cli._resolve_model(BASE, KEY, None, start_cli.LoadOptions(max_seq_length = 8192))
+
+ assert server.loads[0]["force_reload"] is True
+ assert "unloads the current model" in capsys.readouterr().out
+
+
+def test_no_pending_retry_is_still_a_no_op(monkeypatch):
+ server = FakeServer([dict(RESIDENT)], _gguf_status()).install(monkeypatch)
+
+ start_cli._resolve_model(BASE, KEY, None, start_cli.LoadOptions(max_seq_length = 8192))
+
+ assert "force_reload" not in server.loads[0]
diff --git a/unsloth_cli/commands/start.py b/unsloth_cli/commands/start.py
index 616ff5c7789..013ce3f7a8a 100644
--- a/unsloth_cli/commands/start.py
+++ b/unsloth_cli/commands/start.py
@@ -600,6 +600,71 @@ class LoadOptions(NamedTuple):
load_in_4bit: bool = True
tensor_parallel: bool = False
gpu_memory_mode: Optional[Literal["auto", "manual"]] = None
+ # Names the user actually typed: --context-length 0 equals the declared default yet
+ # is a reset the server must hear. Appended last to keep positional callers working.
+ supplied: frozenset = frozenset()
+
+ def overrides(self) -> frozenset:
+ """Fields that must reach the load: typed explicitly, or differing from default."""
+ differing = {
+ name
+ for name, default in (
+ ("gguf_variant", None),
+ ("max_seq_length", 0),
+ ("load_in_4bit", True),
+ ("tensor_parallel", False),
+ ("gpu_memory_mode", None),
+ )
+ if getattr(self, name) != default
+ }
+ # Internal callers never populate `supplied`, so a non-default value counts too.
+ return frozenset(differing) | frozenset(self.supplied)
+
+
+_LOAD_OPTION_PARAMS = (
+ "gguf_variant",
+ "max_seq_length",
+ "load_in_4bit",
+ "tensor_parallel",
+ "gpu_memory_mode",
+)
+
+
+def _supplied_load_params(ctx) -> frozenset:
+ """Which load knobs Click saw on the command line.
+
+ The context must be PASSED IN: Typer invokes callbacks with no active click context,
+ so click.get_current_context() is None. Unaskable -> empty set, and `overrides()`
+ falls back to comparing values.
+ """
+ getter = getattr(ctx, "get_parameter_source", None)
+ if getter is None:
+ return frozenset()
+ supplied = set()
+ for name in _LOAD_OPTION_PARAMS:
+ try:
+ source = getter(name)
+ except Exception:
+ continue
+ # By member NAME, not identity or ordering: Typer vendors its own click, so this
+ # is typer._click's ParameterSource, and click 8.3 reordered the IntEnum.
+ if getattr(source, "name", None) == "COMMANDLINE":
+ supplied.add(name)
+ return frozenset(supplied)
+
+
+def _load_options(
+ ctx, gguf_variant, max_seq_length, load_in_4bit, tensor_parallel, gpu_memory_mode
+) -> LoadOptions:
+ """Build LoadOptions for an agent command, recording what was typed."""
+ return LoadOptions(
+ gguf_variant,
+ max_seq_length,
+ load_in_4bit,
+ tensor_parallel,
+ gpu_memory_mode,
+ _supplied_load_params(ctx),
+ )
class ServerOptions(NamedTuple):
@@ -1645,12 +1710,214 @@ def _model_id_matches(
return str(actual).casefold() == str(requested).casefold()
+def _inference_status(base: str, key: str) -> dict:
+ """Runtime state of the resident model. {} means "cannot prove anything" (older
+ server), never "nothing is set"."""
+ try:
+ return _http_json("GET", f"{base}/api/inference/status", key)
+ except Exception:
+ return {}
+
+
+def _resident_load_target(models: list, status: dict, allow_casefold: bool):
+ """(identifier to post, id it is advertised as) for the running model.
+
+ /v1/models shows only the sanitized basename while _same_loaded_identifier compares
+ resident paths exactly, so the load must carry the identifier status reports.
+ """
+ if status.get("is_diffusion"):
+ # An image runtime answers with an active_model like any other, but it cannot
+ # serve chat: targeting it would tear down the diffusion server and then point
+ # the agent at a model that can never answer it.
+ _fail(
+ "Unsloth is serving an image model, which cannot serve chat, so there are no "
+ "settings to apply. Re-run with --model naming the chat model to load."
+ )
+ active_id = status.get("active_model")
+ entry = None
+ if active_id:
+ entry = next(
+ (
+ m
+ for m in models
+ if _model_id_matches(m.get("id"), active_id, allow_casefold = allow_casefold)
+ and m.get("loaded") is not False
+ ),
+ None,
+ )
+ if entry is None and not status:
+ # Only when there is no status at all (older server). A status that ANSWERED with
+ # active_model null is stating there is no chat resident.
+ # Catalog ORDER is not evidence either: /v1/models lists loaded speech sidecars
+ # too, so the first entry can be one. Answer only when the catalog is unambiguous.
+ loaded = [m for m in models if m.get("loaded") is not False]
+ if len(loaded) == 1:
+ entry = loaded[0]
+ elif loaded:
+ _fail(
+ "This Unsloth cannot say which model is serving chat, and more than one is "
+ "loaded. Re-run with --model naming the one these settings are for."
+ )
+ public_id = active_id or (entry or {}).get("id")
+ if not public_id:
+ if status:
+ # Status answered and named no chat model. Returning empty here would drop
+ # the knobs silently, which is the bug this path exists to fix.
+ _fail(
+ "No chat model is currently loaded, so there are no settings to apply. "
+ "Re-run with --model naming the model to load."
+ )
+ return None, None
+ identifier = status.get("model_identifier")
+ if identifier:
+ return identifier, public_id
+ # A native path lease redacts the internal path; inventing one from the basename
+ # would address the wrong file, so only a hub id can be posted back.
+ if _is_hub_model_id(public_id):
+ return public_id, public_id
+ _fail(
+ f"Unsloth is serving '{public_id}' from a local path it does not expose, so these "
+ "settings cannot be applied by attaching. Re-run with --model naming that path."
+ )
+
+
+# /api/inference/status field -> the /api/inference/load field that reproduces it. The
+# "requested_" values are what the load was INVOKED with, which is what has to be resent;
+# the bare names are the resolved ones and would pin a value the user never chose.
+_RESIDENT_RUNTIME_FIELDS = {
+ "cache_type_kv": "cache_type_kv",
+ "chat_template_override": "chat_template_override",
+ "disable_vision": "disable_vision",
+ "gpu_memory_mode": "gpu_memory_mode",
+ "gpu_layers": "gpu_layers",
+ "n_cpu_moe": "n_cpu_moe",
+ "tensor_split": "tensor_split",
+ "tensor_parallel": "tensor_parallel",
+ "speculative_type": "speculative_type",
+ "spec_draft_n_max": "spec_draft_n_max",
+ # The applied value is null when the runtime refused the request.
+ "mlx_kv_bits_requested": "mlx_kv_bits",
+ # LoadRequest defaults this to True, so omitting it would reload a full-precision
+ # model in 4-bit. Null on GGUF, which has no such setting.
+ "load_in_4bit": "load_in_4bit",
+ # Same trap: max_seq_length defaults to 0, which _gguf_request_intent copies into
+ # n_ctx, so changing another knob would reset a custom context to automatic.
+ "requested_context_length": "max_seq_length",
+ "requested_gpu_ids": "gpu_ids",
+ "requested_parallel_slots": "n_parallel",
+ "requested_n_batch": "n_batch",
+ "requested_n_ubatch": "n_ubatch",
+ "requested_load_mode": "load_mode",
+ "requested_ctx_checkpoints": "ctx_checkpoints",
+ "requested_cache_ram": "cache_ram",
+ "requested_spec_draft_cache_type": "spec_draft_cache_type",
+ "requested_llama_extra_args": "llama_extra_args",
+}
+
+
+def _resident_runtime_payload(status: dict, payload: dict) -> dict:
+ """The resident's own settings for knobs this load did not name.
+
+ None means "never set" for every one of these, so it is dropped rather than sent:
+ omitting a field is what lets the server inherit, while sending null would pin the
+ absence. An explicit empty list is kept, since that is a real "launch with none".
+ """
+ if not status:
+ return {}
+ carried = {}
+ for field, load_field in _RESIDENT_RUNTIME_FIELDS.items():
+ if load_field in payload:
+ continue
+ value = status.get(field)
+ if value is None:
+ continue
+ carried[load_field] = value
+ return carried
+
+
+def _load_settings_differ(status: dict, load: LoadOptions, overrides: frozenset) -> bool:
+ """Whether applying these settings can restart the resident. Unproven equality
+ counts as a difference: a silent restart is worse than a spurious warning."""
+ if not status:
+ return True
+ # _runtime_matches_intent rejects an identical intent while a spec probe, a DFlash
+ # drafter or a changed speculative binary is waiting to be retried, so the server
+ # reloads regardless of what the CLI asked for. Equality of the overrides is then no
+ # proof of a no-op, and claiming one would skip the gate and the warning.
+ if any(
+ status.get(field)
+ for field in (
+ "spec_probe_retry_pending",
+ "spec_dflash_retry_pending",
+ "spec_fallback_binary_changed",
+ )
+ ):
+ return True
+ for name in overrides:
+ if name == "gguf_variant":
+ resident = status.get("gguf_variant") if status.get("is_gguf") else None
+ # Casefold, not _normalized_variant, which strips separators: a mistyped Q4KM
+ # would read as equal here yet still really reload on the server. The preload
+ # gate below already compares this way, and the two have to agree.
+ if (
+ not resident
+ or str(resident).strip().lower() != str(load.gguf_variant).strip().lower()
+ ):
+ return True
+ elif name == "max_seq_length":
+ # Requested, not resolved: llama.cpp clamps n_ctx at fit time.
+ resident = status.get("requested_context_length")
+ if resident is None or int(resident) != int(load.max_seq_length):
+ return True
+ elif name == "load_in_4bit":
+ # GGUF has no 4-bit setting and reports null, which would read as "differs"
+ # and warn about an unload the server is not going to perform.
+ if status.get("is_gguf"):
+ continue
+ resident = status.get("load_in_4bit")
+ if resident is None or bool(resident) != bool(load.load_in_4bit):
+ return True
+ elif name == "tensor_parallel":
+ # llama.cpp only. The standard load never forwards it, so a restart would
+ # apply nothing.
+ if not status.get("is_gguf"):
+ continue
+ # The architecture gate can normalize a tensor request to layer mode and say
+ # so. Asking for it AGAIN is the request already applied, not a difference,
+ # and the backend dedupes exactly this state. Asking to turn it OFF is the
+ # opposite: the backend keeps the tensor intent behind that fallback and does
+ # not read a bare false as an explicit drop (the UI sends false routinely), so
+ # only a real reload can clear it.
+ if status.get("tensor_parallel_dropped_by_arch_gate"):
+ if load.tensor_parallel:
+ continue
+ return True
+ if bool(status.get("tensor_parallel")) != bool(load.tensor_parallel):
+ return True
+ elif name == "gpu_memory_mode":
+ if not status.get("is_gguf"):
+ continue
+ # A paravirtual host pins every placement request to the same runtime, and a
+ # CPU fallback is preserved across reloads by _preserve_cpu_fallback_intent, so
+ # in both cases the raw mode cannot tell two requests apart. resident-config-
+ # match.ts skips placement on exactly these two for the same reason.
+ if status.get("gpu_placement_paravirtual") or status.get("cpu_fallback_reason"):
+ continue
+ if status.get("gpu_memory_mode") != load.gpu_memory_mode:
+ return True
+ # Manual to manual is a real no-op: the payload only sends the implicit
+ # gpu_layers = -1 when switching INTO manual, so a resident already pinned to
+ # a layer count keeps it through the round-trip and nothing changes.
+ return False
+
+
def _resolve_model(
base: str,
key: str,
requested: Optional[str],
load: LoadOptions = LoadOptions(),
preload_check = None,
+ infer_resident: bool = True,
) -> dict:
models = _loaded_models(base, key)
load_requested = False
@@ -1663,13 +1930,22 @@ def _resolve_model(
# /api/inference/load: the server's already-loaded dedup answers "already_loaded"
# without reloading when the variant AND settings match, so a second session running
# the same command still attaches without evicting the first.
- load_has_overrides = bool(
- load.gguf_variant
- or load.max_seq_length
- or not load.load_in_4bit
- or load.tensor_parallel
- or load.gpu_memory_mode is not None
- )
+ overrides = load.overrides()
+ load_has_overrides = bool(overrides)
+ # Inferred-attach path only: `requested` becomes the resident's internal identifier
+ # (possibly a server path), so this is the id to show and to match on.
+ attach_public_id = None
+ status_snapshot = None
+ # Whether the inferred settings can restart the resident. Computed once: the preload
+ # gate, the warning, the consent refusal and force_reload must all agree, and asking
+ # twice against a snapshot taken at different times is how they drift apart.
+ inferred_differs = False
+ if requested is None and load_has_overrides and infer_resident:
+ status_snapshot = _inference_status(base, key)
+ requested, attach_public_id = _resident_load_target(models, status_snapshot, allow_casefold)
+ inferred_differs = _load_settings_differ(status_snapshot, load, overrides)
+ # preload_check deliberately survives: it is the only gate before the load evicts
+ # the shared model (_require_gguf_for_codex runs after _connect returns).
# /v1/models also lists cached-but-unloaded catalog entries (loaded == False);
# matching one would skip /api/inference/load and leave the agent pointed at a
# model that is not resident, so only attach to an entry that is actually loaded.
@@ -1692,17 +1968,27 @@ def _resolve_model(
# resident model already satisfies (a path-loaded GGUF shown as a bare basename
# can collide with a non-GGUF unsloth/<name>).
active = next((m for m in models if m.get("loaded") is not False), None)
+ # On the inferred path the target came from status, so catalog order can name a
+ # different entry (a speech sidecar listed first). Using it would make the
+ # survivor probe below report the wrong model as still serving.
+ if attach_public_id is not None:
+ active = next(
+ (
+ m
+ for m in models
+ if _model_id_matches(
+ m.get("id"), attach_public_id, allow_casefold = allow_casefold
+ )
+ and m.get("loaded") is not False
+ ),
+ None,
+ ) or {"id": attach_public_id}
if preload_check is not None:
# An explicit knob forces match to None so the server's disk-free dedupe can
# answer already_loaded; gating it would reject a second session for the model
# already serving, whose file may have moved. Only the quant is checked below:
# any other run knob changes the runtime intent, a real reload nothing dedupes.
- other_overrides = bool(
- load.max_seq_length
- or not load.load_in_4bit
- or load.tensor_parallel
- or load.gpu_memory_mode is not None
- )
+ other_overrides = bool(overrides - {"gguf_variant"})
# /v1/models shows a path-loaded GGUF under its basename, so match that spelling
# too, or a second session reruns the gate.
wanted_ids = {requested, _public_model_id(requested)} - {None}
@@ -1714,6 +2000,11 @@ def _resolve_model(
)
for m in models
)
+ # A proven no-op evicts nothing, so the gate has nothing to protect, and
+ # running it would reject an attach the disk-free already-loaded path can
+ # still serve (a direct .gguf the server has mapped but that has since moved).
+ if attach_public_id is not None and not inferred_differs:
+ resident_serves_request = True
# /v1/models shows only the basename, so confirm a path request against the
# identifier the server loaded -- else /new/foo.gguf reads as resident because
# /old/foo.gguf is.
@@ -1749,7 +2040,14 @@ def _resolve_model(
preload_check(base, key, requested, load.gguf_variant)
active_id = active.get("id") if active else None
announced_switch = False
- if active_id and not _model_id_matches(
+ if attach_public_id is not None:
+ # An inferred attach never switches model, so the comparison below would
+ # misreport a switch and print the server's path.
+ if inferred_differs:
+ typer.echo(f"Applying new load settings to {attach_public_id}.")
+ typer.echo("This unloads the current model for every attached session.")
+ announced_switch = True
+ elif active_id and not _model_id_matches(
active_id,
requested,
allow_casefold = allow_casefold,
@@ -1773,20 +2071,83 @@ def _resolve_model(
typer.echo("This unloads the current model for every attached session.")
announced_switch = True
# Mirror `unsloth run`'s load knobs; keep the default payload as just
- # model_path so a bare `--model` load is unchanged.
+ # model_path so a bare `--model` load is unchanged. Membership decides, not
+ # truthiness: a reset like --context-length 0 equals the default yet must be sent.
payload = {"model_path": requested}
- if load.gguf_variant:
- payload["gguf_variant"] = load.gguf_variant
- if load.max_seq_length:
+ if "gguf_variant" in overrides and load.gguf_variant:
+ direct_file = attach_public_id is not None and str(requested).lower().endswith(".gguf")
+ if direct_file:
+ # from_identifier consults a variant only for a DIRECTORY, so a DIFFERENT
+ # quant cannot be selected: posting one would reload the very same file and
+ # label it with a quant that does not describe its weights. Restating the
+ # one already running asks for no change, so drop the inapplicable field
+ # and let the other overrides through.
+ resident_variant = status_snapshot.get("gguf_variant")
+ same = (
+ bool(resident_variant)
+ and str(resident_variant).strip().lower()
+ == str(load.gguf_variant).strip().lower()
+ )
+ if not same:
+ _fail(
+ f"'{attach_public_id}' was loaded from a single .gguf file, so "
+ f"--gguf-variant {load.gguf_variant} cannot select a different quant. "
+ "Re-run with --model naming the repository to switch quants."
+ )
+ else:
+ payload["gguf_variant"] = load.gguf_variant
+ elif attach_public_id is not None and status_snapshot.get("is_gguf"):
+ # Re-send the running quant: a repo id carries none, so from_identifier would
+ # auto-pick (_GGUF_QUANT_PREFERENCE, UD-Q4_K_XL first) and changing only the
+ # context would evict a chosen Q8_0 to download a different quant. Skip a
+ # .gguf path, which loads as itself -- the server gates on the same suffix.
+ resident_variant = status_snapshot.get("gguf_variant")
+ if resident_variant and not str(requested).lower().endswith(".gguf"):
+ payload["gguf_variant"] = resident_variant
+ if "max_seq_length" in overrides:
payload["max_seq_length"] = load.max_seq_length
- if not load.load_in_4bit:
- payload["load_in_4bit"] = False
- if load.tensor_parallel:
- payload["tensor_parallel"] = True
- if load.gpu_memory_mode is not None:
+ if "load_in_4bit" in overrides:
+ payload["load_in_4bit"] = load.load_in_4bit
+ if "tensor_parallel" in overrides:
+ payload["tensor_parallel"] = load.tensor_parallel
+ if "gpu_memory_mode" in overrides and load.gpu_memory_mode is not None:
payload["gpu_memory_mode"] = load.gpu_memory_mode
- if load.gpu_memory_mode == "manual":
+ # -1 means "pick the layers", which is right when the user is switching INTO
+ # manual, but on an inferred attach to a resident already in manual it would
+ # throw away the layer count it was pinned to. Leave it for the round-trip.
+ already_manual = (
+ attach_public_id is not None and status_snapshot.get("gpu_memory_mode") == "manual"
+ )
+ if load.gpu_memory_mode == "manual" and not already_manual:
payload["gpu_layers"] = -1
+ if (
+ attach_public_id is not None
+ and inferred_differs
+ and status_snapshot.get("requires_trust_remote_code")
+ ):
+ # The reload cannot reproduce the consent: the payload has no
+ # trust_remote_code and no approval fingerprint, and the standard backend
+ # tears the worker down BEFORE the replacement is accepted, so a rejected
+ # custom-code load leaves nothing resident. Refuse while the model is still
+ # serving; naming it with --model goes through the normal consent path.
+ _fail(
+ f"'{attach_public_id}' was loaded with trust_remote_code, which an attach "
+ "cannot re-authorize. Re-run with --model naming it to apply these settings."
+ )
+ if attach_public_id is not None:
+ # An inferred reload is a full load, not a PATCH: _gguf_request_intent copies
+ # every defaulted LoadRequest field into the new intent, so a knob we leave out
+ # is reset rather than kept. Carry the resident's own values for the ones the
+ # user did not name, or changing the context alone would drop their KV dtype,
+ # slot count, batch sizes and GPU placement.
+ payload.update(_resident_runtime_payload(status_snapshot, payload))
+ # The server cannot tell an explicit `--context-length 0` reset from the 0 that
+ # every UI load sends, so it treats 0 as "no preference" and would answer
+ # already_loaded. Say outright that this one is a reload, but only when status
+ # PROVED a difference: on an older server _load_settings_differ cannot tell,
+ # and forcing there would evict on every attach.
+ if status_snapshot and inferred_differs:
+ payload["force_reload"] = True
try:
loaded = _load_model_with_progress(base, key, requested, load, payload)
except Exception:
@@ -1797,12 +2158,16 @@ def _resolve_model(
typer.echo(f"Nothing was unloaded; {active_id} is still serving.", err = True)
raise
if loaded.get("status") == "already_loaded":
- typer.echo(f"Reusing loaded model: {_display_model_spec(requested, load.gguf_variant)}")
+ # Show the public id on the inferred path; `requested` may be a server path.
+ shown = attach_public_id or requested
+ typer.echo(f"Reusing loaded model: {_display_model_spec(shown, load.gguf_variant)}")
# Unsloth registers the model under a canonical id (resolved identifier,
# casing) that /v1/models echoes but which may differ from the path we
# passed; match on the id the load reports so we don't silently fall
# through to models[0] and connect to a different loaded model.
- wanted = {requested, _public_model_id(requested)} - {None}
+ # attach_public_id: our _public_model_id only strips a basename, while the
+ # server also maps an HF cache path to its repo id, so the two can disagree.
+ wanted = {requested, _public_model_id(requested), attach_public_id} - {None}
if isinstance(loaded, dict):
wanted |= {loaded.get("model"), loaded.get("display_name")} - {None}
models = _loaded_models(base, key)
@@ -3868,6 +4233,9 @@ def _connect(
None if server is not None else model,
load,
preload_check = None if server is not None else preload_check,
+ # That server was started FROM these knobs, so inferring a target here would
+ # reload what was just loaded.
+ infer_resident = server is None,
)
except BaseException:
_shutdown_auto_served()
@@ -4535,7 +4903,9 @@ def claude(
base, key, entry = _connect(
api_key,
model,
- LoadOptions(gguf_variant, max_seq_length, load_in_4bit, tensor_parallel, gpu_memory_mode),
+ _load_options(
+ ctx, gguf_variant, max_seq_length, load_in_4bit, tensor_parallel, gpu_memory_mode
+ ),
serve = serve,
launch = launch,
server_options = ServerOptions(
@@ -4656,7 +5026,9 @@ def codex(
base, key, entry = _connect(
api_key,
model,
- LoadOptions(gguf_variant, max_seq_length, load_in_4bit, tensor_parallel, gpu_memory_mode),
+ _load_options(
+ ctx, gguf_variant, max_seq_length, load_in_4bit, tensor_parallel, gpu_memory_mode
+ ),
serve = serve,
launch = launch,
preload_check = _attach_gguf_check_for_codex,
@@ -4766,7 +5138,9 @@ def openclaw(
base, key, entry = _connect(
api_key,
model,
- LoadOptions(gguf_variant, max_seq_length, load_in_4bit, tensor_parallel, gpu_memory_mode),
+ _load_options(
+ ctx, gguf_variant, max_seq_length, load_in_4bit, tensor_parallel, gpu_memory_mode
+ ),
serve = serve,
launch = launch,
server_options = ServerOptions(
@@ -4856,7 +5230,9 @@ def opencode(
base, key, entry = _connect(
api_key,
model,
- LoadOptions(gguf_variant, max_seq_length, load_in_4bit, tensor_parallel, gpu_memory_mode),
+ _load_options(
+ ctx, gguf_variant, max_seq_length, load_in_4bit, tensor_parallel, gpu_memory_mode
+ ),
serve = serve,
launch = launch,
server_options = ServerOptions(
@@ -5041,7 +5417,9 @@ def hermes(
base, key, entry = _connect(
api_key,
model,
- LoadOptions(gguf_variant, max_seq_length, load_in_4bit, tensor_parallel, gpu_memory_mode),
+ _load_options(
+ ctx, gguf_variant, max_seq_length, load_in_4bit, tensor_parallel, gpu_memory_mode
+ ),
serve = serve,
launch = launch,
server_options = ServerOptions(
@@ -5106,7 +5484,9 @@ def pi(
base, key, entry = _connect(
api_key,
model,
- LoadOptions(gguf_variant, max_seq_length, load_in_4bit, tensor_parallel, gpu_memory_mode),
+ _load_options(
+ ctx, gguf_variant, max_seq_length, load_in_4bit, tensor_parallel, gpu_memory_mode
+ ),
serve = serve,
launch = launch,
server_options = ServerOptions(