Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

Prevent agent startup from waiting on model catalog scans (#10728)

Lee Jacksonc8ed02f69162
diff --git a/studio/backend/routes/inference.py b/studio/backend/routes/inference.py
index a4718728c2d..515ebb43aca 100644
--- a/studio/backend/routes/inference.py
+++ b/studio/backend/routes/inference.py
@@ -26751,9 +26751,10 @@ def _openai_model_objects() -> list[dict]:
entry["task"] = _TTS_MODEL_TASK
models.append(entry)
- # Check Unsloth backend
- backend = get_inference_backend()
- if backend.active_model_name:
+ # Describing residency must not construct an unused orchestrator: its cold
+ # initialization runs device detection even when only llama.cpp is loaded.
+ backend = _peek_inference_backend()
+ if backend is not None and backend.active_model_name:
model_info = backend.models.get(backend.active_model_name, {})
entry = {
# The alias, or an LM Studio model switched to by repo id loses its publisher.
@@ -27349,8 +27350,8 @@ async def _openai_catalog_objects() -> list[dict]:
_created = int(time.time())
# Loaded models first (clean ids + context fields), marked loaded.
by_id: dict[str, dict] = {}
- # Off-loop: _openai_model_objects() is sync and calls get_inference_backend(), whose cold
- # build waits on detection. Inline, an early GET /v1/models held the loop for the import.
+ # Off-loop: _openai_model_objects() is sync and its per-entry quant probe reads the
+ # resolver index. Inline, an early GET /v1/models held the loop for that work.
for entry in await asyncio.to_thread(_openai_model_objects):
by_id[entry["id"]] = {**entry, "loaded": True}
orchestrator = _peek_inference_backend()
@@ -27405,6 +27406,18 @@ async def _openai_catalog_objects() -> list[dict]:
return list(by_id.values())
+@studio_router.get("/loaded-models")
+async def loaded_inference_models(current_subject: str = Depends(get_current_subject)):
+ """Loaded llama.cpp/orchestrator models for agent startup, without a disk catalog scan.
+
+ Keep the public ids and context fields shared with /v1/models. The full catalog
+ also discovers unloaded and media models, which an attaching coding agent does
+ not need and which can take longer than its HTTP deadline on slow scan folders.
+ """
+ models = await asyncio.to_thread(_openai_model_objects)
+ return {"object": "list", "data": [{**entry, "loaded": True} for entry in models]}
+
+
# Some OpenAI-compatible clients (notably DEVONthink) probe ``/v1/models/``
# literally and do not follow FastAPI's automatic slash redirect. Register the
# slash form directly so model discovery reaches authentication and returns the
@@ -27438,7 +27451,7 @@ async def openai_retrieve_model(model_id: str, current_subject: str = Depends(ge
# Loaded models resolve without a catalog scan (the common case); only build
# the full catalog -- which may hit the filesystem -- for unloaded ids. Match
# case-insensitively, like the catalog loop below and the resolver's index.
- # Off-loop like the catalog helper: the singleton's cold build waits on detection.
+ # Off-loop like the catalog helper, and for the same sync resolver-index work.
_loaded = await asyncio.to_thread(_openai_model_objects)
for entry in _loaded:
eid = entry["id"]
diff --git a/studio/backend/tests/test_keyless_api_access.py b/studio/backend/tests/test_keyless_api_access.py
index 1a991cbc84c..e07e8ea38bc 100644
--- a/studio/backend/tests/test_keyless_api_access.py
+++ b/studio/backend/tests/test_keyless_api_access.py
@@ -132,6 +132,44 @@ def test_exact_route_matrix_matches_registered_topology():
assert intended <= registered
+def test_the_allowlisted_studio_paths_name_routes_that_exist():
+ # The matrix above pins the /v1 router. This one covers the studio router, whose
+ # paths carry a mount prefix, so a rename there cannot silently strand an entry.
+ from routes.inference import studio_router
+ from utils.keyless_api_access import _INFERENCE_ROUTES
+
+ registered = {("/api/inference" + route.path, method)
+ for route in studio_router.routes
+ for method in getattr(route, "methods", set())}
+ for method, path in _INFERENCE_ROUTES:
+ if path.startswith("/api/inference/"):
+ assert (path, method) in registered, path
+
+
[email protected]("token", [None, "not-needed"])
+def test_resident_model_discovery_preserves_keyless_inference_access(token):
+ seed_user()
+ set_keyless_api_access("inference", tools = False)
+
+ def discovery_request(method = "GET", path = "/api/inference/loaded-models"):
+ if token is None:
+ return request_for(path = path, method = method)
+ return bearer_request(token, path = path, method = method)
+
+ assert subject_of(discovery_request()) == storage.DEFAULT_ADMIN_USERNAME
+ for method, path in (
+ ("POST", "/api/inference/loaded-models"),
+ ("POST", "/api/inference/load"),
+ ("GET", "/api/inference/status"),
+ ):
+ with pytest.raises(HTTPException):
+ subject_of(discovery_request(method, path))
+
+ set_keyless_api_access("off")
+ with pytest.raises(HTTPException):
+ subject_of(discovery_request())
+
+
def test_settings_are_immediate_and_fail_closed(monkeypatch):
import storage.studio_db as studio_db
diff --git a/studio/backend/tests/test_loaded_inference_models.py b/studio/backend/tests/test_loaded_inference_models.py
new file mode 100644
index 00000000000..bf125f6277c
--- /dev/null
+++ b/studio/backend/tests/test_loaded_inference_models.py
@@ -0,0 +1,142 @@
+# SPDX-License-Identifier: AGPL-3.0-only
+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
+
+"""Resident model discovery must not depend on the local disk/media catalog."""
+
+from types import SimpleNamespace
+
+import pytest
+from fastapi import FastAPI, HTTPException
+from fastapi.testclient import TestClient
+
+import routes.inference as inf
+from auth.authentication import get_current_subject
+
+
+def resident_backends(monkeypatch):
+ llama = SimpleNamespace(
+ is_loaded = True,
+ model_identifier = "/models/Local-Q4.gguf",
+ context_length = 4096,
+ max_context_length = 8192,
+ native_context_length = 32768,
+ )
+ backend = SimpleNamespace(active_model_name = None, models = {})
+ monkeypatch.setattr(inf, "get_llama_cpp_backend", lambda: llama)
+ monkeypatch.setattr(inf, "get_inference_backend", lambda: backend)
+
+ def no_scan(*args, **kwargs):
+ pytest.fail("Resident discovery must not scan the disk or media catalog")
+
+ monkeypatch.setattr(inf, "_cached_local_catalog", no_scan)
+ monkeypatch.setattr(inf, "_servable_catalog_rows", no_scan)
+ monkeypatch.setattr(inf, "_media_model_objects", no_scan)
+ monkeypatch.setattr(inf, "_stt_model_objects", no_scan)
+ return llama, backend
+
+
+def _app():
+ app = FastAPI()
+ app.include_router(inf.studio_router, prefix = "/api/inference")
+ app.dependency_overrides[get_current_subject] = lambda: "test-subject"
+ return app
+
+
+def test_loaded_models_keep_public_ids_and_context_without_scan(resident_backends):
+ llama, backend = resident_backends
+ backend.active_model_name = "org/Other"
+ backend.models = {
+ "org/Other": {
+ "context_length": 2048,
+ "native_context_length": 16384,
+ "max_context_length": 4096,
+ "context_length_enforced": True,
+ }
+ }
+ with TestClient(_app()) as client:
+ response = client.get("/api/inference/loaded-models")
+ assert response.status_code == 200
+ body = response.json()
+ assert body["object"] == "list"
+ models = {entry["id"]: entry for entry in body["data"]}
+ assert set(models) == {"Local-Q4", "org/Other"}
+ assert all(entry["loaded"] is True for entry in models.values())
+ assert models["Local-Q4"]["context_length"] == 4096
+ assert models["Local-Q4"]["max_context_length"] == 8192
+ assert models["Local-Q4"]["native_context_length"] == 32768
+ assert models["org/Other"]["context_length"] == 2048
+ assert models["org/Other"]["context_length_enforced"] is True
+ assert "/models/" not in response.text
+
+
+def test_loaded_models_reread_residency_without_scanning(resident_backends):
+ llama, backend = resident_backends
+ with TestClient(_app()) as client:
+ assert len(client.get("/api/inference/loaded-models").json()["data"]) == 1
+ llama.is_loaded = False
+ assert client.get("/api/inference/loaded-models").json() == {"object": "list", "data": []}
+ backend.active_model_name = "org/New"
+ backend.models = {"org/New": {"context_length": 1024}}
+ models = client.get("/api/inference/loaded-models").json()["data"]
+ assert [entry["id"] for entry in models] == ["org/New"]
+
+
+def test_loaded_models_require_auth_before_reading_backends(monkeypatch):
+ def reject():
+ raise HTTPException(status_code = 401, detail = "Unauthorized")
+
+ def no_backend():
+ pytest.fail("Authentication must run before backend discovery")
+
+ app = _app()
+ app.dependency_overrides[get_current_subject] = reject
+ monkeypatch.setattr(inf, "_openai_model_objects", no_backend)
+ with TestClient(app) as client:
+ assert client.get("/api/inference/loaded-models").status_code == 401
+
+
[email protected]("gguf_loaded", [False, True])
+def test_discovery_does_not_construct_an_unused_orchestrator(
+ resident_backends, monkeypatch, gguf_loaded
+):
+ from core.inference import orchestrator
+
+ llama, _ = resident_backends
+ llama.is_loaded = gguf_loaded
+ # Exercise the real singleton getter/peek, not the fixture's replacement.
+ monkeypatch.setattr(inf, "get_inference_backend", orchestrator.get_inference_backend)
+ monkeypatch.setattr(orchestrator, "_inference_backend", None)
+
+ def no_initialization():
+ raise AssertionError("Resident discovery must not initialize an unused inference backend")
+
+ monkeypatch.setattr(orchestrator, "InferenceOrchestrator", no_initialization)
+ with TestClient(_app()) as client:
+ response = client.get("/api/inference/loaded-models")
+ assert response.status_code == 200
+ assert [entry["id"] for entry in response.json()["data"]] == (
+ ["Local-Q4"] if gguf_loaded else []
+ )
+ assert orchestrator.peek_inference_backend() is None
+
+
+def test_a_resident_quant_is_advertised_without_letting_the_probe_scan(
+ resident_backends, monkeypatch
+):
+ from core.inference import local_model_resolver
+
+ llama, _ = resident_backends
+ llama.hf_variant = "Q4_K_M"
+ seen = []
+
+ def resolve(spec, allow_scan = True):
+ seen.append((spec, allow_scan))
+ return "/models/Local-Q4.gguf"
+
+ monkeypatch.setattr(local_model_resolver, "resolve_local_gguf", resolve)
+ monkeypatch.setattr(local_model_resolver, "index_is_built", lambda: True)
+ with TestClient(_app()) as client:
+ entry = client.get("/api/inference/loaded-models").json()["data"][0]
+ assert entry["quant"] == "Q4_K_M"
+ assert seen == [("Local-Q4:Q4_K_M", False)]
diff --git a/studio/backend/utils/keyless_api_access.py b/studio/backend/utils/keyless_api_access.py
index 11879fb636b..cba0b9f6d4b 100644
--- a/studio/backend/utils/keyless_api_access.py
+++ b/studio/backend/utils/keyless_api_access.py
@@ -37,6 +37,7 @@ _INFERENCE_ROUTES = frozenset(
("POST", "/v1/messages"),
("POST", "/v1/messages/count_tokens"),
("GET", "/v1/models"),
+ ("GET", "/api/inference/loaded-models"),
("POST", "/v1/responses"),
# Discovery probes, read-only. A keyless client that may list models and chat but gets 401 on /props reads that as an auth wall in front of the whole surface and stops, which is the opposite of what the scope grants.
("GET", "/props"),
diff --git a/tests/studio/test_cli_attach_load_knobs.py b/tests/studio/test_cli_attach_load_knobs.py
index 01f9fbffb82..f4e1b5edb0d 100644
--- a/tests/studio/test_cli_attach_load_knobs.py
+++ b/tests/studio/test_cli_attach_load_knobs.py
@@ -18,7 +18,7 @@ RESIDENT = {"id": "unsloth/Qwen3-8B", "loaded": True}
class FakeServer:
- """Serves /v1/models and /api/inference/status; records every load payload."""
+ """Serves the loaded-model listing and /api/inference/status; records every load payload."""
def __init__(self, models, status):
self.models = models
@@ -36,7 +36,9 @@ class FakeServer:
error = None,
):
self.requests.append((method, url))
- if url.endswith("/v1/models"):
+ # Both spellings: the CLI asks for the resident listing and falls back to
+ # /v1/models against a server too old to serve it.
+ if url.endswith(("/api/inference/loaded-models", "/v1/models")):
return {"data": [dict(m) for m in self.models]}
if url.endswith("/api/inference/status"):
return dict(self.status)
@@ -359,7 +361,7 @@ def test_hf_cache_resident_matches_the_advertised_repo_id(monkeypatch):
def test_status_names_the_resident_even_when_the_catalog_lags(monkeypatch):
- """A status id absent from /v1/models still names the resident."""
+ """A status id absent from the listing still names the resident."""
server = FakeServer(
[{"id": "unsloth/whisper-large", "loaded": True}],
{
diff --git a/unsloth_cli/commands/start.py b/unsloth_cli/commands/start.py
index 459ec320ba9..c9cd0e01bc1 100644
--- a/unsloth_cli/commands/start.py
+++ b/unsloth_cli/commands/start.py
@@ -657,7 +657,7 @@ class ServerOptions(NamedTuple):
def _split_repo_variant(model: str) -> tuple:
- """Split ``org/name:QUANT`` into ``("org/name", "QUANT")``. ``unsloth run`` and llama.cpp accept ``--model org/name:QUANT`` as shorthand for ``--model org/name --gguf-variant QUANT``, so mirror that here and a ``:variant`` suffix resolves against the already-loaded ``org/name`` (which /v1/models lists without the suffix) instead of trying to load a repo id containing ``:``, which Hugging Face rejects and which would evict a model another session is using. Local paths, Windows drive letters and ids without a ``:`` pass through unchanged."""
+ """Split ``org/name:QUANT`` into ``("org/name", "QUANT")``. ``unsloth run`` and llama.cpp accept ``--model org/name:QUANT`` as shorthand for ``--model org/name --gguf-variant QUANT``, so mirror that here and a ``:variant`` suffix resolves against the already-loaded ``org/name`` (which the loaded listing shows without the suffix) instead of trying to load a repo id containing ``:``, which Hugging Face rejects and which would evict a model another session is using. Local paths, Windows drive letters and ids without a ``:`` pass through unchanged."""
s = (model or "").strip()
if not s or s.startswith(("/", "./", "../", "~")) or s == ".":
return s, None
@@ -749,6 +749,13 @@ def _http_error_detail(exc: urllib.error.HTTPError) -> str:
return str(exc)
+def _fail_request(exc: Exception, error: str) -> NoReturn:
+ """Fail with `error` plus whatever the server or the transport gave as a reason."""
+ if isinstance(exc, urllib.error.HTTPError):
+ _fail(f"{error}: {_http_error_detail(exc)}")
+ _fail(f"{error}: {getattr(exc, 'reason', None) or exc}")
+
+
def _http_json(
method: str,
url: str,
@@ -757,7 +764,7 @@ def _http_json(
timeout = 30,
error = None,
):
- """On HTTPError: raise if `error` is None, else fail with `error` plus the server's detail."""
+ """On a failed request: raise if `error` is None, else fail with `error` plus the reason."""
request = urllib.request.Request(
url,
data = None if payload is None else json.dumps(payload).encode(),
@@ -774,14 +781,10 @@ def _http_json(
body = json.loads(response.read().decode() or "{}")
# A padded /load or /unload commits its 200 early, so a late failure arrives in-band; raise it as the HTTPError handled below.
return raise_for_deferred_error(url, body)
- except urllib.error.HTTPError as exc:
+ except (urllib.error.HTTPError, urllib.error.URLError, TimeoutError) as exc:
if error is None:
raise
- _fail(f"{error}: {_http_error_detail(exc)}")
- except (urllib.error.URLError, TimeoutError) as exc:
- if error is None:
- raise
- _fail(f"{error}: {getattr(exc, 'reason', None) or exc}")
+ _fail_request(exc, error)
# A server that WE auto-started (never one we merely found). Kept at module scope so failure paths and the atexit backstop can tear it down without threading a handle through all six agent commands. Only one agent runs per process, so one slot is enough.
@@ -1481,10 +1484,35 @@ def _remember_key(cache: Path, base: str, key: str, source: str) -> None:
pass # worst case the next launch mints another key
+def _loaded_models_response(
+ base: str,
+ key: str,
+ timeout = 30,
+) -> dict:
+ """Raw listing of what this server has resident. Startup and key checks need no more.
+
+ /v1/models answers the same question but waits for disk and media discovery first,
+ which on a slow scan folder outlasts the deadline below.
+ """
+ try:
+ answer = _http_json("GET", f"{base}/api/inference/loaded-models", key, timeout = timeout)
+ if isinstance(answer.get("data"), list):
+ return answer
+ # A Studio older than the 404-ing catch-all answers an unknown /api path with a
+ # 200 and {"error": ...}, which read as an empty listing reports a resident model
+ # as unloaded. Anything without a "data" list means the route is not there.
+ except urllib.error.HTTPError as exc:
+ if exc.code != 404:
+ raise
+ # Fall back, never on an auth or server error. An older Studio still answers on
+ # /v1/models, with the unloaded catalog entries every caller below already filters out.
+ return _http_json("GET", f"{base}/v1/models", key, timeout = timeout)
+
+
def _key_accepted(base: str, key: str) -> bool:
# Only a genuine auth rejection (401/403) means "this key is bad, skip it and try the next cached key or mint a fresh one". A 5xx or a network blip is a server-side outage, not a bad key: fail with a clean message instead of silently discarding a working key and minting extras against a struggling server.
try:
- _http_json("GET", f"{base}/v1/models", key)
+ _loaded_models_response(base, key)
return True
except urllib.error.HTTPError as exc:
if exc.code in (401, 403):
@@ -1561,12 +1589,15 @@ def _agent_api_key(
def _loaded_models(base: str, key: str) -> list:
- return _http_json("GET", f"{base}/v1/models", key, error = "Couldn't list models").get("data", [])
+ try:
+ return _loaded_models_response(base, key).get("data", [])
+ except (urllib.error.HTTPError, urllib.error.URLError, TimeoutError) as exc:
+ _fail_request(exc, "Couldn't list models")
def _model_still_loaded(base: str, key: str, model_id: object) -> bool:
try:
- models = _http_json("GET", f"{base}/v1/models", key, timeout = 5).get("data", [])
+ models = _loaded_models_response(base, key, timeout = 5).get("data", [])
except Exception:
return False
return any(m.get("id") == model_id and m.get("loaded") is not False for m in models)
@@ -1611,7 +1642,7 @@ def _is_model_path(value: str) -> bool:
def _public_model_id(value: Optional[str]) -> Optional[str]:
- """The id Unsloth advertises for a model loaded by path. /v1/models never echoes a host path: it reports the file or directory name with any .gguf suffix stripped (core.inference.model_ids.public_model_id), so a path we asked to load has to be matched by that name too."""
+ """The id Unsloth advertises for a model loaded by path. The loaded listing never echoes a host path: it reports the file or directory name with any .gguf suffix stripped (core.inference.model_ids.public_model_id), so a path we asked to load has to be matched by that name too."""
if not value or not _is_model_path(value):
return None
name = os.path.basename(value.replace("\\", "/").rstrip("/"))
@@ -1645,7 +1676,7 @@ def _inference_status(base: str, key: str) -> dict:
def _resident_load_target(models: list, status: dict, allow_casefold: bool):
- """(identifier to post, id it is advertised as) for the running model. /v1/models shows only the sanitized basename while _same_loaded_identifier compares resident paths exactly, so the load must carry the identifier status reports."""
+ """(identifier to post, id it is advertised as) for the running model. The loaded listing shows only the sanitized basename while _same_loaded_identifier compares resident paths exactly, so the load must carry the identifier status reports."""
if status.get("is_diffusion"):
# An image runtime answers with an active_model like any other, but it cannot serve chat: targeting it would tear down the diffusion server and then point the agent at a model that can never answer it.
_fail(
@@ -1665,7 +1696,7 @@ def _resident_load_target(models: list, status: dict, allow_casefold: bool):
None,
)
if entry is None and not status:
- # Only when there is no status at all (older server). A status that ANSWERED with active_model null is stating there is no chat resident. Catalog ORDER is not evidence either: /v1/models lists loaded speech sidecars too, so the first entry can be one. Answer only when the catalog is unambiguous.
+ # Only when there is no status at all (older server). A status that ANSWERED with active_model null is stating there is no chat resident. Listing ORDER is not evidence either: a loaded speech sidecar is listed like any other resident model, so the first entry can be one. Answer only when the catalog is unambiguous.
loaded = [m for m in models if m.get("loaded") is not False]
if len(loaded) == 1:
entry = loaded[0]
@@ -1810,7 +1841,7 @@ def _resolve_model(
load_requested = False
# Only casefold-match ids against a loopback Unsloth, where _is_hub_model_id's local existence probe can actually reject a server-side path; see the note there.
allow_casefold = is_loopback_url(base)
- # /v1/models reports the model id but not the active GGUF variant or runtime load settings, so an id match alone can hide the wrong quant (Q8_0 serving while the user asked for UD-Q4_K_XL). When the user passed any explicit load knob, defer to /api/inference/load: the server's already-loaded dedup answers "already_loaded" without reloading when the variant AND settings match, so a second session running the same command still attaches without evicting the first.
+ # The loaded listing carries the active GGUF variant only while that quant reference still resolves, and never the runtime load settings, so an id match alone can hide the wrong quant (Q8_0 serving while the user asked for UD-Q4_K_XL). When the user passed any explicit load knob, defer to /api/inference/load: the server's already-loaded dedup answers "already_loaded" without reloading when the variant AND settings match, so a second session running the same command still attaches without evicting the first.
overrides = load.overrides()
load_has_overrides = bool(overrides)
# Inferred-attach path only: `requested` becomes the resident's internal identifier (possibly a server path), so this is the id to show and to match on.
@@ -1822,7 +1853,7 @@ def _resolve_model(
status_snapshot = _inference_status(base, key)
requested, attach_public_id = _resident_load_target(models, status_snapshot, allow_casefold)
inferred_differs = _load_settings_differ(status_snapshot, load, overrides)
- # preload_check deliberately survives: it is the only gate before the load evicts the shared model (_require_gguf_for_codex runs after _connect returns). /v1/models also lists cached-but-unloaded catalog entries (loaded == False); matching one would skip /api/inference/load and leave the agent pointed at a model that is not resident, so only attach to an entry that is actually loaded.
+ # preload_check deliberately survives: it is the only gate before the load evicts the shared model (_require_gguf_for_codex runs after _connect returns). An older server answers this listing from the full catalog, which also carries cached-but-unloaded entries (loaded == False); matching one would skip /api/inference/load and leave the agent pointed at a model that is not resident, so only attach to an entry that is actually loaded.
match = (
None
if requested and load_has_overrides
@@ -1856,7 +1887,7 @@ def _resolve_model(
if preload_check is not None:
# An explicit knob forces match to None so the server's disk-free dedupe can answer already_loaded; gating it would reject a second session for the model already serving, whose file may have moved. Only the quant is checked below: any other run knob changes the runtime intent, a real reload nothing dedupes.
other_overrides = bool(overrides - {"gguf_variant"})
- # /v1/models shows a path-loaded GGUF under its basename, so match that spelling too, or a second session reruns the gate.
+ # The loaded listing shows a path-loaded GGUF under its basename, so match that spelling too, or a second session reruns the gate.
wanted_ids = {requested, _public_model_id(requested)} - {None}
resident_serves_request = not other_overrides and any(
m.get("loaded") is not False
@@ -1869,7 +1900,7 @@ def _resolve_model(
# A proven no-op evicts nothing, so the gate has nothing to protect, and running it would reject an attach the disk-free already-loaded path can still serve (a direct .gguf the server has mapped but that has since moved).
if attach_public_id is not None and not inferred_differs:
resident_serves_request = True
- # /v1/models shows only the basename, so confirm a path request against the identifier the server loaded, else /new/foo.gguf reads as resident because /old/foo.gguf is.
+ # The loaded listing shows only the basename, so confirm a path request against the identifier the server loaded, else /new/foo.gguf reads as resident because /old/foo.gguf is.
if resident_serves_request and _is_model_path(requested):
try:
status = _http_json("GET", f"{base}/api/inference/status", key)
@@ -1916,7 +1947,7 @@ def _resolve_model(
typer.echo("This unloads the current model for every attached session.")
announced_switch = True
elif active_id and load.gguf_variant:
- # Same repo id but an explicit quant still replaces the resident weights; /v1/models has no variant, so ask the status endpoint.
+ # Same repo id but an explicit quant still replaces the resident weights; the loaded listing does not carry a variant for every resident model, so ask the status endpoint.
try:
status = _http_json("GET", f"{base}/api/inference/status", key)
except Exception:
@@ -1995,7 +2026,7 @@ def _resolve_model(
# Show the public id on the inferred path; `requested` may be a server path.
shown = attach_public_id or requested
typer.echo(f"Reusing loaded model: {_display_model_spec(shown, load.gguf_variant)}")
- # Unsloth registers the model under a canonical id (resolved identifier, casing) that /v1/models echoes but which may differ from the path we passed; match on the id the load reports so we do not silently fall through to models[0] and connect to a different loaded model. attach_public_id: our _public_model_id only strips a basename, while the server also maps an HF cache path to its repo id, so the two can disagree.
+ # Unsloth registers the model under a canonical id (resolved identifier, casing) that the loaded listing echoes but which may differ from the path we passed; match on the id the load reports so we do not silently fall through to models[0] and connect to a different loaded model. attach_public_id: our _public_model_id only strips a basename, while the server also maps an HF cache path to its repo id, so the two can disagree.
wanted = {requested, _public_model_id(requested), attach_public_id} - {None}
if isinstance(loaded, dict):
wanted |= {loaded.get("model"), loaded.get("display_name")} - {None}
@@ -2016,21 +2047,18 @@ def _resolve_model(
typer.echo(f"Reusing loaded model: {_display_model_spec(requested, load.gguf_variant)}")
return match
if requested:
- # We asked Unsloth to load it and it did not surface in /v1/models; do not silently hand back an unrelated loaded model.
+ # We asked Unsloth to load it and it did not surface as loaded; do not silently hand back an unrelated loaded model.
_fail(
f"Unsloth didn't report '{requested}' as loaded. Double-check the model "
"id, or load it from the model dropdown in the UI."
)
- if not models:
- _fail(
- "No model is loaded in Unsloth. Load one from the model dropdown in "
- "the UI, or pass --model <hf-id-or-path> to load it from here."
- )
resident = next((m for m in models if m.get("loaded") is not False), None)
if resident is None:
+ # An empty listing and one holding only unloaded entries are the same situation
+ # to the user, and which one a server sends depends only on its version.
_fail(
- "No model is currently resident in Unsloth. Pass --model <hf-id-or-path> "
- "to reload one, or load it from the model dropdown in the UI."
+ "No model is loaded in Unsloth. Load one from the model dropdown in "
+ "the UI, or pass --model <hf-id-or-path> to load it from here."
)
return resident
diff --git a/unsloth_cli/tests/test_start.py b/unsloth_cli/tests/test_start.py
index db47c3417de..37c323372f6 100644
--- a/unsloth_cli/tests/test_start.py
+++ b/unsloth_cli/tests/test_start.py
@@ -1509,7 +1509,7 @@ def fake_studio(tmp_path, monkeypatch):
error = None,
):
calls.append((method, url, payload))
- if url.endswith("/v1/models"):
+ if url.endswith("/api/inference/loaded-models"):
return {"object": "list", "data": state["models"]}
if url.endswith("/api/inference/status"):
return {"is_gguf": True, "model_identifier": state["models"][0]["id"]}
@@ -2384,7 +2384,7 @@ def test_resolve_model_matches_loaded_canonical_case_after_load(monkeypatch, cap
error = None,
):
calls.append((method, url, payload))
- if url.endswith("/v1/models"):
+ if url.endswith("/api/inference/loaded-models"):
return {
"data": [
{
@@ -2426,7 +2426,7 @@ def test_resolve_model_matches_snapshot_path_by_public_id(monkeypatch):
timeout = 30,
error = None,
):
- if url.endswith("/v1/models"):
+ if url.endswith("/api/inference/loaded-models"):
return {"data": [{"id": "abc123"}] if state["loaded"] else []}
if url.endswith("/api/inference/load"):
state["loaded"] = True
@@ -2489,7 +2489,7 @@ def test_resolve_model_loads_when_catalog_hit_is_not_loaded(monkeypatch):
error = None,
):
calls.append((method, url))
- if url.endswith("/v1/models"):
+ if url.endswith("/api/inference/loaded-models"):
return {
"data": [
{
@@ -2521,7 +2521,7 @@ def test_resolve_model_does_not_attach_if_catalog_stays_unloaded(monkeypatch):
timeout = 30,
error = None,
):
- if url.endswith("/v1/models"):
+ if url.endswith("/api/inference/loaded-models"):
return {
"data": [
{
@@ -2555,7 +2555,7 @@ def test_resolve_model_attaches_to_loaded_catalog_hit_without_reload(monkeypatch
error = None,
):
calls.append((method, url))
- if url.endswith("/v1/models"):
+ if url.endswith("/api/inference/loaded-models"):
return {
"data": [{"id": "unsloth/Gemma-4-GGUF", "loaded": True, "context_length": 131072}]
}
@@ -2604,7 +2604,7 @@ def test_resolve_model_remote_studio_does_not_casefold_attach(monkeypatch):
error = None,
):
calls.append((method, url))
- if url.endswith("/v1/models"):
+ if url.endswith("/api/inference/loaded-models"):
return {
"data": [{"id": "unsloth/Gemma-4-GGUF", "loaded": True, "context_length": 131072}]
}
@@ -2810,7 +2810,7 @@ def test_connect_skips_cached_keys_the_server_rejects(fake_studio, tmp_path, mon
timeout = 30,
error = None,
):
- if url.endswith("/v1/models") and token == "sk-unsloth-stale":
+ if url.endswith("/api/inference/loaded-models") and token == "sk-unsloth-stale":
raise urllib.error.HTTPError(url, 401, "Unauthorized", None, None)
return inner(method, url, token, payload, timeout, error)
@@ -2838,7 +2838,7 @@ def test_connect_saved_key_server_outage_surfaces_not_reminted(fake_studio, tmp_
timeout = 30,
error = None,
):
- if url.endswith("/v1/models") and token == "sk-unsloth-saved":
+ if url.endswith("/api/inference/loaded-models") and token == "sk-unsloth-saved":
raise urllib.error.HTTPError(url, 503, "Service Unavailable", None, None)
return inner(method, url, token, payload, timeout, error)
@@ -2929,7 +2929,7 @@ def test_connect_model_flag_matches_canonical_id(fake_studio, monkeypatch):
):
if url.endswith("/api/inference/load"):
return {"model": canonical, "display_name": canonical}
- if url.endswith("/v1/models"):
+ if url.endswith("/api/inference/loaded-models"):
# Decoy sorts first, so models[0] is the wrong pick on the old code.
return {"object": "list", "data": [MODEL, {"id": canonical, "context_length": 4096}]}
return inner(method, url, token, payload, timeout, error)
@@ -3548,7 +3548,7 @@ def test_connect_requested_model_not_loaded_fails(fake_studio, monkeypatch):
):
if url.endswith("/api/inference/load"):
return {}
- if url.endswith("/v1/models"):
+ if url.endswith("/api/inference/loaded-models"):
return {"object": "list", "data": [MODEL]} # decoy; request never appears
return inner(method, url, token, payload, timeout, error)
@@ -3684,7 +3684,9 @@ def test_connect_minted_cache_requires_identity_check(fake_studio, tmp_path, mon
result = CliRunner().invoke(start.start_app, ["claude", "--no-launch"])
assert result.exit_code == 1
assert "--api-key" in result.output
- assert not any(c[1].endswith("/v1/models") for c in fake_studio) # minted key never sent
+ assert not any(
+ c[1].endswith("/api/inference/loaded-models") for c in fake_studio
+ ) # minted key never sent
def test_connect_explicit_key_skips_identity_check(fake_studio, monkeypatch):
@@ -4233,7 +4235,7 @@ def test_resolve_model_refused_load_reports_survivor(monkeypatch, capsys):
):
if url.endswith("/api/inference/status"):
return {"is_gguf": True, "gguf_variant": "Q4_K_M"}
- assert url.endswith("/v1/models"), url
+ assert url.endswith("/api/inference/loaded-models"), url
return {"data": models}
def refuse_load(base, key, model, load, payload):
@@ -4300,7 +4302,7 @@ def test_resolve_model_failed_load_stays_quiet_when_model_gone(monkeypatch, caps
):
if url.endswith("/api/inference/status"):
return {"is_gguf": True, "gguf_variant": "Q4_K_M"}
- assert url.endswith("/v1/models"), url
+ assert url.endswith("/api/inference/loaded-models"), url
return {"data": []}
def failing_load(base, key, model, load, payload):
@@ -6430,7 +6432,7 @@ def test_agent_api_key_auto_started_rejected_env_key_falls_back(fake_studio, tmp
timeout = 30,
error = None,
):
- if url.endswith("/v1/models") and token == "sk-unsloth-other-server":
+ if url.endswith("/api/inference/loaded-models") and token == "sk-unsloth-other-server":
raise urllib.error.HTTPError(url, 401, "Unauthorized", None, None)
return inner(method, url, token, payload, timeout, error)
@@ -7867,7 +7869,7 @@ def test_codex_preload_gate_checks_direct_path_identity(fake_studio, monkeypatch
timeout = 30,
error = None,
):
- if url.endswith("/v1/models"):
+ if url.endswith("/api/inference/loaded-models"):
return {"data": [{"id": "foo-Q4_K_M", "loaded": True}]}
if url.endswith("/api/inference/status"):
return {
diff --git a/unsloth_cli/tests/test_start_model_discovery.py b/unsloth_cli/tests/test_start_model_discovery.py
new file mode 100644
index 00000000000..6b4ad6b383c
--- /dev/null
+++ b/unsloth_cli/tests/test_start_model_discovery.py
@@ -0,0 +1,107 @@
+# SPDX-License-Identifier: AGPL-3.0-only
+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
+
+"""Agent attachment, authentication and survivor probes use resident discovery."""
+
+import io
+import urllib.error
+
+import pytest
+import typer
+
+import unsloth_cli.commands.start as start
+
+
+BASE = "http://127.0.0.1:8888"
+
+
[email protected]("operation", ["attach", "key", "survivor"])
+def test_discovery_avoids_full_catalog(monkeypatch, operation):
+ calls = []
+
+ def http_json(method, url, token, **kwargs):
+ calls.append((url, kwargs["timeout"]))
+ assert url == BASE + "/api/inference/loaded-models"
+ return {"data": [{"id": "org/Model", "loaded": True, "context_length": 4096}]}
+
+ monkeypatch.setattr(start, "_http_json", http_json)
+ if operation == "attach":
+ assert start._resolve_model(BASE, "key", None)["context_length"] == 4096
+ elif operation == "key":
+ assert start._key_accepted(BASE, "key")
+ else:
+ assert start._model_still_loaded(BASE, "key", "org/Model")
+ assert calls == [(BASE + "/api/inference/loaded-models", 5 if operation == "survivor" else 30)]
+
+
+ "unsupported",
+ [
+ pytest.param("404", id = "route-404s"),
+ # Studio's SPA catch-all answered an unknown /api path with a 200 body before it
+ # was changed to raise 404, so a resident model must survive that shape too.
+ pytest.param({"error": "API endpoint not found"}, id = "spa-catch-all-answers-200"),
+ ],
+)
+def test_old_server_falls_back_to_compat_listing(monkeypatch, unsupported):
+ calls = []
+
+ def http_json(method, url, token, **kwargs):
+ calls.append((url, kwargs["timeout"]))
+ if url.endswith("/loaded-models"):
+ if unsupported == "404":
+ raise urllib.error.HTTPError(url, 404, "Not Found", {}, None)
+ return unsupported
+ return {"data": [{"id": "org/Model"}]}
+
+ monkeypatch.setattr(start, "_http_json", http_json)
+ assert start._model_still_loaded(BASE, "key", "org/Model")
+ assert calls == [(BASE + "/api/inference/loaded-models", 5), (BASE + "/v1/models", 5)]
+
+
[email protected]("code", [401, 403, 500, 503])
+def test_discovery_does_not_hide_auth_or_server_errors(monkeypatch, code, capsys):
+ calls = []
+
+ def http_json(method, url, token, **kwargs):
+ calls.append(url)
+ raise urllib.error.HTTPError(
+ url, code, "failed", {}, io.BytesIO(b'{"detail":"discovery unavailable"}')
+ )
+
+ monkeypatch.setattr(start, "_http_json", http_json)
+ with pytest.raises(typer.Exit):
+ start._loaded_models(BASE, "key")
+ assert "Couldn't list models: discovery unavailable" in capsys.readouterr().err
+ assert calls == [BASE + "/api/inference/loaded-models"]
+
+
+def test_discovery_timeout_is_not_retried_as_old_server(monkeypatch, capsys):
+ calls = []
+
+ def http_json(method, url, token, **kwargs):
+ calls.append(url)
+ raise TimeoutError("timed out")
+
+ monkeypatch.setattr(start, "_http_json", http_json)
+ with pytest.raises(typer.Exit):
+ start._loaded_models(BASE, "key")
+ assert "Couldn't list models: timed out" in capsys.readouterr().err
+ assert calls == [BASE + "/api/inference/loaded-models"]
+
+
+ "data",
+ [
+ pytest.param([], id = "resident-listing-is-empty"),
+ pytest.param([{"id": "org/Model", "loaded": False}], id = "old-server-lists-only-unloaded"),
+ ],
+)
+def test_nothing_resident_reads_the_same_whatever_the_server_lists(monkeypatch, capsys, data):
+ def http_json(method, url, token, **kwargs):
+ return {"data": data}
+
+ monkeypatch.setattr(start, "_http_json", http_json)
+ with pytest.raises(typer.Exit):
+ start._resolve_model(BASE, "key", None)
+ assert "No model is loaded in Unsloth." in capsys.readouterr().err