Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

Studio: let full-scope keyless callers auto-switch models, and explain the refusal elsewhere (#11180)

Long Yixing7be1d1453a40
diff --git a/studio/backend/routes/inference.py b/studio/backend/routes/inference.py
index 4dc130b2bd2..83bcbc4ae5f 100644
--- a/studio/backend/routes/inference.py
+++ b/studio/backend/routes/inference.py
@@ -9031,6 +9031,16 @@ def disable_openai_auto_switch_for_request(scope) -> None:
scope[_DISABLE_OPENAI_AUTO_SWITCH_SCOPE_KEY] = True
+async def _keyless_caller_held_back(fastapi_request) -> bool:
+ """Keyless caller that may not POST /api/inference/load itself, so it may not switch models."""
+ from auth.authentication import request_admitted_without_credential
+ from utils.keyless_api_access import keyless_request_may_load_models
+
+ if not request_admitted_without_credential(fastapi_request):
+ return False
+ return not await keyless_request_may_load_models(fastapi_request)
+
+
def _automatic_model_load_may_run() -> bool:
"""True when a request can trigger an automatic load: either resolver-based
auto-switch is on, or a standalone idle TTL can reload an idle-freed model. The
@@ -9194,9 +9204,9 @@ async def _no_model_loaded_error(
):
"""``(status, detail)`` for the /v1 sites that fail because nothing is loaded.
- Changes only the case the generic text gets wrong (auto-switch on, a model
- named, that name resolving to nothing local, so the switch silently did
- nothing) into a 404 model_not_found. Everything else keeps ``status`` and the
+ Changes only the cases the generic text gets wrong (auto-switch on, a model named):
+ a name resolving to nothing local becomes a 404 model_not_found, and a downloaded one
+ a held-back keyless caller may not load says so. Everything else keeps ``status`` and the
:func:`_no_model_loaded_detail` text verbatim.
"""
from utils.openai_auto_switch_settings import get_openai_auto_switch_enabled
@@ -9224,6 +9234,11 @@ async def _no_model_loaded_error(
# Resident but on a backend this endpoint can't use, so "not downloaded" is false.
return status, _no_model_loaded_detail(base)
if await asyncio.to_thread(resolve_local_gguf, named) is not None:
+ if fastapi_request is not None and await _keyless_caller_held_back(fastapi_request):
+ return status, (
+ f"No model is loaded, and keyless API access cannot load '{named}' by "
+ "request. Load it in Unsloth Studio, or send an Unsloth API key."
+ )
# Resolvable but unloaded: the switch failed, which the generic text covers.
return status, _no_model_loaded_detail(base)
message = await _unavailable_model_message(named)
@@ -9859,7 +9874,19 @@ async def _reject_unservable_model(
gguf_hub_repo = await asyncio.to_thread(_resident_id_is_namespaced)
if not (quantified or here or gguf_hub_repo):
return
- if switchable:
+ if (
+ switchable
+ and fastapi_request is not None
+ and await _keyless_caller_held_back(fastapi_request)
+ ):
+ # Switching is on but not for this caller, so a retry can never succeed.
+ status_code, code = 404, "model_not_found"
+ message = (
+ f"The model '{requested_model}' is downloaded but not loaded, and keyless API "
+ "access can only use the loaded model. Send an Unsloth API key to switch "
+ "models by request, or load it in Unsloth Studio."
+ )
+ elif switchable:
# On disk and switching allowed, so the swap failed: the resident model is wrong weights.
status_code, code = 503, "model_switch_failed"
message = (
@@ -10125,11 +10152,8 @@ async def _maybe_auto_switch_model(
_claim_slot_for_non_preview(fastapi_request)
return
- from auth.authentication import request_admitted_without_credential
-
- keyless_caller = request_admitted_without_credential(fastapi_request)
- # the keyless dialog offers the loaded model, so a stranger swaps or fetches nothing
- if auto_switch_on and keyless_caller:
+ keyless_held_back = await _keyless_caller_held_back(fastapi_request)
+ if auto_switch_on and keyless_held_back:
auto_switch_on = False
if not idle_unload_is_configured():
await _reject_unservable_model(requested_model, fastapi_request)
@@ -10199,11 +10223,11 @@ async def _maybe_auto_switch_model(
else: # pre-3-tuple stash: fall back to the path as the override key
target_id, variant = last
override_id = target_id
- # A credential-less caller may restore only the model it explicitly
+ # A held-back keyless caller may restore only the model it explicitly
# named (or the reload-only sentinel used when the model is omitted).
# Check before loading so an unrelated name cannot trigger an expensive
# stash restore and only then receive the normal mismatch response.
- if keyless_caller and not reload_only:
+ if keyless_held_back and not reload_only:
requested_base, requested_variant = split_model_ref(requested_model)
if not _matches_any(
requested_base, (target_id, override_id, public_model_id(target_id))
diff --git a/studio/backend/tests/test_keyless_api_access.py b/studio/backend/tests/test_keyless_api_access.py
index e07e8ea38bc..5e6572068be 100644
--- a/studio/backend/tests/test_keyless_api_access.py
+++ b/studio/backend/tests/test_keyless_api_access.py
@@ -1026,8 +1026,9 @@ def test_protected_side_effect_guards_remain_wired():
assert "_require_a_credential_of_its_own" in inspect.getsource(auth.change_password)
assert all(
"request_admitted_without_credential" in inspect.getsource(handler)
- for handler in (inference._maybe_auto_switch_model, inference.openai_chat_completions)
+ for handler in (inference._keyless_caller_held_back, inference.openai_chat_completions)
)
+ assert "_keyless_caller_held_back" in inspect.getsource(inference._maybe_auto_switch_model)
assert all(
"authenticated_without_credential" in inspect.getsource(handler)
and "not no_credential" in inspect.getsource(handler)
@@ -1054,14 +1055,96 @@ def test_protected_side_effect_guards_remain_wired():
assert security.scheme_name == "HTTPBearer"
-def test_keyless_idle_restore_requires_the_requested_model(monkeypatch):
- from core.inference import llama_keepwarm as kw; import auth.authentication as authentication
+def _keyless_switch_hook(scope):
+ from studio.backend.tests import test_openai_auto_switch as auto
+ seed_user(); set_keyless_api_access(scope)
+ request = request_for(headers = {"Host": "localhost:8888", "Authorization": "Bearer no-key-required"})
+ assert admitted_without_session(request)
+ return lambda model: asyncio.run(auto.inference_route._maybe_auto_switch_model(model, request, "unsloth"))
+
+
[email protected]("scope", ["inference", "full"])
+def test_keyless_idle_restore_requires_the_requested_model_unless_the_scope_could_load(monkeypatch, scope):
+ from core.inference import llama_keepwarm as kw
from studio.backend.tests import test_openai_auto_switch as auto
backend = auto._FakeBackend(None); rec = auto._LoadRecorder(backend)
auto._wire(monkeypatch, enabled = False, resolves_to = None, backend = backend, recorder = rec)
monkeypatch.setattr(auto.settings, "idle_unload_is_configured", lambda: True)
monkeypatch.setattr(kw, "_last_unloaded_model", ("/cache/snap/A", "Q4_K_M", "org/A-GGUF"))
- monkeypatch.setattr(authentication, "request_admitted_without_credential", lambda _r: True)
- auto._run_hook("org/B-GGUF"); assert rec.calls == []
- auto._run_hook("org/A-GGUF:Q4_K_M"); assert len(rec.calls) == 1
+ hook = _keyless_switch_hook(scope)
+ hook("gpt-4o-mini")
+ if scope == "full":
+ assert len(rec.calls) == 1; return
+ assert rec.calls == []
+ hook("org/A-GGUF:Q4_K_M"); assert len(rec.calls) == 1
+
+
[email protected](("scope", "loads"), [("full", 1), ("inference", 0)])
+def test_keyless_auto_switch_loads_under_the_full_scope_only(monkeypatch, scope, loads):
+ from studio.backend.tests import test_openai_auto_switch as auto
+ backend, rec = auto._wired(monkeypatch, auto._FakeBackend(None), ("/cache/snap/A", "Q4_K_M", "org/A-GGUF"))
+ monkeypatch.setattr(auto.settings, "idle_unload_is_configured", lambda: False)
+ _keyless_switch_hook(scope)("org/A-GGUF")
+ assert len(rec.calls) == loads
+
+
[email protected](("loaded", "status"), [(None, 400), ("org/Other-GGUF", 404)])
+def test_a_held_back_keyless_caller_is_told_it_cannot_switch(monkeypatch, loaded, status):
+ from studio.backend.tests import test_openai_auto_switch as auto
+ backend, rec = auto._wired(monkeypatch, auto._FakeBackend(loaded), ("/cache/snap/A", "Q4_K_M", "org/A-GGUF"))
+ monkeypatch.setattr(auto.settings, "idle_unload_is_configured", lambda: False)
+ monkeypatch.setattr(auto.inference_route, "get_inference_backend", lambda: SimpleNamespace(active_model_name = None, models = {}))
+ seed_user(); set_keyless_api_access("inference")
+ request = request_for(headers = {"Host": "localhost:8888"})
+ with pytest.raises(HTTPException) as excinfo:
+ asyncio.run(auto.inference_route.openai_chat_completions(auto._chat_request(model = "org/A-GGUF"), request, "unsloth"))
+ assert (excinfo.value.status_code, rec.calls) == (status, [])
+ assert "keyless api access" in str(excinfo.value.detail).lower() and not excinfo.value.headers
+ set_keyless_api_access("full")
+ assert asyncio.run(auto.inference_route._no_model_loaded_error("No model loaded.", "org/A-GGUF", request, status = 400)) == (400, "No model loaded.")
+
+
+def test_the_keyless_load_probe_runs_off_the_event_loop(monkeypatch):
+ """A slow bind-host resolver on the loop would stall every in-flight generation."""
+ from routes import inference
+ from utils import keyless_api_access as keyless
+ seed_user(); set_keyless_api_access("full")
+ request = request_for(headers = {"Host": "localhost:8888"})
+ assert admitted_without_session(request)
+ # as the middleware leaves it, so the admission fallback skips the predicate
+ keyless.mark_keyless_admission(request, True)
+ threads: list[int] = []
+ real = keyless._keyless_request_allowed_for_scope
+ def _spy(*args, **kwargs):
+ threads.append(threading.get_ident()); return real(*args, **kwargs)
+ monkeypatch.setattr(keyless, "_keyless_request_allowed_for_scope", _spy)
+ async def _drive():
+ return await inference._keyless_caller_held_back(request), threading.get_ident()
+ held_back, loop_thread = asyncio.run(_drive())
+ assert held_back is False
+ assert threads and all(thread != loop_thread for thread in threads)
+
+
+def test_the_keyless_load_probe_waits_out_a_settings_refresh(monkeypatch):
+ """The sync settings read fails closed to "off" during a concurrent refresh, dropping the full-scope switch."""
+ from routes import inference
+ from utils import keyless_api_access as keyless
+ seed_user(); set_keyless_api_access("full")
+ request = request_for(headers = {"Host": "localhost:8888"})
+ assert admitted_without_session(request)
+ keyless.mark_keyless_admission(request, True)
+ real = keyless._read_settings_from_db
+ started = threading.Event()
+ def _slow():
+ started.set(); time.sleep(0.05); return real()
+ monkeypatch.setattr(keyless, "_read_settings_from_db", _slow)
+ with keyless._cache_lock:
+ keyless._cached_settings = None
+ refresher = threading.Thread(target = keyless.get_keyless_api_access_scope)
+ refresher.start(); assert started.wait(2)
+ try:
+ held_back = asyncio.run(inference._keyless_caller_held_back(request))
+ finally:
+ refresher.join()
+ assert held_back is False
# fmt: on
diff --git a/studio/backend/utils/keyless_api_access.py b/studio/backend/utils/keyless_api_access.py
index 0ab663c59f8..8e354865d31 100644
--- a/studio/backend/utils/keyless_api_access.py
+++ b/studio/backend/utils/keyless_api_access.py
@@ -505,7 +505,24 @@ def keyless_request_allowed(request: Any) -> bool:
return _keyless_request_allowed_for_scope(request, get_keyless_api_access_scope())
-def _keyless_request_allowed_for_scope(request: Any, scope: str) -> bool:
+async def keyless_request_may_load_models(request: Any) -> bool:
+ """Whether keyless access would admit this request's transport on ``POST /api/inference/load``.
+
+ Reads settings like admission (``_settings_async``): the sync read fails closed to ``off`` during a concurrent cache refresh.
+ """
+ from starlette.concurrency import run_in_threadpool
+
+ scope, _tools, _generation = await _settings_async()
+ return await run_in_threadpool(
+ _keyless_request_allowed_for_scope, request, scope, ("POST", "/api/inference/load")
+ )
+
+
+def _keyless_request_allowed_for_scope(
+ request: Any,
+ scope: str,
+ route: Optional[tuple[str, str]] = None,
+) -> bool:
if scope == KEYLESS_SCOPE_OFF:
return False
from auth.policy import installation_has_managed_accounts
@@ -513,8 +530,7 @@ def _keyless_request_allowed_for_scope(request: Any, scope: str) -> bool:
if installation_has_managed_accounts():
return False
asgi_scope = getattr(request, "scope", {})
- method = asgi_scope.get("method", "")
- path = asgi_scope.get("path", "")
+ method, path = route or (asgi_scope.get("method", ""), asgi_scope.get("path", ""))
root_path = asgi_scope.get("root_path", "")
if not scope_covers(scope, method, path, root_path):
return False
diff --git a/studio/frontend/src/features/settings/components/usage-examples.tsx b/studio/frontend/src/features/settings/components/usage-examples.tsx
index b533c0ddd6b..f63e8fadc37 100644
--- a/studio/frontend/src/features/settings/components/usage-examples.tsx
+++ b/studio/frontend/src/features/settings/components/usage-examples.tsx
@@ -646,7 +646,10 @@ export function UsageExamples({
const keylessBase =
!(useTunnel && cloudflareUrl) &&
keylessBaseEligible(base, keylessScope, keylessExposure);
- const model = useExampleModelName(keylessBase && !apiKey);
+ // Only the inference scope keeps a keyless caller on the loaded model; full can switch.
+ const model = useExampleModelName(
+ keylessBase && !apiKey && keylessScope === "inference",
+ );
const [statusAnswer, setStatusAnswer] = useState<{
key: string;
diff --git a/tests/studio/test_usage_examples_model_source_contract.py b/tests/studio/test_usage_examples_model_source_contract.py
index b19358cf5a2..73e82709a2d 100644
--- a/tests/studio/test_usage_examples_model_source_contract.py
+++ b/tests/studio/test_usage_examples_model_source_contract.py
@@ -229,7 +229,7 @@ def test_keyless_examples_match_transport_tool_and_full_scope_policy():
assert "if (isLoopbackHost(host)) return true;" in eligibility
assert 'return scope === "inference";' in eligibility
assert "!(useTunnel && cloudflareUrl)" in src
- assert "useExampleModelName(keylessBase && !apiKey)" in src
+ assert 'keylessBase && !apiKey && keylessScope === "inference"' in src
section = KEYLESS_SECTION_TSX.read_text(encoding = "utf-8")
assert "[cloudflareUrl, onSettingsChange]" in section
assert "delete" in section[section.find(" full: {") : section.find(" tools: {")]