Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

Studio: improve AMD NPU discovery, CLI support, and chat feedback (#13797)

oobabooga2de9425750aa
diff --git a/studio/backend/core/inference/npu_backend.py b/studio/backend/core/inference/npu_backend.py
index f98ddfbad25..4024d170a61 100644
--- a/studio/backend/core/inference/npu_backend.py
+++ b/studio/backend/core/inference/npu_backend.py
@@ -436,6 +436,11 @@ class LemonadeNpuBackend:
hardware = self.hardware()
if not hardware.get("present"):
raise NpuError("No AMD Ryzen AI NPU was found on this machine.")
+ if hardware.get("family") == "XDNA2" and hardware.get("enabled") is False:
+ raise NpuError(
+ "The AMD NPU is off by default. Start Unsloth Studio with "
+ "UNSLOTH_STUDIO_ENABLE_NPU=1 to use it."
+ )
if not hardware.get("supported"):
raise NpuError(
"FastFlowLM needs an XDNA 2 NPU (Ryzen AI 300/400 or Ryzen AI Max); "
@@ -444,6 +449,8 @@ class LemonadeNpuBackend:
def _ensure_running(self) -> LemonadeServer:
"""Start the installed runtime if needed and return its server."""
+ # A runtime enabled before the flag was turned off must not start again.
+ self._require_supported()
with self._lock:
if self._server is not None and self._server.is_alive():
return self._server
diff --git a/studio/backend/routes/inference.py b/studio/backend/routes/inference.py
index 4cd9aa5816f..c0a979e1ca1 100644
--- a/studio/backend/routes/inference.py
+++ b/studio/backend/routes/inference.py
@@ -10976,7 +10976,10 @@ async def _reject_unservable_model(
if is_npu_model_path(requested_model) and not await asyncio.to_thread(
_loaded_satisfies, requested_model
):
- message = f"The NPU model '{requested_model}' is not loaded. Load it in Unsloth Studio."
+ message = (
+ f"The NPU model '{requested_model}' is not loaded. Load it in Unsloth Studio, or turn "
+ "on 'Switch model by request' under Settings > API to load it from a request."
+ )
path = getattr(getattr(fastapi_request, "url", None), "path", None)
raise HTTPException(
status_code = 404,
@@ -11218,6 +11221,126 @@ def _audio_cpp_switch_target(requested_model: Optional[str]) -> Optional[tuple]:
return (model.id, model.variant.key, model.id, False)
+async def _refuse_unservable_npu_switch(
+ named_model: str,
+ fastapi_request: Request,
+ *,
+ gguf_only: bool,
+ needs_vision: bool,
+ needs_audio: bool,
+ rejection_prefix: str,
+ request_check: Optional[Callable[[Any], None]] = None,
+) -> None:
+ """400 for a request naming an NPU model that could not serve it once loaded.
+
+ ``request_check`` runs the serving route's own checks against the target's catalog entry.
+ """
+ from core.inference.npu_backend import NpuError, get_npu_backend, model_id_from_path
+
+ message = None
+ if gguf_only:
+ message = (
+ f"The NPU model '{named_model}' serves /v1/chat/completions only. "
+ "Load a GGUF model to use this endpoint."
+ )
+ elif needs_audio:
+ message = f"{rejection_prefix}The NPU model '{named_model}' takes no audio or video input."
+ elif (needs_vision or request_check is not None) and not await asyncio.to_thread(
+ _loaded_satisfies, named_model
+ ):
+ try:
+ model = await asyncio.to_thread(
+ get_npu_backend().loadable_model, model_id_from_path(named_model)
+ )
+ except NpuError:
+ model = None # Not downloaded or unknown: the load reports that itself.
+ if model is not None and needs_vision and not model.vision:
+ message = f"{rejection_prefix}The NPU model '{named_model}' cannot read images."
+ elif model is not None and request_check is not None:
+ request_check(model)
+ if message is None:
+ return
+ path = getattr(getattr(fastapi_request, "url", None), "path", None)
+ raise HTTPException(
+ status_code = 400,
+ detail = (
+ error_body_for_path(path, message, status = 400, code = "invalid_value", param = "model")
+ if isinstance(path, str)
+ else message
+ ),
+ )
+
+
+async def _auto_switch_npu_model(
+ named_model: str, fastapi_request: Request, current_subject: str, *, claim_resident: bool
+) -> None:
+ """Load a downloaded ``lemonade:<id>`` model a request names.
+
+ The caller has checked auto-switch is on. Same lock, gates, waiter count, cancellation and
+ deferred claim as a GGUF switch.
+ """
+ from core.inference.gpu_arbiter import require_no_foreign_generations
+ from core.inference.llama_keepwarm import inference_lifecycle_gate, model_load_gate
+ from utils.openai_auto_switch_settings import get_model_override, resolve_fit_max_seq_length
+
+ if await asyncio.to_thread(_loaded_satisfies, named_model):
+ return
+ # The context length saved in the model's settings, as a picker load sends it.
+ max_seq_length = resolve_fit_max_seq_length(get_model_override(named_model), is_gguf = False)
+ cancel_event = getattr(getattr(fastapi_request, "state", None), "generation_cancel_event", None)
+
+ def _raise_if_cancelled() -> None:
+ if cancel_event is not None and cancel_event.is_set():
+ raise HTTPException(status_code = 409, detail = "Generation cancelled")
+
+ _raise_if_cancelled()
+ switch_path = getattr(getattr(fastapi_request, "url", None), "path", None)
+ # Counted as a queued switch, so another switch's drain does not wait on this request.
+ key = _switch_key(named_model, None)
+ _note_switch_waiter(key, 1)
+ try:
+ async with _auto_switch_lock():
+ await _acquire_swap_gate()
+ try:
+ async with model_load_gate(), inference_lifecycle_gate():
+ # Re-read under the gate: another request may have loaded it while this one waited.
+ if await asyncio.to_thread(_loaded_satisfies, named_model):
+ return
+ require_no_foreign_generations(path = switch_path)
+ prior_marker = _get_preview_resident()
+ loaded_ok = False
+ try:
+ await _load_model_impl(
+ LoadRequest(model_path = named_model, max_seq_length = max_seq_length or 0),
+ fastapi_request,
+ current_subject,
+ current_request_counted = True,
+ **(
+ {"load_cancel_event": cancel_event}
+ if cancel_event is not None
+ else {}
+ ),
+ )
+ loaded_ok = True
+ _raise_if_cancelled()
+ finally:
+ if not claim_resident:
+ # Preview-owned until the route's 2xx claims it, so a later rejection
+ # leaves it evictable, as in the GGUF switch.
+ if loaded_ok or _loaded_slot_ident() == named_model:
+ _set_preview_resident(_loaded_slot_ident())
+ else:
+ _set_preview_resident(prior_marker)
+ finally:
+ # Deregister before releasing the gate, as the GGUF switch does.
+ _note_switch_waiter(key, -1)
+ key = None
+ _auto_switch_process_lock.release()
+ finally:
+ if key is not None:
+ _note_switch_waiter(key, -1)
+
+
async def _maybe_auto_switch_model(
requested_model: Optional[str],
fastapi_request: Request,
@@ -11239,6 +11362,8 @@ async def _maybe_auto_switch_model(
require_audio_workflow: Optional[str] = None,
speech_budget: Optional[dict] = None,
tool_images_only: bool = False,
+ serves_npu: bool = False,
+ npu_request_check: Optional[Callable[[Any], None]] = None,
gguf_target_check: Optional[Callable[[], None]] = None,
) -> None:
"""Load a downloaded local model named by an OpenAI request when auto-switch is on.
@@ -11268,7 +11393,9 @@ async def _maybe_auto_switch_model(
checks only the resolved target can decide; see
:func:`_preflight_audio_for_switch`. ``image_preflight`` does the same for
non-GGUF image count and byte validation. ``require_audio_workflow`` refuses a target that
- cannot run that Audio page workflow (``separate``, ``clone``...).
+ cannot run that Audio page workflow (``separate``, ``clone``...). ``serves_npu`` marks
+ the chat route, the only one an NPU model serves; ``npu_request_check``
+ refuses a request an NPU target would reject once loaded, given its catalog entry.
"""
# A text-only GGUF reads tool-result images as a note, so only audio or video needs its mmproj.
gguf_requires_vision = (
@@ -11412,6 +11539,34 @@ async def _maybe_auto_switch_model(
await _reject_unservable_model(requested_model, fastapi_request)
return
+ from core.inference.npu_backend import is_npu_model_path
+
+ if auto_switch_on and named_model and is_npu_model_path(named_model):
+ # Refused before the switch, so a request the NPU cannot serve never evicts the resident model.
+ await _refuse_unservable_npu_switch(
+ named_model,
+ fastapi_request,
+ # gguf_only alone misses routes that relax it for in-process models.
+ gguf_only = gguf_only or not serves_npu,
+ # Not gguf_requires_vision: NPU chat rejects tool-result images too.
+ needs_vision = require_vision,
+ needs_audio = bool(
+ require_audio_input
+ or require_video
+ or require_speech
+ or require_audio_workflow
+ or audio_preflight
+ ),
+ rejection_prefix = rejection_prefix,
+ request_check = npu_request_check,
+ )
+ await _auto_switch_npu_model(
+ named_model, fastapi_request, current_subject, claim_resident = claim_resident
+ )
+ if claim_resident:
+ _claim_slot_for_non_preview(fastapi_request)
+ return
+
async def _resolve_and_switch() -> None:
from core.inference.openai_auto_download import looks_like_quant, split_model_ref
@@ -18655,7 +18810,29 @@ async def _load_npu_model(
await asyncio.to_thread(npu.loadable_model, model_id)
except NpuError as exc:
raise HTTPException(status_code = 400, detail = str(exc)) from None
- if load_cancel_event is not None and load_cancel_event.is_set():
+ llama_backend = get_llama_cpp_backend()
+ unsloth_backend = _peek_inference_backend()
+ serving = bool(
+ getattr(llama_backend, "is_active", getattr(llama_backend, "is_loaded", False))
+ or (unsloth_backend is not None and unsloth_backend.active_model_name)
+ or resident is not None
+ )
+ _raise_if_sidecar_swap_in_progress()
+ if serving:
+ # Drain active inference before unloading, as for GGUF: an auto-switch has no
+ # on_reload_confirmed, so this drain alone keeps it from cutting a reply short.
+ if on_reload_confirmed is not None:
+ on_reload_confirmed(cancel = False)
+ await _wait_for_model_switch_idle(
+ current_request_counted = current_request_counted,
+ cancel_pending = on_reload_confirmed is not None and bool(request.force_cancel_active),
+ )
+ # The rechecks the GGUF and standard paths make before their point of no return: an
+ # auto-switch has no scoped attempt for the shutdown sweep to cancel.
+ _raise_if_sidecar_swap_in_progress()
+ with _scoped_load_attempts_lock:
+ shutting_down = _loads_shutting_down
+ if shutting_down or (load_cancel_event is not None and load_cancel_event.is_set()):
raise HTTPException(status_code = 409, detail = "Model load cancelled")
if on_reload_confirmed is not None:
on_reload_confirmed(cancel = True)
@@ -18665,7 +18842,7 @@ async def _load_npu_model(
timeout_s = _POST_CANCEL_DRAIN_TIMEOUT_S,
)
_set_preview_resident(None)
- await _unload_llama_before_standard_load(get_llama_cpp_backend())
+ await _unload_llama_before_standard_load(llama_backend)
backend = _peek_inference_backend()
if backend is not None and backend.active_model_name:
logger.info(
@@ -18677,8 +18854,17 @@ async def _load_npu_model(
await asyncio.to_thread(release_chat_gpu_claim)
if load_cancel_event is not None and load_cancel_event.is_set():
raise HTTPException(status_code = 409, detail = "Model load cancelled")
+ load = asyncio.ensure_future(asyncio.to_thread(npu.load, model_id, requested_ctx))
+ if load_cancel_event is not None:
+ # Stop the load mid-way, as a GGUF load polls its cancel event.
+ while not load.done():
+ await asyncio.wait({load}, timeout = 0.25)
+ # False until npu.load marks itself in flight, so keep asking.
+ if load_cancel_event.is_set() and not load.done():
+ if await asyncio.to_thread(npu.cancel_load, model_id):
+ break
try:
- await asyncio.to_thread(npu.load, model_id, requested_ctx)
+ await load
except NpuLoadCancelled:
raise HTTPException(status_code = 409, detail = "Model load cancelled") from None
except NpuError as exc:
@@ -29956,6 +30142,21 @@ _NPU_CUT_SHORT_ERROR = {
}
+def _npu_kv_cache_full(usage: Any) -> bool:
+ """Whether FastFlowLM's usage says the reply used the whole loaded context."""
+ if not isinstance(usage, dict):
+ return False
+ # active_kv_tokens is the KV position; total_tokens undercounts by the empty think block's
+ # three uncounted tokens when thinking is off.
+ total = usage.get("active_kv_tokens")
+ if not isinstance(total, int):
+ total = usage.get("total_tokens")
+ capacity = usage.get("max_kv_token_capacity")
+ return (
+ isinstance(total, int) and isinstance(capacity, int) and capacity > 0 and total >= capacity
+ )
+
+
class _NpuStreamRelay:
"""Normalize model IDs, enforce stop sequences, and collect non-streaming replies.
@@ -29997,16 +30198,22 @@ class _NpuStreamRelay:
self._pending = buffer[len(buffer) - keep :] if keep else ""
return buffer[: len(buffer) - keep] if keep else buffer
- def _chunk(self, delta: dict, finish_reason: Optional[str]) -> str:
- return "data: " + json.dumps(
- {
- "id": self.completion_id or "chatcmpl-npu",
- "object": "chat.completion.chunk",
- "created": self.created or int(time.time()),
- "model": self.public_model,
- "choices": [{"index": 0, "delta": delta, "finish_reason": finish_reason}],
- }
- )
+ def _chunk(
+ self,
+ delta: dict,
+ finish_reason: Optional[str],
+ tool_event: Optional[dict] = None,
+ ) -> str:
+ chunk = {
+ "id": self.completion_id or "chatcmpl-npu",
+ "object": "chat.completion.chunk",
+ "created": self.created or int(time.time()),
+ "model": self.public_model,
+ "choices": [{"index": 0, "delta": delta, "finish_reason": finish_reason}],
+ }
+ if tool_event is not None:
+ chunk["_toolEvent"] = tool_event
+ return "data: " + json.dumps(chunk)
def feed(self, line: str) -> list[str]:
"""The lines to send on for one line of the upstream stream."""
@@ -30074,6 +30281,10 @@ class _NpuStreamRelay:
elif isinstance(content, str):
self.content += content
if choice.get("finish_reason") is not None:
+ if choice["finish_reason"] == "stop" and _npu_kv_cache_full(chunk.get("usage")):
+ # FastFlowLM says "stop" when the reply filled the context.
+ choice["finish_reason"] = "length"
+ out.append(self._chunk({}, None, {"type": "context_window_exceeded"}))
self.finish_reason = choice["finish_reason"]
if self._pending:
# The reply ended partway into a stop sequence, so what was held back is text.
@@ -30085,7 +30296,9 @@ class _NpuStreamRelay:
return out
def completion(self) -> dict:
- message: dict[str, Any] = {"role": "assistant", "content": self.content or None}
+ # A tool-calling turn often streams only newlines as text; OpenAI sends null content there.
+ content = self.content if self.content.strip() or not self.tool_calls else ""
+ message: dict[str, Any] = {"role": "assistant", "content": content or None}
if self.reasoning:
message["reasoning_content"] = self.reasoning
if self.tool_calls:
@@ -30109,18 +30322,8 @@ class _NpuStreamRelay:
return body
-async def _npu_chat_completions(payload, request: Request, current_subject: str):
- """Proxy NPU chat, rejecting unsupported parameters.
-
- Always stream upstream: FastFlowLM's non-streaming replies mishandle reasoning
- and finish reasons on length cutoffs. Collect the stream for JSON callers.
- """
- from core.inference.npu_backend import NpuError, get_npu_backend
-
- try:
- upstream = await asyncio.to_thread(get_npu_backend().upstream)
- except NpuError as exc:
- raise HTTPException(status_code = 400, detail = str(exc)) from None
+def _refuse_npu_chat_request(payload, *, supports_vision: bool, supports_tools: bool) -> None:
+ """Reject a chat request FastFlowLM cannot serve; also run before a switch to an NPU model."""
if _wants_multiple_choices(payload):
_raise_unsupported_n("NPU chat completions")
if _response_format_constrains_decoding(payload):
@@ -30151,32 +30354,53 @@ async def _npu_chat_completions(payload, request: Request, current_subject: str)
raise HTTPException(
status_code = 400, detail = "Audio and video input are not supported on the NPU."
)
- if not upstream.supports_vision and _request_has_attached_image(payload):
+ if not supports_vision and _request_has_attached_image(payload):
raise HTTPException(
status_code = 400,
detail = "This NPU model does not read images. Load a vision model to send one.",
)
+ if payload.tools and payload.tool_choice != "none":
+ if payload.parallel_tool_calls is False:
+ _raise_unsupported_openai_parameter(
+ "parallel_tool_calls",
+ "FastFlowLM on the NPU cannot limit a reply to one tool call.",
+ )
+ if payload.tool_choice not in (None, "auto"):
+ _raise_unsupported_openai_parameter(
+ "tool_choice", "FastFlowLM on the NPU chooses its own tool; use auto or none."
+ )
+ if not supports_tools:
+ raise HTTPException(
+ status_code = 400,
+ detail = openai_error_body(
+ "This NPU model does not support tool calling.",
+ status = 400,
+ code = "unsupported_parameter",
+ param = "tools",
+ ),
+ )
+
+
+async def _npu_chat_completions(payload, request: Request, current_subject: str):
+ """Proxy NPU chat, rejecting unsupported parameters.
+
+ Always stream upstream: FastFlowLM's non-streaming replies mishandle reasoning
+ and finish reasons on length cutoffs. Collect the stream for JSON callers.
+ """
+ from core.inference.npu_backend import NpuError, get_npu_backend
+
+ try:
+ upstream = await asyncio.to_thread(get_npu_backend().upstream)
+ except NpuError as exc:
+ raise HTTPException(status_code = 400, detail = str(exc)) from None
+ _refuse_npu_chat_request(
+ payload,
+ supports_vision = upstream.supports_vision,
+ supports_tools = upstream.supports_tools,
+ )
if payload.tool_choice == "none":
# FastFlowLM 1.0.3 ignores tool_choice and calls a tool anyway.
payload.tools = None
- elif payload.tools and payload.parallel_tool_calls is False:
- _raise_unsupported_openai_parameter(
- "parallel_tool_calls", "FastFlowLM on the NPU cannot limit a reply to one tool call."
- )
- elif payload.tools and payload.tool_choice not in (None, "auto"):
- _raise_unsupported_openai_parameter(
- "tool_choice", "FastFlowLM on the NPU chooses its own tool; use auto or none."
- )
- elif payload.tools and not upstream.supports_tools:
- raise HTTPException(
- status_code = 400,
- detail = openai_error_body(
- "This NPU model does not support tool calling.",
- status = 400,
- code = "unsupported_parameter",
- param = "tools",
- ),
- )
_normalize_chat_reasoning_controls(payload)
if not upstream.supports_reasoning:
@@ -31362,6 +31586,11 @@ async def produce_openai_chat_completions(
if _needs_image and _pre_parsed is not None:
_image_preflight = _chat_image_preflight(payload, _pre_parsed[2])
+ def _npu_switch_check(model) -> None:
+ # NPU chat runs no MCP tool loop, so it refuses an MCP image after the switch.
+ _refuse_unused_mcp_image(_mcp_image)
+ _refuse_npu_chat_request(payload, supports_vision = model.vision, supports_tools = model.tools)
+
# Defer the resident claim: chat has several post-switch capability checks that can still
# reject (Whisper without audio, n>1 on a non-GGUF backend, tool or response_format
# passthrough), so claiming here could strand a preview-owned model for a request that
@@ -31380,6 +31609,8 @@ async def produce_openai_chat_completions(
audio_preflight = _audio_preflight,
image_preflight = _image_preflight,
tool_images_only = _tool_images_only,
+ serves_npu = True,
+ npu_request_check = _npu_switch_check,
)
if _audio_preflight is not None:
_predecoded_audio = _audio_preflight.get("decoded")
@@ -44445,6 +44676,16 @@ async def anthropic_messages(
and not _in_process_model_resident()
and not _automatic_model_load_may_run()
):
+ _npu_model = _resident_npu_model()
+ if _npu_model is not None:
+ raise HTTPException(
+ status_code = 400,
+ detail = (
+ f"The loaded NPU model ({_npu_model.model_path}) serves "
+ "/v1/chat/completions only. Load a GGUF model to use the "
+ "Anthropic Messages API."
+ ),
+ )
raise HTTPException(
status_code = 503,
detail = _no_model_loaded_detail("No GGUF model loaded. Load a GGUF model first."),
diff --git a/studio/backend/tests/test_mcp_image.py b/studio/backend/tests/test_mcp_image.py
index f05dbb2f70d..9dee87ad96d 100644
--- a/studio/backend/tests/test_mcp_image.py
+++ b/studio/backend/tests/test_mcp_image.py
@@ -683,14 +683,15 @@ def test_every_local_exit_without_a_tool_loop_refuses_the_image():
"_refuse_unused_mcp_image(_mcp_image, _catalog_names(_sf_tools_to_use) if _sf_use_tools else None)",
):
assert guard in statements
- # NPU, both speech models, audio input and the GGUF passthrough return before either loop.
+ # The NPU switch check, NPU, both speech models, audio input and the GGUF passthrough
+ # return before either loop.
calls = [
node
for node in ast.walk(ast.parse(src))
if isinstance(node, ast.Call)
and getattr(node.func, "id", None) == "_refuse_unused_mcp_image"
]
- assert len(calls) == 7
+ assert len(calls) == 8
# The forced image approval parks the GGUF stream, so the slot must be tracked for reclaim.
slot = [
ast.unparse(node.value)
diff --git a/studio/backend/tests/test_npu_backend.py b/studio/backend/tests/test_npu_backend.py
index 14c490111c3..1b5900e1123 100644
--- a/studio/backend/tests/test_npu_backend.py
+++ b/studio/backend/tests/test_npu_backend.py
@@ -146,6 +146,23 @@ def test_unsupported_npu_is_refused_before_any_download(tmp_path, monkeypatch):
assert installer.installs == 0
[email protected]("call", ["enable", "catalog"])
+def test_an_npu_left_off_is_refused_with_the_flag_named(tmp_path, monkeypatch, call):
+ # catalog: a runtime enabled before the flag went off must not start again.
+ installer = _Installer(_binary(tmp_path))
+ monkeypatch.setattr(nb, "_installer_module", lambda: installer)
+ monkeypatch.setattr(
+ nb,
+ "detect_amd_npu",
+ lambda: {"present": True, "enabled": False, "supported": False, "family": "XDNA2"},
+ )
+ backend = nb.LemonadeNpuBackend(root = tmp_path / "lemonade")
+ with pytest.raises(nb.NpuError, match = "UNSLOTH_STUDIO_ENABLE_NPU=1"):
+ getattr(backend, call)()
+ assert installer.installs == 0
+ assert backend.status()["supported"] is False
+
+
def test_catalog_keeps_flm_chat_models_only(npu):
npu.enable()
ids = [model.id for model in npu.catalog()]
diff --git a/studio/backend/tests/test_npu_chat_route.py b/studio/backend/tests/test_npu_chat_route.py
index 922b9c85570..a751cb6c151 100644
--- a/studio/backend/tests/test_npu_chat_route.py
+++ b/studio/backend/tests/test_npu_chat_route.py
@@ -69,7 +69,38 @@ _FLM_TIMINGS = {
}
+# The end of a reply whose reasoning filled the loaded context, captured on Strix Halo: FastFlowLM
+# says "stop" with the KV cache full.
+_FULL_USAGE = {
+ "prompt_tokens": 45,
+ "completion_tokens": 8147,
+ "total_tokens": 8192,
+ "active_kv_tokens": 8192,
+ "max_kv_token_capacity": 8192,
+ "kv_token_occupancy_rate_percentage": 100.0,
+}
+
+
def _script(prompt: str) -> str:
+ if "FULL" in prompt:
+ return (
+ _chunk({"reasoning_content": " Wait, no!"})
+ + _chunk({"content": None}, "stop", _FULL_USAGE)
+ + "data: [DONE]\n\n"
+ )
+ if "WSTOOL" in prompt:
+ call = {
+ "index": 0,
+ "id": "call_1",
+ "type": "function",
+ "function": {"name": "get_weather", "arguments": '{"city":"Paris"}'},
+ }
+ return (
+ _chunk({"content": "\n\n\n"})
+ + _chunk({"tool_calls": [call]})
+ + _chunk({}, "tool_calls", _USAGE)
+ + "data: [DONE]\n\n"
+ )
if "FAULT" in prompt:
return (
_chunk({"content": "par"})
@@ -826,6 +857,161 @@ def test_an_npu_load_it_would_refuse_keeps_the_gpu_resident(monkeypatch):
assert teardowns == []
+def test_an_npu_load_drains_active_inference_before_unloading(monkeypatch):
+ # An auto-switch passes no on_reload_confirmed, so without the drain it cut a reply short.
+ import routes.inference as routes
+ from core.inference import npu_backend as nb
+ from models.inference import LoadRequest
+
+ order: list[str] = []
+ model = SimpleNamespace(id = "qwen3-0.6b-FLM", model_path = "lemonade:qwen3-0.6b-FLM")
+
+ class _Npu:
+ def resident(self):
+ return None
+
+ def loadable_model(self, model_id):
+ return model
+
+ def load(self, model_id, ctx):
+ order.append("load")
+
+ async def _drain(**kwargs):
+ order.append("drain")
+
+ async def _unload_gpu(_backend):
+ order.append("unload")
+
+ monkeypatch.setattr(nb, "get_npu_backend", lambda: _Npu())
+ monkeypatch.setattr(routes, "get_llama_cpp_backend", lambda: SimpleNamespace(is_active = True))
+ monkeypatch.setattr(routes, "_peek_inference_backend", lambda: None)
+ monkeypatch.setattr(routes, "_wait_for_model_switch_idle", _drain)
+ monkeypatch.setattr(routes, "_unload_llama_before_standard_load", _unload_gpu)
+ monkeypatch.setattr(routes, "release_chat_gpu_claim", lambda: True)
+ with pytest.raises(HTTPException):
+ # The stub never becomes resident, so the load ends in a 409 after the steps above.
+ asyncio.run(
+ routes._load_npu_model(
+ LoadRequest(model_path = model.model_path),
+ current_request_counted = True,
+ on_reload_confirmed = None,
+ load_cancel_event = None,
+ )
+ )
+ assert order == ["drain", "unload", "load"]
+
+
[email protected]("blocker", ["sidecar_swap", "shutdown"])
+def test_an_npu_load_refuses_before_unloading_during_a_swap_or_shutdown(monkeypatch, blocker):
+ # The NPU branch skipped the rechecks the other load paths make, so it unloaded the resident
+ # model under a transformers install or after shutdown had torn everything down.
+ import routes.inference as routes
+ from core.inference import npu_backend as nb
+ from models.inference import LoadRequest
+ from utils import transformers_version
+
+ order: list[str] = []
+ model = SimpleNamespace(id = "qwen3-0.6b-FLM", model_path = "lemonade:qwen3-0.6b-FLM")
+
+ class _Npu:
+ def resident(self):
+ return None
+
+ def loadable_model(self, model_id):
+ return model
+
+ def load(self, model_id, ctx):
+ order.append("load")
+
+ async def _drain(**kwargs):
+ order.append("drain")
+
+ async def _unload_gpu(_backend):
+ order.append("unload")
+
+ monkeypatch.setattr(nb, "get_npu_backend", lambda: _Npu())
+ monkeypatch.setattr(routes, "get_llama_cpp_backend", lambda: SimpleNamespace(is_active = True))
+ monkeypatch.setattr(routes, "_peek_inference_backend", lambda: None)
+ monkeypatch.setattr(routes, "_wait_for_model_switch_idle", _drain)
+ monkeypatch.setattr(routes, "_unload_llama_before_standard_load", _unload_gpu)
+ monkeypatch.setattr(routes, "release_chat_gpu_claim", lambda: True)
+ if blocker == "sidecar_swap":
+ monkeypatch.setattr(transformers_version, "sidecar_swap_in_progress", lambda: True)
+ else:
+ monkeypatch.setattr(routes, "_loads_shutting_down", True)
+ with pytest.raises(HTTPException) as excinfo:
+ asyncio.run(
+ routes._load_npu_model(
+ LoadRequest(model_path = model.model_path),
+ current_request_counted = True,
+ on_reload_confirmed = None,
+ load_cancel_event = None,
+ )
+ )
+ assert excinfo.value.status_code == 409
+ assert "unload" not in order and "load" not in order
+
+
+def test_a_stop_during_an_npu_load_cancels_it(monkeypatch):
+ # The cancel event was read only before and after the blocking load, so a Stop waited it out.
+ import routes.inference as routes
+ from core.inference import npu_backend as nb
+ from models.inference import LoadRequest
+
+ model = SimpleNamespace(id = "qwen3-0.6b-FLM", model_path = "lemonade:qwen3-0.6b-FLM")
+ loading = threading.Event()
+ stopped = threading.Event()
+ stop = threading.Event()
+
+ class _Npu:
+ def resident(self):
+ return None
+
+ def loadable_model(self, model_id):
+ return model
+
+ def load(self, model_id, ctx):
+ loading.set()
+ if not stopped.wait(10):
+ raise AssertionError("the load was never cancelled")
+ raise nb.NpuLoadCancelled("cancelled")
+
+ def cancel_load(self, model_id):
+ # The first ask lands before the load is registered, as when Stop beats npu.load.
+ asks.append(model_id)
+ if len(asks) == 1:
+ return False
+ stopped.set()
+ return True
+
+ asks: list[str] = []
+
+ def _stop_once_loading():
+ loading.wait(10)
+ stop.set()
+
+ async def _nothing(*_args, **_kwargs):
+ return None
+
+ monkeypatch.setattr(nb, "get_npu_backend", lambda: _Npu())
+ monkeypatch.setattr(routes, "get_llama_cpp_backend", lambda: SimpleNamespace(is_active = False))
+ monkeypatch.setattr(routes, "_peek_inference_backend", lambda: None)
+ monkeypatch.setattr(routes, "_unload_llama_before_standard_load", _nothing)
+ monkeypatch.setattr(routes, "release_chat_gpu_claim", lambda: True)
+ threading.Thread(target = _stop_once_loading, daemon = True).start()
+ with pytest.raises(HTTPException) as excinfo:
+ asyncio.run(
+ routes._load_npu_model(
+ LoadRequest(model_path = model.model_path),
+ current_request_counted = True,
+ on_reload_confirmed = None,
+ load_cancel_event = stop,
+ )
+ )
+ assert excinfo.value.status_code == 409
+ assert stopped.is_set() and len(asks) == 2
+
+
def test_a_reply_cut_short_is_not_reported_complete(flm):
flm()
status, body = _call(stream = False, messages = [{"role": "user", "content": "CUT"}])
@@ -1278,3 +1464,57 @@ def test_a_tool_loop_turn_dropped_after_a_tool_call_is_cut_short():
plain = _TurnFinish()
plain.see(_chunk({}, "stop").strip())
assert plain.finished is True
+
+
+def test_a_reply_that_fills_the_context_says_so(flm):
+ flm(reasoning = True)
+ full = [{"role": "user", "content": "FULL"}]
+ status, lines = _call(stream = True, messages = full)
+ assert status == 200
+ chunks = _data(lines)
+ assert [c["_toolEvent"] for c in chunks if "_toolEvent" in c] == [
+ {"type": "context_window_exceeded"}
+ ]
+ finishes = [
+ c["choices"][0]["finish_reason"] for c in chunks if c["choices"][0]["finish_reason"]
+ ]
+ assert finishes == ["length"]
+ _, body = _call(stream = False, messages = full)
+ assert body["choices"][0]["finish_reason"] == "length"
+
+
+ ("usage", "full"),
+ [
+ (_FULL_USAGE, True),
+ # Thinking off: FastFlowLM decodes an empty think block, four tokens counted as one.
+ ({**_FULL_USAGE, "completion_tokens": 8144, "total_tokens": 8189}, True),
+ ({"total_tokens": 8192, "max_kv_token_capacity": 8192}, True),
+ (_FLM_USAGE, False),
+ ({"total_tokens": 8192}, False),
+ (None, False),
+ ],
+)
+def test_a_full_kv_cache_reads_the_kv_position(usage, full):
+ import routes.inference as ri
+ assert ri._npu_kv_cache_full(usage) is full
+
+
+def test_a_reply_with_room_left_keeps_its_stop(flm):
+ flm()
+ status, lines = _call(stream = True, messages = [{"role": "user", "content": "TIMED"}])
+ assert status == 200
+ chunks = _data(lines)
+ assert not any("_toolEvent" in c for c in chunks)
+ assert chunks[-1]["choices"][0]["finish_reason"] == "stop"
+
+
+def test_whitespace_beside_a_tool_call_is_no_content(flm):
+ flm(tools = True)
+ tools = [
+ {"type": "function", "function": {"name": "get_weather", "parameters": {"type": "object"}}}
+ ]
+ _, body = _call(stream = False, tools = tools, messages = [{"role": "user", "content": "WSTOOL"}])
+ message = body["choices"][0]["message"]
+ assert message["content"] is None
+ assert message["tool_calls"][0]["function"]["name"] == "get_weather"
diff --git a/studio/backend/tests/test_npu_detect.py b/studio/backend/tests/test_npu_detect.py
index 36a269e20bf..503b3a6fbeb 100644
--- a/studio/backend/tests/test_npu_detect.py
+++ b/studio/backend/tests/test_npu_detect.py
@@ -78,15 +78,39 @@ def test_only_xdna2_is_supported(tmp_path, monkeypatch, device, supported):
monkeypatch.setattr(
npu, "_linux_probe", lambda: probe(tmp_path / "devices", tmp_path / "module")
)
+ monkeypatch.setenv(npu.ENABLE_ENV, "1")
info = npu.detect_amd_npu()
assert info["present"] is True
assert info["supported"] is supported
[email protected]("value", [None, "", "0", "false"])
+def test_an_xdna2_npu_is_off_without_the_flag(tmp_path, monkeypatch, value):
+ _device(tmp_path, "0000:c4:00.1", "1022", "17f0", driver = "amdxdna")
+ probe = npu._linux_probe
+ monkeypatch.setattr(npu.sys, "platform", "linux")
+ monkeypatch.setattr(
+ npu, "_linux_probe", lambda: probe(tmp_path / "devices", tmp_path / "module")
+ )
+ if value is None:
+ monkeypatch.delenv(npu.ENABLE_ENV, raising = False)
+ else:
+ monkeypatch.setenv(npu.ENABLE_ENV, value)
+ info = npu.detect_amd_npu()
+ assert info["present"] is True and info["family"] == "XDNA2"
+ assert info["enabled"] is False and info["supported"] is False
+
+
def test_detection_never_raises(monkeypatch):
def broken():
raise PermissionError("sysfs")
monkeypatch.setattr(npu.sys, "platform", "linux")
monkeypatch.setattr(npu, "_linux_probe", broken)
- assert npu.detect_amd_npu() == {"present": False, "error": "sysfs", "supported": False}
+ monkeypatch.delenv(npu.ENABLE_ENV, raising = False)
+ assert npu.detect_amd_npu() == {
+ "present": False,
+ "error": "sysfs",
+ "enabled": False,
+ "supported": False,
+ }
diff --git a/studio/backend/tests/test_openai_auto_switch.py b/studio/backend/tests/test_openai_auto_switch.py
index 7244ca532dc..dda419fbcf3 100644
--- a/studio/backend/tests/test_openai_auto_switch.py
+++ b/studio/backend/tests/test_openai_auto_switch.py
@@ -319,8 +319,8 @@ async def _noop_reject(*_args, **_kwargs):
return None
-def _run_hook(model = "some/model"):
- asyncio.run(inference_route._maybe_auto_switch_model(model, object(), "tester"))
+def _run_hook(model = "some/model", **kwargs):
+ asyncio.run(inference_route._maybe_auto_switch_model(model, object(), "tester", **kwargs))
def test_flag_off_never_loads(monkeypatch):
@@ -4410,6 +4410,8 @@ def test_chat_audio_input_guards_target_before_switch(monkeypatch):
audio_preflight = None,
image_preflight = None,
tool_images_only = False,
+ serves_npu = False,
+ npu_request_check = None,
):
captured.update(
require_vision = require_vision,
@@ -9921,6 +9923,8 @@ def test_a_video_request_labels_the_switch_refusal_video(monkeypatch):
audio_preflight = None,
image_preflight = None,
tool_images_only = False,
+ serves_npu = False,
+ npu_request_check = None,
):
captured.update(
require_vision = require_vision,
@@ -12363,6 +12367,281 @@ def test_tensor_split_requires_matching_gpu_ids_at_the_settings_boundary(ids):
)
+_NPU_MODEL = "lemonade:qwen3-0.6b-FLM"
+
+
+def test_an_npu_model_named_by_a_request_switches_when_on(monkeypatch):
+ # The GGUF resolver cannot find an NPU id, so this used to 404 even with the switch on.
+ backend, rec = _wired(monkeypatch, _FakeBackend("unsloth/A-GGUF"), None)
+ _run_hook(_NPU_MODEL, serves_npu = True)
+ assert [call.model_path for call in rec.calls] == [_NPU_MODEL]
+
+
+def test_an_npu_switch_loads_with_the_saved_context_length(monkeypatch):
+ backend, rec = _wired(monkeypatch, _FakeBackend("unsloth/A-GGUF"), None)
+ monkeypatch.setattr(
+ settings,
+ "get_model_override",
+ lambda model_id: {"max_seq_length": 16384} if model_id == _NPU_MODEL else {},
+ )
+ _run_hook(_NPU_MODEL, serves_npu = True)
+ assert [(call.model_path, call.max_seq_length) for call in rec.calls] == [(_NPU_MODEL, 16384)]
+
+
+def test_a_deferred_npu_switch_leaves_the_model_preview_owned(monkeypatch):
+ # Chat defers its claim and can still reject after the switch (n > 1 on the NPU); unmarked, the
+ # NPU model then read as Studio-owned and every public preview got 503.
+ _wired(monkeypatch, _FakeBackend("unsloth/A-GGUF"), None)
+ monkeypatch.setattr(inference_route, "_preview_resident_ident", None)
+ asyncio.run(
+ inference_route._maybe_auto_switch_model(
+ _NPU_MODEL, object(), "tester", claim_resident = False, serves_npu = True
+ )
+ )
+ assert inference_route._get_preview_resident() == _NPU_MODEL
+
+
+def test_an_npu_switch_keeps_the_setting_read_on_entry(monkeypatch):
+ # Turned off mid-request, a second read loaded nothing and the resident GGUF answered instead.
+ _, rec = _wired(monkeypatch, _FakeBackend("unsloth/A-GGUF"), None)
+ state = {"on": True}
+ monkeypatch.setattr(settings, "get_openai_auto_switch_enabled", lambda: state["on"])
+ refuse = inference_route._refuse_unservable_npu_switch
+
+ async def _refuse_then_turn_off(*args, **kwargs):
+ await refuse(*args, **kwargs)
+ state["on"] = False
+
+ monkeypatch.setattr(inference_route, "_refuse_unservable_npu_switch", _refuse_then_turn_off)
+ _run_hook(_NPU_MODEL, serves_npu = True)
+ assert [call.model_path for call in rec.calls] == [_NPU_MODEL]
+
+
+def test_an_npu_model_named_with_the_switch_off_says_how_to_load_it(monkeypatch):
+ backend, rec = _wired(monkeypatch, _FakeBackend("unsloth/A-GGUF"), None, enabled = False)
+ with pytest.raises(HTTPException) as excinfo:
+ _run_hook(_NPU_MODEL)
+ assert excinfo.value.status_code == 404
+ assert "Switch model by request" in str(excinfo.value.detail)
+ assert rec.calls == []
+
+
+def test_a_resident_npu_model_is_not_reloaded(monkeypatch):
+ backend, rec = _wired(monkeypatch, _FakeBackend("unsloth/A-GGUF"), None)
+ monkeypatch.setattr(inference_route, "_loaded_satisfies", lambda model: model == _NPU_MODEL)
+ _run_hook(_NPU_MODEL, serves_npu = True)
+ assert rec.calls == []
+
+
+def test_messages_with_an_npu_model_loaded_names_it(monkeypatch):
+ # "No GGUF model loaded" read as "nothing is loaded" while the NPU model was serving.
+ from models.inference import AnthropicMessagesRequest
+
+ _wired(monkeypatch, _FakeBackend(None), None, enabled = False)
+ monkeypatch.setattr(inference_route, "_automatic_model_load_may_run", lambda: False)
+ monkeypatch.setattr(
+ inference_route,
+ "_resident_npu_model",
+ lambda: SimpleNamespace(model_path = _NPU_MODEL),
+ )
+ request = SimpleNamespace(
+ state = SimpleNamespace(), url = SimpleNamespace(path = "/v1/messages"), method = "POST"
+ )
+ payload = AnthropicMessagesRequest(max_tokens = 16, messages = [{"role": "user", "content": "hi"}])
+ with pytest.raises(HTTPException) as excinfo:
+ asyncio.run(inference_route.anthropic_messages(payload, request, "tester"))
+ assert excinfo.value.status_code == 400
+ assert _NPU_MODEL in excinfo.value.detail
+ assert "No GGUF model loaded" not in excinfo.value.detail
+
+
+def test_messages_with_switching_on_and_an_npu_model_loaded_names_it(monkeypatch):
+ # The early check is skipped once automatic loads may run; the later one must say the same.
+ from models.inference import AnthropicMessagesRequest
+
+ _wired(monkeypatch, _FakeBackend(None), None, enabled = True)
+ monkeypatch.setattr(
+ inference_route,
+ "_resident_npu_model",
+ lambda: SimpleNamespace(model_path = _NPU_MODEL, id = "qwen3-0.6b-FLM"),
+ )
+ monkeypatch.setattr(inference_route, "_loaded_satisfies", lambda model: model == _NPU_MODEL)
+ request = SimpleNamespace(
+ state = SimpleNamespace(), url = SimpleNamespace(path = "/v1/messages"), method = "POST"
+ )
+ payload = AnthropicMessagesRequest(
+ model = _NPU_MODEL, max_tokens = 16, messages = [{"role": "user", "content": "hi"}]
+ )
+ with pytest.raises(HTTPException) as excinfo:
+ asyncio.run(inference_route.anthropic_messages(payload, request, "tester"))
+ assert excinfo.value.status_code == 400
+ assert _NPU_MODEL in str(excinfo.value.detail)
+
+
+def _npu_catalog_entry(
+ monkeypatch,
+ *,
+ vision,
+ tools = False,
+):
+ from core.inference import npu_backend as nb
+ class _Npu:
+ def loadable_model(self, model_id):
+ return SimpleNamespace(id = model_id, vision = vision, tools = tools)
+
+ monkeypatch.setattr(nb, "get_npu_backend", lambda: _Npu())
+
+
+ ("kwargs", "words"),
+ [
+ ({"serves_npu": False}, "serves /v1/chat/completions only"),
+ ({"gguf_only": True}, "serves /v1/chat/completions only"),
+ ({"require_audio_input": True}, "takes no audio or video input"),
+ ({"require_video": True}, "takes no audio or video input"),
+ ({"require_vision": True}, "cannot read images"),
+ ],
+)
+def test_a_request_the_npu_model_cannot_serve_never_switches(monkeypatch, kwargs, words):
+ # Loading first evicted the resident GGUF, then the endpoint failed on the NPU model anyway.
+ backend, rec = _wired(monkeypatch, _FakeBackend("unsloth/A-GGUF"), None)
+ _npu_catalog_entry(monkeypatch, vision = False)
+ with pytest.raises(HTTPException) as excinfo:
+ asyncio.run(
+ inference_route._maybe_auto_switch_model(
+ _NPU_MODEL, object(), "tester", **{"serves_npu": True, **kwargs}
+ )
+ )
+ assert excinfo.value.status_code == 400
+ assert words in str(excinfo.value.detail)
+ assert rec.calls == []
+ assert backend.model_identifier == "unsloth/A-GGUF"
+
+
+ ("fields", "param"),
+ [
+ (
+ {"tools": [{"type": "function", "function": {"name": "f", "parameters": {}}}]},
+ "tools",
+ ),
+ ({"n": 2}, "n"),
+ ],
+)
+def test_a_chat_request_the_npu_model_would_refuse_never_switches(monkeypatch, fields, param):
+ # NPU chat checked these only after the switch had already evicted the resident GGUF.
+ backend, rec = _wired(monkeypatch, _FakeBackend("unsloth/A-GGUF"), None)
+ _npu_catalog_entry(monkeypatch, vision = False, tools = False)
+ payload = _chat_request(model = _NPU_MODEL, **fields)
+ with pytest.raises(HTTPException) as excinfo:
+ asyncio.run(inference_route.openai_chat_completions(payload, object(), "tester"))
+ assert excinfo.value.status_code == 400
+ assert param in str(excinfo.value.detail)
+ assert rec.calls == []
+ assert backend.model_identifier == "unsloth/A-GGUF"
+
+
[email protected]("route", ["/v1/responses", "/v1/messages"])
+def test_a_route_the_npu_model_cannot_serve_never_switches_to_it(monkeypatch, route):
+ # Both routes relax gguf_only for in-process models, so the NPU switch evicted the GGUF first.
+ from starlette.requests import Request
+
+ from models.inference import AnthropicMessagesRequest
+
+ backend, rec = _wired(monkeypatch, _FakeBackend("unsloth/A-GGUF"), None)
+ _npu_catalog_entry(monkeypatch, vision = True, tools = True)
+ request = Request({"type": "http", "headers": [], "path": route, "method": "POST"})
+ if route == "/v1/responses":
+ payload = ResponsesRequest(model = _NPU_MODEL, stream = True, input = "hi")
+ call = inference_route.openai_responses(payload, request, "tester")
+ else:
+ payload = AnthropicMessagesRequest(
+ model = _NPU_MODEL, max_tokens = 16, messages = [{"role": "user", "content": "hi"}]
+ )
+ call = inference_route.anthropic_messages(payload, request, "tester")
+ with pytest.raises(Exception) as excinfo:
+ asyncio.run(call)
+ assert rec.calls == []
+ assert backend.model_identifier == "unsloth/A-GGUF"
+ assert isinstance(excinfo.value, HTTPException) and excinfo.value.status_code == 400
+ assert "serves /v1/chat/completions only" in str(excinfo.value.detail)
+
+
+def test_an_mcp_image_request_never_switches_to_an_npu_model(monkeypatch):
+ # NPU chat runs no MCP tool loop, so it refused the image only after evicting the GGUF.
+ backend, rec = _wired(monkeypatch, _FakeBackend("unsloth/A-GGUF"), None)
+ _npu_catalog_entry(monkeypatch, vision = True, tools = True)
+
+ async def _an_mcp_image(*_args):
+ return object()
+
+ monkeypatch.setattr(inference_route, "_request_mcp_image", _an_mcp_image)
+ payload = _chat_request(model = _NPU_MODEL)
+ with pytest.raises(HTTPException) as excinfo:
+ asyncio.run(inference_route.openai_chat_completions(payload, object(), "tester"))
+ assert excinfo.value.status_code == 400
+ assert "MCP" in str(excinfo.value.detail)
+ assert rec.calls == []
+ assert backend.model_identifier == "unsloth/A-GGUF"
+
+
+def test_an_image_request_switches_to_a_vision_npu_model(monkeypatch):
+ backend, rec = _wired(monkeypatch, _FakeBackend("unsloth/A-GGUF"), None)
+ _npu_catalog_entry(monkeypatch, vision = True)
+ asyncio.run(
+ inference_route._maybe_auto_switch_model(
+ _NPU_MODEL, object(), "tester", require_vision = True, serves_npu = True
+ )
+ )
+ assert [call.model_path for call in rec.calls] == [_NPU_MODEL]
+
+
+def test_a_cancelled_request_never_switches_to_an_npu_model(monkeypatch):
+ backend, rec = _wired(monkeypatch, _FakeBackend("unsloth/A-GGUF"), None)
+ cancel = threading.Event()
+ cancel.set()
+ request = SimpleNamespace(state = SimpleNamespace(generation_cancel_event = cancel))
+ with pytest.raises(HTTPException) as excinfo:
+ asyncio.run(
+ inference_route._maybe_auto_switch_model(_NPU_MODEL, request, "tester", serves_npu = True)
+ )
+ assert excinfo.value.status_code == 409
+ assert rec.calls == []
+
+
+def test_a_tool_result_image_never_switches_to_a_text_only_npu_model(monkeypatch):
+ # A text-only GGUF reads a tool-result image as a note; NPU chat rejects it after the load.
+ backend, rec = _wired(monkeypatch, _FakeBackend("unsloth/A-GGUF"), None)
+ _npu_catalog_entry(monkeypatch, vision = False)
+ with pytest.raises(HTTPException) as excinfo:
+ asyncio.run(
+ inference_route._maybe_auto_switch_model(
+ _NPU_MODEL,
+ object(),
+ "tester",
+ require_vision = True,
+ tool_images_only = True,
+ serves_npu = True,
+ )
+ )
+ assert "cannot read images" in str(excinfo.value.detail)
+ assert rec.calls == []
+
+
+def test_an_npu_switch_counts_as_a_queued_switch_while_it_loads(monkeypatch):
+ # Uncounted, a GGUF switch waiting on the lock this one holds kept its drain from finishing.
+ _wired(monkeypatch, _FakeBackend("unsloth/A-GGUF"), None)
+ seen = []
+
+ async def _load(*_args, **_kwargs):
+ seen.append(inference_route._switch_waiter_count())
+
+ monkeypatch.setattr(inference_route, "_load_model_impl", _load)
+ _run_hook(_NPU_MODEL, serves_npu = True)
+ assert seen == [1]
+ assert inference_route._switch_waiter_count() == 0
+
+
@pytest.mark.parametrize("reload", [False, True], ids = ["switch", "idle reload"])
def test_completions_echo_to_a_gguf_target_is_refused_before_it_loads_on_mlx(monkeypatch, reload):
from utils import hardware
diff --git a/studio/backend/utils/hardware/npu.py b/studio/backend/utils/hardware/npu.py
index 66cf0727bde..b2d735e4c6d 100644
--- a/studio/backend/utils/hardware/npu.py
+++ b/studio/backend/utils/hardware/npu.py
@@ -3,12 +3,13 @@
"""Detect AMD NPUs without starting a runtime.
-XDNA 2 (1022:17f0) is supported; XDNA 1 (1022:1502) is not.
-Runtime readiness is checked separately by ``flm validate``.
+XDNA 2 (1022:17f0) is supported, and only with ``UNSLOTH_STUDIO_ENABLE_NPU=1``; XDNA 1
+(1022:1502) is not. Runtime readiness is checked separately by ``flm validate``.
"""
from __future__ import annotations
+import os
import sys
from pathlib import Path
from typing import Any, Optional
@@ -18,6 +19,12 @@ _XDNA2_DEVICE = "17f0"
_XDNA1_DEVICE = "1502"
_PCI_ROOT = Path("/sys/bus/pci/devices")
_MODULE_ROOT = Path("/sys/module")
+# Off by default: the NPU section, NPU loads and the CLI's NPU ids all wait on this.
+ENABLE_ENV = "UNSLOTH_STUDIO_ENABLE_NPU"
+
+
+def npu_enabled() -> bool:
+ return os.environ.get(ENABLE_ENV, "").strip().lower() in ("1", "true", "yes", "on")
def _read(path: Path) -> Optional[str]:
@@ -126,5 +133,8 @@ def detect_amd_npu() -> dict[str, Any]:
info = {"present": False}
except Exception as exc: # noqa: BLE001 -- detection must never break startup or /status
info = {"present": False, "error": str(exc)}
- info["supported"] = bool(info.get("present")) and info.get("family") == "XDNA2"
+ info["enabled"] = npu_enabled()
+ info["supported"] = (
+ info["enabled"] and bool(info.get("present")) and info.get("family") == "XDNA2"
+ )
return info
diff --git a/studio/frontend/src/features/hub/catalog/npu-catalog-list.tsx b/studio/frontend/src/features/hub/catalog/npu-catalog-list.tsx
index 05fa11dbf5c..30e9fd02648 100644
--- a/studio/frontend/src/features/hub/catalog/npu-catalog-list.tsx
+++ b/studio/frontend/src/features/hub/catalog/npu-catalog-list.tsx
@@ -6,7 +6,6 @@ import { Spinner } from "@/components/ui/spinner";
import {
type NpuModel,
type NpuPickerSource,
- NpuPoweredBy,
NpuSetupNotice,
npuDownloadLabel,
npuResumeLabel,
@@ -71,9 +70,6 @@ export function NpuCatalogList({
data-testid="hub-npu-catalog"
className="mx-auto flex min-h-0 w-full max-w-[var(--hub-measure)] flex-1 flex-col gap-0.5 overflow-y-auto px-3 py-3"
>
- {catalog.ready ? (
- <NpuPoweredBy status={catalog.status} className="px-3 pb-2" />
- ) : null}
{body()}
</div>
);
diff --git a/studio/frontend/src/features/model-picker/components/model-selector/pickers.tsx b/studio/frontend/src/features/model-picker/components/model-selector/pickers.tsx
index b8060ccd9c0..e1b9d16f2ed 100644
--- a/studio/frontend/src/features/model-picker/components/model-selector/pickers.tsx
+++ b/studio/frontend/src/features/model-picker/components/model-selector/pickers.tsx
@@ -497,10 +497,24 @@ function ListLabel({
divider ? "mt-3 border-t border-border pt-3" : "pt-3",
)}
>
- <span className="flex items-center gap-1.5 text-ui-10 font-semibold uppercase tracking-wider text-muted-foreground">
- {icon}
- {children}
- </span>
+ {onToggle ? (
+ // The chevron stays the keyboard control, so the label is not a second tab stop.
+ <button
+ type="button"
+ tabIndex={-1}
+ onClick={onToggle}
+ aria-expanded={!collapsed}
+ className="flex cursor-pointer select-none items-center gap-1.5 text-ui-10 font-semibold uppercase tracking-wider text-muted-foreground hover:text-foreground"
+ >
+ {icon}
+ {children}
+ </button>
+ ) : (
+ <span className="flex items-center gap-1.5 text-ui-10 font-semibold uppercase tracking-wider text-muted-foreground">
+ {icon}
+ {children}
+ </span>
+ )}
{(action || onToggle) && (
<div className="flex items-center gap-0.5">
{action}
diff --git a/studio/frontend/src/features/npu/index.ts b/studio/frontend/src/features/npu/index.ts
index 0c7263e398e..f298794e5a1 100644
--- a/studio/frontend/src/features/npu/index.ts
+++ b/studio/frontend/src/features/npu/index.ts
@@ -12,7 +12,7 @@ export {
npuRowsFor,
npuSizeLabel,
} from "./api";
-export { NpuPoweredBy, NpuSetupNotice } from "./npu-setup-notice";
+export { NpuSetupNotice } from "./npu-setup-notice";
export {
type NpuCatalog,
type NpuPickerSource,
diff --git a/studio/frontend/src/features/npu/npu-setup-notice.tsx b/studio/frontend/src/features/npu/npu-setup-notice.tsx
index 688f32c964c..2d23e13d927 100644
--- a/studio/frontend/src/features/npu/npu-setup-notice.tsx
+++ b/studio/frontend/src/features/npu/npu-setup-notice.tsx
@@ -3,10 +3,8 @@
import { Button } from "@/components/ui/button";
import { Spinner } from "@/components/ui/spinner";
-import { cn } from "@/lib/utils";
import { AlertCircleIcon } from "@hugeicons/core-free-icons";
import { HugeiconsIcon } from "@hugeicons/react";
-import type { NpuStatus } from "./api";
import type { NpuCatalog } from "./use-npu-catalog";
export function NpuSetupNotice({ catalog }: { catalog: NpuCatalog }) {
@@ -15,8 +13,8 @@ export function NpuSetupNotice({ catalog }: { catalog: NpuCatalog }) {
return (
<div className="flex flex-col gap-2 px-2.5 pb-2 text-xs">
<p className="text-muted-foreground">
- {status.hardware.name ?? "AMD NPU"}: FastFlowLM's own NPU builds, run
- through Lemonade. GGUF models run on the GPU.
+ {status.hardware.name ?? "AMD NPU"} found. These models run on the NPU,
+ separately from the GPU models listed below.
</p>
{status.error || problems.length > 0 ? (
<div className="flex gap-2 rounded-lg bg-destructive/10 p-2.5 text-destructive">
@@ -63,45 +61,6 @@ export function NpuSetupNotice({ catalog }: { catalog: NpuCatalog }) {
<Spinner className="size-3.5 text-muted-foreground" />
</div>
) : null}
- <NpuPoweredBy status={status} />
</div>
);
}
-
-const version = (value: string | null | undefined) =>
- value ? ` v${value}` : "";
-
-export function NpuPoweredBy({
- status,
- className,
-}: {
- status: NpuStatus;
- className?: string;
-}) {
- return (
- <p
- data-testid="npu-powered-by"
- className={cn("text-ui-11 text-muted-foreground", className)}
- >
- Powered by{" "}
- <a
- href="https://github.com/ROCm/FastFlowLM"
- target="_blank"
- rel="noreferrer"
- className="underline underline-offset-2"
- >
- FastFlowLM
- </a>
- {version(status.versions?.fastflowlm)} and{" "}
- <a
- href="https://github.com/lemonade-sdk/lemonade"
- target="_blank"
- rel="noreferrer"
- className="underline underline-offset-2"
- >
- Lemonade
- </a>
- {version(status.versions?.lemonade)}.
- </p>
- );
-}
diff --git a/unsloth_cli/_inference.py b/unsloth_cli/_inference.py
index 97ced9e1c86..07a80304b30 100644
--- a/unsloth_cli/_inference.py
+++ b/unsloth_cli/_inference.py
@@ -1119,6 +1119,23 @@ def connect_studio_server(
if not token:
return _refuse("couldn't self-issue an Unsloth token (is Unsloth set up here?).")
backend = HttpChatBackend(base_url, token)
+ from unsloth_cli._npu import npu_model_path, prepare_npu_model
+
+ npu_path = npu_model_path(model)
+ if npu_path:
+ # `unsloth inference` passes the bare id the picker shows; load it by its model path.
+ model = npu_path
+ try:
+ prepare_npu_model(
+ lambda method, path, payload, timeout: backend._request(
+ method, path, payload, timeout = timeout
+ ),
+ model,
+ lambda line: typer.echo(line, err = True),
+ )
+ except RuntimeError as exc:
+ typer.echo(f"Error: {exc}", err = True)
+ raise typer.Exit(code = 1)
backend.ensure_loaded(
model,
hf_token = hf_token,
diff --git a/unsloth_cli/_npu.py b/unsloth_cli/_npu.py
new file mode 100644
index 00000000000..829052bba9a
--- /dev/null
+++ b/unsloth_cli/_npu.py
@@ -0,0 +1,153 @@
+# SPDX-License-Identifier: AGPL-3.0-only
+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
+
+"""AMD Ryzen AI NPU models from the CLI, which drives a running Studio over HTTP."""
+
+from __future__ import annotations
+
+import json
+import os
+from typing import Callable, Optional
+from urllib.parse import quote
+
+NPU_MODEL_PREFIX = "lemonade:"
+
+
+class NpuCliError(RuntimeError):
+ pass
+
+
+def npu_model_path(model: Optional[str]) -> Optional[str]:
+ """`lemonade:<id>` for an NPU model path or bare id: Lemonade's NPU ids end in `-FLM`, no slash."""
+ if not model:
+ return None
+ text = model.strip()
+ if text.startswith(NPU_MODEL_PREFIX):
+ return text if len(text) > len(NPU_MODEL_PREFIX) else None
+ if "/" in text or "\\" in text or ":" in text or not text.lower().endswith("-flm"):
+ return None
+ if os.path.exists(text):
+ return None
+ return NPU_MODEL_PREFIX + text
+
+
+def npu_needs_studio_message(npu_path: str, *, no_server: bool) -> str:
+ """Why `unsloth chat` / `unsloth inference` cannot run `npu_path` without a running Studio."""
+ if no_server:
+ return "NPU models run inside Unsloth Studio; drop --no-server."
+ return (
+ f"{npu_path[len(NPU_MODEL_PREFIX) :]} runs on the AMD NPU inside Unsloth Studio. Start it "
+ f"with `unsloth studio` (or `unsloth run --model {npu_path}`) and run this again."
+ )
+
+
+def _call(request: Callable[..., object], method: str, path: str, timeout: float):
+ """Open `path`, turning an HTTP error into the server's own message."""
+ import urllib.error
+ try:
+ return request(method, path, None, timeout)
+ except urllib.error.HTTPError as exc:
+ try:
+ detail = json.loads(exc.read() or b"{}").get("detail")
+ except Exception:
+ detail = None
+ raise NpuCliError(f"{method} {path} failed ({exc.code}): {detail or exc.reason}") from None
+
+
+def _read_json(response) -> dict:
+ with response:
+ body = json.loads(response.read() or b"{}")
+ return body if isinstance(body, dict) else {}
+
+
+def prepare_npu_model(
+ request: Callable[..., object], model_path: str, echo: Callable[[str], None]
+) -> None:
+ """Enable the NPU runtime and download `model_path` on the Studio behind `request`.
+
+ `request(method, path, payload, timeout)` returns an open HTTP response for that Studio.
+ """
+ import http.client
+ try:
+ _prepare_npu_model(request, model_path, echo)
+ except (OSError, http.client.HTTPException) as exc:
+ # A refused or dropped connection, or a timeout: Studio stopped or restarted mid-way.
+ raise NpuCliError(f"Lost the connection to Unsloth Studio: {exc}") from None
+
+
+def _prepare_npu_model(
+ request: Callable[..., object], model_path: str, echo: Callable[[str], None]
+) -> None:
+ model_id = model_path[len(NPU_MODEL_PREFIX) :]
+ status = _read_json(_call(request, "GET", "/api/npu/status", 60))
+ if not status.get("supported"):
+ hardware = status.get("hardware") or {}
+ if not hardware.get("present"):
+ reason = "no AMD Ryzen AI NPU was found on this machine"
+ elif hardware.get("family") == "XDNA2" and hardware.get("enabled") is False:
+ reason = (
+ "the NPU is off by default. Start Unsloth Studio with "
+ "UNSLOTH_STUDIO_ENABLE_NPU=1 to use it"
+ )
+ else:
+ reason = (
+ f"{hardware.get('name') or 'this NPU'} is not supported (an XDNA2 NPU is needed)"
+ )
+ raise NpuCliError(f"{model_id} runs on the AMD NPU, but {reason}.")
+ if not status.get("ready"):
+ echo("Enabling the NPU runtime (Lemonade and FastFlowLM, about 47 MB)...")
+ status = _read_json(_call(request, "POST", "/api/npu/enable", 1800))
+ if not status.get("ready"):
+ problems = (status.get("validation") or {}).get("problems") or []
+ detail = status.get("error") or " ".join(problems) or "the NPU driver check failed"
+ raise NpuCliError(f"Could not enable the NPU runtime: {detail}")
+
+ models = _read_json(_call(request, "GET", "/api/npu/models", 120)).get("models") or []
+ entry = next((m for m in models if isinstance(m, dict) and m.get("id") == model_id), None)
+ if entry is None:
+ available = sorted(str(m.get("id")) for m in models if isinstance(m, dict))
+ listed = ", ".join(available[:12]) + (", ..." if len(available) > 12 else "")
+ raise NpuCliError(f"No NPU model named {model_id}. Available: {listed}")
+ if entry.get("downloaded"):
+ return
+
+ size = entry.get("size_gb")
+ echo(
+ f"Downloading {model_id}"
+ + (f" ({size:.1f} GB)" if isinstance(size, (int, float)) else "")
+ + "..."
+ )
+ last_tenth = -1
+ finished = False
+ response = _call(
+ request, "POST", f"/api/npu/models/{quote(model_id, safe = '')}/download", 7200
+ )
+ with response:
+ for raw in response:
+ line = (
+ raw.decode("utf-8", "replace").strip()
+ if isinstance(raw, bytes)
+ else str(raw).strip()
+ )
+ if not line.startswith("data:"):
+ continue
+ try:
+ event = json.loads(line[len("data:") :])
+ except ValueError:
+ continue
+ if not isinstance(event, dict):
+ continue
+ if event.get("event") == "error" or event.get("error"):
+ raise NpuCliError(
+ f"Downloading {model_id} failed: {event.get('error') or 'unknown error'}"
+ )
+ percent = event.get("percent")
+ if isinstance(percent, (int, float)) and int(percent) // 10 > last_tenth:
+ last_tenth = int(percent) // 10
+ echo(f" {int(percent)}%")
+ if event.get("event") == "complete":
+ finished = True
+ if not finished:
+ raise NpuCliError(
+ f"The download of {model_id} ended early. Run the command again to resume it."
+ )
diff --git a/unsloth_cli/commands/chat.py b/unsloth_cli/commands/chat.py
index f5fb38e5ab5..cf0a2d3379f 100644
--- a/unsloth_cli/commands/chat.py
+++ b/unsloth_cli/commands/chat.py
@@ -2,6 +2,7 @@
# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
import sys
+from types import SimpleNamespace
from typing import List, Optional
import typer
@@ -285,9 +286,32 @@ def chat(
raise typer.Exit(code = 1)
model = _pick_model(console)
- # Resolve first so --compare can be rejected before the slow load.
- with quiet_if_nonzero_mlx_rank():
- model_config = resolve_model_config(model, hf_token = hf_token)
+ from unsloth_cli._npu import npu_model_path, npu_needs_studio_message
+
+ # An NPU model runs inside Unsloth Studio, never in this process: no HF config to resolve.
+ npu_path = npu_model_path(model)
+ if npu_path:
+ model = npu_path
+ model_config = SimpleNamespace(
+ display_name = npu_path.split(":", 1)[1],
+ is_gguf = False,
+ is_lora = False,
+ path = None,
+ base_model = None,
+ gguf_variant = None,
+ )
+ if no_server or is_mlx_distributed:
+ if should_print:
+ err.print(
+ npu_needs_studio_message(npu_path, no_server = True),
+ style = "red",
+ markup = False,
+ )
+ raise typer.Exit(code = 1)
+ else:
+ # Resolve first so --compare can be rejected before the slow load.
+ with quiet_if_nonzero_mlx_rank():
+ model_config = resolve_model_config(model, hf_token = hf_token)
compare_blocked = _compare_blocked_reason(model_config)
if is_mlx_distributed:
compare_blocked = (
@@ -318,6 +342,14 @@ def chat(
else connect_studio_server(model, **server_load_opts(ctx, load_opts))
)
server_mode = chat_backend is not None
+ if npu_path and not server_mode:
+ if should_print:
+ err.print(
+ npu_needs_studio_message(npu_path, no_server = False),
+ style = "red",
+ markup = False,
+ )
+ raise typer.Exit(code = 1)
if server_mode and should_print:
console.print(
"(Unsloth server connected — model stays warm after /exit)",
diff --git a/unsloth_cli/commands/inference.py b/unsloth_cli/commands/inference.py
index 93af7697a0e..b2b6f9c412e 100644
--- a/unsloth_cli/commands/inference.py
+++ b/unsloth_cli/commands/inference.py
@@ -136,11 +136,21 @@ def inference(
load_opts["speculative_type"] = speculative_type
if spec_draft_n_max is not None:
load_opts["spec_draft_n_max"] = spec_draft_n_max
+ from unsloth_cli._npu import npu_model_path, npu_needs_studio_message
+
+ # An NPU model runs inside Unsloth Studio, never in this process.
+ npu_path = npu_model_path(model)
+ if npu_path and (no_server or is_mlx_distributed):
+ typer.echo(f"Error: {npu_needs_studio_message(npu_path, no_server = True)}", err = True)
+ raise typer.Exit(code = 1)
chat_backend = (
None
if (no_server or is_mlx_distributed)
else connect_studio_server(model, **server_load_opts(ctx, load_opts))
)
+ if chat_backend is None and npu_path:
+ typer.echo(f"Error: {npu_needs_studio_message(npu_path, no_server = False)}", err = True)
+ raise typer.Exit(code = 1)
if chat_backend is None:
chat_backend = load_chat_backend(model, **load_opts)
try:
diff --git a/unsloth_cli/commands/start.py b/unsloth_cli/commands/start.py
index 24982e77ea0..ae20bfd39db 100644
--- a/unsloth_cli/commands/start.py
+++ b/unsloth_cli/commands/start.py
@@ -2775,6 +2775,18 @@ def _fail_gguf_variant_missing(model_id: str, variant: str, variants: list) -> N
_fail(message)
+def _refuse_npu_model(model: Optional[str]) -> None:
+ from unsloth_cli._npu import npu_model_path
+ npu_path = npu_model_path(model)
+ if npu_path:
+ model_id = npu_path.split(":", 1)[1]
+ # Unchecked, `lemonade:<id>` was split into a GGUF lookup for a repo named "lemonade".
+ _fail(
+ f"{model_id} is an AMD NPU model. `unsloth start` runs coding agents on GGUF models; "
+ f"chat with {model_id} in Unsloth Studio or with `unsloth chat {model_id}`."
+ )
+
+
def _fail_agent_needs_gguf(agent: _GgufAgent, model_id: str) -> NoReturn:
message = (
f"{agent.label} needs a GGUF model served by llama-server, " f"but {model_id} is not one."
@@ -2797,6 +2809,7 @@ def _preflight_agent_gguf(
# Only a complete listing with no .gguf files rejects; unknown defers to the
# post-connect check. Mirrors _require_studio's auto-start condition, so with no start
# possible its "no running server" error comes first, without a hub probe.
+ _refuse_npu_model(model)
if not (serve and launch and model):
return
expected = os.environ.get("UNSLOTH_STUDIO_URL", "http://127.0.0.1:8888").rstrip("/")
@@ -4671,6 +4684,7 @@ def _connect(
preload_check = None,
connections: bool = False,
) -> tuple:
+ _refuse_npu_model(model)
requested = model
explicit_connection = bool(model) and model.startswith(_CONNECTION_MODEL_PREFIX)
# `--model org/name:QUANT` is shorthand for `--model org/name --gguf-variant QUANT`. Split it before we match or serve so the attach path resolves against the already-loaded `org/name` (listed without the suffix) instead of reloading a `:`-suffixed repo id, which Unsloth rejects and which would evict a model another session is using.
diff --git a/unsloth_cli/commands/studio.py b/unsloth_cli/commands/studio.py
index 61b275683dd..cf01008f9c1 100644
--- a/unsloth_cli/commands/studio.py
+++ b/unsloth_cli/commands/studio.py
@@ -2378,7 +2378,8 @@ def run(
help = (
"Model path or HF repo. Accepts llama.cpp-style "
"`org/repo:variant` syntax. `-hf` / `--hf-repo` match "
- "llama-server's spelling."
+ "llama-server's spelling. An AMD Ryzen AI NPU model is "
+ "`lemonade:<id>` or its bare id, e.g. `qwen3-0.6b-FLM`."
),
),
gguf_variant: Optional[str] = typer.Option(
@@ -2700,12 +2701,14 @@ def run(
os.environ[_sampling_env] = str(_sampling_value)
# Set before any re-exec. --log-verbose keeps llama-server's own -v passthrough working.
+ added_log_verbose = False
if verbose:
_enable_verbose_access_logs()
if engine == "auto" and not any(
a in ("--verbose", "-v", "--log-verbose") for a in extra_llama_args
):
extra_llama_args.append("--log-verbose")
+ added_log_verbose = True
if disable_dns_pinning:
os.environ["UNSLOTH_STUDIO_DISABLE_DNS_PINNING"] = "1"
else:
@@ -2735,8 +2738,23 @@ def run(
)
raise typer.Exit(2)
+ from unsloth_cli._npu import npu_model_path
+
+ # An NPU model's `lemonade:` prefix is not a llama.cpp `repo:variant` split.
+ npu_path = npu_model_path(model)
+ if npu_path:
+ if added_log_verbose:
+ extra_llama_args.remove("--log-verbose")
+ if gguf_variant or extra_llama_args or engine != "auto":
+ typer.echo(
+ "Error: NPU models run on FastFlowLM; --gguf-variant, --engine and "
+ "llama-server flags do not apply to them.",
+ err = True,
+ )
+ raise typer.Exit(2)
+ model = npu_path
# Parse llama.cpp `repo:variant`; error if it disagrees with --gguf-variant.
- parsed_repo, embedded_variant = _split_repo_variant(model)
+ parsed_repo, embedded_variant = (model, None) if npu_path else _split_repo_variant(model)
if embedded_variant:
if gguf_variant and gguf_variant != embedded_variant:
typer.echo(
@@ -3001,6 +3019,30 @@ def run(
# The engine otherwise gets one GPU, which makes pipeline / data a no-op.
from utils.hardware.hardware import resolve_requested_gpu_ids
gpu_ids = resolve_requested_gpu_ids(None)
+ if npu_path:
+ from unsloth_cli._npu import prepare_npu_model
+ def _npu_request(method, path, payload, timeout):
+ import urllib.request
+ request = urllib.request.Request(
+ f"http://{_url_host(request_host)}:{actual_port}{path}",
+ data = None if payload is None else json.dumps(payload).encode(),
+ headers = {
+ "Content-Type": "application/json",
+ "Authorization": f"Bearer {api_key}",
+ },
+ method = method,
+ )
+ return _direct_urlopen(request, timeout = timeout)
+
+ try:
+ prepare_npu_model(
+ _npu_request,
+ model,
+ (lambda _line: None) if silent else typer.echo,
+ )
+ except RuntimeError as exc:
+ typer.echo(f"Error: {exc}", err = True)
+ raise typer.Exit(1)
if not silent:
typer.echo(f"Loading model: {model}...")
try:
@@ -3077,7 +3119,8 @@ def run(
typer.echo(context_length_line)
typer.echo(f" API Key: {api_key}")
typer.echo("")
- typer.echo(" OpenAI / Anthropic SDK base URL:")
+ # An NPU model serves /v1/chat/completions only: no Anthropic or streaming Responses example.
+ typer.echo(" OpenAI SDK base URL:" if npu_path else " OpenAI / Anthropic SDK base URL:")
typer.echo(f" {sdk_base_url}")
typer.echo("=" * 56)
typer.secho(_tool_notice, fg = _tool_notice_fg, bold = True)
@@ -3090,20 +3133,21 @@ def run(
""" -d '{"messages": [{"role": "user", "content": "Hello"}], "stream": true}'"""
)
typer.echo("")
- typer.echo("Anthropic Messages:")
- typer.echo(f" curl {sdk_base_url}/messages \\")
- typer.echo(f' -H "Authorization: Bearer {api_key}" \\')
- typer.echo(' -H "Content-Type: application/json" \\')
- typer.echo(
- """ -d '{"max_tokens": 256, "messages": [{"role": "user", "content": "Hello"}], "stream": true}'"""
- )
- typer.echo("")
- typer.echo("OpenAI Responses:")
- typer.echo(f" curl {sdk_base_url}/responses \\")
- typer.echo(f' -H "Authorization: Bearer {api_key}" \\')
- typer.echo(' -H "Content-Type: application/json" \\')
- typer.echo(""" -d '{"input": "Hello", "stream": true}'""")
- typer.echo("")
+ if not npu_path:
+ typer.echo("Anthropic Messages:")
+ typer.echo(f" curl {sdk_base_url}/messages \\")
+ typer.echo(f' -H "Authorization: Bearer {api_key}" \\')
+ typer.echo(' -H "Content-Type: application/json" \\')
+ typer.echo(
+ """ -d '{"max_tokens": 256, "messages": [{"role": "user", "content": "Hello"}], "stream": true}'"""
+ )
+ typer.echo("")
+ typer.echo("OpenAI Responses:")
+ typer.echo(f" curl {sdk_base_url}/responses \\")
+ typer.echo(f' -H "Authorization: Bearer {api_key}" \\')
+ typer.echo(' -H "Content-Type: application/json" \\')
+ typer.echo(""" -d '{"input": "Hello", "stream": true}'""")
+ typer.echo("")
else:
if secure and _cf_url:
typer.echo(f"URL: {_cf_url}")
diff --git a/unsloth_cli/tests/test_npu_cli.py b/unsloth_cli/tests/test_npu_cli.py
new file mode 100644
index 00000000000..fabf9426ff2
--- /dev/null
+++ b/unsloth_cli/tests/test_npu_cli.py
@@ -0,0 +1,273 @@
+# SPDX-License-Identifier: AGPL-3.0-only
+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
+
+"""AMD NPU models from `unsloth run`, `unsloth chat` and `unsloth start`."""
+
+from __future__ import annotations
+
+import io
+import json
+import sys
+from pathlib import Path
+
+import pytest
+import typer
+from typer.testing import CliRunner
+
+_REPO_ROOT = Path(__file__).resolve().parents[2]
+if str(_REPO_ROOT) not in sys.path:
+ sys.path.insert(0, str(_REPO_ROOT))
+
+from unsloth_cli._npu import NpuCliError, npu_model_path, prepare_npu_model
+
+
+ ("model", "expected"),
+ [
+ ("lemonade:qwen3-0.6b-FLM", "lemonade:qwen3-0.6b-FLM"),
+ ("qwen3-0.6b-FLM", "lemonade:qwen3-0.6b-FLM"),
+ ("qwen3.8-mtp-27b-flm", "lemonade:qwen3.8-mtp-27b-flm"),
+ ("unsloth/Qwen3-0.6B-GGUF", None),
+ ("unsloth/Qwen3-0.6B-GGUF:Q4_K_M", None),
+ ("org/name-FLM", None),
+ ("lemonade:", None),
+ ("", None),
+ (None, None),
+ ],
+)
+def test_npu_model_path(model, expected):
+ assert npu_model_path(model) == expected
+
+
+class _Studio:
+ """A Studio's /api/npu routes, answering `prepare_npu_model`'s requests."""
+
+ def __init__(
+ self,
+ *,
+ supported = True,
+ ready = False,
+ downloaded = False,
+ events = None,
+ ):
+ self.status = {"supported": supported, "ready": ready, "hardware": {"present": supported}}
+ self.downloaded = downloaded
+ self.events = (
+ events
+ if events is not None
+ else [
+ {"event": "progress", "percent": 0},
+ {"event": "progress", "percent": 55},
+ {"event": "complete", "percent": 100},
+ ]
+ )
+ self.calls = []
+
+ def __call__(self, method, path, payload, timeout):
+ self.calls.append((method, path))
+ if path == "/api/npu/status":
+ return io.BytesIO(json.dumps(self.status).encode())
+ if path == "/api/npu/enable":
+ self.status["ready"] = True
+ return io.BytesIO(json.dumps(self.status).encode())
+ if path == "/api/npu/models":
+ models = [
+ {"id": "qwen3-0.6b-FLM", "downloaded": self.downloaded, "size_gb": 0.66},
+ {"id": "gemma3-1b-FLM", "downloaded": False, "size_gb": 1.2},
+ ]
+ return io.BytesIO(json.dumps({"models": models}).encode())
+ if path.endswith("/download"):
+ return io.BytesIO("".join(f"data: {json.dumps(e)}\n\n" for e in self.events).encode())
+ raise AssertionError(path)
+
+
+def test_a_fresh_npu_is_enabled_and_the_model_downloaded():
+ studio, lines = _Studio(), []
+ prepare_npu_model(studio, "lemonade:qwen3-0.6b-FLM", lines.append)
+ assert studio.calls == [
+ ("GET", "/api/npu/status"),
+ ("POST", "/api/npu/enable"),
+ ("GET", "/api/npu/models"),
+ ("POST", "/api/npu/models/qwen3-0.6b-FLM/download"),
+ ]
+ assert lines[0].startswith("Enabling the NPU runtime")
+ assert " 100%" in lines
+
+
+def test_a_ready_downloaded_model_needs_nothing():
+ studio = _Studio(ready = True, downloaded = True)
+ prepare_npu_model(studio, "lemonade:qwen3-0.6b-FLM", lambda _line: None)
+ assert studio.calls == [("GET", "/api/npu/status"), ("GET", "/api/npu/models")]
+
+
+def test_no_npu_says_so():
+ with pytest.raises(NpuCliError, match = "no AMD Ryzen AI NPU"):
+ prepare_npu_model(_Studio(supported = False), "lemonade:qwen3-0.6b-FLM", print)
+
+
+def test_an_npu_left_off_names_the_flag():
+ studio = _Studio(supported = False)
+ studio.status["hardware"] = {"present": True, "family": "XDNA2", "enabled": False}
+ with pytest.raises(NpuCliError, match = "UNSLOTH_STUDIO_ENABLE_NPU=1"):
+ prepare_npu_model(studio, "lemonade:qwen3-0.6b-FLM", print)
+
+
+def test_an_unknown_model_lists_the_real_ones():
+ with pytest.raises(NpuCliError, match = "gemma3-1b-FLM"):
+ prepare_npu_model(_Studio(ready = True), "lemonade:nope-FLM", print)
+
+
+ "events",
+ [
+ [{"event": "error", "error": "disk full"}],
+ [{"event": "progress", "percent": 40}],
+ ],
+)
+def test_a_failed_or_cut_download_fails(events):
+ with pytest.raises(NpuCliError):
+ prepare_npu_model(_Studio(ready = True, events = events), "lemonade:qwen3-0.6b-FLM", print)
+
+
+ "error",
+ [ConnectionResetError("reset"), TimeoutError("timed out"), OSError("refused")],
+)
+def test_a_lost_studio_connection_is_a_cli_error(error):
+ # Only HTTP errors were converted, so a dropped connection surfaced as a traceback.
+ def _request(*_args):
+ raise error
+
+ with pytest.raises(NpuCliError, match = "Lost the connection to Unsloth Studio"):
+ prepare_npu_model(_request, "lemonade:qwen3-0.6b-FLM", print)
+
+
+def test_a_running_studio_prepares_and_loads_a_bare_npu_id(monkeypatch):
+ # `unsloth inference qwen3-0.6b-FLM` cut the prefix length off the bare id and loaded it as is.
+ import unsloth_cli._inference as inference
+ import unsloth_cli._npu as npu
+
+ seen = {}
+
+ class _Backend:
+ def __init__(self, *_args):
+ pass
+
+ def ensure_loaded(self, model, **_kwargs):
+ seen["loaded"] = model
+
+ monkeypatch.setenv("UNSLOTH_STUDIO_URL", "http://127.0.0.1:8888")
+ monkeypatch.setattr(inference, "find_studio_server", lambda *a, **k: "http://127.0.0.1:8888")
+ monkeypatch.setattr(inference, "verify_studio_identity", lambda *a, **k: True)
+ monkeypatch.setattr(inference, "_studio_token", lambda: "token")
+ monkeypatch.setattr(inference, "HttpChatBackend", _Backend)
+ monkeypatch.setattr(
+ npu, "prepare_npu_model", lambda _request, model, _echo: seen.setdefault("prepared", model)
+ )
+ inference.connect_studio_server(
+ "qwen3-0.6b-FLM", hf_token = None, max_seq_length = 0, load_in_4bit = None
+ )
+ assert seen == {"prepared": "lemonade:qwen3-0.6b-FLM", "loaded": "lemonade:qwen3-0.6b-FLM"}
+
+
+def test_run_refuses_gguf_flags_for_an_npu_model():
+ from unsloth_cli import app
+
+ result = CliRunner().invoke(
+ app, ["run", "--model", "lemonade:qwen3-0.6b-FLM", "--gguf-variant", "Q4_K_M"]
+ )
+ assert result.exit_code == 2
+ assert "NPU models run on FastFlowLM" in result.output
+
+
+def test_run_accepts_verbose_for_an_npu_model(monkeypatch):
+ # --verbose adds --log-verbose for llama-server, which then read as a user flag and exited 2.
+ from unsloth_cli import app
+ from unsloth_cli.commands import studio as studio_mod
+
+ def _past_the_npu_checks(*_args, **_kwargs):
+ raise typer.Exit(7)
+
+ monkeypatch.setattr(studio_mod, "_enable_verbose_access_logs", lambda: None)
+ monkeypatch.setattr(studio_mod, "_require_bind_host", _past_the_npu_checks)
+ result = CliRunner().invoke(app, ["run", "--model", "lemonade:qwen3-0.6b-FLM", "--verbose"])
+ assert result.exit_code == 7, result.output
+
+
+def test_run_does_not_split_the_lemonade_prefix():
+ import inspect
+
+ from unsloth_cli.commands import studio as studio_mod
+
+ src = inspect.getsource(studio_mod.run)
+ # `lemonade:<id>` was split llama.cpp-style into repo "lemonade", variant "<id>".
+ assert "(model, None) if npu_path else _split_repo_variant(model)" in src
+ assert "prepare_npu_model(" in src
+
+
+def test_start_names_an_npu_model_instead_of_looking_up_a_gguf(capsys, monkeypatch):
+ from unsloth_cli.commands import start as start_mod
+
+ # Never reach a real server: a Studio running on this machine would be asked to switch models.
+ def _no_server(*_args, **_kwargs):
+ raise AssertionError("unsloth start looked for a Studio before refusing the NPU model")
+
+ monkeypatch.setattr(start_mod, "_require_studio", _no_server)
+ with pytest.raises((typer.Exit, SystemExit)):
+ start_mod._connect(None, "lemonade:qwen3-0.6b-FLM")
+ assert "qwen3-0.6b-FLM is an AMD NPU model" in capsys.readouterr().err
+
+
+def test_chat_needs_a_running_studio_for_an_npu_model(monkeypatch):
+ from unsloth_cli import app
+ from unsloth_cli.commands import chat as chat_mod
+
+ monkeypatch.setattr(chat_mod, "connect_studio_server", lambda *_a, **_k: None)
+ result = CliRunner().invoke(app, ["chat", "qwen3-0.6b-FLM"])
+ assert result.exit_code == 1
+ assert "runs on the AMD NPU inside Unsloth Studio" in result.output
+
+
+ ("flags", "words"),
+ [([], "runs on the AMD NPU inside Unsloth Studio"), (["--no-server"], "drop --no-server")],
+)
+def test_inference_needs_a_running_studio_for_an_npu_model(monkeypatch, flags, words):
+ # Without a Studio the id fell through to a local load and a Hugging Face lookup.
+ from unsloth_cli import app
+ from unsloth_cli.commands import inference as inference_mod
+
+ def _no_local_load(*_args, **_kwargs):
+ raise AssertionError("an NPU model was loaded in-process")
+
+ monkeypatch.setattr(inference_mod, "connect_studio_server", lambda *_a, **_k: None)
+ monkeypatch.setattr(inference_mod, "load_chat_backend", _no_local_load)
+ result = CliRunner().invoke(app, ["inference", "qwen3-0.6b-FLM", "hi", *flags])
+ assert result.exit_code == 1, result.output
+ assert words in result.output
+
+
+def test_a_gguf_agent_refuses_an_npu_model_before_any_hub_lookup(capsys, monkeypatch):
+ from unsloth_cli.commands import start as start_mod
+
+ def _no_lookup(*_args, **_kwargs):
+ raise AssertionError("looked the NPU id up as a Hugging Face GGUF repo")
+
+ monkeypatch.setattr(start_mod, "find_studio_server", _no_lookup)
+ monkeypatch.setattr(start_mod, "_hub_gguf_files", _no_lookup)
+ with pytest.raises((typer.Exit, SystemExit)):
+ start_mod._preflight_agent_gguf(start_mod._CLAUDE_GGUF_AGENT, "lemonade:qwen3-0.6b-FLM")
+ assert "is an AMD NPU model" in capsys.readouterr().err
+
+
+def test_the_run_banner_offers_no_endpoint_an_npu_model_refuses():
+ import inspect
+
+ from unsloth_cli.commands import studio as studio_mod
+
+ # Structural: the banner prints after a live server load, which this suite cannot start.
+ src = inspect.getsource(studio_mod.run)
+ assert '" OpenAI SDK base URL:" if npu_path' in src
+ banner = src[src.index('typer.echo("OpenAI Chat Completions:")') :]
+ assert banner.index("if not npu_path:") < banner.index('typer.echo("Anthropic Messages:")')
+ assert banner.index("if not npu_path:") < banner.index('typer.echo("OpenAI Responses:")')