Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
Studio: improve AMD NPU discovery, CLI support, and chat feedback (#13797)
oobabooga2de9425750aa
diff --git a/studio/backend/core/inference/npu_backend.py b/studio/backend/core/inference/npu_backend.pyindex f98ddfbad25..4024d170a61 100644--- a/studio/backend/core/inference/npu_backend.py+++ b/studio/backend/core/inference/npu_backend.py@@ -436,6 +436,11 @@ class LemonadeNpuBackend:hardware = self.hardware()if not hardware.get("present"):raise NpuError("No AMD Ryzen AI NPU was found on this machine.")+ if hardware.get("family") == "XDNA2" and hardware.get("enabled") is False:+ raise NpuError(+ "The AMD NPU is off by default. Start Unsloth Studio with "+ "UNSLOTH_STUDIO_ENABLE_NPU=1 to use it."+ )if not hardware.get("supported"):raise NpuError("FastFlowLM needs an XDNA 2 NPU (Ryzen AI 300/400 or Ryzen AI Max); "@@ -444,6 +449,8 @@ class LemonadeNpuBackend:def _ensure_running(self) -> LemonadeServer:"""Start the installed runtime if needed and return its server."""+ # A runtime enabled before the flag was turned off must not start again.+ self._require_supported()with self._lock:if self._server is not None and self._server.is_alive():return self._serverdiff --git a/studio/backend/routes/inference.py b/studio/backend/routes/inference.pyindex 4cd9aa5816f..c0a979e1ca1 100644--- a/studio/backend/routes/inference.py+++ b/studio/backend/routes/inference.py@@ -10976,7 +10976,10 @@ async def _reject_unservable_model(if is_npu_model_path(requested_model) and not await asyncio.to_thread(_loaded_satisfies, requested_model):- message = f"The NPU model '{requested_model}' is not loaded. Load it in Unsloth Studio."+ message = (+ f"The NPU model '{requested_model}' is not loaded. Load it in Unsloth Studio, or turn "+ "on 'Switch model by request' under Settings > API to load it from a request."+ )path = getattr(getattr(fastapi_request, "url", None), "path", None)raise HTTPException(status_code = 404,@@ -11218,6 +11221,126 @@ def _audio_cpp_switch_target(requested_model: Optional[str]) -> Optional[tuple]:return (model.id, model.variant.key, model.id, False)+async def _refuse_unservable_npu_switch(+ named_model: str,+ fastapi_request: Request,+ *,+ gguf_only: bool,+ needs_vision: bool,+ needs_audio: bool,+ rejection_prefix: str,+ request_check: Optional[Callable[[Any], None]] = None,+) -> None:+ """400 for a request naming an NPU model that could not serve it once loaded.++ ``request_check`` runs the serving route's own checks against the target's catalog entry.+ """+ from core.inference.npu_backend import NpuError, get_npu_backend, model_id_from_path++ message = None+ if gguf_only:+ message = (+ f"The NPU model '{named_model}' serves /v1/chat/completions only. "+ "Load a GGUF model to use this endpoint."+ )+ elif needs_audio:+ message = f"{rejection_prefix}The NPU model '{named_model}' takes no audio or video input."+ elif (needs_vision or request_check is not None) and not await asyncio.to_thread(+ _loaded_satisfies, named_model+ ):+ try:+ model = await asyncio.to_thread(+ get_npu_backend().loadable_model, model_id_from_path(named_model)+ )+ except NpuError:+ model = None # Not downloaded or unknown: the load reports that itself.+ if model is not None and needs_vision and not model.vision:+ message = f"{rejection_prefix}The NPU model '{named_model}' cannot read images."+ elif model is not None and request_check is not None:+ request_check(model)+ if message is None:+ return+ path = getattr(getattr(fastapi_request, "url", None), "path", None)+ raise HTTPException(+ status_code = 400,+ detail = (+ error_body_for_path(path, message, status = 400, code = "invalid_value", param = "model")+ if isinstance(path, str)+ else message+ ),+ )+++async def _auto_switch_npu_model(+ named_model: str, fastapi_request: Request, current_subject: str, *, claim_resident: bool+) -> None:+ """Load a downloaded ``lemonade:<id>`` model a request names.++ The caller has checked auto-switch is on. Same lock, gates, waiter count, cancellation and+ deferred claim as a GGUF switch.+ """+ from core.inference.gpu_arbiter import require_no_foreign_generations+ from core.inference.llama_keepwarm import inference_lifecycle_gate, model_load_gate+ from utils.openai_auto_switch_settings import get_model_override, resolve_fit_max_seq_length++ if await asyncio.to_thread(_loaded_satisfies, named_model):+ return+ # The context length saved in the model's settings, as a picker load sends it.+ max_seq_length = resolve_fit_max_seq_length(get_model_override(named_model), is_gguf = False)+ cancel_event = getattr(getattr(fastapi_request, "state", None), "generation_cancel_event", None)++ def _raise_if_cancelled() -> None:+ if cancel_event is not None and cancel_event.is_set():+ raise HTTPException(status_code = 409, detail = "Generation cancelled")++ _raise_if_cancelled()+ switch_path = getattr(getattr(fastapi_request, "url", None), "path", None)+ # Counted as a queued switch, so another switch's drain does not wait on this request.+ key = _switch_key(named_model, None)+ _note_switch_waiter(key, 1)+ try:+ async with _auto_switch_lock():+ await _acquire_swap_gate()+ try:+ async with model_load_gate(), inference_lifecycle_gate():+ # Re-read under the gate: another request may have loaded it while this one waited.+ if await asyncio.to_thread(_loaded_satisfies, named_model):+ return+ require_no_foreign_generations(path = switch_path)+ prior_marker = _get_preview_resident()+ loaded_ok = False+ try:+ await _load_model_impl(+ LoadRequest(model_path = named_model, max_seq_length = max_seq_length or 0),+ fastapi_request,+ current_subject,+ current_request_counted = True,+ **(+ {"load_cancel_event": cancel_event}+ if cancel_event is not None+ else {}+ ),+ )+ loaded_ok = True+ _raise_if_cancelled()+ finally:+ if not claim_resident:+ # Preview-owned until the route's 2xx claims it, so a later rejection+ # leaves it evictable, as in the GGUF switch.+ if loaded_ok or _loaded_slot_ident() == named_model:+ _set_preview_resident(_loaded_slot_ident())+ else:+ _set_preview_resident(prior_marker)+ finally:+ # Deregister before releasing the gate, as the GGUF switch does.+ _note_switch_waiter(key, -1)+ key = None+ _auto_switch_process_lock.release()+ finally:+ if key is not None:+ _note_switch_waiter(key, -1)++async def _maybe_auto_switch_model(requested_model: Optional[str],fastapi_request: Request,@@ -11239,6 +11362,8 @@ async def _maybe_auto_switch_model(require_audio_workflow: Optional[str] = None,speech_budget: Optional[dict] = None,tool_images_only: bool = False,+ serves_npu: bool = False,+ npu_request_check: Optional[Callable[[Any], None]] = None,gguf_target_check: Optional[Callable[[], None]] = None,) -> None:"""Load a downloaded local model named by an OpenAI request when auto-switch is on.@@ -11268,7 +11393,9 @@ async def _maybe_auto_switch_model(checks only the resolved target can decide; see:func:`_preflight_audio_for_switch`. ``image_preflight`` does the same fornon-GGUF image count and byte validation. ``require_audio_workflow`` refuses a target that- cannot run that Audio page workflow (``separate``, ``clone``...).+ cannot run that Audio page workflow (``separate``, ``clone``...). ``serves_npu`` marks+ the chat route, the only one an NPU model serves; ``npu_request_check``+ refuses a request an NPU target would reject once loaded, given its catalog entry."""# A text-only GGUF reads tool-result images as a note, so only audio or video needs its mmproj.gguf_requires_vision = (@@ -11412,6 +11539,34 @@ async def _maybe_auto_switch_model(await _reject_unservable_model(requested_model, fastapi_request)return+ from core.inference.npu_backend import is_npu_model_path++ if auto_switch_on and named_model and is_npu_model_path(named_model):+ # Refused before the switch, so a request the NPU cannot serve never evicts the resident model.+ await _refuse_unservable_npu_switch(+ named_model,+ fastapi_request,+ # gguf_only alone misses routes that relax it for in-process models.+ gguf_only = gguf_only or not serves_npu,+ # Not gguf_requires_vision: NPU chat rejects tool-result images too.+ needs_vision = require_vision,+ needs_audio = bool(+ require_audio_input+ or require_video+ or require_speech+ or require_audio_workflow+ or audio_preflight+ ),+ rejection_prefix = rejection_prefix,+ request_check = npu_request_check,+ )+ await _auto_switch_npu_model(+ named_model, fastapi_request, current_subject, claim_resident = claim_resident+ )+ if claim_resident:+ _claim_slot_for_non_preview(fastapi_request)+ return+async def _resolve_and_switch() -> None:from core.inference.openai_auto_download import looks_like_quant, split_model_ref@@ -18655,7 +18810,29 @@ async def _load_npu_model(await asyncio.to_thread(npu.loadable_model, model_id)except NpuError as exc:raise HTTPException(status_code = 400, detail = str(exc)) from None- if load_cancel_event is not None and load_cancel_event.is_set():+ llama_backend = get_llama_cpp_backend()+ unsloth_backend = _peek_inference_backend()+ serving = bool(+ getattr(llama_backend, "is_active", getattr(llama_backend, "is_loaded", False))+ or (unsloth_backend is not None and unsloth_backend.active_model_name)+ or resident is not None+ )+ _raise_if_sidecar_swap_in_progress()+ if serving:+ # Drain active inference before unloading, as for GGUF: an auto-switch has no+ # on_reload_confirmed, so this drain alone keeps it from cutting a reply short.+ if on_reload_confirmed is not None:+ on_reload_confirmed(cancel = False)+ await _wait_for_model_switch_idle(+ current_request_counted = current_request_counted,+ cancel_pending = on_reload_confirmed is not None and bool(request.force_cancel_active),+ )+ # The rechecks the GGUF and standard paths make before their point of no return: an+ # auto-switch has no scoped attempt for the shutdown sweep to cancel.+ _raise_if_sidecar_swap_in_progress()+ with _scoped_load_attempts_lock:+ shutting_down = _loads_shutting_down+ if shutting_down or (load_cancel_event is not None and load_cancel_event.is_set()):raise HTTPException(status_code = 409, detail = "Model load cancelled")if on_reload_confirmed is not None:on_reload_confirmed(cancel = True)@@ -18665,7 +18842,7 @@ async def _load_npu_model(timeout_s = _POST_CANCEL_DRAIN_TIMEOUT_S,)_set_preview_resident(None)- await _unload_llama_before_standard_load(get_llama_cpp_backend())+ await _unload_llama_before_standard_load(llama_backend)backend = _peek_inference_backend()if backend is not None and backend.active_model_name:logger.info(@@ -18677,8 +18854,17 @@ async def _load_npu_model(await asyncio.to_thread(release_chat_gpu_claim)if load_cancel_event is not None and load_cancel_event.is_set():raise HTTPException(status_code = 409, detail = "Model load cancelled")+ load = asyncio.ensure_future(asyncio.to_thread(npu.load, model_id, requested_ctx))+ if load_cancel_event is not None:+ # Stop the load mid-way, as a GGUF load polls its cancel event.+ while not load.done():+ await asyncio.wait({load}, timeout = 0.25)+ # False until npu.load marks itself in flight, so keep asking.+ if load_cancel_event.is_set() and not load.done():+ if await asyncio.to_thread(npu.cancel_load, model_id):+ breaktry:- await asyncio.to_thread(npu.load, model_id, requested_ctx)+ await loadexcept NpuLoadCancelled:raise HTTPException(status_code = 409, detail = "Model load cancelled") from Noneexcept NpuError as exc:@@ -29956,6 +30142,21 @@ _NPU_CUT_SHORT_ERROR = {}+def _npu_kv_cache_full(usage: Any) -> bool:+ """Whether FastFlowLM's usage says the reply used the whole loaded context."""+ if not isinstance(usage, dict):+ return False+ # active_kv_tokens is the KV position; total_tokens undercounts by the empty think block's+ # three uncounted tokens when thinking is off.+ total = usage.get("active_kv_tokens")+ if not isinstance(total, int):+ total = usage.get("total_tokens")+ capacity = usage.get("max_kv_token_capacity")+ return (+ isinstance(total, int) and isinstance(capacity, int) and capacity > 0 and total >= capacity+ )++class _NpuStreamRelay:"""Normalize model IDs, enforce stop sequences, and collect non-streaming replies.@@ -29997,16 +30198,22 @@ class _NpuStreamRelay:self._pending = buffer[len(buffer) - keep :] if keep else ""return buffer[: len(buffer) - keep] if keep else buffer- def _chunk(self, delta: dict, finish_reason: Optional[str]) -> str:- return "data: " + json.dumps(- {- "id": self.completion_id or "chatcmpl-npu",- "object": "chat.completion.chunk",- "created": self.created or int(time.time()),- "model": self.public_model,- "choices": [{"index": 0, "delta": delta, "finish_reason": finish_reason}],- }- )+ def _chunk(+ self,+ delta: dict,+ finish_reason: Optional[str],+ tool_event: Optional[dict] = None,+ ) -> str:+ chunk = {+ "id": self.completion_id or "chatcmpl-npu",+ "object": "chat.completion.chunk",+ "created": self.created or int(time.time()),+ "model": self.public_model,+ "choices": [{"index": 0, "delta": delta, "finish_reason": finish_reason}],+ }+ if tool_event is not None:+ chunk["_toolEvent"] = tool_event+ return "data: " + json.dumps(chunk)def feed(self, line: str) -> list[str]:"""The lines to send on for one line of the upstream stream."""@@ -30074,6 +30281,10 @@ class _NpuStreamRelay:elif isinstance(content, str):self.content += contentif choice.get("finish_reason") is not None:+ if choice["finish_reason"] == "stop" and _npu_kv_cache_full(chunk.get("usage")):+ # FastFlowLM says "stop" when the reply filled the context.+ choice["finish_reason"] = "length"+ out.append(self._chunk({}, None, {"type": "context_window_exceeded"}))self.finish_reason = choice["finish_reason"]if self._pending:# The reply ended partway into a stop sequence, so what was held back is text.@@ -30085,7 +30296,9 @@ class _NpuStreamRelay:return outdef completion(self) -> dict:- message: dict[str, Any] = {"role": "assistant", "content": self.content or None}+ # A tool-calling turn often streams only newlines as text; OpenAI sends null content there.+ content = self.content if self.content.strip() or not self.tool_calls else ""+ message: dict[str, Any] = {"role": "assistant", "content": content or None}if self.reasoning:message["reasoning_content"] = self.reasoningif self.tool_calls:@@ -30109,18 +30322,8 @@ class _NpuStreamRelay:return body-async def _npu_chat_completions(payload, request: Request, current_subject: str):- """Proxy NPU chat, rejecting unsupported parameters.-- Always stream upstream: FastFlowLM's non-streaming replies mishandle reasoning- and finish reasons on length cutoffs. Collect the stream for JSON callers.- """- from core.inference.npu_backend import NpuError, get_npu_backend-- try:- upstream = await asyncio.to_thread(get_npu_backend().upstream)- except NpuError as exc:- raise HTTPException(status_code = 400, detail = str(exc)) from None+def _refuse_npu_chat_request(payload, *, supports_vision: bool, supports_tools: bool) -> None:+ """Reject a chat request FastFlowLM cannot serve; also run before a switch to an NPU model."""if _wants_multiple_choices(payload):_raise_unsupported_n("NPU chat completions")if _response_format_constrains_decoding(payload):@@ -30151,32 +30354,53 @@ async def _npu_chat_completions(payload, request: Request, current_subject: str)raise HTTPException(status_code = 400, detail = "Audio and video input are not supported on the NPU.")- if not upstream.supports_vision and _request_has_attached_image(payload):+ if not supports_vision and _request_has_attached_image(payload):raise HTTPException(status_code = 400,detail = "This NPU model does not read images. Load a vision model to send one.",)+ if payload.tools and payload.tool_choice != "none":+ if payload.parallel_tool_calls is False:+ _raise_unsupported_openai_parameter(+ "parallel_tool_calls",+ "FastFlowLM on the NPU cannot limit a reply to one tool call.",+ )+ if payload.tool_choice not in (None, "auto"):+ _raise_unsupported_openai_parameter(+ "tool_choice", "FastFlowLM on the NPU chooses its own tool; use auto or none."+ )+ if not supports_tools:+ raise HTTPException(+ status_code = 400,+ detail = openai_error_body(+ "This NPU model does not support tool calling.",+ status = 400,+ code = "unsupported_parameter",+ param = "tools",+ ),+ )+++async def _npu_chat_completions(payload, request: Request, current_subject: str):+ """Proxy NPU chat, rejecting unsupported parameters.++ Always stream upstream: FastFlowLM's non-streaming replies mishandle reasoning+ and finish reasons on length cutoffs. Collect the stream for JSON callers.+ """+ from core.inference.npu_backend import NpuError, get_npu_backend++ try:+ upstream = await asyncio.to_thread(get_npu_backend().upstream)+ except NpuError as exc:+ raise HTTPException(status_code = 400, detail = str(exc)) from None+ _refuse_npu_chat_request(+ payload,+ supports_vision = upstream.supports_vision,+ supports_tools = upstream.supports_tools,+ )if payload.tool_choice == "none":# FastFlowLM 1.0.3 ignores tool_choice and calls a tool anyway.payload.tools = None- elif payload.tools and payload.parallel_tool_calls is False:- _raise_unsupported_openai_parameter(- "parallel_tool_calls", "FastFlowLM on the NPU cannot limit a reply to one tool call."- )- elif payload.tools and payload.tool_choice not in (None, "auto"):- _raise_unsupported_openai_parameter(- "tool_choice", "FastFlowLM on the NPU chooses its own tool; use auto or none."- )- elif payload.tools and not upstream.supports_tools:- raise HTTPException(- status_code = 400,- detail = openai_error_body(- "This NPU model does not support tool calling.",- status = 400,- code = "unsupported_parameter",- param = "tools",- ),- )_normalize_chat_reasoning_controls(payload)if not upstream.supports_reasoning:@@ -31362,6 +31586,11 @@ async def produce_openai_chat_completions(if _needs_image and _pre_parsed is not None:_image_preflight = _chat_image_preflight(payload, _pre_parsed[2])+ def _npu_switch_check(model) -> None:+ # NPU chat runs no MCP tool loop, so it refuses an MCP image after the switch.+ _refuse_unused_mcp_image(_mcp_image)+ _refuse_npu_chat_request(payload, supports_vision = model.vision, supports_tools = model.tools)+# Defer the resident claim: chat has several post-switch capability checks that can still# reject (Whisper without audio, n>1 on a non-GGUF backend, tool or response_format# passthrough), so claiming here could strand a preview-owned model for a request that@@ -31380,6 +31609,8 @@ async def produce_openai_chat_completions(audio_preflight = _audio_preflight,image_preflight = _image_preflight,tool_images_only = _tool_images_only,+ serves_npu = True,+ npu_request_check = _npu_switch_check,)if _audio_preflight is not None:_predecoded_audio = _audio_preflight.get("decoded")@@ -44445,6 +44676,16 @@ async def anthropic_messages(and not _in_process_model_resident()and not _automatic_model_load_may_run()):+ _npu_model = _resident_npu_model()+ if _npu_model is not None:+ raise HTTPException(+ status_code = 400,+ detail = (+ f"The loaded NPU model ({_npu_model.model_path}) serves "+ "/v1/chat/completions only. Load a GGUF model to use the "+ "Anthropic Messages API."+ ),+ )raise HTTPException(status_code = 503,detail = _no_model_loaded_detail("No GGUF model loaded. Load a GGUF model first."),diff --git a/studio/backend/tests/test_mcp_image.py b/studio/backend/tests/test_mcp_image.pyindex f05dbb2f70d..9dee87ad96d 100644--- a/studio/backend/tests/test_mcp_image.py+++ b/studio/backend/tests/test_mcp_image.py@@ -683,14 +683,15 @@ def test_every_local_exit_without_a_tool_loop_refuses_the_image():"_refuse_unused_mcp_image(_mcp_image, _catalog_names(_sf_tools_to_use) if _sf_use_tools else None)",):assert guard in statements- # NPU, both speech models, audio input and the GGUF passthrough return before either loop.+ # The NPU switch check, NPU, both speech models, audio input and the GGUF passthrough+ # return before either loop.calls = [nodefor node in ast.walk(ast.parse(src))if isinstance(node, ast.Call)and getattr(node.func, "id", None) == "_refuse_unused_mcp_image"]- assert len(calls) == 7+ assert len(calls) == 8# The forced image approval parks the GGUF stream, so the slot must be tracked for reclaim.slot = [ast.unparse(node.value)diff --git a/studio/backend/tests/test_npu_backend.py b/studio/backend/tests/test_npu_backend.pyindex 14c490111c3..1b5900e1123 100644--- a/studio/backend/tests/test_npu_backend.py+++ b/studio/backend/tests/test_npu_backend.py@@ -146,6 +146,23 @@ def test_unsupported_npu_is_refused_before_any_download(tmp_path, monkeypatch):assert installer.installs == 0[email protected]("call", ["enable", "catalog"])+def test_an_npu_left_off_is_refused_with_the_flag_named(tmp_path, monkeypatch, call):+ # catalog: a runtime enabled before the flag went off must not start again.+ installer = _Installer(_binary(tmp_path))+ monkeypatch.setattr(nb, "_installer_module", lambda: installer)+ monkeypatch.setattr(+ nb,+ "detect_amd_npu",+ lambda: {"present": True, "enabled": False, "supported": False, "family": "XDNA2"},+ )+ backend = nb.LemonadeNpuBackend(root = tmp_path / "lemonade")+ with pytest.raises(nb.NpuError, match = "UNSLOTH_STUDIO_ENABLE_NPU=1"):+ getattr(backend, call)()+ assert installer.installs == 0+ assert backend.status()["supported"] is False++def test_catalog_keeps_flm_chat_models_only(npu):npu.enable()ids = [model.id for model in npu.catalog()]diff --git a/studio/backend/tests/test_npu_chat_route.py b/studio/backend/tests/test_npu_chat_route.pyindex 922b9c85570..a751cb6c151 100644--- a/studio/backend/tests/test_npu_chat_route.py+++ b/studio/backend/tests/test_npu_chat_route.py@@ -69,7 +69,38 @@ _FLM_TIMINGS = {}+# The end of a reply whose reasoning filled the loaded context, captured on Strix Halo: FastFlowLM+# says "stop" with the KV cache full.+_FULL_USAGE = {+ "prompt_tokens": 45,+ "completion_tokens": 8147,+ "total_tokens": 8192,+ "active_kv_tokens": 8192,+ "max_kv_token_capacity": 8192,+ "kv_token_occupancy_rate_percentage": 100.0,+}++def _script(prompt: str) -> str:+ if "FULL" in prompt:+ return (+ _chunk({"reasoning_content": " Wait, no!"})+ + _chunk({"content": None}, "stop", _FULL_USAGE)+ + "data: [DONE]\n\n"+ )+ if "WSTOOL" in prompt:+ call = {+ "index": 0,+ "id": "call_1",+ "type": "function",+ "function": {"name": "get_weather", "arguments": '{"city":"Paris"}'},+ }+ return (+ _chunk({"content": "\n\n\n"})+ + _chunk({"tool_calls": [call]})+ + _chunk({}, "tool_calls", _USAGE)+ + "data: [DONE]\n\n"+ )if "FAULT" in prompt:return (_chunk({"content": "par"})@@ -826,6 +857,161 @@ def test_an_npu_load_it_would_refuse_keeps_the_gpu_resident(monkeypatch):assert teardowns == []+def test_an_npu_load_drains_active_inference_before_unloading(monkeypatch):+ # An auto-switch passes no on_reload_confirmed, so without the drain it cut a reply short.+ import routes.inference as routes+ from core.inference import npu_backend as nb+ from models.inference import LoadRequest++ order: list[str] = []+ model = SimpleNamespace(id = "qwen3-0.6b-FLM", model_path = "lemonade:qwen3-0.6b-FLM")++ class _Npu:+ def resident(self):+ return None++ def loadable_model(self, model_id):+ return model++ def load(self, model_id, ctx):+ order.append("load")++ async def _drain(**kwargs):+ order.append("drain")++ async def _unload_gpu(_backend):+ order.append("unload")++ monkeypatch.setattr(nb, "get_npu_backend", lambda: _Npu())+ monkeypatch.setattr(routes, "get_llama_cpp_backend", lambda: SimpleNamespace(is_active = True))+ monkeypatch.setattr(routes, "_peek_inference_backend", lambda: None)+ monkeypatch.setattr(routes, "_wait_for_model_switch_idle", _drain)+ monkeypatch.setattr(routes, "_unload_llama_before_standard_load", _unload_gpu)+ monkeypatch.setattr(routes, "release_chat_gpu_claim", lambda: True)+ with pytest.raises(HTTPException):+ # The stub never becomes resident, so the load ends in a 409 after the steps above.+ asyncio.run(+ routes._load_npu_model(+ LoadRequest(model_path = model.model_path),+ current_request_counted = True,+ on_reload_confirmed = None,+ load_cancel_event = None,+ )+ )+ assert order == ["drain", "unload", "load"]++[email protected]("blocker", ["sidecar_swap", "shutdown"])+def test_an_npu_load_refuses_before_unloading_during_a_swap_or_shutdown(monkeypatch, blocker):+ # The NPU branch skipped the rechecks the other load paths make, so it unloaded the resident+ # model under a transformers install or after shutdown had torn everything down.+ import routes.inference as routes+ from core.inference import npu_backend as nb+ from models.inference import LoadRequest+ from utils import transformers_version++ order: list[str] = []+ model = SimpleNamespace(id = "qwen3-0.6b-FLM", model_path = "lemonade:qwen3-0.6b-FLM")++ class _Npu:+ def resident(self):+ return None++ def loadable_model(self, model_id):+ return model++ def load(self, model_id, ctx):+ order.append("load")++ async def _drain(**kwargs):+ order.append("drain")++ async def _unload_gpu(_backend):+ order.append("unload")++ monkeypatch.setattr(nb, "get_npu_backend", lambda: _Npu())+ monkeypatch.setattr(routes, "get_llama_cpp_backend", lambda: SimpleNamespace(is_active = True))+ monkeypatch.setattr(routes, "_peek_inference_backend", lambda: None)+ monkeypatch.setattr(routes, "_wait_for_model_switch_idle", _drain)+ monkeypatch.setattr(routes, "_unload_llama_before_standard_load", _unload_gpu)+ monkeypatch.setattr(routes, "release_chat_gpu_claim", lambda: True)+ if blocker == "sidecar_swap":+ monkeypatch.setattr(transformers_version, "sidecar_swap_in_progress", lambda: True)+ else:+ monkeypatch.setattr(routes, "_loads_shutting_down", True)+ with pytest.raises(HTTPException) as excinfo:+ asyncio.run(+ routes._load_npu_model(+ LoadRequest(model_path = model.model_path),+ current_request_counted = True,+ on_reload_confirmed = None,+ load_cancel_event = None,+ )+ )+ assert excinfo.value.status_code == 409+ assert "unload" not in order and "load" not in order+++def test_a_stop_during_an_npu_load_cancels_it(monkeypatch):+ # The cancel event was read only before and after the blocking load, so a Stop waited it out.+ import routes.inference as routes+ from core.inference import npu_backend as nb+ from models.inference import LoadRequest++ model = SimpleNamespace(id = "qwen3-0.6b-FLM", model_path = "lemonade:qwen3-0.6b-FLM")+ loading = threading.Event()+ stopped = threading.Event()+ stop = threading.Event()++ class _Npu:+ def resident(self):+ return None++ def loadable_model(self, model_id):+ return model++ def load(self, model_id, ctx):+ loading.set()+ if not stopped.wait(10):+ raise AssertionError("the load was never cancelled")+ raise nb.NpuLoadCancelled("cancelled")++ def cancel_load(self, model_id):+ # The first ask lands before the load is registered, as when Stop beats npu.load.+ asks.append(model_id)+ if len(asks) == 1:+ return False+ stopped.set()+ return True++ asks: list[str] = []++ def _stop_once_loading():+ loading.wait(10)+ stop.set()++ async def _nothing(*_args, **_kwargs):+ return None++ monkeypatch.setattr(nb, "get_npu_backend", lambda: _Npu())+ monkeypatch.setattr(routes, "get_llama_cpp_backend", lambda: SimpleNamespace(is_active = False))+ monkeypatch.setattr(routes, "_peek_inference_backend", lambda: None)+ monkeypatch.setattr(routes, "_unload_llama_before_standard_load", _nothing)+ monkeypatch.setattr(routes, "release_chat_gpu_claim", lambda: True)+ threading.Thread(target = _stop_once_loading, daemon = True).start()+ with pytest.raises(HTTPException) as excinfo:+ asyncio.run(+ routes._load_npu_model(+ LoadRequest(model_path = model.model_path),+ current_request_counted = True,+ on_reload_confirmed = None,+ load_cancel_event = stop,+ )+ )+ assert excinfo.value.status_code == 409+ assert stopped.is_set() and len(asks) == 2++def test_a_reply_cut_short_is_not_reported_complete(flm):flm()status, body = _call(stream = False, messages = [{"role": "user", "content": "CUT"}])@@ -1278,3 +1464,57 @@ def test_a_tool_loop_turn_dropped_after_a_tool_call_is_cut_short():plain = _TurnFinish()plain.see(_chunk({}, "stop").strip())assert plain.finished is True+++def test_a_reply_that_fills_the_context_says_so(flm):+ flm(reasoning = True)+ full = [{"role": "user", "content": "FULL"}]+ status, lines = _call(stream = True, messages = full)+ assert status == 200+ chunks = _data(lines)+ assert [c["_toolEvent"] for c in chunks if "_toolEvent" in c] == [+ {"type": "context_window_exceeded"}+ ]+ finishes = [+ c["choices"][0]["finish_reason"] for c in chunks if c["choices"][0]["finish_reason"]+ ]+ assert finishes == ["length"]+ _, body = _call(stream = False, messages = full)+ assert body["choices"][0]["finish_reason"] == "length"+++ ("usage", "full"),+ [+ (_FULL_USAGE, True),+ # Thinking off: FastFlowLM decodes an empty think block, four tokens counted as one.+ ({**_FULL_USAGE, "completion_tokens": 8144, "total_tokens": 8189}, True),+ ({"total_tokens": 8192, "max_kv_token_capacity": 8192}, True),+ (_FLM_USAGE, False),+ ({"total_tokens": 8192}, False),+ (None, False),+ ],+)+def test_a_full_kv_cache_reads_the_kv_position(usage, full):+ import routes.inference as ri+ assert ri._npu_kv_cache_full(usage) is full+++def test_a_reply_with_room_left_keeps_its_stop(flm):+ flm()+ status, lines = _call(stream = True, messages = [{"role": "user", "content": "TIMED"}])+ assert status == 200+ chunks = _data(lines)+ assert not any("_toolEvent" in c for c in chunks)+ assert chunks[-1]["choices"][0]["finish_reason"] == "stop"+++def test_whitespace_beside_a_tool_call_is_no_content(flm):+ flm(tools = True)+ tools = [+ {"type": "function", "function": {"name": "get_weather", "parameters": {"type": "object"}}}+ ]+ _, body = _call(stream = False, tools = tools, messages = [{"role": "user", "content": "WSTOOL"}])+ message = body["choices"][0]["message"]+ assert message["content"] is None+ assert message["tool_calls"][0]["function"]["name"] == "get_weather"diff --git a/studio/backend/tests/test_npu_detect.py b/studio/backend/tests/test_npu_detect.pyindex 36a269e20bf..503b3a6fbeb 100644--- a/studio/backend/tests/test_npu_detect.py+++ b/studio/backend/tests/test_npu_detect.py@@ -78,15 +78,39 @@ def test_only_xdna2_is_supported(tmp_path, monkeypatch, device, supported):monkeypatch.setattr(npu, "_linux_probe", lambda: probe(tmp_path / "devices", tmp_path / "module"))+ monkeypatch.setenv(npu.ENABLE_ENV, "1")info = npu.detect_amd_npu()assert info["present"] is Trueassert info["supported"] is supported[email protected]("value", [None, "", "0", "false"])+def test_an_xdna2_npu_is_off_without_the_flag(tmp_path, monkeypatch, value):+ _device(tmp_path, "0000:c4:00.1", "1022", "17f0", driver = "amdxdna")+ probe = npu._linux_probe+ monkeypatch.setattr(npu.sys, "platform", "linux")+ monkeypatch.setattr(+ npu, "_linux_probe", lambda: probe(tmp_path / "devices", tmp_path / "module")+ )+ if value is None:+ monkeypatch.delenv(npu.ENABLE_ENV, raising = False)+ else:+ monkeypatch.setenv(npu.ENABLE_ENV, value)+ info = npu.detect_amd_npu()+ assert info["present"] is True and info["family"] == "XDNA2"+ assert info["enabled"] is False and info["supported"] is False++def test_detection_never_raises(monkeypatch):def broken():raise PermissionError("sysfs")monkeypatch.setattr(npu.sys, "platform", "linux")monkeypatch.setattr(npu, "_linux_probe", broken)- assert npu.detect_amd_npu() == {"present": False, "error": "sysfs", "supported": False}+ monkeypatch.delenv(npu.ENABLE_ENV, raising = False)+ assert npu.detect_amd_npu() == {+ "present": False,+ "error": "sysfs",+ "enabled": False,+ "supported": False,+ }diff --git a/studio/backend/tests/test_openai_auto_switch.py b/studio/backend/tests/test_openai_auto_switch.pyindex 7244ca532dc..dda419fbcf3 100644--- a/studio/backend/tests/test_openai_auto_switch.py+++ b/studio/backend/tests/test_openai_auto_switch.py@@ -319,8 +319,8 @@ async def _noop_reject(*_args, **_kwargs):return None-def _run_hook(model = "some/model"):- asyncio.run(inference_route._maybe_auto_switch_model(model, object(), "tester"))+def _run_hook(model = "some/model", **kwargs):+ asyncio.run(inference_route._maybe_auto_switch_model(model, object(), "tester", **kwargs))def test_flag_off_never_loads(monkeypatch):@@ -4410,6 +4410,8 @@ def test_chat_audio_input_guards_target_before_switch(monkeypatch):audio_preflight = None,image_preflight = None,tool_images_only = False,+ serves_npu = False,+ npu_request_check = None,):captured.update(require_vision = require_vision,@@ -9921,6 +9923,8 @@ def test_a_video_request_labels_the_switch_refusal_video(monkeypatch):audio_preflight = None,image_preflight = None,tool_images_only = False,+ serves_npu = False,+ npu_request_check = None,):captured.update(require_vision = require_vision,@@ -12363,6 +12367,281 @@ def test_tensor_split_requires_matching_gpu_ids_at_the_settings_boundary(ids):)+_NPU_MODEL = "lemonade:qwen3-0.6b-FLM"+++def test_an_npu_model_named_by_a_request_switches_when_on(monkeypatch):+ # The GGUF resolver cannot find an NPU id, so this used to 404 even with the switch on.+ backend, rec = _wired(monkeypatch, _FakeBackend("unsloth/A-GGUF"), None)+ _run_hook(_NPU_MODEL, serves_npu = True)+ assert [call.model_path for call in rec.calls] == [_NPU_MODEL]+++def test_an_npu_switch_loads_with_the_saved_context_length(monkeypatch):+ backend, rec = _wired(monkeypatch, _FakeBackend("unsloth/A-GGUF"), None)+ monkeypatch.setattr(+ settings,+ "get_model_override",+ lambda model_id: {"max_seq_length": 16384} if model_id == _NPU_MODEL else {},+ )+ _run_hook(_NPU_MODEL, serves_npu = True)+ assert [(call.model_path, call.max_seq_length) for call in rec.calls] == [(_NPU_MODEL, 16384)]+++def test_a_deferred_npu_switch_leaves_the_model_preview_owned(monkeypatch):+ # Chat defers its claim and can still reject after the switch (n > 1 on the NPU); unmarked, the+ # NPU model then read as Studio-owned and every public preview got 503.+ _wired(monkeypatch, _FakeBackend("unsloth/A-GGUF"), None)+ monkeypatch.setattr(inference_route, "_preview_resident_ident", None)+ asyncio.run(+ inference_route._maybe_auto_switch_model(+ _NPU_MODEL, object(), "tester", claim_resident = False, serves_npu = True+ )+ )+ assert inference_route._get_preview_resident() == _NPU_MODEL+++def test_an_npu_switch_keeps_the_setting_read_on_entry(monkeypatch):+ # Turned off mid-request, a second read loaded nothing and the resident GGUF answered instead.+ _, rec = _wired(monkeypatch, _FakeBackend("unsloth/A-GGUF"), None)+ state = {"on": True}+ monkeypatch.setattr(settings, "get_openai_auto_switch_enabled", lambda: state["on"])+ refuse = inference_route._refuse_unservable_npu_switch++ async def _refuse_then_turn_off(*args, **kwargs):+ await refuse(*args, **kwargs)+ state["on"] = False++ monkeypatch.setattr(inference_route, "_refuse_unservable_npu_switch", _refuse_then_turn_off)+ _run_hook(_NPU_MODEL, serves_npu = True)+ assert [call.model_path for call in rec.calls] == [_NPU_MODEL]+++def test_an_npu_model_named_with_the_switch_off_says_how_to_load_it(monkeypatch):+ backend, rec = _wired(monkeypatch, _FakeBackend("unsloth/A-GGUF"), None, enabled = False)+ with pytest.raises(HTTPException) as excinfo:+ _run_hook(_NPU_MODEL)+ assert excinfo.value.status_code == 404+ assert "Switch model by request" in str(excinfo.value.detail)+ assert rec.calls == []+++def test_a_resident_npu_model_is_not_reloaded(monkeypatch):+ backend, rec = _wired(monkeypatch, _FakeBackend("unsloth/A-GGUF"), None)+ monkeypatch.setattr(inference_route, "_loaded_satisfies", lambda model: model == _NPU_MODEL)+ _run_hook(_NPU_MODEL, serves_npu = True)+ assert rec.calls == []+++def test_messages_with_an_npu_model_loaded_names_it(monkeypatch):+ # "No GGUF model loaded" read as "nothing is loaded" while the NPU model was serving.+ from models.inference import AnthropicMessagesRequest++ _wired(monkeypatch, _FakeBackend(None), None, enabled = False)+ monkeypatch.setattr(inference_route, "_automatic_model_load_may_run", lambda: False)+ monkeypatch.setattr(+ inference_route,+ "_resident_npu_model",+ lambda: SimpleNamespace(model_path = _NPU_MODEL),+ )+ request = SimpleNamespace(+ state = SimpleNamespace(), url = SimpleNamespace(path = "/v1/messages"), method = "POST"+ )+ payload = AnthropicMessagesRequest(max_tokens = 16, messages = [{"role": "user", "content": "hi"}])+ with pytest.raises(HTTPException) as excinfo:+ asyncio.run(inference_route.anthropic_messages(payload, request, "tester"))+ assert excinfo.value.status_code == 400+ assert _NPU_MODEL in excinfo.value.detail+ assert "No GGUF model loaded" not in excinfo.value.detail+++def test_messages_with_switching_on_and_an_npu_model_loaded_names_it(monkeypatch):+ # The early check is skipped once automatic loads may run; the later one must say the same.+ from models.inference import AnthropicMessagesRequest++ _wired(monkeypatch, _FakeBackend(None), None, enabled = True)+ monkeypatch.setattr(+ inference_route,+ "_resident_npu_model",+ lambda: SimpleNamespace(model_path = _NPU_MODEL, id = "qwen3-0.6b-FLM"),+ )+ monkeypatch.setattr(inference_route, "_loaded_satisfies", lambda model: model == _NPU_MODEL)+ request = SimpleNamespace(+ state = SimpleNamespace(), url = SimpleNamespace(path = "/v1/messages"), method = "POST"+ )+ payload = AnthropicMessagesRequest(+ model = _NPU_MODEL, max_tokens = 16, messages = [{"role": "user", "content": "hi"}]+ )+ with pytest.raises(HTTPException) as excinfo:+ asyncio.run(inference_route.anthropic_messages(payload, request, "tester"))+ assert excinfo.value.status_code == 400+ assert _NPU_MODEL in str(excinfo.value.detail)+++def _npu_catalog_entry(+ monkeypatch,+ *,+ vision,+ tools = False,+):+ from core.inference import npu_backend as nb+ class _Npu:+ def loadable_model(self, model_id):+ return SimpleNamespace(id = model_id, vision = vision, tools = tools)++ monkeypatch.setattr(nb, "get_npu_backend", lambda: _Npu())+++ ("kwargs", "words"),+ [+ ({"serves_npu": False}, "serves /v1/chat/completions only"),+ ({"gguf_only": True}, "serves /v1/chat/completions only"),+ ({"require_audio_input": True}, "takes no audio or video input"),+ ({"require_video": True}, "takes no audio or video input"),+ ({"require_vision": True}, "cannot read images"),+ ],+)+def test_a_request_the_npu_model_cannot_serve_never_switches(monkeypatch, kwargs, words):+ # Loading first evicted the resident GGUF, then the endpoint failed on the NPU model anyway.+ backend, rec = _wired(monkeypatch, _FakeBackend("unsloth/A-GGUF"), None)+ _npu_catalog_entry(monkeypatch, vision = False)+ with pytest.raises(HTTPException) as excinfo:+ asyncio.run(+ inference_route._maybe_auto_switch_model(+ _NPU_MODEL, object(), "tester", **{"serves_npu": True, **kwargs}+ )+ )+ assert excinfo.value.status_code == 400+ assert words in str(excinfo.value.detail)+ assert rec.calls == []+ assert backend.model_identifier == "unsloth/A-GGUF"+++ ("fields", "param"),+ [+ (+ {"tools": [{"type": "function", "function": {"name": "f", "parameters": {}}}]},+ "tools",+ ),+ ({"n": 2}, "n"),+ ],+)+def test_a_chat_request_the_npu_model_would_refuse_never_switches(monkeypatch, fields, param):+ # NPU chat checked these only after the switch had already evicted the resident GGUF.+ backend, rec = _wired(monkeypatch, _FakeBackend("unsloth/A-GGUF"), None)+ _npu_catalog_entry(monkeypatch, vision = False, tools = False)+ payload = _chat_request(model = _NPU_MODEL, **fields)+ with pytest.raises(HTTPException) as excinfo:+ asyncio.run(inference_route.openai_chat_completions(payload, object(), "tester"))+ assert excinfo.value.status_code == 400+ assert param in str(excinfo.value.detail)+ assert rec.calls == []+ assert backend.model_identifier == "unsloth/A-GGUF"++[email protected]("route", ["/v1/responses", "/v1/messages"])+def test_a_route_the_npu_model_cannot_serve_never_switches_to_it(monkeypatch, route):+ # Both routes relax gguf_only for in-process models, so the NPU switch evicted the GGUF first.+ from starlette.requests import Request++ from models.inference import AnthropicMessagesRequest++ backend, rec = _wired(monkeypatch, _FakeBackend("unsloth/A-GGUF"), None)+ _npu_catalog_entry(monkeypatch, vision = True, tools = True)+ request = Request({"type": "http", "headers": [], "path": route, "method": "POST"})+ if route == "/v1/responses":+ payload = ResponsesRequest(model = _NPU_MODEL, stream = True, input = "hi")+ call = inference_route.openai_responses(payload, request, "tester")+ else:+ payload = AnthropicMessagesRequest(+ model = _NPU_MODEL, max_tokens = 16, messages = [{"role": "user", "content": "hi"}]+ )+ call = inference_route.anthropic_messages(payload, request, "tester")+ with pytest.raises(Exception) as excinfo:+ asyncio.run(call)+ assert rec.calls == []+ assert backend.model_identifier == "unsloth/A-GGUF"+ assert isinstance(excinfo.value, HTTPException) and excinfo.value.status_code == 400+ assert "serves /v1/chat/completions only" in str(excinfo.value.detail)+++def test_an_mcp_image_request_never_switches_to_an_npu_model(monkeypatch):+ # NPU chat runs no MCP tool loop, so it refused the image only after evicting the GGUF.+ backend, rec = _wired(monkeypatch, _FakeBackend("unsloth/A-GGUF"), None)+ _npu_catalog_entry(monkeypatch, vision = True, tools = True)++ async def _an_mcp_image(*_args):+ return object()++ monkeypatch.setattr(inference_route, "_request_mcp_image", _an_mcp_image)+ payload = _chat_request(model = _NPU_MODEL)+ with pytest.raises(HTTPException) as excinfo:+ asyncio.run(inference_route.openai_chat_completions(payload, object(), "tester"))+ assert excinfo.value.status_code == 400+ assert "MCP" in str(excinfo.value.detail)+ assert rec.calls == []+ assert backend.model_identifier == "unsloth/A-GGUF"+++def test_an_image_request_switches_to_a_vision_npu_model(monkeypatch):+ backend, rec = _wired(monkeypatch, _FakeBackend("unsloth/A-GGUF"), None)+ _npu_catalog_entry(monkeypatch, vision = True)+ asyncio.run(+ inference_route._maybe_auto_switch_model(+ _NPU_MODEL, object(), "tester", require_vision = True, serves_npu = True+ )+ )+ assert [call.model_path for call in rec.calls] == [_NPU_MODEL]+++def test_a_cancelled_request_never_switches_to_an_npu_model(monkeypatch):+ backend, rec = _wired(monkeypatch, _FakeBackend("unsloth/A-GGUF"), None)+ cancel = threading.Event()+ cancel.set()+ request = SimpleNamespace(state = SimpleNamespace(generation_cancel_event = cancel))+ with pytest.raises(HTTPException) as excinfo:+ asyncio.run(+ inference_route._maybe_auto_switch_model(_NPU_MODEL, request, "tester", serves_npu = True)+ )+ assert excinfo.value.status_code == 409+ assert rec.calls == []+++def test_a_tool_result_image_never_switches_to_a_text_only_npu_model(monkeypatch):+ # A text-only GGUF reads a tool-result image as a note; NPU chat rejects it after the load.+ backend, rec = _wired(monkeypatch, _FakeBackend("unsloth/A-GGUF"), None)+ _npu_catalog_entry(monkeypatch, vision = False)+ with pytest.raises(HTTPException) as excinfo:+ asyncio.run(+ inference_route._maybe_auto_switch_model(+ _NPU_MODEL,+ object(),+ "tester",+ require_vision = True,+ tool_images_only = True,+ serves_npu = True,+ )+ )+ assert "cannot read images" in str(excinfo.value.detail)+ assert rec.calls == []+++def test_an_npu_switch_counts_as_a_queued_switch_while_it_loads(monkeypatch):+ # Uncounted, a GGUF switch waiting on the lock this one holds kept its drain from finishing.+ _wired(monkeypatch, _FakeBackend("unsloth/A-GGUF"), None)+ seen = []++ async def _load(*_args, **_kwargs):+ seen.append(inference_route._switch_waiter_count())++ monkeypatch.setattr(inference_route, "_load_model_impl", _load)+ _run_hook(_NPU_MODEL, serves_npu = True)+ assert seen == [1]+ assert inference_route._switch_waiter_count() == 0++@pytest.mark.parametrize("reload", [False, True], ids = ["switch", "idle reload"])def test_completions_echo_to_a_gguf_target_is_refused_before_it_loads_on_mlx(monkeypatch, reload):from utils import hardwarediff --git a/studio/backend/utils/hardware/npu.py b/studio/backend/utils/hardware/npu.pyindex 66cf0727bde..b2d735e4c6d 100644--- a/studio/backend/utils/hardware/npu.py+++ b/studio/backend/utils/hardware/npu.py@@ -3,12 +3,13 @@"""Detect AMD NPUs without starting a runtime.-XDNA 2 (1022:17f0) is supported; XDNA 1 (1022:1502) is not.-Runtime readiness is checked separately by ``flm validate``.+XDNA 2 (1022:17f0) is supported, and only with ``UNSLOTH_STUDIO_ENABLE_NPU=1``; XDNA 1+(1022:1502) is not. Runtime readiness is checked separately by ``flm validate``."""from __future__ import annotations+import osimport sysfrom pathlib import Pathfrom typing import Any, Optional@@ -18,6 +19,12 @@ _XDNA2_DEVICE = "17f0"_XDNA1_DEVICE = "1502"_PCI_ROOT = Path("/sys/bus/pci/devices")_MODULE_ROOT = Path("/sys/module")+# Off by default: the NPU section, NPU loads and the CLI's NPU ids all wait on this.+ENABLE_ENV = "UNSLOTH_STUDIO_ENABLE_NPU"+++def npu_enabled() -> bool:+ return os.environ.get(ENABLE_ENV, "").strip().lower() in ("1", "true", "yes", "on")def _read(path: Path) -> Optional[str]:@@ -126,5 +133,8 @@ def detect_amd_npu() -> dict[str, Any]:info = {"present": False}except Exception as exc: # noqa: BLE001 -- detection must never break startup or /statusinfo = {"present": False, "error": str(exc)}- info["supported"] = bool(info.get("present")) and info.get("family") == "XDNA2"+ info["enabled"] = npu_enabled()+ info["supported"] = (+ info["enabled"] and bool(info.get("present")) and info.get("family") == "XDNA2"+ )return infodiff --git a/studio/frontend/src/features/hub/catalog/npu-catalog-list.tsx b/studio/frontend/src/features/hub/catalog/npu-catalog-list.tsxindex 05fa11dbf5c..30e9fd02648 100644--- a/studio/frontend/src/features/hub/catalog/npu-catalog-list.tsx+++ b/studio/frontend/src/features/hub/catalog/npu-catalog-list.tsx@@ -6,7 +6,6 @@ import { Spinner } from "@/components/ui/spinner";import {type NpuModel,type NpuPickerSource,- NpuPoweredBy,NpuSetupNotice,npuDownloadLabel,npuResumeLabel,@@ -71,9 +70,6 @@ export function NpuCatalogList({data-testid="hub-npu-catalog"className="mx-auto flex min-h-0 w-full max-w-[var(--hub-measure)] flex-1 flex-col gap-0.5 overflow-y-auto px-3 py-3">- {catalog.ready ? (- <NpuPoweredBy status={catalog.status} className="px-3 pb-2" />- ) : null}{body()}</div>);diff --git a/studio/frontend/src/features/model-picker/components/model-selector/pickers.tsx b/studio/frontend/src/features/model-picker/components/model-selector/pickers.tsxindex b8060ccd9c0..e1b9d16f2ed 100644--- a/studio/frontend/src/features/model-picker/components/model-selector/pickers.tsx+++ b/studio/frontend/src/features/model-picker/components/model-selector/pickers.tsx@@ -497,10 +497,24 @@ function ListLabel({divider ? "mt-3 border-t border-border pt-3" : "pt-3",)}>- <span className="flex items-center gap-1.5 text-ui-10 font-semibold uppercase tracking-wider text-muted-foreground">- {icon}- {children}- </span>+ {onToggle ? (+ // The chevron stays the keyboard control, so the label is not a second tab stop.+ <button+ type="button"+ tabIndex={-1}+ onClick={onToggle}+ aria-expanded={!collapsed}+ className="flex cursor-pointer select-none items-center gap-1.5 text-ui-10 font-semibold uppercase tracking-wider text-muted-foreground hover:text-foreground"+ >+ {icon}+ {children}+ </button>+ ) : (+ <span className="flex items-center gap-1.5 text-ui-10 font-semibold uppercase tracking-wider text-muted-foreground">+ {icon}+ {children}+ </span>+ )}{(action || onToggle) && (<div className="flex items-center gap-0.5">{action}diff --git a/studio/frontend/src/features/npu/index.ts b/studio/frontend/src/features/npu/index.tsindex 0c7263e398e..f298794e5a1 100644--- a/studio/frontend/src/features/npu/index.ts+++ b/studio/frontend/src/features/npu/index.ts@@ -12,7 +12,7 @@ export {npuRowsFor,npuSizeLabel,} from "./api";-export { NpuPoweredBy, NpuSetupNotice } from "./npu-setup-notice";+export { NpuSetupNotice } from "./npu-setup-notice";export {type NpuCatalog,type NpuPickerSource,diff --git a/studio/frontend/src/features/npu/npu-setup-notice.tsx b/studio/frontend/src/features/npu/npu-setup-notice.tsxindex 688f32c964c..2d23e13d927 100644--- a/studio/frontend/src/features/npu/npu-setup-notice.tsx+++ b/studio/frontend/src/features/npu/npu-setup-notice.tsx@@ -3,10 +3,8 @@import { Button } from "@/components/ui/button";import { Spinner } from "@/components/ui/spinner";-import { cn } from "@/lib/utils";import { AlertCircleIcon } from "@hugeicons/core-free-icons";import { HugeiconsIcon } from "@hugeicons/react";-import type { NpuStatus } from "./api";import type { NpuCatalog } from "./use-npu-catalog";export function NpuSetupNotice({ catalog }: { catalog: NpuCatalog }) {@@ -15,8 +13,8 @@ export function NpuSetupNotice({ catalog }: { catalog: NpuCatalog }) {return (<div className="flex flex-col gap-2 px-2.5 pb-2 text-xs"><p className="text-muted-foreground">- {status.hardware.name ?? "AMD NPU"}: FastFlowLM's own NPU builds, run- through Lemonade. GGUF models run on the GPU.+ {status.hardware.name ?? "AMD NPU"} found. These models run on the NPU,+ separately from the GPU models listed below.</p>{status.error || problems.length > 0 ? (<div className="flex gap-2 rounded-lg bg-destructive/10 p-2.5 text-destructive">@@ -63,45 +61,6 @@ export function NpuSetupNotice({ catalog }: { catalog: NpuCatalog }) {<Spinner className="size-3.5 text-muted-foreground" /></div>) : null}- <NpuPoweredBy status={status} /></div>);}--const version = (value: string | null | undefined) =>- value ? ` v${value}` : "";--export function NpuPoweredBy({- status,- className,-}: {- status: NpuStatus;- className?: string;-}) {- return (- <p- data-testid="npu-powered-by"- className={cn("text-ui-11 text-muted-foreground", className)}- >- Powered by{" "}- <a- href="https://github.com/ROCm/FastFlowLM"- target="_blank"- rel="noreferrer"- className="underline underline-offset-2"- >- FastFlowLM- </a>- {version(status.versions?.fastflowlm)} and{" "}- <a- href="https://github.com/lemonade-sdk/lemonade"- target="_blank"- rel="noreferrer"- className="underline underline-offset-2"- >- Lemonade- </a>- {version(status.versions?.lemonade)}.- </p>- );-}diff --git a/unsloth_cli/_inference.py b/unsloth_cli/_inference.pyindex 97ced9e1c86..07a80304b30 100644--- a/unsloth_cli/_inference.py+++ b/unsloth_cli/_inference.py@@ -1119,6 +1119,23 @@ def connect_studio_server(if not token:return _refuse("couldn't self-issue an Unsloth token (is Unsloth set up here?).")backend = HttpChatBackend(base_url, token)+ from unsloth_cli._npu import npu_model_path, prepare_npu_model++ npu_path = npu_model_path(model)+ if npu_path:+ # `unsloth inference` passes the bare id the picker shows; load it by its model path.+ model = npu_path+ try:+ prepare_npu_model(+ lambda method, path, payload, timeout: backend._request(+ method, path, payload, timeout = timeout+ ),+ model,+ lambda line: typer.echo(line, err = True),+ )+ except RuntimeError as exc:+ typer.echo(f"Error: {exc}", err = True)+ raise typer.Exit(code = 1)backend.ensure_loaded(model,hf_token = hf_token,diff --git a/unsloth_cli/_npu.py b/unsloth_cli/_npu.pynew file mode 100644index 00000000000..829052bba9a--- /dev/null+++ b/unsloth_cli/_npu.py@@ -0,0 +1,153 @@+# SPDX-License-Identifier: AGPL-3.0-only+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0++"""AMD Ryzen AI NPU models from the CLI, which drives a running Studio over HTTP."""++from __future__ import annotations++import json+import os+from typing import Callable, Optional+from urllib.parse import quote++NPU_MODEL_PREFIX = "lemonade:"+++class NpuCliError(RuntimeError):+ pass+++def npu_model_path(model: Optional[str]) -> Optional[str]:+ """`lemonade:<id>` for an NPU model path or bare id: Lemonade's NPU ids end in `-FLM`, no slash."""+ if not model:+ return None+ text = model.strip()+ if text.startswith(NPU_MODEL_PREFIX):+ return text if len(text) > len(NPU_MODEL_PREFIX) else None+ if "/" in text or "\\" in text or ":" in text or not text.lower().endswith("-flm"):+ return None+ if os.path.exists(text):+ return None+ return NPU_MODEL_PREFIX + text+++def npu_needs_studio_message(npu_path: str, *, no_server: bool) -> str:+ """Why `unsloth chat` / `unsloth inference` cannot run `npu_path` without a running Studio."""+ if no_server:+ return "NPU models run inside Unsloth Studio; drop --no-server."+ return (+ f"{npu_path[len(NPU_MODEL_PREFIX) :]} runs on the AMD NPU inside Unsloth Studio. Start it "+ f"with `unsloth studio` (or `unsloth run --model {npu_path}`) and run this again."+ )+++def _call(request: Callable[..., object], method: str, path: str, timeout: float):+ """Open `path`, turning an HTTP error into the server's own message."""+ import urllib.error+ try:+ return request(method, path, None, timeout)+ except urllib.error.HTTPError as exc:+ try:+ detail = json.loads(exc.read() or b"{}").get("detail")+ except Exception:+ detail = None+ raise NpuCliError(f"{method} {path} failed ({exc.code}): {detail or exc.reason}") from None+++def _read_json(response) -> dict:+ with response:+ body = json.loads(response.read() or b"{}")+ return body if isinstance(body, dict) else {}+++def prepare_npu_model(+ request: Callable[..., object], model_path: str, echo: Callable[[str], None]+) -> None:+ """Enable the NPU runtime and download `model_path` on the Studio behind `request`.++ `request(method, path, payload, timeout)` returns an open HTTP response for that Studio.+ """+ import http.client+ try:+ _prepare_npu_model(request, model_path, echo)+ except (OSError, http.client.HTTPException) as exc:+ # A refused or dropped connection, or a timeout: Studio stopped or restarted mid-way.+ raise NpuCliError(f"Lost the connection to Unsloth Studio: {exc}") from None+++def _prepare_npu_model(+ request: Callable[..., object], model_path: str, echo: Callable[[str], None]+) -> None:+ model_id = model_path[len(NPU_MODEL_PREFIX) :]+ status = _read_json(_call(request, "GET", "/api/npu/status", 60))+ if not status.get("supported"):+ hardware = status.get("hardware") or {}+ if not hardware.get("present"):+ reason = "no AMD Ryzen AI NPU was found on this machine"+ elif hardware.get("family") == "XDNA2" and hardware.get("enabled") is False:+ reason = (+ "the NPU is off by default. Start Unsloth Studio with "+ "UNSLOTH_STUDIO_ENABLE_NPU=1 to use it"+ )+ else:+ reason = (+ f"{hardware.get('name') or 'this NPU'} is not supported (an XDNA2 NPU is needed)"+ )+ raise NpuCliError(f"{model_id} runs on the AMD NPU, but {reason}.")+ if not status.get("ready"):+ echo("Enabling the NPU runtime (Lemonade and FastFlowLM, about 47 MB)...")+ status = _read_json(_call(request, "POST", "/api/npu/enable", 1800))+ if not status.get("ready"):+ problems = (status.get("validation") or {}).get("problems") or []+ detail = status.get("error") or " ".join(problems) or "the NPU driver check failed"+ raise NpuCliError(f"Could not enable the NPU runtime: {detail}")++ models = _read_json(_call(request, "GET", "/api/npu/models", 120)).get("models") or []+ entry = next((m for m in models if isinstance(m, dict) and m.get("id") == model_id), None)+ if entry is None:+ available = sorted(str(m.get("id")) for m in models if isinstance(m, dict))+ listed = ", ".join(available[:12]) + (", ..." if len(available) > 12 else "")+ raise NpuCliError(f"No NPU model named {model_id}. Available: {listed}")+ if entry.get("downloaded"):+ return++ size = entry.get("size_gb")+ echo(+ f"Downloading {model_id}"+ + (f" ({size:.1f} GB)" if isinstance(size, (int, float)) else "")+ + "..."+ )+ last_tenth = -1+ finished = False+ response = _call(+ request, "POST", f"/api/npu/models/{quote(model_id, safe = '')}/download", 7200+ )+ with response:+ for raw in response:+ line = (+ raw.decode("utf-8", "replace").strip()+ if isinstance(raw, bytes)+ else str(raw).strip()+ )+ if not line.startswith("data:"):+ continue+ try:+ event = json.loads(line[len("data:") :])+ except ValueError:+ continue+ if not isinstance(event, dict):+ continue+ if event.get("event") == "error" or event.get("error"):+ raise NpuCliError(+ f"Downloading {model_id} failed: {event.get('error') or 'unknown error'}"+ )+ percent = event.get("percent")+ if isinstance(percent, (int, float)) and int(percent) // 10 > last_tenth:+ last_tenth = int(percent) // 10+ echo(f" {int(percent)}%")+ if event.get("event") == "complete":+ finished = True+ if not finished:+ raise NpuCliError(+ f"The download of {model_id} ended early. Run the command again to resume it."+ )diff --git a/unsloth_cli/commands/chat.py b/unsloth_cli/commands/chat.pyindex f5fb38e5ab5..cf0a2d3379f 100644--- a/unsloth_cli/commands/chat.py+++ b/unsloth_cli/commands/chat.py@@ -2,6 +2,7 @@# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0import sys+from types import SimpleNamespacefrom typing import List, Optionalimport typer@@ -285,9 +286,32 @@ def chat(raise typer.Exit(code = 1)model = _pick_model(console)- # Resolve first so --compare can be rejected before the slow load.- with quiet_if_nonzero_mlx_rank():- model_config = resolve_model_config(model, hf_token = hf_token)+ from unsloth_cli._npu import npu_model_path, npu_needs_studio_message++ # An NPU model runs inside Unsloth Studio, never in this process: no HF config to resolve.+ npu_path = npu_model_path(model)+ if npu_path:+ model = npu_path+ model_config = SimpleNamespace(+ display_name = npu_path.split(":", 1)[1],+ is_gguf = False,+ is_lora = False,+ path = None,+ base_model = None,+ gguf_variant = None,+ )+ if no_server or is_mlx_distributed:+ if should_print:+ err.print(+ npu_needs_studio_message(npu_path, no_server = True),+ style = "red",+ markup = False,+ )+ raise typer.Exit(code = 1)+ else:+ # Resolve first so --compare can be rejected before the slow load.+ with quiet_if_nonzero_mlx_rank():+ model_config = resolve_model_config(model, hf_token = hf_token)compare_blocked = _compare_blocked_reason(model_config)if is_mlx_distributed:compare_blocked = (@@ -318,6 +342,14 @@ def chat(else connect_studio_server(model, **server_load_opts(ctx, load_opts)))server_mode = chat_backend is not None+ if npu_path and not server_mode:+ if should_print:+ err.print(+ npu_needs_studio_message(npu_path, no_server = False),+ style = "red",+ markup = False,+ )+ raise typer.Exit(code = 1)if server_mode and should_print:console.print("(Unsloth server connected — model stays warm after /exit)",diff --git a/unsloth_cli/commands/inference.py b/unsloth_cli/commands/inference.pyindex 93af7697a0e..b2b6f9c412e 100644--- a/unsloth_cli/commands/inference.py+++ b/unsloth_cli/commands/inference.py@@ -136,11 +136,21 @@ def inference(load_opts["speculative_type"] = speculative_typeif spec_draft_n_max is not None:load_opts["spec_draft_n_max"] = spec_draft_n_max+ from unsloth_cli._npu import npu_model_path, npu_needs_studio_message++ # An NPU model runs inside Unsloth Studio, never in this process.+ npu_path = npu_model_path(model)+ if npu_path and (no_server or is_mlx_distributed):+ typer.echo(f"Error: {npu_needs_studio_message(npu_path, no_server = True)}", err = True)+ raise typer.Exit(code = 1)chat_backend = (Noneif (no_server or is_mlx_distributed)else connect_studio_server(model, **server_load_opts(ctx, load_opts)))+ if chat_backend is None and npu_path:+ typer.echo(f"Error: {npu_needs_studio_message(npu_path, no_server = False)}", err = True)+ raise typer.Exit(code = 1)if chat_backend is None:chat_backend = load_chat_backend(model, **load_opts)try:diff --git a/unsloth_cli/commands/start.py b/unsloth_cli/commands/start.pyindex 24982e77ea0..ae20bfd39db 100644--- a/unsloth_cli/commands/start.py+++ b/unsloth_cli/commands/start.py@@ -2775,6 +2775,18 @@ def _fail_gguf_variant_missing(model_id: str, variant: str, variants: list) -> N_fail(message)+def _refuse_npu_model(model: Optional[str]) -> None:+ from unsloth_cli._npu import npu_model_path+ npu_path = npu_model_path(model)+ if npu_path:+ model_id = npu_path.split(":", 1)[1]+ # Unchecked, `lemonade:<id>` was split into a GGUF lookup for a repo named "lemonade".+ _fail(+ f"{model_id} is an AMD NPU model. `unsloth start` runs coding agents on GGUF models; "+ f"chat with {model_id} in Unsloth Studio or with `unsloth chat {model_id}`."+ )++def _fail_agent_needs_gguf(agent: _GgufAgent, model_id: str) -> NoReturn:message = (f"{agent.label} needs a GGUF model served by llama-server, " f"but {model_id} is not one."@@ -2797,6 +2809,7 @@ def _preflight_agent_gguf(# Only a complete listing with no .gguf files rejects; unknown defers to the# post-connect check. Mirrors _require_studio's auto-start condition, so with no start# possible its "no running server" error comes first, without a hub probe.+ _refuse_npu_model(model)if not (serve and launch and model):returnexpected = os.environ.get("UNSLOTH_STUDIO_URL", "http://127.0.0.1:8888").rstrip("/")@@ -4671,6 +4684,7 @@ def _connect(preload_check = None,connections: bool = False,) -> tuple:+ _refuse_npu_model(model)requested = modelexplicit_connection = bool(model) and model.startswith(_CONNECTION_MODEL_PREFIX)# `--model org/name:QUANT` is shorthand for `--model org/name --gguf-variant QUANT`. Split it before we match or serve so the attach path resolves against the already-loaded `org/name` (listed without the suffix) instead of reloading a `:`-suffixed repo id, which Unsloth rejects and which would evict a model another session is using.diff --git a/unsloth_cli/commands/studio.py b/unsloth_cli/commands/studio.pyindex 61b275683dd..cf01008f9c1 100644--- a/unsloth_cli/commands/studio.py+++ b/unsloth_cli/commands/studio.py@@ -2378,7 +2378,8 @@ def run(help = ("Model path or HF repo. Accepts llama.cpp-style ""`org/repo:variant` syntax. `-hf` / `--hf-repo` match "- "llama-server's spelling."+ "llama-server's spelling. An AMD Ryzen AI NPU model is "+ "`lemonade:<id>` or its bare id, e.g. `qwen3-0.6b-FLM`."),),gguf_variant: Optional[str] = typer.Option(@@ -2700,12 +2701,14 @@ def run(os.environ[_sampling_env] = str(_sampling_value)# Set before any re-exec. --log-verbose keeps llama-server's own -v passthrough working.+ added_log_verbose = Falseif verbose:_enable_verbose_access_logs()if engine == "auto" and not any(a in ("--verbose", "-v", "--log-verbose") for a in extra_llama_args):extra_llama_args.append("--log-verbose")+ added_log_verbose = Trueif disable_dns_pinning:os.environ["UNSLOTH_STUDIO_DISABLE_DNS_PINNING"] = "1"else:@@ -2735,8 +2738,23 @@ def run()raise typer.Exit(2)+ from unsloth_cli._npu import npu_model_path++ # An NPU model's `lemonade:` prefix is not a llama.cpp `repo:variant` split.+ npu_path = npu_model_path(model)+ if npu_path:+ if added_log_verbose:+ extra_llama_args.remove("--log-verbose")+ if gguf_variant or extra_llama_args or engine != "auto":+ typer.echo(+ "Error: NPU models run on FastFlowLM; --gguf-variant, --engine and "+ "llama-server flags do not apply to them.",+ err = True,+ )+ raise typer.Exit(2)+ model = npu_path# Parse llama.cpp `repo:variant`; error if it disagrees with --gguf-variant.- parsed_repo, embedded_variant = _split_repo_variant(model)+ parsed_repo, embedded_variant = (model, None) if npu_path else _split_repo_variant(model)if embedded_variant:if gguf_variant and gguf_variant != embedded_variant:typer.echo(@@ -3001,6 +3019,30 @@ def run(# The engine otherwise gets one GPU, which makes pipeline / data a no-op.from utils.hardware.hardware import resolve_requested_gpu_idsgpu_ids = resolve_requested_gpu_ids(None)+ if npu_path:+ from unsloth_cli._npu import prepare_npu_model+ def _npu_request(method, path, payload, timeout):+ import urllib.request+ request = urllib.request.Request(+ f"http://{_url_host(request_host)}:{actual_port}{path}",+ data = None if payload is None else json.dumps(payload).encode(),+ headers = {+ "Content-Type": "application/json",+ "Authorization": f"Bearer {api_key}",+ },+ method = method,+ )+ return _direct_urlopen(request, timeout = timeout)++ try:+ prepare_npu_model(+ _npu_request,+ model,+ (lambda _line: None) if silent else typer.echo,+ )+ except RuntimeError as exc:+ typer.echo(f"Error: {exc}", err = True)+ raise typer.Exit(1)if not silent:typer.echo(f"Loading model: {model}...")try:@@ -3077,7 +3119,8 @@ def run(typer.echo(context_length_line)typer.echo(f" API Key: {api_key}")typer.echo("")- typer.echo(" OpenAI / Anthropic SDK base URL:")+ # An NPU model serves /v1/chat/completions only: no Anthropic or streaming Responses example.+ typer.echo(" OpenAI SDK base URL:" if npu_path else " OpenAI / Anthropic SDK base URL:")typer.echo(f" {sdk_base_url}")typer.echo("=" * 56)typer.secho(_tool_notice, fg = _tool_notice_fg, bold = True)@@ -3090,20 +3133,21 @@ def run(""" -d '{"messages": [{"role": "user", "content": "Hello"}], "stream": true}'""")typer.echo("")- typer.echo("Anthropic Messages:")- typer.echo(f" curl {sdk_base_url}/messages \\")- typer.echo(f' -H "Authorization: Bearer {api_key}" \\')- typer.echo(' -H "Content-Type: application/json" \\')- typer.echo(- """ -d '{"max_tokens": 256, "messages": [{"role": "user", "content": "Hello"}], "stream": true}'"""- )- typer.echo("")- typer.echo("OpenAI Responses:")- typer.echo(f" curl {sdk_base_url}/responses \\")- typer.echo(f' -H "Authorization: Bearer {api_key}" \\')- typer.echo(' -H "Content-Type: application/json" \\')- typer.echo(""" -d '{"input": "Hello", "stream": true}'""")- typer.echo("")+ if not npu_path:+ typer.echo("Anthropic Messages:")+ typer.echo(f" curl {sdk_base_url}/messages \\")+ typer.echo(f' -H "Authorization: Bearer {api_key}" \\')+ typer.echo(' -H "Content-Type: application/json" \\')+ typer.echo(+ """ -d '{"max_tokens": 256, "messages": [{"role": "user", "content": "Hello"}], "stream": true}'"""+ )+ typer.echo("")+ typer.echo("OpenAI Responses:")+ typer.echo(f" curl {sdk_base_url}/responses \\")+ typer.echo(f' -H "Authorization: Bearer {api_key}" \\')+ typer.echo(' -H "Content-Type: application/json" \\')+ typer.echo(""" -d '{"input": "Hello", "stream": true}'""")+ typer.echo("")else:if secure and _cf_url:typer.echo(f"URL: {_cf_url}")diff --git a/unsloth_cli/tests/test_npu_cli.py b/unsloth_cli/tests/test_npu_cli.pynew file mode 100644index 00000000000..fabf9426ff2--- /dev/null+++ b/unsloth_cli/tests/test_npu_cli.py@@ -0,0 +1,273 @@+# SPDX-License-Identifier: AGPL-3.0-only+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0++"""AMD NPU models from `unsloth run`, `unsloth chat` and `unsloth start`."""++from __future__ import annotations++import io+import json+import sys+from pathlib import Path++import pytest+import typer+from typer.testing import CliRunner++_REPO_ROOT = Path(__file__).resolve().parents[2]+if str(_REPO_ROOT) not in sys.path:+ sys.path.insert(0, str(_REPO_ROOT))++from unsloth_cli._npu import NpuCliError, npu_model_path, prepare_npu_model+++ ("model", "expected"),+ [+ ("lemonade:qwen3-0.6b-FLM", "lemonade:qwen3-0.6b-FLM"),+ ("qwen3-0.6b-FLM", "lemonade:qwen3-0.6b-FLM"),+ ("qwen3.8-mtp-27b-flm", "lemonade:qwen3.8-mtp-27b-flm"),+ ("unsloth/Qwen3-0.6B-GGUF", None),+ ("unsloth/Qwen3-0.6B-GGUF:Q4_K_M", None),+ ("org/name-FLM", None),+ ("lemonade:", None),+ ("", None),+ (None, None),+ ],+)+def test_npu_model_path(model, expected):+ assert npu_model_path(model) == expected+++class _Studio:+ """A Studio's /api/npu routes, answering `prepare_npu_model`'s requests."""++ def __init__(+ self,+ *,+ supported = True,+ ready = False,+ downloaded = False,+ events = None,+ ):+ self.status = {"supported": supported, "ready": ready, "hardware": {"present": supported}}+ self.downloaded = downloaded+ self.events = (+ events+ if events is not None+ else [+ {"event": "progress", "percent": 0},+ {"event": "progress", "percent": 55},+ {"event": "complete", "percent": 100},+ ]+ )+ self.calls = []++ def __call__(self, method, path, payload, timeout):+ self.calls.append((method, path))+ if path == "/api/npu/status":+ return io.BytesIO(json.dumps(self.status).encode())+ if path == "/api/npu/enable":+ self.status["ready"] = True+ return io.BytesIO(json.dumps(self.status).encode())+ if path == "/api/npu/models":+ models = [+ {"id": "qwen3-0.6b-FLM", "downloaded": self.downloaded, "size_gb": 0.66},+ {"id": "gemma3-1b-FLM", "downloaded": False, "size_gb": 1.2},+ ]+ return io.BytesIO(json.dumps({"models": models}).encode())+ if path.endswith("/download"):+ return io.BytesIO("".join(f"data: {json.dumps(e)}\n\n" for e in self.events).encode())+ raise AssertionError(path)+++def test_a_fresh_npu_is_enabled_and_the_model_downloaded():+ studio, lines = _Studio(), []+ prepare_npu_model(studio, "lemonade:qwen3-0.6b-FLM", lines.append)+ assert studio.calls == [+ ("GET", "/api/npu/status"),+ ("POST", "/api/npu/enable"),+ ("GET", "/api/npu/models"),+ ("POST", "/api/npu/models/qwen3-0.6b-FLM/download"),+ ]+ assert lines[0].startswith("Enabling the NPU runtime")+ assert " 100%" in lines+++def test_a_ready_downloaded_model_needs_nothing():+ studio = _Studio(ready = True, downloaded = True)+ prepare_npu_model(studio, "lemonade:qwen3-0.6b-FLM", lambda _line: None)+ assert studio.calls == [("GET", "/api/npu/status"), ("GET", "/api/npu/models")]+++def test_no_npu_says_so():+ with pytest.raises(NpuCliError, match = "no AMD Ryzen AI NPU"):+ prepare_npu_model(_Studio(supported = False), "lemonade:qwen3-0.6b-FLM", print)+++def test_an_npu_left_off_names_the_flag():+ studio = _Studio(supported = False)+ studio.status["hardware"] = {"present": True, "family": "XDNA2", "enabled": False}+ with pytest.raises(NpuCliError, match = "UNSLOTH_STUDIO_ENABLE_NPU=1"):+ prepare_npu_model(studio, "lemonade:qwen3-0.6b-FLM", print)+++def test_an_unknown_model_lists_the_real_ones():+ with pytest.raises(NpuCliError, match = "gemma3-1b-FLM"):+ prepare_npu_model(_Studio(ready = True), "lemonade:nope-FLM", print)+++ "events",+ [+ [{"event": "error", "error": "disk full"}],+ [{"event": "progress", "percent": 40}],+ ],+)+def test_a_failed_or_cut_download_fails(events):+ with pytest.raises(NpuCliError):+ prepare_npu_model(_Studio(ready = True, events = events), "lemonade:qwen3-0.6b-FLM", print)+++ "error",+ [ConnectionResetError("reset"), TimeoutError("timed out"), OSError("refused")],+)+def test_a_lost_studio_connection_is_a_cli_error(error):+ # Only HTTP errors were converted, so a dropped connection surfaced as a traceback.+ def _request(*_args):+ raise error++ with pytest.raises(NpuCliError, match = "Lost the connection to Unsloth Studio"):+ prepare_npu_model(_request, "lemonade:qwen3-0.6b-FLM", print)+++def test_a_running_studio_prepares_and_loads_a_bare_npu_id(monkeypatch):+ # `unsloth inference qwen3-0.6b-FLM` cut the prefix length off the bare id and loaded it as is.+ import unsloth_cli._inference as inference+ import unsloth_cli._npu as npu++ seen = {}++ class _Backend:+ def __init__(self, *_args):+ pass++ def ensure_loaded(self, model, **_kwargs):+ seen["loaded"] = model++ monkeypatch.setenv("UNSLOTH_STUDIO_URL", "http://127.0.0.1:8888")+ monkeypatch.setattr(inference, "find_studio_server", lambda *a, **k: "http://127.0.0.1:8888")+ monkeypatch.setattr(inference, "verify_studio_identity", lambda *a, **k: True)+ monkeypatch.setattr(inference, "_studio_token", lambda: "token")+ monkeypatch.setattr(inference, "HttpChatBackend", _Backend)+ monkeypatch.setattr(+ npu, "prepare_npu_model", lambda _request, model, _echo: seen.setdefault("prepared", model)+ )+ inference.connect_studio_server(+ "qwen3-0.6b-FLM", hf_token = None, max_seq_length = 0, load_in_4bit = None+ )+ assert seen == {"prepared": "lemonade:qwen3-0.6b-FLM", "loaded": "lemonade:qwen3-0.6b-FLM"}+++def test_run_refuses_gguf_flags_for_an_npu_model():+ from unsloth_cli import app++ result = CliRunner().invoke(+ app, ["run", "--model", "lemonade:qwen3-0.6b-FLM", "--gguf-variant", "Q4_K_M"]+ )+ assert result.exit_code == 2+ assert "NPU models run on FastFlowLM" in result.output+++def test_run_accepts_verbose_for_an_npu_model(monkeypatch):+ # --verbose adds --log-verbose for llama-server, which then read as a user flag and exited 2.+ from unsloth_cli import app+ from unsloth_cli.commands import studio as studio_mod++ def _past_the_npu_checks(*_args, **_kwargs):+ raise typer.Exit(7)++ monkeypatch.setattr(studio_mod, "_enable_verbose_access_logs", lambda: None)+ monkeypatch.setattr(studio_mod, "_require_bind_host", _past_the_npu_checks)+ result = CliRunner().invoke(app, ["run", "--model", "lemonade:qwen3-0.6b-FLM", "--verbose"])+ assert result.exit_code == 7, result.output+++def test_run_does_not_split_the_lemonade_prefix():+ import inspect++ from unsloth_cli.commands import studio as studio_mod++ src = inspect.getsource(studio_mod.run)+ # `lemonade:<id>` was split llama.cpp-style into repo "lemonade", variant "<id>".+ assert "(model, None) if npu_path else _split_repo_variant(model)" in src+ assert "prepare_npu_model(" in src+++def test_start_names_an_npu_model_instead_of_looking_up_a_gguf(capsys, monkeypatch):+ from unsloth_cli.commands import start as start_mod++ # Never reach a real server: a Studio running on this machine would be asked to switch models.+ def _no_server(*_args, **_kwargs):+ raise AssertionError("unsloth start looked for a Studio before refusing the NPU model")++ monkeypatch.setattr(start_mod, "_require_studio", _no_server)+ with pytest.raises((typer.Exit, SystemExit)):+ start_mod._connect(None, "lemonade:qwen3-0.6b-FLM")+ assert "qwen3-0.6b-FLM is an AMD NPU model" in capsys.readouterr().err+++def test_chat_needs_a_running_studio_for_an_npu_model(monkeypatch):+ from unsloth_cli import app+ from unsloth_cli.commands import chat as chat_mod++ monkeypatch.setattr(chat_mod, "connect_studio_server", lambda *_a, **_k: None)+ result = CliRunner().invoke(app, ["chat", "qwen3-0.6b-FLM"])+ assert result.exit_code == 1+ assert "runs on the AMD NPU inside Unsloth Studio" in result.output+++ ("flags", "words"),+ [([], "runs on the AMD NPU inside Unsloth Studio"), (["--no-server"], "drop --no-server")],+)+def test_inference_needs_a_running_studio_for_an_npu_model(monkeypatch, flags, words):+ # Without a Studio the id fell through to a local load and a Hugging Face lookup.+ from unsloth_cli import app+ from unsloth_cli.commands import inference as inference_mod++ def _no_local_load(*_args, **_kwargs):+ raise AssertionError("an NPU model was loaded in-process")++ monkeypatch.setattr(inference_mod, "connect_studio_server", lambda *_a, **_k: None)+ monkeypatch.setattr(inference_mod, "load_chat_backend", _no_local_load)+ result = CliRunner().invoke(app, ["inference", "qwen3-0.6b-FLM", "hi", *flags])+ assert result.exit_code == 1, result.output+ assert words in result.output+++def test_a_gguf_agent_refuses_an_npu_model_before_any_hub_lookup(capsys, monkeypatch):+ from unsloth_cli.commands import start as start_mod++ def _no_lookup(*_args, **_kwargs):+ raise AssertionError("looked the NPU id up as a Hugging Face GGUF repo")++ monkeypatch.setattr(start_mod, "find_studio_server", _no_lookup)+ monkeypatch.setattr(start_mod, "_hub_gguf_files", _no_lookup)+ with pytest.raises((typer.Exit, SystemExit)):+ start_mod._preflight_agent_gguf(start_mod._CLAUDE_GGUF_AGENT, "lemonade:qwen3-0.6b-FLM")+ assert "is an AMD NPU model" in capsys.readouterr().err+++def test_the_run_banner_offers_no_endpoint_an_npu_model_refuses():+ import inspect++ from unsloth_cli.commands import studio as studio_mod++ # Structural: the banner prints after a live server load, which this suite cannot start.+ src = inspect.getsource(studio_mod.run)+ assert '" OpenAI SDK base URL:" if npu_path' in src+ banner = src[src.index('typer.echo("OpenAI Chat Completions:")') :]+ assert banner.index("if not npu_path:") < banner.index('typer.echo("Anthropic Messages:")')+ assert banner.index("if not npu_path:") < banner.index('typer.echo("OpenAI Responses:")')