Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
Studio: serve /v1/completions from managed vLLM and SGLang engines (#14227)
Daniel Han1935c8ab0894
diff --git a/studio/backend/routes/inference.py b/studio/backend/routes/inference.pyindex 0bd6416dcee..d1a2674455f 100644--- a/studio/backend/routes/inference.py+++ b/studio/backend/routes/inference.py@@ -39233,6 +39233,22 @@ def _active_mlx_backend():return None+def _active_managed_engine_backend():+ backend = _peek_inference_backend()+ name = getattr(backend, "active_model_name", None)+ models = getattr(backend, "models", None)+ engine = getattr(backend, "_managed_engine", None)+ if (+ engine is not None+ and name+ and isinstance(models, dict)+ and models.get(name, {}).get("engine") in ("vllm", "sglang")+ and engine.alive()+ ):+ return backend+ return None++def _mlx_completion_sampling(body: dict, model_id) -> dict:"""``generate_completion`` knobs for a /v1/completions body, read as llama-server readsthem: its field names and aliases, its clamps, and its defaults for anything unset."""@@ -39606,7 +39622,7 @@ async def openai_completions(request: Request, current_subject: str = Depends(geOpenAI-compatible text completions endpoint (non-chat).Proxies to the running llama-server's ``/v1/completions`` when a GGUF model is- loaded, and serves an MLX model in process.+ loaded or to a managed vLLM/SGLang engine's, and serves an MLX model in process."""from utils.hardware import DeviceType, get_device@@ -39643,7 +39659,10 @@ async def openai_completions(request: Request, current_subject: str = Depends(ge)if not _completions_prompt_present(_pre):raise HTTPException(status_code = 400, detail = "'prompt' is required for completions.")- _raise_ignored_completions_params(_pre, echo = not _mlx_device)+ # A resident managed engine serves echo; a switch to a GGUF still refuses it below.+ _raise_ignored_completions_params(+ _pre, echo = not (_mlx_device or _active_managed_engine_backend() is not None)+ )elif _pre is not _UNPARSEABLE_BODY:# A valid JSON body that is not an object (e.g. [] or null) is rejected below as# "Request body must be a JSON object"; reject it here, before the switch, so the@@ -39666,7 +39685,12 @@ async def openai_completions(request: Request, current_subject: str = Depends(ge)llama_backend = get_llama_cpp_backend()mlx_backend = None if llama_backend.is_loaded else _active_mlx_backend()- if not llama_backend.is_loaded and mlx_backend is None:+ engine_backend = (+ None+ if llama_backend.is_loaded or mlx_backend is not None+ else _active_managed_engine_backend()+ )+ if not llama_backend.is_loaded and mlx_backend is None and engine_backend is None:_status, _detail = await _no_model_loaded_error("No GGUF model loaded. Load a GGUF model first.",_raw_body_model(body),@@ -39685,6 +39709,10 @@ async def openai_completions(request: Request, current_subject: str = Depends(ge_SCHEMA_VALIDATION_EXECUTOR, _mlx_completions_refusal, body)return await _mlx_completions(request, current_subject, mlx_backend, body, refusal)+ if engine_backend is not None:+ _raise_ignored_completions_params(body, echo = False)+ from routes.managed_engine_chat import managed_engine_completions+ return await managed_engine_completions(request, current_subject, engine_backend, body)_raise_ignored_completions_params(body)# GGUF is loaded and the body is valid. The middleware claims the slot on a successfuldiff --git a/studio/backend/routes/managed_engine_chat.py b/studio/backend/routes/managed_engine_chat.pyindex d295f8e4b5a..93031888697 100644--- a/studio/backend/routes/managed_engine_chat.py+++ b/studio/backend/routes/managed_engine_chat.py@@ -5,6 +5,7 @@import asyncioimport json+import threadingimport timeimport uuidfrom contextlib import aclosing@@ -477,3 +478,132 @@ async def managed_tool_chat(if choice.get("finish_reason"):result["choices"][0]["finish_reason"] = choice["finish_reason"]return JSONResponse(result)+++async def managed_engine_completions(request, current_subject, backend, body):+ from routes import inference as api+ from core.inference.llama_keepwarm import mark_response_failed++ engine = backend._managed_engine+ public = api._orchestrator_public_model_id(backend) or engine.model+ stream = bool(body.get("stream"))+ include_usage = bool((body.get("stream_options") or {}).get("include_usage"))+ upstream = {**body, "model": engine.model}+ if stream:+ # Always asked for so the monitor counts tokens; dropped again unless the caller opted in.+ upstream["stream_options"] = {**(body.get("stream_options") or {}), "include_usage": True}+ monitor_id = api.api_monitor.start(+ endpoint = request.url.path,+ via_api_key = api._request_used_api_key(request),+ method = request.method,+ model = public,+ prompt = api._flatten_monitor_prompt(body.get("prompt", "")),+ context_length = engine.context,+ subject = current_subject,+ )+ cancel = threading.Event()+ tracker = api._TrackedCancel(cancel, model = public, kind = "completions")+ tracker.__enter__()+ client = httpx.AsyncClient(trust_env = False, timeout = engine_request_timeout())+ watcher = asyncio.create_task(+ api._await_cancel_or_disconnect_then_close_client(+ cancel_event = cancel, request = request, client = client+ )+ )+ scope = getattr(request, "scope", None)++ async def close():+ try:+ await api._stop_local_disconnect_cancel_watcher(watcher)+ await client.aclose()+ finally:+ tracker.__exit__(None, None, None)++ url = engine.base_url + "/v1/completions"+ try:+ if stream:+ response = await client.send(+ client.build_request("POST", url, headers = engine.headers, json = upstream),+ stream = True,+ )+ if response.status_code != 200:+ await response.aread()+ else:+ response = await client.post(url, headers = engine.headers, json = upstream)+ except asyncio.CancelledError:+ cancel.set()+ await close()+ api.api_monitor.finish(monitor_id, "cancelled")+ raise+ except Exception as exc:+ await close()+ if cancel.is_set():+ mark_response_failed(scope)+ api.api_monitor.finish(monitor_id, "cancelled")+ raise HTTPException(status_code = 499, detail = "Request cancelled.") from exc+ api.api_monitor.fail(monitor_id, api._friendly_error(exc))+ raise+ if response.status_code != 200:+ await close()+ api.api_monitor.fail(monitor_id, response.text[:500])+ raise api._openai_passthrough_error(response.status_code, response.text)++ if not stream:+ await close()+ data = response.json()+ if isinstance(data, dict) and "model" in data:+ data["model"] = public+ try:+ api._monitor_openai_chunk(monitor_id, data, engine.context)+ except Exception:+ pass+ api.api_monitor.finish(monitor_id)+ return JSONResponse(data)++ async def relay():+ lines = closing_response_lines(response)+ try:+ async for line in lines:+ if not line.startswith("data:"):+ continue+ if api._monitor_openai_sse_line(monitor_id, line, engine.context) == "error":+ mark_response_failed(scope)+ out = api._cmpl_stream_event_out(_with_model(line, public).encode(), include_usage)+ if out is not None:+ yield out + b"\n\n"+ if cancel.is_set():+ mark_response_failed(scope)+ api.api_monitor.finish(monitor_id, "cancelled")+ else:+ api.api_monitor.finish(monitor_id)+ except asyncio.CancelledError:+ cancel.set()+ api.api_monitor.finish(monitor_id, "cancelled")+ raise+ except Exception as exc:+ mark_response_failed(scope)+ if cancel.is_set():+ api.api_monitor.finish(monitor_id, "cancelled")+ return+ api.api_monitor.fail(monitor_id, api._friendly_error(exc))+ yield api._openai_stream_error_sse_bytes(api._openai_stream_error_chunk(exc))+ finally:+ try:+ await lines.aclose()+ finally:+ await close()++ async def unstarted_cleanup():+ cancel.set()+ try:+ # close() releases the active-generation entry even when aclose fails.+ try:+ await response.aclose()+ finally:+ await close()+ finally:+ api.api_monitor.finish(monitor_id, "cancelled")++ return api._sse_streaming_response(+ relay(), unstarted_cleanup = unstarted_cleanup, monitor_id = monitor_id+ )diff --git a/studio/backend/tests/test_managed_engine_completions.py b/studio/backend/tests/test_managed_engine_completions.pynew file mode 100644index 00000000000..aa8cc2ba1cd--- /dev/null+++ b/studio/backend/tests/test_managed_engine_completions.py@@ -0,0 +1,267 @@+# SPDX-License-Identifier: AGPL-3.0-only+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved.++"""/v1/completions served by a resident managed vLLM/SGLang engine (unslothai/unsloth#7703)."""++import asyncio+import json+from types import SimpleNamespace++import pytest+from fastapi import HTTPException++from routes import inference as api+from routes import managed_engine_chat+from core.inference.api_monitor import ApiMonitor+from utils.hardware import DeviceType++LOGPROBS = {+ "tokens": ["2", "+", "2", "=", "4"],+ "token_logprobs": [None, -1.5, -0.2, -0.1, -0.05],+ "top_logprobs": [None, {"+": -1.5}, {"2": -0.2}, {"=": -0.1}, {"4": -0.05}],+ "text_offset": [0, 1, 2, 3, 4],+}+++class _Request:+ state = SimpleNamespace()+ url = SimpleNamespace(path = "/v1/completions")+ method = "POST"+ headers: dict = {}+ scope: dict = {}++ def __init__(self, body):+ self._body = body++ async def json(self):+ return self._body++ async def is_disconnected(self):+ return False+++def _orchestrator(engine = "vllm", alive = True):+ backend = SimpleNamespace(+ active_model_name = "unsloth/Qwen3-0.6B",+ models = {"unsloth/Qwen3-0.6B": {"engine": engine}},+ _managed_engine = SimpleNamespace(+ model = "/models/qwen3",+ context = 4096,+ base_url = "http://engine.test",+ headers = {"Authorization": "Bearer private"},+ engine = engine,+ alive = lambda: alive,+ ),+ )+ return backend+++def engine(monkeypatch):+ httpx = managed_engine_chat.httpx+ requests = []++ async def handle(request):+ assert request.url.path == "/v1/completions"+ assert request.headers["Authorization"] == "Bearer private"+ body = json.loads(request.content)+ requests.append(body)+ if body.get("prompt") == "bad":+ return httpx.Response(400, json = {"error": {"message": "invalid prompt"}})+ usage = {"prompt_tokens": 4, "completion_tokens": 1, "total_tokens": 5}+ choice = {"index": 0, "text": "2+2=4", "logprobs": LOGPROBS, "finish_reason": "length"}+ if not body.get("stream"):+ return httpx.Response(+ 200,+ json = {+ "id": "cmpl-1",+ "object": "text_completion",+ "model": body["model"],+ "choices": [choice],+ "usage": usage,+ },+ )+ events = [+ {+ "id": "cmpl-1",+ "model": body["model"],+ "choices": [{**choice, "finish_reason": None}],+ },+ {"id": "cmpl-1", "model": body["model"], "choices": [], "usage": usage},+ ]+ text = "".join("data: " + json.dumps(e) + "\n\n" for e in events) + "data: [DONE]\n\n"+ return httpx.Response(200, text = text)++ client = httpx.AsyncClient+ monkeypatch.setattr(+ httpx,+ "AsyncClient",+ lambda **kwargs: client(transport = httpx.MockTransport(handle), **kwargs),+ )+ monitor = ApiMonitor(max_entries = 8)+ monkeypatch.setattr(api, "api_monitor", monitor)+ monkeypatch.setattr(+ api, "get_llama_cpp_backend", lambda: SimpleNamespace(is_loaded = False, context_length = None)+ )+ backend = _orchestrator()+ monkeypatch.setattr(api, "_peek_inference_backend", lambda: backend)+ monkeypatch.setattr(api, "_should_validate_before_switch", lambda: False)+ monkeypatch.setattr("utils.hardware.get_device", lambda: DeviceType.CUDA)++ async def no_switch(*_args, **_kwargs):+ return None++ monkeypatch.setattr(api, "_maybe_auto_switch_model", no_switch)+ return SimpleNamespace(backend = backend, requests = requests, monitor = monitor)+++def _call(body):+ async def run():+ response = await api.openai_completions(_Request(body), current_subject = "u")+ if body.get("stream"):+ return [chunk async for chunk in response.body_iterator]+ return json.loads(response.body)++ return asyncio.run(run())+++def test_non_stream_relays_logprobs_and_echo(engine):+ body = {+ "model": "unsloth/Qwen3-0.6B",+ "prompt": "2+2=",+ "logprobs": 2,+ "echo": True,+ "max_tokens": 1,+ }+ result = _call(body)+ [sent] = engine.requests+ assert sent == {**body, "model": "/models/qwen3"}+ assert result["model"] == "unsloth/Qwen3-0.6B"+ assert result["choices"][0]["logprobs"] == LOGPROBS+ [entry] = engine.monitor.snapshot()+ assert entry["status"] == "completed"+ assert engine.monitor.active_count() == 0++[email protected]("include_usage", [False, True])+def test_stream_relays_logprobs_and_honours_usage_opt_in(engine, include_usage):+ body = {"prompt": "2+2=", "logprobs": 1, "stream": True}+ if include_usage:+ body["stream_options"] = {"include_usage": True}+ chunks = _call(body)+ [sent] = engine.requests+ assert sent["stream_options"] == {"include_usage": True}+ events = [+ json.loads(chunk[5:])+ for chunk in b"".join(chunks).decode().split("\n\n")+ if chunk.startswith("data:") and chunk[5:].strip() != "[DONE]"+ ]+ assert events[0]["model"] == "unsloth/Qwen3-0.6B"+ assert events[0]["choices"][0]["logprobs"] == LOGPROBS+ assert any(e.get("usage") for e in events) == include_usage+ assert b"data: [DONE]" in b"".join(chunks)+ assert engine.monitor.active_count() == 0+++def test_engine_rejection_keeps_its_status(engine):+ with pytest.raises(HTTPException) as caught:+ _call({"prompt": "bad"})+ assert caught.value.status_code == 400+ [entry] = engine.monitor.snapshot()+ assert entry["status"] == "error"+ assert engine.monitor.active_count() == 0+++def test_echo_reaches_a_resident_engine_before_any_switch(engine, monkeypatch):+ monkeypatch.setattr(api, "_should_validate_before_switch", lambda: True)+ _call({"prompt": "hi", "echo": True, "logprobs": 1, "max_tokens": 1})+ assert engine.requests[0]["echo"] is True+++def test_dead_engine_and_no_engine_still_503(engine, monkeypatch):+ for backend in (_orchestrator(alive = False), SimpleNamespace(active_model_name = None, models = {})):+ monkeypatch.setattr(api, "_peek_inference_backend", lambda backend = backend: backend)+ with pytest.raises(HTTPException) as caught:+ _call({"prompt": "hi"})+ assert caught.value.status_code == 503+ assert engine.requests == []+++def test_safetensors_model_without_engine_still_503(engine, monkeypatch):+ backend = _orchestrator()+ backend.models["unsloth/Qwen3-0.6B"] = {}+ monkeypatch.setattr(api, "_peek_inference_backend", lambda: backend)+ with pytest.raises(HTTPException) as caught:+ _call({"prompt": "hi"})+ assert caught.value.status_code == 503+ assert engine.requests == []+++def test_echo_still_refused_without_a_resident_engine(engine, monkeypatch):+ monkeypatch.setattr(api, "_should_validate_before_switch", lambda: True)+ monkeypatch.setattr(api, "_peek_inference_backend", lambda: None)+ with pytest.raises(HTTPException) as caught:+ _call({"prompt": "hi", "echo": True})+ assert caught.value.status_code == 400+ assert engine.requests == []+++def test_loaded_gguf_wins_over_engine(engine, monkeypatch):+ captured = []+ monkeypatch.setattr(+ api,+ "get_llama_cpp_backend",+ lambda: SimpleNamespace(+ is_loaded = True,+ base_url = "http://llama.test",+ context_length = 4096,+ model_identifier = "gguf",+ ),+ )+ monkeypatch.setattr(api, "_cancelable_nonstreaming_client", lambda: _PostClient(captured))+ monkeypatch.setattr(api, "_fill_recommended_sampling_completions", lambda *a, **k: None)+ _call({"prompt": "hi"})+ assert captured == ["http://llama.test/v1/completions"]+ assert engine.requests == []+++class _PostClient:+ def __init__(self, captured):+ self.captured = captured++ async def post(self, url, **_kwargs):+ self.captured.append(url)+ return managed_engine_chat.httpx.Response(+ 200, json = {"id": "cmpl-x", "choices": [{"text": "ok", "index": 0}]}+ )++ async def aclose(self):+ pass++ async def __aenter__(self):+ return self++ async def __aexit__(self, *_):+ pass+++def test_unstarted_stream_releases_tracking_when_the_upstream_close_fails(engine, monkeypatch):+ """The SSE wrapper swallows this hook's error, so a skipped release leaves the model busy."""+ from state import active_generations++ async def boom(self):+ raise RuntimeError("transport gone")++ active_generations.reset_for_tests()++ async def run():+ response = await api.openai_completions(+ _Request({"prompt": "2+2=", "stream": True}), current_subject = "u"+ )+ assert active_generations.count() == 1+ monkeypatch.setattr(managed_engine_chat.httpx.Response, "aclose", boom)+ with pytest.raises(RuntimeError):+ await response._unstarted_cleanup()++ asyncio.run(run())+ assert active_generations.count() == 0