Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

Studio: serve /v1/completions from managed vLLM and SGLang engines (#14227)

Daniel Han1935c8ab0894
diff --git a/studio/backend/routes/inference.py b/studio/backend/routes/inference.py
index 0bd6416dcee..d1a2674455f 100644
--- a/studio/backend/routes/inference.py
+++ b/studio/backend/routes/inference.py
@@ -39233,6 +39233,22 @@ def _active_mlx_backend():
return None
+def _active_managed_engine_backend():
+ backend = _peek_inference_backend()
+ name = getattr(backend, "active_model_name", None)
+ models = getattr(backend, "models", None)
+ engine = getattr(backend, "_managed_engine", None)
+ if (
+ engine is not None
+ and name
+ and isinstance(models, dict)
+ and models.get(name, {}).get("engine") in ("vllm", "sglang")
+ and engine.alive()
+ ):
+ return backend
+ return None
+
+
def _mlx_completion_sampling(body: dict, model_id) -> dict:
"""``generate_completion`` knobs for a /v1/completions body, read as llama-server reads
them: its field names and aliases, its clamps, and its defaults for anything unset."""
@@ -39606,7 +39622,7 @@ async def openai_completions(request: Request, current_subject: str = Depends(ge
OpenAI-compatible text completions endpoint (non-chat).
Proxies to the running llama-server's ``/v1/completions`` when a GGUF model is
- loaded, and serves an MLX model in process.
+ loaded or to a managed vLLM/SGLang engine's, and serves an MLX model in process.
"""
from utils.hardware import DeviceType, get_device
@@ -39643,7 +39659,10 @@ async def openai_completions(request: Request, current_subject: str = Depends(ge
)
if not _completions_prompt_present(_pre):
raise HTTPException(status_code = 400, detail = "'prompt' is required for completions.")
- _raise_ignored_completions_params(_pre, echo = not _mlx_device)
+ # A resident managed engine serves echo; a switch to a GGUF still refuses it below.
+ _raise_ignored_completions_params(
+ _pre, echo = not (_mlx_device or _active_managed_engine_backend() is not None)
+ )
elif _pre is not _UNPARSEABLE_BODY:
# A valid JSON body that is not an object (e.g. [] or null) is rejected below as
# "Request body must be a JSON object"; reject it here, before the switch, so the
@@ -39666,7 +39685,12 @@ async def openai_completions(request: Request, current_subject: str = Depends(ge
)
llama_backend = get_llama_cpp_backend()
mlx_backend = None if llama_backend.is_loaded else _active_mlx_backend()
- if not llama_backend.is_loaded and mlx_backend is None:
+ engine_backend = (
+ None
+ if llama_backend.is_loaded or mlx_backend is not None
+ else _active_managed_engine_backend()
+ )
+ if not llama_backend.is_loaded and mlx_backend is None and engine_backend is None:
_status, _detail = await _no_model_loaded_error(
"No GGUF model loaded. Load a GGUF model first.",
_raw_body_model(body),
@@ -39685,6 +39709,10 @@ async def openai_completions(request: Request, current_subject: str = Depends(ge
_SCHEMA_VALIDATION_EXECUTOR, _mlx_completions_refusal, body
)
return await _mlx_completions(request, current_subject, mlx_backend, body, refusal)
+ if engine_backend is not None:
+ _raise_ignored_completions_params(body, echo = False)
+ from routes.managed_engine_chat import managed_engine_completions
+ return await managed_engine_completions(request, current_subject, engine_backend, body)
_raise_ignored_completions_params(body)
# GGUF is loaded and the body is valid. The middleware claims the slot on a successful
diff --git a/studio/backend/routes/managed_engine_chat.py b/studio/backend/routes/managed_engine_chat.py
index d295f8e4b5a..93031888697 100644
--- a/studio/backend/routes/managed_engine_chat.py
+++ b/studio/backend/routes/managed_engine_chat.py
@@ -5,6 +5,7 @@
import asyncio
import json
+import threading
import time
import uuid
from contextlib import aclosing
@@ -477,3 +478,132 @@ async def managed_tool_chat(
if choice.get("finish_reason"):
result["choices"][0]["finish_reason"] = choice["finish_reason"]
return JSONResponse(result)
+
+
+async def managed_engine_completions(request, current_subject, backend, body):
+ from routes import inference as api
+ from core.inference.llama_keepwarm import mark_response_failed
+
+ engine = backend._managed_engine
+ public = api._orchestrator_public_model_id(backend) or engine.model
+ stream = bool(body.get("stream"))
+ include_usage = bool((body.get("stream_options") or {}).get("include_usage"))
+ upstream = {**body, "model": engine.model}
+ if stream:
+ # Always asked for so the monitor counts tokens; dropped again unless the caller opted in.
+ upstream["stream_options"] = {**(body.get("stream_options") or {}), "include_usage": True}
+ monitor_id = api.api_monitor.start(
+ endpoint = request.url.path,
+ via_api_key = api._request_used_api_key(request),
+ method = request.method,
+ model = public,
+ prompt = api._flatten_monitor_prompt(body.get("prompt", "")),
+ context_length = engine.context,
+ subject = current_subject,
+ )
+ cancel = threading.Event()
+ tracker = api._TrackedCancel(cancel, model = public, kind = "completions")
+ tracker.__enter__()
+ client = httpx.AsyncClient(trust_env = False, timeout = engine_request_timeout())
+ watcher = asyncio.create_task(
+ api._await_cancel_or_disconnect_then_close_client(
+ cancel_event = cancel, request = request, client = client
+ )
+ )
+ scope = getattr(request, "scope", None)
+
+ async def close():
+ try:
+ await api._stop_local_disconnect_cancel_watcher(watcher)
+ await client.aclose()
+ finally:
+ tracker.__exit__(None, None, None)
+
+ url = engine.base_url + "/v1/completions"
+ try:
+ if stream:
+ response = await client.send(
+ client.build_request("POST", url, headers = engine.headers, json = upstream),
+ stream = True,
+ )
+ if response.status_code != 200:
+ await response.aread()
+ else:
+ response = await client.post(url, headers = engine.headers, json = upstream)
+ except asyncio.CancelledError:
+ cancel.set()
+ await close()
+ api.api_monitor.finish(monitor_id, "cancelled")
+ raise
+ except Exception as exc:
+ await close()
+ if cancel.is_set():
+ mark_response_failed(scope)
+ api.api_monitor.finish(monitor_id, "cancelled")
+ raise HTTPException(status_code = 499, detail = "Request cancelled.") from exc
+ api.api_monitor.fail(monitor_id, api._friendly_error(exc))
+ raise
+ if response.status_code != 200:
+ await close()
+ api.api_monitor.fail(monitor_id, response.text[:500])
+ raise api._openai_passthrough_error(response.status_code, response.text)
+
+ if not stream:
+ await close()
+ data = response.json()
+ if isinstance(data, dict) and "model" in data:
+ data["model"] = public
+ try:
+ api._monitor_openai_chunk(monitor_id, data, engine.context)
+ except Exception:
+ pass
+ api.api_monitor.finish(monitor_id)
+ return JSONResponse(data)
+
+ async def relay():
+ lines = closing_response_lines(response)
+ try:
+ async for line in lines:
+ if not line.startswith("data:"):
+ continue
+ if api._monitor_openai_sse_line(monitor_id, line, engine.context) == "error":
+ mark_response_failed(scope)
+ out = api._cmpl_stream_event_out(_with_model(line, public).encode(), include_usage)
+ if out is not None:
+ yield out + b"\n\n"
+ if cancel.is_set():
+ mark_response_failed(scope)
+ api.api_monitor.finish(monitor_id, "cancelled")
+ else:
+ api.api_monitor.finish(monitor_id)
+ except asyncio.CancelledError:
+ cancel.set()
+ api.api_monitor.finish(monitor_id, "cancelled")
+ raise
+ except Exception as exc:
+ mark_response_failed(scope)
+ if cancel.is_set():
+ api.api_monitor.finish(monitor_id, "cancelled")
+ return
+ api.api_monitor.fail(monitor_id, api._friendly_error(exc))
+ yield api._openai_stream_error_sse_bytes(api._openai_stream_error_chunk(exc))
+ finally:
+ try:
+ await lines.aclose()
+ finally:
+ await close()
+
+ async def unstarted_cleanup():
+ cancel.set()
+ try:
+ # close() releases the active-generation entry even when aclose fails.
+ try:
+ await response.aclose()
+ finally:
+ await close()
+ finally:
+ api.api_monitor.finish(monitor_id, "cancelled")
+
+ return api._sse_streaming_response(
+ relay(), unstarted_cleanup = unstarted_cleanup, monitor_id = monitor_id
+ )
diff --git a/studio/backend/tests/test_managed_engine_completions.py b/studio/backend/tests/test_managed_engine_completions.py
new file mode 100644
index 00000000000..aa8cc2ba1cd
--- /dev/null
+++ b/studio/backend/tests/test_managed_engine_completions.py
@@ -0,0 +1,267 @@
+# SPDX-License-Identifier: AGPL-3.0-only
+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved.
+
+"""/v1/completions served by a resident managed vLLM/SGLang engine (unslothai/unsloth#7703)."""
+
+import asyncio
+import json
+from types import SimpleNamespace
+
+import pytest
+from fastapi import HTTPException
+
+from routes import inference as api
+from routes import managed_engine_chat
+from core.inference.api_monitor import ApiMonitor
+from utils.hardware import DeviceType
+
+LOGPROBS = {
+ "tokens": ["2", "+", "2", "=", "4"],
+ "token_logprobs": [None, -1.5, -0.2, -0.1, -0.05],
+ "top_logprobs": [None, {"+": -1.5}, {"2": -0.2}, {"=": -0.1}, {"4": -0.05}],
+ "text_offset": [0, 1, 2, 3, 4],
+}
+
+
+class _Request:
+ state = SimpleNamespace()
+ url = SimpleNamespace(path = "/v1/completions")
+ method = "POST"
+ headers: dict = {}
+ scope: dict = {}
+
+ def __init__(self, body):
+ self._body = body
+
+ async def json(self):
+ return self._body
+
+ async def is_disconnected(self):
+ return False
+
+
+def _orchestrator(engine = "vllm", alive = True):
+ backend = SimpleNamespace(
+ active_model_name = "unsloth/Qwen3-0.6B",
+ models = {"unsloth/Qwen3-0.6B": {"engine": engine}},
+ _managed_engine = SimpleNamespace(
+ model = "/models/qwen3",
+ context = 4096,
+ base_url = "http://engine.test",
+ headers = {"Authorization": "Bearer private"},
+ engine = engine,
+ alive = lambda: alive,
+ ),
+ )
+ return backend
+
+
+def engine(monkeypatch):
+ httpx = managed_engine_chat.httpx
+ requests = []
+
+ async def handle(request):
+ assert request.url.path == "/v1/completions"
+ assert request.headers["Authorization"] == "Bearer private"
+ body = json.loads(request.content)
+ requests.append(body)
+ if body.get("prompt") == "bad":
+ return httpx.Response(400, json = {"error": {"message": "invalid prompt"}})
+ usage = {"prompt_tokens": 4, "completion_tokens": 1, "total_tokens": 5}
+ choice = {"index": 0, "text": "2+2=4", "logprobs": LOGPROBS, "finish_reason": "length"}
+ if not body.get("stream"):
+ return httpx.Response(
+ 200,
+ json = {
+ "id": "cmpl-1",
+ "object": "text_completion",
+ "model": body["model"],
+ "choices": [choice],
+ "usage": usage,
+ },
+ )
+ events = [
+ {
+ "id": "cmpl-1",
+ "model": body["model"],
+ "choices": [{**choice, "finish_reason": None}],
+ },
+ {"id": "cmpl-1", "model": body["model"], "choices": [], "usage": usage},
+ ]
+ text = "".join("data: " + json.dumps(e) + "\n\n" for e in events) + "data: [DONE]\n\n"
+ return httpx.Response(200, text = text)
+
+ client = httpx.AsyncClient
+ monkeypatch.setattr(
+ httpx,
+ "AsyncClient",
+ lambda **kwargs: client(transport = httpx.MockTransport(handle), **kwargs),
+ )
+ monitor = ApiMonitor(max_entries = 8)
+ monkeypatch.setattr(api, "api_monitor", monitor)
+ monkeypatch.setattr(
+ api, "get_llama_cpp_backend", lambda: SimpleNamespace(is_loaded = False, context_length = None)
+ )
+ backend = _orchestrator()
+ monkeypatch.setattr(api, "_peek_inference_backend", lambda: backend)
+ monkeypatch.setattr(api, "_should_validate_before_switch", lambda: False)
+ monkeypatch.setattr("utils.hardware.get_device", lambda: DeviceType.CUDA)
+
+ async def no_switch(*_args, **_kwargs):
+ return None
+
+ monkeypatch.setattr(api, "_maybe_auto_switch_model", no_switch)
+ return SimpleNamespace(backend = backend, requests = requests, monitor = monitor)
+
+
+def _call(body):
+ async def run():
+ response = await api.openai_completions(_Request(body), current_subject = "u")
+ if body.get("stream"):
+ return [chunk async for chunk in response.body_iterator]
+ return json.loads(response.body)
+
+ return asyncio.run(run())
+
+
+def test_non_stream_relays_logprobs_and_echo(engine):
+ body = {
+ "model": "unsloth/Qwen3-0.6B",
+ "prompt": "2+2=",
+ "logprobs": 2,
+ "echo": True,
+ "max_tokens": 1,
+ }
+ result = _call(body)
+ [sent] = engine.requests
+ assert sent == {**body, "model": "/models/qwen3"}
+ assert result["model"] == "unsloth/Qwen3-0.6B"
+ assert result["choices"][0]["logprobs"] == LOGPROBS
+ [entry] = engine.monitor.snapshot()
+ assert entry["status"] == "completed"
+ assert engine.monitor.active_count() == 0
+
+
[email protected]("include_usage", [False, True])
+def test_stream_relays_logprobs_and_honours_usage_opt_in(engine, include_usage):
+ body = {"prompt": "2+2=", "logprobs": 1, "stream": True}
+ if include_usage:
+ body["stream_options"] = {"include_usage": True}
+ chunks = _call(body)
+ [sent] = engine.requests
+ assert sent["stream_options"] == {"include_usage": True}
+ events = [
+ json.loads(chunk[5:])
+ for chunk in b"".join(chunks).decode().split("\n\n")
+ if chunk.startswith("data:") and chunk[5:].strip() != "[DONE]"
+ ]
+ assert events[0]["model"] == "unsloth/Qwen3-0.6B"
+ assert events[0]["choices"][0]["logprobs"] == LOGPROBS
+ assert any(e.get("usage") for e in events) == include_usage
+ assert b"data: [DONE]" in b"".join(chunks)
+ assert engine.monitor.active_count() == 0
+
+
+def test_engine_rejection_keeps_its_status(engine):
+ with pytest.raises(HTTPException) as caught:
+ _call({"prompt": "bad"})
+ assert caught.value.status_code == 400
+ [entry] = engine.monitor.snapshot()
+ assert entry["status"] == "error"
+ assert engine.monitor.active_count() == 0
+
+
+def test_echo_reaches_a_resident_engine_before_any_switch(engine, monkeypatch):
+ monkeypatch.setattr(api, "_should_validate_before_switch", lambda: True)
+ _call({"prompt": "hi", "echo": True, "logprobs": 1, "max_tokens": 1})
+ assert engine.requests[0]["echo"] is True
+
+
+def test_dead_engine_and_no_engine_still_503(engine, monkeypatch):
+ for backend in (_orchestrator(alive = False), SimpleNamespace(active_model_name = None, models = {})):
+ monkeypatch.setattr(api, "_peek_inference_backend", lambda backend = backend: backend)
+ with pytest.raises(HTTPException) as caught:
+ _call({"prompt": "hi"})
+ assert caught.value.status_code == 503
+ assert engine.requests == []
+
+
+def test_safetensors_model_without_engine_still_503(engine, monkeypatch):
+ backend = _orchestrator()
+ backend.models["unsloth/Qwen3-0.6B"] = {}
+ monkeypatch.setattr(api, "_peek_inference_backend", lambda: backend)
+ with pytest.raises(HTTPException) as caught:
+ _call({"prompt": "hi"})
+ assert caught.value.status_code == 503
+ assert engine.requests == []
+
+
+def test_echo_still_refused_without_a_resident_engine(engine, monkeypatch):
+ monkeypatch.setattr(api, "_should_validate_before_switch", lambda: True)
+ monkeypatch.setattr(api, "_peek_inference_backend", lambda: None)
+ with pytest.raises(HTTPException) as caught:
+ _call({"prompt": "hi", "echo": True})
+ assert caught.value.status_code == 400
+ assert engine.requests == []
+
+
+def test_loaded_gguf_wins_over_engine(engine, monkeypatch):
+ captured = []
+ monkeypatch.setattr(
+ api,
+ "get_llama_cpp_backend",
+ lambda: SimpleNamespace(
+ is_loaded = True,
+ base_url = "http://llama.test",
+ context_length = 4096,
+ model_identifier = "gguf",
+ ),
+ )
+ monkeypatch.setattr(api, "_cancelable_nonstreaming_client", lambda: _PostClient(captured))
+ monkeypatch.setattr(api, "_fill_recommended_sampling_completions", lambda *a, **k: None)
+ _call({"prompt": "hi"})
+ assert captured == ["http://llama.test/v1/completions"]
+ assert engine.requests == []
+
+
+class _PostClient:
+ def __init__(self, captured):
+ self.captured = captured
+
+ async def post(self, url, **_kwargs):
+ self.captured.append(url)
+ return managed_engine_chat.httpx.Response(
+ 200, json = {"id": "cmpl-x", "choices": [{"text": "ok", "index": 0}]}
+ )
+
+ async def aclose(self):
+ pass
+
+ async def __aenter__(self):
+ return self
+
+ async def __aexit__(self, *_):
+ pass
+
+
+def test_unstarted_stream_releases_tracking_when_the_upstream_close_fails(engine, monkeypatch):
+ """The SSE wrapper swallows this hook's error, so a skipped release leaves the model busy."""
+ from state import active_generations
+
+ async def boom(self):
+ raise RuntimeError("transport gone")
+
+ active_generations.reset_for_tests()
+
+ async def run():
+ response = await api.openai_completions(
+ _Request({"prompt": "2+2=", "stream": True}), current_subject = "u"
+ )
+ assert active_generations.count() == 1
+ monkeypatch.setattr(managed_engine_chat.httpx.Response, "aclose", boom)
+ with pytest.raises(RuntimeError):
+ await response._unstarted_cleanup()
+
+ asyncio.run(run())
+ assert active_generations.count() == 0