Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
Studio: start vLLM on hybrid models whose Mamba state outruns max_num_seqs (#13235)
Daniel Hanf64ed0dc73d5
diff --git a/studio/backend/core/inference/engine_adapters.py b/studio/backend/core/inference/engine_adapters.pyindex 1622088c237..431df6f2fc1 100644--- a/studio/backend/core/inference/engine_adapters.py+++ b/studio/backend/core/inference/engine_adapters.py@@ -209,6 +209,11 @@ class EngineAdapter:*parallel_args,*precision_args,*tool_args,+ *(+ ["--max-num-seqs", str(options["max_num_seqs"])]+ if self.name == "vllm" and options.get("max_num_seqs")+ else []+ ),*(["--trust-remote-code"] if trust_remote_code else []),*self.extra_args,"--host",diff --git a/studio/backend/core/inference/managed_engine.py b/studio/backend/core/inference/managed_engine.pyindex 7f32f20dc27..94aa1be89a7 100644--- a/studio/backend/core/inference/managed_engine.py+++ b/studio/backend/core/inference/managed_engine.py@@ -8,6 +8,7 @@ from __future__ import annotationsimport jsonimport hashlibimport os+import reimport secretsimport socketimport subprocess@@ -246,6 +247,21 @@ def _deep_gemm_unloadable(environment: str) -> bool:STARTUP_STALL_S = 900STARTUP_LIMIT_S = 4 * 3600+# vLLM 0.30 on hybrid models when its default max_num_seqs outruns the KV budget (Qwen3.8 NVFP4 on 32 GB, #8861).+_MAMBA_SEQ_LIMIT = re.compile(+ r"max_num_seqs \((\d+)\) exceeds available Mamba cache blocks \((\d+)\)"+)+++# Per launch config, so a reload skips the failing first launch.+_SEQ_LIMITS: dict[tuple, int] = {}+++class _MambaSeqLimit(RuntimeError):+ def __init__(self, message: str, blocks: int):+ super().__init__(message)+ self.blocks = blocks+def _token_file_token(env) -> str | None:"""The `hf auth login` token a local engine would read from its inherited HF_HOME; the WSL@@ -280,6 +296,7 @@ class ManagedEngine:self._lease = Noneself._reader = Noneself._tail = deque(maxlen = 200)+ self._mamba_blocks = Noneself._last_output = time.monotonic()self.base_url = ""self._guest_environment = None@@ -301,6 +318,44 @@ class ManagedEngine:model_path = None,):"""``model`` is the served name; ``model_path`` is what the engine loads when they differ."""+ args = (model, context, gpu_ids, env, cancel_event)+ options = dict(options or {})+ key = (+ self.engine,+ model_path or model,+ context,+ tuple(gpu_ids or ()),+ repr(sorted(options.items())),+ )+ if key in _SEQ_LIMITS and "max_num_seqs" not in options:+ options["max_num_seqs"] = _SEQ_LIMITS[key]+ try:+ return self._start_once(*args, options or None, trust_remote_code, model_path)+ except _MambaSeqLimit as exc:+ if cancel_event is not None and cancel_event.is_set():+ raise RuntimeError("Model load cancelled") from exc+ if exc.blocks >= options.get("max_num_seqs", exc.blocks + 1):+ raise+ # Once, at the count vLLM measured: fewer sequences only shrink its CUDA graphs.+ _SEQ_LIMITS[key] = exc.blocks+ self._tail.clear()+ self._mamba_blocks = None+ self.phase = "starting"+ return self._start_once(+ *args, {**options, "max_num_seqs": exc.blocks}, trust_remote_code, model_path+ )++ def _start_once(+ self,+ model: str,+ context: int,+ gpu_ids,+ env: dict,+ cancel_event,+ options,+ trust_remote_code,+ model_path,+ ):from utils.process_lifetime import (adopt_pid,child_popen_kwargs,@@ -446,7 +501,12 @@ class ManagedEngine:):raise RuntimeError("Model load cancelled")if not self.alive():- raise RuntimeError("Engine failed to start. " + "\n".join(self._tail))+ if self._reader is not None:+ self._reader.join(timeout = 2)+ message = "Engine failed to start. " + "\n".join(self._tail)+ if self._mamba_blocks:+ raise _MambaSeqLimit(message, self._mamba_blocks)+ raise RuntimeError(message)try:response = client.get(self.base_url + "/health", headers = self.headers)if response.status_code == 200:@@ -458,6 +518,10 @@ class ManagedEngine:raise RuntimeError("Engine startup timed out. Try a smaller model or context length.")+ except _MambaSeqLimit:+ # Not a stop request: an external stop() meanwhile must still cancel the relaunch.+ self._teardown()+ raiseexcept Exception:self.stop()raise@@ -567,6 +631,9 @@ class ManagedEngine:from utils.native_path_leases import redact_native_pathsfor line in proc.stdout:self._last_output = time.monotonic()+ if self.engine == "vllm" and (limit := _MAMBA_SEQ_LIMIT.search(line)):+ # Data-parallel ranks each report their own count; the tightest binds all.+ self._mamba_blocks = min(int(limit.group(2)), self._mamba_blocks or 1 << 30)stage = self.adapter.progress(line)if stage and self.phase != "ready":self.phase = stage@@ -577,9 +644,11 @@ class ManagedEngine:)def stop(self) -> bool:- from utils.process_lifetime import terminate_pid, forget_pid-self._cancel.set()+ return self._teardown()++ def _teardown(self) -> bool:+ from utils.process_lifetime import terminate_pid, forget_pidwith self._lock:if self.process is not None:graceful = Falsediff --git a/studio/backend/tests/test_managed_engines.py b/studio/backend/tests/test_managed_engines.pyindex 1f2a7a10a9d..2346eb0d097 100644--- a/studio/backend/tests/test_managed_engines.py+++ b/studio/backend/tests/test_managed_engines.py@@ -2362,3 +2362,175 @@ def test_legacy_load_in_4bit_is_validated_as_int4(monkeypatch):))assert out.engine_precision == "int4" and out.load_in_4bit is False and out.gpu_ids == [0]+++def test_vllm_argv_caps_max_num_seqs_only_when_asked():+ from core.inference.engine_adapters import ADAPTERS++ def argv(engine, options = None):+ return ADAPTERS[engine].command(+ "/env/bin/python", "org/model", 45678, "key", 4096, 0.8, options = options+ )++ capped = argv("vllm", {"max_num_seqs": 36})+ assert capped[capped.index("--max-num-seqs") + 1] == "36"+ assert "--max-num-seqs" not in argv("vllm")+ assert "--max-num-seqs" not in argv("vllm", {"precision": "auto"})+ assert "--max-num-seqs" not in argv("sglang", {"max_num_seqs": 36})+++# The line vLLM 0.30.0 prints for Qwen3.8-27B-NVFP4 at a 32 GB RTX 5090's budget (#8861), measured on SM 12.0.+_MAMBA_LIMIT_LINE = (+ "(EngineCore pid=34071) ValueError: max_num_seqs (256) exceeds available Mamba cache blocks (36). "+ "Each decode sequence requires one Mamba cache block, so CUDA graph capture cannot proceed. "+ "Please lower max_num_seqs to at most 36 or increase gpu_memory_utilization."+)+++def _scripted_engine(isolated, monkeypatch, outcomes):+ """A vLLM ManagedEngine whose launches follow ``outcomes`` in order: "mamba" prints the+ Mamba limit and exits, "crash" exits without it, "serve" answers /health."""+ import sys+ from types import SimpleNamespace+ from core.inference import managed_engine++ if not (isolated / "vllm" / "active.json").exists():+ active(isolated)+ monkeypatch.setattr(managed_engine, "gpu_memory_fraction", lambda *_: 0.8)+ monkeypatch.setattr(managed_engine, "_SEQ_LIMITS", {})+ engine = ManagedEngine("vllm")+ launches = []++ def command(+ python,+ model,+ port,+ key,+ context,+ memory,+ tensor_parallel_size,+ options = None,+ **_,+ ):+ launches.append(dict(options or {}))+ outcome = outcomes[len(launches) - 1]+ if outcome == "serve":+ code = (+ "from http.server import BaseHTTPRequestHandler, HTTPServer\n"+ "class Handler(BaseHTTPRequestHandler):\n"+ " def do_GET(self):\n"+ " self.send_response(200); self.end_headers()\n"+ " def log_message(self, *args): pass\n"+ f"HTTPServer(('127.0.0.1', {port}), Handler).serve_forever()\n"+ )+ else:+ lines = {+ "mamba": [_MAMBA_LIMIT_LINE],+ # Two data-parallel ranks with different free memory.+ "mamba_dp": [_MAMBA_LIMIT_LINE, _MAMBA_LIMIT_LINE.replace("(36)", "(40)")],+ }.get(outcome, ["RuntimeError: CUDA out of memory"])+ code = f"import sys\nprint({chr(10).join(lines)!r}, flush = True)\nsys.exit(1)\n"+ return [sys.executable, "-u", "-c", code]++ engine.adapter = SimpleNamespace(+ command = command,+ progress = lambda _: None,+ environment = lambda _: {},+ key_environment = lambda _: {},+ )+ return engine, launches+++@_LOCAL_ENGINE_HOST+def test_vllm_relaunches_once_at_the_mamba_block_count(isolated, monkeypatch):+ import os++ engine, launches = _scripted_engine(isolated, monkeypatch, ["mamba", "serve"])+ try:+ engine.start("model", 4096, [0], dict(os.environ), options = {"precision": "auto"})+ assert engine.phase == "ready"+ finally:+ engine.stop()+ assert [launch.get("max_num_seqs") for launch in launches] == [None, 36]+ assert launches[1]["precision"] == "auto"++ # A reload of the same config starts at the learned count instead of failing first.+ from core.inference import managed_engine++ learned = dict(managed_engine._SEQ_LIMITS)+ engine, launches = _scripted_engine(isolated, monkeypatch, ["serve"])+ monkeypatch.setattr(managed_engine, "_SEQ_LIMITS", learned)+ try:+ engine.start("model", 4096, [0], dict(os.environ), options = {"precision": "auto"})+ finally:+ engine.stop()+ assert [launch.get("max_num_seqs") for launch in launches] == [36]+++@_LOCAL_ENGINE_HOST+def test_vllm_relaunches_at_the_tightest_data_parallel_rank(isolated, monkeypatch):+ import os++ engine, launches = _scripted_engine(isolated, monkeypatch, ["mamba_dp", "serve"])+ try:+ engine.start("model", 4096, [0, 1], dict(os.environ), options = {"parallelism": "data"})+ finally:+ engine.stop()+ assert [launch.get("max_num_seqs") for launch in launches] == [None, 36]+++@_LOCAL_ENGINE_HOST[email protected]("outcomes", [["crash"], ["mamba", "mamba"]])+def test_vllm_does_not_retry_other_failures_or_twice(isolated, monkeypatch, outcomes):+ import os++ engine, launches = _scripted_engine(isolated, monkeypatch, outcomes)+ with pytest.raises(RuntimeError, match = "Engine failed to start"):+ engine.start("model", 4096, [0], dict(os.environ))+ engine.stop()+ assert len(launches) == len(outcomes)+++@_LOCAL_ENGINE_HOST+def test_vllm_mamba_retry_respects_a_cancelled_load(isolated, monkeypatch):+ import os+ import threading++ engine, launches = _scripted_engine(isolated, monkeypatch, ["mamba", "serve"])+ cancel = threading.Event()+ original = engine._start_once++ def first_then_cancel(*args):+ try:+ return original(*args)+ finally:+ cancel.set()++ monkeypatch.setattr(engine, "_start_once", first_then_cancel)+ with pytest.raises(RuntimeError, match = "cancelled"):+ engine.start("model", 4096, [0], dict(os.environ), cancel)+ engine.stop()+ assert len(launches) == 1+++@_LOCAL_ENGINE_HOST+def test_vllm_mamba_retry_respects_an_external_stop(isolated, monkeypatch):+ import os+ from core.inference import managed_engine++ engine, launches = _scripted_engine(isolated, monkeypatch, ["mamba", "serve"])+ original = engine._start_once++ def first_then_stop(*args):+ try:+ return original(*args)+ except managed_engine._MambaSeqLimit:+ # An unload or GPU handoff stops the engine without the load's cancel event.+ assert engine.stop() is True+ raise++ monkeypatch.setattr(engine, "_start_once", first_then_stop)+ with pytest.raises(RuntimeError, match = "cancelled"):+ engine.start("model", 4096, [0], dict(os.environ))+ engine.stop()+ assert len(launches) == 1