Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

Studio: start vLLM on hybrid models whose Mamba state outruns max_num_seqs (#13235)

Daniel Hanf64ed0dc73d5
diff --git a/studio/backend/core/inference/engine_adapters.py b/studio/backend/core/inference/engine_adapters.py
index 1622088c237..431df6f2fc1 100644
--- a/studio/backend/core/inference/engine_adapters.py
+++ b/studio/backend/core/inference/engine_adapters.py
@@ -209,6 +209,11 @@ class EngineAdapter:
*parallel_args,
*precision_args,
*tool_args,
+ *(
+ ["--max-num-seqs", str(options["max_num_seqs"])]
+ if self.name == "vllm" and options.get("max_num_seqs")
+ else []
+ ),
*(["--trust-remote-code"] if trust_remote_code else []),
*self.extra_args,
"--host",
diff --git a/studio/backend/core/inference/managed_engine.py b/studio/backend/core/inference/managed_engine.py
index 7f32f20dc27..94aa1be89a7 100644
--- a/studio/backend/core/inference/managed_engine.py
+++ b/studio/backend/core/inference/managed_engine.py
@@ -8,6 +8,7 @@ from __future__ import annotations
import json
import hashlib
import os
+import re
import secrets
import socket
import subprocess
@@ -246,6 +247,21 @@ def _deep_gemm_unloadable(environment: str) -> bool:
STARTUP_STALL_S = 900
STARTUP_LIMIT_S = 4 * 3600
+# vLLM 0.30 on hybrid models when its default max_num_seqs outruns the KV budget (Qwen3.8 NVFP4 on 32 GB, #8861).
+_MAMBA_SEQ_LIMIT = re.compile(
+ r"max_num_seqs \((\d+)\) exceeds available Mamba cache blocks \((\d+)\)"
+)
+
+
+# Per launch config, so a reload skips the failing first launch.
+_SEQ_LIMITS: dict[tuple, int] = {}
+
+
+class _MambaSeqLimit(RuntimeError):
+ def __init__(self, message: str, blocks: int):
+ super().__init__(message)
+ self.blocks = blocks
+
def _token_file_token(env) -> str | None:
"""The `hf auth login` token a local engine would read from its inherited HF_HOME; the WSL
@@ -280,6 +296,7 @@ class ManagedEngine:
self._lease = None
self._reader = None
self._tail = deque(maxlen = 200)
+ self._mamba_blocks = None
self._last_output = time.monotonic()
self.base_url = ""
self._guest_environment = None
@@ -301,6 +318,44 @@ class ManagedEngine:
model_path = None,
):
"""``model`` is the served name; ``model_path`` is what the engine loads when they differ."""
+ args = (model, context, gpu_ids, env, cancel_event)
+ options = dict(options or {})
+ key = (
+ self.engine,
+ model_path or model,
+ context,
+ tuple(gpu_ids or ()),
+ repr(sorted(options.items())),
+ )
+ if key in _SEQ_LIMITS and "max_num_seqs" not in options:
+ options["max_num_seqs"] = _SEQ_LIMITS[key]
+ try:
+ return self._start_once(*args, options or None, trust_remote_code, model_path)
+ except _MambaSeqLimit as exc:
+ if cancel_event is not None and cancel_event.is_set():
+ raise RuntimeError("Model load cancelled") from exc
+ if exc.blocks >= options.get("max_num_seqs", exc.blocks + 1):
+ raise
+ # Once, at the count vLLM measured: fewer sequences only shrink its CUDA graphs.
+ _SEQ_LIMITS[key] = exc.blocks
+ self._tail.clear()
+ self._mamba_blocks = None
+ self.phase = "starting"
+ return self._start_once(
+ *args, {**options, "max_num_seqs": exc.blocks}, trust_remote_code, model_path
+ )
+
+ def _start_once(
+ self,
+ model: str,
+ context: int,
+ gpu_ids,
+ env: dict,
+ cancel_event,
+ options,
+ trust_remote_code,
+ model_path,
+ ):
from utils.process_lifetime import (
adopt_pid,
child_popen_kwargs,
@@ -446,7 +501,12 @@ class ManagedEngine:
):
raise RuntimeError("Model load cancelled")
if not self.alive():
- raise RuntimeError("Engine failed to start. " + "\n".join(self._tail))
+ if self._reader is not None:
+ self._reader.join(timeout = 2)
+ message = "Engine failed to start. " + "\n".join(self._tail)
+ if self._mamba_blocks:
+ raise _MambaSeqLimit(message, self._mamba_blocks)
+ raise RuntimeError(message)
try:
response = client.get(self.base_url + "/health", headers = self.headers)
if response.status_code == 200:
@@ -458,6 +518,10 @@ class ManagedEngine:
raise RuntimeError(
"Engine startup timed out. Try a smaller model or context length."
)
+ except _MambaSeqLimit:
+ # Not a stop request: an external stop() meanwhile must still cancel the relaunch.
+ self._teardown()
+ raise
except Exception:
self.stop()
raise
@@ -567,6 +631,9 @@ class ManagedEngine:
from utils.native_path_leases import redact_native_paths
for line in proc.stdout:
self._last_output = time.monotonic()
+ if self.engine == "vllm" and (limit := _MAMBA_SEQ_LIMIT.search(line)):
+ # Data-parallel ranks each report their own count; the tightest binds all.
+ self._mamba_blocks = min(int(limit.group(2)), self._mamba_blocks or 1 << 30)
stage = self.adapter.progress(line)
if stage and self.phase != "ready":
self.phase = stage
@@ -577,9 +644,11 @@ class ManagedEngine:
)
def stop(self) -> bool:
- from utils.process_lifetime import terminate_pid, forget_pid
-
self._cancel.set()
+ return self._teardown()
+
+ def _teardown(self) -> bool:
+ from utils.process_lifetime import terminate_pid, forget_pid
with self._lock:
if self.process is not None:
graceful = False
diff --git a/studio/backend/tests/test_managed_engines.py b/studio/backend/tests/test_managed_engines.py
index 1f2a7a10a9d..2346eb0d097 100644
--- a/studio/backend/tests/test_managed_engines.py
+++ b/studio/backend/tests/test_managed_engines.py
@@ -2362,3 +2362,175 @@ def test_legacy_load_in_4bit_is_validated_as_int4(monkeypatch):
)
)
assert out.engine_precision == "int4" and out.load_in_4bit is False and out.gpu_ids == [0]
+
+
+def test_vllm_argv_caps_max_num_seqs_only_when_asked():
+ from core.inference.engine_adapters import ADAPTERS
+
+ def argv(engine, options = None):
+ return ADAPTERS[engine].command(
+ "/env/bin/python", "org/model", 45678, "key", 4096, 0.8, options = options
+ )
+
+ capped = argv("vllm", {"max_num_seqs": 36})
+ assert capped[capped.index("--max-num-seqs") + 1] == "36"
+ assert "--max-num-seqs" not in argv("vllm")
+ assert "--max-num-seqs" not in argv("vllm", {"precision": "auto"})
+ assert "--max-num-seqs" not in argv("sglang", {"max_num_seqs": 36})
+
+
+# The line vLLM 0.30.0 prints for Qwen3.8-27B-NVFP4 at a 32 GB RTX 5090's budget (#8861), measured on SM 12.0.
+_MAMBA_LIMIT_LINE = (
+ "(EngineCore pid=34071) ValueError: max_num_seqs (256) exceeds available Mamba cache blocks (36). "
+ "Each decode sequence requires one Mamba cache block, so CUDA graph capture cannot proceed. "
+ "Please lower max_num_seqs to at most 36 or increase gpu_memory_utilization."
+)
+
+
+def _scripted_engine(isolated, monkeypatch, outcomes):
+ """A vLLM ManagedEngine whose launches follow ``outcomes`` in order: "mamba" prints the
+ Mamba limit and exits, "crash" exits without it, "serve" answers /health."""
+ import sys
+ from types import SimpleNamespace
+ from core.inference import managed_engine
+
+ if not (isolated / "vllm" / "active.json").exists():
+ active(isolated)
+ monkeypatch.setattr(managed_engine, "gpu_memory_fraction", lambda *_: 0.8)
+ monkeypatch.setattr(managed_engine, "_SEQ_LIMITS", {})
+ engine = ManagedEngine("vllm")
+ launches = []
+
+ def command(
+ python,
+ model,
+ port,
+ key,
+ context,
+ memory,
+ tensor_parallel_size,
+ options = None,
+ **_,
+ ):
+ launches.append(dict(options or {}))
+ outcome = outcomes[len(launches) - 1]
+ if outcome == "serve":
+ code = (
+ "from http.server import BaseHTTPRequestHandler, HTTPServer\n"
+ "class Handler(BaseHTTPRequestHandler):\n"
+ " def do_GET(self):\n"
+ " self.send_response(200); self.end_headers()\n"
+ " def log_message(self, *args): pass\n"
+ f"HTTPServer(('127.0.0.1', {port}), Handler).serve_forever()\n"
+ )
+ else:
+ lines = {
+ "mamba": [_MAMBA_LIMIT_LINE],
+ # Two data-parallel ranks with different free memory.
+ "mamba_dp": [_MAMBA_LIMIT_LINE, _MAMBA_LIMIT_LINE.replace("(36)", "(40)")],
+ }.get(outcome, ["RuntimeError: CUDA out of memory"])
+ code = f"import sys\nprint({chr(10).join(lines)!r}, flush = True)\nsys.exit(1)\n"
+ return [sys.executable, "-u", "-c", code]
+
+ engine.adapter = SimpleNamespace(
+ command = command,
+ progress = lambda _: None,
+ environment = lambda _: {},
+ key_environment = lambda _: {},
+ )
+ return engine, launches
+
+
+@_LOCAL_ENGINE_HOST
+def test_vllm_relaunches_once_at_the_mamba_block_count(isolated, monkeypatch):
+ import os
+
+ engine, launches = _scripted_engine(isolated, monkeypatch, ["mamba", "serve"])
+ try:
+ engine.start("model", 4096, [0], dict(os.environ), options = {"precision": "auto"})
+ assert engine.phase == "ready"
+ finally:
+ engine.stop()
+ assert [launch.get("max_num_seqs") for launch in launches] == [None, 36]
+ assert launches[1]["precision"] == "auto"
+
+ # A reload of the same config starts at the learned count instead of failing first.
+ from core.inference import managed_engine
+
+ learned = dict(managed_engine._SEQ_LIMITS)
+ engine, launches = _scripted_engine(isolated, monkeypatch, ["serve"])
+ monkeypatch.setattr(managed_engine, "_SEQ_LIMITS", learned)
+ try:
+ engine.start("model", 4096, [0], dict(os.environ), options = {"precision": "auto"})
+ finally:
+ engine.stop()
+ assert [launch.get("max_num_seqs") for launch in launches] == [36]
+
+
+@_LOCAL_ENGINE_HOST
+def test_vllm_relaunches_at_the_tightest_data_parallel_rank(isolated, monkeypatch):
+ import os
+
+ engine, launches = _scripted_engine(isolated, monkeypatch, ["mamba_dp", "serve"])
+ try:
+ engine.start("model", 4096, [0, 1], dict(os.environ), options = {"parallelism": "data"})
+ finally:
+ engine.stop()
+ assert [launch.get("max_num_seqs") for launch in launches] == [None, 36]
+
+
+@_LOCAL_ENGINE_HOST
[email protected]("outcomes", [["crash"], ["mamba", "mamba"]])
+def test_vllm_does_not_retry_other_failures_or_twice(isolated, monkeypatch, outcomes):
+ import os
+
+ engine, launches = _scripted_engine(isolated, monkeypatch, outcomes)
+ with pytest.raises(RuntimeError, match = "Engine failed to start"):
+ engine.start("model", 4096, [0], dict(os.environ))
+ engine.stop()
+ assert len(launches) == len(outcomes)
+
+
+@_LOCAL_ENGINE_HOST
+def test_vllm_mamba_retry_respects_a_cancelled_load(isolated, monkeypatch):
+ import os
+ import threading
+
+ engine, launches = _scripted_engine(isolated, monkeypatch, ["mamba", "serve"])
+ cancel = threading.Event()
+ original = engine._start_once
+
+ def first_then_cancel(*args):
+ try:
+ return original(*args)
+ finally:
+ cancel.set()
+
+ monkeypatch.setattr(engine, "_start_once", first_then_cancel)
+ with pytest.raises(RuntimeError, match = "cancelled"):
+ engine.start("model", 4096, [0], dict(os.environ), cancel)
+ engine.stop()
+ assert len(launches) == 1
+
+
+@_LOCAL_ENGINE_HOST
+def test_vllm_mamba_retry_respects_an_external_stop(isolated, monkeypatch):
+ import os
+ from core.inference import managed_engine
+
+ engine, launches = _scripted_engine(isolated, monkeypatch, ["mamba", "serve"])
+ original = engine._start_once
+
+ def first_then_stop(*args):
+ try:
+ return original(*args)
+ except managed_engine._MambaSeqLimit:
+ # An unload or GPU handoff stops the engine without the load's cancel event.
+ assert engine.stop() is True
+ raise
+
+ monkeypatch.setattr(engine, "_start_once", first_then_stop)
+ with pytest.raises(RuntimeError, match = "cancelled"):
+ engine.start("model", 4096, [0], dict(os.environ))
+ engine.stop()
+ assert len(launches) == 1