Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

Studio: install FlashInfer's prebuilt kernels into the vLLM engine env (#17063)

Daniel Hanbdebdc8331c3
diff --git a/studio/backend/core/inference/engine_install.py b/studio/backend/core/inference/engine_install.py
index 6119260b3b0..c826eabe9a3 100644
--- a/studio/backend/core/inference/engine_install.py
+++ b/studio/backend/core/inference/engine_install.py
@@ -34,6 +34,8 @@ PROFILES = {
"driver": 580,
# FlashInfer fetches trtllm kernels on demand; CUTLASS DSL loads the newest flavour driver 580 runs.
"omit": ("flashinfer-cubin", "nvidia-cutlass-dsl-libs-cu12"),
+ # vLLM samples with FlashInfer (nvcc JIT at first launch); Studio's SGLang flags never do.
+ "flashinfer_kernels": True,
# Newest first; each release pins one torch build (vLLM 0.27+ needs torch 2.13, 0.20-0.26 torch 2.11).
"releases": (
{"version": "0.30.0", "torch": "2.13.0", "lock": "vllm-linux-cu130-torch213"},
@@ -351,6 +353,8 @@ def download_bytes(engine: str) -> int | None:
total = sum(
size or 0 for name, size in sizes.items() if name not in provided and name not in omit
)
+ if not wsl and (kernels := flashinfer_kernel_wheel(engine, wait = False)):
+ total += kernels.get("size") or 0
summary = wsl_host.summary() if wsl else {}
if wsl and not summary.get("distro"):
total += wsl_host.ROOTFS["size"] + wsl_host.UV["size"]
@@ -1332,6 +1336,100 @@ def _index_arguments(engine: str) -> list[str]:
]
+# flashinfer.ai/whl/cu130 jit-cache per locked FlashInfer; UNSLOTH_ENGINE_FLASHINFER_KERNELS=0 skips.
+_FLASHINFER_KERNELS = _REQUIREMENTS / "flashinfer-kernels.json"
+
+
+def _kernels_run_on(arches: list[str], capability: str) -> bool:
+ """SASS only, no PTX: `X.Ya` runs on exactly X.Y, `X.Y` and `X.Yf` on X.Y and later X minors."""
+ major, minor = (int(part) for part in capability.strip().split(".")[:2])
+ for arch in arches:
+ built = tuple(int(part) for part in arch.rstrip("af").split("."))
+ if (major, minor) == built or (
+ not arch.endswith("a") and major == built[0] and minor > built[1]
+ ):
+ return True
+ return False
+
+
+def flashinfer_kernel_wheel(engine: str, *, wait: bool = True) -> dict | None:
+ """The jit-cache wheel if it has SASS for every GPU here (a missing arch fails, never JITs)."""
+ if os.environ.get("UNSLOTH_ENGINE_FLASHINFER_KERNELS", "").strip().lower() in (
+ "0",
+ "false",
+ "no",
+ "off",
+ ):
+ return None
+ wanted = profile(engine)
+ if (
+ not wanted.get("flashinfer_kernels")
+ or sys.platform != "linux"
+ or wanted["platform"] != "cuda"
+ or not wanted["cuda"]
+ ):
+ return None
+ version = _pins(engine).get("flashinfer-python", (None,))[0]
+ try:
+ entry = json.loads(_FLASHINFER_KERNELS.read_text(encoding = "utf-8")).get(
+ f"{version}+{wanted['cuda']}", {}
+ )
+ wheel = entry.get(platform.machine())
+ if not wheel:
+ return None
+ rows = _driver_rows(None, wait = wait)
+ if (
+ rows is _PENDING
+ or not rows
+ or not all(_kernels_run_on(entry["arches"], row[1]) for row in rows)
+ ):
+ return None
+ except (OSError, ValueError, AttributeError, KeyError, IndexError, TypeError):
+ return None
+ return wheel
+
+
+def _install_flashinfer_kernels(
+ engine: str, uv: str, python: str, destination: Path, cancel: threading.Event
+) -> bool:
+ """Best effort: a failure removes the wheel and keeps JIT; one that cannot be removed fails."""
+ wheel = flashinfer_kernel_wheel(engine)
+ if wheel is None:
+ return False
+ requirement = destination / "flashinfer-kernels.txt"
+ requirement.write_text(
+ f"flashinfer-jit-cache @ {wheel['url']} --hash=sha256:{wheel['sha256']}\n", encoding = "utf-8"
+ )
+ _update(engine, message = "Downloading FlashInfer's prebuilt kernels")
+ try:
+ _run(
+ engine,
+ [
+ uv,
+ "pip",
+ "install",
+ "--python",
+ python,
+ "--link-mode",
+ package_link_mode(destination),
+ "--no-deps",
+ "--require-hashes",
+ "--only-binary",
+ ":all:",
+ "-r",
+ str(requirement),
+ ],
+ cancel,
+ )
+ _run(engine, [python, "-I", "-c", "import flashinfer_jit_cache, flashinfer"], cancel)
+ return True
+ except RuntimeError:
+ if cancel.is_set():
+ raise
+ _run(engine, [uv, "pip", "uninstall", "--python", python, "flashinfer-jit-cache"], cancel)
+ return False
+
+
def _smoke_source(engine: str) -> str:
"""Imports the engine and the precision libraries its lock carries, on the locked torch build."""
wanted = profile(engine)
@@ -1424,6 +1522,7 @@ def _install(
(site / _BASE_PTH).write_text(f"import {_BASE_MODULE}\n", encoding = "utf-8")
if profile(engine)["platform"] == "cuda":
link_cuda_home(destination, plan["shared"])
+ kernels = _install_flashinfer_kernels(engine, uv, python, destination, cancel)
_update(engine, phase = "checking", message = "Checking the installed engine")
_run(
engine,
@@ -1463,6 +1562,7 @@ def _install(
"platform": profile(engine)["platform"],
"python": platform.python_version(),
"provided": plan["provided"],
+ **({"flashinfer_kernels": True} if kernels else {}),
"studio_prefix": sys.prefix,
"previous_directory": prior["directory"] if prior else None,
"previous": {
diff --git a/studio/backend/requirements/engines/flashinfer-kernels.json b/studio/backend/requirements/engines/flashinfer-kernels.json
new file mode 100644
index 00000000000..af63edd9dff
--- /dev/null
+++ b/studio/backend/requirements/engines/flashinfer-kernels.json
@@ -0,0 +1,18 @@
+{
+ "0.6.14+cu130": {
+ "arches": ["7.5", "8.0", "8.9", "9.0a", "10.0a", "10.3a", "12.0f"],
+ "x86_64": {
+ "url": "https://github.com/flashinfer-ai/flashinfer/releases/download/v0.6.14/flashinfer_jit_cache-0.6.14+cu130-cp39-abi3-manylinux_2_28_x86_64.whl",
+ "sha256": "22b666dffe4cca7de0ea67668c16c0be819e8a67cdb34c404cf89abeb3d2b510",
+ "size": 1575158877
+ }
+ },
+ "0.6.18.post1+cu130": {
+ "arches": ["8.0", "8.9", "9.0a", "10.0a", "10.3a", "12.0f"],
+ "x86_64": {
+ "url": "https://github.com/flashinfer-ai/flashinfer/releases/download/v0.6.18.post1/flashinfer_jit_cache-0.6.18.post1+cu130-cp39-abi3-manylinux_2_28_x86_64.whl",
+ "sha256": "d1729490636a98f22f158518e10c257eec5a188d3dadf30b4c9de298675d8d4b",
+ "size": 1016190653
+ }
+ }
+}
diff --git a/studio/backend/tests/test_managed_engines.py b/studio/backend/tests/test_managed_engines.py
index 881ce8b002d..432443bcf5e 100644
--- a/studio/backend/tests/test_managed_engines.py
+++ b/studio/backend/tests/test_managed_engines.py
@@ -2910,3 +2910,223 @@ def test_vllm_drafter_retry_respects_an_external_stop(isolated, monkeypatch):
engine.start("model", 4096, [0], dict(os.environ))
engine.stop()
assert len(launches) == 1
+
+
+def kernel_host(monkeypatch):
+ rows = [["580.82.07", "8.0"]]
+ monkeypatch.setattr(install.sys, "platform", "linux")
+ monkeypatch.setattr(install.platform, "machine", lambda: "x86_64")
+ monkeypatch.setattr(install, "_driver_rows", lambda gpu_id, wait = True: rows)
+ monkeypatch.delenv("UNSLOTH_ENGINE_FLASHINFER_KERNELS", raising = False)
+ return rows
+
+
+def test_every_vllm_lock_has_its_flashinfer_kernel_wheel(kernel_host, monkeypatch):
+ table = json.loads(install._FLASHINFER_KERNELS.read_text(encoding = "utf-8"))
+ for engine, entry in install.PROFILES.items():
+ for release in entry["releases"]:
+ monkeypatch.setattr(install, "_release", lambda engine, release = release: release)
+ version = install._pins(engine)["flashinfer-python"][0]
+ wheel = install.flashinfer_kernel_wheel(engine)
+ if engine != "vllm":
+ assert wheel is None, release["lock"]
+ continue
+ assert wheel == table[f"{version}+cu130"]["x86_64"], release["lock"]
+ assert f"/flashinfer_jit_cache-{version}+cu130-" in wheel["url"]
+ assert wheel["url"].endswith("_x86_64.whl")
+ assert len(wheel["sha256"]) == 64 and wheel["size"] > 0
+
+
+ "capability, runs",
+ [
+ ("8.0", True),
+ ("8.6", True),
+ ("8.9", True),
+ ("9.0", True),
+ ("10.0", True),
+ ("10.3", True),
+ ("12.0", True),
+ ("12.1", True),
+ ("7.5", False),
+ ("9.1", False),
+ ("10.1", False),
+ ("11.0", False),
+ ("13.0", False),
+ ],
+)
+def test_flashinfer_kernels_only_for_gpus_they_were_built_for(capability, runs):
+ arches = ["8.0", "8.9", "9.0a", "10.0a", "10.3a", "12.0f"]
+ assert install._kernels_run_on(arches, capability) is runs
+
+
+def test_flashinfer_kernels_skip_hosts_they_cannot_serve(kernel_host, monkeypatch):
+ assert install.flashinfer_kernel_wheel("vllm") is not None
+ kernel_host.append(["580.82.07", "7.5"])
+ assert install.flashinfer_kernel_wheel("vllm") is None
+ kernel_host[:] = [["580.82.07", "12.0"], ["580.82.07", "9.0"]]
+ assert install.flashinfer_kernel_wheel("vllm") is not None
+ for unreadable in (None, [], [["580.82.07"]], [["580", "n/a"]], install._PENDING):
+ monkeypatch.setattr(install, "_driver_rows", lambda gpu_id, wait = True, r = unreadable: r)
+ assert install.flashinfer_kernel_wheel("vllm") is None
+
+
+def test_flashinfer_kernels_only_for_linux_cuda_and_honour_the_switch(kernel_host, monkeypatch):
+ for value in ("0", "false", "off"):
+ monkeypatch.setenv("UNSLOTH_ENGINE_FLASHINFER_KERNELS", value)
+ assert install.flashinfer_kernel_wheel("vllm") is None
+ monkeypatch.setenv("UNSLOTH_ENGINE_FLASHINFER_KERNELS", "1")
+ assert install.flashinfer_kernel_wheel("vllm") is not None
+ monkeypatch.setattr(install.platform, "machine", lambda: "aarch64")
+ assert install.flashinfer_kernel_wheel("vllm") is None
+ monkeypatch.setattr(install.platform, "machine", lambda: "x86_64")
+ monkeypatch.setattr(install.sys, "platform", "win32")
+ assert install.flashinfer_kernel_wheel("vllm") is None
+ monkeypatch.setattr(install.sys, "platform", "linux")
+ monkeypatch.setattr(install, "gpu_platform", lambda: "rocm")
+ assert install.flashinfer_kernel_wheel("vllm") is None
+ monkeypatch.setattr(install, "gpu_platform", lambda: "cuda")
+ monkeypatch.setattr(
+ install, "_FLASHINFER_KERNELS", Path("/nonexistent/flashinfer-kernels.json")
+ )
+ assert install.flashinfer_kernel_wheel("vllm") is None
+
+
+@_LOCAL_ENGINE_HOST
[email protected]("engine", ["vllm", "sglang"])
+def test_install_adds_the_flashinfer_kernel_wheel_before_the_checks(
+ isolated, kernel_host, monkeypatch, engine
+):
+ monkeypatch.setattr(install.shutil, "which", lambda _: "/uv")
+ calls = []
+
+ def run(engine, argv, cancel):
+ calls.append(argv)
+ if "-r" in argv:
+ calls.append(Path(argv[argv.index("-r") + 1]).read_text())
+ fake_venv(engine, argv, cancel)
+
+ monkeypatch.setattr(install, "_run", run)
+ install._install(engine, threading.Event())
+ assert install.status(engine)["job"]["state"] == "success"
+ wheel = install.flashinfer_kernel_wheel(engine)
+ pip = [i for i, argv in enumerate(calls) if isinstance(argv, list) and "-r" in argv]
+ if engine == "sglang":
+ assert wheel is None and pip == []
+ assert "flashinfer_kernels" not in install.installed(engine)
+ return
+ assert len(pip) == 1
+ argv = calls[pip[0]]
+ assert argv[:3] == ["/uv", "pip", "install"]
+ assert {"--no-deps", "--require-hashes"} <= set(argv)
+ assert calls[pip[0] + 1] == (
+ f"flashinfer-jit-cache @ {wheel['url']} --hash=sha256:{wheel['sha256']}\n"
+ )
+ sync = next(i for i, a in enumerate(calls) if isinstance(a, list) and "sync" in a)
+ check = next(i for i, a in enumerate(calls) if isinstance(a, list) and install._CHECK in a)
+ assert sync < pip[0] < check
+ assert any(
+ isinstance(a, list) and "import flashinfer_jit_cache, flashinfer" in a for a in calls
+ )
+ assert install.installed(engine)["flashinfer_kernels"] is True
+
+
+@_LOCAL_ENGINE_HOST
[email protected]("failing", ["download", "import"])
+def test_failed_flashinfer_kernels_keep_the_jit_engine(isolated, kernel_host, monkeypatch, failing):
+ monkeypatch.setattr(install.shutil, "which", lambda _: "/uv")
+ calls = []
+
+ def run(engine, argv, cancel):
+ calls.append(argv)
+ if failing == "download" and "-r" in argv:
+ raise RuntimeError("Engine installation failed. github.com unreachable")
+ if failing == "import" and "import flashinfer_jit_cache, flashinfer" in argv:
+ raise RuntimeError("Engine installation failed. version mismatch")
+ fake_venv(engine, argv, cancel)
+
+ monkeypatch.setattr(install, "_run", run)
+ install._install("vllm", threading.Event())
+ assert install.status("vllm")["job"]["state"] == "success"
+ assert "flashinfer_kernels" not in install.installed("vllm")
+ assert any(a[:3] == ["/uv", "pip", "uninstall"] and "flashinfer-jit-cache" in a for a in calls)
+ assert any(install._CHECK in a for a in calls)
+
+
+@_LOCAL_ENGINE_HOST
+def test_a_kernel_wheel_that_cannot_be_removed_keeps_the_active_environment(
+ isolated, kernel_host, monkeypatch
+):
+ marker = active(isolated)
+ before = marker.read_bytes()
+ monkeypatch.setattr(install.shutil, "which", lambda _: "/uv")
+
+ def run(engine, argv, cancel):
+ if "import flashinfer_jit_cache, flashinfer" in argv or argv[:3] == [
+ "/uv",
+ "pip",
+ "uninstall",
+ ]:
+ raise RuntimeError("Engine installation failed.")
+ fake_venv(engine, argv, cancel)
+
+ monkeypatch.setattr(install, "_run", run)
+ install._install("vllm", threading.Event())
+ assert install.status("vllm")["job"]["state"] == "error"
+ assert marker.read_bytes() == before
+ assert [p.name for p in marker.parent.iterdir() if p.is_dir()] == ["env-prior"]
+
+
+def test_no_gpu_probe_without_a_kernel_wheel(kernel_host, monkeypatch):
+ probes = []
+ monkeypatch.setattr(install, "_driver_rows", lambda gpu_id, wait = True: probes.append(gpu_id))
+ monkeypatch.setattr(install.platform, "machine", lambda: "aarch64")
+ assert install.flashinfer_kernel_wheel("vllm") is None
+ assert install.flashinfer_kernel_wheel("sglang") is None
+ assert probes == []
+
+
+@_LOCAL_ENGINE_HOST
+def test_cancel_during_flashinfer_kernels_cancels_the_install(isolated, kernel_host, monkeypatch):
+ monkeypatch.setattr(install.shutil, "which", lambda _: "/uv")
+ cancel = threading.Event()
+
+ def run(engine, argv, event):
+ if "-r" in argv:
+ event.set()
+ raise RuntimeError("Installation cancelled.")
+ fake_venv(engine, argv, event)
+
+ monkeypatch.setattr(install, "_run", run)
+ install._install("vllm", cancel)
+ assert install.status("vllm")["job"]["state"] == "cancelled"
+ assert install.installed("vllm") is None
+
+
+@_LOCAL_ENGINE_HOST
+def test_switched_off_flashinfer_kernels_install_as_before(isolated, kernel_host, monkeypatch):
+ monkeypatch.setenv("UNSLOTH_ENGINE_FLASHINFER_KERNELS", "0")
+ monkeypatch.setattr(install.shutil, "which", lambda _: "/uv")
+ calls = []
+
+ def run(engine, argv, cancel):
+ calls.append(argv)
+ fake_venv(engine, argv, cancel)
+
+ monkeypatch.setattr(install, "_run", run)
+ install._install("vllm", threading.Event())
+ assert install.status("vllm")["job"]["state"] == "success"
+ assert not any("-r" in argv or "flashinfer-jit-cache" in argv for argv in calls)
+ assert "flashinfer_kernels" not in install.installed("vllm")
+
+
+@_LOCAL_ENGINE_HOST
+def test_download_size_counts_the_flashinfer_kernel_wheel(isolated, kernel_host, monkeypatch):
+ with_kernels = install.download_bytes("vllm")
+ monkeypatch.setenv("UNSLOTH_ENGINE_FLASHINFER_KERNELS", "0")
+ version = install._pins("vllm")["flashinfer-python"][0]
+ table = json.loads(install._FLASHINFER_KERNELS.read_text(encoding = "utf-8"))
+ assert (
+ with_kernels - install.download_bytes("vllm") == table[f"{version}+cu130"]["x86_64"]["size"]
+ )