Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
Studio: install FlashInfer's prebuilt kernels into the vLLM engine env (#17063)
Daniel Hanbdebdc8331c3
diff --git a/studio/backend/core/inference/engine_install.py b/studio/backend/core/inference/engine_install.pyindex 6119260b3b0..c826eabe9a3 100644--- a/studio/backend/core/inference/engine_install.py+++ b/studio/backend/core/inference/engine_install.py@@ -34,6 +34,8 @@ PROFILES = {"driver": 580,# FlashInfer fetches trtllm kernels on demand; CUTLASS DSL loads the newest flavour driver 580 runs."omit": ("flashinfer-cubin", "nvidia-cutlass-dsl-libs-cu12"),+ # vLLM samples with FlashInfer (nvcc JIT at first launch); Studio's SGLang flags never do.+ "flashinfer_kernels": True,# Newest first; each release pins one torch build (vLLM 0.27+ needs torch 2.13, 0.20-0.26 torch 2.11)."releases": ({"version": "0.30.0", "torch": "2.13.0", "lock": "vllm-linux-cu130-torch213"},@@ -351,6 +353,8 @@ def download_bytes(engine: str) -> int | None:total = sum(size or 0 for name, size in sizes.items() if name not in provided and name not in omit)+ if not wsl and (kernels := flashinfer_kernel_wheel(engine, wait = False)):+ total += kernels.get("size") or 0summary = wsl_host.summary() if wsl else {}if wsl and not summary.get("distro"):total += wsl_host.ROOTFS["size"] + wsl_host.UV["size"]@@ -1332,6 +1336,100 @@ def _index_arguments(engine: str) -> list[str]:]+# flashinfer.ai/whl/cu130 jit-cache per locked FlashInfer; UNSLOTH_ENGINE_FLASHINFER_KERNELS=0 skips.+_FLASHINFER_KERNELS = _REQUIREMENTS / "flashinfer-kernels.json"+++def _kernels_run_on(arches: list[str], capability: str) -> bool:+ """SASS only, no PTX: `X.Ya` runs on exactly X.Y, `X.Y` and `X.Yf` on X.Y and later X minors."""+ major, minor = (int(part) for part in capability.strip().split(".")[:2])+ for arch in arches:+ built = tuple(int(part) for part in arch.rstrip("af").split("."))+ if (major, minor) == built or (+ not arch.endswith("a") and major == built[0] and minor > built[1]+ ):+ return True+ return False+++def flashinfer_kernel_wheel(engine: str, *, wait: bool = True) -> dict | None:+ """The jit-cache wheel if it has SASS for every GPU here (a missing arch fails, never JITs)."""+ if os.environ.get("UNSLOTH_ENGINE_FLASHINFER_KERNELS", "").strip().lower() in (+ "0",+ "false",+ "no",+ "off",+ ):+ return None+ wanted = profile(engine)+ if (+ not wanted.get("flashinfer_kernels")+ or sys.platform != "linux"+ or wanted["platform"] != "cuda"+ or not wanted["cuda"]+ ):+ return None+ version = _pins(engine).get("flashinfer-python", (None,))[0]+ try:+ entry = json.loads(_FLASHINFER_KERNELS.read_text(encoding = "utf-8")).get(+ f"{version}+{wanted['cuda']}", {}+ )+ wheel = entry.get(platform.machine())+ if not wheel:+ return None+ rows = _driver_rows(None, wait = wait)+ if (+ rows is _PENDING+ or not rows+ or not all(_kernels_run_on(entry["arches"], row[1]) for row in rows)+ ):+ return None+ except (OSError, ValueError, AttributeError, KeyError, IndexError, TypeError):+ return None+ return wheel+++def _install_flashinfer_kernels(+ engine: str, uv: str, python: str, destination: Path, cancel: threading.Event+) -> bool:+ """Best effort: a failure removes the wheel and keeps JIT; one that cannot be removed fails."""+ wheel = flashinfer_kernel_wheel(engine)+ if wheel is None:+ return False+ requirement = destination / "flashinfer-kernels.txt"+ requirement.write_text(+ f"flashinfer-jit-cache @ {wheel['url']} --hash=sha256:{wheel['sha256']}\n", encoding = "utf-8"+ )+ _update(engine, message = "Downloading FlashInfer's prebuilt kernels")+ try:+ _run(+ engine,+ [+ uv,+ "pip",+ "install",+ "--python",+ python,+ "--link-mode",+ package_link_mode(destination),+ "--no-deps",+ "--require-hashes",+ "--only-binary",+ ":all:",+ "-r",+ str(requirement),+ ],+ cancel,+ )+ _run(engine, [python, "-I", "-c", "import flashinfer_jit_cache, flashinfer"], cancel)+ return True+ except RuntimeError:+ if cancel.is_set():+ raise+ _run(engine, [uv, "pip", "uninstall", "--python", python, "flashinfer-jit-cache"], cancel)+ return False++def _smoke_source(engine: str) -> str:"""Imports the engine and the precision libraries its lock carries, on the locked torch build."""wanted = profile(engine)@@ -1424,6 +1522,7 @@ def _install((site / _BASE_PTH).write_text(f"import {_BASE_MODULE}\n", encoding = "utf-8")if profile(engine)["platform"] == "cuda":link_cuda_home(destination, plan["shared"])+ kernels = _install_flashinfer_kernels(engine, uv, python, destination, cancel)_update(engine, phase = "checking", message = "Checking the installed engine")_run(engine,@@ -1463,6 +1562,7 @@ def _install("platform": profile(engine)["platform"],"python": platform.python_version(),"provided": plan["provided"],+ **({"flashinfer_kernels": True} if kernels else {}),"studio_prefix": sys.prefix,"previous_directory": prior["directory"] if prior else None,"previous": {diff --git a/studio/backend/requirements/engines/flashinfer-kernels.json b/studio/backend/requirements/engines/flashinfer-kernels.jsonnew file mode 100644index 00000000000..af63edd9dff--- /dev/null+++ b/studio/backend/requirements/engines/flashinfer-kernels.json@@ -0,0 +1,18 @@+{+ "0.6.14+cu130": {+ "arches": ["7.5", "8.0", "8.9", "9.0a", "10.0a", "10.3a", "12.0f"],+ "x86_64": {+ "url": "https://github.com/flashinfer-ai/flashinfer/releases/download/v0.6.14/flashinfer_jit_cache-0.6.14+cu130-cp39-abi3-manylinux_2_28_x86_64.whl",+ "sha256": "22b666dffe4cca7de0ea67668c16c0be819e8a67cdb34c404cf89abeb3d2b510",+ "size": 1575158877+ }+ },+ "0.6.18.post1+cu130": {+ "arches": ["8.0", "8.9", "9.0a", "10.0a", "10.3a", "12.0f"],+ "x86_64": {+ "url": "https://github.com/flashinfer-ai/flashinfer/releases/download/v0.6.18.post1/flashinfer_jit_cache-0.6.18.post1+cu130-cp39-abi3-manylinux_2_28_x86_64.whl",+ "sha256": "d1729490636a98f22f158518e10c257eec5a188d3dadf30b4c9de298675d8d4b",+ "size": 1016190653+ }+ }+}diff --git a/studio/backend/tests/test_managed_engines.py b/studio/backend/tests/test_managed_engines.pyindex 881ce8b002d..432443bcf5e 100644--- a/studio/backend/tests/test_managed_engines.py+++ b/studio/backend/tests/test_managed_engines.py@@ -2910,3 +2910,223 @@ def test_vllm_drafter_retry_respects_an_external_stop(isolated, monkeypatch):engine.start("model", 4096, [0], dict(os.environ))engine.stop()assert len(launches) == 1+++def kernel_host(monkeypatch):+ rows = [["580.82.07", "8.0"]]+ monkeypatch.setattr(install.sys, "platform", "linux")+ monkeypatch.setattr(install.platform, "machine", lambda: "x86_64")+ monkeypatch.setattr(install, "_driver_rows", lambda gpu_id, wait = True: rows)+ monkeypatch.delenv("UNSLOTH_ENGINE_FLASHINFER_KERNELS", raising = False)+ return rows+++def test_every_vllm_lock_has_its_flashinfer_kernel_wheel(kernel_host, monkeypatch):+ table = json.loads(install._FLASHINFER_KERNELS.read_text(encoding = "utf-8"))+ for engine, entry in install.PROFILES.items():+ for release in entry["releases"]:+ monkeypatch.setattr(install, "_release", lambda engine, release = release: release)+ version = install._pins(engine)["flashinfer-python"][0]+ wheel = install.flashinfer_kernel_wheel(engine)+ if engine != "vllm":+ assert wheel is None, release["lock"]+ continue+ assert wheel == table[f"{version}+cu130"]["x86_64"], release["lock"]+ assert f"/flashinfer_jit_cache-{version}+cu130-" in wheel["url"]+ assert wheel["url"].endswith("_x86_64.whl")+ assert len(wheel["sha256"]) == 64 and wheel["size"] > 0+++ "capability, runs",+ [+ ("8.0", True),+ ("8.6", True),+ ("8.9", True),+ ("9.0", True),+ ("10.0", True),+ ("10.3", True),+ ("12.0", True),+ ("12.1", True),+ ("7.5", False),+ ("9.1", False),+ ("10.1", False),+ ("11.0", False),+ ("13.0", False),+ ],+)+def test_flashinfer_kernels_only_for_gpus_they_were_built_for(capability, runs):+ arches = ["8.0", "8.9", "9.0a", "10.0a", "10.3a", "12.0f"]+ assert install._kernels_run_on(arches, capability) is runs+++def test_flashinfer_kernels_skip_hosts_they_cannot_serve(kernel_host, monkeypatch):+ assert install.flashinfer_kernel_wheel("vllm") is not None+ kernel_host.append(["580.82.07", "7.5"])+ assert install.flashinfer_kernel_wheel("vllm") is None+ kernel_host[:] = [["580.82.07", "12.0"], ["580.82.07", "9.0"]]+ assert install.flashinfer_kernel_wheel("vllm") is not None+ for unreadable in (None, [], [["580.82.07"]], [["580", "n/a"]], install._PENDING):+ monkeypatch.setattr(install, "_driver_rows", lambda gpu_id, wait = True, r = unreadable: r)+ assert install.flashinfer_kernel_wheel("vllm") is None+++def test_flashinfer_kernels_only_for_linux_cuda_and_honour_the_switch(kernel_host, monkeypatch):+ for value in ("0", "false", "off"):+ monkeypatch.setenv("UNSLOTH_ENGINE_FLASHINFER_KERNELS", value)+ assert install.flashinfer_kernel_wheel("vllm") is None+ monkeypatch.setenv("UNSLOTH_ENGINE_FLASHINFER_KERNELS", "1")+ assert install.flashinfer_kernel_wheel("vllm") is not None+ monkeypatch.setattr(install.platform, "machine", lambda: "aarch64")+ assert install.flashinfer_kernel_wheel("vllm") is None+ monkeypatch.setattr(install.platform, "machine", lambda: "x86_64")+ monkeypatch.setattr(install.sys, "platform", "win32")+ assert install.flashinfer_kernel_wheel("vllm") is None+ monkeypatch.setattr(install.sys, "platform", "linux")+ monkeypatch.setattr(install, "gpu_platform", lambda: "rocm")+ assert install.flashinfer_kernel_wheel("vllm") is None+ monkeypatch.setattr(install, "gpu_platform", lambda: "cuda")+ monkeypatch.setattr(+ install, "_FLASHINFER_KERNELS", Path("/nonexistent/flashinfer-kernels.json")+ )+ assert install.flashinfer_kernel_wheel("vllm") is None+++@_LOCAL_ENGINE_HOST[email protected]("engine", ["vllm", "sglang"])+def test_install_adds_the_flashinfer_kernel_wheel_before_the_checks(+ isolated, kernel_host, monkeypatch, engine+):+ monkeypatch.setattr(install.shutil, "which", lambda _: "/uv")+ calls = []++ def run(engine, argv, cancel):+ calls.append(argv)+ if "-r" in argv:+ calls.append(Path(argv[argv.index("-r") + 1]).read_text())+ fake_venv(engine, argv, cancel)++ monkeypatch.setattr(install, "_run", run)+ install._install(engine, threading.Event())+ assert install.status(engine)["job"]["state"] == "success"+ wheel = install.flashinfer_kernel_wheel(engine)+ pip = [i for i, argv in enumerate(calls) if isinstance(argv, list) and "-r" in argv]+ if engine == "sglang":+ assert wheel is None and pip == []+ assert "flashinfer_kernels" not in install.installed(engine)+ return+ assert len(pip) == 1+ argv = calls[pip[0]]+ assert argv[:3] == ["/uv", "pip", "install"]+ assert {"--no-deps", "--require-hashes"} <= set(argv)+ assert calls[pip[0] + 1] == (+ f"flashinfer-jit-cache @ {wheel['url']} --hash=sha256:{wheel['sha256']}\n"+ )+ sync = next(i for i, a in enumerate(calls) if isinstance(a, list) and "sync" in a)+ check = next(i for i, a in enumerate(calls) if isinstance(a, list) and install._CHECK in a)+ assert sync < pip[0] < check+ assert any(+ isinstance(a, list) and "import flashinfer_jit_cache, flashinfer" in a for a in calls+ )+ assert install.installed(engine)["flashinfer_kernels"] is True+++@_LOCAL_ENGINE_HOST[email protected]("failing", ["download", "import"])+def test_failed_flashinfer_kernels_keep_the_jit_engine(isolated, kernel_host, monkeypatch, failing):+ monkeypatch.setattr(install.shutil, "which", lambda _: "/uv")+ calls = []++ def run(engine, argv, cancel):+ calls.append(argv)+ if failing == "download" and "-r" in argv:+ raise RuntimeError("Engine installation failed. github.com unreachable")+ if failing == "import" and "import flashinfer_jit_cache, flashinfer" in argv:+ raise RuntimeError("Engine installation failed. version mismatch")+ fake_venv(engine, argv, cancel)++ monkeypatch.setattr(install, "_run", run)+ install._install("vllm", threading.Event())+ assert install.status("vllm")["job"]["state"] == "success"+ assert "flashinfer_kernels" not in install.installed("vllm")+ assert any(a[:3] == ["/uv", "pip", "uninstall"] and "flashinfer-jit-cache" in a for a in calls)+ assert any(install._CHECK in a for a in calls)+++@_LOCAL_ENGINE_HOST+def test_a_kernel_wheel_that_cannot_be_removed_keeps_the_active_environment(+ isolated, kernel_host, monkeypatch+):+ marker = active(isolated)+ before = marker.read_bytes()+ monkeypatch.setattr(install.shutil, "which", lambda _: "/uv")++ def run(engine, argv, cancel):+ if "import flashinfer_jit_cache, flashinfer" in argv or argv[:3] == [+ "/uv",+ "pip",+ "uninstall",+ ]:+ raise RuntimeError("Engine installation failed.")+ fake_venv(engine, argv, cancel)++ monkeypatch.setattr(install, "_run", run)+ install._install("vllm", threading.Event())+ assert install.status("vllm")["job"]["state"] == "error"+ assert marker.read_bytes() == before+ assert [p.name for p in marker.parent.iterdir() if p.is_dir()] == ["env-prior"]+++def test_no_gpu_probe_without_a_kernel_wheel(kernel_host, monkeypatch):+ probes = []+ monkeypatch.setattr(install, "_driver_rows", lambda gpu_id, wait = True: probes.append(gpu_id))+ monkeypatch.setattr(install.platform, "machine", lambda: "aarch64")+ assert install.flashinfer_kernel_wheel("vllm") is None+ assert install.flashinfer_kernel_wheel("sglang") is None+ assert probes == []+++@_LOCAL_ENGINE_HOST+def test_cancel_during_flashinfer_kernels_cancels_the_install(isolated, kernel_host, monkeypatch):+ monkeypatch.setattr(install.shutil, "which", lambda _: "/uv")+ cancel = threading.Event()++ def run(engine, argv, event):+ if "-r" in argv:+ event.set()+ raise RuntimeError("Installation cancelled.")+ fake_venv(engine, argv, event)++ monkeypatch.setattr(install, "_run", run)+ install._install("vllm", cancel)+ assert install.status("vllm")["job"]["state"] == "cancelled"+ assert install.installed("vllm") is None+++@_LOCAL_ENGINE_HOST+def test_switched_off_flashinfer_kernels_install_as_before(isolated, kernel_host, monkeypatch):+ monkeypatch.setenv("UNSLOTH_ENGINE_FLASHINFER_KERNELS", "0")+ monkeypatch.setattr(install.shutil, "which", lambda _: "/uv")+ calls = []++ def run(engine, argv, cancel):+ calls.append(argv)+ fake_venv(engine, argv, cancel)++ monkeypatch.setattr(install, "_run", run)+ install._install("vllm", threading.Event())+ assert install.status("vllm")["job"]["state"] == "success"+ assert not any("-r" in argv or "flashinfer-jit-cache" in argv for argv in calls)+ assert "flashinfer_kernels" not in install.installed("vllm")+++@_LOCAL_ENGINE_HOST+def test_download_size_counts_the_flashinfer_kernel_wheel(isolated, kernel_host, monkeypatch):+ with_kernels = install.download_bytes("vllm")+ monkeypatch.setenv("UNSLOTH_ENGINE_FLASHINFER_KERNELS", "0")+ version = install._pins("vllm")["flashinfer-python"][0]+ table = json.loads(install._FLASHINFER_KERNELS.read_text(encoding = "utf-8"))+ assert (+ with_kernels - install.download_bytes("vllm") == table[f"{version}+cu130"]["x86_64"]["size"]+ )