FreeToken brings datacenter-scale model serving to your desktop. Run massive models locally, fast and efficiently.
fix(rocm): gate CUDA-only optional backends on ROCm (#134)
ZihaoMu9b585b72d083
diff --git a/docs/install_amd.md b/docs/install_amd.mdindex d8b0c1bc..81d1ad50 100644--- a/docs/install_amd.md+++ b/docs/install_amd.md@@ -37,3 +37,11 @@ python -m pip install --no-build-isolation -e .Set both architecture variables to `gfx1200` for RX 9060 family GPUs, or to theactual target reported by `rocminfo`.++## Optional kernel backends++FreeToken's FlashInfer, `sgl_kernel`, and vLLM kernel integrations are CUDA-only.+On ROCm, these paths use the built-in fallbacks; do not install the `[accel]`,+`[fi]`, or `[sgl]` extras. Forcing `--attention-backend fi`, `fa`, or `trtllm`,+or NVFP4 Marlin/b12x (e.g. `--quant-backend moe.nvfp4=marlin` or+`--quant-backend moe.nvfp4=b12x`), fails with a ROCm-specific error.diff --git a/python/freetoken/engine/engine.py b/python/freetoken/engine/engine.pyindex bf3960ab..3988caf6 100644--- a/python/freetoken/engine/engine.py+++ b/python/freetoken/engine/engine.py@@ -58,6 +58,10 @@ def _flashinfer_available() -> bool:def _sgl_flash_attn_available() -> bool:+ from freetoken.kernel.backend import is_rocm++ if is_rocm():+ return Falsetry:from sgl_kernel.flash_attn import flash_attn_with_kvcache # noqa: F401except Exception as exc:@@ -167,6 +171,7 @@ def _validate_attention_backend_choice(config, override, required: frozenset[Attwhere a DSV4 or MLA checkpoint rejects a generic backend before weights load,and where a generic model rejects dsa/dsv4_sparse."""from freetoken.attention import validate_attn_backend+ from freetoken.kernel.backend import is_rocm# Name membership first (ArgumentTypeError listing the supported names): the CLI already# ran this, but the programmatic EngineConfig path reaches here unvalidated and would@@ -207,12 +212,22 @@ def _validate_attention_backend_choice(config, override, required: frozenset[Attfor part in backend_parts:info = attention_backend_info(part)if info.requires_flashinfer and not _flashinfer_available():+ if is_rocm():+ raise RuntimeError(+ f"Attention backend {part!r} uses CUDA-only FlashInfer kernels "+ "unavailable on ROCm; use --attention-backend auto or triton."+ )raise RuntimeError(f"Attention backend {config.attention_backend!r} requires flashinfer, which is ""not installed. Install it with `pip install 'freetoken[fi]'` (or ""'freetoken[accel]'), or use --attention-backend triton.")if info.requires_sgl_kernel and not _sgl_flash_attn_available():+ if is_rocm():+ raise RuntimeError(+ f"Attention backend {part!r} uses CUDA-only sgl_kernel kernels "+ "unavailable on ROCm; use --attention-backend auto or triton."+ )raise RuntimeError(f"Attention backend {config.attention_backend!r} requires sgl_kernel, which is ""not installed. Install it with `pip install 'freetoken[sgl]'` (or "diff --git a/python/freetoken/kernel/backend.py b/python/freetoken/kernel/backend.pyindex 6d1b4a9f..73a7dc6f 100644--- a/python/freetoken/kernel/backend.py+++ b/python/freetoken/kernel/backend.py@@ -4,6 +4,7 @@ When flashinfer / sgl_kernel are installed the call-sites use their fused CUDAops; otherwise they fall back to the pure-Triton kernels in``freetoken.kernel.triton``. ``find_spec`` only checks that the package isimportable (no import side effects), and the result is cached.+On ROCm, the optional-package probes return False so callers use the fallbacks."""from __future__ import annotations@@ -25,16 +26,22 @@ def _importable(name: str) -> bool:@functools.cachedef is_flashinfer_installed() -> bool:+ if is_rocm():+ return Falsereturn _importable("flashinfer")@functools.cachedef is_sgl_kernel_installed() -> bool:+ if is_rocm():+ return Falsereturn _importable("sgl_kernel")@functools.cachedef is_vllm_installed() -> bool:+ if is_rocm():+ return Falsereturn _importable("vllm")@@ -54,17 +61,6 @@ def is_rocm() -> bool:return getattr(torch.version, "hip", None) is not None-def driver_hip_version() -> int | None:- """ROCm driver version, or None if undetermined."""- # TODO(ROCm): flashinfer/sgl_kernel have no ROCm builds — Triton fallback is used.- try:- from freetoken.kernel.pinned import _load_pinned_extension- return int(_load_pinned_extension().driver_cuda_version()) or None- except Exception:- return None--@functools.cachedef driver_cuda_version() -> int | None:"""Max CUDA version the installed NVIDIA driver supports (``13000`` == CUDA 13.0),@@ -73,6 +69,8 @@ def driver_cuda_version() -> int | None:toolkit version. Resolved through the ``_pinned_tensor`` extension's link-timecudart, so it works wherever the extension builds (including Windows) -- no dlopenby soname."""+ if is_rocm():+ return Nonetry:from freetoken.kernel.pinned import _load_pinned_extensiondiff --git a/python/freetoken/layers/quantization/linear/nvfp4.py b/python/freetoken/layers/quantization/linear/nvfp4.pyindex 0953b825..80f5b78d 100644--- a/python/freetoken/layers/quantization/linear/nvfp4.py+++ b/python/freetoken/layers/quantization/linear/nvfp4.py@@ -38,6 +38,8 @@ class MarlinNvfp4LinearKernel(LinearKernel):name = "marlin"def unusable_reason(self, cfg: LinearConfig) -> str | None:+ if backend.is_rocm():+ return "Marlin is CUDA-only and unavailable on ROCm; use triton"if not backend.is_vllm_installed():return "vLLM is not installed"if len(cfg.output_sizes) > 1:diff --git a/python/freetoken/layers/quantization/moe/nvfp4.py b/python/freetoken/layers/quantization/moe/nvfp4.pyindex cf66e207..1cd1a1ac 100644--- a/python/freetoken/layers/quantization/moe/nvfp4.py+++ b/python/freetoken/layers/quantization/moe/nvfp4.py@@ -228,6 +228,8 @@ class MarlinNvfp4MoEKernel(MoEKernel):max_slots = MARLIN_MAX_SLOTSdef unusable_reason(self, cfg: MoEConfig) -> str | None:+ if backend.is_rocm():+ return "Marlin is CUDA-only and unavailable on ROCm; use triton"if not backend.is_vllm_installed():return "vLLM is not installed"reason = self._common_reject(cfg, tp_ok=False, cpu_ok=False, plain_silu_only=True)@@ -491,6 +493,8 @@ class B12xNvfp4MoEKernel(MoEKernel):name = "b12x"def unusable_reason(self, cfg: MoEConfig) -> str | None:+ if backend.is_rocm():+ return "b12x is CUDA-only and unavailable on ROCm; use triton"cc = backend.device_capability()if cc < (12, 0):return f"b12x requires sm_120+, got sm_{cc[0]}{cc[1]}"diff --git a/tests/engine/test_attention_backend_arch.py b/tests/engine/test_attention_backend_arch.pyindex 9271bd42..25eb9354 100644--- a/tests/engine/test_attention_backend_arch.py+++ b/tests/engine/test_attention_backend_arch.py@@ -116,3 +116,41 @@ def test_fa_backend_rejects_cc12(monkeypatch):monkeypatch.setattr(arch, "_get_torch_cuda_version", lambda: (12, 0))with pytest.raises(RuntimeError, match="12.x"):FlashAttentionBackend(SimpleNamespace(head_dim=128))++[email protected]("requested", ["auto", "fi", "fa", "trtllm", "triton,fi"])+def test_rocm_attention_selection_avoids_cuda_packages(monkeypatch, requested):+ import builtins++ from freetoken.engine import engine+ from freetoken.kernel import backend++ real_import = builtins.__import__++ def guarded_import(name, *args, **kwargs):+ if name.split(".")[0] in {"flashinfer", "sgl_kernel"}:+ pytest.fail(f"ROCm tried to import {name}")+ return real_import(name, *args, **kwargs)++ def unexpected_probe(_name):+ pytest.fail("ROCm tried to discover a CUDA package")++ monkeypatch.setattr(builtins, "__import__", guarded_import)+ monkeypatch.setattr(backend, "is_rocm", lambda: True)+ monkeypatch.setattr(backend, "_importable", unexpected_probe)+ monkeypatch.setattr(engine, "is_sm90_family", lambda: False)+ monkeypatch.setattr(engine, "is_sm100_family", lambda: False)+ backend.is_flashinfer_installed.cache_clear()+ backend.is_sgl_kernel_installed.cache_clear()+ try:+ assert not engine._sgl_flash_attn_available()+ config = _engine_config(attention_backend=requested)+ if requested == "auto":+ engine._adjust_config(config)+ assert config.attention_backend == "triton"+ else:+ with pytest.raises(RuntimeError, match="CUDA-only.*ROCm"):+ engine._adjust_config(config)+ finally:+ backend.is_flashinfer_installed.cache_clear()+ backend.is_sgl_kernel_installed.cache_clear()