Skip to content

FreeToken brings datacenter-scale model serving to your desktop. Run massive models locally, fast and efficiently.

fix(rocm): gate CUDA-only optional backends on ROCm (#134)

ZihaoMu9b585b72d083
diff --git a/docs/install_amd.md b/docs/install_amd.md
index d8b0c1bc..81d1ad50 100644
--- a/docs/install_amd.md
+++ b/docs/install_amd.md
@@ -37,3 +37,11 @@ python -m pip install --no-build-isolation -e .
Set both architecture variables to `gfx1200` for RX 9060 family GPUs, or to the
actual target reported by `rocminfo`.
+
+## Optional kernel backends
+
+FreeToken's FlashInfer, `sgl_kernel`, and vLLM kernel integrations are CUDA-only.
+On ROCm, these paths use the built-in fallbacks; do not install the `[accel]`,
+`[fi]`, or `[sgl]` extras. Forcing `--attention-backend fi`, `fa`, or `trtllm`,
+or NVFP4 Marlin/b12x (e.g. `--quant-backend moe.nvfp4=marlin` or
+`--quant-backend moe.nvfp4=b12x`), fails with a ROCm-specific error.
diff --git a/python/freetoken/engine/engine.py b/python/freetoken/engine/engine.py
index bf3960ab..3988caf6 100644
--- a/python/freetoken/engine/engine.py
+++ b/python/freetoken/engine/engine.py
@@ -58,6 +58,10 @@ def _flashinfer_available() -> bool:
def _sgl_flash_attn_available() -> bool:
+ from freetoken.kernel.backend import is_rocm
+
+ if is_rocm():
+ return False
try:
from sgl_kernel.flash_attn import flash_attn_with_kvcache # noqa: F401
except Exception as exc:
@@ -167,6 +171,7 @@ def _validate_attention_backend_choice(config, override, required: frozenset[Att
where a DSV4 or MLA checkpoint rejects a generic backend before weights load,
and where a generic model rejects dsa/dsv4_sparse."""
from freetoken.attention import validate_attn_backend
+ from freetoken.kernel.backend import is_rocm
# Name membership first (ArgumentTypeError listing the supported names): the CLI already
# ran this, but the programmatic EngineConfig path reaches here unvalidated and would
@@ -207,12 +212,22 @@ def _validate_attention_backend_choice(config, override, required: frozenset[Att
for part in backend_parts:
info = attention_backend_info(part)
if info.requires_flashinfer and not _flashinfer_available():
+ if is_rocm():
+ raise RuntimeError(
+ f"Attention backend {part!r} uses CUDA-only FlashInfer kernels "
+ "unavailable on ROCm; use --attention-backend auto or triton."
+ )
raise RuntimeError(
f"Attention backend {config.attention_backend!r} requires flashinfer, which is "
"not installed. Install it with `pip install 'freetoken[fi]'` (or "
"'freetoken[accel]'), or use --attention-backend triton."
)
if info.requires_sgl_kernel and not _sgl_flash_attn_available():
+ if is_rocm():
+ raise RuntimeError(
+ f"Attention backend {part!r} uses CUDA-only sgl_kernel kernels "
+ "unavailable on ROCm; use --attention-backend auto or triton."
+ )
raise RuntimeError(
f"Attention backend {config.attention_backend!r} requires sgl_kernel, which is "
"not installed. Install it with `pip install 'freetoken[sgl]'` (or "
diff --git a/python/freetoken/kernel/backend.py b/python/freetoken/kernel/backend.py
index 6d1b4a9f..73a7dc6f 100644
--- a/python/freetoken/kernel/backend.py
+++ b/python/freetoken/kernel/backend.py
@@ -4,6 +4,7 @@ When flashinfer / sgl_kernel are installed the call-sites use their fused CUDA
ops; otherwise they fall back to the pure-Triton kernels in
``freetoken.kernel.triton``. ``find_spec`` only checks that the package is
importable (no import side effects), and the result is cached.
+On ROCm, the optional-package probes return False so callers use the fallbacks.
"""
from __future__ import annotations
@@ -25,16 +26,22 @@ def _importable(name: str) -> bool:
@functools.cache
def is_flashinfer_installed() -> bool:
+ if is_rocm():
+ return False
return _importable("flashinfer")
@functools.cache
def is_sgl_kernel_installed() -> bool:
+ if is_rocm():
+ return False
return _importable("sgl_kernel")
@functools.cache
def is_vllm_installed() -> bool:
+ if is_rocm():
+ return False
return _importable("vllm")
@@ -54,17 +61,6 @@ def is_rocm() -> bool:
return getattr(torch.version, "hip", None) is not None
-def driver_hip_version() -> int | None:
- """ROCm driver version, or None if undetermined."""
- # TODO(ROCm): flashinfer/sgl_kernel have no ROCm builds — Triton fallback is used.
- try:
- from freetoken.kernel.pinned import _load_pinned_extension
- return int(_load_pinned_extension().driver_cuda_version()) or None
- except Exception:
- return None
-
-
@functools.cache
def driver_cuda_version() -> int | None:
"""Max CUDA version the installed NVIDIA driver supports (``13000`` == CUDA 13.0),
@@ -73,6 +69,8 @@ def driver_cuda_version() -> int | None:
toolkit version. Resolved through the ``_pinned_tensor`` extension's link-time
cudart, so it works wherever the extension builds (including Windows) -- no dlopen
by soname."""
+ if is_rocm():
+ return None
try:
from freetoken.kernel.pinned import _load_pinned_extension
diff --git a/python/freetoken/layers/quantization/linear/nvfp4.py b/python/freetoken/layers/quantization/linear/nvfp4.py
index 0953b825..80f5b78d 100644
--- a/python/freetoken/layers/quantization/linear/nvfp4.py
+++ b/python/freetoken/layers/quantization/linear/nvfp4.py
@@ -38,6 +38,8 @@ class MarlinNvfp4LinearKernel(LinearKernel):
name = "marlin"
def unusable_reason(self, cfg: LinearConfig) -> str | None:
+ if backend.is_rocm():
+ return "Marlin is CUDA-only and unavailable on ROCm; use triton"
if not backend.is_vllm_installed():
return "vLLM is not installed"
if len(cfg.output_sizes) > 1:
diff --git a/python/freetoken/layers/quantization/moe/nvfp4.py b/python/freetoken/layers/quantization/moe/nvfp4.py
index cf66e207..1cd1a1ac 100644
--- a/python/freetoken/layers/quantization/moe/nvfp4.py
+++ b/python/freetoken/layers/quantization/moe/nvfp4.py
@@ -228,6 +228,8 @@ class MarlinNvfp4MoEKernel(MoEKernel):
max_slots = MARLIN_MAX_SLOTS
def unusable_reason(self, cfg: MoEConfig) -> str | None:
+ if backend.is_rocm():
+ return "Marlin is CUDA-only and unavailable on ROCm; use triton"
if not backend.is_vllm_installed():
return "vLLM is not installed"
reason = self._common_reject(cfg, tp_ok=False, cpu_ok=False, plain_silu_only=True)
@@ -491,6 +493,8 @@ class B12xNvfp4MoEKernel(MoEKernel):
name = "b12x"
def unusable_reason(self, cfg: MoEConfig) -> str | None:
+ if backend.is_rocm():
+ return "b12x is CUDA-only and unavailable on ROCm; use triton"
cc = backend.device_capability()
if cc < (12, 0):
return f"b12x requires sm_120+, got sm_{cc[0]}{cc[1]}"
diff --git a/tests/engine/test_attention_backend_arch.py b/tests/engine/test_attention_backend_arch.py
index 9271bd42..25eb9354 100644
--- a/tests/engine/test_attention_backend_arch.py
+++ b/tests/engine/test_attention_backend_arch.py
@@ -116,3 +116,41 @@ def test_fa_backend_rejects_cc12(monkeypatch):
monkeypatch.setattr(arch, "_get_torch_cuda_version", lambda: (12, 0))
with pytest.raises(RuntimeError, match="12.x"):
FlashAttentionBackend(SimpleNamespace(head_dim=128))
+
+
[email protected]("requested", ["auto", "fi", "fa", "trtllm", "triton,fi"])
+def test_rocm_attention_selection_avoids_cuda_packages(monkeypatch, requested):
+ import builtins
+
+ from freetoken.engine import engine
+ from freetoken.kernel import backend
+
+ real_import = builtins.__import__
+
+ def guarded_import(name, *args, **kwargs):
+ if name.split(".")[0] in {"flashinfer", "sgl_kernel"}:
+ pytest.fail(f"ROCm tried to import {name}")
+ return real_import(name, *args, **kwargs)
+
+ def unexpected_probe(_name):
+ pytest.fail("ROCm tried to discover a CUDA package")
+
+ monkeypatch.setattr(builtins, "__import__", guarded_import)
+ monkeypatch.setattr(backend, "is_rocm", lambda: True)
+ monkeypatch.setattr(backend, "_importable", unexpected_probe)
+ monkeypatch.setattr(engine, "is_sm90_family", lambda: False)
+ monkeypatch.setattr(engine, "is_sm100_family", lambda: False)
+ backend.is_flashinfer_installed.cache_clear()
+ backend.is_sgl_kernel_installed.cache_clear()
+ try:
+ assert not engine._sgl_flash_attn_available()
+ config = _engine_config(attention_backend=requested)
+ if requested == "auto":
+ engine._adjust_config(config)
+ assert config.attention_backend == "triton"
+ else:
+ with pytest.raises(RuntimeError, match="CUDA-only.*ROCm"):
+ engine._adjust_config(config)
+ finally:
+ backend.is_flashinfer_installed.cache_clear()
+ backend.is_sgl_kernel_installed.cache_clear()