FreeToken brings datacenter-scale model serving to your desktop. Run massive models locally, fast and efficiently.
"""Arch-family gating for attention backends.
The utils.arch helpers come in two flavors: open-ended ``is_smXX_supported``
(capability >= X, for family-portable features like PDL) and closed
``is_smXX_family`` (major == X, for arch-specific cubins: FA3 is sm_90a-only,
trtllm-gen is sm_100a/103a-only). Auto backend selection must use the closed
checks: consumer Blackwell (sm_120/121) passes ``is_sm100_supported`` but
cannot run either kernel family, so it has to resolve to fi.
"""
from types import SimpleNamespace
import pytest
import torch
from freetoken.utils import arch
@pytest.mark.parametrize(
"cc, sm90_sup, sm100_sup, sm90_fam, sm100_fam",
[
((8, 6), False, False, False, False),
((9, 0), True, False, True, False),
((10, 0), True, True, False, True),
((10, 3), True, True, False, True),
((11, 0), True, True, False, False),
((12, 0), True, True, False, False),
((12, 1), True, True, False, False),
(None, False, False, False, False), # no CUDA device
],
)
def test_arch_helper_semantics(monkeypatch, cc, sm90_sup, sm100_sup, sm90_fam, sm100_fam):
monkeypatch.setattr(arch, "_get_torch_cuda_version", lambda: cc)
assert arch.is_sm90_supported() is sm90_sup
assert arch.is_sm100_supported() is sm100_sup
assert arch.is_sm90_family() is sm90_fam
assert arch.is_sm100_family() is sm100_fam
def _engine_config(**overrides):
from freetoken.distributed import DistributedInfo
from freetoken.engine.config import EngineConfig
config = EngineConfig(
model_path="/tmp/freetoken-test-model",
tp_info=DistributedInfo(rank=0, size=1),
dtype=torch.bfloat16,
**overrides,
)
object.__setattr__(
config,
"model_config",
SimpleNamespace(
has_swa_attention=False,
has_linear_attention=False,
is_moe=False,
num_layers=10,
expert_quant="none",
),
)
return config
def _patch_env(monkeypatch, *, major, flashinfer=True, sgl=True):
from freetoken.engine import engine
monkeypatch.setattr(engine, "is_sm100_family", lambda: major == 10)
monkeypatch.setattr(engine, "is_sm90_family", lambda: major == 9)
monkeypatch.setattr(engine, "_flashinfer_available", lambda: flashinfer)
monkeypatch.setattr(engine, "_sgl_flash_attn_available", lambda: sgl)
@pytest.mark.parametrize(
"major, flashinfer, sgl, expected",
[
(10, True, True, "trtllm"), # datacenter Blackwell
(9, True, True, "fa,fi"), # Hopper with sgl_kernel
(9, True, False, "fi"), # Hopper without sgl_kernel
(12, True, True, "fi"), # consumer Blackwell: no FA3/FA4/trtllm-gen kernels
(11, True, True, "fi"), # Thor
(8, True, True, "fi"), # Ampere
(12, False, True, "triton"), # no flashinfer -> only self-contained option
],
)
def test_auto_backend_selection_by_arch(monkeypatch, major, flashinfer, sgl, expected):
from freetoken.engine.engine import _adjust_config
_patch_env(monkeypatch, major=major, flashinfer=flashinfer, sgl=sgl)
config = _engine_config(attention_backend="auto")
_adjust_config(config)
assert config.attention_backend == expected
def test_explicit_trtllm_rejected_outside_sm100_family(monkeypatch):
from freetoken.engine.engine import _adjust_config
_patch_env(monkeypatch, major=12)
config = _engine_config(attention_backend="trtllm")
with pytest.raises(RuntimeError, match="10.x"):
_adjust_config(config)
def test_explicit_trtllm_allowed_on_sm100_family(monkeypatch):
from freetoken.engine.engine import _adjust_config
_patch_env(monkeypatch, major=10)
config = _engine_config(attention_backend="trtllm")
_adjust_config(config)
assert config.attention_backend == "trtllm"
assert config.page_size in (16, 32, 64)
def test_fa_backend_rejects_cc12(monkeypatch):
from freetoken.attention.fa import FlashAttentionBackend
monkeypatch.setattr(arch, "_get_torch_cuda_version", lambda: (12, 0))
with pytest.raises(RuntimeError, match="12.x"):
FlashAttentionBackend(SimpleNamespace(head_dim=128))
@pytest.mark.parametrize("requested", ["auto", "fi", "fa", "trtllm", "triton,fi"])
def test_rocm_attention_selection_avoids_cuda_packages(monkeypatch, requested):
import builtins
from freetoken.engine import engine
from freetoken.kernel import backend
real_import = builtins.__import__
def guarded_import(name, *args, **kwargs):
if name.split(".")[0] in {"flashinfer", "sgl_kernel"}:
pytest.fail(f"ROCm tried to import {name}")
return real_import(name, *args, **kwargs)
def unexpected_probe(_name):
pytest.fail("ROCm tried to discover a CUDA package")
monkeypatch.setattr(builtins, "__import__", guarded_import)
monkeypatch.setattr(backend, "is_rocm", lambda: True)
monkeypatch.setattr(backend, "_importable", unexpected_probe)
monkeypatch.setattr(engine, "is_sm90_family", lambda: False)
monkeypatch.setattr(engine, "is_sm100_family", lambda: False)
backend.is_flashinfer_installed.cache_clear()
backend.is_sgl_kernel_installed.cache_clear()
try:
assert not engine._sgl_flash_attn_available()
config = _engine_config(attention_backend=requested)
if requested == "auto":
engine._adjust_config(config)
assert config.attention_backend == "triton"
else:
with pytest.raises(RuntimeError, match="CUDA-only.*ROCm"):
engine._adjust_config(config)
finally:
backend.is_flashinfer_installed.cache_clear()
backend.is_sgl_kernel_installed.cache_clear()