Skip to content

FreeToken brings datacenter-scale model serving to your desktop. Run massive models locally, fast and efficiently.

test(kernel): cover Triton activation and norm fallbacks (#652)

ZihaoMu6c36af01108b
diff --git a/tests/kernels/test_triton_fallbacks.py b/tests/kernels/test_triton_fallbacks.py
new file mode 100644
index 00000000..45234e66
--- /dev/null
+++ b/tests/kernels/test_triton_fallbacks.py
@@ -0,0 +1,57 @@
+"""Numerical checks for the built-in activation and norm fallback paths."""
+
+import pytest
+import torch
+
+
+pytestmark = pytest.mark.skipif(
+ not torch.cuda.is_available(), reason="a CUDA or ROCm GPU is required"
+)
+
+
[email protected](autouse=True)
+def _use_triton_fallbacks(monkeypatch):
+ from freetoken.kernel import backend
+
+ monkeypatch.setattr(backend, "is_flashinfer_installed", lambda: False)
+ monkeypatch.setattr(backend, "is_sgl_kernel_installed", lambda: False)
+
+
[email protected]("dtype", [torch.float32, torch.float16, torch.bfloat16])
[email protected]("activation", ["silu", "gelu", "gelu_tanh"])
+def test_activation_fallback_matches_torch(dtype, activation):
+ from freetoken.layers.activation import gated_act_and_mul
+
+ torch.manual_seed(0)
+ x = torch.randn((7, 256), device="cuda", dtype=dtype)
+ out = torch.full((7, 128), float("nan"), device="cuda", dtype=dtype)
+ gate, up = x.float().chunk(2, dim=-1)
+ if activation == "silu":
+ expected = torch.nn.functional.silu(gate) * up
+ else:
+ approx = "tanh" if activation == "gelu_tanh" else "none"
+ expected = torch.nn.functional.gelu(gate, approximate=approx) * up
+
+ gated_act_and_mul(activation, x, out)
+
+ # CUDA intentionally uses approximate math (notably tanh.approx.f32),
+ # while ROCm uses libdevice; allow for both against the Torch reference.
+ rtol, atol = (1e-4, 1e-4) if dtype == torch.float32 else (1e-2, 1e-3)
+ torch.testing.assert_close(out, expected.to(dtype), rtol=rtol, atol=atol)
+
+
[email protected]("dtype", [torch.float32, torch.float16, torch.bfloat16])
[email protected]("kind", ["rmsnorm", "gemma"])
+def test_norm_fallback_matches_torch(dtype, kind):
+ from freetoken.layers.norm import GemmaRMSNorm, RMSNorm
+
+ torch.manual_seed(0)
+ norm = (RMSNorm if kind == "rmsnorm" else GemmaRMSNorm)(128, 1e-6)
+ norm.weight = torch.randn(128, device="cuda", dtype=dtype)
+ x = torch.randn((7, 128), device="cuda", dtype=dtype)
+ xf = x.float()
+ expected = xf * torch.rsqrt(xf.square().mean(-1, keepdim=True) + norm.eps)
+ expected *= norm.weight.float()
+
+ rtol, atol = (1e-5, 1e-6) if dtype == torch.float32 else (1e-2, 1e-3)
+ torch.testing.assert_close(norm.forward(x), expected.to(dtype), rtol=rtol, atol=atol)