Skip to content

FreeToken brings datacenter-scale model serving to your desktop. Run massive models locally, fast and efficiently.

3.1 KB
Raw Blame
"""Numerical checks for the built-in activation and norm fallback paths."""

import pytest
import torch


pytestmark = pytest.mark.skipif(
    not torch.cuda.is_available(), reason="a CUDA or ROCm GPU is required"
)


@pytest.fixture(autouse=True)
def _use_triton_fallbacks(monkeypatch):
    from freetoken.kernel import backend

    monkeypatch.setattr(backend, "is_flashinfer_installed", lambda: False)
    monkeypatch.setattr(backend, "is_sgl_kernel_installed", lambda: False)


@pytest.mark.parametrize("dtype", [torch.float32, torch.float16, torch.bfloat16])
@pytest.mark.parametrize("activation", ["silu", "gelu", "gelu_tanh"])
def test_activation_fallback_matches_torch(dtype, activation):
    from freetoken.layers.activation import gated_act_and_mul

    torch.manual_seed(0)
    x = torch.randn((7, 256), device="cuda", dtype=dtype)
    out = torch.full((7, 128), float("nan"), device="cuda", dtype=dtype)
    gate, up = x.float().chunk(2, dim=-1)
    if activation == "silu":
        expected = torch.nn.functional.silu(gate) * up
    else:
        approx = "tanh" if activation == "gelu_tanh" else "none"
        expected = torch.nn.functional.gelu(gate, approximate=approx) * up

    gated_act_and_mul(activation, x, out)

    # CUDA intentionally uses approximate math (notably tanh.approx.f32),
    # while ROCm uses libdevice; allow for both against the Torch reference.
    rtol, atol = (1e-4, 1e-4) if dtype == torch.float32 else (1e-2, 1e-3)
    torch.testing.assert_close(out, expected.to(dtype), rtol=rtol, atol=atol)


@pytest.mark.parametrize("dtype", [torch.float32, torch.float16, torch.bfloat16])
@pytest.mark.parametrize("kind", ["rmsnorm", "gemma"])
def test_norm_fallback_matches_torch(dtype, kind):
    from freetoken.layers.norm import GemmaRMSNorm, RMSNorm

    torch.manual_seed(0)
    norm = (RMSNorm if kind == "rmsnorm" else GemmaRMSNorm)(128, 1e-6)
    norm.weight = torch.randn(128, device="cuda", dtype=dtype)
    x = torch.randn((7, 128), device="cuda", dtype=dtype)
    xf = x.float()
    expected = xf * torch.rsqrt(xf.square().mean(-1, keepdim=True) + norm.eps)
    expected *= norm.weight.float()

    rtol, atol = (1e-5, 1e-6) if dtype == torch.float32 else (1e-2, 1e-3)
    torch.testing.assert_close(norm.forward(x), expected.to(dtype), rtol=rtol, atol=atol)


@pytest.mark.parametrize("mask", ["prefix", "allowed_ids"])
def test_sampling_softmax_fallback_with_masked_logits(mask):
    from freetoken.kernel.triton.sampling import softmax

    torch.manual_seed(0)
    logits = torch.randn((3, 151936), device="cuda") * 4
    if mask == "prefix":
        # Long enough that some column chunk starts with an all -inf block.
        logits[:2, :16384] = float("-inf")
    else:
        allowed = torch.tensor([5, 70000, 151000], device="cuda")
        logits[:2] = torch.full_like(logits[:2], float("-inf")).index_copy(
            1, allowed, logits[:2, allowed])
    temperature = torch.tensor([1.0, 0.7, 1.3], device="cuda")
    expected = torch.softmax(logits / temperature[:, None], dim=-1)

    torch.testing.assert_close(softmax(logits, temperature), expected, rtol=1e-5, atol=1e-7)