Skip to content

FreeToken brings datacenter-scale model serving to your desktop. Run massive models locally, fast and efficiently.

21.7 KB
Raw Blame
"""Skeleton tests: the frozen qwen4_exp interfaces and their torch references.

Everything runs on a scaled-down copy of the real geometry (4 layers, full attention on layer 3,
PLE on layer 1, hc_count 4, 3-gram hash) so the shapes and the layer split are the shipping ones.
The hyper-connection and PLE references transcribed here are HF ``modeling_qwen4_exp.py``
(``Qwen4ExpTextGatedResidual``:941, ``Qwen4ExpTextNGramEmbedding``:1018, ``Qwen4ExpTextPLELayer``
:1117), so the torch implementations are checked against the math, not against themselves.
"""

from __future__ import annotations

import math
from types import SimpleNamespace

import pytest
import torch
import torch.nn.functional as F

from freetoken.layers import BaseOP, LinearReplicated, iter_moe_layers
from freetoken.models.config import ModelConfig
from freetoken.models.qwen4_exp.config import parse_config
from freetoken.models.qwen4_exp.hc import GatedResidual
from freetoken.models.qwen4_exp.ple import GpuResidentTable, PLELayer, PLEMetadata

from .common import EOS, hash_constants, requires_cuda, toy_hf_config


def _config(num_layers: int = 4) -> ModelConfig:
    return parse_config(toy_hf_config(num_layers))


def _fill(op, gen: torch.Generator, scale: float = 0.05) -> None:
    """Random floats / zeroed ints for every state-dict tensor of an op tree, and random expert banks for its resident MoE layers."""
    for tensor in op.state_dict().values():
        if tensor.is_floating_point():
            tensor.normal_(0.0, scale, generator=gen)
        else:
            tensor.zero_()
    for layer in iter_moe_layers(op):
        layer._expert_banks = {
            role: torch.randn((layer.num_experts, *spec.shape), generator=gen, device=gen.device, dtype=spec.dtype) * scale
            for role, spec in layer.quant_method.layout().items()
        }


def _group_norm(x, weight, eps, groups):
    xf = x.float().reshape(*x.shape[:-1], groups, -1)
    xf = xf * torch.rsqrt(xf.pow(2).mean(-1, keepdim=True) + eps)
    return (xf.flatten(-2) * (1.0 + weight.float())).type_as(x)


# --------------------------------------------------------------------------------------
# hyper-connections
# --------------------------------------------------------------------------------------


def _hf_gated_residual(hc, R, w_norm, w_down, w_up, w_inject, hidden, eps):
    xn = _group_norm(R, w_norm, eps, hc)
    mix = F.silu(F.linear(xn, w_down) / hc)
    mix = torch.sigmoid(F.linear(mix, w_up)).unflatten(-1, (hc, hidden))
    mixed = (mix * xn.unflatten(-1, (hc, hidden))).mean(-2)
    if w_inject is None:
        return mixed, None
    return mixed, 2 * torch.sigmoid(F.linear(xn, w_inject) / hc)


@pytest.mark.parametrize("tokens", [1, 7])
def test_hc_mix_and_combine_match_hf(tokens: int):
    torch.manual_seed(0)
    config = _config()
    args = config.qwen4_args
    hc = GatedResidual(config)
    _fill(hc, torch.Generator().manual_seed(1))

    R = torch.randn(tokens, args.ple_state_width)
    y = torch.randn(tokens, args.hidden_size)
    x, s = hc.mix(R)
    got = hc.combine(R, y, s)

    merged = hc.input_mix_weight_down_block_inject.weight
    ref_x, ref_inject = _hf_gated_residual(
        args.hc_count,
        R,
        hc.hc_norm.weight,
        merged[: args.hc_lowrank],
        hc.input_mix_weight_up.weight,
        merged[args.hc_lowrank : args.hc_lowrank + args.hc_count],
        args.hidden_size,
        config.rms_norm_eps,
    )
    ref = R.unflatten(-1, (args.hc_count, args.hidden_size))
    ref = (ref + y.unsqueeze(-2) * ref_inject.unsqueeze(-1)).flatten(-2)

    assert torch.allclose(x, ref_x, rtol=1e-5, atol=1e-6)
    assert torch.allclose(got, ref, rtol=1e-5, atol=1e-6)


def test_hc_merged_gemm_layout_and_top_level_mixer():
    config = _config()
    args = config.qwen4_args
    hc = GatedResidual(config)
    # 320 + 4 lowrank/inject rows padded to a multiple of 16 in the real config
    assert hc.pad_size == (-(args.hc_lowrank + args.hc_count)) % 16
    merged = hc.input_mix_weight_down_block_inject.weight
    assert merged.shape == (
        args.hc_lowrank + args.hc_count + hc.pad_size,
        args.ple_state_width,
    )
    assert set(hc.state_dict()) == {
        "hc_norm.weight",
        "input_mix_weight_down_block_inject.weight",
        "input_mix_weight_up.weight",
    }

    mixer = GatedResidual(config, use_combine=False)
    _fill(mixer, torch.Generator().manual_seed(2))
    assert set(mixer.state_dict()) == {
        "hc_norm.weight",
        "input_mix_weight_down.weight",
        "input_mix_weight_up.weight",
    }
    R = torch.randn(5, args.ple_state_width)
    x, s = mixer.mix(R)
    assert s is None
    ref_x, ref_inject = _hf_gated_residual(
        args.hc_count,
        R,
        mixer.hc_norm.weight,
        mixer.input_mix_weight_down.weight,
        mixer.input_mix_weight_up.weight,
        None,
        args.hidden_size,
        config.rms_norm_eps,
    )
    assert ref_inject is None
    assert torch.allclose(x, ref_x, rtol=1e-5, atol=1e-6)


# --------------------------------------------------------------------------------------
# PLE
# --------------------------------------------------------------------------------------


def _hf_shift_right(tokens, shift, eos):
    if shift == 0:
        return tokens
    batch, seq_len = tokens.shape
    positions = torch.arange(seq_len)
    eos_positions = torch.where(tokens == eos, positions, torch.full_like(positions, -1))
    previous = torch.cummax(eos_positions, dim=1).values
    previous = torch.cat([eos_positions.new_full((batch, 1), -1), previous[:, :-1]], dim=1)
    in_segment = positions.unsqueeze(0) - (previous + 1)
    source = positions - shift
    shifted = tokens.gather(1, source.clamp_min(0).unsqueeze(0).expand(batch, -1))
    valid = (in_segment >= shift) & (source.unsqueeze(0) >= 0)
    return torch.where(valid, shifted, tokens.new_full((), eos))


def _hf_ngram_ids(tokens, context, args, multipliers, sizes, offsets):
    """HF Qwen4ExpTextNGramEmbedding id computation over a dense [B, L] batch."""
    history = torch.cat([context, tokens], dim=-1)
    shifted = [_hf_shift_right(history, s, args.ngram_boundary_token_id) for s in range(args.ngram_size)]
    blocks = []
    for ngram in range(2, args.ngram_size + 1):
        start = (ngram - 2) * args.heads_per_ngram
        end = start + args.heads_per_ngram
        mixed = shifted[0] * multipliers[0]
        for position in range(1, ngram):
            mixed = torch.bitwise_xor(mixed, shifted[position] * multipliers[position])
        ids = torch.remainder(mixed.unsqueeze(-1), sizes[start:end].view(1, 1, -1))
        blocks.append(ids + offsets[start:end].view(1, 1, -1))
    return torch.cat(blocks, dim=-1)[:, -tokens.shape[1] :]


def _ragged(sequences, contexts, args, device="cpu"):
    """Ragged PLEMetadata (prefill) for a list of per-request token lists."""
    lens = [len(s) for s in sequences]
    cu = torch.tensor([0, *lens], dtype=torch.int64).cumsum(0)
    return PLEMetadata(
        input_ids=torch.tensor([t for s in sequences for t in s], dtype=torch.int64, device=device),
        cu_seqlens=cu.to(device),
        seq_lens=tuple(lens),
        ngram_context=torch.tensor(contexts, dtype=torch.int64, device=device),
        state_slots=torch.arange(len(sequences), dtype=torch.int64, device=device),
        fresh_slots=None,
        is_decode=False,
    )


def _make_ple(config, seed: int = 3, rows: int = 4096):
    args = config.qwen4_args
    gen = torch.Generator().manual_seed(seed)
    layer = PLELayer(config, args.ple_layer_ids[0])
    _fill(layer, gen)
    multipliers, sizes, offsets = hash_constants(args)
    layer.ple_embedding.layer_multipliers.copy_(multipliers)
    layer.ple_embedding.ngram_heads_vocab_sizes.copy_(sizes)
    layer.ple_embedding.ngram_heads_offsets.copy_(offsets)
    table = torch.randn(rows, args.ngram_head_dim, generator=gen) * 0.05
    layer.ple_embedding.attach_table(GpuResidentTable(table, dtype=torch.float32))
    return layer, (multipliers, sizes, offsets)


def test_ple_hash_matches_hf():
    """Ragged hash ids vs the HF dense reference, including eos inside and at the start of a request."""
    config = _config()
    args = config.qwen4_args
    layer, (multipliers, sizes, offsets) = _make_ple(config)

    sequences = [[3, 4, EOS, 5, 6], [EOS, 11, 12], [9]]
    contexts = [[EOS, EOS], [21, 22], [EOS, 31]]
    meta = _ragged(sequences, contexts, args)
    got = layer.ple_embedding.row_ids(meta)

    offset = 0
    for tokens, context in zip(sequences, contexts):
        ref = _hf_ngram_ids(
            torch.tensor([tokens]),
            torch.tensor([context]),
            args,
            multipliers,
            sizes,
            offsets,
        )[0]
        assert torch.equal(got[offset : offset + len(tokens)], ref)
        offset += len(tokens)
    # sequences[0][3] sits right after the boundary token, so its window is cut to eos padding
    after_eos = layer.ple_embedding.row_ids(_ragged([[5]], [[EOS, EOS]], args))[0]
    with_history = layer.ple_embedding.row_ids(_ragged([[5]], [[3, 4]], args))[0]
    assert torch.equal(got[3], after_eos)
    assert not torch.equal(got[3], with_history)


def test_ple_forward_matches_hf():
    """Full PLE forward (fp32) against the HF gate/norm chain and an explicit conv tap sum."""
    torch.manual_seed(4)
    config = _config()
    args = config.qwen4_args
    layer, (multipliers, sizes, offsets) = _make_ple(config)
    hidden, hc = args.hidden_size, args.hc_count

    sequences = [[3, 4, EOS, 5, 6, 8], [2, EOS, 11, 12, 13, 14]]
    contexts = [[EOS, EOS], [21, 22]]
    meta = _ragged(sequences, contexts, args)
    total = sum(len(s) for s in sequences)
    R = torch.randn(total, args.ple_state_width)
    states = torch.randn(len(sequences), args.ple_state_width, args.ple_conv_state_len) * 0.1
    got = layer.forward(R, batch=None, meta=meta, conv_states=states.clone())

    offset = 0
    for i, (tokens, context) in enumerate(zip(sequences, contexts)):
        ids = _hf_ngram_ids(
            torch.tensor([tokens]), torch.tensor([context]), args, multipliers, sizes, offsets
        )[0]
        embed = layer.ple_embedding.table.weight[ids.reshape(-1)].view(len(tokens), -1)
        key = _group_norm(
            F.linear(embed, layer.key_proj.weight), layer.norm_key.weight, config.rms_norm_eps, hc
        ).unflatten(-1, (hc, hidden))
        value = F.linear(embed, layer.value_proj.weight)
        rows = R[offset : offset + len(tokens)]
        query = _group_norm(
            rows, layer.norm_query.weight, config.rms_norm_eps, hc
        ).unflatten(-1, (hc, hidden))
        gate = (key * query).sum(-1, keepdim=True) / math.sqrt(hidden)
        gate = torch.sigmoid(gate.sign() * gate.abs().clamp_min(1e-6).sqrt())
        gated = (gate * value.unsqueeze(-2)).flatten(-2)
        normed = _group_norm(gated, layer.norm_conv.weight, config.rms_norm_eps, hc)
        history = torch.cat([states[i], normed.transpose(0, 1)], dim=-1)
        taps = sum(
            layer.conv1d.weight[:, 0, k].unsqueeze(0)
            * history.transpose(0, 1)[k * args.ple_conv_dilation :][: len(tokens)]
            for k in range(args.ple_conv_kernel_size)
        )
        ref = gated + F.silu(taps)
        assert torch.allclose(got[offset : offset + len(tokens)], ref, rtol=1e-4, atol=1e-5)
        offset += len(tokens)


def _fresh_ctx(**fields):
    import freetoken.core as core
    from freetoken.core import Context, set_global_ctx

    core._GLOBAL_CTX = None  # test-only: each scenario builds its own ctx
    ctx = Context(page_size=64)
    for name, value in fields.items():
        setattr(ctx, name, value)
    set_global_ctx(ctx)
    return ctx


def _plus_one_rmsnorm(x, weight, eps):
    xf = x.float()
    xf = xf * torch.rsqrt(xf.pow(2).mean(-1, keepdim=True) + eps)
    return (xf * (1.0 + weight.float())).type_as(x)


def _hf_rope(x, positions, rotary_dim, base):
    """HF apply_rotary_pos_emb on [T, H, D], rotating only the first rotary_dim dims."""
    inv = 1.0 / (
        base ** (torch.arange(0, rotary_dim, 2, device=x.device, dtype=torch.float32) / rotary_dim)
    )
    freqs = positions.float().unsqueeze(-1) * inv
    cos = torch.cat([freqs.cos(), freqs.cos()], dim=-1).unsqueeze(1)
    sin = torch.cat([freqs.sin(), freqs.sin()], dim=-1).unsqueeze(1)
    rot = x[..., :rotary_dim].float()
    half = rotary_dim // 2
    rotated = torch.cat([-rot[..., half:], rot[..., :half]], dim=-1)
    out = (rot * cos + rotated * sin).type_as(x)
    return torch.cat([out, x[..., rotary_dim:]], dim=-1)


def _hf_attention(x, attn, config, positions):
    """HF Qwen4ExpTextAttention with a dense causal mask (QSA selects every block at this length)."""
    num_q, num_kv, dim = attn.num_q, attn.num_kv, attn.head_dim
    qkv = F.linear(x, attn.qkv_proj.weight)
    qg, k, v = qkv.split(attn._qkv_split, dim=-1)
    qg = qg.view(-1, num_q, dim * 2)
    q, gate = qg[..., :dim], qg[..., dim:].reshape(-1, num_q * dim)
    q = _hf_rope(_plus_one_rmsnorm(q, attn.q_norm.weight, config.rms_norm_eps), positions,
                 config.rotary_config.rotary_dim, config.rotary_config.base)
    k = _hf_rope(
        _plus_one_rmsnorm(k.view(-1, num_kv, dim), attn.k_norm.weight, config.rms_norm_eps),
        positions, config.rotary_config.rotary_dim, config.rotary_config.base,
    )
    v = v.view(-1, num_kv, dim)
    rep = num_q // num_kv
    scores = torch.einsum("qhd,khd->hqk", q.float(), k.repeat_interleave(rep, 1).float())
    scores = scores * dim**-0.5
    mask = torch.arange(x.shape[0], device=x.device) > positions.unsqueeze(-1)
    out = torch.einsum(
        "hqk,khd->qhd", scores.masked_fill(mask, float("-inf")).softmax(-1),
        v.repeat_interleave(rep, 1).float(),
    ).to(x.dtype)
    return F.linear(out.reshape(-1, num_q * dim) * torch.sigmoid(gate), attn.o_proj.weight)


@requires_cuda
def test_qsa_layer_matches_hf_dense():
    """The QSA layer under the dense oracle backend equals HF attention, and freezes what the indexer hands the backend."""
    from freetoken.models.qwen4_exp.attention import Qwen4ExpAttention, TorchDenseQSAReference
    from freetoken.utils.torch_utils import torch_dtype

    torch.manual_seed(6)
    config = _config()
    device, dtype = torch.device("cuda"), torch.bfloat16
    with torch.device(device), torch_dtype(dtype):
        attn = Qwen4ExpAttention(config, layer_id=3)
    _fill(attn, torch.Generator(device=device).manual_seed(7))

    seq_len = 24
    x = (torch.randn(seq_len, config.hidden_size, device=device, dtype=dtype) * 0.5)
    positions = torch.arange(seq_len, device=device, dtype=torch.int64)
    req = SimpleNamespace(extend_len=seq_len, cached_len=0, table_idx=1)
    batch = SimpleNamespace(
        padded_reqs=[req], reqs=[req], positions=positions, get_attn_positions=lambda: positions
    )

    backend = TorchDenseQSAReference(config, num_slots=4, max_len=64, device=device, dtype=dtype)
    _fresh_ctx(attn_backend=backend)
    ref = _hf_attention(x, attn, config, positions)
    got = attn.forward(x, batch)
    assert torch.allclose(got.float(), ref.float(), rtol=2e-2, atol=2e-2)

    index = attn.indexer.forward(x)
    args = config.qwen4_args
    raw = F.linear(x, attn.indexer.index_qk_proj.weight)
    assert index.q.shape == (seq_len, args.index_n_heads, args.index_head_dim)
    assert index.k.shape == (seq_len, args.index_head_dim)
    assert torch.equal(index.q.reshape(seq_len, -1), raw[:, : args.index_n_heads * args.index_head_dim])
    assert torch.equal(index.k, raw[:, args.index_n_heads * args.index_head_dim :])
    assert index.q_norm_weight.data_ptr() == attn.indexer.q_layernorm.weight.data_ptr()


class _StubLinearMixer(BaseOP):
    """Stands in for the GDN layer; same [T, hidden] -> [T, hidden] shape."""

    def __init__(self, config, layer_id, prefix=""):
        self.out_proj = LinearReplicated(config.hidden_size, config.hidden_size, has_bias=False)

    def forward(self, x):
        return self.out_proj.forward(x)


@requires_cuda
def test_shared_expert_gate_fusion_matches_eager():
    """Qwen4ExpMoE only swaps qwen3_5's gemv+sigmoid+mul+add gate chain for two triton kernels."""
    from freetoken.models.qwen3_5_moe.moe import Qwen3_5MoE
    from freetoken.models.qwen4_exp.moe import Qwen4ExpMoE
    from freetoken.utils.torch_utils import torch_dtype

    config = _config()
    device, dtype = torch.device("cuda"), torch.bfloat16
    with torch.device(device), torch_dtype(dtype):
        moe = Qwen4ExpMoE(config, 0)
    _fill(moe, torch.Generator(device=device).manual_seed(21), scale=0.2)
    _fresh_ctx(_batch=SimpleNamespace(is_prefill=True))

    x = torch.randn(6, config.hidden_size, device=device, dtype=dtype) * 0.5
    fused = moe.forward(x.clone())
    eager = Qwen3_5MoE.forward(moe, x.clone())

    routed = moe.experts.forward(hidden_states=x.clone(), router_logits=moe.gate.forward(x))
    gate = torch.sigmoid(x.float() @ moe.shared_expert_gate.weight.float().view(-1))
    ref = routed.float() + gate.unsqueeze(1) * moe.shared_expert.forward(x).float()

    assert fused.shape == x.shape and fused.dtype == dtype
    torch.testing.assert_close(fused, eager, rtol=2e-2, atol=2e-2)
    # The fused gate stays in fp32 where the eager chain rounds the scalar to bf16.
    assert (fused.float() - ref).abs().max() <= (eager.float() - ref).abs().max()


@requires_cuda
@pytest.mark.parametrize("dtype", [torch.bfloat16])
@pytest.mark.parametrize("num_tokens,hidden", [(1, 2560), (7, 640)])
def test_shared_gate_kernels_match_torch(num_tokens, hidden, dtype):
    """Shipping hidden size for the two shared-gate kernels, against the torch chain they replace."""
    from freetoken.kernel.triton.moe_shared_gate import shared_gate_mul_add, shared_gate_sigmoid

    gen = torch.Generator(device="cuda").manual_seed(hidden + num_tokens)
    kw = {"generator": gen, "device": "cuda", "dtype": dtype}
    x = torch.randn(num_tokens, hidden, **kw)
    weight = torch.randn(1, hidden, **kw) * 0.05
    shared = torch.randn(num_tokens, hidden, **kw)
    routed = torch.randn(num_tokens, hidden, **kw)

    fused = shared_gate_mul_add(routed, shared, shared_gate_sigmoid(x, weight.view(-1)))
    eager = routed + shared * torch.sigmoid(F.linear(x, weight))
    ref = routed.float() + shared.float() * torch.sigmoid(x.float() @ weight.float().view(-1))[:, None]

    assert fused.dtype == dtype and fused.shape == routed.shape
    torch.testing.assert_close(fused, eager, rtol=2e-2, atol=2e-2)
    assert (fused.float() - ref).abs().max() <= (eager.float() - ref).abs().max() + 1e-6


@requires_cuda
def test_decoder_stack_prefill_and_decode(monkeypatch):
    """Ragged bs=3 prefill then a bs=3 decode step through the whole model with dummy weights."""
    from freetoken.kvcache.linear_state_pool import LinearStatePool
    from freetoken.models.qwen4_exp import model as model_module
    from freetoken.models.qwen4_exp.attention import TorchDenseQSAReference
    from freetoken.models.qwen4_exp.ple import GpuResidentTable
    from freetoken.utils.torch_utils import torch_dtype

    torch.manual_seed(8)
    config = _config()
    args = config.qwen4_args
    device, dtype = torch.device("cuda"), torch.bfloat16
    monkeypatch.setattr(model_module, "build_linear_mixer", _StubLinearMixer)

    with torch.device(device), torch_dtype(dtype):
        model = model_module.Qwen4ExpForCausalLM(config)
    gen = torch.Generator(device=device).manual_seed(9)
    _fill(model, gen)
    multipliers, sizes, offsets = hash_constants(args)
    table = torch.randn(4096, args.ngram_head_dim, generator=gen, device=device, dtype=dtype) * 0.05
    for ple in model.model.ple_layers:
        ple.ple_embedding.layer_multipliers.copy_(multipliers)
        ple.ple_embedding.ngram_heads_vocab_sizes.copy_(sizes)
        ple.ple_embedding.ngram_heads_offsets.copy_(offsets)
        ple.ple_embedding.attach_table(GpuResidentTable(table, dtype=dtype))

    num_slots, max_len = 4, 64
    pool = LinearStatePool(
        config.linear_attention_group(), num_slots, dtype, device,
        slot_states=config.slot_states,
    )
    prompts = [[3, 4, EOS, 5, 6, 8], [2, EOS, 11, 12], [9, 10, 11, 12, 13]]
    ctx = _fresh_ctx(
        attn_backend=TorchDenseQSAReference(config, num_slots, max_len, device, dtype),
        linear_state_pool=pool,
    )
    reqs = [
        SimpleNamespace(
            extend_len=len(p), cached_len=0, table_idx=i + 1, linear_slot_idx=None,
            input_ids=torch.tensor(p, dtype=torch.int64),
        )
        for i, p in enumerate(prompts)
    ]
    flat = [t for p in prompts for t in p]
    last = torch.tensor(
        [sum(len(p) for p in prompts[: i + 1]) - 1 for i in range(len(prompts))], device=device
    )
    positions = torch.cat([torch.arange(len(p)) for p in prompts]).to(device)
    batch = SimpleNamespace(
        padded_reqs=reqs, reqs=reqs, size=len(reqs), is_prefill=True, is_decode=False,
        input_ids=torch.tensor(flat, dtype=torch.int64, device=device),
        positions=positions, get_attn_positions=lambda: positions, mm_embeds=None,
        attn_metadata=SimpleNamespace(get_last_indices=lambda bs: last[:bs]),
    )
    with ctx.forward_batch(batch):
        logits = model.forward()
    assert logits.shape == (len(prompts), config.vocab_size)
    assert torch.isfinite(logits.float()).all()

    for r, p in zip(reqs, prompts):
        r.cached_len = len(p)
        r.extend_len = 1
        r.input_ids = torch.cat([r.input_ids, torch.tensor([14], dtype=torch.int64)])
    decode_positions = torch.tensor([len(p) for p in prompts], dtype=torch.int64, device=device)
    decode = SimpleNamespace(
        padded_reqs=reqs, reqs=reqs, size=len(reqs), is_prefill=False, is_decode=True,
        input_ids=torch.tensor([14] * len(reqs), dtype=torch.int64, device=device),
        positions=decode_positions, get_attn_positions=lambda: decode_positions, mm_embeds=None,
        attn_metadata=None,
    )
    with ctx.forward_batch(decode):
        decode_logits = model.forward()
    assert decode_logits.shape == (len(prompts), config.vocab_size)
    assert torch.isfinite(decode_logits.float()).all()