FreeToken brings datacenter-scale model serving to your desktop. Run massive models locally, fast and efficiently.
"""Glm5NextKDA op vs an eager reference (projection/conv/gate/norm wiring).
The kernel math itself is validated in tests/kernels/test_kda.py; this test checks
the OP-level wiring: the fused in_proj split (q|k|v|b|f_a|g_a), the merged q|k|v
depthwise causal conv (+silu) against the state pool, the low-rank f/g gates, the
sigmoid-gated output RMSNorm, and prefill -> decode state continuity through
``LinearStatePool``.
"""
from __future__ import annotations
from types import SimpleNamespace
import pytest
import torch
pytestmark = pytest.mark.skipif(not torch.cuda.is_available(), reason="needs CUDA")
@pytest.fixture(autouse=True)
def _single_rank_tp():
from freetoken.distributed import set_tp_info, try_get_tp_info
if try_get_tp_info() is None:
set_tp_info(rank=0, size=1)
HIDDEN, H, D, KERNEL = 256, 4, 128, 4
P = H * D
LOWER_BOUND = -5.0
def _make_args():
from freetoken.models.glm5_next.args import Glm5NextArgs
n_layers = 2
return Glm5NextArgs(
hidden_size=HIDDEN, num_heads=8,
q_lora_rank=64, kv_lora_rank=32, qk_nope_head_dim=32, qk_rope_head_dim=0,
v_head_dim=32, mla_nope=True, norm_eps=1e-5, max_position=4096,
index_n_heads=0, index_head_dim=0, index_topk=0, indexer_types=(),
indexer_rope_interleave=True, index_kpool=1, index_kpool_compress=False,
index_kpool_always_select_tail=False,
linear_num_heads=H, linear_head_dim=D, linear_conv_kernel_dim=KERNEL,
linear_lower_bound=LOWER_BOUND,
layer_types=("linear_attention",) * n_layers,
mlp_layer_types=("dense",) * n_layers,
mhc=False, mhc_num_residual_streams=1, hc_eps=1e-6,
mhc_sinkhorn_iterations=0, mhc_tau=0.05, mhc_post_mult_value=2.0,
mhc_no_norm_weight=False, swiglu_limit=None, rope_theta=10000.0,
)
def _make_op(seed=0):
from freetoken.models.glm5_next.kda import Glm5NextKDA
cfg = SimpleNamespace(glm5_args=_make_args(), attn_quant="none", quant=None)
op = Glm5NextKDA(cfg, layer_id=0)
torch.manual_seed(seed)
dev, dt = "cuda", torch.bfloat16
op.in_proj.weight = torch.randn(3 * P + H + 2 * D, HIDDEN, device=dev, dtype=dt) * 0.05
op.f_b_proj.weight = torch.randn(P, D, device=dev, dtype=dt) * 0.05
op.g_b_proj.weight = torch.randn(P, D, device=dev, dtype=dt) * 0.05
op.conv1d.weight = torch.randn(3 * P, 1, KERNEL, device=dev, dtype=dt) * 0.2
op.A_log = torch.randn(H, device=dev, dtype=torch.float32) * 0.5
op.dt_bias = torch.randn(P, device=dev, dtype=torch.float32) * 0.5
op.o_norm.weight = torch.randn(D, device=dev, dtype=dt) * 0.1 + 1.0
op.o_proj.weight = torch.randn(HIDDEN, P, device=dev, dtype=dt) * 0.05
return op
def _make_pool(num_slots=4):
from freetoken.kvcache.linear_state_pool import LinearStatePool
from freetoken.models.config import LinearGatedDeltaGroupConfig
group = LinearGatedDeltaGroupConfig(
name="linear", layer_ids=(0,),
num_key_heads=H, num_value_heads=H, key_head_dim=D, value_head_dim=D,
conv_kernel_dim=KERNEL, output_gate=True, variant="kda",
)
return LinearStatePool(
group, num_slots, dtype=torch.bfloat16,
device=torch.device("cuda"), tp_size=1,
)
def _patch_ctx(monkeypatch, pool, batch):
ctx = SimpleNamespace(batch=batch, linear_state_pool=pool)
monkeypatch.setattr(
"freetoken.models.glm5_next.kda.get_global_ctx", lambda: ctx
)
def _l2norm(x):
return x / torch.sqrt((x * x).sum(-1, keepdim=True) + 1e-6)
def _reference_forward(op, x_seq, conv_ctx=None, h0=None):
"""Eager op reference for one sequence [T, HIDDEN] (fp32 where the kernels are
fp32). Returns (out [T, HIDDEN], conv_tail [3P, KERNEL-1], state [H, D, D])."""
T = x_seq.shape[0]
proj = x_seq.to(torch.bfloat16) @ op.in_proj.weight.T
conv_in, b, f_a, g_a = torch.split(proj, [3 * P, H, D, D], dim=-1)
g1 = (f_a @ op.f_b_proj.weight.T).float()
g2 = g_a @ op.g_b_proj.weight.T
# depthwise causal conv + silu over the merged q|k|v stream, with optional
# left-context from a previous chunk (conv state semantics).
w = op.conv1d.weight.squeeze(1).float() # [3P, KERNEL]
stream = conv_in.T.float() # [3P, T]
left = (
conv_ctx.float()
if conv_ctx is not None
else torch.zeros(3 * P, KERNEL - 1, device=x_seq.device)
)
padded = torch.cat([left, stream], dim=1) # [3P, KERNEL-1+T]
conv = torch.stack(
[(padded[:, t : t + KERNEL] * w).sum(-1) for t in range(T)], dim=1
)
mixed = torch.nn.functional.silu(conv).T # [T, 3P]
conv_tail = padded[:, -(KERNEL - 1):]
q, k, v = (t.reshape(T, H, D) for t in torch.split(mixed, [P, P, P], dim=-1))
# bf16 round-trip like the op (kernel inputs are bf16)
q, k, v = q.to(torch.bfloat16).float(), k.to(torch.bfloat16).float(), v.to(torch.bfloat16).float()
h = h0.clone() if h0 is not None else torch.zeros(H, D, D, device=x_seq.device)
amp = op.A_log.exp().view(H, 1)
bias = op.dt_bias.view(H, D)
core = []
for t in range(T):
gk = LOWER_BOUND * torch.sigmoid(amp * (g1[t].view(H, D) + bias))
h = h * gk.exp().unsqueeze(1)
kt = _l2norm(k[t])
v_err = (v[t] - torch.einsum("hvk,hk->hv", h, kt)) * torch.sigmoid(
b[t].float()
).unsqueeze(-1)
h = h + torch.einsum("hv,hk->hvk", v_err, kt)
core.append(torch.einsum("hvk,hk->hv", h, _l2norm(q[t]) * D**-0.5))
core = torch.stack(core) # [T, H, D]
xn = core.reshape(-1, D)
rms = xn * torch.rsqrt(xn.pow(2).mean(-1, keepdim=True) + op.o_norm.eps)
gated = rms * op.o_norm.weight.float() * torch.sigmoid(g2.reshape(-1, D).float())
out = gated.reshape(T, P).to(torch.bfloat16) @ op.o_proj.weight.T
return out.float(), conv_tail, h
def _fla(cu, indices, has_init=None, fresh=None):
from freetoken.attention.linear import FLAMetadata
return FLAMetadata(
cu_seqlens=cu, cache_indices=indices,
has_initial_state=has_init, fresh_state_indices=fresh,
)
def _assert_close(ours, ref, tag, atol=3e-2):
err = (ours.float() - ref.float()).abs().max().item()
scale = ref.float().abs().max().item() + 1e-8
assert err / scale < atol, f"{tag}: max abs err {err:.5f} (ref scale {scale:.3f})"
def test_prefill_matches_reference(monkeypatch):
op = _make_op()
pool = _make_pool()
lens = [33, 70]
total = sum(lens)
torch.manual_seed(10)
x = torch.randn(total, HIDDEN, device="cuda", dtype=torch.bfloat16)
cu = torch.tensor([0, *torch.tensor(lens).cumsum(0).tolist()], dtype=torch.int32, device="cuda")
indices = torch.tensor([1, 2], dtype=torch.int32, device="cuda")
has_init = torch.tensor([False, False], device="cuda")
fresh = torch.tensor([1, 2], dtype=torch.int64, device="cuda")
batch = SimpleNamespace(is_decode=False, fla_metadata=_fla(cu, indices, has_init, fresh))
_patch_ctx(monkeypatch, pool, batch)
out = op.forward(x)
start = 0
for i, ln in enumerate(lens):
sl = slice(start, start + ln)
ref_out, ref_conv, ref_h = _reference_forward(op, x[sl].float())
_assert_close(out[sl], ref_out, f"seq{i} prefill out")
_assert_close(pool.recurrent_states[0, i + 1], ref_h, f"seq{i} state")
_assert_close(pool.conv_states[0, i + 1], ref_conv, f"seq{i} conv state")
start += ln
def test_prefill_then_decode_continuity(monkeypatch):
op = _make_op(seed=1)
pool = _make_pool()
T0, T1 = 40, 3
torch.manual_seed(11)
x = torch.randn(T0 + T1, HIDDEN, device="cuda", dtype=torch.bfloat16)
ref_out, _, _ = _reference_forward(op, x.float())
cu = torch.tensor([0, T0], dtype=torch.int32, device="cuda")
indices = torch.tensor([1], dtype=torch.int32, device="cuda")
batch = SimpleNamespace(
is_decode=False,
fla_metadata=_fla(
cu, indices,
torch.tensor([False], device="cuda"),
torch.tensor([1], dtype=torch.int64, device="cuda"),
),
)
_patch_ctx(monkeypatch, pool, batch)
out0 = op.forward(x[:T0])
_assert_close(out0, ref_out[:T0], "prefill out")
for t in range(T0, T0 + T1):
batch = SimpleNamespace(
is_decode=True,
fla_metadata=_fla(
torch.tensor([0, 1], dtype=torch.int32, device="cuda"), indices
),
)
_patch_ctx(monkeypatch, pool, batch)
out_t = op.forward(x[t : t + 1])
_assert_close(out_t[0], ref_out[t], f"decode token {t}")
def test_chunked_prefill_continuation(monkeypatch):
"""Second prefill chunk with has_initial_state=True must continue conv AND
recurrent state exactly (the chunked-prefill path)."""
op = _make_op(seed=2)
pool = _make_pool()
T0, T1 = 64, 30
torch.manual_seed(12)
x = torch.randn(T0 + T1, HIDDEN, device="cuda", dtype=torch.bfloat16)
ref_out, ref_conv, ref_h = _reference_forward(op, x.float())
indices = torch.tensor([1], dtype=torch.int32, device="cuda")
batch = SimpleNamespace(
is_decode=False,
fla_metadata=_fla(
torch.tensor([0, T0], dtype=torch.int32, device="cuda"), indices,
torch.tensor([False], device="cuda"),
torch.tensor([1], dtype=torch.int64, device="cuda"),
),
)
_patch_ctx(monkeypatch, pool, batch)
out0 = op.forward(x[:T0])
_assert_close(out0, ref_out[:T0], "chunk0 out")
batch = SimpleNamespace(
is_decode=False,
fla_metadata=_fla(
torch.tensor([0, T1], dtype=torch.int32, device="cuda"), indices,
torch.tensor([True], device="cuda"), None,
),
)
_patch_ctx(monkeypatch, pool, batch)
out1 = op.forward(x[T0:])
_assert_close(out1, ref_out[T0:], "chunk1 out")
_assert_close(pool.recurrent_states[0, 1], ref_h, "final state")
_assert_close(pool.conv_states[0, 1], ref_conv, "final conv state")