FreeToken brings datacenter-scale model serving to your desktop. Run massive models locally, fast and efficiently.
"""GLM-5.3-Flash KDA (Kimi Delta Attention) op.
Per-channel gated delta rule over H=64 heads of D=128, with SEPARATE q/k/v short
convolutions (vs GDN's one fused conv), a low-rank forget gate (``f_a`` ->
``f_b`` -> raw per-channel logits; the bounded safe gate ``lower_bound *
sigmoid(exp(A_log) * (g + dt_bias))`` is computed inside the kernels), a
per-head sigmoid beta (``b_proj``), and a sigmoid-gated output RMSNorm
(``o_norm`` gated by ``g_b(g_a(x))``).
State lives in ``ctx.linear_state_pool`` exactly like GDN: the conv state is the
merged q|k|v stream (width 3*H*D == the pool's ``2*K*dk + V*dv``) and the
recurrent state is one [D, D] matrix per head, stored in the KERNEL's [V, K]
layout (coincides with the pool's [K, V] declaration because D_k == D_v; same
convention as GDN, see layers/linear_attn/gdn.py).
Kernels: ``fused_recurrent_kda`` decodes with in-kernel gate/beta/l2norm and
per-slot state read/write (slot 0 is its NULL sentinel == the pool's padding
slot); ``chunk_kda_with_fused_gate`` prefills from an explicitly gathered
initial state and returns the final state, which this op scatters back (the
kernel CLOBBERS its v buffer -- v here is an ephemeral conv output, so that is
free). Hybrid-radix track snapshots ride the per-chunk h (``return_h``).
"""
from __future__ import annotations
from typing import TYPE_CHECKING
import torch
from freetoken.core import get_global_ctx
from freetoken.kernel.causal_conv1d import causal_conv1d_decode, causal_conv1d_varlen
from freetoken.layers import BaseOP, GatedRMSNorm, LinearColParallelMerged, LinearReplicated
from freetoken.utils import nvtx_annotate
if TYPE_CHECKING:
from freetoken.models.config import ModelConfig
class _DepthwiseConv1d(BaseOP):
"""Merged q|k|v depthwise conv weight ``[3*H*D, 1, K]`` (key ``conv1d.weight``;
the loader concatenates the checkpoint's q/k/v_conv1d along channels)."""
def __init__(self, conv_dim: int, kernel: int):
self.weight = torch.empty(conv_dim, 1, kernel)
class Glm5NextKDA(BaseOP):
"""KDA op; state is held in ``ctx.linear_state_pool`` keyed by the request's
linear slot (``FLAMetadata.cache_indices``). Parameter names follow the
checkpoint modulo two load-time fusions (see weight.py): ``in_proj`` is
q|k|v|b|f_a|g_a concatenated, ``conv1d`` is q|k|v conv concatenated."""
def __init__(self, config: ModelConfig, layer_id: int, *, prefix: str = ""):
args = config.glm5_args
self.layer_id = layer_id
self.num_heads = args.linear_num_heads
self.head_dim = args.linear_head_dim
self.proj_size = self.num_heads * self.head_dim # H * D
self.conv_dim = 3 * self.proj_size # merged q|k|v stream
self.conv_kernel_size = args.linear_conv_kernel_dim
self.lower_bound = args.linear_lower_bound
self.scale = self.head_dim**-0.5
p, h, d = self.proj_size, self.num_heads, self.head_dim
# one fused input GEMM over q|k|v|b|f_a|g_a
self._in_proj_split = [p, p, p, h, d, d]
self.in_proj = LinearColParallelMerged(
args.hidden_size, self._in_proj_split, has_bias=False,
quant_config=config.quant, prefix=f"{prefix}.in_proj",
)
# Low-rank gate up-projections (128 -> 8192): forget gate and output gate.
self.f_b_proj = LinearReplicated(d, p, has_bias=False, quant_config=config.quant, prefix=f"{prefix}.f_b_proj")
self.g_b_proj = LinearReplicated(d, p, has_bias=False, quant_config=config.quant, prefix=f"{prefix}.g_b_proj")
self.conv1d = _DepthwiseConv1d(self.conv_dim, self.conv_kernel_size)
# Gate params stay fp32 (exp/sigmoid precision; the kernels read fp32).
# models/weight.py exempts *.A_log / *.dt_bias from the model-dtype downcast.
self.A_log = torch.empty(h, dtype=torch.float32)
self.dt_bias = torch.empty(p, dtype=torch.float32)
self.o_norm = GatedRMSNorm(d, eps=args.norm_eps, activation="sigmoid")
self.o_proj = LinearReplicated(p, args.hidden_size, has_bias=False, quant_config=config.quant, prefix=f"{prefix}.o_proj")
def _conv_weight(self) -> torch.Tensor:
return self.conv1d.weight.squeeze(1) # [conv_dim, kernel]
def _write_track_snapshot(self, pool, li, conv_in, h, fla) -> None:
"""Hybrid-radix: snapshot recurrent + conv state at the chunk-aligned track
boundary into a donatable pool slot (same contract as GDN, see
layers/linear_attn/gdn.py). h rows are the kernel's per-chunk [V, K] states --
a direct copy into the pool's [K, V] slots (D_k == D_v)."""
rec = pool.recurrent_states[li]
rec.index_copy_(0, fla.track_dst, h[0, fla.track_h_row].to(rec.dtype))
cv = pool.conv_states[li]
conv_win = conv_in[fla.track_conv_src].transpose(-1, -2).contiguous()
cv.index_copy_(0, fla.track_dst, conv_win.to(cv.dtype))
@nvtx_annotate("KDA")
def forward(self, hidden_states: torch.Tensor) -> torch.Tensor:
ctx = get_global_ctx()
batch = ctx.batch
pool = ctx.linear_state_pool
total = hidden_states.shape[0]
dtype = hidden_states.dtype
h, d, p = self.num_heads, self.head_dim, self.proj_size
fla = batch.fla_metadata
if fla is None:
from freetoken.attention.linear import build_fla_metadata
fla = build_fla_metadata(batch, hidden_states.device)
batch.fla_metadata = fla
proj = self.in_proj.forward(hidden_states)
conv_in, b, f_a, g_a = torch.split(
proj, [self.conv_dim, h, d, d], dim=-1
)
g1 = self.f_b_proj.forward(f_a) # raw forget-gate logits [T, H*D]
g2 = self.g_b_proj.forward(g_a) # output-gate logits [T, H*D]
li = pool.local_index(self.layer_id)
if batch.is_decode:
mixed = causal_conv1d_decode(
conv_in, pool.conv_states[li], self._conv_weight(), fla.cache_indices
)
bsz = mixed.shape[0]
q, k, v = (
t.reshape(1, bsz, h, d).to(dtype)
for t in torch.split(mixed, [p, p, p], dim=-1)
)
core_out, _ = _fused_recurrent(
q, k, v,
g=g1.view(1, bsz, h, d),
beta=b.view(1, bsz, h),
state_pool=pool.recurrent_states[li],
indices=fla.cache_indices,
cu_seqlens=fla.cu_seqlens,
a_log=self.A_log,
dt_bias=self.dt_bias,
lower_bound=self.lower_bound,
scale=self.scale,
)
else:
x = conv_in.transpose(0, 1).contiguous() # [conv_dim, total]
mixed = causal_conv1d_varlen(
x, self._conv_weight(), pool.conv_states[li],
fla.cu_seqlens, fla.cache_indices, fla.has_initial_state,
).transpose(0, 1)
q, k, v = (
t.reshape(1, total, h, d).to(dtype)
for t in torch.split(mixed, [p, p, p], dim=-1)
)
# Fresh sequences start from a zeroed slot; then gather every request's
# initial state (the chunk kernel takes it dense, [N, H, D, D]).
rec = pool.recurrent_states[li]
if fla.fresh_state_indices is not None:
rec.index_fill_(0, fla.fresh_state_indices, 0.0)
slot_ids = fla.cache_indices.long()
initial = rec.index_select(0, slot_ids)
from freetoken.kernel.fla import chunk_kda_with_fused_gate
track = fla.track_dst is not None
result = chunk_kda_with_fused_gate(
q=q, k=k, v=v, # NOTE: v (ephemeral conv output) is clobbered
raw_g=g1.view(1, total, h, d),
beta=b.float().sigmoid().view(1, total, h),
A_log=self.A_log,
g_bias=self.dt_bias,
scale=self.scale,
initial_state=initial,
output_final_state=True,
use_qk_l2norm_in_kernel=True,
cu_seqlens=fla.cu_seqlens,
safe_gate=True,
lower_bound=self.lower_bound,
return_h=track,
)
if track:
core_out, final_state, chunk_h = result
self._write_track_snapshot(pool, li, conv_in, chunk_h, fla)
else:
core_out, final_state = result
rec.index_copy_(0, slot_ids, final_state.to(rec.dtype))
core_out = core_out.reshape(-1, d)
out = self.o_norm.forward(core_out, g2.reshape(-1, d)).reshape(total, -1)
return self.o_proj.forward(out.to(dtype))
def _fused_recurrent(
q, k, v, g, beta, state_pool, indices, cu_seqlens,
a_log, dt_bias, lower_bound, scale,
):
"""Decode via the vendored recurrent kernel: gate + beta-sigmoid + q/k l2norm
in-kernel, state read/written in place at ``indices`` (int32, 1 token/req)."""
from freetoken.kernel.fla import fused_recurrent_kda
return fused_recurrent_kda(
q=q, k=k, v=v, g=g, beta=beta,
scale=scale,
initial_state=state_pool,
use_qk_l2norm_in_kernel=True,
cu_seqlens=cu_seqlens,
ssm_state_indices=indices,
sigmoid_beta=True,
a_log=a_log,
g_bias=dt_bias,
compute_gate=True,
lower_bound=lower_bound,
)
__all__ = ["Glm5NextKDA"]