Skip to content

FreeToken brings datacenter-scale model serving to your desktop. Run massive models locally, fast and efficiently.

feat(glm5_next): support GLM-5.3-Flash (#332)

Xiaoze Fana2538a428baa
diff --git a/docs/models.md b/docs/models.md
index 41b79cd6..c9499163 100644
--- a/docs/models.md
+++ b/docs/models.md
@@ -7,6 +7,7 @@ for them; other checkpoints of the same architectures work too.
| Model | HF checkpoints |
|---|---|
| DeepSeek-V4 | [deepseek-ai/DeepSeek-V4-Flash-0731](https://huggingface.co/deepseek-ai/DeepSeek-V4-Flash-0731) |
+| GLM-5.3-Flash | [RedHatAI/GLM-5.3-Flash-NVFP4](https://huggingface.co/RedHatAI/GLM-5.3-Flash-NVFP4) |
| GLM-5.2 | [nvidia/GLM-5.2-NVFP4](https://huggingface.co/nvidia/GLM-5.2-NVFP4) |
| GLM-4.7 | [nvidia/GLM-4.7-NVFP4](https://huggingface.co/nvidia/GLM-4.7-NVFP4) |
| Qwen3.8-Flash-Next | [Qwen/Qwen3.8-Flash-Next-FP8](https://huggingface.co/Qwen/Qwen3.8-Flash-Next-FP8), [RadixArk/Qwen3.8-Flash-Next-NVFP4](https://huggingface.co/RadixArk/Qwen3.8-Flash-Next-NVFP4) |
diff --git a/python/freetoken/attention/__init__.py b/python/freetoken/attention/__init__.py
index 72b01c04..8a410ffe 100644
--- a/python/freetoken/attention/__init__.py
+++ b/python/freetoken/attention/__init__.py
@@ -107,6 +107,11 @@ def create_dsv4_sparse_backend(config: ModelConfig):
BackendInfo(supported_types=frozenset({AttnType.MLA, AttnType.DSA})),
)
def create_dsa_backend(config: ModelConfig):
+ # MLA with a grouped index (index_ratio > 1) is the kpool indexer layout.
+ if any(s.mla and s.index_ratio > 1 for s in config.kv_cache_group_specs()):
+ from .dsa_indexer_kpool import Glm5NextDSABackend
+
+ return Glm5NextDSABackend(config)
from .dsa import DSAAttnBackend
return DSAAttnBackend(config)
diff --git a/python/freetoken/attention/dsa.py b/python/freetoken/attention/dsa.py
index ff4fa7ca..398d1700 100644
--- a/python/freetoken/attention/dsa.py
+++ b/python/freetoken/attention/dsa.py
@@ -31,7 +31,7 @@ split does not fit the generic ``forward(q, k, v)`` contract).
from __future__ import annotations
from dataclasses import dataclass, field
-from typing import TYPE_CHECKING, Dict, List, Tuple
+from typing import TYPE_CHECKING, Dict, List, Tuple, NamedTuple
import torch
from freetoken.core import Batch, get_global_ctx
@@ -50,6 +50,32 @@ _PREFILL_SCORE_BYTES = 128 << 20
_PREFILL_SCORE_CHUNK = 512
+class KpoolPlan(NamedTuple):
+ """glm5_next kpool: one forward's slab/ring routing (layer-invariant; the
+ first DSA layer plans, the rest reuse -- Glm5NextDSABackend._plan_kpool_writes)."""
+
+ cmp_rows: torch.Tensor # [T] shadow row (closing) or scratch row
+ ring_rows: torch.Tensor # [T] tail-ring row, -1 = masked off
+ ring_slots: torch.Tensor # [n_req] Req.table_idx
+ token_to_req: torch.Tensor # [T]
+ cu_seqlens: torch.Tensor # [n_req + 1]
+
+
+@dataclass(frozen=True)
+class DSAIndexerInputs:
+ """Per-forward indexer tensors, built by the MODEL's indexer module and passed
+ per call (QSAIndexerInputs precedent): the backend holds no model parameters.
+ ``gate``/``ape`` are the glm5_next kpool extras (None for GLM-5.2): the raw
+ per-channel compression gate scores and the [kpool, Di] APE weight."""
+ # fmt: off
+ q: torch.Tensor # [T, Hi, Di]
+ k: torch.Tensor # [T, Di]
+ w: torch.Tensor # [T, Hi] fp32
+ gate: torch.Tensor | None = None
+ ape: torch.Tensor | None = None
+ # fmt: on
+
+
@dataclass
class DSAMetadata(BaseAttnMetadata):
# fmt: off
@@ -63,6 +89,11 @@ class DSAMetadata(BaseAttnMetadata):
kvlen: torch.Tensor | None = None
# group leader layer -> (sel_rows, counts); only the LIVE leader is retained
sel: dict = field(default_factory=dict)
+ # glm5_next kpool decode: per-request tail-ring slots (Req.table_idx). Under CUDA
+ # graphs this is a view of the backend's STATIC buffer (restaged per replay by
+ # _stage_decode, QSA ring_slots precedent); eager decode reads active_table_idx.
+ ring_slots: torch.Tensor | None = None
+ kpool_plan: "KpoolPlan | None" = None
# fmt: on
def get_last_indices(self, bs: int) -> torch.Tensor:
@@ -73,8 +104,7 @@ class DSAAttnBackend(DSAIndexerMixin, BaseAttnBackend):
def __init__(self, config: ModelConfig) -> None:
from freetoken.kvcache.dsa_pool import DSAKVCache, MLAKVCache
- args = config.glm_dsa_args
- assert args is not None, "dsa backend needs ModelConfig.glm_dsa_args (MLA dims)"
+ args = self._model_args(config)
self.config = config
self.num_heads = config.num_qo_heads
self.kv_lora_rank = args.kv_lora_rank
@@ -99,15 +129,7 @@ class DSAAttnBackend(DSAIndexerMixin, BaseAttnBackend):
self._leader: Dict[int, int] = {}
self._idx_slot: Dict[int, int] = {}
if self.dsa_enabled:
- lead = None
- # Capped to the SERVED layer count (dev num_layers overrides must not
- # index slots past the pool the factory sized from the same cap).
- for lid, kind in enumerate(args.indexer_types[: config.num_layers]):
- if kind == "full":
- lead = lid
- self._idx_slot[lid] = len(self._idx_slot)
- assert lead is not None, "indexer_types must start with a 'full' layer"
- self._leader[lid] = lead
+ self._build_index_slots(args, config)
# decode staging (static buffers under CUDA graphs; eager decode builds
# per-forward tensors in prepare_metadata instead)
self._rows_buf: torch.Tensor | None = None
@@ -115,6 +137,32 @@ class DSAAttnBackend(DSAIndexerMixin, BaseAttnBackend):
self.max_seq_len = 0
self.capture_bs: List[int] = []
+ # ----- model-family hooks (overridden by the glm5_next kpool backend) ---------------
+ def _model_args(self, config: ModelConfig):
+ """The MLA/indexer dims payload. GLM-5.2 reads glm_dsa_args; glm5_next's
+ subclass reads glm5_args (same duck-typed fields)."""
+ args = config.glm_dsa_args
+ assert args is not None, "dsa backend needs ModelConfig.glm_dsa_args (MLA dims)"
+ return args
+
+ def _build_index_slots(self, args, config: ModelConfig) -> None:
+ """IndexShare (GLM-5.2): "full" layers own an indexer slot; followers reuse
+ the most recent leader's selection."""
+ lead = None
+ # Capped to the SERVED layer count (dev num_layers overrides must not
+ # index slots past the pool the factory sized from the same cap).
+ for lid, kind in enumerate(args.indexer_types[: config.num_layers]):
+ if kind == "full":
+ lead = lid
+ self._idx_slot[lid] = len(self._idx_slot)
+ assert lead is not None, "indexer_types must start with a 'full' layer"
+ self._leader[lid] = lead
+
+ def _store_index(self, inputs: "DSAIndexerInputs", batch: Batch, layer_id: int) -> None:
+ """Scatter this forward's index keys (kpool subclass adds gate scores and
+ pool-completion compression)."""
+ self.kvcache.store_index_k(inputs.k, batch.out_loc, self._idx_slot[layer_id])
+
def forward(self, q, k, v, layer_id, batch, attn_spec: AttentionSpec | None = None):
raise NotImplementedError("MLA models use mla_forward(), not forward().")
@@ -156,14 +204,14 @@ class DSAAttnBackend(DSAIndexerMixin, BaseAttnBackend):
)
def mla_forward(
- self, q_nope, q_pe, c_kv, k_rope, layer_id, batch, indexer_qkw=None
+ self, q_nope, q_pe, c_kv, k_rope, layer_id, batch, indexer_inputs=None
) -> torch.Tensor:
"""Store this forward's latent rows and attend over the paged latent history.
``q_nope`` [T, H, kv_lora_rank] (kv_b-absorbed), ``q_pe`` [T, H, rope_dim],
``c_kv`` [T, kv_lora_rank] / ``k_rope`` [T, rope_dim] (the pool scatters the
- two latent halves). ``indexer_qkw`` = (q [T, Hi, Di], k [T, Di], w [T, Hi])
- on full-indexer layers, None on shared layers. Returns [T, H, kv_lora_rank].
+ two latent halves). ``indexer_inputs`` is a :class:`DSAIndexerInputs` on
+ full-indexer layers, None on shared layers. Returns [T, H, kv_lora_rank].
"""
md = batch.attn_metadata
assert isinstance(md, DSAMetadata)
@@ -174,17 +222,17 @@ class DSAAttnBackend(DSAIndexerMixin, BaseAttnBackend):
md.rows = self._decode_rows(batch).to(torch.int32)
md.kvlen = md.kv_len_cpu.to(self.device, non_blocking=True)
self.kvcache.store_kv(c_kv, k_rope, batch.out_loc, layer_id)
- if self.dsa_enabled and indexer_qkw is not None:
+ if self.dsa_enabled and indexer_inputs is not None:
# Scatter index keys unconditionally: short prefills serve through the
# identity path TODAY, but their keys must exist once decode passes topk.
- self.kvcache.store_index_k(indexer_qkw[1], batch.out_loc, self._idx_slot[layer_id])
+ self._store_index(indexer_inputs, batch, layer_id)
if md.is_decode:
- return self._decode(md, layer_id, q_nope, q_pe, indexer_qkw)
- return self._prefill(md, layer_id, q_nope, q_pe, batch, indexer_qkw)
+ return self._decode(md, layer_id, q_nope, q_pe, indexer_inputs)
+ return self._prefill(md, layer_id, q_nope, q_pe, batch, indexer_inputs)
# ----- decode (CUDA-graph capturable, single code path) -----------------------------
- def _decode(self, md, layer_id, q_nope, q_pe, indexer_qkw) -> torch.Tensor:
+ def _decode(self, md, layer_id, q_nope, q_pe, inputs) -> torch.Tensor:
bs = q_nope.shape[0]
rows, kvlen = md.rows, md.kvlen
if not self.dsa_enabled:
@@ -192,8 +240,8 @@ class DSAAttnBackend(DSAIndexerMixin, BaseAttnBackend):
# whole row list, bounded by the device-side live length.
sel, cnt = rows.view(bs, 1, -1), kvlen.view(bs, 1)
else:
- if indexer_qkw is not None:
- q_idx, _, w = indexer_qkw
+ if inputs is not None:
+ q_idx, w = inputs.q, inputs.w
s = self.dsa_decode_scores(q_idx, w, self._idx_slot[layer_id], rows, kvlen)
k_sel = min(self.index_topk, s.shape[-1])
picks = self.indexer_select_decode(
@@ -212,15 +260,16 @@ class DSAAttnBackend(DSAIndexerMixin, BaseAttnBackend):
# ----- prefill / extend (eager) ------------------------------------------------------
def _select_prefill(
self, slot: int, q_idx: torch.Tensor, w: torch.Tensor,
- rows: torch.Tensor, positions: torch.Tensor,
+ rows: torch.Tensor, positions: torch.Tensor, start_pos: int,
) -> Tuple[torch.Tensor, torch.Tensor]:
- """Per-request causal top-k: ([1, m, K] physical rows, [1, m] counts)."""
+ """Per-request causal top-k: ([1, m, K] physical rows, [1, m] counts).
+ ``start_pos`` is the request's first query position -- a host int
+ (cached_len), so no device->host sync on the prefill path."""
kv_len = rows.numel()
k_all = self.kvcache.index_k_cache(slot).index_select(0, rows.long())
k_sel = min(self.index_topk, kv_len)
m = q_idx.shape[0]
sel = torch.empty(m, k_sel, dtype=torch.int32, device=self.device)
- start_pos = int(positions[0])
# Bound the fp32 [chunk, kv_len] logits transient (worst case is capped by the
# model's max_position: floor 16 x 1M x 4 B = 64 MB, see _PREFILL_SCORE_BYTES).
chunk = max(16, min(_PREFILL_SCORE_CHUNK, _PREFILL_SCORE_BYTES // max(kv_len * 4, 1)))
@@ -236,15 +285,16 @@ class DSAAttnBackend(DSAIndexerMixin, BaseAttnBackend):
cnt = torch.clamp(positions + 1, max=k_sel).to(torch.int32)
return sel.view(1, m, k_sel), cnt.view(1, m)
- def _prefill(self, md, layer_id, q_nope, q_pe, batch, indexer_qkw) -> torch.Tensor:
+ def _prefill(self, md, layer_id, q_nope, q_pe, batch, inputs) -> torch.Tensor:
t = q_nope.shape[0]
q_cat = torch.cat([q_nope, q_pe], dim=-1) # [T, H, 576]
reqs = batch.padded_reqs if hasattr(batch, "padded_reqs") else batch.reqs
page_table = get_global_ctx().page_table
qo = md.qo_indptr_cpu.tolist()
- sparse = self.dsa_enabled and int(md.kv_len_cpu.max()) > self.index_topk
- if sparse and indexer_qkw is not None:
- q_idx, _, w = indexer_qkw
+ kv_lens = md.kv_len_cpu.tolist() # one D2H for both the gate and start_pos
+ sparse = self.dsa_enabled and max(kv_lens) > self.index_topk
+ if sparse and inputs is not None:
+ q_idx, w = inputs.q, inputs.w
md.sel.clear() # one live group leader at a time
md.sel[layer_id] = [
self._select_prefill(
@@ -252,6 +302,7 @@ class DSAAttnBackend(DSAIndexerMixin, BaseAttnBackend):
q_idx[qo[i] : qo[i + 1]], w[qo[i] : qo[i + 1]],
page_table[r.table_idx, : r.device_len],
batch.positions[qo[i] : qo[i + 1]],
+ kv_lens[i] - (qo[i + 1] - qo[i]), # cached_len == first position
)
for i, r in enumerate(reqs)
]
diff --git a/python/freetoken/attention/dsa_indexer_kpool.py b/python/freetoken/attention/dsa_indexer_kpool.py
new file mode 100644
index 00000000..6743a143
--- /dev/null
+++ b/python/freetoken/attention/dsa_indexer_kpool.py
@@ -0,0 +1,288 @@
+"""glm5_next (GLM-5.3-Flash) kpool DSA backend: pooled-indexer addressing.
+
+Extends the GLM-5.2 DSA backend with the kpool compression scheme: the indexer K
+cache is scored at POOL granularity -- every ``index_kpool`` (4) consecutive
+tokens fold into one entry, a per-channel ``softmax(gate + APE)``-weighted sum of
+their raw keys -- so indexer compute and top-k shrink by 4x. Selection picks
+``index_topk // kpool`` pools, each pool expands back to its constituent token
+rows, and the request's trailing incomplete pool ("tail", up to kpool-1 tokens)
+is force-included (``index_kpool_always_select_tail``). Selection widths are
+therefore ``(kpool - 1) + select_k * kpool`` with ``-1`` gather-only sentinels;
+the sparse-MLA kernel masks them.
+
+Slab convention (KpoolDSAKVCache): the index slab is a 1/kpool SHADOW of the KV
+pages -- a pool's entry lives at row ``token_slot // kpool`` (well-defined because
+the engine pins ``page_size % kpool == 0``, so a pool never straddles a page).
+Raw keys + gate scores of the in-progress pool live in per-request tail rings
+written at ``pos % kpool``; a pool that closes reads its older members from the
+ring (so a chunk may start mid-pool), and a decode step whose pool does NOT close
+scatters its (garbage) pooled candidate into the request's scratch row -- an
+unconditional, CUDA-graph-safe write that scoring never reads (QSA precedent).
+
+Faithfulness: pooled entries are stored bf16 (the reference Hadamard-rotates and
+fp8-quantizes them -- a memory device whose rotation cancels in the score); the
+pooling softmax runs fp32, matching the reference kernel. Selection is a plain
+top-k over pool scores plus the tail: the reference does NOT force-include the
+query's own (complete) pool -- the model is trained with that scheme, so neither
+do we. Sharing note: IndexShare does not exist here; every DSA layer owns its
+indexer slot and is its own leader.
+"""
+
+from __future__ import annotations
+
+from typing import TYPE_CHECKING, Tuple
+
+import torch
+
+from .dsa import DSAAttnBackend, DSAMetadata, KpoolPlan
+
+if TYPE_CHECKING:
+ from freetoken.core import Batch
+ from freetoken.models import ModelConfig
+
+
+class Glm5NextDSABackend(DSAAttnBackend):
+ def __init__(self, config: "ModelConfig") -> None:
+ super().__init__(config)
+ self._slots_buf: torch.Tensor | None = None # static graph buffer for ring_slots
+ if self.dsa_enabled:
+ from freetoken.kvcache.dsa_pool import KpoolDSAKVCache
+
+ assert isinstance(self.kvcache, KpoolDSAKVCache), (
+ "glm5_next kpool indexer needs the KpoolDSAKVCache (tail rings); "
+ f"the pool factory built {type(self.kvcache).__name__}"
+ )
+
+ # ----- CUDA-graph decode staging ------------------------------------------------------
+ # The tail rings are keyed by Req.table_idx; a captured decode step must read it
+ # from a static buffer restaged per replay (rows/kvlen precedent in the parent).
+ def init_capture_graph(self, max_seq_len: int, bs_list) -> None:
+ super().init_capture_graph(max_seq_len, bs_list)
+ self._slots_buf = torch.zeros(max(bs_list), dtype=torch.int64, device=self.device)
+
+ def _stage_decode(self, batch: "Batch", bs: int, table_idx: torch.Tensor) -> None:
+ super()._stage_decode(batch, bs, table_idx)
+ self._slots_buf[:bs].copy_(table_idx)
+ batch.attn_metadata.ring_slots = self._slots_buf[:bs]
+
+ def reset_capture(self) -> None:
+ super().reset_capture()
+ self._slots_buf = None
+
+ # ----- model-family hooks -----------------------------------------------------------
+ def _model_args(self, config: "ModelConfig"):
+ args = config.glm5_args
+ assert args is not None, "kpool dsa backend needs ModelConfig.glm5_args"
+ self.kpool = args.index_kpool
+ assert self.kpool > 1 and args.index_kpool_compress, (
+ "Glm5NextDSABackend serves the kpool-compressed indexer; a kpool=1 "
+ "checkpoint should run the plain DSA backend"
+ )
+ assert args.index_topk % self.kpool == 0
+ self.select_k = args.index_topk // self.kpool
+ assert args.index_kpool_always_select_tail, (
+ "tail force-inclusion is baked into the selection layout"
+ )
+ return args
+
+ def _build_index_slots(self, args, config: "ModelConfig") -> None:
+ # No IndexShare: every DSA layer owns its indexer and is its own leader.
+ for lid in args.dsa_layer_ids:
+ if lid >= config.num_layers:
+ continue
+ self._idx_slot[lid] = len(self._idx_slot)
+ self._leader[lid] = lid
+
+ # ----- store: fused single path (prefill AND decode, CUDA-graph capturable) ----------
+ def _plan_kpool_writes(self, md, batch: "Batch", slot: int):
+ """Per-token slab/ring routing for this forward; layer-invariant, so the
+ first DSA layer computes it and the rest reuse it (QSA _plan_index_writes
+ shape). Pure device arithmetic: no host sync, graph-capturable.
+
+ Rebuilt at the FIRST indexer slot of every forward, never trusted across
+ forwards: a capture batch runs its warmup and its capture through ONE
+ metadata object, and a cached plan would bake the warmup's (non-graph-pool)
+ tensor addresses into the graph (QSA precedent)."""
+ if slot != 0 and md.kpool_plan is not None:
+ return md.kpool_plan
+ kp = self.kpool
+ out_loc = batch.out_loc.to(torch.int64)
+ positions = batch.positions.to(torch.int64)
+ t = out_loc.numel()
+ if md.is_decode:
+ # One token per request. ring_slots is the backend's STATIC buffer under
+ # graphs (restaged per replay in _stage_decode); eager reads the
+ # scheduler-staged active_table_idx. arange shapes are fixed per capture.
+ ring_slots = (
+ md.ring_slots if md.ring_slots is not None else batch.active_table_idx
+ ).to(torch.int64)
+ token_to_req = torch.arange(t, device=self.device, dtype=torch.int32)
+ cu_seqlens = torch.arange(t + 1, device=self.device, dtype=torch.int32)
+ else:
+ reqs = batch.padded_reqs if hasattr(batch, "padded_reqs") else batch.reqs
+ ring_slots = torch.tensor(
+ [r.table_idx for r in reqs], dtype=torch.int64, pin_memory=True
+ ).to(self.device, non_blocking=True)
+ cu_cpu = md.qo_indptr_cpu
+ cu_seqlens = cu_cpu.to(self.device, non_blocking=True)
+ token_to_req = torch.repeat_interleave(
+ torch.arange(len(reqs), dtype=torch.int32),
+ (cu_cpu[1:] - cu_cpu[:-1]).to(torch.int64),
+ ).pin_memory().to(self.device, non_blocking=True)
+ slots = ring_slots.index_select(0, token_to_req.to(torch.int64))
+ # page_size % kpool == 0, so out_loc % kp == position % kp: a group closes
+ # exactly on position % kp == kp - 1. Non-closing rows land in the request's
+ # scratch row (never scored).
+ closing = positions % kp == kp - 1
+ cmp_rows = torch.where(
+ closing, out_loc // kp, self.kvcache.cmp_scratch_base + slots
+ ).to(torch.int32)
+ # Ring refresh: only each request's last kp rows survive to the next forward
+ # (one keeper per pos%kp residue -- deterministic, no write races).
+ rows = torch.arange(t, device=self.device, dtype=torch.int64)
+ ends = cu_seqlens.to(torch.int64).index_select(0, token_to_req.to(torch.int64) + 1)
+ keep = rows >= ends - kp
+ ring_row = slots * kp + positions % kp
+ ring_rows = torch.where(keep, ring_row, torch.full_like(ring_row, -1)).to(
+ torch.int32
+ )
+ md.kpool_plan = KpoolPlan(cmp_rows, ring_rows, ring_slots, token_to_req, cu_seqlens)
+ return md.kpool_plan
+
+ def _store_index(self, inputs, batch: "Batch", layer_id: int) -> None:
+ """One fused kernel serves prefill and decode (layout and member
+ resolution: module docstring); the ring refresh follows the read."""
+ from freetoken.kernel.triton.kpool_compress import kpool_compress_store
+ from freetoken.kernel.triton.qsa import qsa_store_rows
+
+ md = batch.attn_metadata
+ assert isinstance(md, DSAMetadata)
+ assert inputs.gate is not None and inputs.ape is not None, (
+ "kpool store needs DSAIndexerInputs.gate/ape from the model's indexer"
+ )
+ k, gate = inputs.k, inputs.gate.to(inputs.k.dtype)
+ slot = self._idx_slot[layer_id]
+ tail_k, tail_g = self.kvcache.tail_k(slot), self.kvcache.tail_gate(slot)
+ plan = self._plan_kpool_writes(md, batch, slot)
+ kpool_compress_store(
+ k, gate,
+ tail_k.view(-1, k.shape[-1]), tail_g.view(-1, k.shape[-1]),
+ inputs.ape,
+ plan.ring_slots, plan.token_to_req, plan.cu_seqlens, batch.positions,
+ self.kvcache.index_k_cache(slot), plan.cmp_rows,
+ self.kpool,
+ )
+ # After the compression read: the ring rows this forward overwrites are
+ # exactly the ones a straddling group just consumed.
+ qsa_store_rows(tail_k, plan.ring_rows, k)
+ qsa_store_rows(tail_g, plan.ring_rows, gate)
+
+ # ----- selection: pools -> token rows + tail ------------------------------------------
+ def _expand_and_tail(
+ self,
+ picks: torch.Tensor, # [B, m, select_k] pool ids, -1 sentinel
+ rows: torch.Tensor, # [B, W] or [B, m, W] position-ordered physical rows
+ q_pos: torch.Tensor, # [B, m] query positions (token-granular)
+ ) -> Tuple[torch.Tensor, torch.Tensor]:
+ """Selected pools -> token rows, tail tokens appended FIRST (fixed kpool-1
+ slots, -1 padded). Returns (sel [B, m, (kpool-1) + select_k*kpool] int32,
+ cnt [B, m] int32)."""
+ kp = self.kpool
+ b, m, k_sel = picks.shape
+ device = picks.device
+ offs = torch.arange(kp, device=device)
+
+ # History: pool pick p -> token positions p*kp + [0, kp).
+ hist_pos = picks.unsqueeze(-1) * kp + offs # [B, m, k_sel, kp]
+ hist_pos = torch.where(
+ picks.unsqueeze(-1) < 0, hist_pos.new_full((), -1), hist_pos
+ ).view(b, m, k_sel * kp)
+
+ # Tail: positions [n_pools*kp, q_pos] of each query's own request/step.
+ n_pools = (q_pos + 1) // kp # complete pools at this query
+ tail_start = n_pools * kp
+ toffs = torch.arange(kp - 1, device=device)
+ tail_pos = tail_start.unsqueeze(-1) + toffs # [B, m, kp-1]
+ tail_pos = torch.where(tail_pos <= q_pos.unsqueeze(-1), tail_pos, tail_pos.new_full((), -1))
+
+ pos = torch.cat([tail_pos, hist_pos], dim=-1) # [B, m, width]
+ rows_b = rows if rows.dim() == 3 else rows.unsqueeze(1).expand(b, m, -1)
+ sel = rows_b.gather(-1, pos.clamp_min(0).long()).to(torch.int32)
+ sel = torch.where(pos < 0, sel.new_full((), -1), sel)
+ # Valid entries: the fixed tail slots + all expanded picked pools. -1
+ # sentinels inside the bound are masked by the sparse kernel.
+ # clamp_max (not minimum(new_tensor)): no H2D copy, CUDA-graph safe.
+ cnt = ((kp - 1) + n_pools.clamp_max(k_sel) * kp).to(torch.int32)
+ return sel, cnt
+
+ def _decode(self, md, layer_id, q_nope, q_pe, inputs) -> torch.Tensor:
+ bs = q_nope.shape[0]
+ rows, kvlen = md.rows, md.kvlen
+ if not self.dsa_enabled:
+ return super()._decode(md, layer_id, q_nope, q_pe, inputs)
+ if inputs is not None:
+ q_idx, w = inputs.q, inputs.w
+ kp = self.kpool
+ n_pools = (kvlen // kp).to(torch.int32)
+ # Pool p's shadow row = any member's token slot // kp (pools never
+ # straddle pages): stride the position-ordered row snapshot down to
+ # pool granularity, then divide into the shadow slab.
+ pool_rows = (rows[:, kp - 1 :: kp] // kp).contiguous()
+ s = self.dsa_decode_scores(q_idx, w, self._idx_slot[layer_id], pool_rows, n_pools)
+ k_sel = min(self.select_k, s.shape[-1])
+ picks = self.indexer_select_decode(
+ s.view(bs, 1, -1), valid=n_pools, topk=k_sel, offset=0
+ ) # [bs, 1, k_sel] pool ids
+ sel, cnt = self._expand_and_tail(
+ picks.transpose(0, 1), # [1, bs, k_sel]
+ rows.unsqueeze(0), # [1, bs, W]
+ (kvlen - 1).view(1, bs),
+ )
+ sel = sel.transpose(0, 1) # [bs, 1, width]
+ cnt = cnt.view(bs, 1)
+ md.sel.clear()
+ md.sel[layer_id] = (sel, cnt)
+ sel, cnt = md.sel[self._leader[layer_id]]
+ q_cat = torch.cat([q_nope, q_pe], dim=-1).view(bs, 1, self.num_heads, self.latent_dim)
+ o = self._attend(q_cat, layer_id, sel, cnt)
+ return o.view(bs, self.num_heads, self.kv_lora_rank)
+
+ def _select_prefill(
+ self, slot: int, q_idx: torch.Tensor, w: torch.Tensor,
+ rows: torch.Tensor, positions: torch.Tensor, start_pos: int,
+ ) -> Tuple[torch.Tensor, torch.Tensor]:
+ """Per-request causal top-k at pool granularity, expanded to token rows.
+ ``start_pos`` is the request's cached_len (host int, no device sync)."""
+ kp = self.kpool
+ kv_len = rows.numel()
+ n_pools_total = kv_len // kp
+ pool_rows = rows[kp - 1 :: kp] // kp # shadow rows (token slot // kpool)
+ k_pool = self.kvcache.index_k_cache(slot).index_select(0, pool_rows.long())
+ k_sel = min(self.select_k, max(n_pools_total, 1))
+ m = q_idx.shape[0]
+ width = (kp - 1) + k_sel * kp
+ if n_pools_total == 0:
+ # No complete pool to score (a sub-kpool request riding a sparse batch):
+ # the selection is the tail alone (select_prefill would return zero pick
+ # columns and break the fixed width below).
+ picks = torch.full((1, m, k_sel), -1, dtype=torch.int32, device=self.device)
+ return self._expand_and_tail(picks, rows.view(1, -1), positions.view(1, -1))
+ sel = torch.empty(m, width, dtype=torch.int32, device=self.device)
+ cnt = torch.empty(m, dtype=torch.int32, device=self.device)
+ chunk = 512
+ for s0 in range(0, m, chunk):
+ s1 = min(s0 + chunk, m)
+ scores = self.dsa_prefill_logits(q_idx[s0:s1], k_pool, w[s0:s1])
+ picks = self.indexer_select_prefill(
+ scores.unsqueeze(0), start_pos=start_pos + s0, seqlen=s1 - s0,
+ ratio=kp, topk=k_sel, offset=0,
+ ) # [1, s1-s0, k_sel] pool ids
+ sel_c, cnt_c = self._expand_and_tail(
+ picks, rows.view(1, -1), positions[s0:s1].view(1, -1)
+ )
+ sel[s0:s1] = sel_c[0]
+ cnt[s0:s1] = cnt_c[0]
+ return sel.view(1, m, width), cnt.view(1, m)
+
+
+__all__ = ["Glm5NextDSABackend"]
diff --git a/python/freetoken/engine/engine.py b/python/freetoken/engine/engine.py
index 33b0d549..b5a6fa3b 100644
--- a/python/freetoken/engine/engine.py
+++ b/python/freetoken/engine/engine.py
@@ -216,13 +216,19 @@ def _validate_attention_backend_choice(config, override, required: frozenset[Att
"Use --attention-backend fi (or triton) instead."
)
- if required & {AttnType.MLA, AttnType.DSA} and config.page_size != 1:
- # The MLA backend's row addressing (latent scatter, DSA index keys, sparse
- # top-k page indices) assumes page_size == 1 throughout; reject explicitly
- # like the SWA models do rather than corrupting addressing silently.
- raise ValueError(
- f"latent-KV MLA models require --page-size 1, got {config.page_size}."
+ if required & {AttnType.MLA, AttnType.DSA}:
+ # Plain MLA/DSA runs on page_size 1; the kpool indexer layout needs 64.
+ _kpool_ratio = max(
+ (s.index_ratio for s in model_config.kv_cache_group_specs() if s.mla),
+ default=1,
)
+ want_page = 64 if _kpool_ratio > 1 else 1
+ if config.page_size != want_page:
+ logger.warning_rank0(
+ f"Page size {config.page_size} is auto-adjusted to {want_page} "
+ f"for latent-KV attention."
+ )
+ override("page_size", want_page)
for part in backend_parts:
info = attention_backend_info(part)
@@ -1135,6 +1141,7 @@ def _resolve_cpu_layers(config: EngineConfig, num_moe_layers: int) -> frozenset[
# expert activations the CPU MoE executor supports (csrc ActKind)
_CPU_MOE_ACTS = (
"silu", "swish", "gelu", "gelu_tanh", "gelu_pytorch_tanh", "swigluoai",
+ "swiglu_clamp",
)
diff --git a/python/freetoken/kernel/aot_models.py b/python/freetoken/kernel/aot_models.py
index 6268ae33..0ce249de 100644
--- a/python/freetoken/kernel/aot_models.py
+++ b/python/freetoken/kernel/aot_models.py
@@ -274,6 +274,20 @@ SUPPORTED_MODELS: tuple[AotModel, ...] = (
moe_intermediate_size=2048,
expert_formats=_NVFP4_FORMATS,
),
+ AotModel(
+ # GLM-5.3-Flash: hybrid KDA + NoPE-MLA/DSA (kpool indexer). Latent writes
+ # go through torch scatter like GLM-5.2 (no paged-KV store groups); the
+ # KDA conv/recurrent state lives in the LinearStatePool, not paged KV.
+ name="RedHatAI/GLM-5.3-Flash-NVFP4",
+ architecture="Glm5NextForCausalLM",
+ arch_aliases=("Glm5NextForConditionalGeneration",),
+ hidden_size=4096,
+ kv_groups=(),
+ top_k=8,
+ moe_intermediate_size=2048,
+ expert_formats=_NVFP4_FORMATS,
+ aliases=("zai-org/GLM-5.3-Flash", "LibertAIDAI/GLM-5.3-Flash-NVFP4"),
+ ),
AotModel(
# MiniMaxAI/MiniMax-M2.5 ships block-fp8, which has no expert-bank
# provider for this arch on main -- the NVFP4 release is the servable
diff --git a/python/freetoken/kernel/csrc/cpu_moe/cpu_moe_ext.cpp b/python/freetoken/kernel/csrc/cpu_moe/cpu_moe_ext.cpp
index 880e8637..8349123b 100644
--- a/python/freetoken/kernel/csrc/cpu_moe/cpu_moe_ext.cpp
+++ b/python/freetoken/kernel/csrc/cpu_moe/cpu_moe_ext.cpp
@@ -71,7 +71,15 @@ inline bf16_t f32_to_bf16(float f) {
// MiniMax-M3): gate/up are combined jointly with the runtime alpha/limit
// scalars, so it is handled in the do_pass1 epilogue (act_apply never sees it;
// the mxfp4 kernel additionally fuses its own copy of the same math).
-enum ActKind { ACT_SILU = 0, ACT_GELU = 1, ACT_GELU_TANH = 2, ACT_SWIGLUOAI = 3 };
+// ACT_SWIGLU_CLAMP (GLM-5.3 "swiglu_limit") is the same clamped form WITHOUT
+// the (up + 1) bias: clamp(gate, max=lim) * sigmoid(alpha*gate) * clamp(up, +-lim).
+enum ActKind {
+ ACT_SILU = 0,
+ ACT_GELU = 1,
+ ACT_GELU_TANH = 2,
+ ACT_SWIGLUOAI = 3,
+ ACT_SWIGLU_CLAMP = 4,
+};
inline float act_apply(int act, float x) {
if (act == ACT_SILU) return x / (1.0f + std::exp(-x));
@@ -1604,7 +1612,8 @@ struct CpuMoeExecutor {
bf16_t* g_row = g_scratch.data() + ((size_t)tok * top_k + k) * I;
const int i0 = static_cast<int>(ib) * IBLK;
const int i1 = std::min(I, i0 + IBLK);
- const bool swigluoai = act == ACT_SWIGLUOAI;
+ const bool clamped = act == ACT_SWIGLUOAI || act == ACT_SWIGLU_CLAMP;
+ const float up_bias = act == ACT_SWIGLUOAI ? 1.0f : 0.0f;
const float lim = swiglu_limit, alpha = swiglu_alpha;
for (int i = i0; i < i1; ++i) {
// gate = row i, up = row I+i
@@ -1612,14 +1621,15 @@ struct CpuMoeExecutor {
gemm1_dot(gate_up_l, gu_packed_l, gu_scale_l, gu_global_l, e, i, x_row, xe, xo, xi8, xas) * w_in;
float up = gemm1_dot(gate_up_l, gu_packed_l, gu_scale_l, gu_global_l, e, I + i, x_row,
xe, xo, xi8, xas) * w_in;
- if (swigluoai) {
- // clamp(gate, max=lim) * sigmoid(alpha * gate) * (clamp(up, +-lim) + 1)
- // -- same math as the mxfp4 kernel's fused epilogue (lim == +inf: no clamp).
+ if (clamped) {
+ // clamp(gate, max=lim) * sigmoid(alpha * gate) * (clamp(up, +-lim) + up_bias)
+ // -- swigluoai carries the +1 up bias (gpt-oss/MiniMax); swiglu_clamp
+ // (GLM-5.3) does not. lim == +inf: no clamp.
if (gate > lim) gate = lim;
if (up > lim) up = lim;
else if (up < -lim) up = -lim;
const float glu = gate / (1.0f + std::exp(-gate * alpha));
- g_row[i] = f32_to_bf16(glu * (up + 1.0f));
+ g_row[i] = f32_to_bf16(glu * (up + up_bias));
} else {
g_row[i] = f32_to_bf16(act_apply(act, gate) * up);
}
@@ -2146,5 +2156,5 @@ PYBIND11_MODULE(TORCH_EXTENSION_NAME, m) {
// accepts id 3 without error and silently computes the wrong activation
// (act_apply falls through to gelu_tanh); the probe turns a stale extension
// into a loud rebuild instruction instead of wrong model outputs.
- m.def("max_generic_act_id", []() { return static_cast<int>(ACT_SWIGLUOAI); });
+ m.def("max_generic_act_id", []() { return static_cast<int>(ACT_SWIGLU_CLAMP); });
}
diff --git a/python/freetoken/kernel/fla/__init__.py b/python/freetoken/kernel/fla/__init__.py
index e3fff018..4d532077 100644
--- a/python/freetoken/kernel/fla/__init__.py
+++ b/python/freetoken/kernel/fla/__init__.py
@@ -21,15 +21,37 @@ Provenance: https://github.com/sgl-project/sglang, ``python/sglang/srt/layers/at
stripped/inlined on vendoring). Keep ``chunk_delta_h.py``'s single fixed ``triton.Config`` — restoring
upstream's multi-config autotune corrupts the in-place state pool. Tune via the env knobs
``SGLANG_GDN_CHUNK_H_BV`` / ``SGLANG_GDN_CHUNK_H_NUM_WARPS`` / ``SGLANG_GDN_CHUNK_H_NUM_STAGES``.
+
+KDA (GLM-5.3-Flash Kimi Delta Attention) kernels are vendored separately from vLLM's
+``third_party/flash_linear_attention`` (same fla lineage): ``kda.py`` (chunked prefill +
+fused gate cumsum + recurrent decode wrapper), ``kda_chunk_delta_h.py`` (exp2-gate chunk
+recurrence), ``fused_recurrent.py`` (pool-indexed decode kernel with in-kernel KDA gate),
+``solve_tril.py``. Public entry points:
+- ``chunk_kda_with_fused_gate`` -- chunked prefill from raw gate logits; gathered initial
+ state in, final state out (scatter back to the pool is the caller's job).
+ WARNING: clobbers the ``v`` argument (the output is written into that buffer to save
+ an allocation, as in vLLM where v is an ephemeral projection). Never pass a tensor
+ that is read again afterwards.
+- ``fused_recurrent_kda`` -- decode; per-slot state read/write via ``ssm_state_indices``,
+ gate + beta-sigmoid + q/k l2norm computed in-kernel. The per-token state store reads
+ ``ssm_state_indices`` as a CONTIGUOUS [N, T] block; materialize, never ``expand()``.
"""
from freetoken.kernel.fla.chunk import chunk_gated_delta_rule
from freetoken.kernel.fla.fused_sigmoid_gating_recurrent import (
fused_sigmoid_gating_delta_rule_update,
)
+from freetoken.kernel.fla.kda import (
+ chunk_kda_with_fused_gate,
+ fused_kda_gate,
+ fused_recurrent_kda,
+)
from freetoken.kernel.fla.layernorm_gated import rms_norm_gated
__all__ = [
"chunk_gated_delta_rule",
"fused_sigmoid_gating_delta_rule_update",
+ "chunk_kda_with_fused_gate",
+ "fused_kda_gate",
+ "fused_recurrent_kda",
"rms_norm_gated",
]
diff --git a/python/freetoken/kernel/fla/fused_recurrent.py b/python/freetoken/kernel/fla/fused_recurrent.py
new file mode 100644
index 00000000..cc15fef9
--- /dev/null
+++ b/python/freetoken/kernel/fla/fused_recurrent.py
@@ -0,0 +1,659 @@
+# Vendored from vLLM's third_party/flash_linear_attention (PR #53906, commit 933876c3),
+# itself copied from the flash-linear-attention project (MIT, (c) 2023-2025 Songlin Yang,
+# Yu Zhang). Imports are remapped onto freetoken.kernel.fla's shared helpers; keep this
+# file in sync with upstream when pulling KDA kernel fixes.
+# SPDX-License-Identifier: Apache-2.0
+# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
+# SPDX-FileCopyrightText: Songlin Yang, Yu Zhang
+#
+# This file contains code copied from the flash-linear-attention project.
+# The original source code was licensed under the MIT license and included
+# the following copyright notice:
+# Copyright (c) 2023-2025, Songlin Yang, Yu Zhang
+# ruff: noqa: E501
+
+import torch
+
+import triton
+import triton.language as tl
+
+from .op import exp, log
+
+
+ {
+ "USE_INITIAL_STATE": lambda args: args["h0"] is not None,
+ "IS_VARLEN": lambda args: args["cu_seqlens"] is not None,
+ "IS_CONTINUOUS_BATCHING": lambda args: args["ssm_state_indices"] is not None,
+ "IS_SPEC_DECODING": lambda args: args["num_accepted_tokens"] is not None,
+ }
+)
[email protected](do_not_specialize=["N", "T"])
+def fused_recurrent_gated_delta_rule_fwd_kernel(
+ q,
+ k,
+ v,
+ g,
+ beta,
+ o,
+ h0,
+ ht,
+ cu_seqlens,
+ ssm_state_indices,
+ num_accepted_tokens,
+ a_log,
+ g_bias,
+ scale,
+ N: tl.int64, # num of sequences
+ T: tl.int64, # num of tokens
+ B: tl.constexpr,
+ H: tl.constexpr,
+ HV: tl.constexpr,
+ K: tl.constexpr,
+ V: tl.constexpr,
+ BK: tl.constexpr,
+ BV: tl.constexpr,
+ stride_init_state_token: tl.constexpr,
+ stride_final_state_token: tl.constexpr,
+ stride_indices_seq: tl.constexpr,
+ stride_indices_tok: tl.constexpr,
+ USE_INITIAL_STATE: tl.constexpr, # whether to use initial state
+ INPLACE_FINAL_STATE: tl.constexpr, # whether to store final state inplace
+ IS_BETA_HEADWISE: tl.constexpr, # whether beta is headwise vector or scalar,
+ USE_QK_L2NORM_IN_KERNEL: tl.constexpr,
+ IS_VARLEN: tl.constexpr,
+ IS_CONTINUOUS_BATCHING: tl.constexpr,
+ IS_SPEC_DECODING: tl.constexpr,
+ IS_KDA: tl.constexpr,
+ SIGMOID_BETA: tl.constexpr, # beta holds raw logits; sigmoid at fp32 load
+ COMPUTE_GATE: tl.constexpr, # g holds raw logits; KDA gate computed in-kernel
+ SAFE_GATE: tl.constexpr, # bounded gate variant (only branch implemented)
+ LOWER_BOUND: tl.constexpr,
+):
+ i_k, i_v, i_nh = tl.program_id(0), tl.program_id(1), tl.program_id(2)
+ i_n, i_hv = i_nh // HV, i_nh % HV
+ i_h = i_hv // (HV // H)
+ if IS_VARLEN:
+ bos, eos = (
+ tl.load(cu_seqlens + i_n).to(tl.int64),
+ tl.load(cu_seqlens + i_n + 1).to(tl.int64),
+ )
+ all = T
+ T = eos - bos
+ else:
+ bos, eos = i_n * T, i_n * T + T
+ all = B * T
+
+ if T == 0:
+ # no tokens to process for this sequence
+ return
+
+ o_k = i_k * BK + tl.arange(0, BK)
+ o_v = i_v * BV + tl.arange(0, BV)
+
+ p_q = q + (bos * H + i_h) * K + o_k
+ p_k = k + (bos * H + i_h) * K + o_k
+ p_v = v + (bos * HV + i_hv) * V + o_v
+ if IS_BETA_HEADWISE:
+ p_beta = beta + (bos * HV + i_hv) * V + o_v
+ else:
+ p_beta = beta + bos * HV + i_hv
+
+ if not IS_KDA:
+ p_g = g + bos * HV + i_hv
+ else:
+ p_gk = g + (bos * HV + i_hv) * K + o_k
+
+ # Per-head gate amplitude, hoisted out of the token loop (COMPUTE_GATE).
+ if COMPUTE_GATE:
+ b_a_log = tl.exp(tl.load(a_log + i_h).to(tl.float32))
+
+ p_o = o + ((i_k * all + bos) * HV + i_hv) * V + o_v
+
+ mask_k = o_k < K
+ mask_v = o_v < V
+ mask_h = mask_v[:, None] & mask_k[None, :]
+
+ b_h = tl.zeros([BV, BK], dtype=tl.float32)
+ if USE_INITIAL_STATE:
+ if IS_CONTINUOUS_BATCHING:
+ if IS_SPEC_DECODING:
+ i_t = tl.load(num_accepted_tokens + i_n).to(tl.int64) - 1
+ else:
+ i_t = 0
+ # Load state index and check for invalid entries
+ state_idx = tl.load(ssm_state_indices + i_n * stride_indices_seq + i_t).to(
+ tl.int64
+ )
+ # DIVERGENCE from upstream: vLLM treats slot 0 as its NULL_BLOCK_ID
+ # sentinel (`state_idx <= 0`), which silently skips a real request that
+ # keys state by raw table_idx == 0 (--cache-type naive). FreeToken's
+ # GDN kernel has no sentinel; match it -- only negative ids are invalid.
+ # Under hybrid, padded rows now read/write the reserved padding slot
+ # instead of early-returning (a benign write-only sink, same as GDN).
+ if state_idx < 0:
+ return
+ p_h0 = h0 + state_idx * stride_init_state_token
+ else:
+ p_h0 = h0 + bos * HV * V * K
+ p_h0 = p_h0 + i_hv * V * K + o_v[:, None] * K + o_k[None, :]
+ b_h += tl.load(p_h0, mask=mask_h, other=0).to(tl.float32)
+
+ for i_t in range(0, T):
+ b_q = tl.load(p_q, mask=mask_k, other=0).to(tl.float32)
+ b_k = tl.load(p_k, mask=mask_k, other=0).to(tl.float32)
+ b_v = tl.load(p_v, mask=mask_v, other=0).to(tl.float32)
+
+ if USE_QK_L2NORM_IN_KERNEL:
+ b_q = b_q / tl.sqrt(tl.sum(b_q * b_q) + 1e-6)
+ b_k = b_k / tl.sqrt(tl.sum(b_k * b_k) + 1e-6)
+ b_q = b_q * scale
+ # [BV, BK]
+ if not IS_KDA:
+ b_g = tl.load(p_g).to(tl.float32)
+ b_h *= exp(b_g)
+ else:
+ b_gk = tl.load(p_gk).to(tl.float32)
+ if COMPUTE_GATE:
+ # Replicates kda_gate_fwd_kernel's SAFE_GATE branch
+ # bit-for-bit (same tl.exp, same fp32 math; the intermediate
+ # gate value this replaces was stored/reloaded as fp32,
+ # which is lossless): y = lb / (1 + exp(-exp(A)*(g+bias))).
+ b_gk += tl.load(
+ g_bias + i_h * K + o_k, mask=mask_k, other=0.0
+ ).to(tl.float32)
+ b_gk = LOWER_BOUND / (1.0 + tl.exp(-(b_a_log * b_gk)))
+ b_h *= exp(b_gk[None, :])
+ # [BV]
+ b_v -= tl.sum(b_h * b_k[None, :], 1)
+ if IS_BETA_HEADWISE:
+ b_beta = tl.load(p_beta, mask=mask_v, other=0).to(tl.float32)
+ else:
+ b_beta = tl.load(p_beta).to(tl.float32)
+ # Matches torch's `x.float().sigmoid()` pre-computation bit-for-bit
+ # on the input side (bf16->fp32 is exact); only the sigmoid impl itself
+ # can differ by <=1 ULP.
+ if SIGMOID_BETA:
+ b_beta = tl.sigmoid(b_beta)
+ b_v *= b_beta
+ # [BV, BK]
+ b_h += b_v[:, None] * b_k[None, :]
+ # [BV]
+ b_o = tl.sum(b_h * b_q[None, :], 1)
+ tl.store(p_o, b_o.to(p_o.dtype.element_ty), mask=mask_v)
+
+ # keep the states for multi-query tokens
+ if INPLACE_FINAL_STATE:
+ # Load state index and check for invalid entries
+ final_state_idx = tl.load(
+ ssm_state_indices + i_n * stride_indices_seq + i_t
+ ).to(tl.int64)
+ # DIVERGENCE from upstream: no slot-0 sentinel (see the load-side note).
+ if final_state_idx >= 0:
+ p_ht = ht + final_state_idx * stride_final_state_token
+ p_ht = p_ht + i_hv * V * K + o_v[:, None] * K + o_k[None, :]
+ tl.store(p_ht, b_h.to(p_ht.dtype.element_ty), mask=mask_h)
+ else:
+ p_ht = ht + (bos + i_t) * stride_final_state_token
+ p_ht = p_ht + i_hv * V * K + o_v[:, None] * K + o_k[None, :]
+ tl.store(p_ht, b_h.to(p_ht.dtype.element_ty), mask=mask_h)
+
+ p_q += H * K
+ p_k += H * K
+ p_o += HV * V
+ p_v += HV * V
+ if not IS_KDA:
+ p_g += HV
+ else:
+ p_gk += HV * K
+ p_beta += HV * (V if IS_BETA_HEADWISE else 1)
+
+
+def fused_recurrent_gated_delta_rule_fwd(
+ q: torch.Tensor,
+ k: torch.Tensor,
+ v: torch.Tensor,
+ g: torch.Tensor,
+ beta: torch.Tensor,
+ scale: float,
+ initial_state: torch.Tensor,
+ inplace_final_state: bool = True,
+ cu_seqlens: torch.Tensor | None = None,
+ ssm_state_indices: torch.Tensor | None = None,
+ num_accepted_tokens: torch.Tensor | None = None,
+ use_qk_l2norm_in_kernel: bool = False,
+) -> tuple[torch.Tensor, torch.Tensor]:
+ B, T, H, K, V = *k.shape, v.shape[-1]
+ HV = v.shape[2]
+ N = B if cu_seqlens is None else len(cu_seqlens) - 1
+ BK, BV = triton.next_power_of_2(K), min(triton.next_power_of_2(V), 32)
+ NK, NV = triton.cdiv(K, BK), triton.cdiv(V, BV)
+ assert NK == 1, "NK > 1 is not supported yet"
+ num_stages = 3
+ num_warps = 1
+
+ o = q.new_empty(NK, *v.shape)
+ if inplace_final_state:
+ final_state = initial_state
+ else:
+ final_state = q.new_empty(T, HV, V, K, dtype=initial_state.dtype)
+
+ stride_init_state_token = initial_state.stride(0)
+ stride_final_state_token = final_state.stride(0)
+
+ if ssm_state_indices is None:
+ stride_indices_seq, stride_indices_tok = 1, 1
+ elif ssm_state_indices.ndim == 1:
+ stride_indices_seq, stride_indices_tok = ssm_state_indices.stride(0), 1
+ else:
+ stride_indices_seq, stride_indices_tok = ssm_state_indices.stride()
+
+ grid = (NK, NV, N * HV)
+ fused_recurrent_gated_delta_rule_fwd_kernel[grid](
+ q=q,
+ k=k,
+ v=v,
+ g=g,
+ beta=beta,
+ o=o,
+ h0=initial_state,
+ ht=final_state,
+ cu_seqlens=cu_seqlens,
+ ssm_state_indices=ssm_state_indices,
+ num_accepted_tokens=num_accepted_tokens,
+ scale=scale,
+ N=N,
+ T=T,
+ B=B,
+ H=H,
+ HV=HV,
+ K=K,
+ V=V,
+ BK=BK,
+ BV=BV,
+ stride_init_state_token=stride_init_state_token,
+ stride_final_state_token=stride_final_state_token,
+ stride_indices_seq=stride_indices_seq,
+ stride_indices_tok=stride_indices_tok,
+ IS_BETA_HEADWISE=beta.ndim == v.ndim,
+ USE_QK_L2NORM_IN_KERNEL=use_qk_l2norm_in_kernel,
+ INPLACE_FINAL_STATE=inplace_final_state,
+ IS_KDA=False,
+ SIGMOID_BETA=False,
+ a_log=None,
+ g_bias=None,
+ COMPUTE_GATE=False,
+ SAFE_GATE=True,
+ LOWER_BOUND=-5.0,
+ num_warps=num_warps,
+ num_stages=num_stages,
+ )
+ o = o.squeeze(0)
+ return o, final_state
+
+
+def fused_recurrent_gated_delta_rule_packed_decode_kernel(
+ mixed_qkv,
+ a,
+ b,
+ A_log,
+ dt_bias,
+ o,
+ h0,
+ ht,
+ ssm_state_indices,
+ scale,
+ stride_mixed_qkv_tok: tl.constexpr,
+ stride_a_tok: tl.constexpr,
+ stride_b_tok: tl.constexpr,
+ stride_init_state_token: tl.constexpr,
+ stride_final_state_token: tl.constexpr,
+ stride_indices_seq: tl.constexpr,
+ H: tl.constexpr,
+ HV: tl.constexpr,
+ K: tl.constexpr,
+ V: tl.constexpr,
+ BK: tl.constexpr,
+ BV: tl.constexpr,
+ SOFTPLUS_THRESHOLD: tl.constexpr,
+ USE_QK_L2NORM_IN_KERNEL: tl.constexpr,
+):
+ i_v, i_nh = tl.program_id(0), tl.program_id(1)
+ i_n, i_hv = i_nh // HV, i_nh % HV
+ i_h = i_hv // (HV // H)
+
+ o_k = tl.arange(0, BK)
+ o_v = i_v * BV + tl.arange(0, BV)
+ mask_k = o_k < K
+ mask_v = o_v < V
+ mask_h = mask_v[:, None] & mask_k[None, :]
+
+ state_idx = tl.load(ssm_state_indices + i_n * stride_indices_seq).to(tl.int64)
+ p_o = o + (i_n * HV + i_hv) * V + o_v
+
+ # Skip if state index is invalid (NULL_BLOCK_ID=0)
+ if state_idx <= 0:
+ zero = tl.zeros([BV], dtype=tl.float32).to(p_o.dtype.element_ty)
+ tl.store(p_o, zero, mask=mask_v)
+ return
+
+ p_h0 = h0 + state_idx * stride_init_state_token
+ p_h0 = p_h0 + i_hv * V * K + o_v[:, None] * K + o_k[None, :]
+ b_h = tl.load(p_h0, mask=mask_h, other=0).to(tl.float32)
+
+ p_mixed = mixed_qkv + i_n * stride_mixed_qkv_tok
+ q_off = i_h * K + o_k
+ k_off = (H * K) + i_h * K + o_k
+ v_off = (2 * H * K) + i_hv * V + o_v
+ b_q = tl.load(p_mixed + q_off, mask=mask_k, other=0).to(tl.float32)
+ b_k = tl.load(p_mixed + k_off, mask=mask_k, other=0).to(tl.float32)
+ b_v = tl.load(p_mixed + v_off, mask=mask_v, other=0).to(tl.float32)
+
+ if USE_QK_L2NORM_IN_KERNEL:
+ b_q = b_q / tl.sqrt(tl.sum(b_q * b_q) + 1e-6)
+ b_k = b_k / tl.sqrt(tl.sum(b_k * b_k) + 1e-6)
+ b_q = b_q * scale
+
+ a_val = tl.load(a + i_n * stride_a_tok + i_hv).to(tl.float32)
+ b_val = tl.load(b + i_n * stride_b_tok + i_hv).to(tl.float32)
+ A_log_val = tl.load(A_log + i_hv).to(tl.float32)
+ dt_bias_val = tl.load(dt_bias + i_hv).to(tl.float32)
+ x = a_val + dt_bias_val
+ softplus_x = tl.where(x <= SOFTPLUS_THRESHOLD, tl.log(1.0 + tl.exp(x)), x)
+ g_val = -tl.exp(A_log_val) * softplus_x
+ beta_val = tl.sigmoid(b_val).to(b.dtype.element_ty).to(tl.float32)
+
+ b_h *= exp(g_val)
+ b_v -= tl.sum(b_h * b_k[None, :], 1)
+ b_v *= beta_val
+ b_h += b_v[:, None] * b_k[None, :]
+ b_o = tl.sum(b_h * b_q[None, :], 1)
+ tl.store(p_o, b_o.to(p_o.dtype.element_ty), mask=mask_v)
+
+ p_ht = ht + state_idx * stride_final_state_token
+ p_ht = p_ht + i_hv * V * K + o_v[:, None] * K + o_k[None, :]
+ tl.store(p_ht, b_h.to(p_ht.dtype.element_ty), mask=mask_h)
+
+
+def fused_recurrent_gated_delta_rule_packed_decode(
+ mixed_qkv: torch.Tensor,
+ a: torch.Tensor,
+ b: torch.Tensor,
+ A_log: torch.Tensor,
+ dt_bias: torch.Tensor,
+ scale: float,
+ initial_state: torch.Tensor,
+ out: torch.Tensor,
+ ssm_state_indices: torch.Tensor,
+ use_qk_l2norm_in_kernel: bool = False,
+) -> tuple[torch.Tensor, torch.Tensor]:
+ if mixed_qkv.ndim != 2:
+ raise ValueError(
+ f"`mixed_qkv` must be a 2D tensor (got ndim={mixed_qkv.ndim})."
+ )
+ if mixed_qkv.stride(-1) != 1:
+ raise ValueError("`mixed_qkv` must be contiguous in the last dim.")
+ if a.ndim != 2 or b.ndim != 2:
+ raise ValueError(
+ f"`a` and `b` must be 2D tensors (got a.ndim={a.ndim}, b.ndim={b.ndim})."
+ )
+ if a.stride(-1) != 1 or b.stride(-1) != 1:
+ raise ValueError("`a`/`b` must be contiguous in the last dim.")
+ if A_log.ndim != 1 or dt_bias.ndim != 1:
+ raise ValueError("`A_log`/`dt_bias` must be 1D tensors.")
+ if A_log.stride(0) != 1 or dt_bias.stride(0) != 1:
+ raise ValueError("`A_log`/`dt_bias` must be contiguous.")
+ if ssm_state_indices.ndim != 1:
+ raise ValueError(
+ f"`ssm_state_indices` must be 1D for packed decode (got ndim={ssm_state_indices.ndim})."
+ )
+ if not out.is_contiguous():
+ raise ValueError("`out` must be contiguous.")
+
+ dev = mixed_qkv.device
+ if (
+ a.device != dev
+ or b.device != dev
+ or A_log.device != dev
+ or dt_bias.device != dev
+ or initial_state.device != dev
+ or out.device != dev
+ or ssm_state_indices.device != dev
+ ):
+ raise ValueError("All inputs must be on the same device.")
+
+ B = mixed_qkv.shape[0]
+ if a.shape[0] != B or b.shape[0] != B:
+ raise ValueError(
+ "Mismatched batch sizes: "
+ f"mixed_qkv.shape[0]={B}, a.shape[0]={a.shape[0]}, b.shape[0]={b.shape[0]}."
+ )
+ if ssm_state_indices.shape[0] != B:
+ raise ValueError(
+ f"`ssm_state_indices` must have shape [B] (got {tuple(ssm_state_indices.shape)}; expected ({B},))."
+ )
+
+ if initial_state.ndim != 4:
+ raise ValueError(
+ f"`initial_state` must be a 4D tensor (got ndim={initial_state.ndim})."
+ )
+ if initial_state.stride(-1) != 1:
+ raise ValueError("`initial_state` must be contiguous in the last dim.")
+ HV, V, K = initial_state.shape[-3:]
+ if a.shape[1] != HV or b.shape[1] != HV:
+ raise ValueError(
+ f"`a`/`b` must have shape [B, HV] with HV={HV} (got a.shape={tuple(a.shape)}, b.shape={tuple(b.shape)})."
+ )
+ if A_log.numel() != HV or dt_bias.numel() != HV:
+ raise ValueError(
+ f"`A_log` and `dt_bias` must have {HV} elements (got A_log.numel()={A_log.numel()}, dt_bias.numel()={dt_bias.numel()})."
+ )
+ if out.shape != (B, 1, HV, V):
+ raise ValueError(
+ f"`out` must have shape {(B, 1, HV, V)} (got out.shape={tuple(out.shape)})."
+ )
+
+ qkv_dim = mixed_qkv.shape[1]
+ qk_dim = qkv_dim - HV * V
+ if qk_dim <= 0 or qk_dim % 2 != 0:
+ raise ValueError(
+ f"Invalid packed `mixed_qkv` last dim={qkv_dim} for HV={HV}, V={V}."
+ )
+ q_dim = qk_dim // 2
+ if q_dim % K != 0:
+ raise ValueError(f"Invalid packed Q size {q_dim}: must be divisible by K={K}.")
+ H = q_dim // K
+ if H <= 0 or HV % H != 0:
+ raise ValueError(
+ f"Invalid head config inferred from mixed_qkv: H={H}, HV={HV}."
+ )
+
+ BK = triton.next_power_of_2(K)
+ if triton.cdiv(K, BK) != 1:
+ raise ValueError(
+ f"Packed decode kernel only supports NK=1 (got K={K}, BK={BK})."
+ )
+ BV = min(triton.next_power_of_2(V), 32)
+ num_stages = 3
+ num_warps = 1
+
+ stride_mixed_qkv_tok = mixed_qkv.stride(0)
+ stride_a_tok = a.stride(0)
+ stride_b_tok = b.stride(0)
+ stride_init_state_token = initial_state.stride(0)
+ stride_final_state_token = initial_state.stride(0)
+ stride_indices_seq = ssm_state_indices.stride(0)
+
+ NV = triton.cdiv(V, BV)
+ grid = (NV, B * HV)
+ fused_recurrent_gated_delta_rule_packed_decode_kernel[grid](
+ mixed_qkv=mixed_qkv,
+ a=a,
+ b=b,
+ A_log=A_log,
+ dt_bias=dt_bias,
+ o=out,
+ h0=initial_state,
+ ht=initial_state,
+ ssm_state_indices=ssm_state_indices,
+ scale=scale,
+ stride_mixed_qkv_tok=stride_mixed_qkv_tok,
+ stride_a_tok=stride_a_tok,
+ stride_b_tok=stride_b_tok,
+ stride_init_state_token=stride_init_state_token,
+ stride_final_state_token=stride_final_state_token,
+ stride_indices_seq=stride_indices_seq,
+ H=H,
+ HV=HV,
+ K=K,
+ V=V,
+ BK=BK,
+ BV=BV,
+ SOFTPLUS_THRESHOLD=20.0,
+ USE_QK_L2NORM_IN_KERNEL=use_qk_l2norm_in_kernel,
+ num_warps=num_warps,
+ num_stages=num_stages,
+ )
+ return out, initial_state
+
+
+class FusedRecurrentFunction(torch.autograd.Function):
+ @staticmethod
+ def forward(
+ ctx,
+ q: torch.Tensor,
+ k: torch.Tensor,
+ v: torch.Tensor,
+ g: torch.Tensor,
+ beta: torch.Tensor,
+ scale: float,
+ initial_state: torch.Tensor,
+ inplace_final_state: bool = True,
+ cu_seqlens: torch.Tensor | None = None,
+ ssm_state_indices: torch.Tensor | None = None,
+ num_accepted_tokens: torch.Tensor | None = None,
+ use_qk_l2norm_in_kernel: bool = False,
+ ):
+ o, final_state = fused_recurrent_gated_delta_rule_fwd(
+ q=q.contiguous(),
+ k=k.contiguous(),
+ v=v.contiguous(),
+ g=g.contiguous(),
+ beta=beta.contiguous(),
+ scale=scale,
+ initial_state=initial_state,
+ inplace_final_state=inplace_final_state,
+ cu_seqlens=cu_seqlens,
+ ssm_state_indices=ssm_state_indices,
+ num_accepted_tokens=num_accepted_tokens,
+ use_qk_l2norm_in_kernel=use_qk_l2norm_in_kernel,
+ )
+
+ return o, final_state
+
+
+def fused_recurrent_gated_delta_rule(
+ q: torch.Tensor,
+ k: torch.Tensor,
+ v: torch.Tensor,
+ g: torch.Tensor,
+ beta: torch.Tensor = None,
+ scale: float = None,
+ initial_state: torch.Tensor = None,
+ inplace_final_state: bool = True,
+ cu_seqlens: torch.Tensor | None = None,
+ ssm_state_indices: torch.Tensor | None = None,
+ num_accepted_tokens: torch.Tensor | None = None,
+ use_qk_l2norm_in_kernel: bool = False,
+) -> tuple[torch.Tensor, torch.Tensor]:
+ r"""
+ Args:
+ q (torch.Tensor):
+ queries of shape `[B, T, H, K]`.
+ k (torch.Tensor):
+ keys of shape `[B, T, H, K]`.
+ v (torch.Tensor):
+ values of shape `[B, T, HV, V]`.
+ GVA is applied if `HV > H`.
+ g (torch.Tensor):
+ g (decays) of shape `[B, T, HV]`.
+ beta (torch.Tensor):
+ betas of shape `[B, T, HV]`.
+ scale (Optional[int]):
+ Scale factor for the RetNet attention scores.
+ If not provided, it will default to `1 / sqrt(K)`. Default: `None`.
+ initial_state (Optional[torch.Tensor]):
+ Initial state of shape `[N, HV, V, K]` for `N` input sequences.
+ For equal-length input sequences, `N` equals the batch size `B`.
+ Default: `None`.
+ inplace_final_state: bool:
+ Whether to store the final state in-place to save memory.
+ Default: `True`.
+ cu_seqlens (torch.Tensor):
+ Cumulative sequence lengths of shape `[N+1]` used for variable-length training,
+ consistent with the FlashAttention API.
+ ssm_state_indices (Optional[torch.Tensor]):
+ Indices to map the input sequences to the initial/final states.
+ num_accepted_tokens (Optional[torch.Tensor]):
+ Number of accepted tokens for each sequence during decoding.
+
+ Returns:
+ o (torch.Tensor):
+ Outputs of shape `[B, T, HV, V]`.
+ final_state (torch.Tensor):
+ Final state of shape `[N, HV, V, K]`.
+
+ Examples::
+ >>> import torch
+ >>> import torch.nn.functional as F
+ >>> from einops import rearrange
+ >>> from fla.ops.gated_delta_rule import fused_recurrent_gated_delta_rule
+ # inputs with equal lengths
+ >>> B, T, H, HV, K, V = 4, 2048, 4, 8, 512, 512
+ >>> q = torch.randn(B, T, H, K, device='cuda')
+ >>> k = F.normalize(torch.randn(B, T, H, K, device='cuda'), p=2, dim=-1)
+ >>> v = torch.randn(B, T, HV, V, device='cuda')
+ >>> g = F.logsigmoid(torch.rand(B, T, HV, device='cuda'))
+ >>> beta = torch.rand(B, T, HV, device='cuda').sigmoid()
+ >>> h0 = torch.randn(B, HV, V, K, device='cuda')
+ >>> o, ht = fused_gated_recurrent_delta_rule(
+ q, k, v, g, beta,
+ initial_state=h0,
+ )
+ # for variable-length inputs, the batch size `B` is expected to be 1 and `cu_seqlens` is required
+ >>> q, k, v, g, beta = map(lambda x: rearrange(x, 'b t ... -> 1 (b t) ...'), (q, k, v, g, beta))
+ # for a batch with 4 sequences, `cu_seqlens` with 5 start/end positions are expected
+ >>> cu_seqlens = q.new_tensor([0, 2048, 4096, 6144, 8192], dtype=torch.int32)
+ >>> o_var, ht_var = fused_gated_recurrent_delta_rule(
+ q, k, v, g, beta,
+ initial_state=h0,
+ cu_seqlens=cu_seqlens
+ )
+ """
+ if cu_seqlens is not None and q.shape[0] != 1:
+ raise ValueError(
+ f"The batch size is expected to be 1 rather than {q.shape[0]} when using `cu_seqlens`."
+ f"Please flatten variable-length inputs before processing."
+ )
+ if scale is None:
+ scale = k.shape[-1] ** -0.5
+ else:
+ assert scale > 0, "scale must be positive"
+ if beta is None:
+ beta = torch.ones_like(q[..., 0])
+ o, final_state = FusedRecurrentFunction.apply(
+ q,
+ k,
+ v,
+ g,
+ beta,
+ scale,
+ initial_state,
+ inplace_final_state,
+ cu_seqlens,
+ ssm_state_indices,
+ num_accepted_tokens,
+ use_qk_l2norm_in_kernel,
+ )
+ return o, final_state
diff --git a/python/freetoken/kernel/fla/kda.py b/python/freetoken/kernel/fla/kda.py
new file mode 100644
index 00000000..ff724049
--- /dev/null
+++ b/python/freetoken/kernel/fla/kda.py
@@ -0,0 +1,1396 @@
+# SPDX-License-Identifier: Apache-2.0
+# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
+# SPDX-FileCopyrightText: Songlin Yang, Yu Zhang
+#
+# This file contains code copied from the flash-linear-attention project.
+# The original source code was licensed under the MIT license and included
+# the following copyright notice:
+# Copyright (c) 2023-2025, Songlin Yang, Yu Zhang
+# ruff: noqa: E501
+
+
+# Vendored from vLLM's third_party/flash_linear_attention (PR #53906, commit 933876c3),
+# itself copied from the flash-linear-attention project (MIT, (c) 2023-2025 Songlin Yang,
+# Yu Zhang). Imports are remapped onto freetoken.kernel.fla's shared helpers; keep this
+# file in sync with upstream when pulling KDA kernel fixes.
+import torch
+import triton
+import triton.language as tl
+
+from freetoken.kernel.fla.cumsum import chunk_local_cumsum
+from freetoken.kernel.fla.fused_recurrent import (
+ fused_recurrent_gated_delta_rule_fwd_kernel,
+)
+from freetoken.kernel.fla.index import prepare_chunk_indices
+from freetoken.kernel.fla.kda_chunk_delta_h import chunk_gated_delta_rule_fwd_h
+from freetoken.kernel.fla.l2norm import l2norm_fwd
+from freetoken.kernel.fla.op import exp2, log
+from freetoken.kernel.fla.solve_tril import solve_tril
+from freetoken.kernel.fla.utils import FLA_CHUNK_SIZE, is_amd
+
+RCP_LN2 = 1.4426950408889634 # 1 / ln(2)
+cdiv = triton.cdiv
+next_power_of_2 = triton.next_power_of_2
+
+BT_LIST_AUTOTUNE = [32, 64, 128]
+NUM_WARPS_AUTOTUNE = [2, 4, 8, 16] if is_amd else [4, 8, 16, 32]
+
+
+def fused_recurrent_kda_fwd(
+ q: torch.Tensor,
+ k: torch.Tensor,
+ v: torch.Tensor,
+ g: torch.Tensor,
+ beta: torch.Tensor,
+ scale: float,
+ initial_state: torch.Tensor,
+ inplace_final_state: bool = True,
+ cu_seqlens: torch.Tensor | None = None,
+ ssm_state_indices: torch.Tensor | None = None,
+ num_accepted_tokens: torch.Tensor | None = None,
+ use_qk_l2norm_in_kernel: bool = False,
+ out: torch.Tensor | None = None,
+ sigmoid_beta: bool = False,
+ a_log: torch.Tensor | None = None,
+ g_bias: torch.Tensor | None = None,
+ compute_gate: bool = False,
+ lower_bound: float | None = -5.0,
+) -> tuple[torch.Tensor, torch.Tensor]:
+ B, T, H, K, V = *k.shape, v.shape[-1]
+ HV = v.shape[2]
+ N = B if cu_seqlens is None else len(cu_seqlens) - 1
+ BK, BV = next_power_of_2(K), min(next_power_of_2(V), 8)
+ NK, NV = cdiv(K, BK), cdiv(V, BV)
+ assert NK == 1, "NK > 1 is not supported yet"
+ num_stages = 3
+ num_warps = 1
+
+ if compute_gate:
+ assert a_log is not None and g_bias is not None, (
+ "compute_gate requires a_log and g_bias"
+ )
+ assert lower_bound is not None, (
+ "compute_gate implements the bounded (safe_gate) branch only"
+ )
+ a_log = a_log.reshape(-1).contiguous()
+ g_bias = g_bias.reshape(-1).contiguous()
+
+ if out is None:
+ o = torch.empty_like(k)
+ else:
+ # Caller-provided output buffer; must be layout-compatible with the
+ # tensor the kernel indexes (contiguous, same shape/dtype as k).
+ assert out.shape == k.shape and out.dtype == k.dtype
+ assert out.is_contiguous()
+ o = out
+ if inplace_final_state:
+ final_state = initial_state
+ else:
+ final_state = q.new_empty(T, HV, V, K, dtype=initial_state.dtype)
+
+ stride_init_state_token = initial_state.stride(0)
+ stride_final_state_token = final_state.stride(0)
+
+ if ssm_state_indices is None:
+ stride_indices_seq, stride_indices_tok = 1, 1
+ elif ssm_state_indices.ndim == 1:
+ stride_indices_seq, stride_indices_tok = ssm_state_indices.stride(0), 1
+ else:
+ stride_indices_seq, stride_indices_tok = ssm_state_indices.stride()
+
+ grid = (NK, NV, N * HV)
+ fused_recurrent_gated_delta_rule_fwd_kernel[grid](
+ q=q,
+ k=k,
+ v=v,
+ g=g,
+ beta=beta,
+ o=o,
+ h0=initial_state,
+ ht=final_state,
+ cu_seqlens=cu_seqlens,
+ ssm_state_indices=ssm_state_indices,
+ num_accepted_tokens=num_accepted_tokens,
+ scale=scale,
+ N=N,
+ T=T,
+ B=B,
+ H=H,
+ HV=HV,
+ K=K,
+ V=V,
+ BK=BK,
+ BV=BV,
+ stride_init_state_token=stride_init_state_token,
+ stride_final_state_token=stride_final_state_token,
+ stride_indices_seq=stride_indices_seq,
+ stride_indices_tok=stride_indices_tok,
+ IS_BETA_HEADWISE=beta.ndim == v.ndim,
+ USE_QK_L2NORM_IN_KERNEL=use_qk_l2norm_in_kernel,
+ INPLACE_FINAL_STATE=inplace_final_state,
+ IS_KDA=True,
+ SIGMOID_BETA=sigmoid_beta,
+ a_log=a_log,
+ g_bias=g_bias,
+ COMPUTE_GATE=compute_gate,
+ SAFE_GATE=True,
+ LOWER_BOUND=lower_bound if lower_bound is not None else -5.0,
+ num_warps=num_warps,
+ num_stages=num_stages,
+ )
+
+ return o, final_state
+
+
+def fused_recurrent_kda(
+ q: torch.Tensor,
+ k: torch.Tensor,
+ v: torch.Tensor,
+ g: torch.Tensor,
+ beta: torch.Tensor = None,
+ scale: float = None,
+ initial_state: torch.Tensor = None,
+ inplace_final_state: bool = True,
+ use_qk_l2norm_in_kernel: bool = True,
+ cu_seqlens: torch.Tensor | None = None,
+ ssm_state_indices: torch.LongTensor | None = None,
+ num_accepted_tokens: torch.Tensor | None = None,
+ out: torch.Tensor | None = None,
+ sigmoid_beta: bool = False,
+ a_log: torch.Tensor | None = None,
+ g_bias: torch.Tensor | None = None,
+ compute_gate: bool = False,
+ lower_bound: float | None = -5.0,
+ **kwargs,
+) -> tuple[torch.Tensor, torch.Tensor]:
+ if cu_seqlens is not None and q.shape[0] != 1:
+ raise ValueError(
+ f"The batch size is expected to be 1 rather than {q.shape[0]} when using `cu_seqlens`."
+ f"Please flatten variable-length inputs before processing."
+ )
+ if scale is None:
+ scale = k.shape[-1] ** -0.5
+
+ o, final_state = fused_recurrent_kda_fwd(
+ q=q.contiguous(),
+ k=k.contiguous(),
+ v=v.contiguous(),
+ g=g.contiguous(),
+ beta=beta.contiguous(),
+ scale=scale,
+ initial_state=initial_state,
+ inplace_final_state=inplace_final_state,
+ cu_seqlens=cu_seqlens,
+ ssm_state_indices=ssm_state_indices,
+ num_accepted_tokens=num_accepted_tokens,
+ use_qk_l2norm_in_kernel=use_qk_l2norm_in_kernel,
+ out=out,
+ sigmoid_beta=sigmoid_beta,
+ a_log=a_log,
+ g_bias=g_bias,
+ compute_gate=compute_gate,
+ lower_bound=lower_bound,
+ )
+ return o, final_state
+
+
[email protected]({"IS_VARLEN": lambda args: args["cu_seqlens"] is not None})
+ configs=[
+ triton.Config({"BK": BK}, num_warps=num_warps, num_stages=num_stages)
+ for BK in [32, 64]
+ for num_warps in [1, 2, 4, 8]
+ for num_stages in [2, 3, 4]
+ ],
+ key=["BC"],
+)
[email protected](do_not_specialize=["T"])
+def chunk_kda_scaled_dot_kkt_fwd_kernel_intra_sub_inter(
+ q,
+ k,
+ g,
+ beta,
+ A,
+ Aqk,
+ scale,
+ cu_seqlens,
+ chunk_indices,
+ T,
+ H: tl.constexpr,
+ K: tl.constexpr,
+ BT: tl.constexpr,
+ BC: tl.constexpr,
+ BK: tl.constexpr,
+ NC: tl.constexpr,
+ IS_VARLEN: tl.constexpr,
+):
+ i_t, i_c, i_bh = tl.program_id(0), tl.program_id(1), tl.program_id(2)
+ i_b, i_h = i_bh // H, i_bh % H
+ i_i, i_j = i_c // NC, i_c % NC
+ if IS_VARLEN:
+ i_n, i_t = (
+ tl.load(chunk_indices + i_t * 2).to(tl.int32),
+ tl.load(chunk_indices + i_t * 2 + 1).to(tl.int32),
+ )
+ bos, eos = (
+ tl.load(cu_seqlens + i_n).to(tl.int32),
+ tl.load(cu_seqlens + i_n + 1).to(tl.int32),
+ )
+ T = eos - bos
+ else:
+ bos, eos = i_b * T, i_b * T + T
+
+ if i_t * BT + i_i * BC >= T:
+ return
+ if i_i <= i_j:
+ return
+
+ q += (bos * H + i_h) * K
+ k += (bos * H + i_h) * K
+ g += (bos * H + i_h) * K
+ A += (bos * H + i_h) * BT
+ Aqk += (bos * H + i_h) * BT
+
+ p_b = tl.make_block_ptr(
+ beta + bos * H + i_h, (T,), (H,), (i_t * BT + i_i * BC,), (BC,), (0,)
+ )
+ b_b = tl.load(p_b, boundary_check=(0,))
+
+ b_A = tl.zeros([BC, BC], dtype=tl.float32)
+ b_Aqk = tl.zeros([BC, BC], dtype=tl.float32)
+ for i_k in range(tl.cdiv(K, BK)):
+ p_q = tl.make_block_ptr(
+ q, (T, K), (H * K, 1), (i_t * BT + i_i * BC, i_k * BK), (BC, BK), (1, 0)
+ )
+ p_k = tl.make_block_ptr(
+ k, (T, K), (H * K, 1), (i_t * BT + i_i * BC, i_k * BK), (BC, BK), (1, 0)
+ )
+ p_g = tl.make_block_ptr(
+ g, (T, K), (H * K, 1), (i_t * BT + i_i * BC, i_k * BK), (BC, BK), (1, 0)
+ )
+ b_kt = tl.make_block_ptr(
+ k, (K, T), (1, H * K), (i_k * BK, i_t * BT + i_j * BC), (BK, BC), (0, 1)
+ )
+ p_gk = tl.make_block_ptr(
+ g, (K, T), (1, H * K), (i_k * BK, i_t * BT + i_j * BC), (BK, BC), (0, 1)
+ )
+
+ o_k = i_k * BK + tl.arange(0, BK)
+ m_k = o_k < K
+ # [BK,]
+ b_gn = tl.load(g + (i_t * BT + i_i * BC) * H * K + o_k, mask=m_k, other=0)
+ # [BC, BK]
+ b_g = tl.load(p_g, boundary_check=(0, 1))
+ b_k = tl.load(p_k, boundary_check=(0, 1)) * exp2(b_g - b_gn[None, :])
+ # [BK, BC]
+ b_gk = tl.load(p_gk, boundary_check=(0, 1))
+ b_kt = tl.load(b_kt, boundary_check=(0, 1))
+ # [BC, BC]
+ b_ktg = b_kt * exp2(b_gn[:, None] - b_gk)
+ b_A += tl.dot(b_k, b_ktg)
+
+ b_q = tl.load(p_q, boundary_check=(0, 1))
+ b_qg = b_q * exp2(b_g - b_gn[None, :]) * scale
+ b_Aqk += tl.dot(b_qg, b_ktg)
+
+ b_A *= b_b[:, None]
+
+ p_A = tl.make_block_ptr(
+ A, (T, BT), (H * BT, 1), (i_t * BT + i_i * BC, i_j * BC), (BC, BC), (1, 0)
+ )
+ tl.store(p_A, b_A.to(A.dtype.element_ty), boundary_check=(0, 1))
+ p_Aqk = tl.make_block_ptr(
+ Aqk, (T, BT), (H * BT, 1), (i_t * BT + i_i * BC, i_j * BC), (BC, BC), (1, 0)
+ )
+ tl.store(p_Aqk, b_Aqk.to(Aqk.dtype.element_ty), boundary_check=(0, 1))
+
+
[email protected]({"IS_VARLEN": lambda args: args["cu_seqlens"] is not None})
+ configs=[triton.Config({}, num_warps=num_warps) for num_warps in [1, 2, 4, 8]],
+ key=["BK", "BT"],
+)
[email protected](do_not_specialize=["T"])
+def chunk_kda_scaled_dot_kkt_fwd_kernel_intra_sub_intra(
+ q,
+ k,
+ g,
+ beta,
+ A,
+ Aqk,
+ scale,
+ cu_seqlens,
+ chunk_indices,
+ T,
+ H: tl.constexpr,
+ K: tl.constexpr,
+ BT: tl.constexpr,
+ BC: tl.constexpr,
+ BK: tl.constexpr,
+ IS_VARLEN: tl.constexpr,
+):
+ i_t, i_i, i_bh = tl.program_id(0), tl.program_id(1), tl.program_id(2)
+ i_b, i_h = i_bh // H, i_bh % H
+ if IS_VARLEN:
+ i_n, i_t = (
+ tl.load(chunk_indices + i_t * 2).to(tl.int32),
+ tl.load(chunk_indices + i_t * 2 + 1).to(tl.int32),
+ )
+ bos, eos = (
+ tl.load(cu_seqlens + i_n).to(tl.int32),
+ tl.load(cu_seqlens + i_n + 1).to(tl.int32),
+ )
+ T = eos - bos
+ else:
+ bos, eos = i_b * T, i_b * T + T
+
+ if i_t * BT + i_i * BC >= T:
+ return
+
+ o_i = tl.arange(0, BC)
+ o_k = tl.arange(0, BK)
+ m_k = o_k < K
+ m_A = (i_t * BT + i_i * BC + o_i) < T
+ o_A = (bos + i_t * BT + i_i * BC + o_i) * H * BT + i_h * BT + i_i * BC
+
+ p_q = tl.make_block_ptr(
+ q + (bos * H + i_h) * K,
+ (T, K),
+ (H * K, 1),
+ (i_t * BT + i_i * BC, 0),
+ (BC, BK),
+ (1, 0),
+ )
+ p_k = tl.make_block_ptr(
+ k + (bos * H + i_h) * K,
+ (T, K),
+ (H * K, 1),
+ (i_t * BT + i_i * BC, 0),
+ (BC, BK),
+ (1, 0),
+ )
+ p_g = tl.make_block_ptr(
+ g + (bos * H + i_h) * K,
+ (T, K),
+ (H * K, 1),
+ (i_t * BT + i_i * BC, 0),
+ (BC, BK),
+ (1, 0),
+ )
+ b_q = tl.load(p_q, boundary_check=(0, 1))
+ b_k = tl.load(p_k, boundary_check=(0, 1))
+ b_g = tl.load(p_g, boundary_check=(0, 1))
+
+ p_b = beta + (bos + i_t * BT + i_i * BC + o_i) * H + i_h
+ b_k = b_k * tl.load(p_b, mask=m_A, other=0)[:, None]
+
+ p_kt = k + (bos + i_t * BT + i_i * BC) * H * K + i_h * K + o_k
+ p_gk = g + (bos + i_t * BT + i_i * BC) * H * K + i_h * K + o_k
+
+ for j in range(0, min(BC, T - i_t * BT - i_i * BC)):
+ b_kt = tl.load(p_kt, mask=m_k, other=0).to(tl.float32)
+ b_gk = tl.load(p_gk, mask=m_k, other=0).to(tl.float32)
+ b_ktg = b_kt[None, :] * exp2(b_g - b_gk[None, :])
+ b_A = tl.sum(b_k * b_ktg, 1)
+ b_A = tl.where(o_i > j, b_A, 0.0)
+ b_Aqk = tl.sum(b_q * b_ktg, 1)
+ b_Aqk = tl.where(o_i >= j, b_Aqk * scale, 0.0)
+ tl.store(A + o_A + j, b_A, mask=m_A)
+ tl.store(Aqk + o_A + j, b_Aqk, mask=m_A)
+ p_kt += H * K
+ p_gk += H * K
+
+
+def chunk_kda_scaled_dot_kkt_fwd(
+ q: torch.Tensor,
+ k: torch.Tensor,
+ gk: torch.Tensor | None = None,
+ beta: torch.Tensor | None = None,
+ scale: float | None = None,
+ cu_seqlens: torch.Tensor | None = None,
+ chunk_indices: torch.Tensor | None = None,
+ chunk_size: int = FLA_CHUNK_SIZE,
+ output_dtype: torch.dtype = torch.float32,
+) -> tuple[torch.Tensor, torch.Tensor]:
+ r"""
+ Compute beta * K * K^T.
+
+ Args:
+ k (torch.Tensor):
+ The key tensor of shape `[B, T, H, K]`.
+ beta (torch.Tensor):
+ The beta tensor of shape `[B, T, H]`.
+ gk (torch.Tensor):
+ The cumulative sum of the gate tensor of shape `[B, T, H, K]` applied to the key tensor. Default: `None`.
+ cu_seqlens (torch.Tensor):
+ The cumulative sequence lengths of the input tensor.
+ Default: None
+ chunk_size (int):
+ The chunk size. Default: 64.
+ output_dtype (torch.dtype):
+ The dtype of the output tensor. Default: `torch.float32`
+
+ Returns:
+ beta * K * K^T of shape `[B, T, H, BT]` where `BT` is the chunk size.
+ """
+ B, T, H, K = k.shape
+ assert K <= 256
+ BT = chunk_size
+ if chunk_indices is None and cu_seqlens is not None:
+ chunk_indices = prepare_chunk_indices(cu_seqlens, BT)
+ NT = cdiv(T, BT) if cu_seqlens is None else len(chunk_indices)
+
+ BC = min(16, BT)
+ NC = cdiv(BT, BC)
+ BK = max(next_power_of_2(K), 16)
+ A = torch.zeros(B, T, H, BT, device=k.device, dtype=output_dtype)
+ Aqk = torch.zeros(B, T, H, BT, device=k.device, dtype=output_dtype)
+ grid = (NT, NC * NC, B * H)
+ chunk_kda_scaled_dot_kkt_fwd_kernel_intra_sub_inter[grid](
+ q=q,
+ k=k,
+ g=gk,
+ beta=beta,
+ A=A,
+ Aqk=Aqk,
+ scale=scale,
+ cu_seqlens=cu_seqlens,
+ chunk_indices=chunk_indices,
+ T=T,
+ H=H,
+ K=K,
+ BT=BT,
+ BC=BC,
+ NC=NC,
+ )
+
+ grid = (NT, NC, B * H)
+ chunk_kda_scaled_dot_kkt_fwd_kernel_intra_sub_intra[grid](
+ q=q,
+ k=k,
+ g=gk,
+ beta=beta,
+ A=A,
+ Aqk=Aqk,
+ scale=scale,
+ cu_seqlens=cu_seqlens,
+ chunk_indices=chunk_indices,
+ T=T,
+ H=H,
+ K=K,
+ BT=BT,
+ BC=BC,
+ BK=BK,
+ )
+ return A, Aqk
+
+
+ {
+ "STORE_QG": lambda args: args["qg"] is not None,
+ "STORE_KG": lambda args: args["kg"] is not None,
+ "IS_VARLEN": lambda args: args["cu_seqlens"] is not None,
+ }
+)
+ configs=[
+ triton.Config({}, num_warps=num_warps, num_stages=num_stages)
+ for num_warps in [2, 4, 8]
+ for num_stages in [2, 3, 4]
+ ],
+ key=["H", "K", "V", "BT", "BK", "BV", "IS_VARLEN"],
+)
[email protected](do_not_specialize=["T"])
+def recompute_w_u_fwd_kernel(
+ q,
+ k,
+ qg,
+ kg,
+ v,
+ beta,
+ w,
+ u,
+ A,
+ gk,
+ cu_seqlens,
+ chunk_indices,
+ T,
+ H: tl.constexpr,
+ K: tl.constexpr,
+ V: tl.constexpr,
+ BT: tl.constexpr,
+ BK: tl.constexpr,
+ BV: tl.constexpr,
+ STORE_QG: tl.constexpr,
+ STORE_KG: tl.constexpr,
+ IS_VARLEN: tl.constexpr,
+ DOT_PRECISION: tl.constexpr,
+):
+ i_t, i_bh = tl.program_id(0), tl.program_id(1)
+ i_b, i_h = i_bh // H, i_bh % H
+ if IS_VARLEN:
+ i_n, i_t = (
+ tl.load(chunk_indices + i_t * 2).to(tl.int32),
+ tl.load(chunk_indices + i_t * 2 + 1).to(tl.int32),
+ )
+ bos, eos = (
+ tl.load(cu_seqlens + i_n).to(tl.int32),
+ tl.load(cu_seqlens + i_n + 1).to(tl.int32),
+ )
+ T = eos - bos
+ else:
+ bos, eos = i_b * T, i_b * T + T
+ p_b = tl.make_block_ptr(beta + bos * H + i_h, (T,), (H,), (i_t * BT,), (BT,), (0,))
+ b_b = tl.load(p_b, boundary_check=(0,))
+
+ p_A = tl.make_block_ptr(
+ A + (bos * H + i_h) * BT, (T, BT), (H * BT, 1), (i_t * BT, 0), (BT, BT), (1, 0)
+ )
+ b_A = tl.load(p_A, boundary_check=(0, 1))
+
+ for i_v in range(tl.cdiv(V, BV)):
+ p_v = tl.make_block_ptr(
+ v + (bos * H + i_h) * V,
+ (T, V),
+ (H * V, 1),
+ (i_t * BT, i_v * BV),
+ (BT, BV),
+ (1, 0),
+ )
+ p_u = tl.make_block_ptr(
+ u + (bos * H + i_h) * V,
+ (T, V),
+ (H * V, 1),
+ (i_t * BT, i_v * BV),
+ (BT, BV),
+ (1, 0),
+ )
+ b_v = tl.load(p_v, boundary_check=(0, 1))
+ b_vb = (b_v * b_b[:, None]).to(b_v.dtype)
+ b_u = tl.dot(b_A, b_vb, input_precision=DOT_PRECISION)
+ tl.store(p_u, b_u.to(p_u.dtype.element_ty), boundary_check=(0, 1))
+
+ for i_k in range(tl.cdiv(K, BK)):
+ p_w = tl.make_block_ptr(
+ w + (bos * H + i_h) * K,
+ (T, K),
+ (H * K, 1),
+ (i_t * BT, i_k * BK),
+ (BT, BK),
+ (1, 0),
+ )
+ p_k = tl.make_block_ptr(
+ k + (bos * H + i_h) * K,
+ (T, K),
+ (H * K, 1),
+ (i_t * BT, i_k * BK),
+ (BT, BK),
+ (1, 0),
+ )
+ b_k = tl.load(p_k, boundary_check=(0, 1))
+ b_kb = b_k * b_b[:, None]
+
+ p_gk = tl.make_block_ptr(
+ gk + (bos * H + i_h) * K,
+ (T, K),
+ (H * K, 1),
+ (i_t * BT, i_k * BK),
+ (BT, BK),
+ (1, 0),
+ )
+ b_gk = tl.load(p_gk, boundary_check=(0, 1))
+ b_kb *= exp2(b_gk)
+ if STORE_QG:
+ p_q = tl.make_block_ptr(
+ q + (bos * H + i_h) * K,
+ (T, K),
+ (H * K, 1),
+ (i_t * BT, i_k * BK),
+ (BT, BK),
+ (1, 0),
+ )
+ p_qg = tl.make_block_ptr(
+ qg + (bos * H + i_h) * K,
+ (T, K),
+ (H * K, 1),
+ (i_t * BT, i_k * BK),
+ (BT, BK),
+ (1, 0),
+ )
+ b_q = tl.load(p_q, boundary_check=(0, 1))
+ b_qg = b_q * exp2(b_gk)
+ tl.store(p_qg, b_qg.to(p_qg.dtype.element_ty), boundary_check=(0, 1))
+ if STORE_KG:
+ last_idx = min(i_t * BT + BT, T) - 1
+
+ o_k = i_k * BK + tl.arange(0, BK)
+ m_k = o_k < K
+ b_gn = tl.load(
+ gk + ((bos + last_idx) * H + i_h) * K + o_k, mask=m_k, other=0.0
+ )
+ b_kg = b_k * exp2(b_gn - b_gk)
+
+ p_kg = tl.make_block_ptr(
+ kg + (bos * H + i_h) * K,
+ (T, K),
+ (H * K, 1),
+ (i_t * BT, i_k * BK),
+ (BT, BK),
+ (1, 0),
+ )
+ tl.store(p_kg, b_kg.to(p_kg.dtype.element_ty), boundary_check=(0, 1))
+
+ b_w = tl.dot(b_A, b_kb.to(b_k.dtype))
+ tl.store(p_w, b_w.to(p_w.dtype.element_ty), boundary_check=(0, 1))
+
+
+def recompute_w_u_fwd(
+ k: torch.Tensor,
+ v: torch.Tensor,
+ beta: torch.Tensor,
+ A: torch.Tensor,
+ q: torch.Tensor | None = None,
+ gk: torch.Tensor | None = None,
+ cu_seqlens: torch.Tensor | None = None,
+ chunk_indices: torch.Tensor | None = None,
+) -> tuple[torch.Tensor, torch.Tensor]:
+ B, T, H, K, V = *k.shape, v.shape[-1]
+ BT = A.shape[-1]
+ BK = 64
+ BV = 64
+
+ if chunk_indices is None and cu_seqlens is not None:
+ chunk_indices = prepare_chunk_indices(cu_seqlens, BT)
+ NT = cdiv(T, BT) if cu_seqlens is None else len(chunk_indices)
+
+ w = torch.empty_like(k)
+ u = torch.empty_like(v)
+ kg = torch.empty_like(k) if gk is not None else None
+ recompute_w_u_fwd_kernel[(NT, B * H)](
+ q=q,
+ k=k,
+ qg=None,
+ kg=kg,
+ v=v,
+ beta=beta,
+ w=w,
+ u=u,
+ A=A,
+ gk=gk,
+ cu_seqlens=cu_seqlens,
+ chunk_indices=chunk_indices,
+ T=T,
+ H=H,
+ K=K,
+ V=V,
+ BT=BT,
+ BK=BK,
+ BV=BV,
+ DOT_PRECISION="ieee",
+ )
+ return w, u, None, kg
+
+
[email protected]({"IS_VARLEN": lambda args: args["cu_seqlens"] is not None})
+ configs=[
+ triton.Config({"BK": BK, "BV": BV}, num_warps=num_warps, num_stages=num_stages)
+ for BK in [32, 64]
+ for BV in [64, 128]
+ for num_warps in [2, 4, 8]
+ for num_stages in [2, 3, 4]
+ ],
+ key=["BT"],
+)
[email protected](do_not_specialize=["T"])
+def chunk_gla_fwd_kernel_o(
+ q,
+ v,
+ g,
+ h,
+ o,
+ A,
+ cu_seqlens,
+ chunk_indices,
+ scale,
+ T,
+ H: tl.constexpr,
+ K: tl.constexpr,
+ V: tl.constexpr,
+ BT: tl.constexpr,
+ BK: tl.constexpr,
+ BV: tl.constexpr,
+ IS_VARLEN: tl.constexpr,
+):
+ i_v, i_t, i_bh = tl.program_id(0), tl.program_id(1), tl.program_id(2)
+ i_b, i_h = i_bh // H, i_bh % H
+ if IS_VARLEN:
+ i_tg = i_t
+ i_n, i_t = (
+ tl.load(chunk_indices + i_t * 2).to(tl.int32),
+ tl.load(chunk_indices + i_t * 2 + 1).to(tl.int32),
+ )
+ bos, eos = (
+ tl.load(cu_seqlens + i_n).to(tl.int32),
+ tl.load(cu_seqlens + i_n + 1).to(tl.int32),
+ )
+ T = eos - bos
+ NT = tl.cdiv(T, BT)
+ else:
+ NT = tl.cdiv(T, BT)
+ i_tg = i_b * NT + i_t
+ bos, eos = i_b * T, i_b * T + T
+
+ m_s = tl.arange(0, BT)[:, None] >= tl.arange(0, BT)[None, :]
+
+ b_o = tl.zeros([BT, BV], dtype=tl.float32)
+ for i_k in range(tl.cdiv(K, BK)):
+ p_q = tl.make_block_ptr(
+ q + (bos * H + i_h) * K,
+ (T, K),
+ (H * K, 1),
+ (i_t * BT, i_k * BK),
+ (BT, BK),
+ (1, 0),
+ )
+ p_g = tl.make_block_ptr(
+ g + (bos * H + i_h) * K,
+ (T, K),
+ (H * K, 1),
+ (i_t * BT, i_k * BK),
+ (BT, BK),
+ (1, 0),
+ )
+ p_h = tl.make_block_ptr(
+ # int64 BEFORE the K*V multiply: the int32 product wraps at chunk
+ # index 2048 (131072-token prefill at H=64, K=V=128).
+ h + (i_tg * H + i_h).to(tl.int64) * K * V,
+ (V, K),
+ (K, 1),
+ (i_v * BV, i_k * BK),
+ (BV, BK),
+ (1, 0),
+ )
+
+ # [BT, BK]
+ b_q = tl.load(p_q, boundary_check=(0, 1))
+ b_q = (b_q * scale).to(b_q.dtype)
+ # [BT, BK]
+ b_g = tl.load(p_g, boundary_check=(0, 1))
+ # [BT, BK]
+ b_qg = (b_q * exp2(b_g)).to(b_q.dtype)
+ # [BV, BK]
+ b_h = tl.load(p_h, boundary_check=(0, 1))
+ # [BT, BV]
+ if i_k >= 0:
+ b_o += tl.dot(b_qg, tl.trans(b_h).to(b_qg.dtype))
+ p_v = tl.make_block_ptr(
+ v + (bos * H + i_h) * V,
+ (T, V),
+ (H * V, 1),
+ (i_t * BT, i_v * BV),
+ (BT, BV),
+ (1, 0),
+ )
+ p_o = tl.make_block_ptr(
+ o + (bos * H + i_h) * V,
+ (T, V),
+ (H * V, 1),
+ (i_t * BT, i_v * BV),
+ (BT, BV),
+ (1, 0),
+ )
+ p_A = tl.make_block_ptr(
+ A + (bos * H + i_h) * BT, (T, BT), (H * BT, 1), (i_t * BT, 0), (BT, BT), (1, 0)
+ )
+ # [BT, BV]
+ b_v = tl.load(p_v, boundary_check=(0, 1))
+ # [BT, BT]
+ b_A = tl.load(p_A, boundary_check=(0, 1))
+ b_A = tl.where(m_s, b_A, 0.0).to(b_v.dtype)
+ b_o += tl.dot(b_A, b_v, allow_tf32=False)
+ tl.store(p_o, b_o.to(p_o.dtype.element_ty), boundary_check=(0, 1))
+
+
+def chunk_gla_fwd_o_gk(
+ q: torch.Tensor,
+ v: torch.Tensor,
+ g: torch.Tensor,
+ A: torch.Tensor,
+ h: torch.Tensor,
+ o: torch.Tensor,
+ scale: float,
+ cu_seqlens: torch.Tensor | None = None,
+ chunk_indices: torch.Tensor | None = None,
+ chunk_size: int = FLA_CHUNK_SIZE,
+):
+ B, T, H, K, V = *q.shape, v.shape[-1]
+ BT = chunk_size
+
+ if chunk_indices is None and cu_seqlens is not None:
+ chunk_indices = prepare_chunk_indices(cu_seqlens, chunk_size)
+ NT = cdiv(T, BT) if cu_seqlens is None else len(chunk_indices)
+
+ def grid(meta):
+ return (cdiv(V, meta["BV"]), NT, B * H)
+
+ chunk_gla_fwd_kernel_o[grid](
+ q=q,
+ v=v,
+ g=g,
+ h=h,
+ o=o,
+ A=A,
+ cu_seqlens=cu_seqlens,
+ chunk_indices=chunk_indices,
+ scale=scale,
+ T=T,
+ H=H,
+ K=K,
+ V=V,
+ BT=BT,
+ )
+ return o
+
+
+ {
+ "HAS_BIAS": lambda args: args["g_bias"] is not None,
+ "IS_VARLEN": lambda args: args["cu_seqlens"] is not None,
+ }
+)
+ configs=[
+ triton.Config({"BD": BD}, num_warps=num_warps)
+ for BD in [32, 64]
+ for num_warps in [2, 4, 8]
+ ],
+ key=["H", "D", "BT", "IS_VARLEN"],
+)
[email protected](do_not_specialize=["T"])
+def kda_gate_cumsum_fwd_kernel(
+ g,
+ A,
+ y,
+ g_bias,
+ cu_seqlens,
+ chunk_indices,
+ cumsum_scale,
+ beta,
+ threshold,
+ SAFE_GATE: tl.constexpr,
+ LOWER_BOUND: tl.constexpr,
+ T,
+ H: tl.constexpr,
+ D: tl.constexpr,
+ BT: tl.constexpr,
+ BD: tl.constexpr,
+ HAS_BIAS: tl.constexpr,
+ IS_VARLEN: tl.constexpr,
+):
+ i_d, i_t, i_bh = tl.program_id(0), tl.program_id(1), tl.program_id(2)
+ i_b, i_h = i_bh // H, i_bh % H
+ if IS_VARLEN:
+ i_n, i_t = (
+ tl.load(chunk_indices + i_t * 2).to(tl.int32),
+ tl.load(chunk_indices + i_t * 2 + 1).to(tl.int32),
+ )
+ bos, eos = (
+ tl.load(cu_seqlens + i_n).to(tl.int32),
+ tl.load(cu_seqlens + i_n + 1).to(tl.int32),
+ )
+ T = eos - bos
+ else:
+ bos = i_b * T
+
+ p_g = tl.make_block_ptr(
+ g + (bos * H + i_h) * D,
+ (T, D),
+ (H * D, 1),
+ (i_t * BT, i_d * BD),
+ (BT, BD),
+ (1, 0),
+ )
+ p_y = tl.make_block_ptr(
+ y + (bos * H + i_h) * D,
+ (T, D),
+ (H * D, 1),
+ (i_t * BT, i_d * BD),
+ (BT, BD),
+ (1, 0),
+ )
+
+ b_g = tl.load(p_g, boundary_check=(0, 1)).to(tl.float32)
+ if HAS_BIAS:
+ o_d = i_d * BD + tl.arange(0, BD)
+ b_bias = tl.load(g_bias + i_h * D + o_d, mask=o_d < D, other=0.0).to(tl.float32)
+ b_g = b_g + b_bias[None, :]
+
+ b_a = tl.load(A + i_h).to(tl.float32)
+ b_a = tl.exp(b_a) if SAFE_GATE else -tl.exp(b_a)
+ if SAFE_GATE:
+ # y = lower_bound * sigmoid(exp(A) * (g + g_bias)), bounded to
+ # (lower_bound, 0) for safe-gate checkpoints.
+ b_gate = LOWER_BOUND / (1.0 + tl.exp(-(b_a * b_g)))
+ else:
+ b_g_scaled = b_g * beta
+ b_softplus = tl.where(
+ b_g_scaled > threshold,
+ b_g,
+ (1.0 / beta) * log(1.0 + tl.exp(b_g_scaled)),
+ )
+ b_gate = b_a * b_softplus
+
+ # Out-of-bounds rows (load returns 0, but softplus/bias can still make
+ # b_gate non-zero) participate in the dot product. They only contribute to
+ # out-of-bounds output rows, which are masked away by `boundary_check` on
+ # the store, so visible output matches unfused gate + chunk-local cumsum.
+ o_t = tl.arange(0, BT)
+ m_cumsum = tl.where(o_t[:, None] >= o_t[None, :], 1.0, 0.0)
+ b_y = tl.dot(m_cumsum, b_gate, allow_tf32=False) * cumsum_scale
+ tl.store(p_y, b_y.to(p_y.dtype.element_ty), boundary_check=(0, 1))
+
+
+def fused_kda_gate_chunk_cumsum(
+ raw_g: torch.Tensor,
+ A_log: torch.Tensor,
+ g_bias: torch.Tensor | None = None,
+ beta: float = 1.0,
+ threshold: float = 20.0,
+ cu_seqlens: torch.Tensor | None = None,
+ chunk_indices: torch.Tensor | None = None,
+ chunk_size: int = FLA_CHUNK_SIZE,
+ output_dtype: torch.dtype | None = torch.float,
+ safe_gate: bool = False,
+ lower_bound: float = -5.0,
+) -> torch.Tensor:
+ if cu_seqlens is not None:
+ assert raw_g.shape[0] == 1, (
+ "Only batch size 1 is supported when cu_seqlens are provided"
+ )
+ B, T, H, D = raw_g.shape
+ if chunk_indices is None and cu_seqlens is not None:
+ chunk_indices = prepare_chunk_indices(cu_seqlens, chunk_size)
+ NT = cdiv(T, chunk_size) if cu_seqlens is None else len(chunk_indices)
+
+ A_log = A_log.reshape(-1)
+ if g_bias is not None:
+ g_bias = g_bias.reshape(-1)
+ y = torch.empty_like(raw_g, dtype=output_dtype or raw_g.dtype)
+
+ def grid(meta):
+ return (cdiv(meta["D"], meta["BD"]), NT, B * H)
+
+ kda_gate_cumsum_fwd_kernel[grid](
+ g=raw_g,
+ A=A_log,
+ y=y,
+ g_bias=g_bias,
+ cu_seqlens=cu_seqlens,
+ chunk_indices=chunk_indices,
+ # RCP_LN2 folds in the natural-log -> log2 conversion so downstream
+ # exp2-based kernels reproduce exp(g). Keep this in sync with the
+ # `use_exp2=True` path in `_chunk_kda_fwd_with_cumulative_g`.
+ cumsum_scale=RCP_LN2,
+ beta=beta,
+ threshold=threshold,
+ SAFE_GATE=safe_gate,
+ LOWER_BOUND=lower_bound,
+ T=T,
+ H=H,
+ D=D,
+ BT=chunk_size,
+ )
+ return y
+
+
+def _chunk_kda_fwd_with_cumulative_g(
+ q: torch.Tensor,
+ k: torch.Tensor,
+ v: torch.Tensor,
+ g: torch.Tensor,
+ beta: torch.Tensor,
+ scale: float,
+ initial_state: torch.Tensor,
+ output_final_state: bool,
+ cu_seqlens: torch.Tensor | None = None,
+ chunk_indices: torch.Tensor | None = None,
+ chunk_size: int = FLA_CHUNK_SIZE,
+ return_h: bool = False,
+):
+ # Token-offset addressing (q/k/g at `(bos*H + i_h) * K`) is still int32 in the
+ # intra-chunk kernels: refuse a varlen batch long enough to wrap rather than
+ # silently corrupting (the chunk-STATE offsets are int64 and do not bind first).
+ _, _T, _H, _K = k.shape # [1, total_tokens, H, K] (varlen packs on dim 1)
+ if _T * _H * _K >= 2**31:
+ raise ValueError(
+ f"KDA prefill batch too long for int32 token addressing: {_T} tokens "
+ f"x H={_H} x K={_K} exceeds 2**31; keep --max-prefill-length below "
+ f"{2**31 // (_H * _K)} tokens."
+ )
+ # `g` must already be chunk-local cumulatively-summed AND scaled by
+ # RCP_LN2 (so the downstream exp2-based kernels reproduce exp(g)).
+ # Use `chunk_kda_fwd` or `chunk_kda_with_fused_gate_fwd` instead of
+ # calling this helper directly unless that invariant is upheld.
+ # the intra Aqk is kept in fp32
+ # the computation has very marginal effect on the entire throughput
+ A, Aqk = chunk_kda_scaled_dot_kkt_fwd(
+ q=q,
+ k=k,
+ gk=g,
+ beta=beta,
+ scale=scale,
+ cu_seqlens=cu_seqlens,
+ chunk_indices=chunk_indices,
+ chunk_size=chunk_size,
+ output_dtype=torch.float32,
+ )
+ A = solve_tril(A=A, cu_seqlens=cu_seqlens, output_dtype=k.dtype)
+ w, u, _, kg = recompute_w_u_fwd(
+ k=k,
+ v=v,
+ beta=beta,
+ A=A,
+ gk=g,
+ cu_seqlens=cu_seqlens,
+ chunk_indices=chunk_indices,
+ )
+ del A
+ h, v_new, final_state = chunk_gated_delta_rule_fwd_h(
+ k=kg,
+ w=w,
+ u=u,
+ gk=g,
+ initial_state=initial_state,
+ output_final_state=output_final_state,
+ cu_seqlens=cu_seqlens,
+ chunk_indices=chunk_indices,
+ use_exp2=True,
+ )
+ del w, u, kg
+ o = chunk_gla_fwd_o_gk(
+ q=q,
+ v=v_new,
+ g=g,
+ A=Aqk,
+ h=h,
+ o=v,
+ scale=scale,
+ cu_seqlens=cu_seqlens,
+ chunk_indices=chunk_indices,
+ chunk_size=chunk_size,
+ )
+ del Aqk, v_new
+ if return_h:
+ # FreeToken addition: expose the per-chunk state snapshots (h[b, i] is the
+ # state at the START of chunk i) for hybrid-radix track checkpoints.
+ return o, final_state, h
+ del h
+ return o, final_state
+
+
+def chunk_kda_fwd(
+ q: torch.Tensor,
+ k: torch.Tensor,
+ v: torch.Tensor,
+ g: torch.Tensor,
+ beta: torch.Tensor,
+ scale: float,
+ initial_state: torch.Tensor,
+ output_final_state: bool,
+ cu_seqlens: torch.Tensor | None = None,
+):
+ chunk_size = FLA_CHUNK_SIZE
+ chunk_indices = (
+ prepare_chunk_indices(cu_seqlens, chunk_size)
+ if cu_seqlens is not None
+ else None
+ )
+ g = chunk_local_cumsum(
+ g,
+ chunk_size=chunk_size,
+ cu_seqlens=cu_seqlens,
+ chunk_indices=chunk_indices,
+ )
+ # KDA evaluates cumulative gate decays with exp2. Convert from natural-log
+ # space so exp(x) is preserved as exp2(x / ln(2)).
+ g = g * RCP_LN2
+ return _chunk_kda_fwd_with_cumulative_g(
+ q=q,
+ k=k,
+ v=v,
+ g=g,
+ beta=beta,
+ scale=scale,
+ initial_state=initial_state,
+ output_final_state=output_final_state,
+ cu_seqlens=cu_seqlens,
+ chunk_indices=chunk_indices,
+ chunk_size=chunk_size,
+ )
+
+
+def chunk_kda_with_fused_gate_fwd(
+ q: torch.Tensor,
+ k: torch.Tensor,
+ v: torch.Tensor,
+ raw_g: torch.Tensor,
+ beta: torch.Tensor,
+ A_log: torch.Tensor,
+ g_bias: torch.Tensor | None,
+ scale: float,
+ initial_state: torch.Tensor,
+ output_final_state: bool,
+ cu_seqlens: torch.Tensor | None = None,
+ safe_gate: bool = False,
+ lower_bound: float = -5.0,
+ return_h: bool = False,
+):
+ chunk_size = FLA_CHUNK_SIZE
+ chunk_indices = (
+ prepare_chunk_indices(cu_seqlens, chunk_size)
+ if cu_seqlens is not None
+ else None
+ )
+ g = fused_kda_gate_chunk_cumsum(
+ raw_g,
+ A_log=A_log,
+ g_bias=g_bias,
+ cu_seqlens=cu_seqlens,
+ chunk_indices=chunk_indices,
+ chunk_size=chunk_size,
+ safe_gate=safe_gate,
+ lower_bound=lower_bound,
+ )
+ return _chunk_kda_fwd_with_cumulative_g(
+ q=q,
+ k=k,
+ v=v,
+ g=g,
+ beta=beta,
+ scale=scale,
+ initial_state=initial_state,
+ output_final_state=output_final_state,
+ cu_seqlens=cu_seqlens,
+ chunk_indices=chunk_indices,
+ chunk_size=chunk_size,
+ return_h=return_h,
+ )
+
+
+def chunk_kda(
+ q: torch.Tensor,
+ k: torch.Tensor,
+ v: torch.Tensor,
+ g: torch.Tensor,
+ beta: torch.Tensor,
+ scale: float = None,
+ initial_state: torch.Tensor = None,
+ output_final_state: bool = False,
+ use_qk_l2norm_in_kernel: bool = False,
+ cu_seqlens: torch.Tensor | None = None,
+ **kwargs,
+):
+ if scale is None:
+ scale = k.shape[-1] ** -0.5
+
+ if use_qk_l2norm_in_kernel:
+ q = l2norm_fwd(q.contiguous())
+ k = l2norm_fwd(k.contiguous())
+
+ o, final_state = chunk_kda_fwd(
+ q=q,
+ k=k,
+ v=v.contiguous(),
+ g=g.contiguous(),
+ beta=beta.contiguous(),
+ scale=scale,
+ initial_state=initial_state.contiguous(),
+ output_final_state=output_final_state,
+ cu_seqlens=cu_seqlens,
+ )
+ return o, final_state
+
+
+def chunk_kda_with_fused_gate(
+ q: torch.Tensor,
+ k: torch.Tensor,
+ v: torch.Tensor,
+ raw_g: torch.Tensor,
+ beta: torch.Tensor,
+ A_log: torch.Tensor,
+ g_bias: torch.Tensor | None,
+ scale: float | None = None,
+ initial_state: torch.Tensor | None = None,
+ output_final_state: bool = False,
+ use_qk_l2norm_in_kernel: bool = False,
+ cu_seqlens: torch.Tensor | None = None,
+ safe_gate: bool = False,
+ lower_bound: float = -5.0,
+ return_h: bool = False,
+ **kwargs,
+):
+ """Run chunk KDA from raw gate projection using fused gate+cumsum.
+
+ WARNING: the output is written into (and returned as) the ``v`` buffer; never
+ pass a tensor that is read again after this call. With ``return_h`` the
+ per-chunk state snapshots (h[b, i] = state at the START of chunk i) are
+ returned as a third value, for hybrid-radix track checkpoints.
+ """
+ if scale is None:
+ scale = k.shape[-1] ** -0.5
+
+ if use_qk_l2norm_in_kernel:
+ q = l2norm_fwd(q.contiguous())
+ k = l2norm_fwd(k.contiguous())
+
+ return chunk_kda_with_fused_gate_fwd(
+ q=q,
+ k=k,
+ v=v.contiguous(),
+ raw_g=raw_g.contiguous(),
+ beta=beta.contiguous(),
+ A_log=A_log,
+ g_bias=g_bias,
+ scale=scale,
+ initial_state=initial_state.contiguous() if initial_state is not None else None,
+ output_final_state=output_final_state,
+ cu_seqlens=cu_seqlens,
+ safe_gate=safe_gate,
+ lower_bound=lower_bound,
+ return_h=return_h,
+ )
+
+
+ configs=[
+ triton.Config({"BT": bt}, num_warps=nw, num_stages=ns)
+ for bt in BT_LIST_AUTOTUNE
+ for nw in NUM_WARPS_AUTOTUNE
+ for ns in [2, 3]
+ ],
+ key=["H", "D"],
+)
+def kda_gate_fwd_kernel(
+ g,
+ A,
+ y,
+ g_bias,
+ beta: tl.constexpr,
+ threshold: tl.constexpr,
+ SAFE_GATE: tl.constexpr,
+ LOWER_BOUND: tl.constexpr,
+ T,
+ H,
+ D: tl.constexpr,
+ BT: tl.constexpr,
+ BD: tl.constexpr,
+ HAS_BIAS: tl.constexpr,
+):
+ i_t, i_h = tl.program_id(0), tl.program_id(1)
+ n_t = i_t * BT
+
+ b_a = tl.load(A + i_h).to(tl.float32)
+ b_a = tl.exp(b_a) if SAFE_GATE else -tl.exp(b_a)
+
+ stride_row = H * D
+ stride_col = 1
+
+ g_ptr = tl.make_block_ptr(
+ base=g + i_h * D,
+ shape=(T, D),
+ strides=(stride_row, stride_col),
+ offsets=(n_t, 0),
+ block_shape=(BT, BD),
+ order=(1, 0),
+ )
+
+ y_ptr = tl.make_block_ptr(
+ base=y + i_h * D,
+ shape=(T, D),
+ strides=(stride_row, stride_col),
+ offsets=(n_t, 0),
+ block_shape=(BT, BD),
+ order=(1, 0),
+ )
+
+ b_g = tl.load(g_ptr, boundary_check=(0, 1)).to(tl.float32)
+
+ if HAS_BIAS:
+ n_d = tl.arange(0, BD)
+ bias_mask = n_d < D
+ b_bias = tl.load(g_bias + i_h * D + n_d, mask=bias_mask, other=0.0).to(
+ tl.float32
+ )
+ b_g = b_g + b_bias[None, :]
+
+ if SAFE_GATE:
+ # y = lower_bound * sigmoid(exp(A) * (g + g_bias)), bounded to
+ # (lower_bound, 0) for safe-gate checkpoints.
+ b_y = LOWER_BOUND / (1.0 + tl.exp(-(b_a * b_g)))
+ else:
+ # softplus(x, beta) = (1/beta) * log(1 + exp(beta * x))
+ # When beta * x > threshold, use linear approximation x
+ # Use threshold to switch to linear when beta*x > threshold
+ g_scaled = b_g * beta
+ use_linear = g_scaled > threshold
+ sp = tl.where(use_linear, b_g, (1.0 / beta) * log(1.0 + tl.exp(g_scaled)))
+ b_y = b_a * sp
+
+ tl.store(y_ptr, b_y.to(y.dtype.element_ty), boundary_check=(0, 1))
+
+
+def fused_kda_gate(
+ g: torch.Tensor,
+ A: torch.Tensor,
+ head_k_dim: int,
+ g_bias: torch.Tensor | None = None,
+ beta: float = 1.0,
+ threshold: float = 20.0,
+ safe_gate: bool = False,
+ lower_bound: float | None = -5.0,
+) -> torch.Tensor:
+ """
+ Forward pass for KDA gate:
+ input g: [..., H*D]
+ param A: [H] or [1, 1, H, 1]
+ beta: softplus beta parameter (softplus branch only)
+ threshold: softplus threshold parameter (softplus branch only)
+ safe_gate: when False (default) compute y = -exp(A)*softplus(g+g_bias);
+ when True compute the bounded y = lower_bound*sigmoid(exp(A)*(g+g_bias))
+ lower_bound: floor for the safe_gate branch (default -5.0)
+ return : [..., H, D]
+ """
+ orig_shape = g.shape[:-1]
+
+ g = g.view(-1, g.shape[-1])
+ T = g.shape[0]
+ HD = g.shape[1]
+ H = A.numel()
+ assert H * head_k_dim == HD
+
+ y = torch.empty_like(g, dtype=torch.float32)
+
+ def grid(meta):
+ return (cdiv(T, meta["BT"]), H)
+
+ kda_gate_fwd_kernel[grid](
+ g,
+ A,
+ y,
+ g_bias,
+ beta,
+ threshold,
+ safe_gate,
+ lower_bound if lower_bound is not None else -5.0,
+ T,
+ H,
+ head_k_dim,
+ BD=next_power_of_2(head_k_dim),
+ HAS_BIAS=g_bias is not None,
+ )
+
+ y = y.view(*orig_shape, H, head_k_dim)
+ return y
+
diff --git a/python/freetoken/kernel/fla/kda_chunk_delta_h.py b/python/freetoken/kernel/fla/kda_chunk_delta_h.py
new file mode 100644
index 00000000..55c0ffb2
--- /dev/null
+++ b/python/freetoken/kernel/fla/kda_chunk_delta_h.py
@@ -0,0 +1,394 @@
+# Vendored from vLLM's third_party/flash_linear_attention (PR #53906, commit 933876c3),
+# itself copied from the flash-linear-attention project (MIT, (c) 2023-2025 Songlin Yang,
+# Yu Zhang). Imports are remapped onto freetoken.kernel.fla's shared helpers; keep this
+# file in sync with upstream when pulling KDA kernel fixes.
+# NOTE: this is the KDA-consistent variant (exp2 gate semantics via use_exp2, returns
+# the final state). freetoken/kernel/fla/chunk_delta_h.py is the GDN variant (natural-exp
+# gk, in-place pool state via initial_state_indices); the two serve different recurrences
+# and are kept separate on purpose.
+# SPDX-License-Identifier: Apache-2.0
+# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
+# SPDX-FileCopyrightText: Songlin Yang, Yu Zhang
+#
+# This file contains code copied from the flash-linear-attention project.
+# The original source code was licensed under the MIT license and included
+# the following copyright notice:
+# Copyright (c) 2023-2025, Songlin Yang, Yu Zhang
+# ruff: noqa: E501
+
+import torch
+
+import triton
+import triton.language as tl
+
+from .index import prepare_chunk_indices, prepare_chunk_offsets
+from .op import exp, exp2
+from .utils import FLA_CHUNK_SIZE, use_cuda_graph
+
+NUM_WARPS = [2, 4, 8, 16]
+# Triton's AMD backend fails to lower this kernel with num_stages=4.
+_CHUNK_DELTA_H_NUM_STAGES = [2, 3] if torch.version.hip else [2, 3, 4]
+
+
+ {
+ "USE_G": lambda args: args["g"] is not None,
+ "USE_GK": lambda args: args["gk"] is not None,
+ "USE_INITIAL_STATE": lambda args: args["h0"] is not None,
+ "STORE_FINAL_STATE": lambda args: args["ht"] is not None,
+ "SAVE_NEW_VALUE": lambda args: args["v_new"] is not None,
+ "IS_VARLEN": lambda args: args["cu_seqlens"] is not None,
+ }
+)
+ configs=[
+ triton.Config({"BV": BV}, num_warps=num_warps, num_stages=num_stages)
+ for num_warps in [2, 4]
+ for num_stages in _CHUNK_DELTA_H_NUM_STAGES
+ for BV in [32, 64]
+ ],
+ key=["H", "K", "V", "BT"],
+ use_cuda_graph=use_cuda_graph,
+)
[email protected](do_not_specialize=["T"])
+def chunk_gated_delta_rule_fwd_kernel_h_blockdim64(
+ k,
+ v,
+ w,
+ v_new,
+ g,
+ gk,
+ h,
+ h0,
+ ht,
+ cu_seqlens,
+ chunk_offsets,
+ T,
+ H: tl.constexpr,
+ Hg: tl.constexpr,
+ K: tl.constexpr,
+ V: tl.constexpr,
+ BT: tl.constexpr,
+ BV: tl.constexpr,
+ USE_G: tl.constexpr,
+ USE_GK: tl.constexpr,
+ USE_INITIAL_STATE: tl.constexpr,
+ STORE_FINAL_STATE: tl.constexpr,
+ SAVE_NEW_VALUE: tl.constexpr,
+ IS_VARLEN: tl.constexpr,
+ USE_EXP2: tl.constexpr,
+):
+ i_v, i_nh = tl.program_id(0), tl.program_id(1)
+ i_n, i_h = i_nh // H, i_nh % H
+ if IS_VARLEN:
+ bos, eos = (
+ tl.load(cu_seqlens + i_n).to(tl.int32),
+ tl.load(cu_seqlens + i_n + 1).to(tl.int32),
+ )
+ T = eos - bos
+ NT = tl.cdiv(T, BT)
+ boh = tl.load(chunk_offsets + i_n).to(tl.int32)
+ else:
+ bos, eos = i_n * T, i_n * T + T
+ NT = tl.cdiv(T, BT)
+ boh = i_n * NT
+
+ # [BV, BK]
+ b_h1 = tl.zeros([BV, 64], dtype=tl.float32)
+ if K > 64:
+ b_h2 = tl.zeros([BV, 64], dtype=tl.float32)
+ if K > 128:
+ b_h3 = tl.zeros([BV, 64], dtype=tl.float32)
+ if K > 192:
+ b_h4 = tl.zeros([BV, 64], dtype=tl.float32)
+
+ # calculate offset. DIVERGENCE from upstream: cast to int64 BEFORE the K*V
+ # multiply (the GDN chunk_o.py idiom) -- upstream casts the already-wrapped
+ # int32 product, which overflows at chunk index 2048 (a 131072-token prefill
+ # at H=64, K=V=128) and silently lands in another head's state.
+ h += (boh * H + i_h).to(tl.int64) * V * K
+ v += (bos * H + i_h).to(tl.int64) * V
+ k += (bos * Hg + i_h // (H // Hg)).to(tl.int64) * K
+ w += (bos * H + i_h).to(tl.int64) * K
+ if SAVE_NEW_VALUE:
+ v_new += (bos * H + i_h).to(tl.int64) * V
+ stride_v = H * V
+ stride_h = H * V * K
+ stride_k = Hg * K
+ stride_w = H * K
+ if USE_INITIAL_STATE:
+ h0 = h0 + i_nh * V * K
+ if STORE_FINAL_STATE:
+ ht = ht + i_nh * V * K
+
+ # load initial state
+ if USE_INITIAL_STATE:
+ p_h0_1 = tl.make_block_ptr(h0, (V, K), (K, 1), (i_v * BV, 0), (BV, 64), (1, 0))
+ b_h1 += tl.load(p_h0_1, boundary_check=(0, 1)).to(tl.float32)
+ if K > 64:
+ p_h0_2 = tl.make_block_ptr(
+ h0, (V, K), (K, 1), (i_v * BV, 64), (BV, 64), (1, 0)
+ )
+ b_h2 += tl.load(p_h0_2, boundary_check=(0, 1)).to(tl.float32)
+ if K > 128:
+ p_h0_3 = tl.make_block_ptr(
+ h0, (V, K), (K, 1), (i_v * BV, 128), (BV, 64), (1, 0)
+ )
+ b_h3 += tl.load(p_h0_3, boundary_check=(0, 1)).to(tl.float32)
+ if K > 192:
+ p_h0_4 = tl.make_block_ptr(
+ h0, (V, K), (K, 1), (i_v * BV, 192), (BV, 64), (1, 0)
+ )
+ b_h4 += tl.load(p_h0_4, boundary_check=(0, 1)).to(tl.float32)
+
+ # main recurrence
+ for i_t in range(NT):
+ p_h1 = tl.make_block_ptr(
+ h + i_t.to(tl.int64) * stride_h,
+ (V, K),
+ (K, 1),
+ (i_v * BV, 0),
+ (BV, 64),
+ (1, 0),
+ )
+ tl.store(p_h1, b_h1.to(p_h1.dtype.element_ty), boundary_check=(0, 1))
+ if K > 64:
+ p_h2 = tl.make_block_ptr(
+ h + i_t.to(tl.int64) * stride_h,
+ (V, K),
+ (K, 1),
+ (i_v * BV, 64),
+ (BV, 64),
+ (1, 0),
+ )
+ tl.store(p_h2, b_h2.to(p_h2.dtype.element_ty), boundary_check=(0, 1))
+ if K > 128:
+ p_h3 = tl.make_block_ptr(
+ h + i_t.to(tl.int64) * stride_h,
+ (V, K),
+ (K, 1),
+ (i_v * BV, 128),
+ (BV, 64),
+ (1, 0),
+ )
+ tl.store(p_h3, b_h3.to(p_h3.dtype.element_ty), boundary_check=(0, 1))
+ if K > 192:
+ p_h4 = tl.make_block_ptr(
+ h + i_t.to(tl.int64) * stride_h,
+ (V, K),
+ (K, 1),
+ (i_v * BV, 192),
+ (BV, 64),
+ (1, 0),
+ )
+ tl.store(p_h4, b_h4.to(p_h4.dtype.element_ty), boundary_check=(0, 1))
+
+ p_w = tl.make_block_ptr(
+ w, (T, K), (stride_w, 1), (i_t * BT, 0), (BT, 64), (1, 0)
+ )
+ b_w = tl.load(p_w, boundary_check=(0, 1))
+ b_v = tl.dot(b_w, tl.trans(b_h1).to(b_w.dtype))
+ if K > 64:
+ p_w = tl.make_block_ptr(
+ w, (T, K), (stride_w, 1), (i_t * BT, 64), (BT, 64), (1, 0)
+ )
+ b_w = tl.load(p_w, boundary_check=(0, 1))
+ b_v += tl.dot(b_w, tl.trans(b_h2).to(b_w.dtype))
+ if K > 128:
+ p_w = tl.make_block_ptr(
+ w, (T, K), (stride_w, 1), (i_t * BT, 128), (BT, 64), (1, 0)
+ )
+ b_w = tl.load(p_w, boundary_check=(0, 1))
+ b_v += tl.dot(b_w, tl.trans(b_h3).to(b_w.dtype))
+ if K > 192:
+ p_w = tl.make_block_ptr(
+ w, (T, K), (stride_w, 1), (i_t * BT, 192), (BT, 64), (1, 0)
+ )
+ b_w = tl.load(p_w, boundary_check=(0, 1))
+ b_v += tl.dot(b_w, tl.trans(b_h4).to(b_w.dtype))
+ p_v = tl.make_block_ptr(
+ v, (T, V), (stride_v, 1), (i_t * BT, i_v * BV), (BT, BV), (1, 0)
+ )
+ b_v = tl.load(p_v, boundary_check=(0, 1)) - b_v
+
+ if SAVE_NEW_VALUE:
+ p_v = tl.make_block_ptr(
+ v_new, (T, V), (stride_v, 1), (i_t * BT, i_v * BV), (BT, BV), (1, 0)
+ )
+ tl.store(p_v, b_v.to(p_v.dtype.element_ty), boundary_check=(0, 1))
+
+ last_idx = min((i_t.to(tl.int64) + 1) * BT, T) - 1
+ if USE_G:
+ m_t = (i_t.to(tl.int64) * BT + tl.arange(0, BT)) < T
+ b_g_last = tl.load(g + bos * H + last_idx * H + i_h)
+ p_g = tl.make_block_ptr(
+ g + bos * H + i_h, (T,), (H,), (i_t * BT,), (BT,), (0,)
+ )
+ b_g = tl.load(p_g, boundary_check=(0,))
+ if USE_EXP2:
+ b_v = b_v * tl.where(m_t, exp2(b_g_last - b_g), 0)[:, None]
+ b_g_last = exp2(b_g_last)
+ else:
+ b_v = b_v * tl.where(m_t, exp(b_g_last - b_g), 0)[:, None]
+ b_g_last = exp(b_g_last)
+ b_h1 *= b_g_last
+ if K > 64:
+ b_h2 *= b_g_last
+ if K > 128:
+ b_h3 *= b_g_last
+ if K > 192:
+ b_h4 *= b_g_last
+
+ if USE_GK:
+ o_k1 = tl.arange(0, 64)
+ b_gk_last1 = tl.load(
+ gk + (bos + last_idx) * H * K + i_h * K + o_k1,
+ mask=(o_k1 < K),
+ other=0.0,
+ )
+ if USE_EXP2:
+ b_h1 *= exp2(b_gk_last1)[None, :]
+ else:
+ b_h1 *= exp(b_gk_last1)[None, :]
+ if K > 64:
+ o_k2 = 64 + o_k1
+ b_gk_last2 = tl.load(
+ gk + (bos + last_idx) * H * K + i_h * K + o_k2,
+ mask=(o_k2 < K),
+ other=0.0,
+ )
+ if USE_EXP2:
+ b_h2 *= exp2(b_gk_last2)[None, :]
+ else:
+ b_h2 *= exp(b_gk_last2)[None, :]
+ if K > 128:
+ o_k3 = 128 + o_k1
+ b_gk_last3 = tl.load(
+ gk + (bos + last_idx) * H * K + i_h * K + o_k3,
+ mask=(o_k3 < K),
+ other=0.0,
+ )
+ if USE_EXP2:
+ b_h3 *= exp2(b_gk_last3)[None, :]
+ else:
+ b_h3 *= exp(b_gk_last3)[None, :]
+ if K > 192:
+ o_k4 = 192 + o_k1
+ b_gk_last4 = tl.load(
+ gk + (bos + last_idx) * H * K + i_h * K + o_k4,
+ mask=(o_k4 < K),
+ other=0.0,
+ )
+ if USE_EXP2:
+ b_h4 *= exp2(b_gk_last4)[None, :]
+ else:
+ b_h4 *= exp(b_gk_last4)[None, :]
+ b_v = b_v.to(k.dtype.element_ty)
+
+ p_k = tl.make_block_ptr(
+ k, (K, T), (1, stride_k), (0, i_t * BT), (64, BT), (0, 1)
+ )
+ b_k = tl.load(p_k, boundary_check=(0, 1))
+ b_h1 += tl.trans(tl.dot(b_k, b_v))
+ if K > 64:
+ p_k = tl.make_block_ptr(
+ k, (K, T), (1, stride_k), (64, i_t * BT), (64, BT), (0, 1)
+ )
+ b_k = tl.load(p_k, boundary_check=(0, 1))
+ b_h2 += tl.trans(tl.dot(b_k, b_v))
+ if K > 128:
+ p_k = tl.make_block_ptr(
+ k, (K, T), (1, stride_k), (128, i_t * BT), (64, BT), (0, 1)
+ )
+ b_k = tl.load(p_k, boundary_check=(0, 1))
+ b_h3 += tl.trans(tl.dot(b_k, b_v))
+ if K > 192:
+ p_k = tl.make_block_ptr(
+ k, (K, T), (1, stride_k), (192, i_t * BT), (64, BT), (0, 1)
+ )
+ b_k = tl.load(p_k, boundary_check=(0, 1))
+ b_h4 += tl.trans(tl.dot(b_k, b_v))
+ # epilogue
+ if STORE_FINAL_STATE:
+ p_ht = tl.make_block_ptr(ht, (V, K), (K, 1), (i_v * BV, 0), (BV, 64), (1, 0))
+ tl.store(p_ht, b_h1.to(p_ht.dtype.element_ty), boundary_check=(0, 1))
+ if K > 64:
+ p_ht = tl.make_block_ptr(
+ ht, (V, K), (K, 1), (i_v * BV, 64), (BV, 64), (1, 0)
+ )
+ tl.store(p_ht, b_h2.to(p_ht.dtype.element_ty), boundary_check=(0, 1))
+ if K > 128:
+ p_ht = tl.make_block_ptr(
+ ht, (V, K), (K, 1), (i_v * BV, 128), (BV, 64), (1, 0)
+ )
+ tl.store(p_ht, b_h3.to(p_ht.dtype.element_ty), boundary_check=(0, 1))
+ if K > 192:
+ p_ht = tl.make_block_ptr(
+ ht, (V, K), (K, 1), (i_v * BV, 192), (BV, 64), (1, 0)
+ )
+ tl.store(p_ht, b_h4.to(p_ht.dtype.element_ty), boundary_check=(0, 1))
+
+
+def chunk_gated_delta_rule_fwd_h(
+ k: torch.Tensor,
+ w: torch.Tensor,
+ u: torch.Tensor,
+ g: torch.Tensor | None = None,
+ gk: torch.Tensor | None = None,
+ initial_state: torch.Tensor | None = None,
+ output_final_state: bool = False,
+ chunk_size: int = FLA_CHUNK_SIZE,
+ save_new_value: bool = True,
+ cu_seqlens: torch.Tensor | None = None,
+ chunk_indices: torch.Tensor | None = None,
+ chunk_offsets: torch.Tensor | None = None,
+ use_exp2: bool = False,
+) -> tuple[torch.Tensor, torch.Tensor]:
+ # This kernel is slightly different from fla to support Q/K with different head numbers.
+ # In fla, Q/K always have the same head number, so Hg is always equal to H.
+ B, T, Hg, K, V = *k.shape, u.shape[-1]
+ H = u.shape[-2]
+ BT = chunk_size
+
+ if chunk_indices is None and cu_seqlens is not None:
+ chunk_indices = prepare_chunk_indices(cu_seqlens, chunk_size)
+ # N: the actual number of sequences in the batch with either equal or variable lengths
+ if cu_seqlens is None:
+ N, NT, chunk_offsets = B, triton.cdiv(T, BT), None
+ else:
+ N, NT = len(cu_seqlens) - 1, len(chunk_indices)
+ if chunk_offsets is None:
+ chunk_offsets = prepare_chunk_offsets(cu_seqlens, BT)
+ assert K <= 256, "current kernel does not support head dimension larger than 256."
+
+ h = k.new_empty(B, NT, H, V, K)
+ final_state = (
+ k.new_empty(N, H, V, K, dtype=torch.float32) if output_final_state else None
+ )
+
+ v_new = torch.empty_like(u) if save_new_value else None
+
+ def grid(meta):
+ return (triton.cdiv(V, meta["BV"]), N * H)
+
+ chunk_gated_delta_rule_fwd_kernel_h_blockdim64[grid](
+ k=k,
+ v=u,
+ w=w,
+ v_new=v_new,
+ g=g,
+ gk=gk,
+ h=h,
+ h0=initial_state,
+ ht=final_state,
+ cu_seqlens=cu_seqlens,
+ chunk_offsets=chunk_offsets,
+ T=T,
+ H=H,
+ Hg=Hg,
+ K=K,
+ V=V,
+ BT=BT,
+ USE_EXP2=use_exp2,
+ )
+ return h, v_new, final_state
diff --git a/python/freetoken/kernel/fla/solve_tril.py b/python/freetoken/kernel/fla/solve_tril.py
new file mode 100644
index 00000000..a8cad058
--- /dev/null
+++ b/python/freetoken/kernel/fla/solve_tril.py
@@ -0,0 +1,563 @@
+# Vendored from vLLM's third_party/flash_linear_attention (PR #53906, commit 933876c3),
+# itself copied from the flash-linear-attention project (MIT, (c) 2023-2025 Songlin Yang,
+# Yu Zhang). Imports are remapped onto freetoken.kernel.fla's shared helpers; keep this
+# file in sync with upstream when pulling KDA kernel fixes.
+# SPDX-License-Identifier: Apache-2.0
+# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
+# SPDX-FileCopyrightText: Songlin Yang, Yu Zhang
+#
+# This file contains code copied from the flash-linear-attention project.
+# The original source code was licensed under the MIT license and included
+# the following copyright notice:
+# Copyright (c) 2023-2025, Songlin Yang, Yu Zhang
+# ruff: noqa: E501
+
+import os
+
+import torch
+
+import triton
+import triton.language as tl
+
+from .index import prepare_chunk_indices
+from .op import make_tensor_descriptor
+from .utils import input_guard, is_amd, is_tma_supported
+
+FLA_TRIL_PRECISION = os.environ.get("FLA_TRIL_PRECISION", "ieee")
+ALLOWED_TRIL_PRECISIONS = ["ieee", "tf32"] if is_amd else ["ieee", "tf32", "tf32x3"]
+assert FLA_TRIL_PRECISION in ALLOWED_TRIL_PRECISIONS, (
+ f"FLA_TRIL_PRECISION must be one of {ALLOWED_TRIL_PRECISIONS}, but got {FLA_TRIL_PRECISION}"
+)
+
+
[email protected]({"IS_VARLEN": lambda args: args["cu_seqlens"] is not None})
+ configs=[
+ triton.Config({}, num_warps=num_warps, num_stages=num_stages)
+ for num_warps in [1, 2, 4, 8]
+ for num_stages in [2, 3, 4, 5]
+ ],
+ key=["BT"],
+)
[email protected](do_not_specialize=["T"])
+def solve_tril_16x16_kernel(
+ A,
+ Ai,
+ cu_seqlens,
+ chunk_indices,
+ T,
+ H: tl.constexpr,
+ BT: tl.constexpr,
+ USE_TMA: tl.constexpr,
+ IS_VARLEN: tl.constexpr,
+ DOT_PRECISION: tl.constexpr,
+):
+ i_t, i_bh = tl.program_id(0), tl.program_id(1)
+ i_b, i_h = i_bh // H, i_bh % H
+ if IS_VARLEN:
+ i_n, i_t = (
+ tl.load(chunk_indices + i_t * 2).to(tl.int32),
+ tl.load(chunk_indices + i_t * 2 + 1).to(tl.int32),
+ )
+ bos, eos = (
+ tl.load(cu_seqlens + i_n).to(tl.int32),
+ tl.load(cu_seqlens + i_n + 1).to(tl.int32),
+ )
+ T = eos - bos
+ else:
+ bos, eos = i_b * T, i_b * T + T
+ o_i = tl.arange(0, 16)
+ m_A = o_i[:, None] > o_i[None, :]
+ m_I = o_i[:, None] == o_i[None, :]
+
+ A = A + (bos * H + i_h) * BT
+ Ai = Ai + (bos * H + i_h) * 16
+
+ offset = (i_t * 16) % BT
+ if not USE_TMA:
+ p_A = tl.make_block_ptr(
+ A, (T, BT), (H * BT, 1), (i_t * 16, offset), (16, 16), (1, 0)
+ )
+ # [16, 16]
+ b_A = tl.load(p_A, boundary_check=(0, 1)).to(tl.float32)
+ else:
+ desc = make_tensor_descriptor(A, [T, BT], [H * BT, 1], [16, 16])
+ desc_o = make_tensor_descriptor(Ai, [T, 16], [H * 16, 1], [16, 16])
+ b_A = desc.load([i_t * 16, offset]).to(tl.float32)
+ b_A = -tl.where(m_A, b_A, 0)
+
+ for i in range(2, min(16, T - i_t * 16)):
+ # [16]
+ b_a = -tl.load(A + (i_t * 16 + i) * H * BT + o_i + offset)
+ b_a = b_a + tl.sum(b_a[:, None] * b_A, 0)
+ b_A = tl.where((o_i == i)[:, None], b_a, b_A)
+ b_A += m_I
+ if not USE_TMA:
+ p_Ai = tl.make_block_ptr(
+ Ai, (T, 16), (H * 16, 1), (i_t * 16, 0), (16, 16), (1, 0)
+ )
+ tl.store(
+ p_Ai,
+ b_A.to(p_Ai.dtype.element_ty, fp_downcast_rounding="rtne"),
+ boundary_check=(0, 1),
+ )
+ else:
+ desc_o.store([i_t * 16, 0], b_A.to(desc_o.dtype, fp_downcast_rounding="rtne"))
+
+
[email protected]({"IS_VARLEN": lambda args: args["cu_seqlens"] is not None})
+ configs=[
+ triton.Config({}, num_warps=num_warps, num_stages=num_stages)
+ for num_warps in [1, 2, 4, 8]
+ for num_stages in [2, 3, 4, 5]
+ ],
+ key=["H", "BT", "IS_VARLEN"],
+)
[email protected](do_not_specialize=["T"])
+def merge_16x16_to_32x32_inverse_kernel(
+ A,
+ Ai,
+ cu_seqlens,
+ chunk_indices,
+ T,
+ H: tl.constexpr,
+ BT: tl.constexpr,
+ USE_TMA: tl.constexpr,
+ IS_VARLEN: tl.constexpr,
+ DOT_PRECISION: tl.constexpr,
+):
+ i_t, i_bh = tl.program_id(0), tl.program_id(1)
+ i_b, i_h = i_bh // H, i_bh % H
+ if IS_VARLEN:
+ i_n, i_t = (
+ tl.load(chunk_indices + i_t * 2).to(tl.int32),
+ tl.load(chunk_indices + i_t * 2 + 1).to(tl.int32),
+ )
+ bos, eos = (
+ tl.load(cu_seqlens + i_n).to(tl.int32),
+ tl.load(cu_seqlens + i_n + 1).to(tl.int32),
+ )
+ T = eos - bos
+ else:
+ bos, eos = i_b * T, i_b * T + T
+
+ o_i = tl.arange(0, 16)
+ m_A = o_i[:, None] > o_i[None, :]
+ m_I = o_i[:, None] == o_i[None, :]
+ A += (bos * H + i_h) * BT
+ Ai += (bos * H + i_h) * BT
+
+ if not USE_TMA:
+ p_A_11 = tl.make_block_ptr(
+ A, (T, BT), (H * BT, 1), (i_t * BT, 0), (16, 16), (1, 0)
+ )
+ p_A_22 = tl.make_block_ptr(
+ A, (T, BT), (H * BT, 1), (i_t * BT + 16, 16), (16, 16), (1, 0)
+ )
+ b_Ai_11 = tl.load(p_A_11, boundary_check=(0, 1)).to(tl.float32)
+ b_Ai_22 = tl.load(p_A_22, boundary_check=(0, 1)).to(tl.float32)
+ else:
+ desc = make_tensor_descriptor(A, [T, BT], [H * BT, 1], [16, 16])
+ desc_o = make_tensor_descriptor(Ai, [T, BT], [H * BT, 1], [16, 16])
+ b_Ai_11 = desc.load([i_t * BT + 0, 0]).to(tl.float32)
+ b_Ai_22 = desc.load([i_t * BT + 16, 16]).to(tl.float32)
+
+ # [16, 16]
+ b_Ai_11 = -tl.where(m_A, b_Ai_11, 0)
+ b_Ai_22 = -tl.where(m_A, b_Ai_22, 0)
+
+ for i in range(2, min(16, T - i_t * BT)):
+ b_a_11 = -tl.load(A + (i_t * BT + i) * H * BT + o_i)
+ b_a_11 += tl.sum(b_a_11[:, None] * b_Ai_11, 0)
+ b_Ai_11 = tl.where((o_i == i)[:, None], b_a_11, b_Ai_11)
+ for i in range(16 + 2, min(32, T - i_t * BT)):
+ b_a_22 = -tl.load(A + (i_t * BT + i) * H * BT + o_i + 16)
+ b_a_22 += tl.sum(b_a_22[:, None] * b_Ai_22, 0)
+ b_Ai_22 = tl.where((o_i == i - 16)[:, None], b_a_22, b_Ai_22)
+
+ b_Ai_11 += m_I
+ b_Ai_22 += m_I
+
+ if not USE_TMA:
+ p_A_21 = tl.make_block_ptr(
+ A, (T, BT), (H * BT, 1), (i_t * BT + 16, 0), (16, 16), (1, 0)
+ )
+ b_A_21 = tl.load(p_A_21, boundary_check=(0, 1)).to(tl.float32)
+ else:
+ b_A_21 = desc.load([i_t * BT + 16, 0]).to(tl.float32)
+
+ b_Ai_21 = -tl.dot(
+ tl.dot(b_Ai_22, b_A_21, input_precision=DOT_PRECISION),
+ b_Ai_11,
+ input_precision=DOT_PRECISION,
+ )
+
+ if not USE_TMA:
+ p_Ai_11 = tl.make_block_ptr(
+ Ai, (T, BT), (H * BT, 1), (i_t * BT, 0), (16, 16), (1, 0)
+ )
+ p_Ai_21 = tl.make_block_ptr(
+ Ai, (T, BT), (H * BT, 1), (i_t * BT + 16, 0), (16, 16), (1, 0)
+ )
+ p_Ai_22 = tl.make_block_ptr(
+ Ai, (T, BT), (H * BT, 1), (i_t * BT + 16, 16), (16, 16), (1, 0)
+ )
+ tl.store(
+ p_Ai_11,
+ b_Ai_11.to(p_Ai_11.dtype.element_ty, fp_downcast_rounding="rtne"),
+ boundary_check=(0, 1),
+ )
+ tl.store(
+ p_Ai_22,
+ b_Ai_22.to(p_Ai_22.dtype.element_ty, fp_downcast_rounding="rtne"),
+ boundary_check=(0, 1),
+ )
+ tl.store(
+ p_Ai_21,
+ b_Ai_21.to(p_Ai_21.dtype.element_ty, fp_downcast_rounding="rtne"),
+ boundary_check=(0, 1),
+ )
+ else:
+ desc_o.store(
+ [i_t * BT + 0, 0], b_Ai_11.to(desc_o.dtype, fp_downcast_rounding="rtne")
+ )
+ desc_o.store(
+ [i_t * BT + 16, 0], b_Ai_21.to(desc_o.dtype, fp_downcast_rounding="rtne")
+ )
+ desc_o.store(
+ [i_t * BT + 16, 16], b_Ai_22.to(desc_o.dtype, fp_downcast_rounding="rtne")
+ )
+
+
[email protected]({"IS_VARLEN": lambda args: args["cu_seqlens"] is not None})
+ configs=[
+ triton.Config({}, num_warps=num_warps, num_stages=num_stages)
+ for num_warps in [2, 4, 8]
+ for num_stages in [2, 3, 4, 5]
+ ],
+ key=["H", "BT", "IS_VARLEN"],
+)
[email protected](do_not_specialize=["T"])
+def merge_16x16_to_64x64_inverse_kernel(
+ A,
+ Ai,
+ cu_seqlens,
+ chunk_indices,
+ T,
+ H: tl.constexpr,
+ BT: tl.constexpr,
+ USE_TMA: tl.constexpr,
+ IS_VARLEN: tl.constexpr,
+ DOT_PRECISION: tl.constexpr,
+):
+ i_t, i_bh = tl.program_id(0), tl.program_id(1)
+ i_b, i_h = i_bh // H, i_bh % H
+ if IS_VARLEN:
+ i_n, i_t = (
+ tl.load(chunk_indices + i_t * 2).to(tl.int32),
+ tl.load(chunk_indices + i_t * 2 + 1).to(tl.int32),
+ )
+ bos, eos = (
+ tl.load(cu_seqlens + i_n).to(tl.int32),
+ tl.load(cu_seqlens + i_n + 1).to(tl.int32),
+ )
+ T = eos - bos
+ else:
+ bos, eos = i_b * T, i_b * T + T
+
+ o_i = tl.arange(0, 16)
+ m_A = o_i[:, None] > o_i[None, :]
+ m_I = o_i[:, None] == o_i[None, :]
+ A += (bos * H + i_h) * BT
+ Ai += (bos * H + i_h) * BT
+
+ if not USE_TMA:
+ p_A_11 = tl.make_block_ptr(
+ A, (T, BT), (H * BT, 1), (i_t * BT, 0), (16, 16), (1, 0)
+ )
+ p_A_22 = tl.make_block_ptr(
+ A, (T, BT), (H * BT, 1), (i_t * BT + 16, 16), (16, 16), (1, 0)
+ )
+ p_A_33 = tl.make_block_ptr(
+ A, (T, BT), (H * BT, 1), (i_t * BT + 32, 32), (16, 16), (1, 0)
+ )
+ p_A_44 = tl.make_block_ptr(
+ A, (T, BT), (H * BT, 1), (i_t * BT + 48, 48), (16, 16), (1, 0)
+ )
+ b_Ai_11 = tl.load(p_A_11, boundary_check=(0, 1)).to(tl.float32)
+ b_Ai_22 = tl.load(p_A_22, boundary_check=(0, 1)).to(tl.float32)
+ b_Ai_33 = tl.load(p_A_33, boundary_check=(0, 1)).to(tl.float32)
+ b_Ai_44 = tl.load(p_A_44, boundary_check=(0, 1)).to(tl.float32)
+ else:
+ desc = make_tensor_descriptor(A, [T, BT], [H * BT, 1], [16, 16])
+ desc_o = make_tensor_descriptor(Ai, [T, BT], [H * BT, 1], [16, 16])
+ b_Ai_11 = desc.load([i_t * BT + 0, 0]).to(tl.float32)
+ b_Ai_22 = desc.load([i_t * BT + 16, 16]).to(tl.float32)
+ b_Ai_33 = desc.load([i_t * BT + 32, 32]).to(tl.float32)
+ b_Ai_44 = desc.load([i_t * BT + 48, 48]).to(tl.float32)
+
+ # [16, 16]
+ b_Ai_11 = -tl.where(m_A, b_Ai_11, 0)
+ b_Ai_22 = -tl.where(m_A, b_Ai_22, 0)
+ b_Ai_33 = -tl.where(m_A, b_Ai_33, 0)
+ b_Ai_44 = -tl.where(m_A, b_Ai_44, 0)
+
+ for i in range(2, min(16, T - i_t * BT)):
+ b_a_11 = -tl.load(A + (i_t * BT + i) * H * BT + o_i)
+ b_a_11 += tl.sum(b_a_11[:, None] * b_Ai_11, 0)
+ b_Ai_11 = tl.where((o_i == i)[:, None], b_a_11, b_Ai_11)
+ for i in range(16 + 2, min(32, T - i_t * BT)):
+ b_a_22 = -tl.load(A + (i_t * BT + i) * H * BT + o_i + 16)
+ b_a_22 += tl.sum(b_a_22[:, None] * b_Ai_22, 0)
+ b_Ai_22 = tl.where((o_i == i - 16)[:, None], b_a_22, b_Ai_22)
+ for i in range(32 + 2, min(48, T - i_t * BT)):
+ b_a_33 = -tl.load(A + (i_t * BT + i) * H * BT + o_i + 32)
+ b_a_33 += tl.sum(b_a_33[:, None] * b_Ai_33, 0)
+ b_Ai_33 = tl.where((o_i == i - 32)[:, None], b_a_33, b_Ai_33)
+ for i in range(48 + 2, min(64, T - i_t * BT)):
+ b_a_44 = -tl.load(A + (i_t * BT + i) * H * BT + o_i + 48)
+ b_a_44 += tl.sum(b_a_44[:, None] * b_Ai_44, 0)
+ b_Ai_44 = tl.where((o_i == i - 48)[:, None], b_a_44, b_Ai_44)
+ b_Ai_11 += m_I
+ b_Ai_22 += m_I
+ b_Ai_33 += m_I
+ b_Ai_44 += m_I
+
+ if not USE_TMA:
+ p_A_21 = tl.make_block_ptr(
+ A, (T, BT), (H * BT, 1), (i_t * BT + 16, 0), (16, 16), (1, 0)
+ )
+ p_A_31 = tl.make_block_ptr(
+ A, (T, BT), (H * BT, 1), (i_t * BT + 32, 0), (16, 16), (1, 0)
+ )
+ p_A_32 = tl.make_block_ptr(
+ A, (T, BT), (H * BT, 1), (i_t * BT + 32, 16), (16, 16), (1, 0)
+ )
+ p_A_41 = tl.make_block_ptr(
+ A, (T, BT), (H * BT, 1), (i_t * BT + 48, 0), (16, 16), (1, 0)
+ )
+ p_A_42 = tl.make_block_ptr(
+ A, (T, BT), (H * BT, 1), (i_t * BT + 48, 16), (16, 16), (1, 0)
+ )
+ p_A_43 = tl.make_block_ptr(
+ A, (T, BT), (H * BT, 1), (i_t * BT + 48, 32), (16, 16), (1, 0)
+ )
+ b_A_21 = tl.load(p_A_21, boundary_check=(0, 1)).to(tl.float32)
+ b_A_31 = tl.load(p_A_31, boundary_check=(0, 1)).to(tl.float32)
+ b_A_32 = tl.load(p_A_32, boundary_check=(0, 1)).to(tl.float32)
+ b_A_41 = tl.load(p_A_41, boundary_check=(0, 1)).to(tl.float32)
+ b_A_42 = tl.load(p_A_42, boundary_check=(0, 1)).to(tl.float32)
+ b_A_43 = tl.load(p_A_43, boundary_check=(0, 1)).to(tl.float32)
+ else:
+ b_A_21 = desc.load([i_t * BT + 16, 0]).to(tl.float32)
+ b_A_31 = desc.load([i_t * BT + 32, 0]).to(tl.float32)
+ b_A_32 = desc.load([i_t * BT + 32, 16]).to(tl.float32)
+ b_A_41 = desc.load([i_t * BT + 48, 0]).to(tl.float32)
+ b_A_42 = desc.load([i_t * BT + 48, 16]).to(tl.float32)
+ b_A_43 = desc.load([i_t * BT + 48, 32]).to(tl.float32)
+
+ b_Ai_21 = -tl.dot(
+ tl.dot(b_Ai_22, b_A_21, input_precision=DOT_PRECISION),
+ b_Ai_11,
+ input_precision=DOT_PRECISION,
+ )
+ b_Ai_32 = -tl.dot(
+ tl.dot(b_Ai_33, b_A_32, input_precision=DOT_PRECISION),
+ b_Ai_22,
+ input_precision=DOT_PRECISION,
+ )
+ b_Ai_43 = -tl.dot(
+ tl.dot(b_Ai_44, b_A_43, input_precision=DOT_PRECISION),
+ b_Ai_33,
+ input_precision=DOT_PRECISION,
+ )
+
+ b_Ai_31 = -tl.dot(
+ b_Ai_33,
+ tl.dot(b_A_31, b_Ai_11, input_precision=DOT_PRECISION)
+ + tl.dot(b_A_32, b_Ai_21, input_precision=DOT_PRECISION),
+ input_precision=DOT_PRECISION,
+ )
+ b_Ai_42 = -tl.dot(
+ b_Ai_44,
+ tl.dot(b_A_42, b_Ai_22, input_precision=DOT_PRECISION)
+ + tl.dot(b_A_43, b_Ai_32, input_precision=DOT_PRECISION),
+ input_precision=DOT_PRECISION,
+ )
+ b_Ai_41 = -tl.dot(
+ b_Ai_44,
+ tl.dot(b_A_41, b_Ai_11, input_precision=DOT_PRECISION)
+ + tl.dot(b_A_42, b_Ai_21, input_precision=DOT_PRECISION)
+ + tl.dot(b_A_43, b_Ai_31, input_precision=DOT_PRECISION),
+ input_precision=DOT_PRECISION,
+ )
+
+ if not USE_TMA:
+ p_Ai_11 = tl.make_block_ptr(
+ Ai, (T, BT), (H * BT, 1), (i_t * BT, 0), (16, 16), (1, 0)
+ )
+ p_Ai_22 = tl.make_block_ptr(
+ Ai, (T, BT), (H * BT, 1), (i_t * BT + 16, 16), (16, 16), (1, 0)
+ )
+ p_Ai_33 = tl.make_block_ptr(
+ Ai, (T, BT), (H * BT, 1), (i_t * BT + 32, 32), (16, 16), (1, 0)
+ )
+ p_Ai_44 = tl.make_block_ptr(
+ Ai, (T, BT), (H * BT, 1), (i_t * BT + 48, 48), (16, 16), (1, 0)
+ )
+ p_Ai_21 = tl.make_block_ptr(
+ Ai, (T, BT), (H * BT, 1), (i_t * BT + 16, 0), (16, 16), (1, 0)
+ )
+ p_Ai_31 = tl.make_block_ptr(
+ Ai, (T, BT), (H * BT, 1), (i_t * BT + 32, 0), (16, 16), (1, 0)
+ )
+ p_Ai_32 = tl.make_block_ptr(
+ Ai, (T, BT), (H * BT, 1), (i_t * BT + 32, 16), (16, 16), (1, 0)
+ )
+ p_Ai_41 = tl.make_block_ptr(
+ Ai, (T, BT), (H * BT, 1), (i_t * BT + 48, 0), (16, 16), (1, 0)
+ )
+ p_Ai_42 = tl.make_block_ptr(
+ Ai, (T, BT), (H * BT, 1), (i_t * BT + 48, 16), (16, 16), (1, 0)
+ )
+ p_Ai_43 = tl.make_block_ptr(
+ Ai, (T, BT), (H * BT, 1), (i_t * BT + 48, 32), (16, 16), (1, 0)
+ )
+ tl.store(
+ p_Ai_11,
+ b_Ai_11.to(p_Ai_11.dtype.element_ty, fp_downcast_rounding="rtne"),
+ boundary_check=(0, 1),
+ )
+ tl.store(
+ p_Ai_22,
+ b_Ai_22.to(p_Ai_22.dtype.element_ty, fp_downcast_rounding="rtne"),
+ boundary_check=(0, 1),
+ )
+ tl.store(
+ p_Ai_33,
+ b_Ai_33.to(p_Ai_33.dtype.element_ty, fp_downcast_rounding="rtne"),
+ boundary_check=(0, 1),
+ )
+ tl.store(
+ p_Ai_44,
+ b_Ai_44.to(p_Ai_44.dtype.element_ty, fp_downcast_rounding="rtne"),
+ boundary_check=(0, 1),
+ )
+ tl.store(
+ p_Ai_21,
+ b_Ai_21.to(p_Ai_21.dtype.element_ty, fp_downcast_rounding="rtne"),
+ boundary_check=(0, 1),
+ )
+ tl.store(
+ p_Ai_31,
+ b_Ai_31.to(p_Ai_31.dtype.element_ty, fp_downcast_rounding="rtne"),
+ boundary_check=(0, 1),
+ )
+ tl.store(
+ p_Ai_32,
+ b_Ai_32.to(p_Ai_32.dtype.element_ty, fp_downcast_rounding="rtne"),
+ boundary_check=(0, 1),
+ )
+ tl.store(
+ p_Ai_41,
+ b_Ai_41.to(p_Ai_41.dtype.element_ty, fp_downcast_rounding="rtne"),
+ boundary_check=(0, 1),
+ )
+ tl.store(
+ p_Ai_42,
+ b_Ai_42.to(p_Ai_42.dtype.element_ty, fp_downcast_rounding="rtne"),
+ boundary_check=(0, 1),
+ )
+ tl.store(
+ p_Ai_43,
+ b_Ai_43.to(p_Ai_43.dtype.element_ty, fp_downcast_rounding="rtne"),
+ boundary_check=(0, 1),
+ )
+ else:
+ desc_o.store(
+ [i_t * BT + 0, 0], b_Ai_11.to(desc_o.dtype, fp_downcast_rounding="rtne")
+ )
+ desc_o.store(
+ [i_t * BT + 16, 16], b_Ai_22.to(desc_o.dtype, fp_downcast_rounding="rtne")
+ )
+ desc_o.store(
+ [i_t * BT + 32, 32], b_Ai_33.to(desc_o.dtype, fp_downcast_rounding="rtne")
+ )
+ desc_o.store(
+ [i_t * BT + 48, 48], b_Ai_44.to(desc_o.dtype, fp_downcast_rounding="rtne")
+ )
+ desc_o.store(
+ [i_t * BT + 16, 0], b_Ai_21.to(desc_o.dtype, fp_downcast_rounding="rtne")
+ )
+ desc_o.store(
+ [i_t * BT + 32, 0], b_Ai_31.to(desc_o.dtype, fp_downcast_rounding="rtne")
+ )
+ desc_o.store(
+ [i_t * BT + 32, 16], b_Ai_32.to(desc_o.dtype, fp_downcast_rounding="rtne")
+ )
+ desc_o.store(
+ [i_t * BT + 48, 0], b_Ai_41.to(desc_o.dtype, fp_downcast_rounding="rtne")
+ )
+ desc_o.store(
+ [i_t * BT + 48, 16], b_Ai_42.to(desc_o.dtype, fp_downcast_rounding="rtne")
+ )
+ desc_o.store(
+ [i_t * BT + 48, 32], b_Ai_43.to(desc_o.dtype, fp_downcast_rounding="rtne")
+ )
+
+
+@input_guard
+def solve_tril(
+ A: torch.Tensor,
+ cu_seqlens: torch.Tensor | None = None,
+ chunk_indices: torch.Tensor | None = None,
+ output_dtype: torch.dtype = torch.float,
+) -> torch.Tensor:
+ """
+ Compute the inverse of the matrix I + A
+ A should be strictly lower triangular, i.e., A.triu() == 0.
+
+ Args:
+ A (torch.Tensor):
+ [B, T, H, BT], where BT should only be 16, 32, or 64.
+ cu_seqlens (torch.Tensor):
+ The cumulative sequence lengths of the input tensor. Default: `None`.
+ chunk_indices (torch.Tensor):
+ Pre-computed chunk indices. Default: `None`.
+ output_dtype (torch.dtype):
+ The dtype of the output tensor. Default: `torch.float`.
+ If `None`, the output dtype will be the same as the input dtype.
+
+ Returns:
+ (I + A)^-1 with the same shape as A
+ """
+ assert A.shape[-1] in [16, 32, 64]
+ output_dtype = A.dtype if output_dtype is None else output_dtype
+
+ B, T, H, BT = A.shape
+ if chunk_indices is None and cu_seqlens is not None:
+ chunk_indices = prepare_chunk_indices(cu_seqlens, BT)
+ NT = len(chunk_indices) if cu_seqlens is not None else triton.cdiv(T, BT)
+
+ Ai = torch.zeros_like(A, dtype=output_dtype)
+ if BT == 16:
+ merge_fn = solve_tril_16x16_kernel
+ elif BT == 32:
+ merge_fn = merge_16x16_to_32x32_inverse_kernel
+ elif BT == 64:
+ merge_fn = merge_16x16_to_64x64_inverse_kernel
+
+ merge_fn[NT, B * H](
+ A=A,
+ Ai=Ai,
+ cu_seqlens=cu_seqlens,
+ chunk_indices=chunk_indices,
+ T=T,
+ H=H,
+ BT=BT,
+ USE_TMA=is_tma_supported,
+ DOT_PRECISION=FLA_TRIL_PRECISION,
+ )
+ return Ai
diff --git a/python/freetoken/kernel/fla/utils.py b/python/freetoken/kernel/fla/utils.py
index e9e0b697..a0999311 100644
--- a/python/freetoken/kernel/fla/utils.py
+++ b/python/freetoken/kernel/fla/utils.py
@@ -280,6 +280,20 @@ use_cuda_graph = is_nvidia and os.environ.get("FLA_USE_CUDA_GRAPH", "0") == "1"
is_tf32_supported = is_nvidia and torch.cuda.get_device_capability()[0] >= 8
is_gather_supported = hasattr(triton.language, "gather")
+# Shared chunk length of the fla chunked kernels (KDA vendor reads it from here).
+FLA_CHUNK_SIZE = 64
+
+# TMA descriptors (solve_tril fast path): Hopper+, opt-in via FLA_USE_TMA=1, and only
+# when this triton exposes a make_tensor_descriptor (see kernel/fla/op.py).
+is_tma_supported = (
+ is_nvidia_hopper
+ and os.getenv("FLA_USE_TMA", "0") == "1"
+ and (
+ hasattr(triton.language, "_experimental_make_tensor_descriptor")
+ or hasattr(triton.language, "make_tensor_descriptor")
+ )
+)
+
def get_all_max_shared_mem():
try:
diff --git a/python/freetoken/kernel/triton/activation.py b/python/freetoken/kernel/triton/activation.py
index 2c38b533..0e4d28c8 100644
--- a/python/freetoken/kernel/triton/activation.py
+++ b/python/freetoken/kernel/triton/activation.py
@@ -31,6 +31,8 @@ GELU_TANH = 2
# variant lives in triton/mxfp4_moe.py):
# y = clamp(gate, max=limit) * sigmoid(alpha * gate) * (clamp(up, +-limit) + 1)
SWIGLUOAI = 3
+# GLM-5.3 "swiglu_limit": swigluoai's clamped form WITHOUT the (up + 1) bias.
+SWIGLU_CLAMP = 4
_SQRT_2_OVER_PI = 0.7978845608028654 # sqrt(2/pi)
_GELU_C = 0.044715
@@ -105,6 +107,11 @@ def _act_and_mul_kernel(
up = tl.minimum(tl.maximum(up, -limit), limit)
act = gate / (1.0 + _fast_ex2(-gate * alpha * _LOG2E))
y = act * (up + 1.0)
+ elif ACT == 4: # SWIGLU_CLAMP (GLM-5.3): swigluoai without the (up + 1) bias
+ gate = tl.minimum(gate, limit)
+ up = tl.minimum(tl.maximum(up, -limit), limit)
+ act = gate / (1.0 + _fast_ex2(-gate * alpha * _LOG2E))
+ y = act * up
else: # GELU (erf)
act = 0.5 * gate * (1.0 + libdevice.erf(gate * 0.7071067811865476))
y = act * up
@@ -166,4 +173,21 @@ def swigluoai_and_mul(
return _act_and_mul(SWIGLUOAI, x, out, alpha=alpha, limit=limit)
-__all__ = ["silu_and_mul", "gelu_and_mul", "gelu_tanh_and_mul", "swigluoai_and_mul"]
+def swiglu_clamp_and_mul(
+ x: torch.Tensor,
+ out: torch.Tensor | None = None,
+ *,
+ alpha: float = 1.0,
+ limit: float = 10.0,
+) -> torch.Tensor:
+ """GLM-5.3 clamped SwiGLU over UNINTERLEAVED halves: ``clamp(gate, max=limit) * sigmoid(alpha * gate) * clamp(up, +-limit)``."""
+ return _act_and_mul(SWIGLU_CLAMP, x, out, alpha=alpha, limit=limit)
+
+
+__all__ = [
+ "silu_and_mul",
+ "gelu_and_mul",
+ "gelu_tanh_and_mul",
+ "swigluoai_and_mul",
+ "swiglu_clamp_and_mul",
+]
diff --git a/python/freetoken/kernel/triton/glm_dsa_sparse.py b/python/freetoken/kernel/triton/glm_dsa_sparse.py
index d3219f95..8aaeeb9b 100644
--- a/python/freetoken/kernel/triton/glm_dsa_sparse.py
+++ b/python/freetoken/kernel/triton/glm_dsa_sparse.py
@@ -49,6 +49,7 @@ def _glm_dsa_sparse_kernel(
BLOCK_H: tl.constexpr,
BLOCK_T: tl.constexpr,
HAS_COUNTS: tl.constexpr,
+ HAS_ROPE: tl.constexpr,
):
pid_m = tl.program_id(0)
pid_b = tl.program_id(1)
@@ -57,11 +58,14 @@ def _glm_dsa_sparse_kernel(
offs_h = pid_h * BLOCK_H + tl.arange(0, BLOCK_H)
h_mask = offs_h < H
offs_v = tl.arange(0, D_V)
- offs_r = tl.arange(0, D_R)
q_base = q_ptr + pid_b * stride_qb + pid_m * stride_qm + offs_h[:, None] * stride_qh
q_v = tl.load(q_base + offs_v[None, :] * stride_qd, mask=h_mask[:, None], other=0.0).to(tl.float32)
- q_r = tl.load(q_base + (D_V + offs_r[None, :]) * stride_qd, mask=h_mask[:, None], other=0.0).to(tl.float32)
+ if HAS_ROPE:
+ # NoPE checkpoints (glm5_next) have D_R == 0: tl.arange needs a non-empty
+ # span, so the whole rope half is compiled out on the constexpr.
+ offs_r = tl.arange(0, D_R)
+ q_r = tl.load(q_base + (D_V + offs_r[None, :]) * stride_qd, mask=h_mask[:, None], other=0.0).to(tl.float32)
m_i = tl.full((BLOCK_H,), -float("inf"), dtype=tl.float32)
l_i = tl.zeros((BLOCK_H,), dtype=tl.float32)
@@ -79,9 +83,12 @@ def _glm_dsa_sparse_kernel(
valid = idxs >= 0
kv_base = pool_ptr + idxs[:, None] * stride_pn
kv_v = tl.load(kv_base + offs_v[None, :] * stride_pd, mask=valid[:, None], other=0.0).to(tl.float32)
- kv_r = tl.load(kv_base + (D_V + offs_r[None, :]) * stride_pd, mask=valid[:, None], other=0.0).to(tl.float32)
- scores = (tl.dot(q_v, tl.trans(kv_v)) + tl.dot(q_r, tl.trans(kv_r))) * scale
+ scores = tl.dot(q_v, tl.trans(kv_v))
+ if HAS_ROPE:
+ kv_r = tl.load(kv_base + (D_V + offs_r[None, :]) * stride_pd, mask=valid[:, None], other=0.0).to(tl.float32)
+ scores += tl.dot(q_r, tl.trans(kv_r))
+ scores = scores * scale
scores = tl.where(valid[None, :], scores, -float("inf"))
m_new = tl.maximum(m_i, tl.max(scores, axis=1))
@@ -207,6 +214,7 @@ def _glm_dsa_splitk_kernel(
BLOCK_H: tl.constexpr,
BLOCK_T: tl.constexpr,
HAS_COUNTS: tl.constexpr,
+ HAS_ROPE: tl.constexpr,
NUM_SPLITS: tl.constexpr,
):
"""Stage 1 (decode flash-decoding): each program reduces one BLOCK_T-aligned slice of
@@ -221,7 +229,6 @@ def _glm_dsa_splitk_kernel(
offs_h = pid_h * BLOCK_H + tl.arange(0, BLOCK_H)
h_mask = offs_h < H
offs_v = tl.arange(0, D_V)
- offs_r = tl.arange(0, D_R)
n_active = TOPK
if HAS_COUNTS:
@@ -238,7 +245,9 @@ def _glm_dsa_splitk_kernel(
if split_end > split_start:
q_base = q_ptr + pid_b * stride_qb + pid_m * stride_qm + offs_h[:, None] * stride_qh
q_v = tl.load(q_base + offs_v[None, :] * stride_qd, mask=h_mask[:, None], other=0.0).to(tl.float32)
- q_r = tl.load(q_base + (D_V + offs_r[None, :]) * stride_qd, mask=h_mask[:, None], other=0.0).to(tl.float32)
+ if HAS_ROPE:
+ offs_r = tl.arange(0, D_R)
+ q_r = tl.load(q_base + (D_V + offs_r[None, :]) * stride_qd, mask=h_mask[:, None], other=0.0).to(tl.float32)
idx_base = idx_ptr + pid_b * stride_ib + pid_m * stride_im
for start in range(split_start, split_end, BLOCK_T):
@@ -248,9 +257,12 @@ def _glm_dsa_splitk_kernel(
valid = idxs >= 0
kv_base = pool_ptr + idxs[:, None] * stride_pn
kv_v = tl.load(kv_base + offs_v[None, :] * stride_pd, mask=valid[:, None], other=0.0).to(tl.float32)
- kv_r = tl.load(kv_base + (D_V + offs_r[None, :]) * stride_pd, mask=valid[:, None], other=0.0).to(tl.float32)
- scores = (tl.dot(q_v, tl.trans(kv_v)) + tl.dot(q_r, tl.trans(kv_r))) * scale
+ scores = tl.dot(q_v, tl.trans(kv_v))
+ if HAS_ROPE:
+ kv_r = tl.load(kv_base + (D_V + offs_r[None, :]) * stride_pd, mask=valid[:, None], other=0.0).to(tl.float32)
+ scores += tl.dot(q_r, tl.trans(kv_r))
+ scores = scores * scale
scores = tl.where(valid[None, :], scores, -float("inf"))
m_new = tl.maximum(m_i, tl.max(scores, axis=1))
@@ -384,7 +396,7 @@ def glm_dsa_sparse_attn(
stride_nb, stride_nm,
D_V=d_v, D_R=d_r,
BLOCK_H=BLOCK_H, BLOCK_T=BLOCK_T,
- HAS_COUNTS=has_counts, NUM_SPLITS=n_splits,
+ HAS_COUNTS=has_counts, HAS_ROPE=d_r > 0, NUM_SPLITS=n_splits,
num_warps=4, num_stages=2,
)
grid2 = (m, b, h)
@@ -410,7 +422,7 @@ def glm_dsa_sparse_attn(
stride_nb, stride_nm,
D_V=d_v, D_R=d_r,
BLOCK_H=BLOCK_H, BLOCK_T=BLOCK_T,
- HAS_COUNTS=has_counts,
+ HAS_COUNTS=has_counts, HAS_ROPE=d_r > 0,
num_warps=4, num_stages=2,
)
return o
diff --git a/python/freetoken/kernel/triton/kpool_compress.py b/python/freetoken/kernel/triton/kpool_compress.py
new file mode 100644
index 00000000..61cacb1f
--- /dev/null
+++ b/python/freetoken/kernel/triton/kpool_compress.py
@@ -0,0 +1,146 @@
+"""Pool one token row's group into ``slab[cmp_rows[row]]``: members inside this
+forward read the raw K/gate rows, older members read the per-request tail ring.
+Adapted from ``qsa/compress.py`` with the mean replaced by a per-channel
+softmax(gate + APE) weighted sum (hence the gate stream/ring and the APE input).
+Non-closing rows land on a scratch row that scoring never reads."""
+
+from __future__ import annotations
+
+import torch
+import triton
+import triton.language as tl
+
+
+def _compress_kpool_groups_kernel(
+ raw_k_ptr,
+ raw_g_ptr,
+ ring_k_ptr,
+ ring_g_ptr,
+ ape_ptr,
+ ring_slots_ptr,
+ token_to_req_ptr,
+ query_start_loc_ptr,
+ positions_ptr,
+ slab_ptr,
+ cmp_rows_ptr,
+ stride_raw_k_row,
+ stride_raw_g_row,
+ stride_ring_row,
+ stride_slab_row,
+ num_rows,
+ num_ring_rows,
+ num_requests,
+ RATIO: tl.constexpr,
+ HEAD_DIM: tl.constexpr,
+ BLOCK_D: tl.constexpr,
+) -> None:
+ row = tl.program_id(0)
+ dims = tl.arange(0, BLOCK_D)
+ in_dim = dims < HEAD_DIM
+
+ request = tl.load(token_to_req_ptr + row, mask=row < num_rows, other=-1)
+ end_position = tl.load(positions_ptr + row, mask=row < num_rows, other=0).to(tl.int64)
+ valid_request = (request >= 0) & (request < num_requests)
+ safe_request = tl.minimum(tl.maximum(request, 0), num_requests - 1)
+ query_row_start = tl.load(
+ query_start_loc_ptr + safe_request, mask=valid_request, other=0
+ ).to(tl.int64)
+ chunk_start_position = end_position - (row - query_row_start)
+ ring_slot = tl.load(ring_slots_ptr + safe_request, mask=valid_request, other=0).to(
+ tl.int64
+ )
+ # A row whose group has members before position 0 (end_position < RATIO - 1)
+ # can never close; keep its loads masked off entirely -- member positions would
+ # be negative and C-style % would produce NEGATIVE ring rows (illegal address).
+ valid_row = (row < num_rows) & valid_request & (end_position >= RATIO - 1)
+
+ # Two-pass per-channel softmax over the RATIO group members (static loop; the
+ # doubled loads are 4 x 512B and free next to the fp32 math).
+ m = tl.full((BLOCK_D,), float("-inf"), tl.float32)
+ for off in tl.static_range(RATIO):
+ position = end_position - (RATIO - 1 - off)
+ use_raw = position >= chunk_start_position
+ raw_row = query_row_start + position - chunk_start_position
+ ring_row = ring_slot * RATIO + position % RATIO
+ g_raw = tl.load(
+ raw_g_ptr + raw_row * stride_raw_g_row + dims,
+ mask=valid_row & use_raw & (raw_row >= 0) & (raw_row < num_rows) & in_dim,
+ other=0.0,
+ ).to(tl.float32)
+ g_ring = tl.load(
+ ring_g_ptr + tl.maximum(ring_row, 0) * stride_ring_row + dims,
+ mask=valid_row & (~use_raw) & (ring_row >= 0) & (ring_row < num_ring_rows) & in_dim,
+ other=0.0,
+ ).to(tl.float32)
+ ape = tl.load(ape_ptr + off * HEAD_DIM + dims, mask=in_dim, other=0.0)
+ m = tl.maximum(m, tl.where(use_raw, g_raw, g_ring) + ape)
+
+ s = tl.zeros((BLOCK_D,), dtype=tl.float32)
+ acc = tl.zeros((BLOCK_D,), dtype=tl.float32)
+ for off in tl.static_range(RATIO):
+ position = end_position - (RATIO - 1 - off)
+ use_raw = position >= chunk_start_position
+ raw_row = query_row_start + position - chunk_start_position
+ ring_row = ring_slot * RATIO + position % RATIO
+ raw_mask = valid_row & use_raw & (raw_row >= 0) & (raw_row < num_rows) & in_dim
+ ring_mask = (
+ valid_row & (~use_raw) & (ring_row >= 0) & (ring_row < num_ring_rows) & in_dim
+ )
+ g_raw = tl.load(
+ raw_g_ptr + raw_row * stride_raw_g_row + dims, mask=raw_mask, other=0.0
+ ).to(tl.float32)
+ g_ring = tl.load(
+ ring_g_ptr + tl.maximum(ring_row, 0) * stride_ring_row + dims,
+ mask=ring_mask, other=0.0,
+ ).to(tl.float32)
+ k_raw = tl.load(
+ raw_k_ptr + raw_row * stride_raw_k_row + dims, mask=raw_mask, other=0.0
+ ).to(tl.float32)
+ k_ring = tl.load(
+ ring_k_ptr + tl.maximum(ring_row, 0) * stride_ring_row + dims,
+ mask=ring_mask, other=0.0,
+ ).to(tl.float32)
+ ape = tl.load(ape_ptr + off * HEAD_DIM + dims, mask=in_dim, other=0.0)
+ e = tl.exp(tl.where(use_raw, g_raw, g_ring) + ape - m)
+ s += e
+ acc += e * tl.where(use_raw, k_raw, k_ring)
+
+ pooled = acc / s
+ dest = tl.load(cmp_rows_ptr + row, mask=row < num_rows, other=0).to(tl.int64)
+ tl.store(
+ slab_ptr + dest * stride_slab_row + dims,
+ pooled.to(slab_ptr.dtype.element_ty),
+ mask=(row < num_rows) & in_dim,
+ )
+
+
+def kpool_compress_store(
+ k: torch.Tensor, # [T, D] raw index keys (this forward)
+ gate: torch.Tensor, # [T, D] raw gate scores
+ ring_k: torch.Tensor, # [slots * ratio, D] flat tail ring (keys)
+ ring_g: torch.Tensor, # [slots * ratio, D] flat tail ring (gates)
+ ape: torch.Tensor, # [ratio, D] fp32 model parameter
+ ring_slots: torch.Tensor, # [n_req] Req.table_idx
+ token_to_req: torch.Tensor, # [T]
+ cu_seqlens: torch.Tensor, # [n_req + 1]
+ positions: torch.Tensor, # [T]
+ slab: torch.Tensor, # [shadow_rows + scratch, D]
+ cmp_rows: torch.Tensor, # [T] shadow row (closing) or scratch row
+ ratio: int,
+) -> None:
+ t, d = k.shape
+ if t == 0:
+ return
+ assert ape.dtype == torch.float32 and ape.shape == (ratio, d)
+ _compress_kpool_groups_kernel[(t,)](
+ k, gate, ring_k, ring_g, ape,
+ ring_slots, token_to_req, cu_seqlens, positions,
+ slab, cmp_rows,
+ k.stride(0), gate.stride(0), ring_k.stride(0), slab.stride(0),
+ t, ring_k.shape[0], ring_slots.numel(),
+ RATIO=ratio, HEAD_DIM=d, BLOCK_D=triton.next_power_of_2(d),
+ )
+
+
+__all__ = ["kpool_compress_store"]
diff --git a/python/freetoken/kernel/triton/mhc.py b/python/freetoken/kernel/triton/mhc.py
new file mode 100644
index 00000000..0782efe3
--- /dev/null
+++ b/python/freetoken/kernel/triton/mhc.py
@@ -0,0 +1,232 @@
+"""Fused mHC (Manifold-Constrained Hyper-Connections) triton kernels.
+
+One program per token fuses the sublayer-boundary mix: apply the previous
+sublayer's hc_post (comb^T @ res + post * x), then this sublayer's hc_pre --
+the fn GEMV over the flattened streams, flat-RMS normalization, and the three
+gates (sigmoid pre, sigmoid*mult post, row-softmax + Sinkhorn comb, all on an
+n x n held in registers) -- and the pre-mixed layer input.
+
+Semantics are defined by layers/mhc.py's torch reference (bit-comparable in
+fp32 up to reduction order); tests/layers/test_mhc.py pins the parity. N
+(hc_mult) is a constexpr; only N == 4 is exercised.
+"""
+
+from __future__ import annotations
+
+import torch
+import triton
+import triton.language as tl
+
+
+def _mhc_stage1_kernel(
+ x_ptr, res_ptr, post_ptr, comb_ptr, fn_ptr,
+ res_out_ptr,
+ sq_part_ptr, # [T, NS] fp32
+ mix_part_ptr, # [T, NS, BLK_MIX] fp32
+ H: tl.constexpr, N: tl.constexpr, MIX: tl.constexpr, BLK_MIX: tl.constexpr,
+ SPLIT: tl.constexpr, # hidden elems per split (multiple of BLOCK_H)
+ BLOCK_H: tl.constexpr,
+ NS: tl.constexpr,
+ HAS_POST: tl.constexpr,
+):
+ """Split-K stage of the fused mHC: each program owns one hidden slice of one
+ token -- applies hc_post there, stores the updated streams, and reduces its
+ partial sq-sum + fn-GEMV contribution over NS splits."""
+ t = tl.program_id(0).to(tl.int64)
+ s = tl.program_id(1)
+ offs_mix = tl.arange(0, BLK_MIX)
+ mix_mask = offs_mix < MIX
+ offs_n = tl.arange(0, N)
+
+ if HAS_POST:
+ b_post = tl.load(post_ptr + t * N + offs_n)
+ b_comb = tl.load(comb_ptr + t * N * N + offs_n[:, None] * N + offs_n[None, :])
+
+ sqsum = 0.0
+ acc = tl.zeros([BLK_MIX], dtype=tl.float32)
+ for h0 in range(s * SPLIT, tl.minimum((s + 1) * SPLIT, H), BLOCK_H):
+ offs_h = h0 + tl.arange(0, BLOCK_H)
+ h_mask = offs_h < H
+ if HAS_POST:
+ b_x = tl.load(x_ptr + t * H + offs_h, mask=h_mask, other=0.0).to(tl.float32)
+ for n in tl.static_range(N):
+ if HAS_POST:
+ r_new = tl.zeros([BLOCK_H], dtype=tl.float32)
+ for i in tl.static_range(N):
+ r_i = tl.load(res_ptr + (t * N + i) * H + offs_h, mask=h_mask, other=0.0).to(tl.float32)
+ c_in = tl.sum(tl.where((offs_n == i)[:, None] & (offs_n == n)[None, :], b_comb, 0.0))
+ r_new += c_in * r_i
+ p_n = tl.sum(tl.where(offs_n == n, b_post, 0.0))
+ r_new += p_n * b_x
+ # Round through the STORAGE dtype before the sq-sum and fn-GEMV:
+ # the torch reference reads back what it stored (dtype-generic --
+ # fp16/fp32 residuals must not be silently bf16-rounded).
+ r_new = r_new.to(res_out_ptr.dtype.element_ty).to(tl.float32)
+ tl.store(res_out_ptr + (t * N + n) * H + offs_h, r_new.to(res_out_ptr.dtype.element_ty), mask=h_mask)
+ else:
+ r_new = tl.load(res_ptr + (t * N + n) * H + offs_h, mask=h_mask, other=0.0).to(tl.float32)
+ tl.store(res_out_ptr + (t * N + n) * H + offs_h, r_new.to(res_out_ptr.dtype.element_ty), mask=h_mask)
+ sqsum += tl.sum(r_new * r_new)
+ fn_tile = tl.load(
+ fn_ptr + offs_mix[:, None] * (N * H) + (n * H + offs_h)[None, :],
+ mask=mix_mask[:, None] & h_mask[None, :], other=0.0,
+ )
+ acc += tl.sum(fn_tile * r_new[None, :], axis=1)
+
+ tl.store(sq_part_ptr + t * NS + s, sqsum)
+ tl.store(mix_part_ptr + (t * NS + s) * BLK_MIX + offs_mix, acc)
+
+
+def _mhc_stage2_kernel(
+ sq_part_ptr, mix_part_ptr, scale_ptr, base_ptr,
+ post_out_ptr, comb_out_ptr, pre_out_ptr,
+ rms_eps, hc_eps, post_mult,
+ SINKHORN: tl.constexpr,
+ H: tl.constexpr, N: tl.constexpr, MIX: tl.constexpr, BLK_MIX: tl.constexpr,
+ NS: tl.constexpr,
+):
+ """Reduce the split partials and run the tiny gate math (sigmoid gates,
+ row-softmax + in-register 4x4 Sinkhorn); emits pre gates for stage 3."""
+ t = tl.program_id(0).to(tl.int64)
+ offs_mix = tl.arange(0, BLK_MIX)
+ mix_mask = offs_mix < MIX
+ offs_s = tl.arange(0, NS)
+
+ sqsum = tl.sum(tl.load(sq_part_ptr + t * NS + offs_s))
+ acc = tl.sum(
+ tl.load(mix_part_ptr + (t * NS + offs_s)[:, None] * BLK_MIX + offs_mix[None, :]),
+ axis=0,
+ )
+ inv_rms = tl.math.rsqrt(sqsum / (N * H) + rms_eps)
+ mixes = acc * inv_rms
+ s0 = tl.load(scale_ptr + 0)
+ s1 = tl.load(scale_ptr + 1)
+ s2 = tl.load(scale_ptr + 2)
+ b_base = tl.load(base_ptr + offs_mix, mask=mix_mask, other=0.0)
+
+ is_pre = offs_mix < N
+ is_post = (offs_mix >= N) & (offs_mix < 2 * N)
+ gate_scale = tl.where(is_pre, s0, tl.where(is_post, s1, s2))
+ logits = mixes * gate_scale + b_base
+ pre = tl.sigmoid(logits) + hc_eps
+ post_new = tl.sigmoid(logits) * post_mult
+
+ offs_n2 = tl.arange(0, N)
+ comb_logits = tl.zeros([N, N], dtype=tl.float32)
+ for r in tl.static_range(N):
+ for c in tl.static_range(N):
+ lane = 2 * N + r * N + c
+ v = tl.sum(tl.where(offs_mix == lane, logits, 0.0))
+ comb_logits += tl.where(
+ (offs_n2 == r)[:, None] & (offs_n2 == c)[None, :], v, 0.0
+ )
+ row_max = tl.max(comb_logits, axis=1)
+ e = tl.exp(comb_logits - row_max[:, None])
+ comb = e / tl.sum(e, axis=1)[:, None] + hc_eps
+ comb = comb / (tl.sum(comb, axis=0)[None, :] + hc_eps)
+ for _ in range(SINKHORN - 1):
+ comb = comb / (tl.sum(comb, axis=1)[:, None] + hc_eps)
+ comb = comb / (tl.sum(comb, axis=0)[None, :] + hc_eps)
+
+ post_g = tl.sum(
+ tl.where((offs_mix[None, :] - N) == offs_n2[:, None], post_new[None, :], 0.0),
+ axis=1,
+ )
+ pre_g = tl.sum(
+ tl.where(offs_mix[None, :] == offs_n2[:, None], pre[None, :], 0.0), axis=1
+ )
+ tl.store(post_out_ptr + t * N + offs_n2, post_g)
+ tl.store(pre_out_ptr + t * N + offs_n2, pre_g)
+ tl.store(comb_out_ptr + t * N * N + offs_n2[:, None] * N + offs_n2[None, :], comb)
+
+
+def _mhc_stage3_kernel(
+ res_out_ptr, pre_ptr, li_out_ptr,
+ H: tl.constexpr, N: tl.constexpr, BLOCK_H: tl.constexpr,
+):
+ """layer_input = sum_n pre_n * res_new_n, parallel over hidden chunks."""
+ t = tl.program_id(0).to(tl.int64)
+ hb = tl.program_id(1)
+ offs_h = hb * BLOCK_H + tl.arange(0, BLOCK_H)
+ h_mask = offs_h < H
+ offs_n = tl.arange(0, N)
+ pre = tl.load(pre_ptr + t * N + offs_n)
+ li = tl.zeros([BLOCK_H], dtype=tl.float32)
+ for n in tl.static_range(N):
+ r = tl.load(res_out_ptr + (t * N + n) * H + offs_h, mask=h_mask, other=0.0).to(tl.float32)
+ li += tl.sum(tl.where(offs_n == n, pre, 0.0)) * r
+ tl.store(li_out_ptr + t * H + offs_h, li.to(li_out_ptr.dtype.element_ty), mask=h_mask)
+
+
+def mhc_fused_post_pre_triton(
+ x: torch.Tensor,
+ residual: torch.Tensor,
+ post_mix: torch.Tensor | None,
+ comb_mix: torch.Tensor | None,
+ fn: torch.Tensor,
+ hc_scale: torch.Tensor,
+ hc_base: torch.Tensor,
+ rms_eps: float,
+ hc_eps: float,
+ post_mult: float,
+ sinkhorn_repeat: int,
+) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor]:
+ """Fused hc_post (skipped when ``post_mix is None``) + hc_pre. Three-stage
+ split-K: the GEMV/sq-sum reduction fans out over NS hidden slices. Returns
+ (residual_new [T,N,H] bf16, post [T,N,1] fp32, comb [T,N,N] fp32,
+ layer_input [T,H] bf16)."""
+ t, n, h = residual.shape
+ mix = 2 * n + n * n
+ assert fn.shape == (mix, n * h) and fn.dtype == torch.float32
+ residual = residual.contiguous()
+ has_post = post_mix is not None
+ dev = residual.device
+
+ res_out = torch.empty_like(residual)
+ post_out = torch.empty(t, n, dtype=torch.float32, device=dev)
+ comb_out = torch.empty(t, n, n, dtype=torch.float32, device=dev)
+ li_out = torch.empty(t, h, dtype=residual.dtype, device=dev)
+
+ block_h = min(512, triton.next_power_of_2(h))
+ # NS feeds a tl.arange in stage 2 -> keep it a power of two.
+ ns = 1
+ while ns * 2 <= min(16, h // block_h):
+ ns *= 2
+ split = triton.cdiv(triton.cdiv(h, ns), block_h) * block_h
+ ns = triton.cdiv(h, split)
+ blk_mix = triton.next_power_of_2(mix)
+ sq_part = torch.empty(t, ns, dtype=torch.float32, device=dev)
+ mix_part = torch.empty(t, ns, blk_mix, dtype=torch.float32, device=dev)
+ pre_out = torch.empty(t, n, dtype=torch.float32, device=dev)
+
+ _mhc_stage1_kernel[(t, ns)](
+ x.contiguous() if has_post else residual, # dummy ptr when unused
+ residual,
+ post_mix.contiguous().view(t, n) if has_post else post_out,
+ comb_mix.contiguous() if has_post else comb_out,
+ fn, res_out, sq_part, mix_part,
+ H=h, N=n, MIX=mix, BLK_MIX=blk_mix,
+ SPLIT=split, BLOCK_H=block_h, NS=ns,
+ HAS_POST=has_post,
+ num_warps=4, num_stages=2,
+ )
+ _mhc_stage2_kernel[(t,)](
+ sq_part, mix_part, hc_scale, hc_base,
+ post_out, comb_out, pre_out,
+ rms_eps, hc_eps, post_mult,
+ SINKHORN=sinkhorn_repeat,
+ H=h, N=n, MIX=mix, BLK_MIX=blk_mix, NS=ns,
+ num_warps=1,
+ )
+ _mhc_stage3_kernel[(t, triton.cdiv(h, 1024))](
+ res_out, pre_out, li_out,
+ H=h, N=n, BLOCK_H=min(1024, triton.next_power_of_2(h)),
+ num_warps=4,
+ )
+ return res_out, post_out.view(t, n, 1), comb_out, li_out
+
+
+__all__ = ["mhc_fused_post_pre_triton"]
diff --git a/python/freetoken/kvcache/__init__.py b/python/freetoken/kvcache/__init__.py
index c6c0f1bb..41b82f1d 100644
--- a/python/freetoken/kvcache/__init__.py
+++ b/python/freetoken/kvcache/__init__.py
@@ -40,7 +40,8 @@ def resolve_pool_class(model_config: ModelConfig) -> type[BaseKVCachePool]:
from .mha_pool import MHAKVCache
return MHAKVCache
- types = {spec.attn_type for spec in specs_fn()}
+ specs = list(specs_fn())
+ types = {spec.attn_type for spec in specs}
if AttnType.DSV4 in types:
from .dsv4_paged_pool import DSV4PagedKVCache
@@ -50,6 +51,11 @@ def resolve_pool_class(model_config: ModelConfig) -> type[BaseKVCachePool]:
return HybridSWAKVCache
if AttnType.DSA in types:
+ # kpool-compressed indexer (glm5_next): shadow slab + tail rings.
+ if any(s.attn_type == AttnType.DSA and s.index_ratio > 1 for s in specs):
+ from .dsa_pool import KpoolDSAKVCache
+
+ return KpoolDSAKVCache
from .dsa_pool import DSAKVCache
return DSAKVCache
@@ -138,36 +144,31 @@ def create_kvcache_pool(
from .mha_pool import MHAKVCache
- # Hybrid linear-attention models (e.g. Qwen3.5 GatedDeltaNet) only store paged KV
- # for their full-attention layers; the linear layers keep a separate recurrent
- # state. Back just those layers and remap their global ids to dense storage slots
- # so we don't over-allocate slabs for the (majority) linear layers.
+ # Hybrid linear-attention models only store paged KV for their non-linear
+ # layers; the linear layers keep a separate recurrent state (LinearStatePool).
+ # The linear group emits no paged spec, so the remaining paged spec(s) drive
+ # the dispatch below; the paged pool backs JUST those layers via a global-id
+ # -> dense-slot remap (layer_ids) so the (majority) linear layers cost no
+ # slabs.
layer_ids: tuple[int, ...] | None = None
- num_kv_heads = model_config.num_kv_heads
- head_dim = model_config.head_dim
+ kv_specs = [s for s in model_config.kv_cache_group_specs() if s.num_layers > 0]
if model_config.has_linear_attention:
- specs = [s for s in model_config.kv_cache_group_specs() if s.num_layers > 0]
- assert len(specs) == 1, f"expected one paged-KV group, got {[s.name for s in specs]}"
- spec = specs[0]
- layer_ids = spec.layer_ids
- num_kv_heads = spec.num_kv_heads
- head_dim = spec.head_dim
-
- # Latent-KV MLA models declare it on their single full-attention group: they get
- # the latent pool (one slab, V aliases K), plus the DSA index-key slab when the
- # spec carries indexer dims. The same spec fields drive the KV cost model, so the
- # factory and the budget can never disagree.
- kv_specs = model_config.kv_cache_group_specs()
-
- # GQA block-sparse (MiniMax-M3): one full-attention group carrying the index dims
- # with mla=False -> the MHA pool plus the index-key slab. The same spec fields
- # drive the KV cost model, so the factory and the budget can never disagree.
+ assert len(kv_specs) == 1, (
+ f"hybrid-linear models support one paged-KV group, got "
+ f"{[s.name for s in kv_specs]}"
+ )
+ layer_ids = kv_specs[0].layer_ids
+
+ # Latent-KV MLA / GQA block-sparse models declare their geometry on the single
+ # paged spec; the same spec fields drive the KV cost model, so the factory and
+ # the budget can never disagree.
from freetoken.attention import AttnType as _AttnType
if len(kv_specs) == 1 and kv_specs[0].attn_type == _AttnType.BSA:
from .bsa_pool import BSAKVCache
spec = kv_specs[0]
+ assert layer_ids is None, "hybrid-linear x BSA has no pool support yet"
return BSAKVCache(
num_kv_heads=spec.num_kv_heads,
num_layers=model_config.num_layers,
@@ -206,35 +207,51 @@ def create_kvcache_pool(
)
if len(kv_specs) == 1 and kv_specs[0].mla:
- from .dsa_pool import DSAKVCache, MLAKVCache
+ from .dsa_pool import DSAKVCache, KpoolDSAKVCache, MLAKVCache
spec = kv_specs[0]
+ # With a layer remap the pool allocates len(layer_ids) slabs; without one
+ # it backs every model layer (all-MLA models, GLM-5.2).
+ num_layers = model_config.num_layers if layer_ids is None else len(layer_ids)
if spec.index_head_dim > 0 and spec.num_index_layers > 0:
- return DSAKVCache(
+ common = dict(
latent_dim=spec.head_dim,
- num_layers=model_config.num_layers,
+ num_layers=num_layers,
num_pages=num_pages,
page_size=page_size,
dtype=dtype,
device=device,
index_head_dim=spec.index_head_dim,
num_index_layers=spec.num_index_layers,
+ layer_ids=layer_ids,
)
+ if spec.index_ratio > 1:
+ # kpool tail rings are keyed by Req.table_idx; + 1 covers the dummy request row.
+ if num_req_slots is None:
+ raise ValueError("kpool pools need num_req_slots (max_running_req + 1)")
+ return KpoolDSAKVCache(
+ **common,
+ index_ratio=spec.index_ratio,
+ num_req_slots=num_req_slots,
+ )
+ return DSAKVCache(**common)
return MLAKVCache(
latent_dim=spec.head_dim,
- num_layers=model_config.num_layers,
+ num_layers=num_layers,
num_pages=num_pages,
page_size=page_size,
dtype=dtype,
device=device,
+ layer_ids=layer_ids,
)
+ spec = kv_specs[0] if len(kv_specs) == 1 else None
return MHAKVCache(
- num_kv_heads=num_kv_heads,
+ num_kv_heads=spec.num_kv_heads if spec is not None else model_config.num_kv_heads,
num_pages=num_pages,
page_size=page_size,
num_layers=model_config.num_layers,
- head_dim=head_dim,
+ head_dim=spec.head_dim if spec is not None else model_config.head_dim,
device=device,
dtype=dtype,
layer_ids=layer_ids,
diff --git a/python/freetoken/kvcache/dsa_pool.py b/python/freetoken/kvcache/dsa_pool.py
index e6a51ac9..13225cec 100644
--- a/python/freetoken/kvcache/dsa_pool.py
+++ b/python/freetoken/kvcache/dsa_pool.py
@@ -5,9 +5,10 @@ per-token latent ``ckv (kv_lora_rank) | kpe (qk_rope_head_dim)`` -- there is no
separate V (``v_cache`` aliases ``k_cache``, same convention as dsv4_paged_pool's
single-latent tiers). ``DSAKVCache`` extends it with the DeepSeek-Sparse-Attention
index-key slab: one ``index_head_dim``-wide bf16 row per token per full-indexer
-layer, addressed by the SAME physical rows as the latent slab (page_size == 1), and
-``rebuild`` resizes BOTH slabs atomically so the allocator can never hand out a slot
-one slab has and the other lacks.
+layer, addressed by the SAME physical rows as the latent slab (GLM-5.2, page 1;
+``KpoolDSAKVCache`` overrides the geometry to a 1/ratio shadow). ``rebuild``
+resizes ALL slabs atomically so the allocator can never hand out a slot one slab
+has and the other lacks.
Storage lives here -- not in the attention backend -- so the engine's rebuild path
(``MHAKVCache.rebuild``-shaped: fresh allocation, object identity preserved, views
@@ -27,6 +28,10 @@ class MLAKVCache(BaseKVCachePool):
The leading singleton keeps the buffer shape-compatible with MHAKVCache's
(tokens = shape[2] * shape[3]).
+
+ ``layer_ids`` backs only a SUBSET of the model's layers (hybrid linear x
+ MLA/DSA) while callers keep addressing by GLOBAL layer id -- the same remap
+ contract as MHAKVCache; None keeps the identity mapping.
"""
def __init__(
@@ -37,14 +42,23 @@ class MLAKVCache(BaseKVCachePool):
page_size: int,
dtype: torch.dtype,
device: torch.device,
+ layer_ids: "tuple[int, ...] | None" = None,
) -> None:
self._latent_dim = latent_dim
- self._num_layers = num_layers
+ if layer_ids is None:
+ self._num_layers = num_layers
+ self._layer_index: dict[int, int] | None = None
+ else:
+ self._num_layers = len(layer_ids)
+ self._layer_index = {int(g): i for i, g in enumerate(layer_ids)}
self._page_size = page_size
self._dtype = dtype
self._device = device
self._alloc(num_pages)
+ def _local_layer(self, layer_id: int) -> int:
+ return layer_id if self._layer_index is None else self._layer_index[layer_id]
+
def _alloc(self, num_pages: int) -> None:
self._num_pages = num_pages
self._kv_buffer = torch.empty(
@@ -53,10 +67,12 @@ class MLAKVCache(BaseKVCachePool):
dtype=self._dtype,
)
- # -- views ------------------------------------------------------------------
+ # -- views (addressed by GLOBAL layer id; remapped when layer_ids was given) --
def k_cache(self, layer_id: int) -> torch.Tensor:
"""Paged latent view ``[num_pages, page_size, latent_dim]``."""
- return self._kv_buffer[0, layer_id].view(self._num_pages, self._page_size, -1)
+ return self._kv_buffer[0, self._local_layer(layer_id)].view(
+ self._num_pages, self._page_size, -1
+ )
def v_cache(self, layer_id: int) -> torch.Tensor:
# MLA: K == V (single latent); same buffer, dsv4_paged_pool precedent.
@@ -64,7 +80,7 @@ class MLAKVCache(BaseKVCachePool):
def latent_rows(self, layer_id: int) -> torch.Tensor:
"""Row-flat latent view ``[num_pages * page_size, latent_dim]``."""
- return self._kv_buffer[0, layer_id].view(-1, self._latent_dim)
+ return self._kv_buffer[0, self._local_layer(layer_id)].view(-1, self._latent_dim)
# -- writes -----------------------------------------------------------------
def store_kv(
@@ -77,8 +93,8 @@ class MLAKVCache(BaseKVCachePool):
"""Scatter this forward's latent rows: ``c_kv`` [T, kv_lora_rank] and
``k_rope`` [T, qk_rope_head_dim] land in the row's two halves.
- v0: two narrow ``index_put_`` scatters. TODO: generalize kernel/csrc
- store.cu to a two-width fused store and route this through it.
+ Two narrow ``index_put_`` scatters. TODO: fuse into kernel/csrc
+ store.cu (two-width store).
"""
rows = self.latent_rows(layer_id)
split = rows.shape[1] - k_rope.shape[-1]
@@ -141,20 +157,29 @@ class DSAKVCache(MLAKVCache):
device: torch.device,
index_head_dim: int,
num_index_layers: int,
+ layer_ids: "tuple[int, ...] | None" = None,
) -> None:
self._index_head_dim = index_head_dim
self._num_index_layers = num_index_layers
- super().__init__(latent_dim, num_layers, num_pages, page_size, dtype, device)
+ super().__init__(
+ latent_dim, num_layers, num_pages, page_size, dtype, device,
+ layer_ids=layer_ids,
+ )
+
+ def _index_rows(self, num_pages: int) -> int:
+ """Index-slab row count: one row per token (KpoolDSAKVCache overrides to the
+ 1/ratio shadow + scratch layout)."""
+ return num_pages * self._page_size
def _alloc(self, num_pages: int) -> None:
- # Both slabs in one allocation step: rebuild can never leave the pool with a
- # grown latent slab and a stale index slab (the OOB class this type exists for).
+ # Both slabs in one allocation step so rebuild can never leave one grown
+ # and the other stale.
super()._alloc(num_pages)
# bf16 == the 2 bytes/token/layer the KV cost model budgets for this slab
# (cache_status._kv_cost_model); keep the two in lockstep.
self._index_k_buffer = torch.zeros(
self._num_index_layers,
- num_pages * self._page_size,
+ self._index_rows(num_pages),
self._index_head_dim,
dtype=torch.bfloat16,
device=self._device,
@@ -180,4 +205,56 @@ class DSAKVCache(MLAKVCache):
self._index_k_buffer[slot][out_loc] = k
-__all__ = ["MLAKVCache", "DSAKVCache"]
+class KpoolDSAKVCache(DSAKVCache):
+ """DSAKVCache with the index slab as a 1/ratio SHADOW of the KV pages, plus
+ per-request scratch rows and tail rings.
+
+ A pooled entry lives at ``token_slot // index_ratio`` (``page_size %
+ index_ratio == 0``), so compressed rows follow page sharing/eviction for
+ free. Rows whose pool does not close write to the request's scratch row
+ instead (scoring never reads it). The tail rings hold the in-progress
+ pool's raw K + gate at ``pos % index_ratio`` for pools straddling two
+ forwards; never cleared -- prefix-cache resume points are page-aligned,
+ so a stale ring is never read.
+ """
+
+ def __init__(self, *args, num_req_slots: int, index_ratio: int, **kwargs) -> None:
+ self._num_req_slots = num_req_slots
+ self._index_ratio = index_ratio
+ super().__init__(*args, **kwargs)
+ assert self._page_size % index_ratio == 0, (
+ f"kpool needs page_size ({self._page_size}) divisible by "
+ f"index_ratio ({index_ratio})"
+ )
+
+ def _index_rows(self, num_pages: int) -> int:
+ # 1/ratio shadow of every token slot + one scratch row per request slot.
+ return num_pages * self._page_size // self._index_ratio + self._num_req_slots
+
+ @property
+ def cmp_scratch_base(self) -> int:
+ """First scratch row (== shadow row count); request ``table_idx`` offsets it."""
+ return self._num_pages * self._page_size // self._index_ratio
+
+ def _alloc(self, num_pages: int) -> None:
+ super()._alloc(num_pages)
+ self._tail_k = torch.zeros(
+ self._num_index_layers, self._num_req_slots, self._index_ratio,
+ self._index_head_dim, dtype=torch.bfloat16, device=self._device,
+ )
+ self._tail_gate = torch.zeros_like(self._tail_k)
+
+ def rebuild(self, num_pages: int) -> None:
+ self._tail_k = None
+ self._tail_gate = None
+ super().rebuild(num_pages)
+
+ def tail_k(self, slot: int) -> torch.Tensor:
+ """Tail raw keys for an indexer layer slot: ``[num_req_slots, ratio, head_dim]``."""
+ return self._tail_k[slot]
+
+ def tail_gate(self, slot: int) -> torch.Tensor:
+ return self._tail_gate[slot]
+
+
+__all__ = ["MLAKVCache", "DSAKVCache", "KpoolDSAKVCache"]
diff --git a/python/freetoken/layers/__init__.py b/python/freetoken/layers/__init__.py
index 2dbd6373..31c19bf9 100644
--- a/python/freetoken/layers/__init__.py
+++ b/python/freetoken/layers/__init__.py
@@ -1,4 +1,10 @@
-from .activation import gelu_and_mul, gelu_tanh_and_mul, silu_and_mul, swigluoai_and_mul
+from .activation import (
+ gelu_and_mul,
+ gelu_tanh_and_mul,
+ silu_and_mul,
+ swiglu_clamp_and_mul,
+ swigluoai_and_mul,
+)
from .base import BaseOP, OPList, StateLessOP
from .embedding import ParallelLMHead, VocabParallelEmbedding
from .linear import (
@@ -23,6 +29,7 @@ __all__ = [
"gelu_and_mul",
"gelu_tanh_and_mul",
"swigluoai_and_mul",
+ "swiglu_clamp_and_mul",
"BaseOP",
"StateLessOP",
"OPList",
diff --git a/python/freetoken/layers/activation.py b/python/freetoken/layers/activation.py
index 93602b6c..ae1d9881 100644
--- a/python/freetoken/layers/activation.py
+++ b/python/freetoken/layers/activation.py
@@ -56,4 +56,19 @@ def swigluoai_and_mul(
return swigluoai_and_mul(x, out=out, alpha=alpha, limit=limit)
-__all__ = ["silu_and_mul", "gelu_and_mul", "gelu_tanh_and_mul", "swigluoai_and_mul"]
+def swiglu_clamp_and_mul(
+ x, out=None, *, alpha: float = 1.0, limit: float = 10.0
+):
+ """GLM-5.3 clamped SwiGLU over UNINTERLEAVED halves: ``clamp(gate, max=limit) * sigmoid(alpha * gate) * clamp(up, +-limit)``."""
+ from freetoken.kernel.triton.activation import swiglu_clamp_and_mul
+
+ return swiglu_clamp_and_mul(x, out=out, alpha=alpha, limit=limit)
+
+
+__all__ = [
+ "silu_and_mul",
+ "gelu_and_mul",
+ "gelu_tanh_and_mul",
+ "swigluoai_and_mul",
+ "swiglu_clamp_and_mul",
+]
diff --git a/python/freetoken/layers/mhc.py b/python/freetoken/layers/mhc.py
new file mode 100644
index 00000000..1cd6459d
--- /dev/null
+++ b/python/freetoken/layers/mhc.py
@@ -0,0 +1,150 @@
+"""mHC -- Manifold-Constrained Hyper-Connections (GLM-5.3-Flash; arXiv 2512.24880).
+
+The residual stream is widened to ``hc_mult`` (n) parallel streams. Around every
+sublayer the streams are mixed by three learned, token-dependent maps computed
+from ONE fp32 GEMM over the flattened streams (``fn [2n+n^2, n*hidden]``),
+RMS-normalized over the full ``n*hidden`` vector and split into:
+
+* ``pre_mix [n]`` sigmoid gates: the sublayer input is ``sum_i pre_i * res_i``
+* ``post_mix [n]`` sigmoid*mult gates: how much sublayer output enters each stream
+* ``comb_mix [n, n]`` softmax + Sinkhorn-projected (approximately doubly-stochastic)
+ stream-mixing matrix -- the "manifold constraint": mixing
+ neither amplifies nor loses residual mass.
+
+``mhc_post`` then rebuilds the streams: ``out_j = sum_i comb_ij * res_i + post_j * x``.
+
+Semantics match vLLM's reference (``model_executor/kernels/mhc/torch.py``,
+PR #53906) bit-for-bit in fp32; the per-layer weights are ``hc_{attn,ffn}_fn`` /
+``_scale`` / ``_base`` from the checkpoint. This torch implementation is the
+correctness baseline; the fused triton kernel (``kernel/triton/mhc.py``,
+dispatched by ``mhc_fused_post_pre`` below) replaces it on CUDA and is
+validated against this file.
+"""
+
+from __future__ import annotations
+
+import torch
+
+
+def hc_expand(x: torch.Tensor, n: int) -> torch.Tensor:
+ """[T, hidden] -> [T, n, hidden] by replication (model entry)."""
+ return x.unsqueeze(1).expand(-1, n, -1).contiguous()
+
+
+def hc_contract(x: torch.Tensor) -> torch.Tensor:
+ """[T, n, hidden] -> [T, hidden] by averaging (model exit)."""
+ return x.mean(dim=1)
+
+
+def mhc_pre(
+ residual: torch.Tensor, # [T, n, hidden] bf16
+ fn: torch.Tensor, # [2n + n^2, n*hidden] fp32
+ hc_scale: torch.Tensor, # [3] fp32
+ hc_base: torch.Tensor, # [2n + n^2] fp32
+ rms_eps: float,
+ hc_eps: float,
+ post_mult: float,
+ sinkhorn_repeat: int,
+) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor]:
+ """Returns (post_mix [T, n, 1] fp32, comb_mix [T, n, n] fp32,
+ layer_input [T, hidden] bf16)."""
+ n, hidden = residual.shape[-2], residual.shape[-1]
+ t = residual.shape[0]
+
+ x = residual.reshape(t, n * hidden).to(torch.float32)
+ mixes = x @ fn.t()
+ # RMS over the FULL flattened n*hidden vector (not per stream).
+ mixes = mixes * torch.rsqrt(x.square().sum(-1, keepdim=True) / (n * hidden) + rms_eps)
+
+ pre_mix = torch.sigmoid(mixes[:, :n] * hc_scale[0] + hc_base[:n]) + hc_eps
+ post_mix = torch.sigmoid(mixes[:, n : 2 * n] * hc_scale[1] + hc_base[n : 2 * n])
+ post_mix = post_mix * post_mult
+
+ comb = mixes[:, 2 * n :].view(t, n, n) * hc_scale[2] + hc_base[2 * n :].view(1, n, n)
+ comb = torch.softmax(comb, dim=-1) + hc_eps
+ # Sinkhorn-Knopp projection toward the doubly-stochastic manifold: alternate
+ # column / row normalization, ``sinkhorn_repeat`` column steps in total.
+ comb = comb / (comb.sum(dim=-2, keepdim=True) + hc_eps)
+ for _ in range(sinkhorn_repeat - 1):
+ comb = comb / (comb.sum(dim=-1, keepdim=True) + hc_eps)
+ comb = comb / (comb.sum(dim=-2, keepdim=True) + hc_eps)
+
+ layer_input = (
+ (pre_mix.unsqueeze(-1) * residual.to(torch.float32)).sum(dim=1).to(residual.dtype)
+ )
+ return post_mix.view(t, n, 1), comb, layer_input
+
+
+def mhc_post(
+ x: torch.Tensor, # [T, hidden] sublayer output
+ residual: torch.Tensor, # [T, n, hidden]
+ post_mix: torch.Tensor, # [T, n, 1] fp32
+ comb_mix: torch.Tensor, # [T, n, n] fp32
+) -> torch.Tensor:
+ """out_j = sum_i comb_ij * residual_i + post_j * x; returns [T, n, hidden]."""
+ mixed = torch.einsum(
+ "tij,tih->tjh", comb_mix.to(torch.float32), residual.to(torch.float32)
+ )
+ post = post_mix.to(torch.float32) * x.unsqueeze(-2).to(torch.float32)
+ return (mixed + post).to(residual.dtype)
+
+
+def mhc_fused_post_pre_torch(
+ x: torch.Tensor,
+ residual: torch.Tensor,
+ post_mix: torch.Tensor,
+ comb_mix: torch.Tensor,
+ fn: torch.Tensor,
+ hc_scale: torch.Tensor,
+ hc_base: torch.Tensor,
+ rms_eps: float,
+ hc_eps: float,
+ post_mult: float,
+ sinkhorn_repeat: int,
+) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor]:
+ """Decomposed reference: hc_post then hc_pre; the fused triton kernel
+ replaces it on CUDA and is validated against this function."""
+ residual_new = mhc_post(x, residual, post_mix, comb_mix)
+ post_new, comb_new, layer_input = mhc_pre(
+ residual_new, fn, hc_scale, hc_base, rms_eps, hc_eps, post_mult, sinkhorn_repeat
+ )
+ return residual_new, post_new, comb_new, layer_input
+
+
+def mhc_fused_post_pre(
+ x: torch.Tensor,
+ residual: torch.Tensor,
+ post_mix: torch.Tensor,
+ comb_mix: torch.Tensor,
+ fn: torch.Tensor,
+ hc_scale: torch.Tensor,
+ hc_base: torch.Tensor,
+ rms_eps: float,
+ hc_eps: float,
+ post_mult: float,
+ sinkhorn_repeat: int,
+) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor]:
+ """Apply the previous sublayer's hc_post, then this sublayer's hc_pre on the
+ updated streams. The fused triton kernel serves every batch size on CUDA
+ (deliberately no T threshold); the decomposed torch path serves CPU/tests."""
+ if residual.is_cuda:
+ from freetoken.kernel.triton.mhc import mhc_fused_post_pre_triton
+
+ return mhc_fused_post_pre_triton(
+ x, residual, post_mix, comb_mix, fn, hc_scale, hc_base,
+ rms_eps, hc_eps, post_mult, sinkhorn_repeat,
+ )
+ return mhc_fused_post_pre_torch(
+ x, residual, post_mix, comb_mix, fn, hc_scale, hc_base,
+ rms_eps, hc_eps, post_mult, sinkhorn_repeat,
+ )
+
+
+__all__ = [
+ "hc_expand",
+ "hc_contract",
+ "mhc_pre",
+ "mhc_post",
+ "mhc_fused_post_pre",
+ "mhc_fused_post_pre_torch",
+]
diff --git a/python/freetoken/models/config.py b/python/freetoken/models/config.py
index 229cce81..8ce69d65 100644
--- a/python/freetoken/models/config.py
+++ b/python/freetoken/models/config.py
@@ -20,18 +20,24 @@ def vision_load_enabled() -> bool:
def detect_expert_quant(hf_config: Any) -> str:
"""Routed-expert quantization from a checkpoint's ``quantization_config``: ``"nvfp4"`` for
- a ModelOpt FP4 build, else the lowercased algo string (``"none"`` when unquantized). Models
- with mixed-precision configs (e.g. qwen3_5_moe) need their own detector."""
+ a ModelOpt FP4 build (``quant_algo: NVFP4``) OR an llm-compressor NVFP4 export
+ (``quant_method: compressed-tensors`` + ``format: nvfp4-pack-quantized``, e.g.
+ RedHatAI/GLM-5.3-Flash-NVFP4), else the lowercased algo string (``"none"`` when
+ unquantized). Models with mixed-precision configs (e.g. qwen3_5_moe) need their
+ own detector."""
quant = getattr(hf_config, "quantization_config", None)
if quant is None:
return "none"
- if isinstance(quant, dict):
- algo = quant.get("quant_algo") or quant.get("quant_method")
- else:
- algo = getattr(quant, "quant_algo", None) or getattr(quant, "quant_method", None)
+ get = quant.get if isinstance(quant, dict) else (lambda k, d=None: getattr(quant, k, d))
+ algo = get("quant_algo") or get("quant_method")
if algo is None:
return "none"
- return "nvfp4" if "fp4" in str(algo).lower() else str(algo).lower()
+ if "fp4" in str(algo).lower():
+ return "nvfp4"
+ # exact "nvfp4" (not the "fp4" substring) so MXFP4 exports don't misroute
+ if "nvfp4" in str(get("format") or "").lower():
+ return "nvfp4"
+ return str(algo).lower()
def detect_compressed_tensors_nvfp4(hf_config: Any) -> bool:
@@ -97,8 +103,9 @@ class KVCacheGroupSpec:
mla: bool = False
index_head_dim: int = 0
num_index_layers: int = 0
- # QSA compression: one index-key row per index_ratio tokens (1 keeps the BSA/DSA
- # per-token slab). The pool factory and the cost model divide by the same value.
+ # Grouped index-key compression: one index-key row per ``index_ratio`` tokens
+ # (QSA groups, glm5_next kpool pools; 1 keeps the per-token BSA/DSA slab). The
+ # pool factory and the KV cost model divide by the same value.
index_ratio: int = 1
# Attention-type taxonomy value for this group; drives the backend capability
# matrix and (with the pool factory) selects the KV pool family.
@@ -137,8 +144,9 @@ class FullAttentionGroupConfig(BaseAttentionGroupConfig):
mla: bool = False
index_head_dim: int = 0
num_index_layers: int = 0
- # QSA compression ratio (> 1 -> AttnType.QSA): index-key rows are per token group,
- # so the index slab costs index_head_dim * num_index_layers * 2 // index_ratio per token.
+ # Grouped index-key compression ratio (see KVCacheGroupSpec.index_ratio).
+ # GQA + ratio > 1 -> AttnType.QSA (Qwen3.8); MLA + ratio > 1 -> the glm5_next
+ # kpool DSA layout (attn type stays DSA; the pool factory branches on mla).
index_ratio: int = 1
@@ -165,6 +173,9 @@ class LinearGatedDeltaGroupConfig(BaseAttentionGroupConfig):
conv_kernel_dim: int
# Output-gate activation name ("silu", "sigmoid"), forwarded to rms_norm_gated.
output_gate: str
+ # "gdn" and "kda" share the same state geometry (one LinearStatePool serves
+ # both); the variant selects the kernels.
+ variant: Literal["gdn", "kda"] = "gdn"
@dataclass(frozen=True)
@@ -307,6 +318,10 @@ class ModelConfig:
# DSA indexer geometry the model module needs. Opaque to model-agnostic engine code;
# None for every other model.
glm_dsa_args: Any | None = None
+ # GLM-5.3-Flash (glm5_next) payload (Glm5NextArgs): NoPE-MLA dims, the kpool indexer
+ # geometry, the KDA head config, and the mHC knobs. Opaque to model-agnostic engine
+ # code; None for every other model.
+ glm5_args: Any | None = None
# MiniMax-M3 (minimax_m3) payload (MiniMaxM3Args): the block-sparse indexer geometry
# (index heads/dim, top-k blocks, init/local blocks, sparse layer set) plus the
# swigluoai/dense-MLP scalars the model module needs. Opaque to model-agnostic engine
diff --git a/python/freetoken/models/glm5_next/__init__.py b/python/freetoken/models/glm5_next/__init__.py
new file mode 100644
index 00000000..c1c05140
--- /dev/null
+++ b/python/freetoken/models/glm5_next/__init__.py
@@ -0,0 +1,10 @@
+from .config import parse_config
+from .model import Glm5NextForCausalLM
+from .weight import iter_weights, load_nvfp4_expert_sources
+
+__all__ = [
+ "Glm5NextForCausalLM",
+ "parse_config",
+ "iter_weights",
+ "load_nvfp4_expert_sources",
+]
diff --git a/python/freetoken/models/glm5_next/args.py b/python/freetoken/models/glm5_next/args.py
new file mode 100644
index 00000000..c25fef48
--- /dev/null
+++ b/python/freetoken/models/glm5_next/args.py
@@ -0,0 +1,225 @@
+"""GLM-5.3-Flash (``glm5_next``) hyperparameters.
+
+GLM-5.3-Flash is the first hybrid GLM: 34 of 45 decoder layers run KDA linear
+attention (Kimi-Delta-Attention: per-channel gated delta rule with separate q/k/v
+short convolutions) and the remaining 11 run DeepSeek-sparse attention -- MLA
+latent KV with a Lightning indexer whose K cache is *pool-compressed*
+(``index_kpool`` tokens fold into one stored entry). The main attention is NoPE
+(``qk_rope_head_dim == 0``, ``mla_use_nope``): no rotary embedding on Q/K at all;
+positional information enters only through the indexer's pool-compression APE.
+The residual stream is widened by mHC (Manifold-Constrained Hyper-Connections,
+``hc_mult`` streams mixed by a Sinkhorn-projected matrix around every sublayer).
+
+This payload carries everything the model module needs beyond the generic
+``ModelConfig`` fields; it is stashed on ``ModelConfig.glm5_args`` (opaque to the
+engine). Field spellings follow the checkpoint's ``config.json`` (transformers
+5.16 ``Glm5NextTextConfig``); ``load_args`` folds the checkpoint aliases
+(``hc_mult``, ``hc_sinkhorn_iters``, ``mla_use_nope``, the nested
+``linear_attn_config`` dict) the same way vLLM's config class does, so a future
+flattened checkpoint keeps loading.
+"""
+
+from __future__ import annotations
+
+from dataclasses import dataclass
+from typing import Any, Tuple
+
+# Layer-type strings used by the checkpoint's ``layer_types`` field.
+KDA_LAYER = "linear_attention"
+DSA_LAYER = "deepseek_sparse_attention"
+
+
+@dataclass(frozen=True)
+class Glm5NextArgs:
+ hidden_size: int
+ num_heads: int
+ # ---- MLA (the 11 "deepseek_sparse_attention" layers) ----
+ q_lora_rank: int
+ kv_lora_rank: int
+ qk_nope_head_dim: int
+ qk_rope_head_dim: int # 0: NoPE -- main attention carries no rotary dims
+ v_head_dim: int
+ mla_nope: bool
+ norm_eps: float
+ max_position: int
+ # ---- DSA indexer ----
+ index_n_heads: int
+ index_head_dim: int
+ index_topk: int
+ indexer_types: Tuple[str, ...]
+ indexer_rope_interleave: bool
+ # kpool compression: every ``index_kpool`` indexer-K entries pool into one
+ # stored entry (softmax(gate + APE)-weighted sum); top-k selects pools
+ # (select_k = index_topk // index_kpool) and ``always_select_tail``
+ # force-includes the in-progress tail pool.
+ index_kpool: int
+ index_kpool_compress: bool
+ index_kpool_always_select_tail: bool
+ # ---- KDA linear attention (the 34 "linear_attention" layers) ----
+ linear_num_heads: int
+ linear_head_dim: int
+ linear_conv_kernel_dim: int
+ linear_lower_bound: float
+ # ---- per-layer layout ----
+ layer_types: Tuple[str, ...]
+ mlp_layer_types: Tuple[str, ...]
+ # ---- mHC (Manifold-Constrained Hyper-Connections) ----
+ mhc: bool
+ mhc_num_residual_streams: int
+ hc_eps: float
+ mhc_sinkhorn_iterations: int
+ mhc_tau: float
+ mhc_post_mult_value: float
+ mhc_no_norm_weight: bool
+ # ---- misc ----
+ swiglu_limit: float | None
+ rope_theta: float # indexer-side only; the main attention is NoPE
+
+ @property
+ def qk_head_dim(self) -> int:
+ return self.qk_nope_head_dim + self.qk_rope_head_dim
+
+ @property
+ def latent_dim(self) -> int:
+ """Width of one MLA latent row in the paged pool: ckv | kpe."""
+ return self.kv_lora_rank + self.qk_rope_head_dim
+
+ @property
+ def kda_layer_ids(self) -> Tuple[int, ...]:
+ return tuple(i for i, t in enumerate(self.layer_types) if t == KDA_LAYER)
+
+ @property
+ def dsa_layer_ids(self) -> Tuple[int, ...]:
+ return tuple(i for i, t in enumerate(self.layer_types) if t == DSA_LAYER)
+
+ def is_kda_layer(self, layer_id: int) -> bool:
+ return self.layer_types[layer_id] == KDA_LAYER
+
+
+def _require_mhc(mhc: bool) -> bool:
+ """The decoder layer wires the hyper-connection tensors unconditionally
+ (model.py touches hc_attn_fn on every forward), so an mhc=False checkpoint
+ would die with an AttributeError mid-forward -- refuse it at parse until a
+ real one exists to implement against."""
+ if not mhc:
+ raise NotImplementedError(
+ "glm5_next requires mhc=True (manifold-constrained hyper-connections); "
+ "an mhc=False checkpoint has no implementation yet."
+ )
+ return mhc
+
+
+def _get(cfg: Any, name: str, default: Any = None) -> Any:
+ return getattr(cfg, name, default)
+
+
+def load_args(hf_config: Any) -> Glm5NextArgs:
+ """Build ``Glm5NextArgs`` from the checkpoint config (nested ``text_config`` or a
+ flat text-only config). Raw checkpoint spellings win; the vLLM-normalized names
+ are accepted as fallbacks."""
+ text = _get(hf_config, "text_config", hf_config)
+
+ num_layers = int(text.num_hidden_layers)
+ layer_types = tuple(_get(text, "layer_types", ()) or ())
+ if not layer_types:
+ raise ValueError("glm5_next config is missing layer_types")
+ if len(layer_types) != num_layers:
+ raise ValueError(
+ f"layer_types has {len(layer_types)} entries for {num_layers} layers"
+ )
+ unknown = sorted(set(layer_types) - {KDA_LAYER, DSA_LAYER})
+ if unknown:
+ raise ValueError(f"unsupported layer_types entries: {unknown}")
+
+ mlp_layer_types = tuple(_get(text, "mlp_layer_types", ()) or ())
+ if not mlp_layer_types:
+ # Older-schema fallback (mirrors vLLM): derive from first_k_dense_replace.
+ first_dense = int(_get(text, "first_k_dense_replace", 0) or 0)
+ mlp_layer_types = ("dense",) * first_dense + ("sparse",) * (
+ num_layers - first_dense
+ )
+
+ # Checkpoint alias folding (transformers 5.16 spellings first).
+ mla_nope = _get(text, "mla_use_nope", _get(text, "mla_nope", False))
+ hc_mult = _get(text, "hc_mult", _get(text, "mhc_num_residual_streams", 4))
+ hc_sinkhorn = _get(
+ text, "hc_sinkhorn_iters", _get(text, "mhc_sinkhorn_iterations", 20)
+ )
+
+ # KDA head geometry ships as the nested ``linear_attn_config`` dict; a future
+ # flattened schema would carry vLLM-style ``linear_*`` top-level fields.
+ linear_cfg = _get(text, "linear_attn_config", None)
+ if linear_cfg is not None and not isinstance(linear_cfg, dict):
+ linear_cfg = {
+ k: getattr(linear_cfg, k)
+ for k in (
+ "num_heads",
+ "head_dim",
+ "short_conv_kernel_size",
+ "gate_lower_bound",
+ )
+ if hasattr(linear_cfg, k)
+ }
+ linear_cfg = linear_cfg or {}
+ linear_num_heads = int(
+ linear_cfg.get("num_heads", _get(text, "linear_num_heads", 0))
+ )
+ linear_head_dim = int(linear_cfg.get("head_dim", _get(text, "linear_head_dim", 0)))
+ linear_conv = int(
+ linear_cfg.get(
+ "short_conv_kernel_size", _get(text, "linear_conv_kernel_dim", 4)
+ )
+ )
+ linear_lower_bound = float(
+ linear_cfg.get("gate_lower_bound", _get(text, "linear_lower_bound", -5.0))
+ )
+ if KDA_LAYER in layer_types and (linear_num_heads <= 0 or linear_head_dim <= 0):
+ raise ValueError("glm5_next config is missing the KDA linear_attn_config dims")
+
+ # The main attention is NoPE; rope exists only on the indexer side. The
+ # checkpoint ships no rope_theta -- fall back to the transformers default.
+ rope = _get(text, "rope_parameters", None) or {}
+ rope_theta = float(rope.get("rope_theta", _get(text, "rope_theta", 10000.0)))
+
+ swiglu_limit = _get(text, "swiglu_limit", None)
+
+ return Glm5NextArgs(
+ hidden_size=int(text.hidden_size),
+ num_heads=int(text.num_attention_heads),
+ q_lora_rank=int(text.q_lora_rank),
+ kv_lora_rank=int(text.kv_lora_rank),
+ qk_nope_head_dim=int(text.qk_nope_head_dim),
+ qk_rope_head_dim=int(text.qk_rope_head_dim),
+ v_head_dim=int(text.v_head_dim),
+ mla_nope=bool(mla_nope),
+ norm_eps=float(text.rms_norm_eps),
+ max_position=int(text.max_position_embeddings),
+ index_n_heads=int(_get(text, "index_n_heads", 0) or 0),
+ index_head_dim=int(_get(text, "index_head_dim", 0) or 0),
+ index_topk=int(_get(text, "index_topk", 0) or 0),
+ indexer_types=tuple(_get(text, "indexer_types", ()) or ()),
+ indexer_rope_interleave=bool(_get(text, "indexer_rope_interleave", False)),
+ index_kpool=int(_get(text, "index_kpool", 1) or 1),
+ index_kpool_compress=bool(_get(text, "index_kpool_compress", False)),
+ index_kpool_always_select_tail=bool(
+ _get(text, "index_kpool_always_select_tail", False)
+ ),
+ linear_num_heads=linear_num_heads,
+ linear_head_dim=linear_head_dim,
+ linear_conv_kernel_dim=linear_conv,
+ linear_lower_bound=linear_lower_bound,
+ layer_types=layer_types,
+ mlp_layer_types=mlp_layer_types,
+ mhc=_require_mhc(bool(_get(text, "mhc", False))),
+ mhc_num_residual_streams=int(hc_mult),
+ hc_eps=float(_get(text, "hc_eps", 1e-6)),
+ mhc_sinkhorn_iterations=int(hc_sinkhorn),
+ mhc_tau=float(_get(text, "mhc_tau", 0.05)),
+ mhc_post_mult_value=float(_get(text, "mhc_post_mult_value", 2.0)),
+ mhc_no_norm_weight=bool(_get(text, "mhc_no_norm_weight", False)),
+ swiglu_limit=(None if swiglu_limit is None else float(swiglu_limit)),
+ rope_theta=rope_theta,
+ )
+
+
+__all__ = ["Glm5NextArgs", "load_args", "KDA_LAYER", "DSA_LAYER"]
diff --git a/python/freetoken/models/glm5_next/attention.py b/python/freetoken/models/glm5_next/attention.py
new file mode 100644
index 00000000..1ab6c340
--- /dev/null
+++ b/python/freetoken/models/glm5_next/attention.py
@@ -0,0 +1,184 @@
+"""GLM-5.3-Flash NoPE Multi-head Latent Attention with a kpool DSA indexer.
+
+MLA weight-absorption as in glm_moe_dsa (kv_b absorbed into Q and onto the
+output; the paged pool stores one latent row per token) with two GLM-5.3
+differences:
+
+* **NoPE** (``mla_use_nope``, ``qk_rope_head_dim == 0``): no rotary embedding
+ anywhere in the main attention -- Q is all-nope [T, H, 256], the latent row is
+ bare ckv (512, no kpe half). All rope plumbing degenerates to zero-width
+ tensors, which the DSA backend's cat/scatter handle natively. Positional
+ information enters ONLY through the indexer's pool-compression APE.
+* **kpool indexer**: every DSA layer owns its indexer (no IndexShare). The
+ indexer K cache stores one entry per ``index_kpool`` (4) tokens: a per-channel
+ ``softmax(gate + ape)``-weighted sum of the raw keys. Scoring runs at pool
+ granularity (select_k = index_topk / kpool) and the selected pools expand back
+ to token rows, with the in-progress tail pool force-included
+ (``index_kpool_always_select_tail``). This module owns the PROJECTIONS (wq_b /
+ wk+k_norm / weights_proj / compress gate + APE); pooling, scoring, selection
+ and the tail buffer live in the backend (attention/dsa_indexer_kpool.py).
+
+Faithfulness note: the reference stack stores pooled entries as
+Hadamard-rotated fp8 (a quantization device; the rotation cancels in the dot
+product). FreeToken stores pooled entries in bf16 -- mathematically the same
+score with strictly less quantization error -- matching the GLM-5.2 precedent
+of bf16 indexer keys.
+TODO: fp8 index slab (+ Hadamard rotation, upstream parity) to halve the slab bytes.
+"""
+
+from __future__ import annotations
+
+from typing import TYPE_CHECKING
+
+import torch
+from freetoken.core import get_global_ctx
+from freetoken.layers import BaseOP, LinearReplicated, RMSNorm
+# Shared with GLM-5.2 (weight.py imports privately from the same package).
+from freetoken.models.glm_moe_dsa.attention import _IdxLayerNorm, _make_proj
+from freetoken.utils import nvtx_annotate
+
+if TYPE_CHECKING:
+ from freetoken.models.config import ModelConfig
+
+
+class Glm5NextIndexer(BaseOP):
+ """kpool DSA indexer projections (every DSA layer owns one; no rope -- the
+ checkpoint's NoPE geometry leaves ``qk_rope_head_dim == 0`` so position enters
+ only via the compression APE).
+
+ Kept bf16 in every quant mode: small (~17 MB/layer) and the top-k boundary is
+ precision-sensitive (same reasoning as glm_moe_dsa's indexer).
+ """
+
+ def __init__(self, config: ModelConfig, layer_id: int):
+ args = config.glm5_args
+ self.n_heads = args.index_n_heads
+ self.head_dim = args.index_head_dim
+ self.kpool = args.index_kpool
+ self.wq_b = LinearReplicated(
+ args.q_lora_rank, self.n_heads * self.head_dim, has_bias=False
+ )
+ self.wk = LinearReplicated(args.hidden_size, self.head_dim, has_bias=False)
+ self.k_norm = _IdxLayerNorm(self.head_dim, eps=1e-6)
+ self.weights_proj = LinearReplicated(
+ args.hidden_size, self.n_heads, has_bias=False
+ )
+ # Pool-compression parameters (checkpoint names, no ".weight" suffix on
+ # the gate: it is stored as a bare [head_dim, hidden] tensor).
+ self.index_kpool_compress_gate = torch.empty(
+ self.head_dim, args.hidden_size
+ )
+ # Per-pool-slot position bias, fp32 (models/weight.py exempts it from the
+ # model-dtype downcast alongside A_log/dt_bias).
+ self.index_kpool_compress_ape = torch.empty(
+ self.kpool, self.head_dim, dtype=torch.float32
+ )
+
+ def compute(self, x: torch.Tensor, q_resid: torch.Tensor) -> "DSAIndexerInputs":
+ """Per-token indexer projections as a typed inputs object: q [T, Hi, Di],
+ k [T, Di], weights [T, Hi] fp32, plus the kpool gate scores [T, Di] and
+ the [kpool, Di] APE parameter (passed per call; the backend keeps no copy)."""
+ from freetoken.attention.dsa import DSAIndexerInputs
+
+ t = x.shape[0]
+ q = self.wq_b.forward(q_resid).view(t, self.n_heads, self.head_dim)
+ k = self.k_norm.forward(self.wk.forward(x))
+ w = self.weights_proj.forward(x).float() * (self.n_heads**-0.5)
+ gate = torch.nn.functional.linear(x, self.index_kpool_compress_gate)
+ return DSAIndexerInputs(
+ q=q, k=k, w=w, gate=gate, ape=self.index_kpool_compress_ape
+ )
+
+
+class Glm5NextAttention(BaseOP):
+ def __init__(self, config: ModelConfig, layer_id: int):
+ args = config.glm5_args
+ self.layer_id = layer_id
+ self.indexer = Glm5NextIndexer(config, layer_id)
+ self.num_heads = args.num_heads
+ self.qk_nope_head_dim = args.qk_nope_head_dim
+ self.qk_rope_head_dim = args.qk_rope_head_dim # 0 (NoPE)
+ self.qk_head_dim = args.qk_head_dim
+ self.v_head_dim = args.v_head_dim
+ self.kv_lora_rank = args.kv_lora_rank
+ assert args.qk_rope_head_dim == 0 and args.mla_nope, (
+ "glm5_next attention implements the NoPE geometry; a roped variant "
+ "would need the glm_moe_dsa rope plumbing back"
+ )
+
+ quant = config.attn_quant
+ self.q_a_proj = _make_proj(quant, args.hidden_size, args.q_lora_rank)
+ self.q_a_layernorm = RMSNorm(args.q_lora_rank, eps=args.norm_eps)
+ self.q_b_proj = _make_proj(
+ quant, args.q_lora_rank, self.num_heads * self.qk_head_dim
+ )
+ # NoPE: kv_a projects to bare ckv (no +qk_rope_head_dim rows).
+ self.kv_a_proj_with_mqa = _make_proj(
+ quant, args.hidden_size, self.kv_lora_rank
+ )
+ self.kv_a_layernorm = RMSNorm(self.kv_lora_rank, eps=args.norm_eps)
+ # kv_b stays bf16 in every mode (bmm absorption operand, not a Linear).
+ self.kv_b_proj = LinearReplicated(
+ self.kv_lora_rank,
+ self.num_heads * (self.qk_nope_head_dim + self.v_head_dim),
+ has_bias=False,
+ )
+ self.o_proj = _make_proj(
+ quant, self.num_heads * self.v_head_dim, args.hidden_size
+ )
+ self._w_uk: torch.Tensor | None = None
+ self._w_uv: torch.Tensor | None = None
+
+ def _kv_b(self) -> tuple[torch.Tensor, torch.Tensor]:
+ """Per-head kv_b split in bmm-ready bf16 layout (same contract and
+ prepare_for_runtime budgeting as glm_moe_dsa; see that module)."""
+ if self._w_uk is None:
+ w = self.kv_b_proj.weight.view(
+ self.num_heads,
+ self.qk_nope_head_dim + self.v_head_dim,
+ self.kv_lora_rank,
+ )
+ self._w_uk = w[:, : self.qk_nope_head_dim, :].contiguous()
+ self._w_uv = w[:, self.qk_nope_head_dim :, :].transpose(1, 2).contiguous()
+ return self._w_uk, self._w_uv
+
+ def prepare_for_runtime(self) -> None:
+ self._kv_b()
+ self.kv_b_proj.weight = None # checkpoint layout freed; repacked forms serve
+
+ @nvtx_annotate("MLA")
+ def forward(self, x: torch.Tensor) -> torch.Tensor:
+ ctx = get_global_ctx()
+ t = x.shape[0]
+ w_uk, w_uv = self._kv_b()
+
+ q_a_resid = self.q_a_layernorm.forward(self.q_a_proj.forward(x))
+ q = self.q_b_proj.forward(q_a_resid)
+ # NoPE: the whole head is the nope part; no rope split, no rope kernel.
+ q_nope = q.view(t, self.num_heads, self.qk_head_dim)
+
+ c_kv = self.kv_a_layernorm.forward(self.kv_a_proj_with_mqa.forward(x))
+
+ # Absorb kv_b's k-part into the query: q_nope[H,T,nope] @ W_uk[H,nope,lora].
+ q_absorbed = torch.bmm(q_nope.transpose(0, 1).contiguous(), w_uk).transpose(0, 1)
+
+ indexer_inputs = (
+ self.indexer.compute(x, q_a_resid)
+ if getattr(ctx.attn_backend, "dsa_enabled", False)
+ else None
+ )
+
+ # Zero-width rope halves: cat/scatter no-ops in the backend.
+ q_pe = q.new_empty(t, self.num_heads, 0)
+ k_rope = q.new_empty(t, 0)
+ o_latent = ctx.attn_backend.mla_forward(
+ q_absorbed.contiguous(), q_pe, c_kv.contiguous(), k_rope,
+ self.layer_id, ctx.batch, indexer_inputs=indexer_inputs,
+ ) # [T, H, kv_lora_rank]
+
+ # Absorb kv_b's v-part onto the output: o_latent[H,T,lora] @ W_uv_t[H,lora,v].
+ o = torch.bmm(o_latent.transpose(0, 1).contiguous(), w_uv).transpose(0, 1)
+ return self.o_proj.forward(o.reshape(t, self.num_heads * self.v_head_dim))
+
+
+__all__ = ["Glm5NextAttention", "Glm5NextIndexer"]
diff --git a/python/freetoken/models/glm5_next/config.py b/python/freetoken/models/glm5_next/config.py
new file mode 100644
index 00000000..294ef579
--- /dev/null
+++ b/python/freetoken/models/glm5_next/config.py
@@ -0,0 +1,198 @@
+"""Engine-facing config for GLM-5.3-Flash (``glm5_next``).
+
+Two attention groups, ordered by first layer id:
+
+* ``linear`` -- 34 KDA layers (``LinearGatedDeltaGroupConfig`` with
+ ``variant="kda"``). KDA's state geometry coincides with GDN's (conv width
+ ``2*K*dk + V*dv`` == q|k|v at ``H*d`` each; recurrent ``[H, d, d]``), so the
+ existing ``LinearStatePool`` shapes serve it unchanged.
+* ``full`` -- 11 DSA layers (``FullAttentionGroupConfig`` with ``mla=True`` and
+ the indexer dims). The MLA is NoPE (``qk_rope_head_dim == 0``): the latent row
+ is bare ckv (512), and the indexer K slab is kpool-compressed
+ (``index_kpool=4`` -> tokens/4 stored entries; the spec's ``index_ratio``
+ drives both the pool factory and the KV cost model).
+
+MoE routing mirrors glm_moe_dsa (sigmoid ``noaux_tc``, shared expert, routed
+scaling 2.5) with 288 routed experts and a clamped SwiGLU
+(``swiglu_limit=10``). Everything model-specific beyond ``ModelConfig`` rides in
+``glm5_args`` (``Glm5NextArgs``).
+
+Resident-weight quantization (attn/dense/lm_head fp8-at-load, the GLM-5.2
+bandwidth trick) is OFF by default: the NVFP4 exports deliberately quantize only
+the routed experts (attention / shared expert / lm_head ship bf16 behind the
+quantization_config ignore list), and a serving engine must not override the
+checkpoint author's precision decision silently -- the checkpoint is served
+as-is, like vLLM/sglang. FREETOKEN_GLM5_ATTN_FP8=1 / FREETOKEN_GLM5_MLP_FP8=1
+opt into the W8A16 requantization (measured decode 36 -> 45.5 tok/s on the
+hybrid reference setup; the win needs CUDA graphs -- launch-bound eager decode
+gets slower).
+"""
+
+from __future__ import annotations
+
+import os
+from typing import Any
+
+from freetoken.models.config import (
+ FullAttentionGroupConfig,
+ LinearGatedDeltaGroupConfig,
+ ModelConfig,
+ RotaryConfig,
+ detect_expert_quant,
+)
+
+from .args import load_args
+
+# Load-time W8A16 fp8 for resident weights: default OFF, env opt-in (see module
+# docstring for the rationale and measured numbers). attn covers the KDA
+# in_proj_qkv/o_proj + DSA projections (the precision-sensitive b|f_a|g_a gate
+# slice stays bf16, see kda.py); mlp covers dense MLPs, shared expert, lm_head.
+_ATTN_FP8 = os.getenv("FREETOKEN_GLM5_ATTN_FP8", "0") != "0"
+_MLP_FP8 = os.getenv("FREETOKEN_GLM5_MLP_FP8", "0") != "0"
+
+
+def _dsa_on(args, dsa_layer_ids) -> bool:
+ """DSA serving switch, resolved ONCE into the attention-group spec (the pool
+ factory, KV cost model, and backend read the spec, never the env)."""
+ return (
+ len(dsa_layer_ids) > 0
+ and args.index_topk > 0
+ and args.index_head_dim > 0
+ and os.getenv("FREETOKEN_GLM5_DSA", "1") != "0"
+ )
+
+
+def parse_config(hf_config: Any) -> ModelConfig:
+ args = load_args(hf_config)
+ text = getattr(hf_config, "text_config", hf_config)
+
+ num_layers = len(args.layer_types)
+ # Dev/testing only: cap the layer count so the forward path / KV / offload cache
+ # can be exercised without the full ~175 GB of experts. Unset in normal use.
+ _cap = os.environ.get("FREETOKEN_GLM5_MAX_LAYERS")
+ if _cap:
+ num_layers = min(num_layers, int(_cap))
+
+ kda_ids = tuple(i for i in args.kda_layer_ids if i < num_layers)
+ dsa_ids = tuple(i for i in args.dsa_layer_ids if i < num_layers)
+
+ # NoPE: the main attention carries no rotary dims (rotary_dim == 0); rope
+ # survives only in the indexer geometry (args.rope_theta / interleave).
+ rotary_config = RotaryConfig(
+ head_dim=args.qk_head_dim,
+ rotary_dim=args.qk_rope_head_dim,
+ max_position=args.max_position,
+ base=args.rope_theta,
+ scaling=None,
+ )
+ latent_dim = args.latent_dim # 512: bare ckv, no kpe rows
+
+ dsa_on = _dsa_on(args, dsa_ids)
+ # Each DSA layer owns its indexer ("full" in indexer_types); count only the
+ # layers that actually exist under a dev layer cap.
+ num_index_layers = (
+ sum(
+ 1
+ for i in dsa_ids
+ if i < len(args.indexer_types) and args.indexer_types[i] == "full"
+ )
+ if dsa_on
+ else 0
+ )
+
+ linear_group = LinearGatedDeltaGroupConfig(
+ name="linear",
+ layer_ids=kda_ids,
+ # KDA: one qk-sized and one v-sized head set (H=64, d=128 each). Mapping onto
+ # the GDN field names keeps LinearStatePool's conv-width formula exact:
+ # 2*K*dk + V*dv = (q|k) + v = 3 * H * d.
+ num_key_heads=args.linear_num_heads,
+ num_value_heads=args.linear_num_heads,
+ key_head_dim=args.linear_head_dim,
+ value_head_dim=args.linear_head_dim,
+ conv_kernel_dim=args.linear_conv_kernel_dim,
+ output_gate="sigmoid", # KDA o_norm gates with sigmoid (kda.py)
+ variant="kda",
+ )
+ full_group = FullAttentionGroupConfig(
+ name="full",
+ layer_ids=dsa_ids,
+ num_kv_heads=1, # single shared MLA latent
+ head_dim=latent_dim,
+ rotary_config=rotary_config,
+ mla=True,
+ index_head_dim=args.index_head_dim if dsa_on else 0,
+ num_index_layers=num_index_layers,
+ index_ratio=(args.index_kpool if dsa_on and args.index_kpool_compress else 1),
+ )
+ groups = tuple(
+ sorted(
+ (linear_group, full_group),
+ key=lambda g: g.layer_ids[0] if g.layer_ids else 1 << 30,
+ )
+ )
+
+ # The MLP layout is a dense prefix + sparse tail; ModelConfig models exactly that
+ # via first_k_dense_replace, so assert the checkpoint matches before collapsing.
+ mlp_types = args.mlp_layer_types[:num_layers]
+ first_dense = next(
+ (i for i, t in enumerate(mlp_types) if t == "sparse"), len(mlp_types)
+ )
+ assert all(t == "dense" for t in mlp_types[:first_dense]) and all(
+ t == "sparse" for t in mlp_types[first_dense:]
+ ), f"mlp_layer_types is not a dense-prefix layout: {mlp_types}"
+
+ return ModelConfig(
+ num_layers=num_layers,
+ num_qo_heads=args.num_heads,
+ num_kv_heads=1,
+ head_dim=latent_dim,
+ hidden_size=args.hidden_size,
+ vocab_size=text.vocab_size,
+ intermediate_size=text.intermediate_size,
+ # hidden_act stands proxy for the EXPERT activation everywhere the engine
+ # gates on it (NVFP4 backend selection, CPU-executor capability): GLM-5.3
+ # experts and dense MLPs both run clamped SwiGLU when swiglu_limit is set,
+ # even though the HF config still says "silu". Passing "silu" through would
+ # let auto-selection repack experts into the marlin/b12x silu-only epilogue.
+ hidden_act=(
+ "swiglu_clamp" if args.swiglu_limit is not None else text.hidden_act
+ ),
+ rms_norm_eps=args.norm_eps,
+ tie_word_embeddings=bool(getattr(text, "tie_word_embeddings", False)),
+ rotary_config=rotary_config,
+ attention_groups=groups,
+ num_experts=(
+ getattr(text, "n_routed_experts", None) or getattr(text, "num_experts", 0)
+ ),
+ num_experts_per_tok=(
+ getattr(text, "num_experts_per_tok", None)
+ or getattr(text, "num_experts_per_token", 0)
+ ),
+ moe_intermediate_size=getattr(text, "moe_intermediate_size", 0)
+ or text.intermediate_size,
+ norm_topk_prob=bool(getattr(text, "norm_topk_prob", True)),
+ model_type=getattr(hf_config, "model_type", "glm5_next"),
+ architectures=getattr(
+ hf_config, "architectures", ["Glm5NextForConditionalGeneration"]
+ ),
+ moe_enabled=True,
+ expert_quant=detect_expert_quant(hf_config),
+ first_k_dense_replace=first_dense,
+ n_shared_experts=int(getattr(text, "n_shared_experts", 0) or 0),
+ routed_scaling_factor=float(getattr(text, "routed_scaling_factor", 1.0)),
+ n_group=int(getattr(text, "n_group", 1) or 1),
+ topk_group=int(getattr(text, "topk_group", 1) or 1),
+ attn_sm_scale=args.qk_head_dim**-0.5,
+ has_attn_bias=bool(getattr(text, "attention_bias", False)),
+ swiglu_limit=args.swiglu_limit,
+ attn_quant="fp8_pertensor" if _ATTN_FP8 else "none",
+ dense_quant="fp8_pertensor" if _MLP_FP8 else "none",
+ lm_head_quant="fp8_pertensor" if _MLP_FP8 else "none",
+ vision_config=None, # text-only milestone; model.visual.* weights are dropped
+ image_token_id=getattr(hf_config, "image_token_id", None),
+ glm5_args=args,
+ )
+
+
+__all__ = ["parse_config"]
diff --git a/python/freetoken/models/glm5_next/kda.py b/python/freetoken/models/glm5_next/kda.py
new file mode 100644
index 00000000..1d44e3a4
--- /dev/null
+++ b/python/freetoken/models/glm5_next/kda.py
@@ -0,0 +1,252 @@
+"""GLM-5.3-Flash KDA (Kimi Delta Attention) op.
+
+Per-channel gated delta rule over H=64 heads of D=128, with SEPARATE q/k/v short
+convolutions (vs GDN's one fused conv), a low-rank forget gate (``f_a`` ->
+``f_b`` -> raw per-channel logits; the bounded safe gate ``lower_bound *
+sigmoid(exp(A_log) * (g + dt_bias))`` is computed inside the kernels), a
+per-head sigmoid beta (``b_proj``), and a sigmoid-gated output RMSNorm
+(``o_norm`` gated by ``g_b(g_a(x))``).
+
+State lives in ``ctx.linear_state_pool`` exactly like GDN: the conv state is the
+merged q|k|v stream (width 3*H*D == the pool's ``2*K*dk + V*dv``) and the
+recurrent state is one [D, D] matrix per head, stored in the KERNEL's [V, K]
+layout (coincides with the pool's [K, V] declaration because D_k == D_v; same
+convention as GDN, see qwen3_5_moe/gdn.py).
+
+Kernels: ``fused_recurrent_kda`` decodes with in-kernel gate/beta/l2norm and
+per-slot state read/write (slot 0 is its NULL sentinel == the pool's padding
+slot); ``chunk_kda_with_fused_gate`` prefills from an explicitly gathered
+initial state and returns the final state, which this op scatters back (the
+kernel CLOBBERS its v buffer -- v here is an ephemeral conv output, so that is
+free). Hybrid-radix track snapshots ride the per-chunk h (``return_h``).
+"""
+
+from __future__ import annotations
+
+from typing import TYPE_CHECKING
+
+import torch
+from freetoken.core import get_global_ctx
+from freetoken.kernel.causal_conv1d import causal_conv1d_decode, causal_conv1d_varlen
+from freetoken.layers import BaseOP, LinearColParallelMerged, LinearReplicated
+from freetoken.utils import nvtx_annotate
+
+if TYPE_CHECKING:
+ from freetoken.models.config import ModelConfig
+
+
+class _DepthwiseConv1d(BaseOP):
+ """Merged q|k|v depthwise conv weight ``[3*H*D, 1, K]`` (key ``conv1d.weight``;
+ the loader concatenates the checkpoint's q/k/v_conv1d along channels)."""
+
+ def __init__(self, conv_dim: int, kernel: int):
+ self.weight = torch.empty(conv_dim, 1, kernel)
+
+
+class _GatedRMSNormSigmoid(BaseOP):
+ """RMSNorm(x) * sigmoid(z), fused (KDA's o_norm; GDN's variant gates with silu)."""
+
+ def __init__(self, dim: int, eps: float):
+ self.weight = torch.empty(dim)
+ self.eps = eps
+
+ def forward(self, x: torch.Tensor, z: torch.Tensor) -> torch.Tensor:
+ from freetoken.kernel.fla import rms_norm_gated
+
+ return rms_norm_gated(
+ x=x, weight=self.weight, bias=None, z=z, eps=self.eps,
+ is_rms_norm=True, norm_before_gate=True, activation="sigmoid",
+ )
+
+
+class Glm5NextKDA(BaseOP):
+ """KDA op; state is held in ``ctx.linear_state_pool`` keyed by the request's
+ linear slot (``FLAMetadata.cache_indices``). Parameter names follow the
+ checkpoint modulo two load-time fusions (see weight.py): ``in_proj`` is
+ q|k|v|b|f_a|g_a concatenated, ``conv1d`` is q|k|v conv concatenated."""
+
+ def __init__(self, config: ModelConfig, layer_id: int):
+ args = config.glm5_args
+ self.layer_id = layer_id
+ self.num_heads = args.linear_num_heads
+ self.head_dim = args.linear_head_dim
+ self.proj_size = self.num_heads * self.head_dim # H * D
+ self.conv_dim = 3 * self.proj_size # merged q|k|v stream
+ self.conv_kernel_size = args.linear_conv_kernel_dim
+ self.lower_bound = args.linear_lower_bound
+ self.scale = self.head_dim**-0.5
+
+ p, h, d = self.proj_size, self.num_heads, self.head_dim
+ # q|k|v dominate the resident read (3 * 8192 x 4096 = 201 MB/layer bf16 --
+ # the single biggest dense-weight stream in the model); under the fp8
+ # resident mode they split into their own W8A16 GEMM while the small,
+ # precision-sensitive gate projections (b|f_a|g_a) stay bf16 (the GDN
+ # qkvz/ba split precedent). BF16 mode keeps the single fused GEMM.
+ self._fp8 = config.attn_quant == "fp8_pertensor"
+ self._bfg_split = [h, d, d]
+ if self._fp8:
+ from freetoken.kernel.triton.fp8_pertensor_linear import Fp8PerTensorColMerged
+
+ self.in_proj_qkv = Fp8PerTensorColMerged(
+ args.hidden_size, [p, p, p], has_bias=False
+ )
+ self.in_proj_bfg = LinearColParallelMerged(
+ args.hidden_size, self._bfg_split, has_bias=False
+ )
+ else:
+ self._in_proj_split = [p, p, p, h, d, d]
+ self.in_proj = LinearColParallelMerged(
+ args.hidden_size, self._in_proj_split, has_bias=False
+ )
+ # Low-rank gate up-projections (128 -> 8192): forget gate and output gate.
+ self.f_b_proj = LinearReplicated(d, p, has_bias=False)
+ self.g_b_proj = LinearReplicated(d, p, has_bias=False)
+ self.conv1d = _DepthwiseConv1d(self.conv_dim, self.conv_kernel_size)
+ # Gate params stay fp32 (exp/sigmoid precision; the kernels read fp32).
+ # models/weight.py exempts *.A_log / *.dt_bias from the model-dtype downcast.
+ self.A_log = torch.empty(h, dtype=torch.float32)
+ self.dt_bias = torch.empty(p, dtype=torch.float32)
+ self.o_norm = _GatedRMSNormSigmoid(d, eps=args.norm_eps)
+ # o_proj follows the resident quant mode (rationale: the split comment
+ # at _fp8 above); f_b/g_b stay bf16 alongside the gate slice.
+ from .attention import _make_proj
+
+ self.o_proj = _make_proj(config.attn_quant, p, args.hidden_size)
+
+ def _conv_weight(self) -> torch.Tensor:
+ return self.conv1d.weight.squeeze(1) # [conv_dim, kernel]
+
+ def _write_track_snapshot(self, pool, li, conv_in, h, fla) -> None:
+ """Hybrid-radix: snapshot recurrent + conv state at the chunk-aligned track
+ boundary into a donatable pool slot (same contract as GDN, see
+ qwen3_5_moe/gdn.py). h rows are the kernel's per-chunk [V, K] states --
+ a direct copy into the pool's [K, V] slots (D_k == D_v)."""
+ rec = pool.recurrent_states[li]
+ rec.index_copy_(0, fla.track_dst, h[0, fla.track_h_row].to(rec.dtype))
+ cv = pool.conv_states[li]
+ conv_win = conv_in[fla.track_conv_src].transpose(-1, -2).contiguous()
+ cv.index_copy_(0, fla.track_dst, conv_win.to(cv.dtype))
+
+ @nvtx_annotate("KDA")
+ def forward(self, hidden_states: torch.Tensor) -> torch.Tensor:
+ ctx = get_global_ctx()
+ batch = ctx.batch
+ pool = ctx.linear_state_pool
+ total = hidden_states.shape[0]
+ dtype = hidden_states.dtype
+ h, d, p = self.num_heads, self.head_dim, self.proj_size
+
+ fla = batch.fla_metadata
+ if fla is None:
+ from freetoken.attention.linear import build_fla_metadata
+
+ fla = build_fla_metadata(batch, hidden_states.device)
+ batch.fla_metadata = fla
+
+ if self._fp8:
+ conv_in = self.in_proj_qkv.forward(hidden_states)
+ b, f_a, g_a = torch.split(
+ self.in_proj_bfg.forward(hidden_states), self._bfg_split, dim=-1
+ )
+ else:
+ proj = self.in_proj.forward(hidden_states)
+ conv_in, b, f_a, g_a = torch.split(
+ proj, [self.conv_dim, h, d, d], dim=-1
+ )
+ g1 = self.f_b_proj.forward(f_a) # raw forget-gate logits [T, H*D]
+ g2 = self.g_b_proj.forward(g_a) # output-gate logits [T, H*D]
+ li = pool.local_index(self.layer_id)
+
+ if batch.is_decode:
+ mixed = causal_conv1d_decode(
+ conv_in, pool.conv_states[li], self._conv_weight(), fla.cache_indices
+ )
+ bsz = mixed.shape[0]
+ q, k, v = (
+ t.reshape(1, bsz, h, d).to(dtype)
+ for t in torch.split(mixed, [p, p, p], dim=-1)
+ )
+ core_out, _ = _fused_recurrent(
+ q, k, v,
+ g=g1.view(1, bsz, h, d),
+ beta=b.view(1, bsz, h),
+ state_pool=pool.recurrent_states[li],
+ indices=fla.cache_indices,
+ cu_seqlens=fla.cu_seqlens,
+ a_log=self.A_log,
+ dt_bias=self.dt_bias,
+ lower_bound=self.lower_bound,
+ scale=self.scale,
+ )
+ else:
+ x = conv_in.transpose(0, 1).contiguous() # [conv_dim, total]
+ mixed = causal_conv1d_varlen(
+ x, self._conv_weight(), pool.conv_states[li],
+ fla.cu_seqlens, fla.cache_indices, fla.has_initial_state,
+ ).transpose(0, 1)
+ q, k, v = (
+ t.reshape(1, total, h, d).to(dtype)
+ for t in torch.split(mixed, [p, p, p], dim=-1)
+ )
+ # Fresh sequences start from a zeroed slot; then gather every request's
+ # initial state (the chunk kernel takes it dense, [N, H, D, D]).
+ rec = pool.recurrent_states[li]
+ if fla.fresh_state_indices is not None:
+ rec.index_fill_(0, fla.fresh_state_indices, 0.0)
+ slot_ids = fla.cache_indices.long()
+ initial = rec.index_select(0, slot_ids)
+
+ from freetoken.kernel.fla import chunk_kda_with_fused_gate
+
+ track = fla.track_dst is not None
+ result = chunk_kda_with_fused_gate(
+ q=q, k=k, v=v, # NOTE: v (ephemeral conv output) is clobbered
+ raw_g=g1.view(1, total, h, d),
+ beta=b.float().sigmoid().view(1, total, h),
+ A_log=self.A_log,
+ g_bias=self.dt_bias,
+ scale=self.scale,
+ initial_state=initial,
+ output_final_state=True,
+ use_qk_l2norm_in_kernel=True,
+ cu_seqlens=fla.cu_seqlens,
+ safe_gate=True,
+ lower_bound=self.lower_bound,
+ return_h=track,
+ )
+ if track:
+ core_out, final_state, chunk_h = result
+ self._write_track_snapshot(pool, li, conv_in, chunk_h, fla)
+ else:
+ core_out, final_state = result
+ rec.index_copy_(0, slot_ids, final_state.to(rec.dtype))
+
+ core_out = core_out.reshape(-1, d)
+ out = self.o_norm.forward(core_out, g2.reshape(-1, d)).reshape(total, -1)
+ return self.o_proj.forward(out.to(dtype))
+
+
+def _fused_recurrent(
+ q, k, v, g, beta, state_pool, indices, cu_seqlens,
+ a_log, dt_bias, lower_bound, scale,
+):
+ """Decode via the vendored recurrent kernel: gate + beta-sigmoid + q/k l2norm
+ in-kernel, state read/written in place at ``indices`` (int32, 1 token/req)."""
+ from freetoken.kernel.fla import fused_recurrent_kda
+
+ return fused_recurrent_kda(
+ q=q, k=k, v=v, g=g, beta=beta,
+ scale=scale,
+ initial_state=state_pool,
+ use_qk_l2norm_in_kernel=True,
+ cu_seqlens=cu_seqlens,
+ ssm_state_indices=indices,
+ sigmoid_beta=True,
+ a_log=a_log,
+ g_bias=dt_bias,
+ compute_gate=True,
+ lower_bound=lower_bound,
+ )
+
+
+__all__ = ["Glm5NextKDA"]
diff --git a/python/freetoken/models/glm5_next/mlp.py b/python/freetoken/models/glm5_next/mlp.py
new file mode 100644
index 00000000..1ee508bf
--- /dev/null
+++ b/python/freetoken/models/glm5_next/mlp.py
@@ -0,0 +1,47 @@
+"""Clamped-SwiGLU MLP for GLM-5.3-Flash's leading dense layers and shared experts.
+
+Same shape as glm_moe_dsa's GlmDsaGatedMLP (bf16 in the NVFP4 checkpoint;
+optional W8A16 fp8-at-load via ``ModelConfig.dense_quant``), but the activation
+is the GLM-5.3 clamped SwiGLU (``swiglu_limit``):
+``clamp(gate, max=L) * sigmoid(gate_clamped) * clamp(up, +-L)``.
+"""
+
+from __future__ import annotations
+
+
+import torch
+from freetoken.layers import BaseOP, swiglu_clamp_and_mul
+from freetoken.utils import nvtx_annotate
+
+from .attention import _make_proj
+
+
+class Glm5NextGatedMLP(BaseOP):
+ def __init__(
+ self,
+ hidden_size: int,
+ intermediate_size: int,
+ quant: str = "none",
+ swiglu_limit: float | None = None,
+ ):
+ self.gate_proj = _make_proj(quant, hidden_size, intermediate_size)
+ self.up_proj = _make_proj(quant, hidden_size, intermediate_size)
+ self.down_proj = _make_proj(quant, intermediate_size, hidden_size)
+ self.swiglu_limit = swiglu_limit
+
+ @nvtx_annotate("MLP")
+ def forward(self, x: torch.Tensor) -> torch.Tensor:
+ gate = self.gate_proj.forward(x)
+ up = self.up_proj.forward(x)
+ del x
+ if self.swiglu_limit is None:
+ import torch.nn.functional as F
+
+ return self.down_proj.forward(F.silu(gate) * up)
+ gated = swiglu_clamp_and_mul(
+ torch.cat([gate, up], dim=-1), alpha=1.0, limit=self.swiglu_limit
+ )
+ return self.down_proj.forward(gated)
+
+
+__all__ = ["Glm5NextGatedMLP"]
diff --git a/python/freetoken/models/glm5_next/model.py b/python/freetoken/models/glm5_next/model.py
new file mode 100644
index 00000000..4c011dd6
--- /dev/null
+++ b/python/freetoken/models/glm5_next/model.py
@@ -0,0 +1,207 @@
+"""GLM-5.3-Flash (glm5_next) model: hybrid KDA/DSA decoder with mHC residual streams.
+
+Layer layout comes from the checkpoint's ``layer_types`` (34 KDA linear-attention
+layers, 11 NoPE-MLA/DSA layers at 3:1) and ``mlp_layer_types`` (3 dense + 42 MoE).
+The residual stream is mHC-widened to ``hc_mult`` (4) parallel streams:
+
+ layer 0: residual = hc_expand(x); (post, comb, x) = mhc_pre(residual, hc_attn_*)
+ each sublayer boundary fuses the previous hc_post with the next hc_pre
+ (mhc_fused_post_pre), and the sublayer input is RMS-normed AFTER the mix
+ (the reference fuses the norm into its hc kernels; decomposed here, same math).
+ last layer: x = mhc_post(...); x = hc_contract(x) -> final norm -> lm_head.
+
+The deferred (post, comb) pair threads through the layer loop exactly like
+glm_moe_dsa's (x, residual) pair. lm_head quant mirrors glm_moe_dsa (optional
+W8A16 fp8 at load; the ~1.2 GiB bf16 head is read every decode step).
+"""
+
+from __future__ import annotations
+
+from typing import TYPE_CHECKING, Tuple
+
+import torch
+from freetoken.core import get_global_ctx
+from freetoken.layers import (
+ BaseOP,
+ OPList,
+ ParallelLMHead,
+ RMSNorm,
+ VocabParallelEmbedding,
+)
+from freetoken.layers.mhc import hc_contract, hc_expand, mhc_fused_post_pre, mhc_post, mhc_pre
+from freetoken.models.blocks import BaseLLMModel
+from freetoken.utils import nvtx_annotate
+
+from .attention import Glm5NextAttention
+from .kda import Glm5NextKDA
+from .mlp import Glm5NextGatedMLP
+from .moe import Glm5NextSparseBlock
+
+if TYPE_CHECKING:
+ from freetoken.models.config import ModelConfig
+
+
+class Glm5Fp8LMHead(ParallelLMHead):
+ """W8A16 lm_head (fp8-e4m3 weight + per-row scale, quantized at load); the
+ full-vocab decode GEMV reads the whole head every step -- fp8 halves it.
+ Same contract as glm_moe_dsa's GlmFp8LMHead."""
+
+ def __init__(self, num_embeddings: int, embedding_dim: int):
+ super().__init__(num_embeddings, embedding_dim, tie_word_embeddings=False)
+ self.weight = torch.empty(num_embeddings, embedding_dim, dtype=torch.float8_e4m3fn)
+ self.weight_scale = torch.empty(num_embeddings, dtype=torch.float32)
+
+ def forward(self, x: torch.Tensor) -> torch.Tensor:
+ from freetoken.kernel.triton.fp8_pertensor_linear import fp8_pertensor_linear
+
+ batch = get_global_ctx().batch
+ if batch.is_prefill:
+ indices = batch.attn_metadata.get_last_indices(batch.size)
+ x = x[indices].contiguous()
+ return fp8_pertensor_linear(x, self.weight, self.weight_scale)
+
+
+class Glm5NextDecoderLayer(BaseOP):
+ def __init__(self, config: ModelConfig, layer_id: int):
+ args = config.glm5_args
+ self._layer_id = layer_id
+ self._is_last = layer_id == config.num_layers - 1
+ self.mhc = args.mhc
+ self._n = args.mhc_num_residual_streams
+ self._hc_eps = args.hc_eps
+ self._rms_eps = args.norm_eps
+ self._post_mult = args.mhc_post_mult_value
+ self._sinkhorn = args.mhc_sinkhorn_iterations
+
+ if args.is_kda_layer(layer_id):
+ self.self_attn: BaseOP = Glm5NextKDA(config, layer_id)
+ else:
+ self.self_attn = Glm5NextAttention(config, layer_id)
+ if layer_id >= config.first_k_dense_replace:
+ self.mlp: BaseOP = Glm5NextSparseBlock(config, layer_id)
+ else:
+ self.mlp = Glm5NextGatedMLP(
+ config.hidden_size, config.intermediate_size,
+ quant=config.dense_quant, swiglu_limit=config.swiglu_limit,
+ )
+ self.input_layernorm = RMSNorm(size=config.hidden_size, eps=args.norm_eps)
+ self.post_attention_layernorm = RMSNorm(size=config.hidden_size, eps=args.norm_eps)
+
+ if self.mhc:
+ n, hidden = self._n, config.hidden_size
+ mix = 2 * n + n * n
+ # fp32 mHC weights (models/weight.py exempts hc_* from the dtype downcast).
+ self.hc_attn_fn = torch.empty(mix, n * hidden, dtype=torch.float32)
+ self.hc_attn_base = torch.empty(mix, dtype=torch.float32)
+ self.hc_attn_scale = torch.empty(3, dtype=torch.float32)
+ self.hc_ffn_fn = torch.empty(mix, n * hidden, dtype=torch.float32)
+ self.hc_ffn_base = torch.empty(mix, dtype=torch.float32)
+ self.hc_ffn_scale = torch.empty(3, dtype=torch.float32)
+
+ def _pre(self, residual, fn, scale, base):
+ # Layer 0's standalone pre rides the fused kernel too (HAS_POST=False
+ # path; x/post/comb are the no-post sentinels) -- same dispatch, same
+ # numerics, and the kernel wins at every batch size (see layers/mhc.py).
+ if residual.is_cuda:
+ _, post, comb, x = mhc_fused_post_pre(
+ residual.new_empty(residual.shape[0], residual.shape[-1]),
+ residual, None, None, fn, scale, base,
+ self._rms_eps, self._hc_eps, self._post_mult, self._sinkhorn,
+ )
+ return post, comb, x
+ return mhc_pre(
+ residual, fn, scale, base,
+ self._rms_eps, self._hc_eps, self._post_mult, self._sinkhorn,
+ )
+
+ def _fused(self, x, residual, post, comb, fn, scale, base):
+ return mhc_fused_post_pre(
+ x, residual, post, comb, fn, scale, base,
+ self._rms_eps, self._hc_eps, self._post_mult, self._sinkhorn,
+ )
+
+ @nvtx_annotate("Layer_{}", layer_id_field="_layer_id")
+ def forward(
+ self,
+ x: torch.Tensor,
+ residual: torch.Tensor | None,
+ post: torch.Tensor | None,
+ comb: torch.Tensor | None,
+ ) -> Tuple[torch.Tensor, torch.Tensor | None, torch.Tensor | None, torch.Tensor | None]:
+ if post is None:
+ if residual is None:
+ residual = hc_expand(x, self._n)
+ post, comb, x = self._pre(
+ residual, self.hc_attn_fn, self.hc_attn_scale, self.hc_attn_base
+ )
+ else:
+ residual, post, comb, x = self._fused(
+ x, residual, post, comb,
+ self.hc_attn_fn, self.hc_attn_scale, self.hc_attn_base,
+ )
+ x = self.input_layernorm.forward(x)
+ x = self.self_attn.forward(x)
+
+ residual, post, comb, x = self._fused(
+ x, residual, post, comb,
+ self.hc_ffn_fn, self.hc_ffn_scale, self.hc_ffn_base,
+ )
+ x = self.post_attention_layernorm.forward(x)
+ x = self.mlp.forward(x)
+
+ if self._is_last:
+ x = mhc_post(x, residual, post, comb)
+ return hc_contract(x), None, None, None
+ return x, residual, post, comb
+
+
+class Glm5NextModel(BaseOP):
+ def __init__(self, config: ModelConfig):
+ self.embed_tokens = VocabParallelEmbedding(
+ num_embeddings=config.vocab_size,
+ embedding_dim=config.hidden_size,
+ )
+ self.layers = OPList(
+ [Glm5NextDecoderLayer(config, i) for i in range(config.num_layers)]
+ )
+ self.norm = RMSNorm(size=config.hidden_size, eps=config.rms_norm_eps)
+
+ def forward(self, input_ids: torch.Tensor) -> torch.Tensor:
+ x = self.embed_tokens.forward(input_ids)
+ residual = post = comb = None
+ for layer in self.layers.op_list:
+ x, residual, post, comb = layer.forward(x, residual, post, comb)
+ return self.norm.forward(x)
+
+
+class Glm5NextForCausalLM(BaseLLMModel):
+ def __init__(self, config: ModelConfig):
+ self._config = config
+ self.model = Glm5NextModel(config)
+ if config.lm_head_quant == "fp8_pertensor" and not config.tie_word_embeddings:
+ self.lm_head: BaseOP = Glm5Fp8LMHead(
+ num_embeddings=config.vocab_size, embedding_dim=config.hidden_size
+ )
+ else:
+ self.lm_head = ParallelLMHead(
+ num_embeddings=config.vocab_size,
+ embedding_dim=config.hidden_size,
+ tie_word_embeddings=config.tie_word_embeddings,
+ tied_embedding=self.model.embed_tokens if config.tie_word_embeddings else None,
+ )
+
+ def prepare_for_runtime(self) -> None:
+ """Post-load, pre-KV-sizing hook: materialize the DSA layers' bmm-ready
+ kv_b splits and free the checkpoint-layout originals (glm_moe_dsa
+ precedent)."""
+ for layer in self.model.layers.op_list:
+ if isinstance(layer.self_attn, Glm5NextAttention):
+ layer.self_attn.prepare_for_runtime()
+ torch.cuda.empty_cache()
+
+ def forward(self) -> torch.Tensor:
+ output = self.model.forward(get_global_ctx().batch.input_ids)
+ return self.lm_head.forward(output)
+
+
+__all__ = ["Glm5NextForCausalLM"]
diff --git a/python/freetoken/models/glm5_next/moe.py b/python/freetoken/models/glm5_next/moe.py
new file mode 100644
index 00000000..038ffc0d
--- /dev/null
+++ b/python/freetoken/models/glm5_next/moe.py
@@ -0,0 +1,92 @@
+"""GLM-5.3-Flash sparse MoE block.
+
+Routing is identical to glm_moe_dsa (sigmoid scores + selection-only
+``e_score_correction_bias``, optional group-limited top-k, gather unbiased
+scores, renormalize, scale by ``routed_scaling_factor``); the deltas are the
+expert count (288, top-8) and the clamped-SwiGLU activation (``swiglu_limit`` =
+10), which rides ``make_moe_layer``'s ``extra_attrs`` into the offload kernels
+(triton NVFP4 in-GPU, generic-epilogue CPU GEMV). The marlin/b12x borrowed
+kernels hard-code silu; the backend selector already falls back to triton for
+non-silu experts.
+"""
+
+from __future__ import annotations
+
+from typing import TYPE_CHECKING, Tuple
+
+import torch
+import torch.nn.functional as F
+from freetoken.layers import BaseOP, LinearReplicated, make_moe_layer
+
+from .mlp import Glm5NextGatedMLP
+
+if TYPE_CHECKING:
+ from freetoken.models.config import ModelConfig
+
+TopK = Tuple[torch.Tensor, torch.Tensor]
+
+
+class Glm5NextSparseBlock(BaseOP):
+ def __init__(self, config: ModelConfig, layer_id: int):
+ self.top_k = config.num_experts_per_tok
+ self.num_experts = config.num_experts
+ self.norm_topk_prob = config.norm_topk_prob
+ self.routed_scaling_factor = config.routed_scaling_factor
+ self.n_group = config.n_group
+ self.topk_group = config.topk_group
+
+ self.gate = LinearReplicated(config.hidden_size, config.num_experts, has_bias=False)
+ self.e_score_correction_bias = torch.empty(config.num_experts, dtype=torch.float32)
+
+ # The offload cache indexes experts by MoE layer (global minus dense prefix).
+ self.experts = make_moe_layer(
+ config,
+ layer_id=layer_id - config.first_k_dense_replace,
+ activation="swiglu_clamp" if config.swiglu_limit is not None else "silu",
+ renormalize=config.norm_topk_prob,
+ extra_attrs={
+ "swiglu_limit": config.swiglu_limit,
+ "hidden_act_alpha": 1.0, # plain sigmoid inside the clamped swiglu
+ },
+ )
+ self.shared_experts = Glm5NextGatedMLP(
+ config.hidden_size,
+ config.moe_intermediate_size * max(1, config.n_shared_experts),
+ quant=config.dense_quant,
+ swiglu_limit=config.swiglu_limit,
+ )
+
+ def _group_limited(self, scores_for_choice: torch.Tensor) -> torch.Tensor:
+ m = scores_for_choice.shape[0]
+ e, g = self.num_experts, self.n_group
+ group_scores = scores_for_choice.view(m, g, e // g).topk(2, dim=-1)[0].sum(dim=-1)
+ group_idx = torch.topk(group_scores, self.topk_group, dim=-1, sorted=False)[1]
+ group_mask = torch.zeros_like(group_scores)
+ group_mask.scatter_(1, group_idx, 1.0)
+ score_mask = group_mask.unsqueeze(-1).expand(m, g, e // g).reshape(m, e)
+ return scores_for_choice.masked_fill(~score_mask.bool(), float("-inf"))
+
+ def _route(self, hidden_states: torch.Tensor) -> TopK:
+ # HF computes router logits in fp32 (moe_router_dtype: float32); match exactly.
+ logits = F.linear(hidden_states.float(), self.gate.weight.float())
+ scores = logits.sigmoid()
+ scores_for_choice = scores + self.e_score_correction_bias.float()
+ if self.n_group > 1:
+ scores_for_choice = self._group_limited(scores_for_choice)
+ _, topk_ids = torch.topk(scores_for_choice, self.top_k, dim=-1)
+ topk_weights = scores.gather(-1, topk_ids)
+ if self.norm_topk_prob:
+ topk_weights = topk_weights / (topk_weights.sum(dim=-1, keepdim=True) + 1e-20)
+ topk_weights = topk_weights * self.routed_scaling_factor
+ return topk_weights.to(torch.float32).contiguous(), topk_ids.to(torch.int32).contiguous()
+
+ def forward(self, hidden_states: torch.Tensor) -> torch.Tensor:
+ num_tokens, hidden_dim = hidden_states.shape
+ hidden_states = hidden_states.view(-1, hidden_dim)
+ topk_weights, topk_ids = self._route(hidden_states)
+ out = self.experts.routed_forward(hidden_states, topk_weights, topk_ids)
+ out = out + self.shared_experts.forward(hidden_states)
+ return out.view(num_tokens, hidden_dim)
+
+
+__all__ = ["Glm5NextSparseBlock"]
diff --git a/python/freetoken/models/glm5_next/weight.py b/python/freetoken/models/glm5_next/weight.py
new file mode 100644
index 00000000..fda85225
--- /dev/null
+++ b/python/freetoken/models/glm5_next/weight.py
@@ -0,0 +1,275 @@
+"""Weight loading for GLM-5.3-Flash (``glm5_next``).
+
+Supported checkpoints: NVFP4 exports of GLM-5.3-Flash in the multimodal-wrapper
+layout (``model.language_model.*``) -- ModelOpt tensor kinds (LibertAIDAI) or
+compressed-tensors kinds (RedHatAI), selected by ``quantization_config``. Not
+supported: bf16-expert originals (zai-org), text-only key layouts, TP > 1.
+
+Routed experts go to the offload cache via ``load_nvfp4_expert_sources``;
+everything else loads bf16 with keys renamed ``model.language_model.X`` ->
+``model.X``. ``model.visual.*`` and the trailing MTP layer are never read.
+
+Load-time fusions (must mirror the module split orders):
+
+* KDA ``in_proj`` = q|k|v|b|f_a|g_a projections concatenated on the output axis
+* KDA ``conv1d`` = q|k|v depthwise conv weights concatenated on the channel axis
+
+fp32-kept tensors: ``A_log`` / ``dt_bias``, the mHC ``hc_*`` tensors, the indexer
+APE, and the router ``e_score_correction_bias``. Optional W8A16 fp8-at-load
+follows ``ModelConfig.attn_quant`` / ``dense_quant`` / ``lm_head_quant``
+(defaults and env opt-ins: see config.py).
+"""
+
+from __future__ import annotations
+
+import json
+import os
+import re
+from typing import Iterator
+
+import torch
+from freetoken.distributed import get_tp_info
+from freetoken.models.glm_moe_dsa.weight import _ShardReader, _quant_fp8_per_row
+from freetoken.models.loader import drop_page_cache
+from freetoken.models.nvfp4_banks import (
+ Nvfp4ExpertSourceSpec,
+ load_nvfp4_expert_source_banks,
+)
+from freetoken.utils import cached_load_hf_config, download_hf_weight
+from tqdm import tqdm
+
+from .args import Glm5NextArgs
+from .config import parse_config
+
+# Checkpoint prefix (multimodal wrapper) -> model prefix.
+_CKPT = "model.language_model"
+_MODEL = "model"
+
+# MTP-layer experts (layer == num_layers under the full checkpoint) map to None
+# alongside the dense prefix; the bank loader skips them.
+def _layer_to_bank(layer, config):
+ return (
+ None
+ if layer < config.first_k_dense_replace or layer >= config.num_layers
+ else layer - config.first_k_dense_replace
+ )
+
+
+# ModelOpt export (LibertAIDAI/GLM-5.3-Flash-NVFP4): weight | weight_scale |
+# weight_scale_2 (dequant-side global).
+_NVFP4_SOURCE_SPEC = Nvfp4ExpertSourceSpec(
+ key_pattern=re.compile(
+ r"^model\.language_model\.layers\.(?P<layer>\d+)\.mlp\.experts\.(?P<expert>\d+)\."
+ r"(?P<proj>gate_proj|up_proj|down_proj)\.(?P<kind>weight|weight_scale|weight_scale_2)$"
+ ),
+ proj_to_role={"gate_proj": "gate", "up_proj": "up", "down_proj": "down"},
+ layer_to_bank=_layer_to_bank,
+ desc="GLM-5.3 NVFP4 experts",
+)
+
+# llm-compressor export (RedHatAI/GLM-5.3-Flash-NVFP4): weight_packed |
+# weight_scale | weight_global_scale (quant-side global -> reciprocal at ingest).
+# ``input_global_scale`` (the calibrated W4A4 activation scale) deliberately does
+# not match: our routed-expert paths are W4A16 and never quantize activations.
+_NVFP4_CT_SOURCE_SPEC = Nvfp4ExpertSourceSpec(
+ key_pattern=re.compile(
+ r"^model\.language_model\.layers\.(?P<layer>\d+)\.mlp\.experts\.(?P<expert>\d+)\."
+ r"(?P<proj>gate_proj|up_proj|down_proj)\."
+ r"(?P<kind>weight_packed|weight_global_scale|weight_scale)$"
+ ),
+ proj_to_role={"gate_proj": "gate", "up_proj": "up", "down_proj": "down"},
+ layer_to_bank=_layer_to_bank,
+ desc="GLM-5.3 NVFP4 experts (compressed-tensors)",
+ kind_map={"weight_packed": "weight", "weight_global_scale": "weight_scale_2"},
+ global_reciprocal=True,
+)
+
+
+def _select_expert_source_spec(model_path: str) -> Nvfp4ExpertSourceSpec:
+ quant = getattr(cached_load_hf_config(model_path), "quantization_config", None) or {}
+ get = quant.get if isinstance(quant, dict) else (lambda k, d=None: getattr(quant, k, d))
+ method = str(get("quant_method") or "").lower()
+ return _NVFP4_CT_SOURCE_SPEC if method == "compressed-tensors" else _NVFP4_SOURCE_SPEC
+
+# KDA in_proj fusion order; MUST match Glm5NextKDA._in_proj_split.
+_KDA_IN_PROJ = ("q_proj", "k_proj", "v_proj", "b_proj", "f_a_proj", "g_a_proj")
+
+
+def load_nvfp4_expert_sources(model_path: str, config, layer_sink=None):
+ return load_nvfp4_expert_source_banks(
+ model_path,
+ config,
+ _select_expert_source_spec(model_path),
+ drop_page_cache=drop_page_cache,
+ primary=get_tp_info().is_primary(),
+ layer_sink=layer_sink,
+ )
+
+
+def _maybe_fp8(key: str, w: torch.Tensor, fp8: bool):
+ if fp8:
+ q, scale = _quant_fp8_per_row(w)
+ yield f"{key}.weight", q
+ yield f"{key}.weight_scale", scale
+ else:
+ yield f"{key}.weight", w.to(torch.bfloat16)
+
+
+def _iter_kda_layer(reader, layer: int, attn_fp8: bool) -> Iterator[tuple[str, torch.Tensor]]:
+ src = f"{_CKPT}.layers.{layer}.self_attn"
+ dst = f"{_MODEL}.layers.{layer}.self_attn"
+ if attn_fp8:
+ # fp8 resident: q|k|v (the 201 MB/layer read) as one W8A16 GEMM with
+ # per-row scales; the small gate projections b|f_a|g_a stay bf16.
+ qkv = torch.cat(
+ [reader.get(f"{src}.{p}.weight").to(torch.bfloat16) for p in ("q_proj", "k_proj", "v_proj")],
+ dim=0,
+ )
+ q, scale = _quant_fp8_per_row(qkv)
+ yield f"{dst}.in_proj_qkv.weight", q
+ yield f"{dst}.in_proj_qkv.weight_scale", scale
+ bfg = torch.cat(
+ [reader.get(f"{src}.{p}.weight").to(torch.bfloat16) for p in ("b_proj", "f_a_proj", "g_a_proj")],
+ dim=0,
+ )
+ yield f"{dst}.in_proj_bfg.weight", bfg
+ else:
+ # One fused input GEMM: q|k|v|b|f_a|g_a (output-axis concat).
+ fused = torch.cat(
+ [reader.get(f"{src}.{p}.weight").to(torch.bfloat16) for p in _KDA_IN_PROJ], dim=0
+ )
+ yield f"{dst}.in_proj.weight", fused
+ # One merged depthwise conv over the q|k|v stream (channel-axis concat).
+ conv = torch.cat(
+ [reader.get(f"{src}.{p}_conv1d.weight").to(torch.bfloat16) for p in ("q", "k", "v")],
+ dim=0,
+ )
+ yield f"{dst}.conv1d.weight", conv
+ for p in ("f_b_proj", "g_b_proj"):
+ yield f"{dst}.{p}.weight", reader.get(f"{src}.{p}.weight").to(torch.bfloat16)
+ yield from _maybe_fp8(f"{dst}.o_proj", reader.get(f"{src}.o_proj.weight"), attn_fp8)
+ # Gate params stay fp32 (the recurrent kernels read them as fp32).
+ yield f"{dst}.A_log", reader.get(f"{src}.A_log").to(torch.float32)
+ yield f"{dst}.dt_bias", reader.get(f"{src}.dt_bias").to(torch.float32)
+ yield f"{dst}.o_norm.weight", reader.get(f"{src}.o_norm.weight").to(torch.bfloat16)
+
+
+def _iter_dsa_layer(reader, layer: int, attn_fp8: bool) -> Iterator[tuple[str, torch.Tensor]]:
+ src = f"{_CKPT}.layers.{layer}.self_attn"
+ dst = f"{_MODEL}.layers.{layer}.self_attn"
+ fp8_projs = ("q_a_proj", "q_b_proj", "kv_a_proj_with_mqa", "o_proj") if attn_fp8 else ()
+ for proj in ("q_a_proj", "q_b_proj", "kv_a_proj_with_mqa", "kv_b_proj", "o_proj"):
+ w = reader.get(f"{src}.{proj}.weight")
+ yield from _maybe_fp8(f"{dst}.{proj}", w, proj in fp8_projs)
+ for norm in ("q_a_layernorm", "kv_a_layernorm"):
+ yield f"{dst}.{norm}.weight", reader.get(f"{src}.{norm}.weight").to(torch.bfloat16)
+ # kpool indexer (every DSA layer owns one). Kept bf16; the APE is fp32.
+ for proj in ("wq_b", "wk", "weights_proj"):
+ yield f"{dst}.indexer.{proj}.weight", reader.get(
+ f"{src}.indexer.{proj}.weight"
+ ).to(torch.bfloat16)
+ for part, dtype in (
+ ("k_norm.weight", torch.bfloat16),
+ ("k_norm.bias", torch.bfloat16),
+ ("index_kpool_compress_gate", torch.bfloat16),
+ ("index_kpool_compress_ape", torch.float32),
+ ):
+ yield f"{dst}.indexer.{part}", reader.get(f"{src}.indexer.{part}").to(dtype)
+
+
+def iter_weights(
+ model_path: str,
+ device: torch.device,
+ *,
+ include_moe_experts: bool,
+ include_non_moe: bool,
+) -> Iterator[tuple[str, torch.Tensor]]:
+ assert not include_moe_experts, (
+ "GLM-5.3 stores routed experts as NVFP4 and only supports the offload backend; "
+ "experts are loaded into the offload cache via load_nvfp4_expert_sources()."
+ )
+ assert include_non_moe
+ if get_tp_info().size > 1:
+ # The loader emits full fused KDA/DSA tensors; TP sharding (per-head q|k|v|b
+ # splits, replicated f_a|g_a, row-parallel o_proj) is not implemented yet --
+ # same status as every other linear-hybrid / offload-family model in tree.
+ raise NotImplementedError("glm5_next weight loading currently supports TP=1 only")
+ config = parse_config(cached_load_hf_config(model_path))
+ args: Glm5NextArgs = config.glm5_args
+ folder = download_hf_weight(model_path)
+ with open(os.path.join(folder, "model.safetensors.index.json")) as f:
+ weight_map = json.load(f)["weight_map"]
+ reader = _ShardReader(folder, weight_map, device)
+ primary = get_tp_info().is_primary()
+ attn_fp8 = config.attn_quant == "fp8_pertensor"
+ mlp_fp8 = config.dense_quant == "fp8_pertensor"
+ head_fp8 = config.lm_head_quant == "fp8_pertensor"
+ if primary:
+ from freetoken.utils import init_logger
+
+ init_logger(__name__).info(
+ f"GLM-5.3 resident quant: attn={config.attn_quant} dense={config.dense_quant} "
+ f"lm_head={config.lm_head_quant} (FREETOKEN_GLM5_ATTN_FP8/FREETOKEN_GLM5_MLP_FP8; "
+ "an FTW conversion records these choices implicitly -- serve with the same flags)"
+ )
+ try:
+ for layer in tqdm(
+ range(config.num_layers),
+ desc="Loading GLM-5.3 dense weights",
+ disable=not primary,
+ ):
+ src = f"{_CKPT}.layers.{layer}"
+ dst = f"{_MODEL}.layers.{layer}"
+ if args.is_kda_layer(layer):
+ yield from _iter_kda_layer(reader, layer, attn_fp8)
+ else:
+ yield from _iter_dsa_layer(reader, layer, attn_fp8)
+
+ # mHC mixing tensors, fp32 on every layer.
+ for hc in ("hc_attn_fn", "hc_attn_base", "hc_attn_scale",
+ "hc_ffn_fn", "hc_ffn_base", "hc_ffn_scale"):
+ yield f"{dst}.{hc}", reader.get(f"{src}.{hc}").to(torch.float32)
+
+ for norm in ("input_layernorm", "post_attention_layernorm"):
+ yield f"{dst}.{norm}.weight", reader.get(f"{src}.{norm}.weight").to(
+ torch.bfloat16
+ )
+
+ if layer < config.first_k_dense_replace:
+ for proj in ("gate_proj", "up_proj", "down_proj"):
+ yield from _maybe_fp8(
+ f"{dst}.mlp.{proj}", reader.get(f"{src}.mlp.{proj}.weight"), mlp_fp8
+ )
+ else:
+ yield f"{dst}.mlp.gate.weight", reader.get(f"{src}.mlp.gate.weight").to(
+ torch.bfloat16
+ )
+ yield (
+ f"{dst}.mlp.e_score_correction_bias",
+ # fp32 like HF's router math (the module declares fp32; a bf16
+ # cast would perturb top-8 selection on fp32-bias checkpoints).
+ reader.get(f"{src}.mlp.gate.e_score_correction_bias").to(torch.float32),
+ )
+ for proj in ("gate_proj", "up_proj", "down_proj"):
+ yield from _maybe_fp8(
+ f"{dst}.mlp.shared_experts.{proj}",
+ reader.get(f"{src}.mlp.shared_experts.{proj}.weight"),
+ mlp_fp8,
+ )
+
+ yield f"{_MODEL}.embed_tokens.weight", reader.get(
+ f"{_CKPT}.embed_tokens.weight"
+ ).to(torch.bfloat16)
+ yield f"{_MODEL}.norm.weight", reader.get(f"{_CKPT}.norm.weight").to(torch.bfloat16)
+ head = reader.get("lm_head.weight")
+ if head_fp8 and not config.tie_word_embeddings:
+ q, scale = _quant_fp8_per_row(head)
+ yield "lm_head.weight", q
+ yield "lm_head.weight_scale", scale
+ else:
+ yield "lm_head.weight", head.to(torch.bfloat16)
+ finally:
+ reader.close()
+
+
+__all__ = ["iter_weights", "load_nvfp4_expert_sources"]
diff --git a/python/freetoken/models/glm_moe_dsa/attention.py b/python/freetoken/models/glm_moe_dsa/attention.py
index 14b1c100..e25d961f 100644
--- a/python/freetoken/models/glm_moe_dsa/attention.py
+++ b/python/freetoken/models/glm_moe_dsa/attention.py
@@ -93,14 +93,16 @@ class GlmDsaIndexer(BaseOP):
def compute(
self, x: torch.Tensor, q_resid: torch.Tensor, positions: torch.Tensor
- ) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor]:
- """Per-token indexer projections: (q [T, H, D], k [T, D], weights [T, H] fp32)."""
+ ) -> "DSAIndexerInputs":
+ """Per-token indexer projections: q [T, H, D], k [T, D], weights [T, H] fp32."""
+ from freetoken.attention.dsa import DSAIndexerInputs
+
t = x.shape[0]
q = self.wq_b.forward(q_resid).view(t, self.n_heads * self.head_dim)
k = self.k_norm.forward(self.wk.forward(x)).view(t, self.head_dim)
q, k = self._rope.forward(positions, q, k)
w = self.weights_proj.forward(x).float() * (self.n_heads**-0.5)
- return q.view(t, self.n_heads, self.head_dim), k, w
+ return DSAIndexerInputs(q=q.view(t, self.n_heads, self.head_dim), k=k, w=w)
class GlmMoeDsaAttention(BaseOP):
@@ -213,7 +215,7 @@ class GlmMoeDsaAttention(BaseOP):
# DSA: full layers hand the backend this token's indexer projections (the
# backend caches the keys, scores the history, and selects top-k); shared
# layers pass None and reuse their group leader's selection.
- indexer_qkw = (
+ indexer_inputs = (
self.indexer.compute(x, q_a_resid, ctx.batch.positions)
if self.indexer is not None and getattr(ctx.attn_backend, "dsa_enabled", False)
else None
@@ -223,7 +225,7 @@ class GlmMoeDsaAttention(BaseOP):
# concatenated latent copy on the hot path.
o_latent = ctx.attn_backend.mla_forward(
q_absorbed.contiguous(), q_rope.contiguous(), c_kv.contiguous(),
- k_rope.contiguous(), self.layer_id, ctx.batch, indexer_qkw=indexer_qkw,
+ k_rope.contiguous(), self.layer_id, ctx.batch, indexer_inputs=indexer_inputs,
) # [T, H, kv_lora_rank]
# Absorb kv_b's v-part onto the output: o_latent[H,T,lora] @ W_uv_t[H,lora,v].
diff --git a/python/freetoken/models/nvfp4_banks.py b/python/freetoken/models/nvfp4_banks.py
index 0e3ab6a5..6b933ff1 100644
--- a/python/freetoken/models/nvfp4_banks.py
+++ b/python/freetoken/models/nvfp4_banks.py
@@ -22,6 +22,22 @@ class Nvfp4ExpertSourceSpec:
proj_to_role: dict[str, str]
layer_to_bank: LayerToBank
desc: str
+ # Maps checkpoint tensor-kind names onto the canonical (modelopt) kinds, e.g.
+ # compressed-tensors' weight_packed -> weight, weight_global_scale -> weight_scale_2.
+ kind_map: dict[str, str] | None = None
+ # The checkpoint stores the QUANT-side global scale (local fp8 scales were
+ # multiplied by it before the cast); the banks keep its reciprocal.
+ global_reciprocal: bool = False
+
+
+def _canon_kind(spec: "Nvfp4ExpertSourceSpec", kind: str) -> str:
+ return spec.kind_map.get(kind, kind) if spec.kind_map else kind
+
+
+def _ingest_global(spec: "Nvfp4ExpertSourceSpec", tensor: torch.Tensor) -> torch.Tensor:
+ if spec.global_reciprocal:
+ tensor = 1.0 / tensor.float()
+ return tensor.to(torch.float16)
def _num_moe_layers(config) -> int:
@@ -112,7 +128,7 @@ def load_nvfp4_expert_source_banks(
proj = match.group("proj")
if proj not in spec.proj_to_role:
raise ValueError(f"{spec.desc}: unknown NVFP4 expert projection {proj!r}")
- kind = match.group("kind")
+ kind = _canon_kind(spec, match.group("kind"))
if kind == "weight_scale_2":
global_shards[shard].append((name, match, bank_layer))
elif kind in {"weight", "weight_scale"}:
@@ -130,7 +146,7 @@ def load_nvfp4_expert_source_banks(
int(match.group("expert")),
match.group("proj"),
)
- globals_map[key] = f.get_tensor(name).to(torch.float16)
+ globals_map[key] = _ingest_global(spec, f.get_tensor(name))
drop_page_cache(path)
_hb = _alloc_nvfp4_host_banks(num_layers, E, H, I) # unpinned; pinned after fill
@@ -154,7 +170,7 @@ def load_nvfp4_expert_source_banks(
expert = int(match.group("expert"))
proj = match.group("proj")
role = spec.proj_to_role[proj]
- kind = match.group("kind")
+ kind = _canon_kind(spec, match.group("kind"))
tensor = f.get_tensor(name)
if kind == "weight":
if role == "gate":
@@ -236,7 +252,7 @@ def load_nvfp4_expert_source_banks_parallel(
bank_layer = _bank_layer(spec, int(match.group("layer")), config)
if bank_layer is None:
continue
- kind = match.group("kind")
+ kind = _canon_kind(spec, match.group("kind"))
if kind == "weight_scale_2":
global_names_by_shard[shard].append(name)
elif kind in {"weight", "weight_scale"}:
@@ -253,7 +269,7 @@ def load_nvfp4_expert_source_banks_parallel(
for name in global_names_by_shard[shard]:
m = spec.key_pattern.match(name)
globals_map[(int(m.group("layer")), int(m.group("expert")), m.group("proj"))] = (
- f.get_tensor(name).to(torch.float16)
+ _ingest_global(spec, f.get_tensor(name))
)
drop_page_cache(path)
@@ -279,7 +295,7 @@ def load_nvfp4_expert_source_banks_parallel(
expert = int(match.group("expert"))
proj = match.group("proj")
role = spec.proj_to_role[proj]
- kind = match.group("kind")
+ kind = _canon_kind(spec, match.group("kind"))
if kind == "weight":
if role == "gate":
gate_up_packed[bank_layer_id][expert, :I] = tensor
diff --git a/python/freetoken/models/register.py b/python/freetoken/models/register.py
index b94d8291..b8c17322 100644
--- a/python/freetoken/models/register.py
+++ b/python/freetoken/models/register.py
@@ -130,6 +130,19 @@ _MODEL_REGISTRY: dict[str, ModelSpec] = {
"freetoken.models.glm_moe_dsa",
"GlmMoeDsaForCausalLM",
),
+ # GLM-5.3-Flash (model_type glm5_next): hybrid KDA linear attention (34/45 layers)
+ # + NoPE-MLA/DSA with a kpool-compressed indexer (11/45), mHC x4 residual streams,
+ # 288-expert sigmoid/noaux_tc MoE; natively-multimodal wrapper config (text tower
+ # in text_config, weights under model.language_model.), served text-only.
+ "Glm5NextForConditionalGeneration": ModelSpec(
+ "freetoken.models.glm5_next",
+ "Glm5NextForCausalLM",
+ ),
+ # Text-only sibling (the text_config's own architectures entry).
+ "Glm5NextForCausalLM": ModelSpec(
+ "freetoken.models.glm5_next",
+ "Glm5NextForCausalLM",
+ ),
}
diff --git a/python/freetoken/moe/benchbw.py b/python/freetoken/moe/benchbw.py
index f3e5359a..83b3d196 100644
--- a/python/freetoken/moe/benchbw.py
+++ b/python/freetoken/moe/benchbw.py
@@ -122,6 +122,10 @@ WORKLOADS: dict[str, Workload] = {
activation="gpt_oss_swiglu", swiglu_limit=7.0),
"dsv4": Workload("dsv4", 4096, 2048, 256, 6, ("ds_fp4",), swiglu_limit=7.0),
"glm4.7-nvfp4": Workload("glm4.7-nvfp4", 5120, 1536, 160, 8, ("nvfp4",)),
+ "glm5.3-flash-nvfp4": Workload(
+ "glm5.3-flash-nvfp4", 4096, 2048, 288, 8, ("nvfp4",),
+ activation="swiglu_clamp", swiglu_alpha=1.0, swiglu_limit=10.0,
+ ),
"minimax-m2.5": Workload("minimax-m2.5", 3072, 1536, 256, 8, ("nvfp4",)),
}
diff --git a/python/freetoken/moe/cpu_executor.py b/python/freetoken/moe/cpu_executor.py
index b96205aa..d9cb43c1 100644
--- a/python/freetoken/moe/cpu_executor.py
+++ b/python/freetoken/moe/cpu_executor.py
@@ -65,6 +65,7 @@ _ACT_IDS = {
"gelu_pytorch_tanh": 2,
"gpt_oss_swiglu": 3,
"swigluoai": 3,
+ "swiglu_clamp": 4,
}
# Weight-format ids must match WFmt in csrc/cpu_moe/cpu_moe_ext.cpp.
diff --git a/python/freetoken/moe/fused_nvfp4.py b/python/freetoken/moe/fused_nvfp4.py
index 29a561e6..f736b80c 100644
--- a/python/freetoken/moe/fused_nvfp4.py
+++ b/python/freetoken/moe/fused_nvfp4.py
@@ -25,6 +25,7 @@ from freetoken.layers import (
gelu_and_mul,
gelu_tanh_and_mul,
silu_and_mul,
+ swiglu_clamp_and_mul,
swigluoai_and_mul,
)
from freetoken.moe.fused import moe_align_block_size
@@ -40,12 +41,16 @@ def _run_act(
act_limit: float,
) -> None:
"""gemm1 -> gemm2 activation dispatch. ``swigluoai`` (MiniMax-M3, clamped
- gpt-oss swiglu over the banks' uninterleaved [gate; up] halves) carries the
+ gpt-oss swiglu over the banks' uninterleaved [gate; up] halves) and
+ ``swiglu_clamp`` (GLM-5.3, same clamp without the +1 up bias) carry the
per-model ``act_alpha``/``act_limit`` scalars; the plain *_and_mul kinds
ignore them."""
if activation == "swigluoai":
swigluoai_and_mul(gate_up, out, alpha=act_alpha, limit=act_limit)
return
+ if activation == "swiglu_clamp":
+ swiglu_clamp_and_mul(gate_up, out, alpha=act_alpha, limit=act_limit)
+ return
_ACT[activation](gate_up, out)
# Decode is captured into a CUDA graph, so the config must be fixed (no triton.autotune,
diff --git a/tests/attention/test_dsa_kpool.py b/tests/attention/test_dsa_kpool.py
new file mode 100644
index 00000000..fb856e17
--- /dev/null
+++ b/tests/attention/test_dsa_kpool.py
@@ -0,0 +1,369 @@
+"""Glm5NextDSABackend (kpool indexer) vs an eager reference.
+
+Exercises the full path -- raw K/gate stores, pool-completion compression,
+pool-granular scoring, top-k + expansion + tail, gathered sparse MLA -- on a
+single request at toy dims (Hi=16 index heads, Di=64, kpool=4, topk=32):
+
+* kv_len <= index_topk: the identity/dense path must equal full softmax MLA.
+* kv_len > index_topk: prefill queries must match a subset-softmax reference
+ built from an independently computed pooled-score top-k (+ tail).
+* decode: pooled entries appear exactly at pool-completion steps and match the
+ softmax(gate+APE) reference; decode outputs match the same subset reference.
+"""
+
+from __future__ import annotations
+
+from types import SimpleNamespace
+
+import pytest
+import torch
+
+pytestmark = pytest.mark.skipif(not torch.cuda.is_available(), reason="needs CUDA")
+
+H, LATENT = 2, 64 # MLA heads, kv_lora_rank (== latent width: NoPE has no kpe half)
+HI, DI = 16, 64 # index heads (kernel needs >= 16), index head dim (pow2)
+KPOOL, TOPK = 4, 32
+SM_SCALE = 0.125
+DEV = "cuda"
+
+
+def _args(num_layers=1):
+ from freetoken.models.glm5_next.args import Glm5NextArgs
+
+ return Glm5NextArgs(
+ hidden_size=32, num_heads=H,
+ q_lora_rank=16, kv_lora_rank=LATENT, qk_nope_head_dim=LATENT,
+ qk_rope_head_dim=0, v_head_dim=LATENT, mla_nope=True, norm_eps=1e-5,
+ max_position=4096,
+ index_n_heads=HI, index_head_dim=DI, index_topk=TOPK,
+ indexer_types=("full",) * num_layers, indexer_rope_interleave=True,
+ index_kpool=KPOOL, index_kpool_compress=True,
+ index_kpool_always_select_tail=True,
+ linear_num_heads=0, linear_head_dim=0, linear_conv_kernel_dim=4,
+ linear_lower_bound=-5.0,
+ layer_types=("deepseek_sparse_attention",) * num_layers,
+ mlp_layer_types=("dense",) * num_layers,
+ mhc=False, mhc_num_residual_streams=1, hc_eps=1e-6,
+ mhc_sinkhorn_iterations=0, mhc_tau=0.05, mhc_post_mult_value=2.0,
+ mhc_no_norm_weight=False, swiglu_limit=None, rope_theta=10000.0,
+ )
+
+
+def harness(monkeypatch):
+ from freetoken.attention.dsa_indexer_kpool import Glm5NextDSABackend
+ from freetoken.kvcache.dsa_pool import KpoolDSAKVCache
+
+ pool = KpoolDSAKVCache(
+ latent_dim=LATENT, num_layers=1, num_pages=8, page_size=64,
+ dtype=torch.bfloat16, device=torch.device(DEV),
+ index_head_dim=DI, num_index_layers=1,
+ index_ratio=KPOOL, num_req_slots=4,
+ )
+ page_table = torch.full((4, 512), -1, dtype=torch.int32, device=DEV)
+ page_table[0, :512] = torch.arange(512, dtype=torch.int32, device=DEV)
+ ctx = SimpleNamespace(kv_cache=pool, page_table=page_table)
+ monkeypatch.setattr("freetoken.attention.dsa.get_global_ctx", lambda: ctx)
+
+ config = SimpleNamespace(
+ glm5_args=_args(), glm_dsa_args=None, num_qo_heads=H,
+ attn_sm_scale=SM_SCALE, num_layers=1,
+ )
+ backend = Glm5NextDSABackend(config)
+ torch.manual_seed(0)
+ # The APE is a MODEL parameter, passed per call via DSAIndexerInputs.
+ ape = torch.randn(KPOOL, DI, device=DEV, dtype=torch.float32) * 0.3
+ return backend, pool, ape
+
+
+def _req(device_len, cached_len=0):
+ return SimpleNamespace(
+ table_idx=0, device_len=device_len, extend_len=device_len - cached_len,
+ cached_len=cached_len, linear_slot_idx=None,
+ )
+
+
+def _prefill_batch(t0, t1):
+ return SimpleNamespace(
+ phase="prefill", padded_reqs=[_req(t1, t0)], reqs=[_req(t1, t0)],
+ positions=torch.arange(t0, t1, device=DEV),
+ out_loc=torch.arange(t0, t1, device=DEV),
+ active_table_idx=None,
+ )
+
+
+def _decode_batch(pos):
+ return SimpleNamespace(
+ phase="decode", padded_reqs=[_req(pos + 1, pos)], reqs=[_req(pos + 1, pos)],
+ positions=torch.tensor([pos], device=DEV),
+ out_loc=torch.tensor([pos], device=DEV),
+ active_table_idx=torch.tensor([0], device=DEV),
+ )
+
+
+def _rand_seq(total, seed=1):
+ torch.manual_seed(seed)
+ mk = lambda *s: torch.randn(*s, device=DEV, dtype=torch.bfloat16)
+ return dict(
+ q_nope=mk(total, H, LATENT), c_kv=mk(total, LATENT),
+ qi=mk(total, HI, DI), ki=mk(total, DI),
+ wi=(torch.randn(total, HI, device=DEV).float() * 0.5),
+ gate=mk(total, DI),
+ )
+
+
+def _run(backend, batch, d, sl, ape):
+ from freetoken.attention.dsa import DSAIndexerInputs
+
+ t = batch.positions.shape[0]
+ backend.prepare_metadata(batch)
+ return backend.mla_forward(
+ d["q_nope"][sl], d["q_nope"].new_empty(t, H, 0),
+ d["c_kv"][sl], d["c_kv"].new_empty(t, 0),
+ 0, batch,
+ indexer_inputs=DSAIndexerInputs(
+ q=d["qi"][sl], k=d["ki"][sl], w=d["wi"][sl],
+ gate=d["gate"][sl], ape=ape,
+ ),
+ )
+
+
+# ---- eager references -----------------------------------------------------------------
+
+
+def _ref_pooled(d, ape, n_pools):
+ """[n_pools, DI] softmax(gate+ape)-weighted pooled keys (fp32 -> bf16)."""
+ k = d["ki"][: n_pools * KPOOL].view(n_pools, KPOOL, DI).float()
+ g = d["gate"][: n_pools * KPOOL].view(n_pools, KPOOL, DI).float()
+ w = torch.softmax(g + ape, dim=1)
+ return (w * k).sum(1).to(torch.bfloat16)
+
+
+def _ref_scores(d, ape, q_idx_t, w_t, n_pools):
+ """Pool scores for one query: sum_h w_h * relu(q_h . k_pool) * DI**-0.5."""
+ kp = _ref_pooled(d, ape, n_pools).float()
+ s = torch.relu(q_idx_t.float() @ kp.T) # [HI, n_pools]
+ return ((w_t * DI**-0.5).unsqueeze(1) * s).sum(0)
+
+
+def _ref_attend(d, q_t, positions):
+ """Full softmax MLA over latent rows at ``positions`` for one query [H, LATENT]."""
+ lat = d["c_kv"][positions].float() # [n, LATENT]
+ logits = q_t.float() @ lat.T * SM_SCALE # [H, n]
+ p = torch.softmax(logits, dim=-1)
+ return (p @ lat).to(torch.bfloat16)
+
+
+def _ref_selected_positions(d, ape, q_idx_t, w_t, pos):
+ """Reference kpool selection for a query at ``pos``: top-k complete pools
+ expanded to tokens, plus the tail [n_pools*KPOOL, pos]."""
+ n_pools = (pos + 1) // KPOOL
+ sel_pools = min(TOPK // KPOOL, n_pools)
+ picked = torch.topk(
+ _ref_scores(d, ape, q_idx_t, w_t, n_pools), sel_pools
+ ).indices.tolist()
+ positions = [p * KPOOL + o for p in picked for o in range(KPOOL)]
+ positions += list(range(n_pools * KPOOL, pos + 1))
+ return sorted(set(positions))
+
+
+def test_dense_path_matches_full_softmax(harness):
+ backend, pool, ape = harness
+ total = 20 # < TOPK -> identity/dense path
+ d = _rand_seq(total)
+ out = _run(backend, _prefill_batch(0, total), d, slice(0, total), ape)
+ for t in range(total):
+ ref = _ref_attend(d, d["q_nope"][t], list(range(t + 1)))
+ err = (out[t].float() - ref.float()).abs().max().item()
+ assert err < 2e-2, f"dense query {t}: err {err}"
+
+
+def test_sparse_prefill_matches_reference(harness):
+ backend, pool, ape = harness
+ total = 60 # > TOPK -> kpool scoring
+ d = _rand_seq(total, seed=2)
+ out = _run(backend, _prefill_batch(0, total), d, slice(0, total), ape)
+
+ # Pooled entries in the slab match the compression reference.
+ n_pools = total // KPOOL
+ # Shadow slab: pool p lives at token_slot // KPOOL == p (identity page table).
+ slab = pool.index_k_cache(0)[torch.arange(n_pools, device=DEV)]
+ ref_pool = _ref_pooled(d, ape, n_pools)
+ assert (slab.float() - ref_pool.float()).abs().max().item() < 2e-2
+
+ for t in (35, 47, 59): # queries past TOPK (sparse regime)
+ sel = _ref_selected_positions(d, ape, d["qi"][t], d["wi"][t], t)
+ ref = _ref_attend(d, d["q_nope"][t], sel)
+ err = (out[t].float() - ref.float()).abs().max().item()
+ assert err < 3e-2, f"sparse query {t}: err {err}"
+
+
+def test_decode_completion_and_selection(harness):
+ backend, pool, ape = harness
+ total, extra = 60, 6 # decode positions 60..65; completion at 63
+ d = _rand_seq(total + extra, seed=3)
+ _run(backend, _prefill_batch(0, total), d, slice(0, total), ape)
+
+ for pos in range(total, total + extra):
+ out = _run(backend, _decode_batch(pos), d, slice(pos, pos + 1), ape)
+ sel = _ref_selected_positions(d, ape, d["qi"][pos], d["wi"][pos], pos)
+ ref = _ref_attend(d, d["q_nope"][pos], sel)
+ err = (out[0].float() - ref.float()).abs().max().item()
+ assert err < 3e-2, f"decode pos {pos}: err {err}"
+
+ if pos % KPOOL == KPOOL - 1: # a pool completed this step
+ n_pools = (pos + 1) // KPOOL
+ row = pos // KPOOL # the pool's shadow row
+ got = pool.index_k_cache(0)[row]
+ want = _ref_pooled(d, ape, n_pools)[-1]
+ assert (got.float() - want.float()).abs().max().item() < 2e-2
+
+
+def test_sparse_batch_with_sub_pool_request(harness):
+ """A sparse prefill batch may carry a request shorter than one pool: it has nothing
+ to score and must come out as the dense (tail-only) attention over its own rows."""
+ from freetoken.attention import dsa as dsa_mod
+
+ backend, pool, ape = harness
+ dsa_mod.get_global_ctx().page_table[1, :64] = torch.arange(
+ 448, 512, dtype=torch.int32, device=DEV
+ )
+ ta = 60 # > TOPK -> the batch takes the sparse path
+ for tb in (1, 2, 3):
+ d = _rand_seq(ta + tb, seed=6)
+ req_b = _req(tb)
+ req_b.table_idx = 1
+ reqs = [_req(ta), req_b]
+ batch = SimpleNamespace(
+ phase="prefill", padded_reqs=reqs, reqs=reqs,
+ positions=torch.cat([torch.arange(ta), torch.arange(tb)]).to(DEV),
+ out_loc=torch.cat([torch.arange(ta), torch.arange(448, 448 + tb)]).to(DEV),
+ active_table_idx=None,
+ )
+ out = _run(backend, batch, d, slice(0, ta + tb), ape)
+ for j in range(tb):
+ t = ta + j
+ ref = _ref_attend(d, d["q_nope"][t], list(range(ta, t + 1)))
+ err = (out[t].float() - ref.float()).abs().max().item()
+ assert err < 3e-2, f"sub-pool request len {tb}, query {j}: err {err}"
+ t = ta - 1
+ sel = _ref_selected_positions(d, ape, d["qi"][t], d["wi"][t], t)
+ ref = _ref_attend(d, d["q_nope"][t], sel)
+ assert (out[t].float() - ref.float()).abs().max().item() < 3e-2
+
+
+def test_chunked_prefill_mid_pool_start(harness):
+ """A chunk may start MID-POOL (main's soft prefill_chunk_align keeps an
+ unaligned end when the budget cannot fill a page): the straddling pool's
+ older members come from the tail ring and the pooled slab + outputs must
+ match the single-shot run."""
+ backend, pool, ape = harness
+ total, split = 60, 30 # split % KPOOL == 2 -> pool 7 straddles the chunks
+ d = _rand_seq(total, seed=4)
+ out1 = _run(backend, _prefill_batch(0, split), d, slice(0, split), ape)
+ out2 = _run(backend, _prefill_batch(split, total), d, slice(split, total), ape)
+
+ n_pools = total // KPOOL
+ slab = pool.index_k_cache(0)[torch.arange(n_pools, device=DEV)]
+ ref_pool = _ref_pooled(d, ape, n_pools)
+ assert (slab.float() - ref_pool.float()).abs().max().item() < 2e-2
+
+ for t in (35, 47, 59):
+ sel = _ref_selected_positions(d, ape, d["qi"][t], d["wi"][t], t)
+ ref = _ref_attend(d, d["q_nope"][t], sel)
+ err = (out2[t - split].float() - ref.float()).abs().max().item()
+ assert err < 3e-2, f"mid-pool chunked query {t}: err {err}"
+
+
+def test_interleaved_decode_requests_do_not_pollute_rings(harness):
+ """Two requests decoding in alternation: tail rings and shadow rows are keyed
+ by table_idx, so neither request's pools may absorb the other's raw K/gate."""
+ from freetoken.attention import dsa as dsa_mod
+
+ backend, pool, ape = harness
+ dsa_mod.get_global_ctx().page_table[1, :128] = torch.arange(
+ 256, 384, dtype=torch.int32, device=DEV
+ )
+ total, extra = 60, 6
+ streams = { # table_idx -> (data, physical row base)
+ 0: (_rand_seq(total + extra, seed=7), 0),
+ 1: (_rand_seq(total + extra, seed=8), 256),
+ }
+
+ def _batch_for(table, phase, t0, t1):
+ base = streams[table][1]
+ req = _req(t1, t0)
+ req.table_idx = table
+ return SimpleNamespace(
+ phase=phase, padded_reqs=[req], reqs=[req],
+ positions=torch.arange(t0, t1, device=DEV),
+ out_loc=torch.arange(base + t0, base + t1, device=DEV),
+ active_table_idx=(
+ torch.tensor([table], device=DEV) if phase == "decode" else None
+ ),
+ )
+
+ for table in (0, 1):
+ d = streams[table][0]
+ _run(backend, _batch_for(table, "prefill", 0, total), d, slice(0, total), ape)
+
+ for pos in range(total, total + extra):
+ for table in (0, 1): # alternate every step
+ d, base = streams[table]
+ out = _run(
+ backend, _batch_for(table, "decode", pos, pos + 1), d,
+ slice(pos, pos + 1), ape,
+ )
+ sel = _ref_selected_positions(d, ape, d["qi"][pos], d["wi"][pos], pos)
+ ref = _ref_attend(d, d["q_nope"][pos], sel)
+ err = (out[0].float() - ref.float()).abs().max().item()
+ assert err < 3e-2, f"table {table} decode pos {pos}: err {err}"
+
+ if pos % KPOOL == KPOOL - 1:
+ n_pools = (pos + 1) // KPOOL
+ row = (base + pos) // KPOOL
+ got = pool.index_k_cache(0)[row]
+ want = _ref_pooled(d, ape, n_pools)[-1]
+ err = (got.float() - want.float()).abs().max().item()
+ assert err < 2e-2, f"table {table} pool at pos {pos}: err {err}"
+
+
+def test_padding_and_empty_batch_leave_shadow_rows_clean():
+ """The compression kernel writes every row somewhere, but masked-off rows
+ (padding request == -1, or a pool that cannot close yet) must land only on
+ their designated scratch rows -- the shadow region and the rings stay clean.
+ An empty batch is a no-op."""
+ from freetoken.kernel.triton.kpool_compress import kpool_compress_store
+
+ shadow_n, n_req = 8, 2
+ slab = torch.full((shadow_n + n_req, DI), 7.0, dtype=torch.bfloat16, device=DEV)
+ ring_k = torch.full((n_req * KPOOL, DI), 3.0, dtype=torch.bfloat16, device=DEV)
+ ring_g = ring_k.clone()
+ ape = torch.randn(KPOOL, DI, dtype=torch.float32, device=DEV)
+ k = torch.randn(2, DI, dtype=torch.bfloat16, device=DEV)
+ gate = torch.randn(2, DI, dtype=torch.bfloat16, device=DEV)
+
+ kpool_compress_store(
+ k, gate, ring_k, ring_g, ape,
+ ring_slots=torch.tensor([0, 1], dtype=torch.int32, device=DEV),
+ token_to_req=torch.tensor([0, -1], dtype=torch.int32, device=DEV),
+ cu_seqlens=torch.tensor([0, 1, 1], dtype=torch.int32, device=DEV),
+ positions=torch.tensor([0, 0], device=DEV), # pos 0: no pool can close
+ slab=slab, cmp_rows=torch.tensor([8, 9], dtype=torch.int32, device=DEV),
+ ratio=KPOOL,
+ )
+ assert torch.equal(slab[:shadow_n], torch.full_like(slab[:shadow_n], 7.0))
+ assert torch.equal(ring_k, torch.full_like(ring_k, 3.0))
+ assert torch.equal(ring_g, ring_k)
+
+ before = slab.clone()
+ kpool_compress_store(
+ k[:0], gate[:0], ring_k, ring_g, ape,
+ ring_slots=torch.tensor([0], dtype=torch.int32, device=DEV),
+ token_to_req=torch.empty(0, dtype=torch.int32, device=DEV),
+ cu_seqlens=torch.tensor([0, 0], dtype=torch.int32, device=DEV),
+ positions=torch.empty(0, dtype=torch.int64, device=DEV),
+ slab=slab, cmp_rows=torch.empty(0, dtype=torch.int32, device=DEV),
+ ratio=KPOOL,
+ )
+ assert torch.equal(slab, before)
diff --git a/tests/e2e/test_aime.py b/tests/e2e/test_aime.py
index 0a4c3a67..f2803539 100644
--- a/tests/e2e/test_aime.py
+++ b/tests/e2e/test_aime.py
@@ -18,6 +18,10 @@ Env knobs:
FREETOKEN_AIME_SAMPLES pass@N sample count when sampling (default 3)
FREETOKEN_AIME_MIN_FREE_GIB free-GPU-memory gate (default 70; raise for a big resident model)
FREETOKEN_TEST_MOE_CACHE_SIZE >0 switches to the offload MoE backend (fp8/GLM/MiniMax)
+ FREETOKEN_TEST_MOE_BACKEND offload-family backend override (default offload; e.g. hybrid)
+ FREETOKEN_TEST_MOE_CPU_THREADS CPU MoE worker threads (0 = physical cores)
+ FREETOKEN_TEST_MOE_CACHE_AUTO 1 sizes the MoE cache from free VRAM instead of MOE_CACHE_SIZE
+ FREETOKEN_TEST_KV_RESERVE KV token floor reserved before auto-cache fills experts
FREETOKEN_TEST_MEM_RATIO offload memory ratio (default 0.9)
fp8 / offload recipe: point FREETOKEN_TEST_MODEL at the fp8 checkpoint dir and set
@@ -164,10 +168,14 @@ def build_llm(model_path: Path) -> LLM:
max_extend_tokens=8192,
)
cache_size = int(os.environ.get("FREETOKEN_TEST_MOE_CACHE_SIZE", "0"))
- if cache_size > 0:
+ cache_auto = os.environ.get("FREETOKEN_TEST_MOE_CACHE_AUTO") == "1"
+ if cache_size > 0 or cache_auto:
kwargs.update(
- moe_backend="offload",
+ moe_backend=os.environ.get("FREETOKEN_TEST_MOE_BACKEND", "offload"),
+ moe_cpu_threads=int(os.environ.get("FREETOKEN_TEST_MOE_CPU_THREADS", "0")),
+ moe_cache_auto=cache_auto,
moe_cache_size=cache_size,
+ kv_reserve_tokens=int(os.environ.get("FREETOKEN_TEST_KV_RESERVE", "8192")),
moe_cache_policy="lru",
memory_ratio=float(os.environ.get("FREETOKEN_TEST_MEM_RATIO", "0.9")),
max_seq_len_override=max_tokens() + 2048,
diff --git a/tests/engine/test_attention_backend_matrix.py b/tests/engine/test_attention_backend_matrix.py
index fc9abfbe..ef7dc7ff 100644
--- a/tests/engine/test_attention_backend_matrix.py
+++ b/tests/engine/test_attention_backend_matrix.py
@@ -259,13 +259,15 @@ def test_legal_explicit_combinations_pass(monkeypatch, kind, backend):
@pytest.mark.parametrize("kind", ["mla", "dsa"])
-def test_mla_requires_page_size_one(monkeypatch, kind):
+def test_mla_auto_adjusts_page_size(monkeypatch, kind):
+ """Any --page-size is auto-adjusted (with a warning) to the layout's
+ required size: 1 for plain latent-KV, 64 for the kpool variant."""
from freetoken.engine.engine import _adjust_config
_patch_env(monkeypatch)
config = _config(kind, attention_backend="auto", page_size=16)
- with pytest.raises(ValueError, match="page-size 1"):
- _adjust_config(config)
+ _adjust_config(config)
+ assert config.page_size == 1
def test_trtllm_page_size_coercion_is_part_aware(monkeypatch):
diff --git a/tests/kernels/test_kda.py b/tests/kernels/test_kda.py
new file mode 100644
index 00000000..b217ee32
--- /dev/null
+++ b/tests/kernels/test_kda.py
@@ -0,0 +1,106 @@
+"""Pins for OUR divergences from the upstream-vendored KDA kernels.
+
+The vendored kernels (freetoken/kernel/fla) are tested upstream and not re-tested
+here. The eager reference replicates their exact math (safe gate ``gk =
+lower_bound * sigmoid(exp(A_log) * (g_raw + dt_bias))``, ``beta =
+sigmoid(beta_raw)``, in-loop q/k l2norm, per-channel-decayed delta rule on a
+[V, K] state) so a divergence pin can assert numerics, not just reachability.
+"""
+
+from __future__ import annotations
+
+import pytest
+import torch
+
+pytestmark = pytest.mark.skipif(not torch.cuda.is_available(), reason="needs CUDA")
+
+H, D = 4, 128 # head count trimmed; head_dim matches GLM-5.3 (kernel specializes on D)
+LOWER_BOUND = -5.0
+SCALE = D**-0.5
+
+
+def _l2norm(x: torch.Tensor) -> torch.Tensor:
+ return x / torch.sqrt((x * x).sum(-1, keepdim=True) + 1e-6)
+
+
+def _reference(
+ q: torch.Tensor, # [T, H, D] bf16
+ k: torch.Tensor,
+ v: torch.Tensor,
+ g_raw: torch.Tensor, # [T, H, D] bf16
+ beta_raw: torch.Tensor, # [T, H] bf16
+ a_log: torch.Tensor, # [H] fp32
+ dt_bias: torch.Tensor, # [H*D] fp32
+ h0: torch.Tensor | None = None, # [H, V, K] fp32
+) -> tuple[torch.Tensor, torch.Tensor]:
+ T = q.shape[0]
+ h = (
+ h0.clone().float()
+ if h0 is not None
+ else torch.zeros(H, D, D, dtype=torch.float32, device=q.device)
+ )
+ amp = a_log.float().exp().view(H, 1)
+ bias = dt_bias.float().view(H, D)
+ outs = []
+ for t in range(T):
+ gk = LOWER_BOUND * torch.sigmoid(amp * (g_raw[t].float() + bias)) # [H, K]
+ h = h * gk.exp().unsqueeze(1) # decay per k-channel: [H, V, K] * [H, 1, K]
+ kt = _l2norm(k[t].float())
+ v_err = v[t].float() - torch.einsum("hvk,hk->hv", h, kt)
+ v_err = v_err * torch.sigmoid(beta_raw[t].float()).unsqueeze(-1)
+ h = h + torch.einsum("hv,hk->hvk", v_err, kt)
+ qt = _l2norm(q[t].float()) * SCALE
+ outs.append(torch.einsum("hvk,hk->hv", h, qt))
+ return torch.stack(outs), h
+
+
+def _rand_inputs(T: int, seed: int = 0, device="cuda"):
+ torch.manual_seed(seed)
+ mk = lambda *s: torch.randn(*s, device=device, dtype=torch.bfloat16)
+ q, k, v, g_raw = mk(T, H, D), mk(T, H, D), mk(T, H, D), mk(T, H, D)
+ beta_raw = mk(T, H)
+ a_log = torch.randn(H, device=device, dtype=torch.float32) * 0.5
+ dt_bias = torch.randn(H * D, device=device, dtype=torch.float32) * 0.5
+ return q, k, v, g_raw, beta_raw, a_log, dt_bias
+
+
+def _assert_close(ours, ref, tag, atol=2e-2, rtol=2e-2):
+ ours, ref = ours.float(), ref.float()
+ err = (ours - ref).abs().max().item()
+ rel = err / (ref.abs().max().item() + 1e-8)
+ assert torch.allclose(ours, ref, atol=atol, rtol=rtol), (
+ f"{tag}: max abs err {err:.5f}, rel {rel:.5f}"
+ )
+
+
+def test_fused_recurrent_serves_slot_zero():
+ """--cache-type naive keys state by raw table_idx, so a real request can sit
+ on slot 0. Upstream vLLM's kernel treats 0 as its NULL_BLOCK_ID sentinel and
+ silently skips it (state frozen, garbage output); our vendored copy diverges
+ to accept every non-negative slot (GDN-kernel parity). Same math as the
+ parametrized reference test, just on slot 0."""
+ from freetoken.kernel.fla import fused_recurrent_kda
+
+ T = 7
+ q, k, v, g_raw, beta_raw, a_log, dt_bias = _rand_inputs(T)
+ ref_o, ref_h = _reference(q, k, v, g_raw, beta_raw, a_log, dt_bias)
+
+ pool = torch.zeros(2, H, D, D, dtype=torch.float32, device="cuda")
+ indices = torch.zeros((1, T), dtype=torch.int64, device="cuda") # slot 0
+ cu = torch.tensor([0, T], dtype=torch.int32, device="cuda")
+ o, _ = fused_recurrent_kda(
+ q=q.unsqueeze(0), k=k.unsqueeze(0), v=v.unsqueeze(0),
+ g=g_raw.unsqueeze(0), beta=beta_raw.unsqueeze(0),
+ initial_state=pool,
+ use_qk_l2norm_in_kernel=True,
+ cu_seqlens=cu,
+ ssm_state_indices=indices,
+ sigmoid_beta=True,
+ a_log=a_log,
+ g_bias=dt_bias,
+ compute_gate=True,
+ lower_bound=LOWER_BOUND,
+ )
+ _assert_close(o[0], ref_o, "slot-0 output")
+ _assert_close(pool[0], ref_h, "slot-0 final state")
+ assert pool[1].abs().max().item() == 0.0 # only slot 0 was touched
diff --git a/tests/kernels/test_swiglu_clamp.py b/tests/kernels/test_swiglu_clamp.py
new file mode 100644
index 00000000..2515bcdd
--- /dev/null
+++ b/tests/kernels/test_swiglu_clamp.py
@@ -0,0 +1,43 @@
+"""swiglu_clamp (GLM-5.3 ``swiglu_limit``) activation parity.
+
+Reference: vLLM's SiluAndMulWithClamp with alpha=1, beta=0 --
+``clamp(gate, max=L) * sigmoid(gate_clamped) * clamp(up, +-L)``. Checks the
+Triton kernel, its distinction from swigluoai (the +1 up bias), and that the
+compiled CPU MoE extension advertises the new generic act id.
+"""
+
+from __future__ import annotations
+
+import pytest
+import torch
+
+LIMIT = 10.0
+
+
+def _ref(x: torch.Tensor, limit: float = LIMIT) -> torch.Tensor:
+ d = x.shape[-1] // 2
+ gate = x[..., :d].float().clamp(max=limit)
+ up = x[..., d:].float().clamp(min=-limit, max=limit)
+ return (gate * torch.sigmoid(gate) * up).to(x.dtype)
+
+
[email protected](not torch.cuda.is_available(), reason="needs CUDA")
+def test_triton_matches_reference():
+ from freetoken.layers import swiglu_clamp_and_mul
+
+ torch.manual_seed(0)
+ # Scale up so the clamp actually engages on a good fraction of elements.
+ x = torch.randn(129, 2 * 512, device="cuda", dtype=torch.bfloat16) * 8.0
+ out = swiglu_clamp_and_mul(x, alpha=1.0, limit=LIMIT)
+ ref = _ref(x)
+ assert (out.float() - ref.float()).abs().max().item() < 2e-2
+ assert (x[..., :512].float() > LIMIT).any(), "test data never hit the clamp"
+
+
+def test_cpu_extension_supports_swiglu_clamp():
+ from freetoken.moe.cpu_executor import compiled_extension_supports
+
+ assert compiled_extension_supports("swiglu_clamp"), (
+ "compiled _cpu_moe extension is stale -- rebuild with ACT_SWIGLU_CLAMP "
+ "(python setup.py build_ext --inplace)"
+ )
diff --git a/tests/kvcache/test_hybrid_linear_paged_pools.py b/tests/kvcache/test_hybrid_linear_paged_pools.py
new file mode 100644
index 00000000..8b60801c
--- /dev/null
+++ b/tests/kvcache/test_hybrid_linear_paged_pools.py
@@ -0,0 +1,117 @@
+"""Pool-factory generalization: hybrid linear x ANY paged family.
+
+The old factory hard-required "linear + one GQA group" (Qwen3.5 GDN shape);
+glm5_next is linear (KDA) x DSA. Checks the factory dispatch, the MLA/DSA
+layer-id remap (34 KDA layers cost no latent slabs), the kpool pool selection
+(gate slab), and the KV cost model's kpool double-count of the index slabs.
+"""
+
+from __future__ import annotations
+
+import pytest
+import torch
+
+from freetoken.kvcache import create_kvcache_pool, resolve_pool_class
+from freetoken.kvcache.dsa_pool import DSAKVCache, KpoolDSAKVCache
+from freetoken.models.config import (
+ FullAttentionGroupConfig,
+ LinearGatedDeltaGroupConfig,
+ ModelConfig,
+ RotaryConfig,
+)
+
+
[email protected](autouse=True)
+def _single_rank_tp():
+ from freetoken.distributed import set_tp_info, try_get_tp_info
+
+ if try_get_tp_info() is None:
+ set_tp_info(rank=0, size=1)
+
+
+def _glm5_like_config(index_kpool=4, index_head_dim=128):
+ n_layers = 12
+ dsa_ids = tuple(range(3, n_layers, 4)) # 3, 7, 11
+ kda_ids = tuple(i for i in range(n_layers) if i not in dsa_ids)
+ rotary = RotaryConfig(head_dim=256, rotary_dim=0, max_position=4096, base=1e4, scaling=None)
+ groups = (
+ LinearGatedDeltaGroupConfig(
+ name="linear", layer_ids=kda_ids,
+ num_key_heads=4, num_value_heads=4, key_head_dim=128, value_head_dim=128,
+ conv_kernel_dim=4, output_gate="sigmoid", variant="kda",
+ ),
+ FullAttentionGroupConfig(
+ name="full", layer_ids=dsa_ids, num_kv_heads=1, head_dim=512,
+ rotary_config=rotary, mla=True,
+ index_head_dim=index_head_dim, num_index_layers=len(dsa_ids),
+ index_ratio=index_kpool,
+ ),
+ )
+ return ModelConfig(
+ num_layers=n_layers, num_qo_heads=4, num_kv_heads=1, head_dim=512,
+ hidden_size=256, vocab_size=1000, intermediate_size=512,
+ rms_norm_eps=1e-5, rotary_config=rotary, hidden_act="silu",
+ tie_word_embeddings=False, num_experts=8, num_experts_per_tok=2,
+ moe_intermediate_size=64, norm_topk_prob=True, model_type="glm5_next",
+ architectures=["Glm5NextForCausalLM"], moe_enabled=True,
+ attention_groups=groups,
+ )
+
+
+def test_factory_builds_kpool_pool_with_layer_remap():
+ cfg = _glm5_like_config()
+ assert resolve_pool_class(cfg) is KpoolDSAKVCache
+
+ pool = create_kvcache_pool(
+ model_config=cfg, num_pages=4, page_size=64,
+ dtype=torch.bfloat16, device=torch.device("cpu"), num_req_slots=5,
+ )
+ assert isinstance(pool, KpoolDSAKVCache)
+ # Latent slabs back ONLY the 3 DSA layers (34-of-45 economy at real scale).
+ assert pool._kv_buffer.shape[1] == 3
+ # Global layer-id addressing: DSA layers resolve, KDA layers have no slab.
+ for lid in (3, 7, 11):
+ assert pool.latent_rows(lid).shape == (256, 512)
+ with pytest.raises(KeyError):
+ pool.latent_rows(0) # a KDA layer
+ # Shadow index slab: tokens/ratio rows + one scratch row per request slot.
+ assert pool.index_k_cache(0).shape == (256 // 4 + 5, 128)
+ assert pool.cmp_scratch_base == 256 // 4
+ # kpool tail rings exist at [num_req_slots, ratio, head_dim] per indexer layer.
+ assert pool.tail_k(0).shape == pool.tail_gate(0).shape
+ assert pool.tail_k(0).shape == (5, 4, 128)
+
+
+def test_factory_kpool1_builds_plain_dsa_pool():
+ cfg = _glm5_like_config(index_kpool=1)
+ assert resolve_pool_class(cfg) is DSAKVCache
+ pool = create_kvcache_pool(
+ model_config=cfg, num_pages=16, page_size=1,
+ dtype=torch.bfloat16, device=torch.device("cpu"),
+ )
+ assert type(pool) is DSAKVCache
+
+
+def test_cost_model_kpool_shadow_slab_quarter_cost():
+ """The shadow slab stores one row per index_ratio tokens: the kpool spec's
+ index bytes are 1/ratio of the plain DSA slab (rings/scratch are per-request
+ and not part of the per-token price)."""
+ from types import SimpleNamespace
+
+ from freetoken.kvcache.base import spec_kv_bytes_per_token
+
+ tp = SimpleNamespace(size=1)
+ econf = SimpleNamespace(tp_info=tp, dtype=torch.bfloat16)
+ (spec,) = [
+ s for s in _glm5_like_config().kv_cache_group_specs() if s.num_layers > 0
+ ]
+ (spec1,) = [
+ s
+ for s in _glm5_like_config(index_kpool=1).kv_cache_group_specs()
+ if s.num_layers > 0
+ ]
+ index_full = spec.index_head_dim * spec.num_index_layers * 2
+ assert (
+ spec_kv_bytes_per_token(spec1, econf) - spec_kv_bytes_per_token(spec, econf)
+ == index_full - index_full // 4
+ )
\ No newline at end of file
diff --git a/tests/layers/test_mhc.py b/tests/layers/test_mhc.py
new file mode 100644
index 00000000..4cf144e7
--- /dev/null
+++ b/tests/layers/test_mhc.py
@@ -0,0 +1,178 @@
+"""mHC (Manifold-Constrained Hyper-Connections) unit tests.
+
+Checks the algebraic contracts of layers/mhc.py at GLM-5.3 geometry (n=4):
+Sinkhorn projection yields an (approximately) doubly-stochastic comb matrix,
+pre/post mixing matches naive per-token einsums, and identity-ish weights give
+the classic single-stream residual behaviour.
+"""
+
+from __future__ import annotations
+
+import pytest
+import torch
+
+from freetoken.layers.mhc import (
+ hc_contract,
+ hc_expand,
+ mhc_fused_post_pre,
+ mhc_post,
+ mhc_pre,
+)
+
+N, HIDDEN, T = 4, 64, 9
+MIX = 2 * N + N * N
+EPS = 1e-6
+RMS_EPS = 1e-5
+POST_MULT = 2.0
+SINKHORN = 20
+
+
+def _weights(seed=0, device="cpu"):
+ torch.manual_seed(seed)
+ fn = torch.randn(MIX, N * HIDDEN, dtype=torch.float32, device=device) * 0.05
+ scale = torch.randn(3, dtype=torch.float32, device=device).abs() + 0.5
+ base = torch.randn(MIX, dtype=torch.float32, device=device) * 0.3
+ return fn, scale, base
+
+
+def _residual(seed=1, device="cpu"):
+ torch.manual_seed(seed)
+ return torch.randn(T, N, HIDDEN, dtype=torch.bfloat16, device=device)
+
+
+def test_comb_is_doubly_stochastic():
+ fn, scale, base = _weights(seed=2)
+ res = _residual(seed=3)
+ _, comb, _ = mhc_pre(res, fn, scale, base, RMS_EPS, EPS, POST_MULT, SINKHORN)
+ rows = comb.sum(dim=-1)
+ cols = comb.sum(dim=-2)
+ assert torch.allclose(rows, torch.ones_like(rows), atol=1e-3)
+ assert torch.allclose(cols, torch.ones_like(cols), atol=1e-3)
+ assert (comb > 0).all()
+
+
+def test_post_matches_naive():
+ res = _residual(seed=4)
+ x = torch.randn(T, HIDDEN, dtype=torch.bfloat16)
+ post = torch.rand(T, N, 1, dtype=torch.float32) * POST_MULT
+ comb = torch.softmax(torch.randn(T, N, N), dim=-1)
+ out = mhc_post(x, res, post, comb)
+ assert out.shape == (T, N, HIDDEN)
+
+ ref = torch.zeros(T, N, HIDDEN, dtype=torch.float32)
+ for t in range(T):
+ for j in range(N):
+ acc = post[t, j, 0] * x[t].float()
+ for i in range(N):
+ acc = acc + comb[t, i, j] * res[t, i].float()
+ ref[t, j] = acc
+ assert torch.allclose(out.float(), ref.to(torch.bfloat16).float())
+
+
+def test_fused_equals_decomposed():
+ fn, scale, base = _weights(seed=7)
+ res = _residual(seed=8)
+ x = torch.randn(T, HIDDEN, dtype=torch.bfloat16)
+ post0, comb0, _ = mhc_pre(res, fn, scale, base, RMS_EPS, EPS, POST_MULT, SINKHORN)
+
+ r1, p1, c1, li1 = mhc_fused_post_pre(
+ x, res, post0, comb0, fn, scale, base, RMS_EPS, EPS, POST_MULT, SINKHORN
+ )
+ r_ref = mhc_post(x, res, post0, comb0)
+ p_ref, c_ref, li_ref = mhc_pre(
+ r_ref, fn, scale, base, RMS_EPS, EPS, POST_MULT, SINKHORN
+ )
+ assert torch.equal(r1, r_ref)
+ assert torch.equal(p1, p_ref)
+ assert torch.equal(c1, c_ref)
+ assert torch.equal(li1, li_ref)
+
+
[email protected](not torch.cuda.is_available(), reason="needs CUDA")
[email protected]("t,hidden", [(1, 64), (9, 64), (3, 4096)])
+def test_triton_fused_matches_torch(t, hidden):
+ """The fused triton kernel must reproduce the decomposed torch reference
+ (hc_post -> hc_pre) on every output, including GLM-5.3's real hidden size."""
+ from freetoken.layers.mhc import mhc_fused_post_pre_torch
+ from freetoken.kernel.triton.mhc import mhc_fused_post_pre_triton
+
+ torch.manual_seed(11)
+ mix = 2 * N + N * N
+ fn = torch.randn(mix, N * hidden, dtype=torch.float32, device="cuda") * 0.05
+ scale = torch.rand(3, dtype=torch.float32, device="cuda") + 0.5
+ base = torch.randn(mix, dtype=torch.float32, device="cuda") * 0.3
+ res = torch.randn(t, N, hidden, dtype=torch.bfloat16, device="cuda")
+ x = torch.randn(t, hidden, dtype=torch.bfloat16, device="cuda")
+ post0 = torch.rand(t, N, 1, dtype=torch.float32, device="cuda") * POST_MULT
+ comb0 = torch.softmax(torch.randn(t, N, N, device="cuda"), dim=-1)
+
+ ref = mhc_fused_post_pre_torch(
+ x, res, post0, comb0, fn, scale, base, RMS_EPS, EPS, POST_MULT, SINKHORN
+ )
+ got = mhc_fused_post_pre_triton(
+ x, res, post0, comb0, fn, scale, base, RMS_EPS, EPS, POST_MULT, SINKHORN
+ )
+ names = ("residual", "post", "comb", "layer_input")
+ tols = (2e-2, 2e-3, 2e-3, 2e-2)
+ for name, r, g, tol in zip(names, ref, got, tols):
+ assert g.shape == r.shape, (name, g.shape, r.shape)
+ err = (g.float() - r.float()).abs().max().item()
+ assert err < tol, f"{name}: max abs err {err}"
+
+
[email protected](not torch.cuda.is_available(), reason="needs CUDA")
[email protected]("dtype,tol", [(torch.float16, 2e-2), (torch.float32, 1e-4)])
+def test_triton_fused_respects_input_dtype(dtype, tol):
+ """--dtype float16/float32 must not be silently bf16-rounded: the kernel
+ stores in the OUTPUT tensor's dtype (regression for the hard-coded
+ tl.bfloat16 stores; fp32's tolerance is far below bf16's 2^-8 grid)."""
+ from freetoken.layers.mhc import mhc_fused_post_pre_torch
+ from freetoken.kernel.triton.mhc import mhc_fused_post_pre_triton
+
+ torch.manual_seed(13)
+ t, hidden = 4, 4096
+ mix = 2 * N + N * N
+ fn = torch.randn(mix, N * hidden, dtype=torch.float32, device="cuda") * 0.05
+ scale = torch.rand(3, dtype=torch.float32, device="cuda") + 0.5
+ base = torch.randn(mix, dtype=torch.float32, device="cuda") * 0.3
+ res = torch.randn(t, N, hidden, dtype=dtype, device="cuda")
+ x = torch.randn(t, hidden, dtype=dtype, device="cuda")
+ post0 = torch.rand(t, N, 1, dtype=torch.float32, device="cuda") * POST_MULT
+ comb0 = torch.softmax(torch.randn(t, N, N, device="cuda"), dim=-1)
+
+ ref = mhc_fused_post_pre_torch(
+ x, res, post0, comb0, fn, scale, base, RMS_EPS, EPS, POST_MULT, SINKHORN
+ )
+ got = mhc_fused_post_pre_triton(
+ x, res, post0, comb0, fn, scale, base, RMS_EPS, EPS, POST_MULT, SINKHORN
+ )
+ assert got[0].dtype == dtype and got[3].dtype == dtype
+ for name, r, g in zip(("residual", "layer_input"), (ref[0], ref[3]), (got[0], got[3])):
+ err = (g.float() - r.float()).abs().max().item()
+ assert err < tol, f"{name} [{dtype}]: max abs err {err}"
+
+
[email protected](not torch.cuda.is_available(), reason="needs CUDA")
+def test_triton_pre_only_matches_torch():
+ """HAS_POST=False path (layer 0's standalone hc_pre through the fused kernel)."""
+ from freetoken.kernel.triton.mhc import mhc_fused_post_pre_triton
+
+ torch.manual_seed(12)
+ t, hidden = 5, 128
+ mix = 2 * N + N * N
+ fn = torch.randn(mix, N * hidden, dtype=torch.float32, device="cuda") * 0.05
+ scale = torch.rand(3, dtype=torch.float32, device="cuda") + 0.5
+ base = torch.randn(mix, dtype=torch.float32, device="cuda") * 0.3
+ res = torch.randn(t, N, hidden, dtype=torch.bfloat16, device="cuda")
+
+ ref_post, ref_comb, ref_li = mhc_pre(
+ res, fn, scale, base, RMS_EPS, EPS, POST_MULT, SINKHORN
+ )
+ got_res, got_post, got_comb, got_li = mhc_fused_post_pre_triton(
+ res.new_empty(t, hidden), res, None, None, fn, scale, base,
+ RMS_EPS, EPS, POST_MULT, SINKHORN,
+ )
+ assert torch.equal(got_res, res) # pass-through when no post
+ assert (got_post.float() - ref_post.float()).abs().max().item() < 2e-3
+ assert (got_comb.float() - ref_comb.float()).abs().max().item() < 2e-3
+ assert (got_li.float() - ref_li.float()).abs().max().item() < 2e-2
diff --git a/tests/models/test_glm5_next_config.py b/tests/models/test_glm5_next_config.py
new file mode 100644
index 00000000..ae4daf5b
--- /dev/null
+++ b/tests/models/test_glm5_next_config.py
@@ -0,0 +1,281 @@
+"""glm5_next (GLM-5.3-Flash) config parsing: attention groups, kpool spec, aliases.
+
+Runs off a trimmed copy of the real checkpoint config.json via RawConfigShim (the
+exact object cached_load_hf_config falls back to when transformers doesn't know
+``glm5_next`` yet), so the parse path under test is the one production hits.
+"""
+
+from __future__ import annotations
+
+import pytest
+
+from freetoken.attention.base import AttnType
+from freetoken.models.config import (
+ FullAttentionGroupConfig,
+ LinearGatedDeltaGroupConfig,
+)
+from freetoken.models.glm5_next.args import load_args
+from freetoken.models.glm5_next.config import parse_config
+from freetoken.utils.hf import RawConfigShim
+
+_NUM_LAYERS = 45
+_DSA_IDS = tuple(range(3, _NUM_LAYERS, 4)) # 3, 7, ..., 43
+_KDA_IDS = tuple(i for i in range(_NUM_LAYERS) if i not in _DSA_IDS)
+
+
+def _layer_types() -> list[str]:
+ return [
+ "deepseek_sparse_attention" if i in _DSA_IDS else "linear_attention"
+ for i in range(_NUM_LAYERS)
+ ]
+
+
+def _text_config() -> dict:
+ # Trimmed from zai-org/GLM-5.3-Flash config.json (text_config).
+ return {
+ "hidden_size": 4096,
+ "intermediate_size": 12288,
+ "num_hidden_layers": _NUM_LAYERS,
+ "num_attention_heads": 64,
+ "num_key_value_heads": 64,
+ "vocab_size": 154880,
+ "hidden_act": "silu",
+ "rms_norm_eps": 1e-5,
+ "max_position_embeddings": 1048576,
+ "tie_word_embeddings": False,
+ # MLA (NoPE)
+ "q_lora_rank": 1536,
+ "kv_lora_rank": 512,
+ "qk_nope_head_dim": 256,
+ "qk_rope_head_dim": 0,
+ "qk_head_dim": 256,
+ "v_head_dim": 256,
+ "mla_use_nope": True,
+ # DSA indexer + kpool
+ "index_n_heads": 32,
+ "index_head_dim": 128,
+ "index_topk": 2048,
+ "indexer_types": ["full"] * _NUM_LAYERS,
+ "indexer_rope_interleave": True,
+ "index_kpool": 4,
+ "index_kpool_compress": True,
+ "index_kpool_always_select_tail": True,
+ # KDA
+ "linear_attn_config": {
+ "num_heads": 64,
+ "head_dim": 128,
+ "short_conv_kernel_size": 4,
+ "gate_lower_bound": -5.0,
+ "kda_layers": list(_KDA_IDS),
+ "full_attn_layers": list(_DSA_IDS),
+ },
+ # layout
+ "layer_types": _layer_types(),
+ "mlp_layer_types": ["dense"] * 3 + ["sparse"] * (_NUM_LAYERS - 3),
+ "first_k_dense_replace": 3,
+ # mHC (checkpoint spellings)
+ "mhc": True,
+ "hc_mult": 4,
+ "hc_eps": 1e-6,
+ "hc_sinkhorn_iters": 20,
+ # MoE
+ "n_routed_experts": 288,
+ "num_experts_per_tok": 8,
+ "n_shared_experts": 1,
+ "moe_intermediate_size": 2048,
+ "norm_topk_prob": True,
+ "routed_scaling_factor": 2.5,
+ "scoring_func": "sigmoid",
+ "topk_method": "noaux_tc",
+ "n_group": 1,
+ "topk_group": 1,
+ "swiglu_limit": 10.0,
+ "num_nextn_predict_layers": 1,
+ "attention_bias": False,
+ "model_type": "glm5_next_text",
+ }
+
+
+def _hf_config(quantization_config: dict | None = None) -> RawConfigShim:
+ data: dict = {
+ "architectures": ["Glm5NextForConditionalGeneration"],
+ "model_type": "glm5_next",
+ "text_config": _text_config(),
+ "vision_config": {"model_type": "glm5_next_vision", "depth": 24},
+ "image_token_id": 154854,
+ }
+ if quantization_config is not None:
+ data["quantization_config"] = quantization_config
+ return RawConfigShim(data)
+
+
+_CT_NVFP4_QUANT = {
+ # From RedHatAI/GLM-5.3-Flash-NVFP4 (llm-compressor, experts-only calibrated NVFP4).
+ "quant_method": "compressed-tensors",
+ "format": "nvfp4-pack-quantized",
+ "config_groups": {
+ "group_0": {
+ "targets": ["re:.*mlp\\.experts\\..*(gate|up|down)_proj$"],
+ "weights": {"num_bits": 4, "type": "float", "group_size": 16},
+ }
+ },
+}
+
+_NVFP4_QUANT = {
+ # From LibertAIDAI/GLM-5.3-Flash-NVFP4 (ModelOpt weight-only NVFP4).
+ "quant_algo": "NVFP4",
+ "quant_method": "modelopt",
+ "config_groups": {
+ "group_0": {
+ "targets": ["Linear"],
+ "weights": {"num_bits": 4, "type": "float", "group_size": 16},
+ }
+ },
+ "ignore": ["lm_head", "model.visual.*"],
+}
+
+
+def test_attention_groups():
+ cfg = parse_config(_hf_config())
+
+ assert cfg.num_layers == _NUM_LAYERS
+ assert len(cfg.attention_groups) == 2
+ linear, full = cfg.attention_groups # ordered by first layer id (0 < 3)
+
+ assert isinstance(linear, LinearGatedDeltaGroupConfig)
+ assert linear.variant == "kda"
+ assert linear.layer_ids == _KDA_IDS
+ assert (linear.num_key_heads, linear.key_head_dim) == (64, 128)
+ assert (linear.num_value_heads, linear.value_head_dim) == (64, 128)
+ assert linear.conv_kernel_dim == 4
+
+ assert isinstance(full, FullAttentionGroupConfig)
+ assert full.layer_ids == _DSA_IDS
+ assert full.mla is True
+ assert full.head_dim == 512 # bare ckv latent: kv_lora_rank + 0 rope dims
+ assert full.num_kv_heads == 1
+ assert full.index_head_dim == 128
+ assert full.num_index_layers == len(_DSA_IDS)
+ assert full.index_ratio == 4
+
+ assert cfg.has_linear_attention and cfg.has_hybrid_attention
+ assert cfg.attn_type_for_layer(0) == AttnType.LINEAR
+ assert cfg.attn_type_for_layer(3) == AttnType.DSA
+
+
+def test_kv_cache_group_specs_skip_linear_and_carry_kpool():
+ cfg = parse_config(_hf_config())
+ specs = [s for s in cfg.kv_cache_group_specs() if s.num_layers > 0]
+ # The linear group keeps recurrent state (no paged KV); exactly one paged spec.
+ assert len(specs) == 1
+ (spec,) = specs
+ assert spec.attn_type == AttnType.DSA
+ assert spec.layer_ids == _DSA_IDS
+ assert (spec.mla, spec.head_dim, spec.index_head_dim) == (True, 512, 128)
+ assert spec.index_ratio == 4
+ assert spec.num_index_layers == len(_DSA_IDS)
+
+
+def test_moe_and_scalars():
+ cfg = parse_config(_hf_config(_NVFP4_QUANT))
+ assert cfg.expert_quant == "nvfp4"
+ # Only DERIVED facts are pinned here; fixture echoes (num_experts == 288
+ # and friends) assert nothing the parse could get wrong.
+ assert cfg.attn_sm_scale == pytest.approx(256**-0.5)
+ assert cfg.num_moe_layers == _NUM_LAYERS - 3
+ assert cfg.is_moe
+ # Checkpoint-faithful default; the FREETOKEN_GLM5_*_FP8 env flags opt into
+ # the W8A16 fp8 load.
+ assert (cfg.attn_quant, cfg.dense_quant, cfg.lm_head_quant) == ("none",) * 3
+ # Text-only serving: the vision tower is never built.
+ assert cfg.vision_config is None
+
+
+def test_args_alias_folding_and_nope():
+ args = load_args(_hf_config())
+ # Checkpoint spellings fold into the canonical fields.
+ assert args.mla_nope is True # from mla_use_nope
+ assert args.mhc_num_residual_streams == 4 # from hc_mult
+ assert args.mhc_sinkhorn_iterations == 20 # from hc_sinkhorn_iters
+ assert args.linear_num_heads == 64 # from nested linear_attn_config
+ assert args.linear_lower_bound == -5.0
+ # NoPE geometry.
+ assert args.qk_rope_head_dim == 0
+ assert args.qk_head_dim == 256
+ assert args.latent_dim == 512
+ assert args.kda_layer_ids == _KDA_IDS
+ assert args.dsa_layer_ids == _DSA_IDS
+ # Defaults for fields the checkpoint doesn't ship.
+ assert args.rope_theta == 10000.0
+ assert args.mhc_tau == 0.05
+ assert args.mhc_post_mult_value == 2.0
+
+
+def test_registry_resolves_glm5_next():
+ from freetoken.models.register import get_model_spec
+
+ spec = get_model_spec("Glm5NextForConditionalGeneration")
+ assert spec.module == "freetoken.models.glm5_next"
+ assert spec.model_cls == "Glm5NextForCausalLM"
+ assert get_model_spec("Glm5NextForCausalLM").module == spec.module
+
+
+def test_dev_layer_cap(monkeypatch):
+ monkeypatch.setenv("FREETOKEN_GLM5_MAX_LAYERS", "5")
+ cfg = parse_config(_hf_config())
+ assert cfg.num_layers == 5
+ linear, full = cfg.attention_groups
+ assert linear.layer_ids == (0, 1, 2, 4)
+ assert full.layer_ids == (3,)
+ assert full.num_index_layers == 1
+ # 3 dense + 2 sparse under the cap.
+ assert cfg.first_k_dense_replace == 3
+
+
+def test_rejects_unknown_layer_types():
+ data = _hf_config().to_dict()
+ data["text_config"]["layer_types"][0] = "full_attention"
+ with pytest.raises(ValueError, match="unsupported layer_types"):
+ load_args(RawConfigShim(data))
+
+
+def test_compressed_tensors_nvfp4_detected():
+ """RedHatAI/GLM-5.3-Flash-NVFP4 (llm-compressor): expert_quant resolves to
+ nvfp4 off the ``format`` field (quant_algo is absent for compressed-tensors)."""
+ cfg = parse_config(_hf_config(_CT_NVFP4_QUANT))
+ assert cfg.expert_quant == "nvfp4"
+
+
+def test_expert_source_spec_selection():
+ """quant_method picks the bank source spec: compressed-tensors maps
+ weight_packed/weight_global_scale onto the canonical kinds with a reciprocal
+ global; modelopt stays identity."""
+ from freetoken.models.glm5_next.weight import (
+ _NVFP4_CT_SOURCE_SPEC,
+ _NVFP4_SOURCE_SPEC,
+ )
+
+ ct = _NVFP4_CT_SOURCE_SPEC
+ m = ct.key_pattern.match(
+ "model.language_model.layers.5.mlp.experts.7.gate_proj.weight_packed"
+ )
+ assert m and m.group("kind") == "weight_packed"
+ assert ct.kind_map["weight_packed"] == "weight"
+ assert ct.kind_map["weight_global_scale"] == "weight_scale_2"
+ assert ct.global_reciprocal
+ # W4A16 serving never consumes the calibrated activation scale.
+ assert ct.key_pattern.match(
+ "model.language_model.layers.5.mlp.experts.7.gate_proj.input_global_scale"
+ ) is None
+ assert _NVFP4_SOURCE_SPEC.kind_map is None
+ assert not _NVFP4_SOURCE_SPEC.global_reciprocal
+
+
+def test_ingest_global_reciprocal():
+ import torch
+ from freetoken.models.glm5_next.weight import _NVFP4_CT_SOURCE_SPEC, _NVFP4_SOURCE_SPEC
+ from freetoken.models.nvfp4_banks import _ingest_global
+
+ g = torch.tensor(4.0)
+ assert _ingest_global(_NVFP4_CT_SOURCE_SPEC, g).item() == 0.25
+ assert _ingest_global(_NVFP4_SOURCE_SPEC, g).item() == 4.0
diff --git a/tests/models/test_glm5_next_kda_op.py b/tests/models/test_glm5_next_kda_op.py
new file mode 100644
index 00000000..11adbf11
--- /dev/null
+++ b/tests/models/test_glm5_next_kda_op.py
@@ -0,0 +1,258 @@
+"""Glm5NextKDA op vs an eager reference (projection/conv/gate/norm wiring).
+
+The kernel math itself is validated in tests/kernels/test_kda.py; this test checks
+the OP-level wiring: the fused in_proj split (q|k|v|b|f_a|g_a), the merged q|k|v
+depthwise causal conv (+silu) against the state pool, the low-rank f/g gates, the
+sigmoid-gated output RMSNorm, and prefill -> decode state continuity through
+``LinearStatePool``.
+"""
+
+from __future__ import annotations
+
+from types import SimpleNamespace
+
+import pytest
+import torch
+
+pytestmark = pytest.mark.skipif(not torch.cuda.is_available(), reason="needs CUDA")
+
+
[email protected](autouse=True)
+def _single_rank_tp():
+ from freetoken.distributed import set_tp_info, try_get_tp_info
+
+ if try_get_tp_info() is None:
+ set_tp_info(rank=0, size=1)
+
+HIDDEN, H, D, KERNEL = 256, 4, 128, 4
+P = H * D
+LOWER_BOUND = -5.0
+
+
+def _make_args():
+ from freetoken.models.glm5_next.args import Glm5NextArgs
+
+ n_layers = 2
+ return Glm5NextArgs(
+ hidden_size=HIDDEN, num_heads=8,
+ q_lora_rank=64, kv_lora_rank=32, qk_nope_head_dim=32, qk_rope_head_dim=0,
+ v_head_dim=32, mla_nope=True, norm_eps=1e-5, max_position=4096,
+ index_n_heads=0, index_head_dim=0, index_topk=0, indexer_types=(),
+ indexer_rope_interleave=True, index_kpool=1, index_kpool_compress=False,
+ index_kpool_always_select_tail=False,
+ linear_num_heads=H, linear_head_dim=D, linear_conv_kernel_dim=KERNEL,
+ linear_lower_bound=LOWER_BOUND,
+ layer_types=("linear_attention",) * n_layers,
+ mlp_layer_types=("dense",) * n_layers,
+ mhc=False, mhc_num_residual_streams=1, hc_eps=1e-6,
+ mhc_sinkhorn_iterations=0, mhc_tau=0.05, mhc_post_mult_value=2.0,
+ mhc_no_norm_weight=False, swiglu_limit=None, rope_theta=10000.0,
+ )
+
+
+def _make_op(seed=0):
+ from freetoken.models.glm5_next.kda import Glm5NextKDA
+
+ cfg = SimpleNamespace(glm5_args=_make_args(), attn_quant="none")
+ op = Glm5NextKDA(cfg, layer_id=0)
+ torch.manual_seed(seed)
+ dev, dt = "cuda", torch.bfloat16
+ op.in_proj.weight = torch.randn(3 * P + H + 2 * D, HIDDEN, device=dev, dtype=dt) * 0.05
+ op.f_b_proj.weight = torch.randn(P, D, device=dev, dtype=dt) * 0.05
+ op.g_b_proj.weight = torch.randn(P, D, device=dev, dtype=dt) * 0.05
+ op.conv1d.weight = torch.randn(3 * P, 1, KERNEL, device=dev, dtype=dt) * 0.2
+ op.A_log = torch.randn(H, device=dev, dtype=torch.float32) * 0.5
+ op.dt_bias = torch.randn(P, device=dev, dtype=torch.float32) * 0.5
+ op.o_norm.weight = torch.randn(D, device=dev, dtype=dt) * 0.1 + 1.0
+ op.o_proj.weight = torch.randn(HIDDEN, P, device=dev, dtype=dt) * 0.05
+ return op
+
+
+def _make_pool(num_slots=4):
+ from freetoken.kvcache.linear_state_pool import LinearStatePool
+ from freetoken.models.config import LinearGatedDeltaGroupConfig
+
+ group = LinearGatedDeltaGroupConfig(
+ name="linear", layer_ids=(0,),
+ num_key_heads=H, num_value_heads=H, key_head_dim=D, value_head_dim=D,
+ conv_kernel_dim=KERNEL, output_gate=True, variant="kda",
+ )
+ return LinearStatePool(
+ group, num_slots, dtype=torch.bfloat16,
+ device=torch.device("cuda"), tp_size=1,
+ )
+
+
+def _patch_ctx(monkeypatch, pool, batch):
+ ctx = SimpleNamespace(batch=batch, linear_state_pool=pool)
+ monkeypatch.setattr(
+ "freetoken.models.glm5_next.kda.get_global_ctx", lambda: ctx
+ )
+
+
+def _l2norm(x):
+ return x / torch.sqrt((x * x).sum(-1, keepdim=True) + 1e-6)
+
+
+def _reference_forward(op, x_seq, conv_ctx=None, h0=None):
+ """Eager op reference for one sequence [T, HIDDEN] (fp32 where the kernels are
+ fp32). Returns (out [T, HIDDEN], conv_tail [3P, KERNEL-1], state [H, D, D])."""
+ T = x_seq.shape[0]
+ proj = x_seq.to(torch.bfloat16) @ op.in_proj.weight.T
+ conv_in, b, f_a, g_a = torch.split(proj, [3 * P, H, D, D], dim=-1)
+ g1 = (f_a @ op.f_b_proj.weight.T).float()
+ g2 = g_a @ op.g_b_proj.weight.T
+
+ # depthwise causal conv + silu over the merged q|k|v stream, with optional
+ # left-context from a previous chunk (conv state semantics).
+ w = op.conv1d.weight.squeeze(1).float() # [3P, KERNEL]
+ stream = conv_in.T.float() # [3P, T]
+ left = (
+ conv_ctx.float()
+ if conv_ctx is not None
+ else torch.zeros(3 * P, KERNEL - 1, device=x_seq.device)
+ )
+ padded = torch.cat([left, stream], dim=1) # [3P, KERNEL-1+T]
+ conv = torch.stack(
+ [(padded[:, t : t + KERNEL] * w).sum(-1) for t in range(T)], dim=1
+ )
+ mixed = torch.nn.functional.silu(conv).T # [T, 3P]
+ conv_tail = padded[:, -(KERNEL - 1):]
+
+ q, k, v = (t.reshape(T, H, D) for t in torch.split(mixed, [P, P, P], dim=-1))
+ # bf16 round-trip like the op (kernel inputs are bf16)
+ q, k, v = q.to(torch.bfloat16).float(), k.to(torch.bfloat16).float(), v.to(torch.bfloat16).float()
+
+ h = h0.clone() if h0 is not None else torch.zeros(H, D, D, device=x_seq.device)
+ amp = op.A_log.exp().view(H, 1)
+ bias = op.dt_bias.view(H, D)
+ core = []
+ for t in range(T):
+ gk = LOWER_BOUND * torch.sigmoid(amp * (g1[t].view(H, D) + bias))
+ h = h * gk.exp().unsqueeze(1)
+ kt = _l2norm(k[t])
+ v_err = (v[t] - torch.einsum("hvk,hk->hv", h, kt)) * torch.sigmoid(
+ b[t].float()
+ ).unsqueeze(-1)
+ h = h + torch.einsum("hv,hk->hvk", v_err, kt)
+ core.append(torch.einsum("hvk,hk->hv", h, _l2norm(q[t]) * D**-0.5))
+ core = torch.stack(core) # [T, H, D]
+
+ xn = core.reshape(-1, D)
+ rms = xn * torch.rsqrt(xn.pow(2).mean(-1, keepdim=True) + op.o_norm.eps)
+ gated = rms * op.o_norm.weight.float() * torch.sigmoid(g2.reshape(-1, D).float())
+ out = gated.reshape(T, P).to(torch.bfloat16) @ op.o_proj.weight.T
+ return out.float(), conv_tail, h
+
+
+def _fla(cu, indices, has_init=None, fresh=None):
+ from freetoken.attention.linear import FLAMetadata
+
+ return FLAMetadata(
+ cu_seqlens=cu, cache_indices=indices,
+ has_initial_state=has_init, fresh_state_indices=fresh,
+ )
+
+
+def _assert_close(ours, ref, tag, atol=3e-2):
+ err = (ours.float() - ref.float()).abs().max().item()
+ scale = ref.float().abs().max().item() + 1e-8
+ assert err / scale < atol, f"{tag}: max abs err {err:.5f} (ref scale {scale:.3f})"
+
+
+def test_prefill_matches_reference(monkeypatch):
+ op = _make_op()
+ pool = _make_pool()
+ lens = [33, 70]
+ total = sum(lens)
+ torch.manual_seed(10)
+ x = torch.randn(total, HIDDEN, device="cuda", dtype=torch.bfloat16)
+
+ cu = torch.tensor([0, *torch.tensor(lens).cumsum(0).tolist()], dtype=torch.int32, device="cuda")
+ indices = torch.tensor([1, 2], dtype=torch.int32, device="cuda")
+ has_init = torch.tensor([False, False], device="cuda")
+ fresh = torch.tensor([1, 2], dtype=torch.int64, device="cuda")
+ batch = SimpleNamespace(is_decode=False, fla_metadata=_fla(cu, indices, has_init, fresh))
+ _patch_ctx(monkeypatch, pool, batch)
+
+ out = op.forward(x)
+
+ start = 0
+ for i, ln in enumerate(lens):
+ sl = slice(start, start + ln)
+ ref_out, ref_conv, ref_h = _reference_forward(op, x[sl].float())
+ _assert_close(out[sl], ref_out, f"seq{i} prefill out")
+ _assert_close(pool.recurrent_states[0, i + 1], ref_h, f"seq{i} state")
+ _assert_close(pool.conv_states[0, i + 1], ref_conv, f"seq{i} conv state")
+ start += ln
+
+
+def test_prefill_then_decode_continuity(monkeypatch):
+ op = _make_op(seed=1)
+ pool = _make_pool()
+ T0, T1 = 40, 3
+ torch.manual_seed(11)
+ x = torch.randn(T0 + T1, HIDDEN, device="cuda", dtype=torch.bfloat16)
+ ref_out, _, _ = _reference_forward(op, x.float())
+
+ cu = torch.tensor([0, T0], dtype=torch.int32, device="cuda")
+ indices = torch.tensor([1], dtype=torch.int32, device="cuda")
+ batch = SimpleNamespace(
+ is_decode=False,
+ fla_metadata=_fla(
+ cu, indices,
+ torch.tensor([False], device="cuda"),
+ torch.tensor([1], dtype=torch.int64, device="cuda"),
+ ),
+ )
+ _patch_ctx(monkeypatch, pool, batch)
+ out0 = op.forward(x[:T0])
+ _assert_close(out0, ref_out[:T0], "prefill out")
+
+ for t in range(T0, T0 + T1):
+ batch = SimpleNamespace(
+ is_decode=True,
+ fla_metadata=_fla(
+ torch.tensor([0, 1], dtype=torch.int32, device="cuda"), indices
+ ),
+ )
+ _patch_ctx(monkeypatch, pool, batch)
+ out_t = op.forward(x[t : t + 1])
+ _assert_close(out_t[0], ref_out[t], f"decode token {t}")
+
+
+def test_chunked_prefill_continuation(monkeypatch):
+ """Second prefill chunk with has_initial_state=True must continue conv AND
+ recurrent state exactly (the chunked-prefill path)."""
+ op = _make_op(seed=2)
+ pool = _make_pool()
+ T0, T1 = 64, 30
+ torch.manual_seed(12)
+ x = torch.randn(T0 + T1, HIDDEN, device="cuda", dtype=torch.bfloat16)
+ ref_out, ref_conv, ref_h = _reference_forward(op, x.float())
+
+ indices = torch.tensor([1], dtype=torch.int32, device="cuda")
+ batch = SimpleNamespace(
+ is_decode=False,
+ fla_metadata=_fla(
+ torch.tensor([0, T0], dtype=torch.int32, device="cuda"), indices,
+ torch.tensor([False], device="cuda"),
+ torch.tensor([1], dtype=torch.int64, device="cuda"),
+ ),
+ )
+ _patch_ctx(monkeypatch, pool, batch)
+ out0 = op.forward(x[:T0])
+ _assert_close(out0, ref_out[:T0], "chunk0 out")
+
+ batch = SimpleNamespace(
+ is_decode=False,
+ fla_metadata=_fla(
+ torch.tensor([0, T1], dtype=torch.int32, device="cuda"), indices,
+ torch.tensor([True], device="cuda"), None,
+ ),
+ )
+ _patch_ctx(monkeypatch, pool, batch)
+ out1 = op.forward(x[T0:])
+ _assert_close(out1, ref_out[T0:], "chunk1 out")
+ _assert_close(pool.recurrent_states[0, 1], ref_h, "final state")
+ _assert_close(pool.conv_states[0, 1], ref_conv, "final conv state")
diff --git a/tests/models/test_glm5_next_kda_snapshot.py b/tests/models/test_glm5_next_kda_snapshot.py
new file mode 100644
index 00000000..e040b925
--- /dev/null
+++ b/tests/models/test_glm5_next_kda_snapshot.py
@@ -0,0 +1,98 @@
+"""KDA hybrid-radix track-snapshot contract (prefix caching for glm5_next).
+
+The scheduler snapshots each request's linear state at the deepest chunk-aligned
+(x64) boundary of a prefill into a donatable pool slot (FLAMetadata.track_*; the
+op writes it from the chunk kernel's per-chunk h + the raw conv window). A later
+request restores by copying that slot and continuing with
+``has_initial_state=True``. Checks, at the KDA-op level:
+
+* the snapshot equals the TRUE state after exactly 64 tokens (independent run)
+* restore + continuation reproduces the uninterrupted run's outputs
+* the 64-boundary is kpool-aligned by construction (64 % 4 == 0)
+"""
+
+from __future__ import annotations
+
+from types import SimpleNamespace
+
+import pytest
+import torch
+
+from tests.models.test_glm5_next_kda_op import ( # reuse the op harness
+ _make_op,
+ _make_pool,
+ _patch_ctx,
+ _fla,
+ _assert_close,
+ HIDDEN,
+)
+
+pytestmark = pytest.mark.skipif(not torch.cuda.is_available(), reason="needs CUDA")
+
+CHUNK = 64
+
+
[email protected](autouse=True)
+def _single_rank_tp():
+ from freetoken.distributed import set_tp_info, try_get_tp_info
+
+ if try_get_tp_info() is None:
+ set_tp_info(rank=0, size=1)
+
+
+def _prefill(op, pool, monkeypatch, x, slot, t0=0, has_init=False, track=None):
+ t = x.shape[0]
+ fla = _fla(
+ torch.tensor([0, t], dtype=torch.int32, device="cuda"),
+ torch.tensor([slot], dtype=torch.int32, device="cuda"),
+ torch.tensor([has_init], device="cuda"),
+ None if has_init else torch.tensor([slot], dtype=torch.int64, device="cuda"),
+ )
+ if track is not None:
+ fla.track_dst, fla.track_h_row, fla.track_conv_src = track
+ batch = SimpleNamespace(is_decode=False, fla_metadata=fla)
+ _patch_ctx(monkeypatch, pool, batch)
+ return op.forward(x)
+
+
+def test_snapshot_restore_roundtrip(monkeypatch):
+ from freetoken.kernel.fla.index import prepare_chunk_offsets
+
+ op = _make_op(seed=5)
+ pool = _make_pool(num_slots=6)
+ total = 100 # crosses one x64 boundary; tail 36 tokens
+ torch.manual_seed(20)
+ x = torch.randn(total, HIDDEN, device="cuda", dtype=torch.bfloat16)
+
+ # --- ground truth: state after exactly CHUNK tokens (independent run, slot 3)
+ _prefill(op, pool, monkeypatch, x[:CHUNK], slot=3)
+ true_rec = pool.recurrent_states[0, 3].clone()
+ true_conv = pool.conv_states[0, 3].clone()
+
+ # --- tracked run (slot 1, snapshot into slot 2), as _build_track_metadata would
+ km1 = pool.conv_states.shape[-1]
+ cu_host = torch.tensor([0, total], dtype=torch.int64)
+ boh = prepare_chunk_offsets(cu_host, CHUNK).tolist()
+ c = (total - 1) // CHUNK # deepest mid-chunk boundary: 1 -> position 64
+ track = (
+ torch.tensor([2], dtype=torch.int64, device="cuda"),
+ torch.tensor([boh[0] + c], dtype=torch.int64, device="cuda"),
+ torch.tensor([[c * CHUNK - km1 + j for j in range(km1)]], dtype=torch.int64, device="cuda"),
+ )
+ out_full = _prefill(op, pool, monkeypatch, x, slot=1, track=track)
+
+ _assert_close(pool.recurrent_states[0, 2], true_rec, "snapshot recurrent state")
+ _assert_close(pool.conv_states[0, 2], true_conv, "snapshot conv state")
+
+ # --- restore: copy snapshot -> fresh slot 4, continue [64, 100)
+ pool.copy_from(2, 4)
+ out_cont = _prefill(
+ op, pool, monkeypatch, x[CHUNK:], slot=4, t0=CHUNK, has_init=True
+ )
+ _assert_close(out_cont, out_full[CHUNK:], "restored continuation outputs")
+ _assert_close(
+ pool.recurrent_states[0, 4], pool.recurrent_states[0, 1], "final states agree"
+ )
+
+ # kpool alignment is subsumed by the x64 snapshot boundary.
+ assert CHUNK % 4 == 0
\ No newline at end of file
diff --git a/tests/models/test_glm5_next_model.py b/tests/models/test_glm5_next_model.py
new file mode 100644
index 00000000..9a499fba
--- /dev/null
+++ b/tests/models/test_glm5_next_model.py
@@ -0,0 +1,210 @@
+"""Glm5NextForCausalLM wiring smoke test (tiny random model, dense MLPs).
+
+The per-op math is covered elsewhere (KDA kernels/op, kpool backend, mHC); this
+test checks the ASSEMBLY: a 2-layer hybrid (KDA + DSA) model with mHC threading
+runs prefill and decode through the real backends/pools, and the strongest
+cache invariant holds -- decoding token T after prefilling [0, T) produces the
+same logits as prefilling [0, T] outright (state handoff across the KDA
+recurrent pool, the MLA latent pool, and the kpool indexer cache).
+"""
+
+from __future__ import annotations
+
+from types import SimpleNamespace
+
+import pytest
+import torch
+
+pytestmark = pytest.mark.skipif(not torch.cuda.is_available(), reason="needs CUDA")
+
+HIDDEN, VOCAB = 64, 128
+KDA_H, KDA_D = 2, 128 # KDA kernels specialize on D=128
+IDX_H, IDX_D = 16, 64
+LATENT = 32
+DEV = "cuda"
+
+
+def _hf_config():
+ from freetoken.utils.hf import RawConfigShim
+
+ text = {
+ "hidden_size": HIDDEN, "intermediate_size": 96, "num_hidden_layers": 2,
+ "num_attention_heads": 2, "vocab_size": VOCAB, "hidden_act": "silu",
+ "rms_norm_eps": 1e-5, "max_position_embeddings": 4096,
+ "tie_word_embeddings": False,
+ "q_lora_rank": 48, "kv_lora_rank": LATENT, "qk_nope_head_dim": 32,
+ "qk_rope_head_dim": 0, "v_head_dim": 32, "mla_use_nope": True,
+ "index_n_heads": IDX_H, "index_head_dim": IDX_D, "index_topk": 32,
+ "indexer_types": ["full", "full"], "indexer_rope_interleave": True,
+ "index_kpool": 4, "index_kpool_compress": True,
+ "index_kpool_always_select_tail": True,
+ "linear_attn_config": {
+ "num_heads": KDA_H, "head_dim": KDA_D,
+ "short_conv_kernel_size": 4, "gate_lower_bound": -5.0,
+ },
+ "layer_types": ["linear_attention", "deepseek_sparse_attention"],
+ "mlp_layer_types": ["dense", "dense"], # no MoE machinery in this test
+ "first_k_dense_replace": 2,
+ "mhc": True, "hc_mult": 4, "hc_eps": 1e-6, "hc_sinkhorn_iters": 20,
+ "n_routed_experts": 8, "num_experts_per_tok": 2, "n_shared_experts": 1,
+ "moe_intermediate_size": 32, "norm_topk_prob": True,
+ "routed_scaling_factor": 2.5, "scoring_func": "sigmoid",
+ "n_group": 1, "topk_group": 1, "swiglu_limit": 10.0,
+ "attention_bias": False, "model_type": "glm5_next_text",
+ }
+ return RawConfigShim({
+ "architectures": ["Glm5NextForConditionalGeneration"],
+ "model_type": "glm5_next", "text_config": text,
+ })
+
+
+def rig(monkeypatch):
+ from freetoken.attention.dsa_indexer_kpool import Glm5NextDSABackend
+ from freetoken.distributed import set_tp_info, try_get_tp_info
+ from freetoken.kvcache.dsa_pool import KpoolDSAKVCache
+ from freetoken.kvcache.linear_state_pool import LinearStatePool
+ from freetoken.models.glm5_next.config import parse_config
+ from freetoken.models.glm5_next.model import Glm5NextForCausalLM
+
+ if try_get_tp_info() is None:
+ set_tp_info(rank=0, size=1)
+ config = parse_config(_hf_config())
+
+ prev_dtype = torch.get_default_dtype()
+ torch.set_default_dtype(torch.bfloat16)
+ prev_dev = torch.get_default_device()
+ torch.set_default_device(DEV)
+ try:
+ model = Glm5NextForCausalLM(config)
+ finally:
+ torch.set_default_dtype(prev_dtype)
+ torch.set_default_device(prev_dev)
+
+ # Random weights via the state-dict round trip (keeps shapes/dtypes honest).
+ torch.manual_seed(0)
+ sd = model.state_dict()
+ rand = {}
+ for k, v in sd.items():
+ t = torch.randn(v.shape, dtype=torch.float32, device=DEV) * 0.05
+ if k.endswith("norm.weight") or ".o_norm.weight" in k:
+ t = t.abs() + 0.5
+ rand[k] = t.to(v.dtype)
+ model.load_state_dict(rand)
+
+ kv = KpoolDSAKVCache(
+ latent_dim=LATENT, num_layers=2, num_pages=4, page_size=64,
+ dtype=torch.bfloat16, device=torch.device(DEV),
+ index_head_dim=IDX_D, num_index_layers=1,
+ index_ratio=4, num_req_slots=4,
+ )
+ page_table = torch.full((2, 256), -1, dtype=torch.int32, device=DEV)
+ page_table[0] = torch.arange(256, dtype=torch.int32, device=DEV)
+ linear_pool = LinearStatePool(
+ config.linear_attention_group(), num_slots=4,
+ dtype=torch.bfloat16, device=torch.device(DEV), tp_size=1,
+ )
+
+ ctx = SimpleNamespace(
+ kv_cache=kv, page_table=page_table, linear_state_pool=linear_pool,
+ attn_backend=None, batch=None,
+ )
+ for mod in (
+ "freetoken.attention.dsa.get_global_ctx",
+ "freetoken.models.glm5_next.kda.get_global_ctx",
+ "freetoken.models.glm5_next.attention.get_global_ctx",
+ "freetoken.models.glm5_next.model.get_global_ctx",
+ "freetoken.layers.embedding.get_global_ctx",
+ ):
+ monkeypatch.setattr(mod, lambda: ctx)
+ ctx.attn_backend = Glm5NextDSABackend(config)
+ return model, ctx
+
+
+def _req(device_len, cached_len):
+ return SimpleNamespace(
+ table_idx=0, device_len=device_len, extend_len=device_len - cached_len,
+ cached_len=cached_len, linear_slot_idx=1, mamba_ping_pong=None,
+ )
+
+
+def _batch(ctx, ids, t0, phase):
+ from freetoken.attention.linear import FLAMetadata
+
+ t1 = t0 + len(ids)
+ is_decode = phase == "decode"
+ batch = SimpleNamespace(
+ phase=phase,
+ is_prefill=not is_decode, is_decode=is_decode, size=1,
+ reqs=[_req(t1, t0)], padded_reqs=[_req(t1, t0)],
+ input_ids=torch.tensor(ids, device=DEV),
+ positions=torch.arange(t0, t1, device=DEV),
+ out_loc=torch.arange(t0, t1, device=DEV),
+ active_table_idx=torch.tensor([0], device=DEV) if is_decode else None,
+ fla_metadata=FLAMetadata(
+ cu_seqlens=torch.tensor([0, len(ids)], dtype=torch.int32, device=DEV),
+ cache_indices=torch.tensor([1], dtype=torch.int32, device=DEV),
+ has_initial_state=None if is_decode else torch.tensor([t0 > 0], device=DEV),
+ fresh_state_indices=(
+ None if (is_decode or t0 > 0)
+ else torch.tensor([1], dtype=torch.int64, device=DEV)
+ ),
+ ),
+ mm_embeds=None,
+ )
+ ctx.batch = batch
+ ctx.attn_backend.prepare_metadata(batch)
+ return batch
+
+
+def _reset(ctx):
+ ctx.linear_state_pool.reset(1)
+ ctx.kv_cache._kv_buffer.zero_()
+ ctx.kv_cache._index_k_buffer.zero_()
+ ctx.kv_cache._tail_k.zero_()
+ ctx.kv_cache._tail_gate.zero_()
+
+
+def test_prefill_decode_consistency(rig):
+ model, ctx = rig
+ torch.manual_seed(1)
+ total = 24
+ ids = torch.randint(0, VOCAB, (total,)).tolist()
+
+ # One-shot prefill over the full sequence: last-token logits per position
+ # are only produced for the final token, so run it twice at different splits.
+ _reset(ctx)
+ _batch(ctx, ids, 0, "prefill")
+ full_logits = model.forward() # [1, VOCAB] logits of the last position
+ assert full_logits.shape == (1, VOCAB)
+ assert torch.isfinite(full_logits.float()).all()
+
+ # Prefill [0, total-1) then decode the last token: must match the one-shot run.
+ _reset(ctx)
+ _batch(ctx, ids[:-1], 0, "prefill")
+ model.forward()
+ _batch(ctx, ids[-1:], total - 1, "decode")
+ dec_logits = model.forward()
+ err = (dec_logits.float() - full_logits.float()).abs().max().item()
+ scale = full_logits.float().abs().max().item() + 1e-8
+ assert err / scale < 3e-2, f"decode/prefill divergence: {err} (scale {scale})"
+
+
+def test_chunked_prefill_consistency(rig):
+ model, ctx = rig
+ torch.manual_seed(2)
+ total = 28 # split 16 + 12; chunk boundary pool-aligned (16 % 4 == 0)
+ ids = torch.randint(0, VOCAB, (total,)).tolist()
+
+ _reset(ctx)
+ _batch(ctx, ids, 0, "prefill")
+ full_logits = model.forward()
+
+ _reset(ctx)
+ _batch(ctx, ids[:16], 0, "prefill")
+ model.forward()
+ _batch(ctx, ids[16:], 16, "prefill")
+ chunk_logits = model.forward()
+ err = (chunk_logits.float() - full_logits.float()).abs().max().item()
+ scale = full_logits.float().abs().max().item() + 1e-8
+ assert err / scale < 3e-2, f"chunked/one-shot divergence: {err} (scale {scale})"
diff --git a/tests/models/test_glm_dsa.py b/tests/models/test_glm_dsa.py
index 84d6c1b3..0dceabad 100644
--- a/tests/models/test_glm_dsa.py
+++ b/tests/models/test_glm_dsa.py
@@ -298,11 +298,15 @@ def test_backend_ragged_prefill_identity_and_selection():
q_pe = torch.randn(t, h, dr, device="cuda", dtype=torch.bfloat16)
c_kv = torch.randn(t, dv, device="cuda", dtype=torch.bfloat16)
k_rope = torch.randn(t, dr, device="cuda", dtype=torch.bfloat16)
- qkw = (torch.randn(t, idx_h, idx_d, device="cuda", dtype=torch.bfloat16),
- torch.randn(t, idx_d, device="cuda", dtype=torch.bfloat16),
- torch.randn(t, idx_h, device="cuda").abs())
+ from freetoken.attention.dsa import DSAIndexerInputs
- o0 = backend.mla_forward(q_nope, q_pe, c_kv, k_rope, 0, batch, indexer_qkw=qkw)
+ qkw = DSAIndexerInputs(
+ q=torch.randn(t, idx_h, idx_d, device="cuda", dtype=torch.bfloat16),
+ k=torch.randn(t, idx_d, device="cuda", dtype=torch.bfloat16),
+ w=torch.randn(t, idx_h, device="cuda").abs(),
+ )
+
+ o0 = backend.mla_forward(q_nope, q_pe, c_kv, k_rope, 0, batch, indexer_inputs=qkw)
# request A (kv <= topk): selection covers all live -> equals dense reference
q_cat = torch.cat([q_nope, q_pe], -1)
@@ -313,7 +317,7 @@ def test_backend_ragged_prefill_identity_and_selection():
assert (o0[j].float() - ref).abs().max().item() < 3e-2, f"A q{j}"
# request B (kv > topk): causal top-k reference from the same scoring math
- q_idx, k_idx, w = qkw
+ q_idx, k_idx, w = qkw.q, qkw.k, qkw.w
for j in (0, 11): # first and last of B's queries
row = 8 + j
pos = 88 + j
@@ -327,7 +331,7 @@ def test_backend_ragged_prefill_identity_and_selection():
# leader/follower: layer 1 (shared, no indexer) reuses layer 0's selection; with
# identical latent content its output must match layer 0's
- o1 = backend.mla_forward(q_nope, q_pe, c_kv, k_rope, 1, batch, indexer_qkw=None)
+ o1 = backend.mla_forward(q_nope, q_pe, c_kv, k_rope, 1, batch, indexer_inputs=None)
assert (o0.float() - o1.float()).abs().max().item() < 3e-2
# identity wiring (dense ablation): same batch through an MLAKVCache backend
@@ -338,7 +342,7 @@ def test_backend_ragged_prefill_identity_and_selection():
batch_d = SimpleNamespace(reqs=reqs, positions=positions, out_loc=out_loc,
active_table_idx=None, attn_metadata=None)
backend_d.prepare_metadata(batch_d)
- od = backend_d.mla_forward(q_nope, q_pe, c_kv, k_rope, 0, batch_d, indexer_qkw=None)
+ od = backend_d.mla_forward(q_nope, q_pe, c_kv, k_rope, 0, batch_d, indexer_inputs=None)
slab_d = ctx_d.kv_cache.latent_rows(0)
for j in range(8):
live = ctx_d.page_table[0, : 33 + j]