FreeToken brings datacenter-scale model serving to your desktop. Run massive models locally, fast and efficiently.
feat(glm5_next): support GLM-5.3-Flash (#332)
Xiaoze Fana2538a428baa
diff --git a/docs/models.md b/docs/models.mdindex 41b79cd6..c9499163 100644--- a/docs/models.md+++ b/docs/models.md@@ -7,6 +7,7 @@ for them; other checkpoints of the same architectures work too.| Model | HF checkpoints ||---|---|| DeepSeek-V4 | [deepseek-ai/DeepSeek-V4-Flash-0731](https://huggingface.co/deepseek-ai/DeepSeek-V4-Flash-0731) |+| GLM-5.3-Flash | [RedHatAI/GLM-5.3-Flash-NVFP4](https://huggingface.co/RedHatAI/GLM-5.3-Flash-NVFP4) || GLM-5.2 | [nvidia/GLM-5.2-NVFP4](https://huggingface.co/nvidia/GLM-5.2-NVFP4) || GLM-4.7 | [nvidia/GLM-4.7-NVFP4](https://huggingface.co/nvidia/GLM-4.7-NVFP4) || Qwen3.8-Flash-Next | [Qwen/Qwen3.8-Flash-Next-FP8](https://huggingface.co/Qwen/Qwen3.8-Flash-Next-FP8), [RadixArk/Qwen3.8-Flash-Next-NVFP4](https://huggingface.co/RadixArk/Qwen3.8-Flash-Next-NVFP4) |diff --git a/python/freetoken/attention/__init__.py b/python/freetoken/attention/__init__.pyindex 72b01c04..8a410ffe 100644--- a/python/freetoken/attention/__init__.py+++ b/python/freetoken/attention/__init__.py@@ -107,6 +107,11 @@ def create_dsv4_sparse_backend(config: ModelConfig):BackendInfo(supported_types=frozenset({AttnType.MLA, AttnType.DSA})),)def create_dsa_backend(config: ModelConfig):+ # MLA with a grouped index (index_ratio > 1) is the kpool indexer layout.+ if any(s.mla and s.index_ratio > 1 for s in config.kv_cache_group_specs()):+ from .dsa_indexer_kpool import Glm5NextDSABackend++ return Glm5NextDSABackend(config)from .dsa import DSAAttnBackendreturn DSAAttnBackend(config)diff --git a/python/freetoken/attention/dsa.py b/python/freetoken/attention/dsa.pyindex ff4fa7ca..398d1700 100644--- a/python/freetoken/attention/dsa.py+++ b/python/freetoken/attention/dsa.py@@ -31,7 +31,7 @@ split does not fit the generic ``forward(q, k, v)`` contract).from __future__ import annotationsfrom dataclasses import dataclass, field-from typing import TYPE_CHECKING, Dict, List, Tuple+from typing import TYPE_CHECKING, Dict, List, Tuple, NamedTupleimport torchfrom freetoken.core import Batch, get_global_ctx@@ -50,6 +50,32 @@ _PREFILL_SCORE_BYTES = 128 << 20_PREFILL_SCORE_CHUNK = 512+class KpoolPlan(NamedTuple):+ """glm5_next kpool: one forward's slab/ring routing (layer-invariant; the+ first DSA layer plans, the rest reuse -- Glm5NextDSABackend._plan_kpool_writes)."""++ cmp_rows: torch.Tensor # [T] shadow row (closing) or scratch row+ ring_rows: torch.Tensor # [T] tail-ring row, -1 = masked off+ ring_slots: torch.Tensor # [n_req] Req.table_idx+ token_to_req: torch.Tensor # [T]+ cu_seqlens: torch.Tensor # [n_req + 1]+++@dataclass(frozen=True)+class DSAIndexerInputs:+ """Per-forward indexer tensors, built by the MODEL's indexer module and passed+ per call (QSAIndexerInputs precedent): the backend holds no model parameters.+ ``gate``/``ape`` are the glm5_next kpool extras (None for GLM-5.2): the raw+ per-channel compression gate scores and the [kpool, Di] APE weight."""+ # fmt: off+ q: torch.Tensor # [T, Hi, Di]+ k: torch.Tensor # [T, Di]+ w: torch.Tensor # [T, Hi] fp32+ gate: torch.Tensor | None = None+ ape: torch.Tensor | None = None+ # fmt: on++@dataclassclass DSAMetadata(BaseAttnMetadata):# fmt: off@@ -63,6 +89,11 @@ class DSAMetadata(BaseAttnMetadata):kvlen: torch.Tensor | None = None# group leader layer -> (sel_rows, counts); only the LIVE leader is retainedsel: dict = field(default_factory=dict)+ # glm5_next kpool decode: per-request tail-ring slots (Req.table_idx). Under CUDA+ # graphs this is a view of the backend's STATIC buffer (restaged per replay by+ # _stage_decode, QSA ring_slots precedent); eager decode reads active_table_idx.+ ring_slots: torch.Tensor | None = None+ kpool_plan: "KpoolPlan | None" = None# fmt: ondef get_last_indices(self, bs: int) -> torch.Tensor:@@ -73,8 +104,7 @@ class DSAAttnBackend(DSAIndexerMixin, BaseAttnBackend):def __init__(self, config: ModelConfig) -> None:from freetoken.kvcache.dsa_pool import DSAKVCache, MLAKVCache- args = config.glm_dsa_args- assert args is not None, "dsa backend needs ModelConfig.glm_dsa_args (MLA dims)"+ args = self._model_args(config)self.config = configself.num_heads = config.num_qo_headsself.kv_lora_rank = args.kv_lora_rank@@ -99,15 +129,7 @@ class DSAAttnBackend(DSAIndexerMixin, BaseAttnBackend):self._leader: Dict[int, int] = {}self._idx_slot: Dict[int, int] = {}if self.dsa_enabled:- lead = None- # Capped to the SERVED layer count (dev num_layers overrides must not- # index slots past the pool the factory sized from the same cap).- for lid, kind in enumerate(args.indexer_types[: config.num_layers]):- if kind == "full":- lead = lid- self._idx_slot[lid] = len(self._idx_slot)- assert lead is not None, "indexer_types must start with a 'full' layer"- self._leader[lid] = lead+ self._build_index_slots(args, config)# decode staging (static buffers under CUDA graphs; eager decode builds# per-forward tensors in prepare_metadata instead)self._rows_buf: torch.Tensor | None = None@@ -115,6 +137,32 @@ class DSAAttnBackend(DSAIndexerMixin, BaseAttnBackend):self.max_seq_len = 0self.capture_bs: List[int] = []+ # ----- model-family hooks (overridden by the glm5_next kpool backend) ---------------+ def _model_args(self, config: ModelConfig):+ """The MLA/indexer dims payload. GLM-5.2 reads glm_dsa_args; glm5_next's+ subclass reads glm5_args (same duck-typed fields)."""+ args = config.glm_dsa_args+ assert args is not None, "dsa backend needs ModelConfig.glm_dsa_args (MLA dims)"+ return args++ def _build_index_slots(self, args, config: ModelConfig) -> None:+ """IndexShare (GLM-5.2): "full" layers own an indexer slot; followers reuse+ the most recent leader's selection."""+ lead = None+ # Capped to the SERVED layer count (dev num_layers overrides must not+ # index slots past the pool the factory sized from the same cap).+ for lid, kind in enumerate(args.indexer_types[: config.num_layers]):+ if kind == "full":+ lead = lid+ self._idx_slot[lid] = len(self._idx_slot)+ assert lead is not None, "indexer_types must start with a 'full' layer"+ self._leader[lid] = lead++ def _store_index(self, inputs: "DSAIndexerInputs", batch: Batch, layer_id: int) -> None:+ """Scatter this forward's index keys (kpool subclass adds gate scores and+ pool-completion compression)."""+ self.kvcache.store_index_k(inputs.k, batch.out_loc, self._idx_slot[layer_id])+def forward(self, q, k, v, layer_id, batch, attn_spec: AttentionSpec | None = None):raise NotImplementedError("MLA models use mla_forward(), not forward().")@@ -156,14 +204,14 @@ class DSAAttnBackend(DSAIndexerMixin, BaseAttnBackend):)def mla_forward(- self, q_nope, q_pe, c_kv, k_rope, layer_id, batch, indexer_qkw=None+ self, q_nope, q_pe, c_kv, k_rope, layer_id, batch, indexer_inputs=None) -> torch.Tensor:"""Store this forward's latent rows and attend over the paged latent history.``q_nope`` [T, H, kv_lora_rank] (kv_b-absorbed), ``q_pe`` [T, H, rope_dim],``c_kv`` [T, kv_lora_rank] / ``k_rope`` [T, rope_dim] (the pool scatters the- two latent halves). ``indexer_qkw`` = (q [T, Hi, Di], k [T, Di], w [T, Hi])- on full-indexer layers, None on shared layers. Returns [T, H, kv_lora_rank].+ two latent halves). ``indexer_inputs`` is a :class:`DSAIndexerInputs` on+ full-indexer layers, None on shared layers. Returns [T, H, kv_lora_rank]."""md = batch.attn_metadataassert isinstance(md, DSAMetadata)@@ -174,17 +222,17 @@ class DSAAttnBackend(DSAIndexerMixin, BaseAttnBackend):md.rows = self._decode_rows(batch).to(torch.int32)md.kvlen = md.kv_len_cpu.to(self.device, non_blocking=True)self.kvcache.store_kv(c_kv, k_rope, batch.out_loc, layer_id)- if self.dsa_enabled and indexer_qkw is not None:+ if self.dsa_enabled and indexer_inputs is not None:# Scatter index keys unconditionally: short prefills serve through the# identity path TODAY, but their keys must exist once decode passes topk.- self.kvcache.store_index_k(indexer_qkw[1], batch.out_loc, self._idx_slot[layer_id])+ self._store_index(indexer_inputs, batch, layer_id)if md.is_decode:- return self._decode(md, layer_id, q_nope, q_pe, indexer_qkw)- return self._prefill(md, layer_id, q_nope, q_pe, batch, indexer_qkw)+ return self._decode(md, layer_id, q_nope, q_pe, indexer_inputs)+ return self._prefill(md, layer_id, q_nope, q_pe, batch, indexer_inputs)# ----- decode (CUDA-graph capturable, single code path) ------------------------------ def _decode(self, md, layer_id, q_nope, q_pe, indexer_qkw) -> torch.Tensor:+ def _decode(self, md, layer_id, q_nope, q_pe, inputs) -> torch.Tensor:bs = q_nope.shape[0]rows, kvlen = md.rows, md.kvlenif not self.dsa_enabled:@@ -192,8 +240,8 @@ class DSAAttnBackend(DSAIndexerMixin, BaseAttnBackend):# whole row list, bounded by the device-side live length.sel, cnt = rows.view(bs, 1, -1), kvlen.view(bs, 1)else:- if indexer_qkw is not None:- q_idx, _, w = indexer_qkw+ if inputs is not None:+ q_idx, w = inputs.q, inputs.ws = self.dsa_decode_scores(q_idx, w, self._idx_slot[layer_id], rows, kvlen)k_sel = min(self.index_topk, s.shape[-1])picks = self.indexer_select_decode(@@ -212,15 +260,16 @@ class DSAAttnBackend(DSAIndexerMixin, BaseAttnBackend):# ----- prefill / extend (eager) ------------------------------------------------------def _select_prefill(self, slot: int, q_idx: torch.Tensor, w: torch.Tensor,- rows: torch.Tensor, positions: torch.Tensor,+ rows: torch.Tensor, positions: torch.Tensor, start_pos: int,) -> Tuple[torch.Tensor, torch.Tensor]:- """Per-request causal top-k: ([1, m, K] physical rows, [1, m] counts)."""+ """Per-request causal top-k: ([1, m, K] physical rows, [1, m] counts).+ ``start_pos`` is the request's first query position -- a host int+ (cached_len), so no device->host sync on the prefill path."""kv_len = rows.numel()k_all = self.kvcache.index_k_cache(slot).index_select(0, rows.long())k_sel = min(self.index_topk, kv_len)m = q_idx.shape[0]sel = torch.empty(m, k_sel, dtype=torch.int32, device=self.device)- start_pos = int(positions[0])# Bound the fp32 [chunk, kv_len] logits transient (worst case is capped by the# model's max_position: floor 16 x 1M x 4 B = 64 MB, see _PREFILL_SCORE_BYTES).chunk = max(16, min(_PREFILL_SCORE_CHUNK, _PREFILL_SCORE_BYTES // max(kv_len * 4, 1)))@@ -236,15 +285,16 @@ class DSAAttnBackend(DSAIndexerMixin, BaseAttnBackend):cnt = torch.clamp(positions + 1, max=k_sel).to(torch.int32)return sel.view(1, m, k_sel), cnt.view(1, m)- def _prefill(self, md, layer_id, q_nope, q_pe, batch, indexer_qkw) -> torch.Tensor:+ def _prefill(self, md, layer_id, q_nope, q_pe, batch, inputs) -> torch.Tensor:t = q_nope.shape[0]q_cat = torch.cat([q_nope, q_pe], dim=-1) # [T, H, 576]reqs = batch.padded_reqs if hasattr(batch, "padded_reqs") else batch.reqspage_table = get_global_ctx().page_tableqo = md.qo_indptr_cpu.tolist()- sparse = self.dsa_enabled and int(md.kv_len_cpu.max()) > self.index_topk- if sparse and indexer_qkw is not None:- q_idx, _, w = indexer_qkw+ kv_lens = md.kv_len_cpu.tolist() # one D2H for both the gate and start_pos+ sparse = self.dsa_enabled and max(kv_lens) > self.index_topk+ if sparse and inputs is not None:+ q_idx, w = inputs.q, inputs.wmd.sel.clear() # one live group leader at a timemd.sel[layer_id] = [self._select_prefill(@@ -252,6 +302,7 @@ class DSAAttnBackend(DSAIndexerMixin, BaseAttnBackend):q_idx[qo[i] : qo[i + 1]], w[qo[i] : qo[i + 1]],page_table[r.table_idx, : r.device_len],batch.positions[qo[i] : qo[i + 1]],+ kv_lens[i] - (qo[i + 1] - qo[i]), # cached_len == first position)for i, r in enumerate(reqs)]diff --git a/python/freetoken/attention/dsa_indexer_kpool.py b/python/freetoken/attention/dsa_indexer_kpool.pynew file mode 100644index 00000000..6743a143--- /dev/null+++ b/python/freetoken/attention/dsa_indexer_kpool.py@@ -0,0 +1,288 @@+"""glm5_next (GLM-5.3-Flash) kpool DSA backend: pooled-indexer addressing.++Extends the GLM-5.2 DSA backend with the kpool compression scheme: the indexer K+cache is scored at POOL granularity -- every ``index_kpool`` (4) consecutive+tokens fold into one entry, a per-channel ``softmax(gate + APE)``-weighted sum of+their raw keys -- so indexer compute and top-k shrink by 4x. Selection picks+``index_topk // kpool`` pools, each pool expands back to its constituent token+rows, and the request's trailing incomplete pool ("tail", up to kpool-1 tokens)+is force-included (``index_kpool_always_select_tail``). Selection widths are+therefore ``(kpool - 1) + select_k * kpool`` with ``-1`` gather-only sentinels;+the sparse-MLA kernel masks them.++Slab convention (KpoolDSAKVCache): the index slab is a 1/kpool SHADOW of the KV+pages -- a pool's entry lives at row ``token_slot // kpool`` (well-defined because+the engine pins ``page_size % kpool == 0``, so a pool never straddles a page).+Raw keys + gate scores of the in-progress pool live in per-request tail rings+written at ``pos % kpool``; a pool that closes reads its older members from the+ring (so a chunk may start mid-pool), and a decode step whose pool does NOT close+scatters its (garbage) pooled candidate into the request's scratch row -- an+unconditional, CUDA-graph-safe write that scoring never reads (QSA precedent).++Faithfulness: pooled entries are stored bf16 (the reference Hadamard-rotates and+fp8-quantizes them -- a memory device whose rotation cancels in the score); the+pooling softmax runs fp32, matching the reference kernel. Selection is a plain+top-k over pool scores plus the tail: the reference does NOT force-include the+query's own (complete) pool -- the model is trained with that scheme, so neither+do we. Sharing note: IndexShare does not exist here; every DSA layer owns its+indexer slot and is its own leader.+"""++from __future__ import annotations++from typing import TYPE_CHECKING, Tuple++import torch++from .dsa import DSAAttnBackend, DSAMetadata, KpoolPlan++if TYPE_CHECKING:+ from freetoken.core import Batch+ from freetoken.models import ModelConfig+++class Glm5NextDSABackend(DSAAttnBackend):+ def __init__(self, config: "ModelConfig") -> None:+ super().__init__(config)+ self._slots_buf: torch.Tensor | None = None # static graph buffer for ring_slots+ if self.dsa_enabled:+ from freetoken.kvcache.dsa_pool import KpoolDSAKVCache++ assert isinstance(self.kvcache, KpoolDSAKVCache), (+ "glm5_next kpool indexer needs the KpoolDSAKVCache (tail rings); "+ f"the pool factory built {type(self.kvcache).__name__}"+ )++ # ----- CUDA-graph decode staging ------------------------------------------------------+ # The tail rings are keyed by Req.table_idx; a captured decode step must read it+ # from a static buffer restaged per replay (rows/kvlen precedent in the parent).+ def init_capture_graph(self, max_seq_len: int, bs_list) -> None:+ super().init_capture_graph(max_seq_len, bs_list)+ self._slots_buf = torch.zeros(max(bs_list), dtype=torch.int64, device=self.device)++ def _stage_decode(self, batch: "Batch", bs: int, table_idx: torch.Tensor) -> None:+ super()._stage_decode(batch, bs, table_idx)+ self._slots_buf[:bs].copy_(table_idx)+ batch.attn_metadata.ring_slots = self._slots_buf[:bs]++ def reset_capture(self) -> None:+ super().reset_capture()+ self._slots_buf = None++ # ----- model-family hooks -----------------------------------------------------------+ def _model_args(self, config: "ModelConfig"):+ args = config.glm5_args+ assert args is not None, "kpool dsa backend needs ModelConfig.glm5_args"+ self.kpool = args.index_kpool+ assert self.kpool > 1 and args.index_kpool_compress, (+ "Glm5NextDSABackend serves the kpool-compressed indexer; a kpool=1 "+ "checkpoint should run the plain DSA backend"+ )+ assert args.index_topk % self.kpool == 0+ self.select_k = args.index_topk // self.kpool+ assert args.index_kpool_always_select_tail, (+ "tail force-inclusion is baked into the selection layout"+ )+ return args++ def _build_index_slots(self, args, config: "ModelConfig") -> None:+ # No IndexShare: every DSA layer owns its indexer and is its own leader.+ for lid in args.dsa_layer_ids:+ if lid >= config.num_layers:+ continue+ self._idx_slot[lid] = len(self._idx_slot)+ self._leader[lid] = lid++ # ----- store: fused single path (prefill AND decode, CUDA-graph capturable) ----------+ def _plan_kpool_writes(self, md, batch: "Batch", slot: int):+ """Per-token slab/ring routing for this forward; layer-invariant, so the+ first DSA layer computes it and the rest reuse it (QSA _plan_index_writes+ shape). Pure device arithmetic: no host sync, graph-capturable.++ Rebuilt at the FIRST indexer slot of every forward, never trusted across+ forwards: a capture batch runs its warmup and its capture through ONE+ metadata object, and a cached plan would bake the warmup's (non-graph-pool)+ tensor addresses into the graph (QSA precedent)."""+ if slot != 0 and md.kpool_plan is not None:+ return md.kpool_plan+ kp = self.kpool+ out_loc = batch.out_loc.to(torch.int64)+ positions = batch.positions.to(torch.int64)+ t = out_loc.numel()+ if md.is_decode:+ # One token per request. ring_slots is the backend's STATIC buffer under+ # graphs (restaged per replay in _stage_decode); eager reads the+ # scheduler-staged active_table_idx. arange shapes are fixed per capture.+ ring_slots = (+ md.ring_slots if md.ring_slots is not None else batch.active_table_idx+ ).to(torch.int64)+ token_to_req = torch.arange(t, device=self.device, dtype=torch.int32)+ cu_seqlens = torch.arange(t + 1, device=self.device, dtype=torch.int32)+ else:+ reqs = batch.padded_reqs if hasattr(batch, "padded_reqs") else batch.reqs+ ring_slots = torch.tensor(+ [r.table_idx for r in reqs], dtype=torch.int64, pin_memory=True+ ).to(self.device, non_blocking=True)+ cu_cpu = md.qo_indptr_cpu+ cu_seqlens = cu_cpu.to(self.device, non_blocking=True)+ token_to_req = torch.repeat_interleave(+ torch.arange(len(reqs), dtype=torch.int32),+ (cu_cpu[1:] - cu_cpu[:-1]).to(torch.int64),+ ).pin_memory().to(self.device, non_blocking=True)+ slots = ring_slots.index_select(0, token_to_req.to(torch.int64))+ # page_size % kpool == 0, so out_loc % kp == position % kp: a group closes+ # exactly on position % kp == kp - 1. Non-closing rows land in the request's+ # scratch row (never scored).+ closing = positions % kp == kp - 1+ cmp_rows = torch.where(+ closing, out_loc // kp, self.kvcache.cmp_scratch_base + slots+ ).to(torch.int32)+ # Ring refresh: only each request's last kp rows survive to the next forward+ # (one keeper per pos%kp residue -- deterministic, no write races).+ rows = torch.arange(t, device=self.device, dtype=torch.int64)+ ends = cu_seqlens.to(torch.int64).index_select(0, token_to_req.to(torch.int64) + 1)+ keep = rows >= ends - kp+ ring_row = slots * kp + positions % kp+ ring_rows = torch.where(keep, ring_row, torch.full_like(ring_row, -1)).to(+ torch.int32+ )+ md.kpool_plan = KpoolPlan(cmp_rows, ring_rows, ring_slots, token_to_req, cu_seqlens)+ return md.kpool_plan++ def _store_index(self, inputs, batch: "Batch", layer_id: int) -> None:+ """One fused kernel serves prefill and decode (layout and member+ resolution: module docstring); the ring refresh follows the read."""+ from freetoken.kernel.triton.kpool_compress import kpool_compress_store+ from freetoken.kernel.triton.qsa import qsa_store_rows++ md = batch.attn_metadata+ assert isinstance(md, DSAMetadata)+ assert inputs.gate is not None and inputs.ape is not None, (+ "kpool store needs DSAIndexerInputs.gate/ape from the model's indexer"+ )+ k, gate = inputs.k, inputs.gate.to(inputs.k.dtype)+ slot = self._idx_slot[layer_id]+ tail_k, tail_g = self.kvcache.tail_k(slot), self.kvcache.tail_gate(slot)+ plan = self._plan_kpool_writes(md, batch, slot)+ kpool_compress_store(+ k, gate,+ tail_k.view(-1, k.shape[-1]), tail_g.view(-1, k.shape[-1]),+ inputs.ape,+ plan.ring_slots, plan.token_to_req, plan.cu_seqlens, batch.positions,+ self.kvcache.index_k_cache(slot), plan.cmp_rows,+ self.kpool,+ )+ # After the compression read: the ring rows this forward overwrites are+ # exactly the ones a straddling group just consumed.+ qsa_store_rows(tail_k, plan.ring_rows, k)+ qsa_store_rows(tail_g, plan.ring_rows, gate)++ # ----- selection: pools -> token rows + tail ------------------------------------------+ def _expand_and_tail(+ self,+ picks: torch.Tensor, # [B, m, select_k] pool ids, -1 sentinel+ rows: torch.Tensor, # [B, W] or [B, m, W] position-ordered physical rows+ q_pos: torch.Tensor, # [B, m] query positions (token-granular)+ ) -> Tuple[torch.Tensor, torch.Tensor]:+ """Selected pools -> token rows, tail tokens appended FIRST (fixed kpool-1+ slots, -1 padded). Returns (sel [B, m, (kpool-1) + select_k*kpool] int32,+ cnt [B, m] int32)."""+ kp = self.kpool+ b, m, k_sel = picks.shape+ device = picks.device+ offs = torch.arange(kp, device=device)++ # History: pool pick p -> token positions p*kp + [0, kp).+ hist_pos = picks.unsqueeze(-1) * kp + offs # [B, m, k_sel, kp]+ hist_pos = torch.where(+ picks.unsqueeze(-1) < 0, hist_pos.new_full((), -1), hist_pos+ ).view(b, m, k_sel * kp)++ # Tail: positions [n_pools*kp, q_pos] of each query's own request/step.+ n_pools = (q_pos + 1) // kp # complete pools at this query+ tail_start = n_pools * kp+ toffs = torch.arange(kp - 1, device=device)+ tail_pos = tail_start.unsqueeze(-1) + toffs # [B, m, kp-1]+ tail_pos = torch.where(tail_pos <= q_pos.unsqueeze(-1), tail_pos, tail_pos.new_full((), -1))++ pos = torch.cat([tail_pos, hist_pos], dim=-1) # [B, m, width]+ rows_b = rows if rows.dim() == 3 else rows.unsqueeze(1).expand(b, m, -1)+ sel = rows_b.gather(-1, pos.clamp_min(0).long()).to(torch.int32)+ sel = torch.where(pos < 0, sel.new_full((), -1), sel)+ # Valid entries: the fixed tail slots + all expanded picked pools. -1+ # sentinels inside the bound are masked by the sparse kernel.+ # clamp_max (not minimum(new_tensor)): no H2D copy, CUDA-graph safe.+ cnt = ((kp - 1) + n_pools.clamp_max(k_sel) * kp).to(torch.int32)+ return sel, cnt++ def _decode(self, md, layer_id, q_nope, q_pe, inputs) -> torch.Tensor:+ bs = q_nope.shape[0]+ rows, kvlen = md.rows, md.kvlen+ if not self.dsa_enabled:+ return super()._decode(md, layer_id, q_nope, q_pe, inputs)+ if inputs is not None:+ q_idx, w = inputs.q, inputs.w+ kp = self.kpool+ n_pools = (kvlen // kp).to(torch.int32)+ # Pool p's shadow row = any member's token slot // kp (pools never+ # straddle pages): stride the position-ordered row snapshot down to+ # pool granularity, then divide into the shadow slab.+ pool_rows = (rows[:, kp - 1 :: kp] // kp).contiguous()+ s = self.dsa_decode_scores(q_idx, w, self._idx_slot[layer_id], pool_rows, n_pools)+ k_sel = min(self.select_k, s.shape[-1])+ picks = self.indexer_select_decode(+ s.view(bs, 1, -1), valid=n_pools, topk=k_sel, offset=0+ ) # [bs, 1, k_sel] pool ids+ sel, cnt = self._expand_and_tail(+ picks.transpose(0, 1), # [1, bs, k_sel]+ rows.unsqueeze(0), # [1, bs, W]+ (kvlen - 1).view(1, bs),+ )+ sel = sel.transpose(0, 1) # [bs, 1, width]+ cnt = cnt.view(bs, 1)+ md.sel.clear()+ md.sel[layer_id] = (sel, cnt)+ sel, cnt = md.sel[self._leader[layer_id]]+ q_cat = torch.cat([q_nope, q_pe], dim=-1).view(bs, 1, self.num_heads, self.latent_dim)+ o = self._attend(q_cat, layer_id, sel, cnt)+ return o.view(bs, self.num_heads, self.kv_lora_rank)++ def _select_prefill(+ self, slot: int, q_idx: torch.Tensor, w: torch.Tensor,+ rows: torch.Tensor, positions: torch.Tensor, start_pos: int,+ ) -> Tuple[torch.Tensor, torch.Tensor]:+ """Per-request causal top-k at pool granularity, expanded to token rows.+ ``start_pos`` is the request's cached_len (host int, no device sync)."""+ kp = self.kpool+ kv_len = rows.numel()+ n_pools_total = kv_len // kp+ pool_rows = rows[kp - 1 :: kp] // kp # shadow rows (token slot // kpool)+ k_pool = self.kvcache.index_k_cache(slot).index_select(0, pool_rows.long())+ k_sel = min(self.select_k, max(n_pools_total, 1))+ m = q_idx.shape[0]+ width = (kp - 1) + k_sel * kp+ if n_pools_total == 0:+ # No complete pool to score (a sub-kpool request riding a sparse batch):+ # the selection is the tail alone (select_prefill would return zero pick+ # columns and break the fixed width below).+ picks = torch.full((1, m, k_sel), -1, dtype=torch.int32, device=self.device)+ return self._expand_and_tail(picks, rows.view(1, -1), positions.view(1, -1))+ sel = torch.empty(m, width, dtype=torch.int32, device=self.device)+ cnt = torch.empty(m, dtype=torch.int32, device=self.device)+ chunk = 512+ for s0 in range(0, m, chunk):+ s1 = min(s0 + chunk, m)+ scores = self.dsa_prefill_logits(q_idx[s0:s1], k_pool, w[s0:s1])+ picks = self.indexer_select_prefill(+ scores.unsqueeze(0), start_pos=start_pos + s0, seqlen=s1 - s0,+ ratio=kp, topk=k_sel, offset=0,+ ) # [1, s1-s0, k_sel] pool ids+ sel_c, cnt_c = self._expand_and_tail(+ picks, rows.view(1, -1), positions[s0:s1].view(1, -1)+ )+ sel[s0:s1] = sel_c[0]+ cnt[s0:s1] = cnt_c[0]+ return sel.view(1, m, width), cnt.view(1, m)+++__all__ = ["Glm5NextDSABackend"]diff --git a/python/freetoken/engine/engine.py b/python/freetoken/engine/engine.pyindex 33b0d549..b5a6fa3b 100644--- a/python/freetoken/engine/engine.py+++ b/python/freetoken/engine/engine.py@@ -216,13 +216,19 @@ def _validate_attention_backend_choice(config, override, required: frozenset[Att"Use --attention-backend fi (or triton) instead.")- if required & {AttnType.MLA, AttnType.DSA} and config.page_size != 1:- # The MLA backend's row addressing (latent scatter, DSA index keys, sparse- # top-k page indices) assumes page_size == 1 throughout; reject explicitly- # like the SWA models do rather than corrupting addressing silently.- raise ValueError(- f"latent-KV MLA models require --page-size 1, got {config.page_size}."+ if required & {AttnType.MLA, AttnType.DSA}:+ # Plain MLA/DSA runs on page_size 1; the kpool indexer layout needs 64.+ _kpool_ratio = max(+ (s.index_ratio for s in model_config.kv_cache_group_specs() if s.mla),+ default=1,)+ want_page = 64 if _kpool_ratio > 1 else 1+ if config.page_size != want_page:+ logger.warning_rank0(+ f"Page size {config.page_size} is auto-adjusted to {want_page} "+ f"for latent-KV attention."+ )+ override("page_size", want_page)for part in backend_parts:info = attention_backend_info(part)@@ -1135,6 +1141,7 @@ def _resolve_cpu_layers(config: EngineConfig, num_moe_layers: int) -> frozenset[# expert activations the CPU MoE executor supports (csrc ActKind)_CPU_MOE_ACTS = ("silu", "swish", "gelu", "gelu_tanh", "gelu_pytorch_tanh", "swigluoai",+ "swiglu_clamp",)diff --git a/python/freetoken/kernel/aot_models.py b/python/freetoken/kernel/aot_models.pyindex 6268ae33..0ce249de 100644--- a/python/freetoken/kernel/aot_models.py+++ b/python/freetoken/kernel/aot_models.py@@ -274,6 +274,20 @@ SUPPORTED_MODELS: tuple[AotModel, ...] = (moe_intermediate_size=2048,expert_formats=_NVFP4_FORMATS,),+ AotModel(+ # GLM-5.3-Flash: hybrid KDA + NoPE-MLA/DSA (kpool indexer). Latent writes+ # go through torch scatter like GLM-5.2 (no paged-KV store groups); the+ # KDA conv/recurrent state lives in the LinearStatePool, not paged KV.+ name="RedHatAI/GLM-5.3-Flash-NVFP4",+ architecture="Glm5NextForCausalLM",+ arch_aliases=("Glm5NextForConditionalGeneration",),+ hidden_size=4096,+ kv_groups=(),+ top_k=8,+ moe_intermediate_size=2048,+ expert_formats=_NVFP4_FORMATS,+ aliases=("zai-org/GLM-5.3-Flash", "LibertAIDAI/GLM-5.3-Flash-NVFP4"),+ ),AotModel(# MiniMaxAI/MiniMax-M2.5 ships block-fp8, which has no expert-bank# provider for this arch on main -- the NVFP4 release is the servablediff --git a/python/freetoken/kernel/csrc/cpu_moe/cpu_moe_ext.cpp b/python/freetoken/kernel/csrc/cpu_moe/cpu_moe_ext.cppindex 880e8637..8349123b 100644--- a/python/freetoken/kernel/csrc/cpu_moe/cpu_moe_ext.cpp+++ b/python/freetoken/kernel/csrc/cpu_moe/cpu_moe_ext.cpp@@ -71,7 +71,15 @@ inline bf16_t f32_to_bf16(float f) {// MiniMax-M3): gate/up are combined jointly with the runtime alpha/limit// scalars, so it is handled in the do_pass1 epilogue (act_apply never sees it;// the mxfp4 kernel additionally fuses its own copy of the same math).-enum ActKind { ACT_SILU = 0, ACT_GELU = 1, ACT_GELU_TANH = 2, ACT_SWIGLUOAI = 3 };+// ACT_SWIGLU_CLAMP (GLM-5.3 "swiglu_limit") is the same clamped form WITHOUT+// the (up + 1) bias: clamp(gate, max=lim) * sigmoid(alpha*gate) * clamp(up, +-lim).+enum ActKind {+ ACT_SILU = 0,+ ACT_GELU = 1,+ ACT_GELU_TANH = 2,+ ACT_SWIGLUOAI = 3,+ ACT_SWIGLU_CLAMP = 4,+};inline float act_apply(int act, float x) {if (act == ACT_SILU) return x / (1.0f + std::exp(-x));@@ -1604,7 +1612,8 @@ struct CpuMoeExecutor {bf16_t* g_row = g_scratch.data() + ((size_t)tok * top_k + k) * I;const int i0 = static_cast<int>(ib) * IBLK;const int i1 = std::min(I, i0 + IBLK);- const bool swigluoai = act == ACT_SWIGLUOAI;+ const bool clamped = act == ACT_SWIGLUOAI || act == ACT_SWIGLU_CLAMP;+ const float up_bias = act == ACT_SWIGLUOAI ? 1.0f : 0.0f;const float lim = swiglu_limit, alpha = swiglu_alpha;for (int i = i0; i < i1; ++i) {// gate = row i, up = row I+i@@ -1612,14 +1621,15 @@ struct CpuMoeExecutor {gemm1_dot(gate_up_l, gu_packed_l, gu_scale_l, gu_global_l, e, i, x_row, xe, xo, xi8, xas) * w_in;float up = gemm1_dot(gate_up_l, gu_packed_l, gu_scale_l, gu_global_l, e, I + i, x_row,xe, xo, xi8, xas) * w_in;- if (swigluoai) {- // clamp(gate, max=lim) * sigmoid(alpha * gate) * (clamp(up, +-lim) + 1)- // -- same math as the mxfp4 kernel's fused epilogue (lim == +inf: no clamp).+ if (clamped) {+ // clamp(gate, max=lim) * sigmoid(alpha * gate) * (clamp(up, +-lim) + up_bias)+ // -- swigluoai carries the +1 up bias (gpt-oss/MiniMax); swiglu_clamp+ // (GLM-5.3) does not. lim == +inf: no clamp.if (gate > lim) gate = lim;if (up > lim) up = lim;else if (up < -lim) up = -lim;const float glu = gate / (1.0f + std::exp(-gate * alpha));- g_row[i] = f32_to_bf16(glu * (up + 1.0f));+ g_row[i] = f32_to_bf16(glu * (up + up_bias));} else {g_row[i] = f32_to_bf16(act_apply(act, gate) * up);}@@ -2146,5 +2156,5 @@ PYBIND11_MODULE(TORCH_EXTENSION_NAME, m) {// accepts id 3 without error and silently computes the wrong activation// (act_apply falls through to gelu_tanh); the probe turns a stale extension// into a loud rebuild instruction instead of wrong model outputs.- m.def("max_generic_act_id", []() { return static_cast<int>(ACT_SWIGLUOAI); });+ m.def("max_generic_act_id", []() { return static_cast<int>(ACT_SWIGLU_CLAMP); });}diff --git a/python/freetoken/kernel/fla/__init__.py b/python/freetoken/kernel/fla/__init__.pyindex e3fff018..4d532077 100644--- a/python/freetoken/kernel/fla/__init__.py+++ b/python/freetoken/kernel/fla/__init__.py@@ -21,15 +21,37 @@ Provenance: https://github.com/sgl-project/sglang, ``python/sglang/srt/layers/atstripped/inlined on vendoring). Keep ``chunk_delta_h.py``'s single fixed ``triton.Config`` — restoringupstream's multi-config autotune corrupts the in-place state pool. Tune via the env knobs``SGLANG_GDN_CHUNK_H_BV`` / ``SGLANG_GDN_CHUNK_H_NUM_WARPS`` / ``SGLANG_GDN_CHUNK_H_NUM_STAGES``.++KDA (GLM-5.3-Flash Kimi Delta Attention) kernels are vendored separately from vLLM's+``third_party/flash_linear_attention`` (same fla lineage): ``kda.py`` (chunked prefill ++fused gate cumsum + recurrent decode wrapper), ``kda_chunk_delta_h.py`` (exp2-gate chunk+recurrence), ``fused_recurrent.py`` (pool-indexed decode kernel with in-kernel KDA gate),+``solve_tril.py``. Public entry points:+- ``chunk_kda_with_fused_gate`` -- chunked prefill from raw gate logits; gathered initial+ state in, final state out (scatter back to the pool is the caller's job).+ WARNING: clobbers the ``v`` argument (the output is written into that buffer to save+ an allocation, as in vLLM where v is an ephemeral projection). Never pass a tensor+ that is read again afterwards.+- ``fused_recurrent_kda`` -- decode; per-slot state read/write via ``ssm_state_indices``,+ gate + beta-sigmoid + q/k l2norm computed in-kernel. The per-token state store reads+ ``ssm_state_indices`` as a CONTIGUOUS [N, T] block; materialize, never ``expand()``."""from freetoken.kernel.fla.chunk import chunk_gated_delta_rulefrom freetoken.kernel.fla.fused_sigmoid_gating_recurrent import (fused_sigmoid_gating_delta_rule_update,)+from freetoken.kernel.fla.kda import (+ chunk_kda_with_fused_gate,+ fused_kda_gate,+ fused_recurrent_kda,+)from freetoken.kernel.fla.layernorm_gated import rms_norm_gated__all__ = ["chunk_gated_delta_rule","fused_sigmoid_gating_delta_rule_update",+ "chunk_kda_with_fused_gate",+ "fused_kda_gate",+ "fused_recurrent_kda","rms_norm_gated",]diff --git a/python/freetoken/kernel/fla/fused_recurrent.py b/python/freetoken/kernel/fla/fused_recurrent.pynew file mode 100644index 00000000..cc15fef9--- /dev/null+++ b/python/freetoken/kernel/fla/fused_recurrent.py@@ -0,0 +1,659 @@+# Vendored from vLLM's third_party/flash_linear_attention (PR #53906, commit 933876c3),+# itself copied from the flash-linear-attention project (MIT, (c) 2023-2025 Songlin Yang,+# Yu Zhang). Imports are remapped onto freetoken.kernel.fla's shared helpers; keep this+# file in sync with upstream when pulling KDA kernel fixes.+# SPDX-License-Identifier: Apache-2.0+# SPDX-FileCopyrightText: Copyright contributors to the vLLM project+# SPDX-FileCopyrightText: Songlin Yang, Yu Zhang+#+# This file contains code copied from the flash-linear-attention project.+# The original source code was licensed under the MIT license and included+# the following copyright notice:+# Copyright (c) 2023-2025, Songlin Yang, Yu Zhang+# ruff: noqa: E501++import torch++import triton+import triton.language as tl++from .op import exp, log+++ {+ "USE_INITIAL_STATE": lambda args: args["h0"] is not None,+ "IS_VARLEN": lambda args: args["cu_seqlens"] is not None,+ "IS_CONTINUOUS_BATCHING": lambda args: args["ssm_state_indices"] is not None,+ "IS_SPEC_DECODING": lambda args: args["num_accepted_tokens"] is not None,+ }+)[email protected](do_not_specialize=["N", "T"])+def fused_recurrent_gated_delta_rule_fwd_kernel(+ q,+ k,+ v,+ g,+ beta,+ o,+ h0,+ ht,+ cu_seqlens,+ ssm_state_indices,+ num_accepted_tokens,+ a_log,+ g_bias,+ scale,+ N: tl.int64, # num of sequences+ T: tl.int64, # num of tokens+ B: tl.constexpr,+ H: tl.constexpr,+ HV: tl.constexpr,+ K: tl.constexpr,+ V: tl.constexpr,+ BK: tl.constexpr,+ BV: tl.constexpr,+ stride_init_state_token: tl.constexpr,+ stride_final_state_token: tl.constexpr,+ stride_indices_seq: tl.constexpr,+ stride_indices_tok: tl.constexpr,+ USE_INITIAL_STATE: tl.constexpr, # whether to use initial state+ INPLACE_FINAL_STATE: tl.constexpr, # whether to store final state inplace+ IS_BETA_HEADWISE: tl.constexpr, # whether beta is headwise vector or scalar,+ USE_QK_L2NORM_IN_KERNEL: tl.constexpr,+ IS_VARLEN: tl.constexpr,+ IS_CONTINUOUS_BATCHING: tl.constexpr,+ IS_SPEC_DECODING: tl.constexpr,+ IS_KDA: tl.constexpr,+ SIGMOID_BETA: tl.constexpr, # beta holds raw logits; sigmoid at fp32 load+ COMPUTE_GATE: tl.constexpr, # g holds raw logits; KDA gate computed in-kernel+ SAFE_GATE: tl.constexpr, # bounded gate variant (only branch implemented)+ LOWER_BOUND: tl.constexpr,+):+ i_k, i_v, i_nh = tl.program_id(0), tl.program_id(1), tl.program_id(2)+ i_n, i_hv = i_nh // HV, i_nh % HV+ i_h = i_hv // (HV // H)+ if IS_VARLEN:+ bos, eos = (+ tl.load(cu_seqlens + i_n).to(tl.int64),+ tl.load(cu_seqlens + i_n + 1).to(tl.int64),+ )+ all = T+ T = eos - bos+ else:+ bos, eos = i_n * T, i_n * T + T+ all = B * T++ if T == 0:+ # no tokens to process for this sequence+ return++ o_k = i_k * BK + tl.arange(0, BK)+ o_v = i_v * BV + tl.arange(0, BV)++ p_q = q + (bos * H + i_h) * K + o_k+ p_k = k + (bos * H + i_h) * K + o_k+ p_v = v + (bos * HV + i_hv) * V + o_v+ if IS_BETA_HEADWISE:+ p_beta = beta + (bos * HV + i_hv) * V + o_v+ else:+ p_beta = beta + bos * HV + i_hv++ if not IS_KDA:+ p_g = g + bos * HV + i_hv+ else:+ p_gk = g + (bos * HV + i_hv) * K + o_k++ # Per-head gate amplitude, hoisted out of the token loop (COMPUTE_GATE).+ if COMPUTE_GATE:+ b_a_log = tl.exp(tl.load(a_log + i_h).to(tl.float32))++ p_o = o + ((i_k * all + bos) * HV + i_hv) * V + o_v++ mask_k = o_k < K+ mask_v = o_v < V+ mask_h = mask_v[:, None] & mask_k[None, :]++ b_h = tl.zeros([BV, BK], dtype=tl.float32)+ if USE_INITIAL_STATE:+ if IS_CONTINUOUS_BATCHING:+ if IS_SPEC_DECODING:+ i_t = tl.load(num_accepted_tokens + i_n).to(tl.int64) - 1+ else:+ i_t = 0+ # Load state index and check for invalid entries+ state_idx = tl.load(ssm_state_indices + i_n * stride_indices_seq + i_t).to(+ tl.int64+ )+ # DIVERGENCE from upstream: vLLM treats slot 0 as its NULL_BLOCK_ID+ # sentinel (`state_idx <= 0`), which silently skips a real request that+ # keys state by raw table_idx == 0 (--cache-type naive). FreeToken's+ # GDN kernel has no sentinel; match it -- only negative ids are invalid.+ # Under hybrid, padded rows now read/write the reserved padding slot+ # instead of early-returning (a benign write-only sink, same as GDN).+ if state_idx < 0:+ return+ p_h0 = h0 + state_idx * stride_init_state_token+ else:+ p_h0 = h0 + bos * HV * V * K+ p_h0 = p_h0 + i_hv * V * K + o_v[:, None] * K + o_k[None, :]+ b_h += tl.load(p_h0, mask=mask_h, other=0).to(tl.float32)++ for i_t in range(0, T):+ b_q = tl.load(p_q, mask=mask_k, other=0).to(tl.float32)+ b_k = tl.load(p_k, mask=mask_k, other=0).to(tl.float32)+ b_v = tl.load(p_v, mask=mask_v, other=0).to(tl.float32)++ if USE_QK_L2NORM_IN_KERNEL:+ b_q = b_q / tl.sqrt(tl.sum(b_q * b_q) + 1e-6)+ b_k = b_k / tl.sqrt(tl.sum(b_k * b_k) + 1e-6)+ b_q = b_q * scale+ # [BV, BK]+ if not IS_KDA:+ b_g = tl.load(p_g).to(tl.float32)+ b_h *= exp(b_g)+ else:+ b_gk = tl.load(p_gk).to(tl.float32)+ if COMPUTE_GATE:+ # Replicates kda_gate_fwd_kernel's SAFE_GATE branch+ # bit-for-bit (same tl.exp, same fp32 math; the intermediate+ # gate value this replaces was stored/reloaded as fp32,+ # which is lossless): y = lb / (1 + exp(-exp(A)*(g+bias))).+ b_gk += tl.load(+ g_bias + i_h * K + o_k, mask=mask_k, other=0.0+ ).to(tl.float32)+ b_gk = LOWER_BOUND / (1.0 + tl.exp(-(b_a_log * b_gk)))+ b_h *= exp(b_gk[None, :])+ # [BV]+ b_v -= tl.sum(b_h * b_k[None, :], 1)+ if IS_BETA_HEADWISE:+ b_beta = tl.load(p_beta, mask=mask_v, other=0).to(tl.float32)+ else:+ b_beta = tl.load(p_beta).to(tl.float32)+ # Matches torch's `x.float().sigmoid()` pre-computation bit-for-bit+ # on the input side (bf16->fp32 is exact); only the sigmoid impl itself+ # can differ by <=1 ULP.+ if SIGMOID_BETA:+ b_beta = tl.sigmoid(b_beta)+ b_v *= b_beta+ # [BV, BK]+ b_h += b_v[:, None] * b_k[None, :]+ # [BV]+ b_o = tl.sum(b_h * b_q[None, :], 1)+ tl.store(p_o, b_o.to(p_o.dtype.element_ty), mask=mask_v)++ # keep the states for multi-query tokens+ if INPLACE_FINAL_STATE:+ # Load state index and check for invalid entries+ final_state_idx = tl.load(+ ssm_state_indices + i_n * stride_indices_seq + i_t+ ).to(tl.int64)+ # DIVERGENCE from upstream: no slot-0 sentinel (see the load-side note).+ if final_state_idx >= 0:+ p_ht = ht + final_state_idx * stride_final_state_token+ p_ht = p_ht + i_hv * V * K + o_v[:, None] * K + o_k[None, :]+ tl.store(p_ht, b_h.to(p_ht.dtype.element_ty), mask=mask_h)+ else:+ p_ht = ht + (bos + i_t) * stride_final_state_token+ p_ht = p_ht + i_hv * V * K + o_v[:, None] * K + o_k[None, :]+ tl.store(p_ht, b_h.to(p_ht.dtype.element_ty), mask=mask_h)++ p_q += H * K+ p_k += H * K+ p_o += HV * V+ p_v += HV * V+ if not IS_KDA:+ p_g += HV+ else:+ p_gk += HV * K+ p_beta += HV * (V if IS_BETA_HEADWISE else 1)+++def fused_recurrent_gated_delta_rule_fwd(+ q: torch.Tensor,+ k: torch.Tensor,+ v: torch.Tensor,+ g: torch.Tensor,+ beta: torch.Tensor,+ scale: float,+ initial_state: torch.Tensor,+ inplace_final_state: bool = True,+ cu_seqlens: torch.Tensor | None = None,+ ssm_state_indices: torch.Tensor | None = None,+ num_accepted_tokens: torch.Tensor | None = None,+ use_qk_l2norm_in_kernel: bool = False,+) -> tuple[torch.Tensor, torch.Tensor]:+ B, T, H, K, V = *k.shape, v.shape[-1]+ HV = v.shape[2]+ N = B if cu_seqlens is None else len(cu_seqlens) - 1+ BK, BV = triton.next_power_of_2(K), min(triton.next_power_of_2(V), 32)+ NK, NV = triton.cdiv(K, BK), triton.cdiv(V, BV)+ assert NK == 1, "NK > 1 is not supported yet"+ num_stages = 3+ num_warps = 1++ o = q.new_empty(NK, *v.shape)+ if inplace_final_state:+ final_state = initial_state+ else:+ final_state = q.new_empty(T, HV, V, K, dtype=initial_state.dtype)++ stride_init_state_token = initial_state.stride(0)+ stride_final_state_token = final_state.stride(0)++ if ssm_state_indices is None:+ stride_indices_seq, stride_indices_tok = 1, 1+ elif ssm_state_indices.ndim == 1:+ stride_indices_seq, stride_indices_tok = ssm_state_indices.stride(0), 1+ else:+ stride_indices_seq, stride_indices_tok = ssm_state_indices.stride()++ grid = (NK, NV, N * HV)+ fused_recurrent_gated_delta_rule_fwd_kernel[grid](+ q=q,+ k=k,+ v=v,+ g=g,+ beta=beta,+ o=o,+ h0=initial_state,+ ht=final_state,+ cu_seqlens=cu_seqlens,+ ssm_state_indices=ssm_state_indices,+ num_accepted_tokens=num_accepted_tokens,+ scale=scale,+ N=N,+ T=T,+ B=B,+ H=H,+ HV=HV,+ K=K,+ V=V,+ BK=BK,+ BV=BV,+ stride_init_state_token=stride_init_state_token,+ stride_final_state_token=stride_final_state_token,+ stride_indices_seq=stride_indices_seq,+ stride_indices_tok=stride_indices_tok,+ IS_BETA_HEADWISE=beta.ndim == v.ndim,+ USE_QK_L2NORM_IN_KERNEL=use_qk_l2norm_in_kernel,+ INPLACE_FINAL_STATE=inplace_final_state,+ IS_KDA=False,+ SIGMOID_BETA=False,+ a_log=None,+ g_bias=None,+ COMPUTE_GATE=False,+ SAFE_GATE=True,+ LOWER_BOUND=-5.0,+ num_warps=num_warps,+ num_stages=num_stages,+ )+ o = o.squeeze(0)+ return o, final_state+++def fused_recurrent_gated_delta_rule_packed_decode_kernel(+ mixed_qkv,+ a,+ b,+ A_log,+ dt_bias,+ o,+ h0,+ ht,+ ssm_state_indices,+ scale,+ stride_mixed_qkv_tok: tl.constexpr,+ stride_a_tok: tl.constexpr,+ stride_b_tok: tl.constexpr,+ stride_init_state_token: tl.constexpr,+ stride_final_state_token: tl.constexpr,+ stride_indices_seq: tl.constexpr,+ H: tl.constexpr,+ HV: tl.constexpr,+ K: tl.constexpr,+ V: tl.constexpr,+ BK: tl.constexpr,+ BV: tl.constexpr,+ SOFTPLUS_THRESHOLD: tl.constexpr,+ USE_QK_L2NORM_IN_KERNEL: tl.constexpr,+):+ i_v, i_nh = tl.program_id(0), tl.program_id(1)+ i_n, i_hv = i_nh // HV, i_nh % HV+ i_h = i_hv // (HV // H)++ o_k = tl.arange(0, BK)+ o_v = i_v * BV + tl.arange(0, BV)+ mask_k = o_k < K+ mask_v = o_v < V+ mask_h = mask_v[:, None] & mask_k[None, :]++ state_idx = tl.load(ssm_state_indices + i_n * stride_indices_seq).to(tl.int64)+ p_o = o + (i_n * HV + i_hv) * V + o_v++ # Skip if state index is invalid (NULL_BLOCK_ID=0)+ if state_idx <= 0:+ zero = tl.zeros([BV], dtype=tl.float32).to(p_o.dtype.element_ty)+ tl.store(p_o, zero, mask=mask_v)+ return++ p_h0 = h0 + state_idx * stride_init_state_token+ p_h0 = p_h0 + i_hv * V * K + o_v[:, None] * K + o_k[None, :]+ b_h = tl.load(p_h0, mask=mask_h, other=0).to(tl.float32)++ p_mixed = mixed_qkv + i_n * stride_mixed_qkv_tok+ q_off = i_h * K + o_k+ k_off = (H * K) + i_h * K + o_k+ v_off = (2 * H * K) + i_hv * V + o_v+ b_q = tl.load(p_mixed + q_off, mask=mask_k, other=0).to(tl.float32)+ b_k = tl.load(p_mixed + k_off, mask=mask_k, other=0).to(tl.float32)+ b_v = tl.load(p_mixed + v_off, mask=mask_v, other=0).to(tl.float32)++ if USE_QK_L2NORM_IN_KERNEL:+ b_q = b_q / tl.sqrt(tl.sum(b_q * b_q) + 1e-6)+ b_k = b_k / tl.sqrt(tl.sum(b_k * b_k) + 1e-6)+ b_q = b_q * scale++ a_val = tl.load(a + i_n * stride_a_tok + i_hv).to(tl.float32)+ b_val = tl.load(b + i_n * stride_b_tok + i_hv).to(tl.float32)+ A_log_val = tl.load(A_log + i_hv).to(tl.float32)+ dt_bias_val = tl.load(dt_bias + i_hv).to(tl.float32)+ x = a_val + dt_bias_val+ softplus_x = tl.where(x <= SOFTPLUS_THRESHOLD, tl.log(1.0 + tl.exp(x)), x)+ g_val = -tl.exp(A_log_val) * softplus_x+ beta_val = tl.sigmoid(b_val).to(b.dtype.element_ty).to(tl.float32)++ b_h *= exp(g_val)+ b_v -= tl.sum(b_h * b_k[None, :], 1)+ b_v *= beta_val+ b_h += b_v[:, None] * b_k[None, :]+ b_o = tl.sum(b_h * b_q[None, :], 1)+ tl.store(p_o, b_o.to(p_o.dtype.element_ty), mask=mask_v)++ p_ht = ht + state_idx * stride_final_state_token+ p_ht = p_ht + i_hv * V * K + o_v[:, None] * K + o_k[None, :]+ tl.store(p_ht, b_h.to(p_ht.dtype.element_ty), mask=mask_h)+++def fused_recurrent_gated_delta_rule_packed_decode(+ mixed_qkv: torch.Tensor,+ a: torch.Tensor,+ b: torch.Tensor,+ A_log: torch.Tensor,+ dt_bias: torch.Tensor,+ scale: float,+ initial_state: torch.Tensor,+ out: torch.Tensor,+ ssm_state_indices: torch.Tensor,+ use_qk_l2norm_in_kernel: bool = False,+) -> tuple[torch.Tensor, torch.Tensor]:+ if mixed_qkv.ndim != 2:+ raise ValueError(+ f"`mixed_qkv` must be a 2D tensor (got ndim={mixed_qkv.ndim})."+ )+ if mixed_qkv.stride(-1) != 1:+ raise ValueError("`mixed_qkv` must be contiguous in the last dim.")+ if a.ndim != 2 or b.ndim != 2:+ raise ValueError(+ f"`a` and `b` must be 2D tensors (got a.ndim={a.ndim}, b.ndim={b.ndim})."+ )+ if a.stride(-1) != 1 or b.stride(-1) != 1:+ raise ValueError("`a`/`b` must be contiguous in the last dim.")+ if A_log.ndim != 1 or dt_bias.ndim != 1:+ raise ValueError("`A_log`/`dt_bias` must be 1D tensors.")+ if A_log.stride(0) != 1 or dt_bias.stride(0) != 1:+ raise ValueError("`A_log`/`dt_bias` must be contiguous.")+ if ssm_state_indices.ndim != 1:+ raise ValueError(+ f"`ssm_state_indices` must be 1D for packed decode (got ndim={ssm_state_indices.ndim})."+ )+ if not out.is_contiguous():+ raise ValueError("`out` must be contiguous.")++ dev = mixed_qkv.device+ if (+ a.device != dev+ or b.device != dev+ or A_log.device != dev+ or dt_bias.device != dev+ or initial_state.device != dev+ or out.device != dev+ or ssm_state_indices.device != dev+ ):+ raise ValueError("All inputs must be on the same device.")++ B = mixed_qkv.shape[0]+ if a.shape[0] != B or b.shape[0] != B:+ raise ValueError(+ "Mismatched batch sizes: "+ f"mixed_qkv.shape[0]={B}, a.shape[0]={a.shape[0]}, b.shape[0]={b.shape[0]}."+ )+ if ssm_state_indices.shape[0] != B:+ raise ValueError(+ f"`ssm_state_indices` must have shape [B] (got {tuple(ssm_state_indices.shape)}; expected ({B},))."+ )++ if initial_state.ndim != 4:+ raise ValueError(+ f"`initial_state` must be a 4D tensor (got ndim={initial_state.ndim})."+ )+ if initial_state.stride(-1) != 1:+ raise ValueError("`initial_state` must be contiguous in the last dim.")+ HV, V, K = initial_state.shape[-3:]+ if a.shape[1] != HV or b.shape[1] != HV:+ raise ValueError(+ f"`a`/`b` must have shape [B, HV] with HV={HV} (got a.shape={tuple(a.shape)}, b.shape={tuple(b.shape)})."+ )+ if A_log.numel() != HV or dt_bias.numel() != HV:+ raise ValueError(+ f"`A_log` and `dt_bias` must have {HV} elements (got A_log.numel()={A_log.numel()}, dt_bias.numel()={dt_bias.numel()})."+ )+ if out.shape != (B, 1, HV, V):+ raise ValueError(+ f"`out` must have shape {(B, 1, HV, V)} (got out.shape={tuple(out.shape)})."+ )++ qkv_dim = mixed_qkv.shape[1]+ qk_dim = qkv_dim - HV * V+ if qk_dim <= 0 or qk_dim % 2 != 0:+ raise ValueError(+ f"Invalid packed `mixed_qkv` last dim={qkv_dim} for HV={HV}, V={V}."+ )+ q_dim = qk_dim // 2+ if q_dim % K != 0:+ raise ValueError(f"Invalid packed Q size {q_dim}: must be divisible by K={K}.")+ H = q_dim // K+ if H <= 0 or HV % H != 0:+ raise ValueError(+ f"Invalid head config inferred from mixed_qkv: H={H}, HV={HV}."+ )++ BK = triton.next_power_of_2(K)+ if triton.cdiv(K, BK) != 1:+ raise ValueError(+ f"Packed decode kernel only supports NK=1 (got K={K}, BK={BK})."+ )+ BV = min(triton.next_power_of_2(V), 32)+ num_stages = 3+ num_warps = 1++ stride_mixed_qkv_tok = mixed_qkv.stride(0)+ stride_a_tok = a.stride(0)+ stride_b_tok = b.stride(0)+ stride_init_state_token = initial_state.stride(0)+ stride_final_state_token = initial_state.stride(0)+ stride_indices_seq = ssm_state_indices.stride(0)++ NV = triton.cdiv(V, BV)+ grid = (NV, B * HV)+ fused_recurrent_gated_delta_rule_packed_decode_kernel[grid](+ mixed_qkv=mixed_qkv,+ a=a,+ b=b,+ A_log=A_log,+ dt_bias=dt_bias,+ o=out,+ h0=initial_state,+ ht=initial_state,+ ssm_state_indices=ssm_state_indices,+ scale=scale,+ stride_mixed_qkv_tok=stride_mixed_qkv_tok,+ stride_a_tok=stride_a_tok,+ stride_b_tok=stride_b_tok,+ stride_init_state_token=stride_init_state_token,+ stride_final_state_token=stride_final_state_token,+ stride_indices_seq=stride_indices_seq,+ H=H,+ HV=HV,+ K=K,+ V=V,+ BK=BK,+ BV=BV,+ SOFTPLUS_THRESHOLD=20.0,+ USE_QK_L2NORM_IN_KERNEL=use_qk_l2norm_in_kernel,+ num_warps=num_warps,+ num_stages=num_stages,+ )+ return out, initial_state+++class FusedRecurrentFunction(torch.autograd.Function):+ @staticmethod+ def forward(+ ctx,+ q: torch.Tensor,+ k: torch.Tensor,+ v: torch.Tensor,+ g: torch.Tensor,+ beta: torch.Tensor,+ scale: float,+ initial_state: torch.Tensor,+ inplace_final_state: bool = True,+ cu_seqlens: torch.Tensor | None = None,+ ssm_state_indices: torch.Tensor | None = None,+ num_accepted_tokens: torch.Tensor | None = None,+ use_qk_l2norm_in_kernel: bool = False,+ ):+ o, final_state = fused_recurrent_gated_delta_rule_fwd(+ q=q.contiguous(),+ k=k.contiguous(),+ v=v.contiguous(),+ g=g.contiguous(),+ beta=beta.contiguous(),+ scale=scale,+ initial_state=initial_state,+ inplace_final_state=inplace_final_state,+ cu_seqlens=cu_seqlens,+ ssm_state_indices=ssm_state_indices,+ num_accepted_tokens=num_accepted_tokens,+ use_qk_l2norm_in_kernel=use_qk_l2norm_in_kernel,+ )++ return o, final_state+++def fused_recurrent_gated_delta_rule(+ q: torch.Tensor,+ k: torch.Tensor,+ v: torch.Tensor,+ g: torch.Tensor,+ beta: torch.Tensor = None,+ scale: float = None,+ initial_state: torch.Tensor = None,+ inplace_final_state: bool = True,+ cu_seqlens: torch.Tensor | None = None,+ ssm_state_indices: torch.Tensor | None = None,+ num_accepted_tokens: torch.Tensor | None = None,+ use_qk_l2norm_in_kernel: bool = False,+) -> tuple[torch.Tensor, torch.Tensor]:+ r"""+ Args:+ q (torch.Tensor):+ queries of shape `[B, T, H, K]`.+ k (torch.Tensor):+ keys of shape `[B, T, H, K]`.+ v (torch.Tensor):+ values of shape `[B, T, HV, V]`.+ GVA is applied if `HV > H`.+ g (torch.Tensor):+ g (decays) of shape `[B, T, HV]`.+ beta (torch.Tensor):+ betas of shape `[B, T, HV]`.+ scale (Optional[int]):+ Scale factor for the RetNet attention scores.+ If not provided, it will default to `1 / sqrt(K)`. Default: `None`.+ initial_state (Optional[torch.Tensor]):+ Initial state of shape `[N, HV, V, K]` for `N` input sequences.+ For equal-length input sequences, `N` equals the batch size `B`.+ Default: `None`.+ inplace_final_state: bool:+ Whether to store the final state in-place to save memory.+ Default: `True`.+ cu_seqlens (torch.Tensor):+ Cumulative sequence lengths of shape `[N+1]` used for variable-length training,+ consistent with the FlashAttention API.+ ssm_state_indices (Optional[torch.Tensor]):+ Indices to map the input sequences to the initial/final states.+ num_accepted_tokens (Optional[torch.Tensor]):+ Number of accepted tokens for each sequence during decoding.++ Returns:+ o (torch.Tensor):+ Outputs of shape `[B, T, HV, V]`.+ final_state (torch.Tensor):+ Final state of shape `[N, HV, V, K]`.++ Examples::+ >>> import torch+ >>> import torch.nn.functional as F+ >>> from einops import rearrange+ >>> from fla.ops.gated_delta_rule import fused_recurrent_gated_delta_rule+ # inputs with equal lengths+ >>> B, T, H, HV, K, V = 4, 2048, 4, 8, 512, 512+ >>> q = torch.randn(B, T, H, K, device='cuda')+ >>> k = F.normalize(torch.randn(B, T, H, K, device='cuda'), p=2, dim=-1)+ >>> v = torch.randn(B, T, HV, V, device='cuda')+ >>> g = F.logsigmoid(torch.rand(B, T, HV, device='cuda'))+ >>> beta = torch.rand(B, T, HV, device='cuda').sigmoid()+ >>> h0 = torch.randn(B, HV, V, K, device='cuda')+ >>> o, ht = fused_gated_recurrent_delta_rule(+ q, k, v, g, beta,+ initial_state=h0,+ )+ # for variable-length inputs, the batch size `B` is expected to be 1 and `cu_seqlens` is required+ >>> q, k, v, g, beta = map(lambda x: rearrange(x, 'b t ... -> 1 (b t) ...'), (q, k, v, g, beta))+ # for a batch with 4 sequences, `cu_seqlens` with 5 start/end positions are expected+ >>> cu_seqlens = q.new_tensor([0, 2048, 4096, 6144, 8192], dtype=torch.int32)+ >>> o_var, ht_var = fused_gated_recurrent_delta_rule(+ q, k, v, g, beta,+ initial_state=h0,+ cu_seqlens=cu_seqlens+ )+ """+ if cu_seqlens is not None and q.shape[0] != 1:+ raise ValueError(+ f"The batch size is expected to be 1 rather than {q.shape[0]} when using `cu_seqlens`."+ f"Please flatten variable-length inputs before processing."+ )+ if scale is None:+ scale = k.shape[-1] ** -0.5+ else:+ assert scale > 0, "scale must be positive"+ if beta is None:+ beta = torch.ones_like(q[..., 0])+ o, final_state = FusedRecurrentFunction.apply(+ q,+ k,+ v,+ g,+ beta,+ scale,+ initial_state,+ inplace_final_state,+ cu_seqlens,+ ssm_state_indices,+ num_accepted_tokens,+ use_qk_l2norm_in_kernel,+ )+ return o, final_statediff --git a/python/freetoken/kernel/fla/kda.py b/python/freetoken/kernel/fla/kda.pynew file mode 100644index 00000000..ff724049--- /dev/null+++ b/python/freetoken/kernel/fla/kda.py@@ -0,0 +1,1396 @@+# SPDX-License-Identifier: Apache-2.0+# SPDX-FileCopyrightText: Copyright contributors to the vLLM project+# SPDX-FileCopyrightText: Songlin Yang, Yu Zhang+#+# This file contains code copied from the flash-linear-attention project.+# The original source code was licensed under the MIT license and included+# the following copyright notice:+# Copyright (c) 2023-2025, Songlin Yang, Yu Zhang+# ruff: noqa: E501+++# Vendored from vLLM's third_party/flash_linear_attention (PR #53906, commit 933876c3),+# itself copied from the flash-linear-attention project (MIT, (c) 2023-2025 Songlin Yang,+# Yu Zhang). Imports are remapped onto freetoken.kernel.fla's shared helpers; keep this+# file in sync with upstream when pulling KDA kernel fixes.+import torch+import triton+import triton.language as tl++from freetoken.kernel.fla.cumsum import chunk_local_cumsum+from freetoken.kernel.fla.fused_recurrent import (+ fused_recurrent_gated_delta_rule_fwd_kernel,+)+from freetoken.kernel.fla.index import prepare_chunk_indices+from freetoken.kernel.fla.kda_chunk_delta_h import chunk_gated_delta_rule_fwd_h+from freetoken.kernel.fla.l2norm import l2norm_fwd+from freetoken.kernel.fla.op import exp2, log+from freetoken.kernel.fla.solve_tril import solve_tril+from freetoken.kernel.fla.utils import FLA_CHUNK_SIZE, is_amd++RCP_LN2 = 1.4426950408889634 # 1 / ln(2)+cdiv = triton.cdiv+next_power_of_2 = triton.next_power_of_2++BT_LIST_AUTOTUNE = [32, 64, 128]+NUM_WARPS_AUTOTUNE = [2, 4, 8, 16] if is_amd else [4, 8, 16, 32]+++def fused_recurrent_kda_fwd(+ q: torch.Tensor,+ k: torch.Tensor,+ v: torch.Tensor,+ g: torch.Tensor,+ beta: torch.Tensor,+ scale: float,+ initial_state: torch.Tensor,+ inplace_final_state: bool = True,+ cu_seqlens: torch.Tensor | None = None,+ ssm_state_indices: torch.Tensor | None = None,+ num_accepted_tokens: torch.Tensor | None = None,+ use_qk_l2norm_in_kernel: bool = False,+ out: torch.Tensor | None = None,+ sigmoid_beta: bool = False,+ a_log: torch.Tensor | None = None,+ g_bias: torch.Tensor | None = None,+ compute_gate: bool = False,+ lower_bound: float | None = -5.0,+) -> tuple[torch.Tensor, torch.Tensor]:+ B, T, H, K, V = *k.shape, v.shape[-1]+ HV = v.shape[2]+ N = B if cu_seqlens is None else len(cu_seqlens) - 1+ BK, BV = next_power_of_2(K), min(next_power_of_2(V), 8)+ NK, NV = cdiv(K, BK), cdiv(V, BV)+ assert NK == 1, "NK > 1 is not supported yet"+ num_stages = 3+ num_warps = 1++ if compute_gate:+ assert a_log is not None and g_bias is not None, (+ "compute_gate requires a_log and g_bias"+ )+ assert lower_bound is not None, (+ "compute_gate implements the bounded (safe_gate) branch only"+ )+ a_log = a_log.reshape(-1).contiguous()+ g_bias = g_bias.reshape(-1).contiguous()++ if out is None:+ o = torch.empty_like(k)+ else:+ # Caller-provided output buffer; must be layout-compatible with the+ # tensor the kernel indexes (contiguous, same shape/dtype as k).+ assert out.shape == k.shape and out.dtype == k.dtype+ assert out.is_contiguous()+ o = out+ if inplace_final_state:+ final_state = initial_state+ else:+ final_state = q.new_empty(T, HV, V, K, dtype=initial_state.dtype)++ stride_init_state_token = initial_state.stride(0)+ stride_final_state_token = final_state.stride(0)++ if ssm_state_indices is None:+ stride_indices_seq, stride_indices_tok = 1, 1+ elif ssm_state_indices.ndim == 1:+ stride_indices_seq, stride_indices_tok = ssm_state_indices.stride(0), 1+ else:+ stride_indices_seq, stride_indices_tok = ssm_state_indices.stride()++ grid = (NK, NV, N * HV)+ fused_recurrent_gated_delta_rule_fwd_kernel[grid](+ q=q,+ k=k,+ v=v,+ g=g,+ beta=beta,+ o=o,+ h0=initial_state,+ ht=final_state,+ cu_seqlens=cu_seqlens,+ ssm_state_indices=ssm_state_indices,+ num_accepted_tokens=num_accepted_tokens,+ scale=scale,+ N=N,+ T=T,+ B=B,+ H=H,+ HV=HV,+ K=K,+ V=V,+ BK=BK,+ BV=BV,+ stride_init_state_token=stride_init_state_token,+ stride_final_state_token=stride_final_state_token,+ stride_indices_seq=stride_indices_seq,+ stride_indices_tok=stride_indices_tok,+ IS_BETA_HEADWISE=beta.ndim == v.ndim,+ USE_QK_L2NORM_IN_KERNEL=use_qk_l2norm_in_kernel,+ INPLACE_FINAL_STATE=inplace_final_state,+ IS_KDA=True,+ SIGMOID_BETA=sigmoid_beta,+ a_log=a_log,+ g_bias=g_bias,+ COMPUTE_GATE=compute_gate,+ SAFE_GATE=True,+ LOWER_BOUND=lower_bound if lower_bound is not None else -5.0,+ num_warps=num_warps,+ num_stages=num_stages,+ )++ return o, final_state+++def fused_recurrent_kda(+ q: torch.Tensor,+ k: torch.Tensor,+ v: torch.Tensor,+ g: torch.Tensor,+ beta: torch.Tensor = None,+ scale: float = None,+ initial_state: torch.Tensor = None,+ inplace_final_state: bool = True,+ use_qk_l2norm_in_kernel: bool = True,+ cu_seqlens: torch.Tensor | None = None,+ ssm_state_indices: torch.LongTensor | None = None,+ num_accepted_tokens: torch.Tensor | None = None,+ out: torch.Tensor | None = None,+ sigmoid_beta: bool = False,+ a_log: torch.Tensor | None = None,+ g_bias: torch.Tensor | None = None,+ compute_gate: bool = False,+ lower_bound: float | None = -5.0,+ **kwargs,+) -> tuple[torch.Tensor, torch.Tensor]:+ if cu_seqlens is not None and q.shape[0] != 1:+ raise ValueError(+ f"The batch size is expected to be 1 rather than {q.shape[0]} when using `cu_seqlens`."+ f"Please flatten variable-length inputs before processing."+ )+ if scale is None:+ scale = k.shape[-1] ** -0.5++ o, final_state = fused_recurrent_kda_fwd(+ q=q.contiguous(),+ k=k.contiguous(),+ v=v.contiguous(),+ g=g.contiguous(),+ beta=beta.contiguous(),+ scale=scale,+ initial_state=initial_state,+ inplace_final_state=inplace_final_state,+ cu_seqlens=cu_seqlens,+ ssm_state_indices=ssm_state_indices,+ num_accepted_tokens=num_accepted_tokens,+ use_qk_l2norm_in_kernel=use_qk_l2norm_in_kernel,+ out=out,+ sigmoid_beta=sigmoid_beta,+ a_log=a_log,+ g_bias=g_bias,+ compute_gate=compute_gate,+ lower_bound=lower_bound,+ )+ return o, final_state++[email protected]({"IS_VARLEN": lambda args: args["cu_seqlens"] is not None})+ configs=[+ triton.Config({"BK": BK}, num_warps=num_warps, num_stages=num_stages)+ for BK in [32, 64]+ for num_warps in [1, 2, 4, 8]+ for num_stages in [2, 3, 4]+ ],+ key=["BC"],+)[email protected](do_not_specialize=["T"])+def chunk_kda_scaled_dot_kkt_fwd_kernel_intra_sub_inter(+ q,+ k,+ g,+ beta,+ A,+ Aqk,+ scale,+ cu_seqlens,+ chunk_indices,+ T,+ H: tl.constexpr,+ K: tl.constexpr,+ BT: tl.constexpr,+ BC: tl.constexpr,+ BK: tl.constexpr,+ NC: tl.constexpr,+ IS_VARLEN: tl.constexpr,+):+ i_t, i_c, i_bh = tl.program_id(0), tl.program_id(1), tl.program_id(2)+ i_b, i_h = i_bh // H, i_bh % H+ i_i, i_j = i_c // NC, i_c % NC+ if IS_VARLEN:+ i_n, i_t = (+ tl.load(chunk_indices + i_t * 2).to(tl.int32),+ tl.load(chunk_indices + i_t * 2 + 1).to(tl.int32),+ )+ bos, eos = (+ tl.load(cu_seqlens + i_n).to(tl.int32),+ tl.load(cu_seqlens + i_n + 1).to(tl.int32),+ )+ T = eos - bos+ else:+ bos, eos = i_b * T, i_b * T + T++ if i_t * BT + i_i * BC >= T:+ return+ if i_i <= i_j:+ return++ q += (bos * H + i_h) * K+ k += (bos * H + i_h) * K+ g += (bos * H + i_h) * K+ A += (bos * H + i_h) * BT+ Aqk += (bos * H + i_h) * BT++ p_b = tl.make_block_ptr(+ beta + bos * H + i_h, (T,), (H,), (i_t * BT + i_i * BC,), (BC,), (0,)+ )+ b_b = tl.load(p_b, boundary_check=(0,))++ b_A = tl.zeros([BC, BC], dtype=tl.float32)+ b_Aqk = tl.zeros([BC, BC], dtype=tl.float32)+ for i_k in range(tl.cdiv(K, BK)):+ p_q = tl.make_block_ptr(+ q, (T, K), (H * K, 1), (i_t * BT + i_i * BC, i_k * BK), (BC, BK), (1, 0)+ )+ p_k = tl.make_block_ptr(+ k, (T, K), (H * K, 1), (i_t * BT + i_i * BC, i_k * BK), (BC, BK), (1, 0)+ )+ p_g = tl.make_block_ptr(+ g, (T, K), (H * K, 1), (i_t * BT + i_i * BC, i_k * BK), (BC, BK), (1, 0)+ )+ b_kt = tl.make_block_ptr(+ k, (K, T), (1, H * K), (i_k * BK, i_t * BT + i_j * BC), (BK, BC), (0, 1)+ )+ p_gk = tl.make_block_ptr(+ g, (K, T), (1, H * K), (i_k * BK, i_t * BT + i_j * BC), (BK, BC), (0, 1)+ )++ o_k = i_k * BK + tl.arange(0, BK)+ m_k = o_k < K+ # [BK,]+ b_gn = tl.load(g + (i_t * BT + i_i * BC) * H * K + o_k, mask=m_k, other=0)+ # [BC, BK]+ b_g = tl.load(p_g, boundary_check=(0, 1))+ b_k = tl.load(p_k, boundary_check=(0, 1)) * exp2(b_g - b_gn[None, :])+ # [BK, BC]+ b_gk = tl.load(p_gk, boundary_check=(0, 1))+ b_kt = tl.load(b_kt, boundary_check=(0, 1))+ # [BC, BC]+ b_ktg = b_kt * exp2(b_gn[:, None] - b_gk)+ b_A += tl.dot(b_k, b_ktg)++ b_q = tl.load(p_q, boundary_check=(0, 1))+ b_qg = b_q * exp2(b_g - b_gn[None, :]) * scale+ b_Aqk += tl.dot(b_qg, b_ktg)++ b_A *= b_b[:, None]++ p_A = tl.make_block_ptr(+ A, (T, BT), (H * BT, 1), (i_t * BT + i_i * BC, i_j * BC), (BC, BC), (1, 0)+ )+ tl.store(p_A, b_A.to(A.dtype.element_ty), boundary_check=(0, 1))+ p_Aqk = tl.make_block_ptr(+ Aqk, (T, BT), (H * BT, 1), (i_t * BT + i_i * BC, i_j * BC), (BC, BC), (1, 0)+ )+ tl.store(p_Aqk, b_Aqk.to(Aqk.dtype.element_ty), boundary_check=(0, 1))++[email protected]({"IS_VARLEN": lambda args: args["cu_seqlens"] is not None})+ configs=[triton.Config({}, num_warps=num_warps) for num_warps in [1, 2, 4, 8]],+ key=["BK", "BT"],+)[email protected](do_not_specialize=["T"])+def chunk_kda_scaled_dot_kkt_fwd_kernel_intra_sub_intra(+ q,+ k,+ g,+ beta,+ A,+ Aqk,+ scale,+ cu_seqlens,+ chunk_indices,+ T,+ H: tl.constexpr,+ K: tl.constexpr,+ BT: tl.constexpr,+ BC: tl.constexpr,+ BK: tl.constexpr,+ IS_VARLEN: tl.constexpr,+):+ i_t, i_i, i_bh = tl.program_id(0), tl.program_id(1), tl.program_id(2)+ i_b, i_h = i_bh // H, i_bh % H+ if IS_VARLEN:+ i_n, i_t = (+ tl.load(chunk_indices + i_t * 2).to(tl.int32),+ tl.load(chunk_indices + i_t * 2 + 1).to(tl.int32),+ )+ bos, eos = (+ tl.load(cu_seqlens + i_n).to(tl.int32),+ tl.load(cu_seqlens + i_n + 1).to(tl.int32),+ )+ T = eos - bos+ else:+ bos, eos = i_b * T, i_b * T + T++ if i_t * BT + i_i * BC >= T:+ return++ o_i = tl.arange(0, BC)+ o_k = tl.arange(0, BK)+ m_k = o_k < K+ m_A = (i_t * BT + i_i * BC + o_i) < T+ o_A = (bos + i_t * BT + i_i * BC + o_i) * H * BT + i_h * BT + i_i * BC++ p_q = tl.make_block_ptr(+ q + (bos * H + i_h) * K,+ (T, K),+ (H * K, 1),+ (i_t * BT + i_i * BC, 0),+ (BC, BK),+ (1, 0),+ )+ p_k = tl.make_block_ptr(+ k + (bos * H + i_h) * K,+ (T, K),+ (H * K, 1),+ (i_t * BT + i_i * BC, 0),+ (BC, BK),+ (1, 0),+ )+ p_g = tl.make_block_ptr(+ g + (bos * H + i_h) * K,+ (T, K),+ (H * K, 1),+ (i_t * BT + i_i * BC, 0),+ (BC, BK),+ (1, 0),+ )+ b_q = tl.load(p_q, boundary_check=(0, 1))+ b_k = tl.load(p_k, boundary_check=(0, 1))+ b_g = tl.load(p_g, boundary_check=(0, 1))++ p_b = beta + (bos + i_t * BT + i_i * BC + o_i) * H + i_h+ b_k = b_k * tl.load(p_b, mask=m_A, other=0)[:, None]++ p_kt = k + (bos + i_t * BT + i_i * BC) * H * K + i_h * K + o_k+ p_gk = g + (bos + i_t * BT + i_i * BC) * H * K + i_h * K + o_k++ for j in range(0, min(BC, T - i_t * BT - i_i * BC)):+ b_kt = tl.load(p_kt, mask=m_k, other=0).to(tl.float32)+ b_gk = tl.load(p_gk, mask=m_k, other=0).to(tl.float32)+ b_ktg = b_kt[None, :] * exp2(b_g - b_gk[None, :])+ b_A = tl.sum(b_k * b_ktg, 1)+ b_A = tl.where(o_i > j, b_A, 0.0)+ b_Aqk = tl.sum(b_q * b_ktg, 1)+ b_Aqk = tl.where(o_i >= j, b_Aqk * scale, 0.0)+ tl.store(A + o_A + j, b_A, mask=m_A)+ tl.store(Aqk + o_A + j, b_Aqk, mask=m_A)+ p_kt += H * K+ p_gk += H * K+++def chunk_kda_scaled_dot_kkt_fwd(+ q: torch.Tensor,+ k: torch.Tensor,+ gk: torch.Tensor | None = None,+ beta: torch.Tensor | None = None,+ scale: float | None = None,+ cu_seqlens: torch.Tensor | None = None,+ chunk_indices: torch.Tensor | None = None,+ chunk_size: int = FLA_CHUNK_SIZE,+ output_dtype: torch.dtype = torch.float32,+) -> tuple[torch.Tensor, torch.Tensor]:+ r"""+ Compute beta * K * K^T.++ Args:+ k (torch.Tensor):+ The key tensor of shape `[B, T, H, K]`.+ beta (torch.Tensor):+ The beta tensor of shape `[B, T, H]`.+ gk (torch.Tensor):+ The cumulative sum of the gate tensor of shape `[B, T, H, K]` applied to the key tensor. Default: `None`.+ cu_seqlens (torch.Tensor):+ The cumulative sequence lengths of the input tensor.+ Default: None+ chunk_size (int):+ The chunk size. Default: 64.+ output_dtype (torch.dtype):+ The dtype of the output tensor. Default: `torch.float32`++ Returns:+ beta * K * K^T of shape `[B, T, H, BT]` where `BT` is the chunk size.+ """+ B, T, H, K = k.shape+ assert K <= 256+ BT = chunk_size+ if chunk_indices is None and cu_seqlens is not None:+ chunk_indices = prepare_chunk_indices(cu_seqlens, BT)+ NT = cdiv(T, BT) if cu_seqlens is None else len(chunk_indices)++ BC = min(16, BT)+ NC = cdiv(BT, BC)+ BK = max(next_power_of_2(K), 16)+ A = torch.zeros(B, T, H, BT, device=k.device, dtype=output_dtype)+ Aqk = torch.zeros(B, T, H, BT, device=k.device, dtype=output_dtype)+ grid = (NT, NC * NC, B * H)+ chunk_kda_scaled_dot_kkt_fwd_kernel_intra_sub_inter[grid](+ q=q,+ k=k,+ g=gk,+ beta=beta,+ A=A,+ Aqk=Aqk,+ scale=scale,+ cu_seqlens=cu_seqlens,+ chunk_indices=chunk_indices,+ T=T,+ H=H,+ K=K,+ BT=BT,+ BC=BC,+ NC=NC,+ )++ grid = (NT, NC, B * H)+ chunk_kda_scaled_dot_kkt_fwd_kernel_intra_sub_intra[grid](+ q=q,+ k=k,+ g=gk,+ beta=beta,+ A=A,+ Aqk=Aqk,+ scale=scale,+ cu_seqlens=cu_seqlens,+ chunk_indices=chunk_indices,+ T=T,+ H=H,+ K=K,+ BT=BT,+ BC=BC,+ BK=BK,+ )+ return A, Aqk+++ {+ "STORE_QG": lambda args: args["qg"] is not None,+ "STORE_KG": lambda args: args["kg"] is not None,+ "IS_VARLEN": lambda args: args["cu_seqlens"] is not None,+ }+)+ configs=[+ triton.Config({}, num_warps=num_warps, num_stages=num_stages)+ for num_warps in [2, 4, 8]+ for num_stages in [2, 3, 4]+ ],+ key=["H", "K", "V", "BT", "BK", "BV", "IS_VARLEN"],+)[email protected](do_not_specialize=["T"])+def recompute_w_u_fwd_kernel(+ q,+ k,+ qg,+ kg,+ v,+ beta,+ w,+ u,+ A,+ gk,+ cu_seqlens,+ chunk_indices,+ T,+ H: tl.constexpr,+ K: tl.constexpr,+ V: tl.constexpr,+ BT: tl.constexpr,+ BK: tl.constexpr,+ BV: tl.constexpr,+ STORE_QG: tl.constexpr,+ STORE_KG: tl.constexpr,+ IS_VARLEN: tl.constexpr,+ DOT_PRECISION: tl.constexpr,+):+ i_t, i_bh = tl.program_id(0), tl.program_id(1)+ i_b, i_h = i_bh // H, i_bh % H+ if IS_VARLEN:+ i_n, i_t = (+ tl.load(chunk_indices + i_t * 2).to(tl.int32),+ tl.load(chunk_indices + i_t * 2 + 1).to(tl.int32),+ )+ bos, eos = (+ tl.load(cu_seqlens + i_n).to(tl.int32),+ tl.load(cu_seqlens + i_n + 1).to(tl.int32),+ )+ T = eos - bos+ else:+ bos, eos = i_b * T, i_b * T + T+ p_b = tl.make_block_ptr(beta + bos * H + i_h, (T,), (H,), (i_t * BT,), (BT,), (0,))+ b_b = tl.load(p_b, boundary_check=(0,))++ p_A = tl.make_block_ptr(+ A + (bos * H + i_h) * BT, (T, BT), (H * BT, 1), (i_t * BT, 0), (BT, BT), (1, 0)+ )+ b_A = tl.load(p_A, boundary_check=(0, 1))++ for i_v in range(tl.cdiv(V, BV)):+ p_v = tl.make_block_ptr(+ v + (bos * H + i_h) * V,+ (T, V),+ (H * V, 1),+ (i_t * BT, i_v * BV),+ (BT, BV),+ (1, 0),+ )+ p_u = tl.make_block_ptr(+ u + (bos * H + i_h) * V,+ (T, V),+ (H * V, 1),+ (i_t * BT, i_v * BV),+ (BT, BV),+ (1, 0),+ )+ b_v = tl.load(p_v, boundary_check=(0, 1))+ b_vb = (b_v * b_b[:, None]).to(b_v.dtype)+ b_u = tl.dot(b_A, b_vb, input_precision=DOT_PRECISION)+ tl.store(p_u, b_u.to(p_u.dtype.element_ty), boundary_check=(0, 1))++ for i_k in range(tl.cdiv(K, BK)):+ p_w = tl.make_block_ptr(+ w + (bos * H + i_h) * K,+ (T, K),+ (H * K, 1),+ (i_t * BT, i_k * BK),+ (BT, BK),+ (1, 0),+ )+ p_k = tl.make_block_ptr(+ k + (bos * H + i_h) * K,+ (T, K),+ (H * K, 1),+ (i_t * BT, i_k * BK),+ (BT, BK),+ (1, 0),+ )+ b_k = tl.load(p_k, boundary_check=(0, 1))+ b_kb = b_k * b_b[:, None]++ p_gk = tl.make_block_ptr(+ gk + (bos * H + i_h) * K,+ (T, K),+ (H * K, 1),+ (i_t * BT, i_k * BK),+ (BT, BK),+ (1, 0),+ )+ b_gk = tl.load(p_gk, boundary_check=(0, 1))+ b_kb *= exp2(b_gk)+ if STORE_QG:+ p_q = tl.make_block_ptr(+ q + (bos * H + i_h) * K,+ (T, K),+ (H * K, 1),+ (i_t * BT, i_k * BK),+ (BT, BK),+ (1, 0),+ )+ p_qg = tl.make_block_ptr(+ qg + (bos * H + i_h) * K,+ (T, K),+ (H * K, 1),+ (i_t * BT, i_k * BK),+ (BT, BK),+ (1, 0),+ )+ b_q = tl.load(p_q, boundary_check=(0, 1))+ b_qg = b_q * exp2(b_gk)+ tl.store(p_qg, b_qg.to(p_qg.dtype.element_ty), boundary_check=(0, 1))+ if STORE_KG:+ last_idx = min(i_t * BT + BT, T) - 1++ o_k = i_k * BK + tl.arange(0, BK)+ m_k = o_k < K+ b_gn = tl.load(+ gk + ((bos + last_idx) * H + i_h) * K + o_k, mask=m_k, other=0.0+ )+ b_kg = b_k * exp2(b_gn - b_gk)++ p_kg = tl.make_block_ptr(+ kg + (bos * H + i_h) * K,+ (T, K),+ (H * K, 1),+ (i_t * BT, i_k * BK),+ (BT, BK),+ (1, 0),+ )+ tl.store(p_kg, b_kg.to(p_kg.dtype.element_ty), boundary_check=(0, 1))++ b_w = tl.dot(b_A, b_kb.to(b_k.dtype))+ tl.store(p_w, b_w.to(p_w.dtype.element_ty), boundary_check=(0, 1))+++def recompute_w_u_fwd(+ k: torch.Tensor,+ v: torch.Tensor,+ beta: torch.Tensor,+ A: torch.Tensor,+ q: torch.Tensor | None = None,+ gk: torch.Tensor | None = None,+ cu_seqlens: torch.Tensor | None = None,+ chunk_indices: torch.Tensor | None = None,+) -> tuple[torch.Tensor, torch.Tensor]:+ B, T, H, K, V = *k.shape, v.shape[-1]+ BT = A.shape[-1]+ BK = 64+ BV = 64++ if chunk_indices is None and cu_seqlens is not None:+ chunk_indices = prepare_chunk_indices(cu_seqlens, BT)+ NT = cdiv(T, BT) if cu_seqlens is None else len(chunk_indices)++ w = torch.empty_like(k)+ u = torch.empty_like(v)+ kg = torch.empty_like(k) if gk is not None else None+ recompute_w_u_fwd_kernel[(NT, B * H)](+ q=q,+ k=k,+ qg=None,+ kg=kg,+ v=v,+ beta=beta,+ w=w,+ u=u,+ A=A,+ gk=gk,+ cu_seqlens=cu_seqlens,+ chunk_indices=chunk_indices,+ T=T,+ H=H,+ K=K,+ V=V,+ BT=BT,+ BK=BK,+ BV=BV,+ DOT_PRECISION="ieee",+ )+ return w, u, None, kg++[email protected]({"IS_VARLEN": lambda args: args["cu_seqlens"] is not None})+ configs=[+ triton.Config({"BK": BK, "BV": BV}, num_warps=num_warps, num_stages=num_stages)+ for BK in [32, 64]+ for BV in [64, 128]+ for num_warps in [2, 4, 8]+ for num_stages in [2, 3, 4]+ ],+ key=["BT"],+)[email protected](do_not_specialize=["T"])+def chunk_gla_fwd_kernel_o(+ q,+ v,+ g,+ h,+ o,+ A,+ cu_seqlens,+ chunk_indices,+ scale,+ T,+ H: tl.constexpr,+ K: tl.constexpr,+ V: tl.constexpr,+ BT: tl.constexpr,+ BK: tl.constexpr,+ BV: tl.constexpr,+ IS_VARLEN: tl.constexpr,+):+ i_v, i_t, i_bh = tl.program_id(0), tl.program_id(1), tl.program_id(2)+ i_b, i_h = i_bh // H, i_bh % H+ if IS_VARLEN:+ i_tg = i_t+ i_n, i_t = (+ tl.load(chunk_indices + i_t * 2).to(tl.int32),+ tl.load(chunk_indices + i_t * 2 + 1).to(tl.int32),+ )+ bos, eos = (+ tl.load(cu_seqlens + i_n).to(tl.int32),+ tl.load(cu_seqlens + i_n + 1).to(tl.int32),+ )+ T = eos - bos+ NT = tl.cdiv(T, BT)+ else:+ NT = tl.cdiv(T, BT)+ i_tg = i_b * NT + i_t+ bos, eos = i_b * T, i_b * T + T++ m_s = tl.arange(0, BT)[:, None] >= tl.arange(0, BT)[None, :]++ b_o = tl.zeros([BT, BV], dtype=tl.float32)+ for i_k in range(tl.cdiv(K, BK)):+ p_q = tl.make_block_ptr(+ q + (bos * H + i_h) * K,+ (T, K),+ (H * K, 1),+ (i_t * BT, i_k * BK),+ (BT, BK),+ (1, 0),+ )+ p_g = tl.make_block_ptr(+ g + (bos * H + i_h) * K,+ (T, K),+ (H * K, 1),+ (i_t * BT, i_k * BK),+ (BT, BK),+ (1, 0),+ )+ p_h = tl.make_block_ptr(+ # int64 BEFORE the K*V multiply: the int32 product wraps at chunk+ # index 2048 (131072-token prefill at H=64, K=V=128).+ h + (i_tg * H + i_h).to(tl.int64) * K * V,+ (V, K),+ (K, 1),+ (i_v * BV, i_k * BK),+ (BV, BK),+ (1, 0),+ )++ # [BT, BK]+ b_q = tl.load(p_q, boundary_check=(0, 1))+ b_q = (b_q * scale).to(b_q.dtype)+ # [BT, BK]+ b_g = tl.load(p_g, boundary_check=(0, 1))+ # [BT, BK]+ b_qg = (b_q * exp2(b_g)).to(b_q.dtype)+ # [BV, BK]+ b_h = tl.load(p_h, boundary_check=(0, 1))+ # [BT, BV]+ if i_k >= 0:+ b_o += tl.dot(b_qg, tl.trans(b_h).to(b_qg.dtype))+ p_v = tl.make_block_ptr(+ v + (bos * H + i_h) * V,+ (T, V),+ (H * V, 1),+ (i_t * BT, i_v * BV),+ (BT, BV),+ (1, 0),+ )+ p_o = tl.make_block_ptr(+ o + (bos * H + i_h) * V,+ (T, V),+ (H * V, 1),+ (i_t * BT, i_v * BV),+ (BT, BV),+ (1, 0),+ )+ p_A = tl.make_block_ptr(+ A + (bos * H + i_h) * BT, (T, BT), (H * BT, 1), (i_t * BT, 0), (BT, BT), (1, 0)+ )+ # [BT, BV]+ b_v = tl.load(p_v, boundary_check=(0, 1))+ # [BT, BT]+ b_A = tl.load(p_A, boundary_check=(0, 1))+ b_A = tl.where(m_s, b_A, 0.0).to(b_v.dtype)+ b_o += tl.dot(b_A, b_v, allow_tf32=False)+ tl.store(p_o, b_o.to(p_o.dtype.element_ty), boundary_check=(0, 1))+++def chunk_gla_fwd_o_gk(+ q: torch.Tensor,+ v: torch.Tensor,+ g: torch.Tensor,+ A: torch.Tensor,+ h: torch.Tensor,+ o: torch.Tensor,+ scale: float,+ cu_seqlens: torch.Tensor | None = None,+ chunk_indices: torch.Tensor | None = None,+ chunk_size: int = FLA_CHUNK_SIZE,+):+ B, T, H, K, V = *q.shape, v.shape[-1]+ BT = chunk_size++ if chunk_indices is None and cu_seqlens is not None:+ chunk_indices = prepare_chunk_indices(cu_seqlens, chunk_size)+ NT = cdiv(T, BT) if cu_seqlens is None else len(chunk_indices)++ def grid(meta):+ return (cdiv(V, meta["BV"]), NT, B * H)++ chunk_gla_fwd_kernel_o[grid](+ q=q,+ v=v,+ g=g,+ h=h,+ o=o,+ A=A,+ cu_seqlens=cu_seqlens,+ chunk_indices=chunk_indices,+ scale=scale,+ T=T,+ H=H,+ K=K,+ V=V,+ BT=BT,+ )+ return o+++ {+ "HAS_BIAS": lambda args: args["g_bias"] is not None,+ "IS_VARLEN": lambda args: args["cu_seqlens"] is not None,+ }+)+ configs=[+ triton.Config({"BD": BD}, num_warps=num_warps)+ for BD in [32, 64]+ for num_warps in [2, 4, 8]+ ],+ key=["H", "D", "BT", "IS_VARLEN"],+)[email protected](do_not_specialize=["T"])+def kda_gate_cumsum_fwd_kernel(+ g,+ A,+ y,+ g_bias,+ cu_seqlens,+ chunk_indices,+ cumsum_scale,+ beta,+ threshold,+ SAFE_GATE: tl.constexpr,+ LOWER_BOUND: tl.constexpr,+ T,+ H: tl.constexpr,+ D: tl.constexpr,+ BT: tl.constexpr,+ BD: tl.constexpr,+ HAS_BIAS: tl.constexpr,+ IS_VARLEN: tl.constexpr,+):+ i_d, i_t, i_bh = tl.program_id(0), tl.program_id(1), tl.program_id(2)+ i_b, i_h = i_bh // H, i_bh % H+ if IS_VARLEN:+ i_n, i_t = (+ tl.load(chunk_indices + i_t * 2).to(tl.int32),+ tl.load(chunk_indices + i_t * 2 + 1).to(tl.int32),+ )+ bos, eos = (+ tl.load(cu_seqlens + i_n).to(tl.int32),+ tl.load(cu_seqlens + i_n + 1).to(tl.int32),+ )+ T = eos - bos+ else:+ bos = i_b * T++ p_g = tl.make_block_ptr(+ g + (bos * H + i_h) * D,+ (T, D),+ (H * D, 1),+ (i_t * BT, i_d * BD),+ (BT, BD),+ (1, 0),+ )+ p_y = tl.make_block_ptr(+ y + (bos * H + i_h) * D,+ (T, D),+ (H * D, 1),+ (i_t * BT, i_d * BD),+ (BT, BD),+ (1, 0),+ )++ b_g = tl.load(p_g, boundary_check=(0, 1)).to(tl.float32)+ if HAS_BIAS:+ o_d = i_d * BD + tl.arange(0, BD)+ b_bias = tl.load(g_bias + i_h * D + o_d, mask=o_d < D, other=0.0).to(tl.float32)+ b_g = b_g + b_bias[None, :]++ b_a = tl.load(A + i_h).to(tl.float32)+ b_a = tl.exp(b_a) if SAFE_GATE else -tl.exp(b_a)+ if SAFE_GATE:+ # y = lower_bound * sigmoid(exp(A) * (g + g_bias)), bounded to+ # (lower_bound, 0) for safe-gate checkpoints.+ b_gate = LOWER_BOUND / (1.0 + tl.exp(-(b_a * b_g)))+ else:+ b_g_scaled = b_g * beta+ b_softplus = tl.where(+ b_g_scaled > threshold,+ b_g,+ (1.0 / beta) * log(1.0 + tl.exp(b_g_scaled)),+ )+ b_gate = b_a * b_softplus++ # Out-of-bounds rows (load returns 0, but softplus/bias can still make+ # b_gate non-zero) participate in the dot product. They only contribute to+ # out-of-bounds output rows, which are masked away by `boundary_check` on+ # the store, so visible output matches unfused gate + chunk-local cumsum.+ o_t = tl.arange(0, BT)+ m_cumsum = tl.where(o_t[:, None] >= o_t[None, :], 1.0, 0.0)+ b_y = tl.dot(m_cumsum, b_gate, allow_tf32=False) * cumsum_scale+ tl.store(p_y, b_y.to(p_y.dtype.element_ty), boundary_check=(0, 1))+++def fused_kda_gate_chunk_cumsum(+ raw_g: torch.Tensor,+ A_log: torch.Tensor,+ g_bias: torch.Tensor | None = None,+ beta: float = 1.0,+ threshold: float = 20.0,+ cu_seqlens: torch.Tensor | None = None,+ chunk_indices: torch.Tensor | None = None,+ chunk_size: int = FLA_CHUNK_SIZE,+ output_dtype: torch.dtype | None = torch.float,+ safe_gate: bool = False,+ lower_bound: float = -5.0,+) -> torch.Tensor:+ if cu_seqlens is not None:+ assert raw_g.shape[0] == 1, (+ "Only batch size 1 is supported when cu_seqlens are provided"+ )+ B, T, H, D = raw_g.shape+ if chunk_indices is None and cu_seqlens is not None:+ chunk_indices = prepare_chunk_indices(cu_seqlens, chunk_size)+ NT = cdiv(T, chunk_size) if cu_seqlens is None else len(chunk_indices)++ A_log = A_log.reshape(-1)+ if g_bias is not None:+ g_bias = g_bias.reshape(-1)+ y = torch.empty_like(raw_g, dtype=output_dtype or raw_g.dtype)++ def grid(meta):+ return (cdiv(meta["D"], meta["BD"]), NT, B * H)++ kda_gate_cumsum_fwd_kernel[grid](+ g=raw_g,+ A=A_log,+ y=y,+ g_bias=g_bias,+ cu_seqlens=cu_seqlens,+ chunk_indices=chunk_indices,+ # RCP_LN2 folds in the natural-log -> log2 conversion so downstream+ # exp2-based kernels reproduce exp(g). Keep this in sync with the+ # `use_exp2=True` path in `_chunk_kda_fwd_with_cumulative_g`.+ cumsum_scale=RCP_LN2,+ beta=beta,+ threshold=threshold,+ SAFE_GATE=safe_gate,+ LOWER_BOUND=lower_bound,+ T=T,+ H=H,+ D=D,+ BT=chunk_size,+ )+ return y+++def _chunk_kda_fwd_with_cumulative_g(+ q: torch.Tensor,+ k: torch.Tensor,+ v: torch.Tensor,+ g: torch.Tensor,+ beta: torch.Tensor,+ scale: float,+ initial_state: torch.Tensor,+ output_final_state: bool,+ cu_seqlens: torch.Tensor | None = None,+ chunk_indices: torch.Tensor | None = None,+ chunk_size: int = FLA_CHUNK_SIZE,+ return_h: bool = False,+):+ # Token-offset addressing (q/k/g at `(bos*H + i_h) * K`) is still int32 in the+ # intra-chunk kernels: refuse a varlen batch long enough to wrap rather than+ # silently corrupting (the chunk-STATE offsets are int64 and do not bind first).+ _, _T, _H, _K = k.shape # [1, total_tokens, H, K] (varlen packs on dim 1)+ if _T * _H * _K >= 2**31:+ raise ValueError(+ f"KDA prefill batch too long for int32 token addressing: {_T} tokens "+ f"x H={_H} x K={_K} exceeds 2**31; keep --max-prefill-length below "+ f"{2**31 // (_H * _K)} tokens."+ )+ # `g` must already be chunk-local cumulatively-summed AND scaled by+ # RCP_LN2 (so the downstream exp2-based kernels reproduce exp(g)).+ # Use `chunk_kda_fwd` or `chunk_kda_with_fused_gate_fwd` instead of+ # calling this helper directly unless that invariant is upheld.+ # the intra Aqk is kept in fp32+ # the computation has very marginal effect on the entire throughput+ A, Aqk = chunk_kda_scaled_dot_kkt_fwd(+ q=q,+ k=k,+ gk=g,+ beta=beta,+ scale=scale,+ cu_seqlens=cu_seqlens,+ chunk_indices=chunk_indices,+ chunk_size=chunk_size,+ output_dtype=torch.float32,+ )+ A = solve_tril(A=A, cu_seqlens=cu_seqlens, output_dtype=k.dtype)+ w, u, _, kg = recompute_w_u_fwd(+ k=k,+ v=v,+ beta=beta,+ A=A,+ gk=g,+ cu_seqlens=cu_seqlens,+ chunk_indices=chunk_indices,+ )+ del A+ h, v_new, final_state = chunk_gated_delta_rule_fwd_h(+ k=kg,+ w=w,+ u=u,+ gk=g,+ initial_state=initial_state,+ output_final_state=output_final_state,+ cu_seqlens=cu_seqlens,+ chunk_indices=chunk_indices,+ use_exp2=True,+ )+ del w, u, kg+ o = chunk_gla_fwd_o_gk(+ q=q,+ v=v_new,+ g=g,+ A=Aqk,+ h=h,+ o=v,+ scale=scale,+ cu_seqlens=cu_seqlens,+ chunk_indices=chunk_indices,+ chunk_size=chunk_size,+ )+ del Aqk, v_new+ if return_h:+ # FreeToken addition: expose the per-chunk state snapshots (h[b, i] is the+ # state at the START of chunk i) for hybrid-radix track checkpoints.+ return o, final_state, h+ del h+ return o, final_state+++def chunk_kda_fwd(+ q: torch.Tensor,+ k: torch.Tensor,+ v: torch.Tensor,+ g: torch.Tensor,+ beta: torch.Tensor,+ scale: float,+ initial_state: torch.Tensor,+ output_final_state: bool,+ cu_seqlens: torch.Tensor | None = None,+):+ chunk_size = FLA_CHUNK_SIZE+ chunk_indices = (+ prepare_chunk_indices(cu_seqlens, chunk_size)+ if cu_seqlens is not None+ else None+ )+ g = chunk_local_cumsum(+ g,+ chunk_size=chunk_size,+ cu_seqlens=cu_seqlens,+ chunk_indices=chunk_indices,+ )+ # KDA evaluates cumulative gate decays with exp2. Convert from natural-log+ # space so exp(x) is preserved as exp2(x / ln(2)).+ g = g * RCP_LN2+ return _chunk_kda_fwd_with_cumulative_g(+ q=q,+ k=k,+ v=v,+ g=g,+ beta=beta,+ scale=scale,+ initial_state=initial_state,+ output_final_state=output_final_state,+ cu_seqlens=cu_seqlens,+ chunk_indices=chunk_indices,+ chunk_size=chunk_size,+ )+++def chunk_kda_with_fused_gate_fwd(+ q: torch.Tensor,+ k: torch.Tensor,+ v: torch.Tensor,+ raw_g: torch.Tensor,+ beta: torch.Tensor,+ A_log: torch.Tensor,+ g_bias: torch.Tensor | None,+ scale: float,+ initial_state: torch.Tensor,+ output_final_state: bool,+ cu_seqlens: torch.Tensor | None = None,+ safe_gate: bool = False,+ lower_bound: float = -5.0,+ return_h: bool = False,+):+ chunk_size = FLA_CHUNK_SIZE+ chunk_indices = (+ prepare_chunk_indices(cu_seqlens, chunk_size)+ if cu_seqlens is not None+ else None+ )+ g = fused_kda_gate_chunk_cumsum(+ raw_g,+ A_log=A_log,+ g_bias=g_bias,+ cu_seqlens=cu_seqlens,+ chunk_indices=chunk_indices,+ chunk_size=chunk_size,+ safe_gate=safe_gate,+ lower_bound=lower_bound,+ )+ return _chunk_kda_fwd_with_cumulative_g(+ q=q,+ k=k,+ v=v,+ g=g,+ beta=beta,+ scale=scale,+ initial_state=initial_state,+ output_final_state=output_final_state,+ cu_seqlens=cu_seqlens,+ chunk_indices=chunk_indices,+ chunk_size=chunk_size,+ return_h=return_h,+ )+++def chunk_kda(+ q: torch.Tensor,+ k: torch.Tensor,+ v: torch.Tensor,+ g: torch.Tensor,+ beta: torch.Tensor,+ scale: float = None,+ initial_state: torch.Tensor = None,+ output_final_state: bool = False,+ use_qk_l2norm_in_kernel: bool = False,+ cu_seqlens: torch.Tensor | None = None,+ **kwargs,+):+ if scale is None:+ scale = k.shape[-1] ** -0.5++ if use_qk_l2norm_in_kernel:+ q = l2norm_fwd(q.contiguous())+ k = l2norm_fwd(k.contiguous())++ o, final_state = chunk_kda_fwd(+ q=q,+ k=k,+ v=v.contiguous(),+ g=g.contiguous(),+ beta=beta.contiguous(),+ scale=scale,+ initial_state=initial_state.contiguous(),+ output_final_state=output_final_state,+ cu_seqlens=cu_seqlens,+ )+ return o, final_state+++def chunk_kda_with_fused_gate(+ q: torch.Tensor,+ k: torch.Tensor,+ v: torch.Tensor,+ raw_g: torch.Tensor,+ beta: torch.Tensor,+ A_log: torch.Tensor,+ g_bias: torch.Tensor | None,+ scale: float | None = None,+ initial_state: torch.Tensor | None = None,+ output_final_state: bool = False,+ use_qk_l2norm_in_kernel: bool = False,+ cu_seqlens: torch.Tensor | None = None,+ safe_gate: bool = False,+ lower_bound: float = -5.0,+ return_h: bool = False,+ **kwargs,+):+ """Run chunk KDA from raw gate projection using fused gate+cumsum.++ WARNING: the output is written into (and returned as) the ``v`` buffer; never+ pass a tensor that is read again after this call. With ``return_h`` the+ per-chunk state snapshots (h[b, i] = state at the START of chunk i) are+ returned as a third value, for hybrid-radix track checkpoints.+ """+ if scale is None:+ scale = k.shape[-1] ** -0.5++ if use_qk_l2norm_in_kernel:+ q = l2norm_fwd(q.contiguous())+ k = l2norm_fwd(k.contiguous())++ return chunk_kda_with_fused_gate_fwd(+ q=q,+ k=k,+ v=v.contiguous(),+ raw_g=raw_g.contiguous(),+ beta=beta.contiguous(),+ A_log=A_log,+ g_bias=g_bias,+ scale=scale,+ initial_state=initial_state.contiguous() if initial_state is not None else None,+ output_final_state=output_final_state,+ cu_seqlens=cu_seqlens,+ safe_gate=safe_gate,+ lower_bound=lower_bound,+ return_h=return_h,+ )+++ configs=[+ triton.Config({"BT": bt}, num_warps=nw, num_stages=ns)+ for bt in BT_LIST_AUTOTUNE+ for nw in NUM_WARPS_AUTOTUNE+ for ns in [2, 3]+ ],+ key=["H", "D"],+)+def kda_gate_fwd_kernel(+ g,+ A,+ y,+ g_bias,+ beta: tl.constexpr,+ threshold: tl.constexpr,+ SAFE_GATE: tl.constexpr,+ LOWER_BOUND: tl.constexpr,+ T,+ H,+ D: tl.constexpr,+ BT: tl.constexpr,+ BD: tl.constexpr,+ HAS_BIAS: tl.constexpr,+):+ i_t, i_h = tl.program_id(0), tl.program_id(1)+ n_t = i_t * BT++ b_a = tl.load(A + i_h).to(tl.float32)+ b_a = tl.exp(b_a) if SAFE_GATE else -tl.exp(b_a)++ stride_row = H * D+ stride_col = 1++ g_ptr = tl.make_block_ptr(+ base=g + i_h * D,+ shape=(T, D),+ strides=(stride_row, stride_col),+ offsets=(n_t, 0),+ block_shape=(BT, BD),+ order=(1, 0),+ )++ y_ptr = tl.make_block_ptr(+ base=y + i_h * D,+ shape=(T, D),+ strides=(stride_row, stride_col),+ offsets=(n_t, 0),+ block_shape=(BT, BD),+ order=(1, 0),+ )++ b_g = tl.load(g_ptr, boundary_check=(0, 1)).to(tl.float32)++ if HAS_BIAS:+ n_d = tl.arange(0, BD)+ bias_mask = n_d < D+ b_bias = tl.load(g_bias + i_h * D + n_d, mask=bias_mask, other=0.0).to(+ tl.float32+ )+ b_g = b_g + b_bias[None, :]++ if SAFE_GATE:+ # y = lower_bound * sigmoid(exp(A) * (g + g_bias)), bounded to+ # (lower_bound, 0) for safe-gate checkpoints.+ b_y = LOWER_BOUND / (1.0 + tl.exp(-(b_a * b_g)))+ else:+ # softplus(x, beta) = (1/beta) * log(1 + exp(beta * x))+ # When beta * x > threshold, use linear approximation x+ # Use threshold to switch to linear when beta*x > threshold+ g_scaled = b_g * beta+ use_linear = g_scaled > threshold+ sp = tl.where(use_linear, b_g, (1.0 / beta) * log(1.0 + tl.exp(g_scaled)))+ b_y = b_a * sp++ tl.store(y_ptr, b_y.to(y.dtype.element_ty), boundary_check=(0, 1))+++def fused_kda_gate(+ g: torch.Tensor,+ A: torch.Tensor,+ head_k_dim: int,+ g_bias: torch.Tensor | None = None,+ beta: float = 1.0,+ threshold: float = 20.0,+ safe_gate: bool = False,+ lower_bound: float | None = -5.0,+) -> torch.Tensor:+ """+ Forward pass for KDA gate:+ input g: [..., H*D]+ param A: [H] or [1, 1, H, 1]+ beta: softplus beta parameter (softplus branch only)+ threshold: softplus threshold parameter (softplus branch only)+ safe_gate: when False (default) compute y = -exp(A)*softplus(g+g_bias);+ when True compute the bounded y = lower_bound*sigmoid(exp(A)*(g+g_bias))+ lower_bound: floor for the safe_gate branch (default -5.0)+ return : [..., H, D]+ """+ orig_shape = g.shape[:-1]++ g = g.view(-1, g.shape[-1])+ T = g.shape[0]+ HD = g.shape[1]+ H = A.numel()+ assert H * head_k_dim == HD++ y = torch.empty_like(g, dtype=torch.float32)++ def grid(meta):+ return (cdiv(T, meta["BT"]), H)++ kda_gate_fwd_kernel[grid](+ g,+ A,+ y,+ g_bias,+ beta,+ threshold,+ safe_gate,+ lower_bound if lower_bound is not None else -5.0,+ T,+ H,+ head_k_dim,+ BD=next_power_of_2(head_k_dim),+ HAS_BIAS=g_bias is not None,+ )++ y = y.view(*orig_shape, H, head_k_dim)+ return y+diff --git a/python/freetoken/kernel/fla/kda_chunk_delta_h.py b/python/freetoken/kernel/fla/kda_chunk_delta_h.pynew file mode 100644index 00000000..55c0ffb2--- /dev/null+++ b/python/freetoken/kernel/fla/kda_chunk_delta_h.py@@ -0,0 +1,394 @@+# Vendored from vLLM's third_party/flash_linear_attention (PR #53906, commit 933876c3),+# itself copied from the flash-linear-attention project (MIT, (c) 2023-2025 Songlin Yang,+# Yu Zhang). Imports are remapped onto freetoken.kernel.fla's shared helpers; keep this+# file in sync with upstream when pulling KDA kernel fixes.+# NOTE: this is the KDA-consistent variant (exp2 gate semantics via use_exp2, returns+# the final state). freetoken/kernel/fla/chunk_delta_h.py is the GDN variant (natural-exp+# gk, in-place pool state via initial_state_indices); the two serve different recurrences+# and are kept separate on purpose.+# SPDX-License-Identifier: Apache-2.0+# SPDX-FileCopyrightText: Copyright contributors to the vLLM project+# SPDX-FileCopyrightText: Songlin Yang, Yu Zhang+#+# This file contains code copied from the flash-linear-attention project.+# The original source code was licensed under the MIT license and included+# the following copyright notice:+# Copyright (c) 2023-2025, Songlin Yang, Yu Zhang+# ruff: noqa: E501++import torch++import triton+import triton.language as tl++from .index import prepare_chunk_indices, prepare_chunk_offsets+from .op import exp, exp2+from .utils import FLA_CHUNK_SIZE, use_cuda_graph++NUM_WARPS = [2, 4, 8, 16]+# Triton's AMD backend fails to lower this kernel with num_stages=4.+_CHUNK_DELTA_H_NUM_STAGES = [2, 3] if torch.version.hip else [2, 3, 4]+++ {+ "USE_G": lambda args: args["g"] is not None,+ "USE_GK": lambda args: args["gk"] is not None,+ "USE_INITIAL_STATE": lambda args: args["h0"] is not None,+ "STORE_FINAL_STATE": lambda args: args["ht"] is not None,+ "SAVE_NEW_VALUE": lambda args: args["v_new"] is not None,+ "IS_VARLEN": lambda args: args["cu_seqlens"] is not None,+ }+)+ configs=[+ triton.Config({"BV": BV}, num_warps=num_warps, num_stages=num_stages)+ for num_warps in [2, 4]+ for num_stages in _CHUNK_DELTA_H_NUM_STAGES+ for BV in [32, 64]+ ],+ key=["H", "K", "V", "BT"],+ use_cuda_graph=use_cuda_graph,+)[email protected](do_not_specialize=["T"])+def chunk_gated_delta_rule_fwd_kernel_h_blockdim64(+ k,+ v,+ w,+ v_new,+ g,+ gk,+ h,+ h0,+ ht,+ cu_seqlens,+ chunk_offsets,+ T,+ H: tl.constexpr,+ Hg: tl.constexpr,+ K: tl.constexpr,+ V: tl.constexpr,+ BT: tl.constexpr,+ BV: tl.constexpr,+ USE_G: tl.constexpr,+ USE_GK: tl.constexpr,+ USE_INITIAL_STATE: tl.constexpr,+ STORE_FINAL_STATE: tl.constexpr,+ SAVE_NEW_VALUE: tl.constexpr,+ IS_VARLEN: tl.constexpr,+ USE_EXP2: tl.constexpr,+):+ i_v, i_nh = tl.program_id(0), tl.program_id(1)+ i_n, i_h = i_nh // H, i_nh % H+ if IS_VARLEN:+ bos, eos = (+ tl.load(cu_seqlens + i_n).to(tl.int32),+ tl.load(cu_seqlens + i_n + 1).to(tl.int32),+ )+ T = eos - bos+ NT = tl.cdiv(T, BT)+ boh = tl.load(chunk_offsets + i_n).to(tl.int32)+ else:+ bos, eos = i_n * T, i_n * T + T+ NT = tl.cdiv(T, BT)+ boh = i_n * NT++ # [BV, BK]+ b_h1 = tl.zeros([BV, 64], dtype=tl.float32)+ if K > 64:+ b_h2 = tl.zeros([BV, 64], dtype=tl.float32)+ if K > 128:+ b_h3 = tl.zeros([BV, 64], dtype=tl.float32)+ if K > 192:+ b_h4 = tl.zeros([BV, 64], dtype=tl.float32)++ # calculate offset. DIVERGENCE from upstream: cast to int64 BEFORE the K*V+ # multiply (the GDN chunk_o.py idiom) -- upstream casts the already-wrapped+ # int32 product, which overflows at chunk index 2048 (a 131072-token prefill+ # at H=64, K=V=128) and silently lands in another head's state.+ h += (boh * H + i_h).to(tl.int64) * V * K+ v += (bos * H + i_h).to(tl.int64) * V+ k += (bos * Hg + i_h // (H // Hg)).to(tl.int64) * K+ w += (bos * H + i_h).to(tl.int64) * K+ if SAVE_NEW_VALUE:+ v_new += (bos * H + i_h).to(tl.int64) * V+ stride_v = H * V+ stride_h = H * V * K+ stride_k = Hg * K+ stride_w = H * K+ if USE_INITIAL_STATE:+ h0 = h0 + i_nh * V * K+ if STORE_FINAL_STATE:+ ht = ht + i_nh * V * K++ # load initial state+ if USE_INITIAL_STATE:+ p_h0_1 = tl.make_block_ptr(h0, (V, K), (K, 1), (i_v * BV, 0), (BV, 64), (1, 0))+ b_h1 += tl.load(p_h0_1, boundary_check=(0, 1)).to(tl.float32)+ if K > 64:+ p_h0_2 = tl.make_block_ptr(+ h0, (V, K), (K, 1), (i_v * BV, 64), (BV, 64), (1, 0)+ )+ b_h2 += tl.load(p_h0_2, boundary_check=(0, 1)).to(tl.float32)+ if K > 128:+ p_h0_3 = tl.make_block_ptr(+ h0, (V, K), (K, 1), (i_v * BV, 128), (BV, 64), (1, 0)+ )+ b_h3 += tl.load(p_h0_3, boundary_check=(0, 1)).to(tl.float32)+ if K > 192:+ p_h0_4 = tl.make_block_ptr(+ h0, (V, K), (K, 1), (i_v * BV, 192), (BV, 64), (1, 0)+ )+ b_h4 += tl.load(p_h0_4, boundary_check=(0, 1)).to(tl.float32)++ # main recurrence+ for i_t in range(NT):+ p_h1 = tl.make_block_ptr(+ h + i_t.to(tl.int64) * stride_h,+ (V, K),+ (K, 1),+ (i_v * BV, 0),+ (BV, 64),+ (1, 0),+ )+ tl.store(p_h1, b_h1.to(p_h1.dtype.element_ty), boundary_check=(0, 1))+ if K > 64:+ p_h2 = tl.make_block_ptr(+ h + i_t.to(tl.int64) * stride_h,+ (V, K),+ (K, 1),+ (i_v * BV, 64),+ (BV, 64),+ (1, 0),+ )+ tl.store(p_h2, b_h2.to(p_h2.dtype.element_ty), boundary_check=(0, 1))+ if K > 128:+ p_h3 = tl.make_block_ptr(+ h + i_t.to(tl.int64) * stride_h,+ (V, K),+ (K, 1),+ (i_v * BV, 128),+ (BV, 64),+ (1, 0),+ )+ tl.store(p_h3, b_h3.to(p_h3.dtype.element_ty), boundary_check=(0, 1))+ if K > 192:+ p_h4 = tl.make_block_ptr(+ h + i_t.to(tl.int64) * stride_h,+ (V, K),+ (K, 1),+ (i_v * BV, 192),+ (BV, 64),+ (1, 0),+ )+ tl.store(p_h4, b_h4.to(p_h4.dtype.element_ty), boundary_check=(0, 1))++ p_w = tl.make_block_ptr(+ w, (T, K), (stride_w, 1), (i_t * BT, 0), (BT, 64), (1, 0)+ )+ b_w = tl.load(p_w, boundary_check=(0, 1))+ b_v = tl.dot(b_w, tl.trans(b_h1).to(b_w.dtype))+ if K > 64:+ p_w = tl.make_block_ptr(+ w, (T, K), (stride_w, 1), (i_t * BT, 64), (BT, 64), (1, 0)+ )+ b_w = tl.load(p_w, boundary_check=(0, 1))+ b_v += tl.dot(b_w, tl.trans(b_h2).to(b_w.dtype))+ if K > 128:+ p_w = tl.make_block_ptr(+ w, (T, K), (stride_w, 1), (i_t * BT, 128), (BT, 64), (1, 0)+ )+ b_w = tl.load(p_w, boundary_check=(0, 1))+ b_v += tl.dot(b_w, tl.trans(b_h3).to(b_w.dtype))+ if K > 192:+ p_w = tl.make_block_ptr(+ w, (T, K), (stride_w, 1), (i_t * BT, 192), (BT, 64), (1, 0)+ )+ b_w = tl.load(p_w, boundary_check=(0, 1))+ b_v += tl.dot(b_w, tl.trans(b_h4).to(b_w.dtype))+ p_v = tl.make_block_ptr(+ v, (T, V), (stride_v, 1), (i_t * BT, i_v * BV), (BT, BV), (1, 0)+ )+ b_v = tl.load(p_v, boundary_check=(0, 1)) - b_v++ if SAVE_NEW_VALUE:+ p_v = tl.make_block_ptr(+ v_new, (T, V), (stride_v, 1), (i_t * BT, i_v * BV), (BT, BV), (1, 0)+ )+ tl.store(p_v, b_v.to(p_v.dtype.element_ty), boundary_check=(0, 1))++ last_idx = min((i_t.to(tl.int64) + 1) * BT, T) - 1+ if USE_G:+ m_t = (i_t.to(tl.int64) * BT + tl.arange(0, BT)) < T+ b_g_last = tl.load(g + bos * H + last_idx * H + i_h)+ p_g = tl.make_block_ptr(+ g + bos * H + i_h, (T,), (H,), (i_t * BT,), (BT,), (0,)+ )+ b_g = tl.load(p_g, boundary_check=(0,))+ if USE_EXP2:+ b_v = b_v * tl.where(m_t, exp2(b_g_last - b_g), 0)[:, None]+ b_g_last = exp2(b_g_last)+ else:+ b_v = b_v * tl.where(m_t, exp(b_g_last - b_g), 0)[:, None]+ b_g_last = exp(b_g_last)+ b_h1 *= b_g_last+ if K > 64:+ b_h2 *= b_g_last+ if K > 128:+ b_h3 *= b_g_last+ if K > 192:+ b_h4 *= b_g_last++ if USE_GK:+ o_k1 = tl.arange(0, 64)+ b_gk_last1 = tl.load(+ gk + (bos + last_idx) * H * K + i_h * K + o_k1,+ mask=(o_k1 < K),+ other=0.0,+ )+ if USE_EXP2:+ b_h1 *= exp2(b_gk_last1)[None, :]+ else:+ b_h1 *= exp(b_gk_last1)[None, :]+ if K > 64:+ o_k2 = 64 + o_k1+ b_gk_last2 = tl.load(+ gk + (bos + last_idx) * H * K + i_h * K + o_k2,+ mask=(o_k2 < K),+ other=0.0,+ )+ if USE_EXP2:+ b_h2 *= exp2(b_gk_last2)[None, :]+ else:+ b_h2 *= exp(b_gk_last2)[None, :]+ if K > 128:+ o_k3 = 128 + o_k1+ b_gk_last3 = tl.load(+ gk + (bos + last_idx) * H * K + i_h * K + o_k3,+ mask=(o_k3 < K),+ other=0.0,+ )+ if USE_EXP2:+ b_h3 *= exp2(b_gk_last3)[None, :]+ else:+ b_h3 *= exp(b_gk_last3)[None, :]+ if K > 192:+ o_k4 = 192 + o_k1+ b_gk_last4 = tl.load(+ gk + (bos + last_idx) * H * K + i_h * K + o_k4,+ mask=(o_k4 < K),+ other=0.0,+ )+ if USE_EXP2:+ b_h4 *= exp2(b_gk_last4)[None, :]+ else:+ b_h4 *= exp(b_gk_last4)[None, :]+ b_v = b_v.to(k.dtype.element_ty)++ p_k = tl.make_block_ptr(+ k, (K, T), (1, stride_k), (0, i_t * BT), (64, BT), (0, 1)+ )+ b_k = tl.load(p_k, boundary_check=(0, 1))+ b_h1 += tl.trans(tl.dot(b_k, b_v))+ if K > 64:+ p_k = tl.make_block_ptr(+ k, (K, T), (1, stride_k), (64, i_t * BT), (64, BT), (0, 1)+ )+ b_k = tl.load(p_k, boundary_check=(0, 1))+ b_h2 += tl.trans(tl.dot(b_k, b_v))+ if K > 128:+ p_k = tl.make_block_ptr(+ k, (K, T), (1, stride_k), (128, i_t * BT), (64, BT), (0, 1)+ )+ b_k = tl.load(p_k, boundary_check=(0, 1))+ b_h3 += tl.trans(tl.dot(b_k, b_v))+ if K > 192:+ p_k = tl.make_block_ptr(+ k, (K, T), (1, stride_k), (192, i_t * BT), (64, BT), (0, 1)+ )+ b_k = tl.load(p_k, boundary_check=(0, 1))+ b_h4 += tl.trans(tl.dot(b_k, b_v))+ # epilogue+ if STORE_FINAL_STATE:+ p_ht = tl.make_block_ptr(ht, (V, K), (K, 1), (i_v * BV, 0), (BV, 64), (1, 0))+ tl.store(p_ht, b_h1.to(p_ht.dtype.element_ty), boundary_check=(0, 1))+ if K > 64:+ p_ht = tl.make_block_ptr(+ ht, (V, K), (K, 1), (i_v * BV, 64), (BV, 64), (1, 0)+ )+ tl.store(p_ht, b_h2.to(p_ht.dtype.element_ty), boundary_check=(0, 1))+ if K > 128:+ p_ht = tl.make_block_ptr(+ ht, (V, K), (K, 1), (i_v * BV, 128), (BV, 64), (1, 0)+ )+ tl.store(p_ht, b_h3.to(p_ht.dtype.element_ty), boundary_check=(0, 1))+ if K > 192:+ p_ht = tl.make_block_ptr(+ ht, (V, K), (K, 1), (i_v * BV, 192), (BV, 64), (1, 0)+ )+ tl.store(p_ht, b_h4.to(p_ht.dtype.element_ty), boundary_check=(0, 1))+++def chunk_gated_delta_rule_fwd_h(+ k: torch.Tensor,+ w: torch.Tensor,+ u: torch.Tensor,+ g: torch.Tensor | None = None,+ gk: torch.Tensor | None = None,+ initial_state: torch.Tensor | None = None,+ output_final_state: bool = False,+ chunk_size: int = FLA_CHUNK_SIZE,+ save_new_value: bool = True,+ cu_seqlens: torch.Tensor | None = None,+ chunk_indices: torch.Tensor | None = None,+ chunk_offsets: torch.Tensor | None = None,+ use_exp2: bool = False,+) -> tuple[torch.Tensor, torch.Tensor]:+ # This kernel is slightly different from fla to support Q/K with different head numbers.+ # In fla, Q/K always have the same head number, so Hg is always equal to H.+ B, T, Hg, K, V = *k.shape, u.shape[-1]+ H = u.shape[-2]+ BT = chunk_size++ if chunk_indices is None and cu_seqlens is not None:+ chunk_indices = prepare_chunk_indices(cu_seqlens, chunk_size)+ # N: the actual number of sequences in the batch with either equal or variable lengths+ if cu_seqlens is None:+ N, NT, chunk_offsets = B, triton.cdiv(T, BT), None+ else:+ N, NT = len(cu_seqlens) - 1, len(chunk_indices)+ if chunk_offsets is None:+ chunk_offsets = prepare_chunk_offsets(cu_seqlens, BT)+ assert K <= 256, "current kernel does not support head dimension larger than 256."++ h = k.new_empty(B, NT, H, V, K)+ final_state = (+ k.new_empty(N, H, V, K, dtype=torch.float32) if output_final_state else None+ )++ v_new = torch.empty_like(u) if save_new_value else None++ def grid(meta):+ return (triton.cdiv(V, meta["BV"]), N * H)++ chunk_gated_delta_rule_fwd_kernel_h_blockdim64[grid](+ k=k,+ v=u,+ w=w,+ v_new=v_new,+ g=g,+ gk=gk,+ h=h,+ h0=initial_state,+ ht=final_state,+ cu_seqlens=cu_seqlens,+ chunk_offsets=chunk_offsets,+ T=T,+ H=H,+ Hg=Hg,+ K=K,+ V=V,+ BT=BT,+ USE_EXP2=use_exp2,+ )+ return h, v_new, final_statediff --git a/python/freetoken/kernel/fla/solve_tril.py b/python/freetoken/kernel/fla/solve_tril.pynew file mode 100644index 00000000..a8cad058--- /dev/null+++ b/python/freetoken/kernel/fla/solve_tril.py@@ -0,0 +1,563 @@+# Vendored from vLLM's third_party/flash_linear_attention (PR #53906, commit 933876c3),+# itself copied from the flash-linear-attention project (MIT, (c) 2023-2025 Songlin Yang,+# Yu Zhang). Imports are remapped onto freetoken.kernel.fla's shared helpers; keep this+# file in sync with upstream when pulling KDA kernel fixes.+# SPDX-License-Identifier: Apache-2.0+# SPDX-FileCopyrightText: Copyright contributors to the vLLM project+# SPDX-FileCopyrightText: Songlin Yang, Yu Zhang+#+# This file contains code copied from the flash-linear-attention project.+# The original source code was licensed under the MIT license and included+# the following copyright notice:+# Copyright (c) 2023-2025, Songlin Yang, Yu Zhang+# ruff: noqa: E501++import os++import torch++import triton+import triton.language as tl++from .index import prepare_chunk_indices+from .op import make_tensor_descriptor+from .utils import input_guard, is_amd, is_tma_supported++FLA_TRIL_PRECISION = os.environ.get("FLA_TRIL_PRECISION", "ieee")+ALLOWED_TRIL_PRECISIONS = ["ieee", "tf32"] if is_amd else ["ieee", "tf32", "tf32x3"]+assert FLA_TRIL_PRECISION in ALLOWED_TRIL_PRECISIONS, (+ f"FLA_TRIL_PRECISION must be one of {ALLOWED_TRIL_PRECISIONS}, but got {FLA_TRIL_PRECISION}"+)++[email protected]({"IS_VARLEN": lambda args: args["cu_seqlens"] is not None})+ configs=[+ triton.Config({}, num_warps=num_warps, num_stages=num_stages)+ for num_warps in [1, 2, 4, 8]+ for num_stages in [2, 3, 4, 5]+ ],+ key=["BT"],+)[email protected](do_not_specialize=["T"])+def solve_tril_16x16_kernel(+ A,+ Ai,+ cu_seqlens,+ chunk_indices,+ T,+ H: tl.constexpr,+ BT: tl.constexpr,+ USE_TMA: tl.constexpr,+ IS_VARLEN: tl.constexpr,+ DOT_PRECISION: tl.constexpr,+):+ i_t, i_bh = tl.program_id(0), tl.program_id(1)+ i_b, i_h = i_bh // H, i_bh % H+ if IS_VARLEN:+ i_n, i_t = (+ tl.load(chunk_indices + i_t * 2).to(tl.int32),+ tl.load(chunk_indices + i_t * 2 + 1).to(tl.int32),+ )+ bos, eos = (+ tl.load(cu_seqlens + i_n).to(tl.int32),+ tl.load(cu_seqlens + i_n + 1).to(tl.int32),+ )+ T = eos - bos+ else:+ bos, eos = i_b * T, i_b * T + T+ o_i = tl.arange(0, 16)+ m_A = o_i[:, None] > o_i[None, :]+ m_I = o_i[:, None] == o_i[None, :]++ A = A + (bos * H + i_h) * BT+ Ai = Ai + (bos * H + i_h) * 16++ offset = (i_t * 16) % BT+ if not USE_TMA:+ p_A = tl.make_block_ptr(+ A, (T, BT), (H * BT, 1), (i_t * 16, offset), (16, 16), (1, 0)+ )+ # [16, 16]+ b_A = tl.load(p_A, boundary_check=(0, 1)).to(tl.float32)+ else:+ desc = make_tensor_descriptor(A, [T, BT], [H * BT, 1], [16, 16])+ desc_o = make_tensor_descriptor(Ai, [T, 16], [H * 16, 1], [16, 16])+ b_A = desc.load([i_t * 16, offset]).to(tl.float32)+ b_A = -tl.where(m_A, b_A, 0)++ for i in range(2, min(16, T - i_t * 16)):+ # [16]+ b_a = -tl.load(A + (i_t * 16 + i) * H * BT + o_i + offset)+ b_a = b_a + tl.sum(b_a[:, None] * b_A, 0)+ b_A = tl.where((o_i == i)[:, None], b_a, b_A)+ b_A += m_I+ if not USE_TMA:+ p_Ai = tl.make_block_ptr(+ Ai, (T, 16), (H * 16, 1), (i_t * 16, 0), (16, 16), (1, 0)+ )+ tl.store(+ p_Ai,+ b_A.to(p_Ai.dtype.element_ty, fp_downcast_rounding="rtne"),+ boundary_check=(0, 1),+ )+ else:+ desc_o.store([i_t * 16, 0], b_A.to(desc_o.dtype, fp_downcast_rounding="rtne"))++[email protected]({"IS_VARLEN": lambda args: args["cu_seqlens"] is not None})+ configs=[+ triton.Config({}, num_warps=num_warps, num_stages=num_stages)+ for num_warps in [1, 2, 4, 8]+ for num_stages in [2, 3, 4, 5]+ ],+ key=["H", "BT", "IS_VARLEN"],+)[email protected](do_not_specialize=["T"])+def merge_16x16_to_32x32_inverse_kernel(+ A,+ Ai,+ cu_seqlens,+ chunk_indices,+ T,+ H: tl.constexpr,+ BT: tl.constexpr,+ USE_TMA: tl.constexpr,+ IS_VARLEN: tl.constexpr,+ DOT_PRECISION: tl.constexpr,+):+ i_t, i_bh = tl.program_id(0), tl.program_id(1)+ i_b, i_h = i_bh // H, i_bh % H+ if IS_VARLEN:+ i_n, i_t = (+ tl.load(chunk_indices + i_t * 2).to(tl.int32),+ tl.load(chunk_indices + i_t * 2 + 1).to(tl.int32),+ )+ bos, eos = (+ tl.load(cu_seqlens + i_n).to(tl.int32),+ tl.load(cu_seqlens + i_n + 1).to(tl.int32),+ )+ T = eos - bos+ else:+ bos, eos = i_b * T, i_b * T + T++ o_i = tl.arange(0, 16)+ m_A = o_i[:, None] > o_i[None, :]+ m_I = o_i[:, None] == o_i[None, :]+ A += (bos * H + i_h) * BT+ Ai += (bos * H + i_h) * BT++ if not USE_TMA:+ p_A_11 = tl.make_block_ptr(+ A, (T, BT), (H * BT, 1), (i_t * BT, 0), (16, 16), (1, 0)+ )+ p_A_22 = tl.make_block_ptr(+ A, (T, BT), (H * BT, 1), (i_t * BT + 16, 16), (16, 16), (1, 0)+ )+ b_Ai_11 = tl.load(p_A_11, boundary_check=(0, 1)).to(tl.float32)+ b_Ai_22 = tl.load(p_A_22, boundary_check=(0, 1)).to(tl.float32)+ else:+ desc = make_tensor_descriptor(A, [T, BT], [H * BT, 1], [16, 16])+ desc_o = make_tensor_descriptor(Ai, [T, BT], [H * BT, 1], [16, 16])+ b_Ai_11 = desc.load([i_t * BT + 0, 0]).to(tl.float32)+ b_Ai_22 = desc.load([i_t * BT + 16, 16]).to(tl.float32)++ # [16, 16]+ b_Ai_11 = -tl.where(m_A, b_Ai_11, 0)+ b_Ai_22 = -tl.where(m_A, b_Ai_22, 0)++ for i in range(2, min(16, T - i_t * BT)):+ b_a_11 = -tl.load(A + (i_t * BT + i) * H * BT + o_i)+ b_a_11 += tl.sum(b_a_11[:, None] * b_Ai_11, 0)+ b_Ai_11 = tl.where((o_i == i)[:, None], b_a_11, b_Ai_11)+ for i in range(16 + 2, min(32, T - i_t * BT)):+ b_a_22 = -tl.load(A + (i_t * BT + i) * H * BT + o_i + 16)+ b_a_22 += tl.sum(b_a_22[:, None] * b_Ai_22, 0)+ b_Ai_22 = tl.where((o_i == i - 16)[:, None], b_a_22, b_Ai_22)++ b_Ai_11 += m_I+ b_Ai_22 += m_I++ if not USE_TMA:+ p_A_21 = tl.make_block_ptr(+ A, (T, BT), (H * BT, 1), (i_t * BT + 16, 0), (16, 16), (1, 0)+ )+ b_A_21 = tl.load(p_A_21, boundary_check=(0, 1)).to(tl.float32)+ else:+ b_A_21 = desc.load([i_t * BT + 16, 0]).to(tl.float32)++ b_Ai_21 = -tl.dot(+ tl.dot(b_Ai_22, b_A_21, input_precision=DOT_PRECISION),+ b_Ai_11,+ input_precision=DOT_PRECISION,+ )++ if not USE_TMA:+ p_Ai_11 = tl.make_block_ptr(+ Ai, (T, BT), (H * BT, 1), (i_t * BT, 0), (16, 16), (1, 0)+ )+ p_Ai_21 = tl.make_block_ptr(+ Ai, (T, BT), (H * BT, 1), (i_t * BT + 16, 0), (16, 16), (1, 0)+ )+ p_Ai_22 = tl.make_block_ptr(+ Ai, (T, BT), (H * BT, 1), (i_t * BT + 16, 16), (16, 16), (1, 0)+ )+ tl.store(+ p_Ai_11,+ b_Ai_11.to(p_Ai_11.dtype.element_ty, fp_downcast_rounding="rtne"),+ boundary_check=(0, 1),+ )+ tl.store(+ p_Ai_22,+ b_Ai_22.to(p_Ai_22.dtype.element_ty, fp_downcast_rounding="rtne"),+ boundary_check=(0, 1),+ )+ tl.store(+ p_Ai_21,+ b_Ai_21.to(p_Ai_21.dtype.element_ty, fp_downcast_rounding="rtne"),+ boundary_check=(0, 1),+ )+ else:+ desc_o.store(+ [i_t * BT + 0, 0], b_Ai_11.to(desc_o.dtype, fp_downcast_rounding="rtne")+ )+ desc_o.store(+ [i_t * BT + 16, 0], b_Ai_21.to(desc_o.dtype, fp_downcast_rounding="rtne")+ )+ desc_o.store(+ [i_t * BT + 16, 16], b_Ai_22.to(desc_o.dtype, fp_downcast_rounding="rtne")+ )++[email protected]({"IS_VARLEN": lambda args: args["cu_seqlens"] is not None})+ configs=[+ triton.Config({}, num_warps=num_warps, num_stages=num_stages)+ for num_warps in [2, 4, 8]+ for num_stages in [2, 3, 4, 5]+ ],+ key=["H", "BT", "IS_VARLEN"],+)[email protected](do_not_specialize=["T"])+def merge_16x16_to_64x64_inverse_kernel(+ A,+ Ai,+ cu_seqlens,+ chunk_indices,+ T,+ H: tl.constexpr,+ BT: tl.constexpr,+ USE_TMA: tl.constexpr,+ IS_VARLEN: tl.constexpr,+ DOT_PRECISION: tl.constexpr,+):+ i_t, i_bh = tl.program_id(0), tl.program_id(1)+ i_b, i_h = i_bh // H, i_bh % H+ if IS_VARLEN:+ i_n, i_t = (+ tl.load(chunk_indices + i_t * 2).to(tl.int32),+ tl.load(chunk_indices + i_t * 2 + 1).to(tl.int32),+ )+ bos, eos = (+ tl.load(cu_seqlens + i_n).to(tl.int32),+ tl.load(cu_seqlens + i_n + 1).to(tl.int32),+ )+ T = eos - bos+ else:+ bos, eos = i_b * T, i_b * T + T++ o_i = tl.arange(0, 16)+ m_A = o_i[:, None] > o_i[None, :]+ m_I = o_i[:, None] == o_i[None, :]+ A += (bos * H + i_h) * BT+ Ai += (bos * H + i_h) * BT++ if not USE_TMA:+ p_A_11 = tl.make_block_ptr(+ A, (T, BT), (H * BT, 1), (i_t * BT, 0), (16, 16), (1, 0)+ )+ p_A_22 = tl.make_block_ptr(+ A, (T, BT), (H * BT, 1), (i_t * BT + 16, 16), (16, 16), (1, 0)+ )+ p_A_33 = tl.make_block_ptr(+ A, (T, BT), (H * BT, 1), (i_t * BT + 32, 32), (16, 16), (1, 0)+ )+ p_A_44 = tl.make_block_ptr(+ A, (T, BT), (H * BT, 1), (i_t * BT + 48, 48), (16, 16), (1, 0)+ )+ b_Ai_11 = tl.load(p_A_11, boundary_check=(0, 1)).to(tl.float32)+ b_Ai_22 = tl.load(p_A_22, boundary_check=(0, 1)).to(tl.float32)+ b_Ai_33 = tl.load(p_A_33, boundary_check=(0, 1)).to(tl.float32)+ b_Ai_44 = tl.load(p_A_44, boundary_check=(0, 1)).to(tl.float32)+ else:+ desc = make_tensor_descriptor(A, [T, BT], [H * BT, 1], [16, 16])+ desc_o = make_tensor_descriptor(Ai, [T, BT], [H * BT, 1], [16, 16])+ b_Ai_11 = desc.load([i_t * BT + 0, 0]).to(tl.float32)+ b_Ai_22 = desc.load([i_t * BT + 16, 16]).to(tl.float32)+ b_Ai_33 = desc.load([i_t * BT + 32, 32]).to(tl.float32)+ b_Ai_44 = desc.load([i_t * BT + 48, 48]).to(tl.float32)++ # [16, 16]+ b_Ai_11 = -tl.where(m_A, b_Ai_11, 0)+ b_Ai_22 = -tl.where(m_A, b_Ai_22, 0)+ b_Ai_33 = -tl.where(m_A, b_Ai_33, 0)+ b_Ai_44 = -tl.where(m_A, b_Ai_44, 0)++ for i in range(2, min(16, T - i_t * BT)):+ b_a_11 = -tl.load(A + (i_t * BT + i) * H * BT + o_i)+ b_a_11 += tl.sum(b_a_11[:, None] * b_Ai_11, 0)+ b_Ai_11 = tl.where((o_i == i)[:, None], b_a_11, b_Ai_11)+ for i in range(16 + 2, min(32, T - i_t * BT)):+ b_a_22 = -tl.load(A + (i_t * BT + i) * H * BT + o_i + 16)+ b_a_22 += tl.sum(b_a_22[:, None] * b_Ai_22, 0)+ b_Ai_22 = tl.where((o_i == i - 16)[:, None], b_a_22, b_Ai_22)+ for i in range(32 + 2, min(48, T - i_t * BT)):+ b_a_33 = -tl.load(A + (i_t * BT + i) * H * BT + o_i + 32)+ b_a_33 += tl.sum(b_a_33[:, None] * b_Ai_33, 0)+ b_Ai_33 = tl.where((o_i == i - 32)[:, None], b_a_33, b_Ai_33)+ for i in range(48 + 2, min(64, T - i_t * BT)):+ b_a_44 = -tl.load(A + (i_t * BT + i) * H * BT + o_i + 48)+ b_a_44 += tl.sum(b_a_44[:, None] * b_Ai_44, 0)+ b_Ai_44 = tl.where((o_i == i - 48)[:, None], b_a_44, b_Ai_44)+ b_Ai_11 += m_I+ b_Ai_22 += m_I+ b_Ai_33 += m_I+ b_Ai_44 += m_I++ if not USE_TMA:+ p_A_21 = tl.make_block_ptr(+ A, (T, BT), (H * BT, 1), (i_t * BT + 16, 0), (16, 16), (1, 0)+ )+ p_A_31 = tl.make_block_ptr(+ A, (T, BT), (H * BT, 1), (i_t * BT + 32, 0), (16, 16), (1, 0)+ )+ p_A_32 = tl.make_block_ptr(+ A, (T, BT), (H * BT, 1), (i_t * BT + 32, 16), (16, 16), (1, 0)+ )+ p_A_41 = tl.make_block_ptr(+ A, (T, BT), (H * BT, 1), (i_t * BT + 48, 0), (16, 16), (1, 0)+ )+ p_A_42 = tl.make_block_ptr(+ A, (T, BT), (H * BT, 1), (i_t * BT + 48, 16), (16, 16), (1, 0)+ )+ p_A_43 = tl.make_block_ptr(+ A, (T, BT), (H * BT, 1), (i_t * BT + 48, 32), (16, 16), (1, 0)+ )+ b_A_21 = tl.load(p_A_21, boundary_check=(0, 1)).to(tl.float32)+ b_A_31 = tl.load(p_A_31, boundary_check=(0, 1)).to(tl.float32)+ b_A_32 = tl.load(p_A_32, boundary_check=(0, 1)).to(tl.float32)+ b_A_41 = tl.load(p_A_41, boundary_check=(0, 1)).to(tl.float32)+ b_A_42 = tl.load(p_A_42, boundary_check=(0, 1)).to(tl.float32)+ b_A_43 = tl.load(p_A_43, boundary_check=(0, 1)).to(tl.float32)+ else:+ b_A_21 = desc.load([i_t * BT + 16, 0]).to(tl.float32)+ b_A_31 = desc.load([i_t * BT + 32, 0]).to(tl.float32)+ b_A_32 = desc.load([i_t * BT + 32, 16]).to(tl.float32)+ b_A_41 = desc.load([i_t * BT + 48, 0]).to(tl.float32)+ b_A_42 = desc.load([i_t * BT + 48, 16]).to(tl.float32)+ b_A_43 = desc.load([i_t * BT + 48, 32]).to(tl.float32)++ b_Ai_21 = -tl.dot(+ tl.dot(b_Ai_22, b_A_21, input_precision=DOT_PRECISION),+ b_Ai_11,+ input_precision=DOT_PRECISION,+ )+ b_Ai_32 = -tl.dot(+ tl.dot(b_Ai_33, b_A_32, input_precision=DOT_PRECISION),+ b_Ai_22,+ input_precision=DOT_PRECISION,+ )+ b_Ai_43 = -tl.dot(+ tl.dot(b_Ai_44, b_A_43, input_precision=DOT_PRECISION),+ b_Ai_33,+ input_precision=DOT_PRECISION,+ )++ b_Ai_31 = -tl.dot(+ b_Ai_33,+ tl.dot(b_A_31, b_Ai_11, input_precision=DOT_PRECISION)+ + tl.dot(b_A_32, b_Ai_21, input_precision=DOT_PRECISION),+ input_precision=DOT_PRECISION,+ )+ b_Ai_42 = -tl.dot(+ b_Ai_44,+ tl.dot(b_A_42, b_Ai_22, input_precision=DOT_PRECISION)+ + tl.dot(b_A_43, b_Ai_32, input_precision=DOT_PRECISION),+ input_precision=DOT_PRECISION,+ )+ b_Ai_41 = -tl.dot(+ b_Ai_44,+ tl.dot(b_A_41, b_Ai_11, input_precision=DOT_PRECISION)+ + tl.dot(b_A_42, b_Ai_21, input_precision=DOT_PRECISION)+ + tl.dot(b_A_43, b_Ai_31, input_precision=DOT_PRECISION),+ input_precision=DOT_PRECISION,+ )++ if not USE_TMA:+ p_Ai_11 = tl.make_block_ptr(+ Ai, (T, BT), (H * BT, 1), (i_t * BT, 0), (16, 16), (1, 0)+ )+ p_Ai_22 = tl.make_block_ptr(+ Ai, (T, BT), (H * BT, 1), (i_t * BT + 16, 16), (16, 16), (1, 0)+ )+ p_Ai_33 = tl.make_block_ptr(+ Ai, (T, BT), (H * BT, 1), (i_t * BT + 32, 32), (16, 16), (1, 0)+ )+ p_Ai_44 = tl.make_block_ptr(+ Ai, (T, BT), (H * BT, 1), (i_t * BT + 48, 48), (16, 16), (1, 0)+ )+ p_Ai_21 = tl.make_block_ptr(+ Ai, (T, BT), (H * BT, 1), (i_t * BT + 16, 0), (16, 16), (1, 0)+ )+ p_Ai_31 = tl.make_block_ptr(+ Ai, (T, BT), (H * BT, 1), (i_t * BT + 32, 0), (16, 16), (1, 0)+ )+ p_Ai_32 = tl.make_block_ptr(+ Ai, (T, BT), (H * BT, 1), (i_t * BT + 32, 16), (16, 16), (1, 0)+ )+ p_Ai_41 = tl.make_block_ptr(+ Ai, (T, BT), (H * BT, 1), (i_t * BT + 48, 0), (16, 16), (1, 0)+ )+ p_Ai_42 = tl.make_block_ptr(+ Ai, (T, BT), (H * BT, 1), (i_t * BT + 48, 16), (16, 16), (1, 0)+ )+ p_Ai_43 = tl.make_block_ptr(+ Ai, (T, BT), (H * BT, 1), (i_t * BT + 48, 32), (16, 16), (1, 0)+ )+ tl.store(+ p_Ai_11,+ b_Ai_11.to(p_Ai_11.dtype.element_ty, fp_downcast_rounding="rtne"),+ boundary_check=(0, 1),+ )+ tl.store(+ p_Ai_22,+ b_Ai_22.to(p_Ai_22.dtype.element_ty, fp_downcast_rounding="rtne"),+ boundary_check=(0, 1),+ )+ tl.store(+ p_Ai_33,+ b_Ai_33.to(p_Ai_33.dtype.element_ty, fp_downcast_rounding="rtne"),+ boundary_check=(0, 1),+ )+ tl.store(+ p_Ai_44,+ b_Ai_44.to(p_Ai_44.dtype.element_ty, fp_downcast_rounding="rtne"),+ boundary_check=(0, 1),+ )+ tl.store(+ p_Ai_21,+ b_Ai_21.to(p_Ai_21.dtype.element_ty, fp_downcast_rounding="rtne"),+ boundary_check=(0, 1),+ )+ tl.store(+ p_Ai_31,+ b_Ai_31.to(p_Ai_31.dtype.element_ty, fp_downcast_rounding="rtne"),+ boundary_check=(0, 1),+ )+ tl.store(+ p_Ai_32,+ b_Ai_32.to(p_Ai_32.dtype.element_ty, fp_downcast_rounding="rtne"),+ boundary_check=(0, 1),+ )+ tl.store(+ p_Ai_41,+ b_Ai_41.to(p_Ai_41.dtype.element_ty, fp_downcast_rounding="rtne"),+ boundary_check=(0, 1),+ )+ tl.store(+ p_Ai_42,+ b_Ai_42.to(p_Ai_42.dtype.element_ty, fp_downcast_rounding="rtne"),+ boundary_check=(0, 1),+ )+ tl.store(+ p_Ai_43,+ b_Ai_43.to(p_Ai_43.dtype.element_ty, fp_downcast_rounding="rtne"),+ boundary_check=(0, 1),+ )+ else:+ desc_o.store(+ [i_t * BT + 0, 0], b_Ai_11.to(desc_o.dtype, fp_downcast_rounding="rtne")+ )+ desc_o.store(+ [i_t * BT + 16, 16], b_Ai_22.to(desc_o.dtype, fp_downcast_rounding="rtne")+ )+ desc_o.store(+ [i_t * BT + 32, 32], b_Ai_33.to(desc_o.dtype, fp_downcast_rounding="rtne")+ )+ desc_o.store(+ [i_t * BT + 48, 48], b_Ai_44.to(desc_o.dtype, fp_downcast_rounding="rtne")+ )+ desc_o.store(+ [i_t * BT + 16, 0], b_Ai_21.to(desc_o.dtype, fp_downcast_rounding="rtne")+ )+ desc_o.store(+ [i_t * BT + 32, 0], b_Ai_31.to(desc_o.dtype, fp_downcast_rounding="rtne")+ )+ desc_o.store(+ [i_t * BT + 32, 16], b_Ai_32.to(desc_o.dtype, fp_downcast_rounding="rtne")+ )+ desc_o.store(+ [i_t * BT + 48, 0], b_Ai_41.to(desc_o.dtype, fp_downcast_rounding="rtne")+ )+ desc_o.store(+ [i_t * BT + 48, 16], b_Ai_42.to(desc_o.dtype, fp_downcast_rounding="rtne")+ )+ desc_o.store(+ [i_t * BT + 48, 32], b_Ai_43.to(desc_o.dtype, fp_downcast_rounding="rtne")+ )+++@input_guard+def solve_tril(+ A: torch.Tensor,+ cu_seqlens: torch.Tensor | None = None,+ chunk_indices: torch.Tensor | None = None,+ output_dtype: torch.dtype = torch.float,+) -> torch.Tensor:+ """+ Compute the inverse of the matrix I + A+ A should be strictly lower triangular, i.e., A.triu() == 0.++ Args:+ A (torch.Tensor):+ [B, T, H, BT], where BT should only be 16, 32, or 64.+ cu_seqlens (torch.Tensor):+ The cumulative sequence lengths of the input tensor. Default: `None`.+ chunk_indices (torch.Tensor):+ Pre-computed chunk indices. Default: `None`.+ output_dtype (torch.dtype):+ The dtype of the output tensor. Default: `torch.float`.+ If `None`, the output dtype will be the same as the input dtype.++ Returns:+ (I + A)^-1 with the same shape as A+ """+ assert A.shape[-1] in [16, 32, 64]+ output_dtype = A.dtype if output_dtype is None else output_dtype++ B, T, H, BT = A.shape+ if chunk_indices is None and cu_seqlens is not None:+ chunk_indices = prepare_chunk_indices(cu_seqlens, BT)+ NT = len(chunk_indices) if cu_seqlens is not None else triton.cdiv(T, BT)++ Ai = torch.zeros_like(A, dtype=output_dtype)+ if BT == 16:+ merge_fn = solve_tril_16x16_kernel+ elif BT == 32:+ merge_fn = merge_16x16_to_32x32_inverse_kernel+ elif BT == 64:+ merge_fn = merge_16x16_to_64x64_inverse_kernel++ merge_fn[NT, B * H](+ A=A,+ Ai=Ai,+ cu_seqlens=cu_seqlens,+ chunk_indices=chunk_indices,+ T=T,+ H=H,+ BT=BT,+ USE_TMA=is_tma_supported,+ DOT_PRECISION=FLA_TRIL_PRECISION,+ )+ return Aidiff --git a/python/freetoken/kernel/fla/utils.py b/python/freetoken/kernel/fla/utils.pyindex e9e0b697..a0999311 100644--- a/python/freetoken/kernel/fla/utils.py+++ b/python/freetoken/kernel/fla/utils.py@@ -280,6 +280,20 @@ use_cuda_graph = is_nvidia and os.environ.get("FLA_USE_CUDA_GRAPH", "0") == "1"is_tf32_supported = is_nvidia and torch.cuda.get_device_capability()[0] >= 8is_gather_supported = hasattr(triton.language, "gather")+# Shared chunk length of the fla chunked kernels (KDA vendor reads it from here).+FLA_CHUNK_SIZE = 64++# TMA descriptors (solve_tril fast path): Hopper+, opt-in via FLA_USE_TMA=1, and only+# when this triton exposes a make_tensor_descriptor (see kernel/fla/op.py).+is_tma_supported = (+ is_nvidia_hopper+ and os.getenv("FLA_USE_TMA", "0") == "1"+ and (+ hasattr(triton.language, "_experimental_make_tensor_descriptor")+ or hasattr(triton.language, "make_tensor_descriptor")+ )+)+def get_all_max_shared_mem():try:diff --git a/python/freetoken/kernel/triton/activation.py b/python/freetoken/kernel/triton/activation.pyindex 2c38b533..0e4d28c8 100644--- a/python/freetoken/kernel/triton/activation.py+++ b/python/freetoken/kernel/triton/activation.py@@ -31,6 +31,8 @@ GELU_TANH = 2# variant lives in triton/mxfp4_moe.py):# y = clamp(gate, max=limit) * sigmoid(alpha * gate) * (clamp(up, +-limit) + 1)SWIGLUOAI = 3+# GLM-5.3 "swiglu_limit": swigluoai's clamped form WITHOUT the (up + 1) bias.+SWIGLU_CLAMP = 4_SQRT_2_OVER_PI = 0.7978845608028654 # sqrt(2/pi)_GELU_C = 0.044715@@ -105,6 +107,11 @@ def _act_and_mul_kernel(up = tl.minimum(tl.maximum(up, -limit), limit)act = gate / (1.0 + _fast_ex2(-gate * alpha * _LOG2E))y = act * (up + 1.0)+ elif ACT == 4: # SWIGLU_CLAMP (GLM-5.3): swigluoai without the (up + 1) bias+ gate = tl.minimum(gate, limit)+ up = tl.minimum(tl.maximum(up, -limit), limit)+ act = gate / (1.0 + _fast_ex2(-gate * alpha * _LOG2E))+ y = act * upelse: # GELU (erf)act = 0.5 * gate * (1.0 + libdevice.erf(gate * 0.7071067811865476))y = act * up@@ -166,4 +173,21 @@ def swigluoai_and_mul(return _act_and_mul(SWIGLUOAI, x, out, alpha=alpha, limit=limit)-__all__ = ["silu_and_mul", "gelu_and_mul", "gelu_tanh_and_mul", "swigluoai_and_mul"]+def swiglu_clamp_and_mul(+ x: torch.Tensor,+ out: torch.Tensor | None = None,+ *,+ alpha: float = 1.0,+ limit: float = 10.0,+) -> torch.Tensor:+ """GLM-5.3 clamped SwiGLU over UNINTERLEAVED halves: ``clamp(gate, max=limit) * sigmoid(alpha * gate) * clamp(up, +-limit)``."""+ return _act_and_mul(SWIGLU_CLAMP, x, out, alpha=alpha, limit=limit)+++__all__ = [+ "silu_and_mul",+ "gelu_and_mul",+ "gelu_tanh_and_mul",+ "swigluoai_and_mul",+ "swiglu_clamp_and_mul",+]diff --git a/python/freetoken/kernel/triton/glm_dsa_sparse.py b/python/freetoken/kernel/triton/glm_dsa_sparse.pyindex d3219f95..8aaeeb9b 100644--- a/python/freetoken/kernel/triton/glm_dsa_sparse.py+++ b/python/freetoken/kernel/triton/glm_dsa_sparse.py@@ -49,6 +49,7 @@ def _glm_dsa_sparse_kernel(BLOCK_H: tl.constexpr,BLOCK_T: tl.constexpr,HAS_COUNTS: tl.constexpr,+ HAS_ROPE: tl.constexpr,):pid_m = tl.program_id(0)pid_b = tl.program_id(1)@@ -57,11 +58,14 @@ def _glm_dsa_sparse_kernel(offs_h = pid_h * BLOCK_H + tl.arange(0, BLOCK_H)h_mask = offs_h < Hoffs_v = tl.arange(0, D_V)- offs_r = tl.arange(0, D_R)q_base = q_ptr + pid_b * stride_qb + pid_m * stride_qm + offs_h[:, None] * stride_qhq_v = tl.load(q_base + offs_v[None, :] * stride_qd, mask=h_mask[:, None], other=0.0).to(tl.float32)- q_r = tl.load(q_base + (D_V + offs_r[None, :]) * stride_qd, mask=h_mask[:, None], other=0.0).to(tl.float32)+ if HAS_ROPE:+ # NoPE checkpoints (glm5_next) have D_R == 0: tl.arange needs a non-empty+ # span, so the whole rope half is compiled out on the constexpr.+ offs_r = tl.arange(0, D_R)+ q_r = tl.load(q_base + (D_V + offs_r[None, :]) * stride_qd, mask=h_mask[:, None], other=0.0).to(tl.float32)m_i = tl.full((BLOCK_H,), -float("inf"), dtype=tl.float32)l_i = tl.zeros((BLOCK_H,), dtype=tl.float32)@@ -79,9 +83,12 @@ def _glm_dsa_sparse_kernel(valid = idxs >= 0kv_base = pool_ptr + idxs[:, None] * stride_pnkv_v = tl.load(kv_base + offs_v[None, :] * stride_pd, mask=valid[:, None], other=0.0).to(tl.float32)- kv_r = tl.load(kv_base + (D_V + offs_r[None, :]) * stride_pd, mask=valid[:, None], other=0.0).to(tl.float32)- scores = (tl.dot(q_v, tl.trans(kv_v)) + tl.dot(q_r, tl.trans(kv_r))) * scale+ scores = tl.dot(q_v, tl.trans(kv_v))+ if HAS_ROPE:+ kv_r = tl.load(kv_base + (D_V + offs_r[None, :]) * stride_pd, mask=valid[:, None], other=0.0).to(tl.float32)+ scores += tl.dot(q_r, tl.trans(kv_r))+ scores = scores * scalescores = tl.where(valid[None, :], scores, -float("inf"))m_new = tl.maximum(m_i, tl.max(scores, axis=1))@@ -207,6 +214,7 @@ def _glm_dsa_splitk_kernel(BLOCK_H: tl.constexpr,BLOCK_T: tl.constexpr,HAS_COUNTS: tl.constexpr,+ HAS_ROPE: tl.constexpr,NUM_SPLITS: tl.constexpr,):"""Stage 1 (decode flash-decoding): each program reduces one BLOCK_T-aligned slice of@@ -221,7 +229,6 @@ def _glm_dsa_splitk_kernel(offs_h = pid_h * BLOCK_H + tl.arange(0, BLOCK_H)h_mask = offs_h < Hoffs_v = tl.arange(0, D_V)- offs_r = tl.arange(0, D_R)n_active = TOPKif HAS_COUNTS:@@ -238,7 +245,9 @@ def _glm_dsa_splitk_kernel(if split_end > split_start:q_base = q_ptr + pid_b * stride_qb + pid_m * stride_qm + offs_h[:, None] * stride_qhq_v = tl.load(q_base + offs_v[None, :] * stride_qd, mask=h_mask[:, None], other=0.0).to(tl.float32)- q_r = tl.load(q_base + (D_V + offs_r[None, :]) * stride_qd, mask=h_mask[:, None], other=0.0).to(tl.float32)+ if HAS_ROPE:+ offs_r = tl.arange(0, D_R)+ q_r = tl.load(q_base + (D_V + offs_r[None, :]) * stride_qd, mask=h_mask[:, None], other=0.0).to(tl.float32)idx_base = idx_ptr + pid_b * stride_ib + pid_m * stride_imfor start in range(split_start, split_end, BLOCK_T):@@ -248,9 +257,12 @@ def _glm_dsa_splitk_kernel(valid = idxs >= 0kv_base = pool_ptr + idxs[:, None] * stride_pnkv_v = tl.load(kv_base + offs_v[None, :] * stride_pd, mask=valid[:, None], other=0.0).to(tl.float32)- kv_r = tl.load(kv_base + (D_V + offs_r[None, :]) * stride_pd, mask=valid[:, None], other=0.0).to(tl.float32)- scores = (tl.dot(q_v, tl.trans(kv_v)) + tl.dot(q_r, tl.trans(kv_r))) * scale+ scores = tl.dot(q_v, tl.trans(kv_v))+ if HAS_ROPE:+ kv_r = tl.load(kv_base + (D_V + offs_r[None, :]) * stride_pd, mask=valid[:, None], other=0.0).to(tl.float32)+ scores += tl.dot(q_r, tl.trans(kv_r))+ scores = scores * scalescores = tl.where(valid[None, :], scores, -float("inf"))m_new = tl.maximum(m_i, tl.max(scores, axis=1))@@ -384,7 +396,7 @@ def glm_dsa_sparse_attn(stride_nb, stride_nm,D_V=d_v, D_R=d_r,BLOCK_H=BLOCK_H, BLOCK_T=BLOCK_T,- HAS_COUNTS=has_counts, NUM_SPLITS=n_splits,+ HAS_COUNTS=has_counts, HAS_ROPE=d_r > 0, NUM_SPLITS=n_splits,num_warps=4, num_stages=2,)grid2 = (m, b, h)@@ -410,7 +422,7 @@ def glm_dsa_sparse_attn(stride_nb, stride_nm,D_V=d_v, D_R=d_r,BLOCK_H=BLOCK_H, BLOCK_T=BLOCK_T,- HAS_COUNTS=has_counts,+ HAS_COUNTS=has_counts, HAS_ROPE=d_r > 0,num_warps=4, num_stages=2,)return odiff --git a/python/freetoken/kernel/triton/kpool_compress.py b/python/freetoken/kernel/triton/kpool_compress.pynew file mode 100644index 00000000..61cacb1f--- /dev/null+++ b/python/freetoken/kernel/triton/kpool_compress.py@@ -0,0 +1,146 @@+"""Pool one token row's group into ``slab[cmp_rows[row]]``: members inside this+forward read the raw K/gate rows, older members read the per-request tail ring.+Adapted from ``qsa/compress.py`` with the mean replaced by a per-channel+softmax(gate + APE) weighted sum (hence the gate stream/ring and the APE input).+Non-closing rows land on a scratch row that scoring never reads."""++from __future__ import annotations++import torch+import triton+import triton.language as tl+++def _compress_kpool_groups_kernel(+ raw_k_ptr,+ raw_g_ptr,+ ring_k_ptr,+ ring_g_ptr,+ ape_ptr,+ ring_slots_ptr,+ token_to_req_ptr,+ query_start_loc_ptr,+ positions_ptr,+ slab_ptr,+ cmp_rows_ptr,+ stride_raw_k_row,+ stride_raw_g_row,+ stride_ring_row,+ stride_slab_row,+ num_rows,+ num_ring_rows,+ num_requests,+ RATIO: tl.constexpr,+ HEAD_DIM: tl.constexpr,+ BLOCK_D: tl.constexpr,+) -> None:+ row = tl.program_id(0)+ dims = tl.arange(0, BLOCK_D)+ in_dim = dims < HEAD_DIM++ request = tl.load(token_to_req_ptr + row, mask=row < num_rows, other=-1)+ end_position = tl.load(positions_ptr + row, mask=row < num_rows, other=0).to(tl.int64)+ valid_request = (request >= 0) & (request < num_requests)+ safe_request = tl.minimum(tl.maximum(request, 0), num_requests - 1)+ query_row_start = tl.load(+ query_start_loc_ptr + safe_request, mask=valid_request, other=0+ ).to(tl.int64)+ chunk_start_position = end_position - (row - query_row_start)+ ring_slot = tl.load(ring_slots_ptr + safe_request, mask=valid_request, other=0).to(+ tl.int64+ )+ # A row whose group has members before position 0 (end_position < RATIO - 1)+ # can never close; keep its loads masked off entirely -- member positions would+ # be negative and C-style % would produce NEGATIVE ring rows (illegal address).+ valid_row = (row < num_rows) & valid_request & (end_position >= RATIO - 1)++ # Two-pass per-channel softmax over the RATIO group members (static loop; the+ # doubled loads are 4 x 512B and free next to the fp32 math).+ m = tl.full((BLOCK_D,), float("-inf"), tl.float32)+ for off in tl.static_range(RATIO):+ position = end_position - (RATIO - 1 - off)+ use_raw = position >= chunk_start_position+ raw_row = query_row_start + position - chunk_start_position+ ring_row = ring_slot * RATIO + position % RATIO+ g_raw = tl.load(+ raw_g_ptr + raw_row * stride_raw_g_row + dims,+ mask=valid_row & use_raw & (raw_row >= 0) & (raw_row < num_rows) & in_dim,+ other=0.0,+ ).to(tl.float32)+ g_ring = tl.load(+ ring_g_ptr + tl.maximum(ring_row, 0) * stride_ring_row + dims,+ mask=valid_row & (~use_raw) & (ring_row >= 0) & (ring_row < num_ring_rows) & in_dim,+ other=0.0,+ ).to(tl.float32)+ ape = tl.load(ape_ptr + off * HEAD_DIM + dims, mask=in_dim, other=0.0)+ m = tl.maximum(m, tl.where(use_raw, g_raw, g_ring) + ape)++ s = tl.zeros((BLOCK_D,), dtype=tl.float32)+ acc = tl.zeros((BLOCK_D,), dtype=tl.float32)+ for off in tl.static_range(RATIO):+ position = end_position - (RATIO - 1 - off)+ use_raw = position >= chunk_start_position+ raw_row = query_row_start + position - chunk_start_position+ ring_row = ring_slot * RATIO + position % RATIO+ raw_mask = valid_row & use_raw & (raw_row >= 0) & (raw_row < num_rows) & in_dim+ ring_mask = (+ valid_row & (~use_raw) & (ring_row >= 0) & (ring_row < num_ring_rows) & in_dim+ )+ g_raw = tl.load(+ raw_g_ptr + raw_row * stride_raw_g_row + dims, mask=raw_mask, other=0.0+ ).to(tl.float32)+ g_ring = tl.load(+ ring_g_ptr + tl.maximum(ring_row, 0) * stride_ring_row + dims,+ mask=ring_mask, other=0.0,+ ).to(tl.float32)+ k_raw = tl.load(+ raw_k_ptr + raw_row * stride_raw_k_row + dims, mask=raw_mask, other=0.0+ ).to(tl.float32)+ k_ring = tl.load(+ ring_k_ptr + tl.maximum(ring_row, 0) * stride_ring_row + dims,+ mask=ring_mask, other=0.0,+ ).to(tl.float32)+ ape = tl.load(ape_ptr + off * HEAD_DIM + dims, mask=in_dim, other=0.0)+ e = tl.exp(tl.where(use_raw, g_raw, g_ring) + ape - m)+ s += e+ acc += e * tl.where(use_raw, k_raw, k_ring)++ pooled = acc / s+ dest = tl.load(cmp_rows_ptr + row, mask=row < num_rows, other=0).to(tl.int64)+ tl.store(+ slab_ptr + dest * stride_slab_row + dims,+ pooled.to(slab_ptr.dtype.element_ty),+ mask=(row < num_rows) & in_dim,+ )+++def kpool_compress_store(+ k: torch.Tensor, # [T, D] raw index keys (this forward)+ gate: torch.Tensor, # [T, D] raw gate scores+ ring_k: torch.Tensor, # [slots * ratio, D] flat tail ring (keys)+ ring_g: torch.Tensor, # [slots * ratio, D] flat tail ring (gates)+ ape: torch.Tensor, # [ratio, D] fp32 model parameter+ ring_slots: torch.Tensor, # [n_req] Req.table_idx+ token_to_req: torch.Tensor, # [T]+ cu_seqlens: torch.Tensor, # [n_req + 1]+ positions: torch.Tensor, # [T]+ slab: torch.Tensor, # [shadow_rows + scratch, D]+ cmp_rows: torch.Tensor, # [T] shadow row (closing) or scratch row+ ratio: int,+) -> None:+ t, d = k.shape+ if t == 0:+ return+ assert ape.dtype == torch.float32 and ape.shape == (ratio, d)+ _compress_kpool_groups_kernel[(t,)](+ k, gate, ring_k, ring_g, ape,+ ring_slots, token_to_req, cu_seqlens, positions,+ slab, cmp_rows,+ k.stride(0), gate.stride(0), ring_k.stride(0), slab.stride(0),+ t, ring_k.shape[0], ring_slots.numel(),+ RATIO=ratio, HEAD_DIM=d, BLOCK_D=triton.next_power_of_2(d),+ )+++__all__ = ["kpool_compress_store"]diff --git a/python/freetoken/kernel/triton/mhc.py b/python/freetoken/kernel/triton/mhc.pynew file mode 100644index 00000000..0782efe3--- /dev/null+++ b/python/freetoken/kernel/triton/mhc.py@@ -0,0 +1,232 @@+"""Fused mHC (Manifold-Constrained Hyper-Connections) triton kernels.++One program per token fuses the sublayer-boundary mix: apply the previous+sublayer's hc_post (comb^T @ res + post * x), then this sublayer's hc_pre --+the fn GEMV over the flattened streams, flat-RMS normalization, and the three+gates (sigmoid pre, sigmoid*mult post, row-softmax + Sinkhorn comb, all on an+n x n held in registers) -- and the pre-mixed layer input.++Semantics are defined by layers/mhc.py's torch reference (bit-comparable in+fp32 up to reduction order); tests/layers/test_mhc.py pins the parity. N+(hc_mult) is a constexpr; only N == 4 is exercised.+"""++from __future__ import annotations++import torch+import triton+import triton.language as tl+++def _mhc_stage1_kernel(+ x_ptr, res_ptr, post_ptr, comb_ptr, fn_ptr,+ res_out_ptr,+ sq_part_ptr, # [T, NS] fp32+ mix_part_ptr, # [T, NS, BLK_MIX] fp32+ H: tl.constexpr, N: tl.constexpr, MIX: tl.constexpr, BLK_MIX: tl.constexpr,+ SPLIT: tl.constexpr, # hidden elems per split (multiple of BLOCK_H)+ BLOCK_H: tl.constexpr,+ NS: tl.constexpr,+ HAS_POST: tl.constexpr,+):+ """Split-K stage of the fused mHC: each program owns one hidden slice of one+ token -- applies hc_post there, stores the updated streams, and reduces its+ partial sq-sum + fn-GEMV contribution over NS splits."""+ t = tl.program_id(0).to(tl.int64)+ s = tl.program_id(1)+ offs_mix = tl.arange(0, BLK_MIX)+ mix_mask = offs_mix < MIX+ offs_n = tl.arange(0, N)++ if HAS_POST:+ b_post = tl.load(post_ptr + t * N + offs_n)+ b_comb = tl.load(comb_ptr + t * N * N + offs_n[:, None] * N + offs_n[None, :])++ sqsum = 0.0+ acc = tl.zeros([BLK_MIX], dtype=tl.float32)+ for h0 in range(s * SPLIT, tl.minimum((s + 1) * SPLIT, H), BLOCK_H):+ offs_h = h0 + tl.arange(0, BLOCK_H)+ h_mask = offs_h < H+ if HAS_POST:+ b_x = tl.load(x_ptr + t * H + offs_h, mask=h_mask, other=0.0).to(tl.float32)+ for n in tl.static_range(N):+ if HAS_POST:+ r_new = tl.zeros([BLOCK_H], dtype=tl.float32)+ for i in tl.static_range(N):+ r_i = tl.load(res_ptr + (t * N + i) * H + offs_h, mask=h_mask, other=0.0).to(tl.float32)+ c_in = tl.sum(tl.where((offs_n == i)[:, None] & (offs_n == n)[None, :], b_comb, 0.0))+ r_new += c_in * r_i+ p_n = tl.sum(tl.where(offs_n == n, b_post, 0.0))+ r_new += p_n * b_x+ # Round through the STORAGE dtype before the sq-sum and fn-GEMV:+ # the torch reference reads back what it stored (dtype-generic --+ # fp16/fp32 residuals must not be silently bf16-rounded).+ r_new = r_new.to(res_out_ptr.dtype.element_ty).to(tl.float32)+ tl.store(res_out_ptr + (t * N + n) * H + offs_h, r_new.to(res_out_ptr.dtype.element_ty), mask=h_mask)+ else:+ r_new = tl.load(res_ptr + (t * N + n) * H + offs_h, mask=h_mask, other=0.0).to(tl.float32)+ tl.store(res_out_ptr + (t * N + n) * H + offs_h, r_new.to(res_out_ptr.dtype.element_ty), mask=h_mask)+ sqsum += tl.sum(r_new * r_new)+ fn_tile = tl.load(+ fn_ptr + offs_mix[:, None] * (N * H) + (n * H + offs_h)[None, :],+ mask=mix_mask[:, None] & h_mask[None, :], other=0.0,+ )+ acc += tl.sum(fn_tile * r_new[None, :], axis=1)++ tl.store(sq_part_ptr + t * NS + s, sqsum)+ tl.store(mix_part_ptr + (t * NS + s) * BLK_MIX + offs_mix, acc)+++def _mhc_stage2_kernel(+ sq_part_ptr, mix_part_ptr, scale_ptr, base_ptr,+ post_out_ptr, comb_out_ptr, pre_out_ptr,+ rms_eps, hc_eps, post_mult,+ SINKHORN: tl.constexpr,+ H: tl.constexpr, N: tl.constexpr, MIX: tl.constexpr, BLK_MIX: tl.constexpr,+ NS: tl.constexpr,+):+ """Reduce the split partials and run the tiny gate math (sigmoid gates,+ row-softmax + in-register 4x4 Sinkhorn); emits pre gates for stage 3."""+ t = tl.program_id(0).to(tl.int64)+ offs_mix = tl.arange(0, BLK_MIX)+ mix_mask = offs_mix < MIX+ offs_s = tl.arange(0, NS)++ sqsum = tl.sum(tl.load(sq_part_ptr + t * NS + offs_s))+ acc = tl.sum(+ tl.load(mix_part_ptr + (t * NS + offs_s)[:, None] * BLK_MIX + offs_mix[None, :]),+ axis=0,+ )+ inv_rms = tl.math.rsqrt(sqsum / (N * H) + rms_eps)+ mixes = acc * inv_rms+ s0 = tl.load(scale_ptr + 0)+ s1 = tl.load(scale_ptr + 1)+ s2 = tl.load(scale_ptr + 2)+ b_base = tl.load(base_ptr + offs_mix, mask=mix_mask, other=0.0)++ is_pre = offs_mix < N+ is_post = (offs_mix >= N) & (offs_mix < 2 * N)+ gate_scale = tl.where(is_pre, s0, tl.where(is_post, s1, s2))+ logits = mixes * gate_scale + b_base+ pre = tl.sigmoid(logits) + hc_eps+ post_new = tl.sigmoid(logits) * post_mult++ offs_n2 = tl.arange(0, N)+ comb_logits = tl.zeros([N, N], dtype=tl.float32)+ for r in tl.static_range(N):+ for c in tl.static_range(N):+ lane = 2 * N + r * N + c+ v = tl.sum(tl.where(offs_mix == lane, logits, 0.0))+ comb_logits += tl.where(+ (offs_n2 == r)[:, None] & (offs_n2 == c)[None, :], v, 0.0+ )+ row_max = tl.max(comb_logits, axis=1)+ e = tl.exp(comb_logits - row_max[:, None])+ comb = e / tl.sum(e, axis=1)[:, None] + hc_eps+ comb = comb / (tl.sum(comb, axis=0)[None, :] + hc_eps)+ for _ in range(SINKHORN - 1):+ comb = comb / (tl.sum(comb, axis=1)[:, None] + hc_eps)+ comb = comb / (tl.sum(comb, axis=0)[None, :] + hc_eps)++ post_g = tl.sum(+ tl.where((offs_mix[None, :] - N) == offs_n2[:, None], post_new[None, :], 0.0),+ axis=1,+ )+ pre_g = tl.sum(+ tl.where(offs_mix[None, :] == offs_n2[:, None], pre[None, :], 0.0), axis=1+ )+ tl.store(post_out_ptr + t * N + offs_n2, post_g)+ tl.store(pre_out_ptr + t * N + offs_n2, pre_g)+ tl.store(comb_out_ptr + t * N * N + offs_n2[:, None] * N + offs_n2[None, :], comb)+++def _mhc_stage3_kernel(+ res_out_ptr, pre_ptr, li_out_ptr,+ H: tl.constexpr, N: tl.constexpr, BLOCK_H: tl.constexpr,+):+ """layer_input = sum_n pre_n * res_new_n, parallel over hidden chunks."""+ t = tl.program_id(0).to(tl.int64)+ hb = tl.program_id(1)+ offs_h = hb * BLOCK_H + tl.arange(0, BLOCK_H)+ h_mask = offs_h < H+ offs_n = tl.arange(0, N)+ pre = tl.load(pre_ptr + t * N + offs_n)+ li = tl.zeros([BLOCK_H], dtype=tl.float32)+ for n in tl.static_range(N):+ r = tl.load(res_out_ptr + (t * N + n) * H + offs_h, mask=h_mask, other=0.0).to(tl.float32)+ li += tl.sum(tl.where(offs_n == n, pre, 0.0)) * r+ tl.store(li_out_ptr + t * H + offs_h, li.to(li_out_ptr.dtype.element_ty), mask=h_mask)+++def mhc_fused_post_pre_triton(+ x: torch.Tensor,+ residual: torch.Tensor,+ post_mix: torch.Tensor | None,+ comb_mix: torch.Tensor | None,+ fn: torch.Tensor,+ hc_scale: torch.Tensor,+ hc_base: torch.Tensor,+ rms_eps: float,+ hc_eps: float,+ post_mult: float,+ sinkhorn_repeat: int,+) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor]:+ """Fused hc_post (skipped when ``post_mix is None``) + hc_pre. Three-stage+ split-K: the GEMV/sq-sum reduction fans out over NS hidden slices. Returns+ (residual_new [T,N,H] bf16, post [T,N,1] fp32, comb [T,N,N] fp32,+ layer_input [T,H] bf16)."""+ t, n, h = residual.shape+ mix = 2 * n + n * n+ assert fn.shape == (mix, n * h) and fn.dtype == torch.float32+ residual = residual.contiguous()+ has_post = post_mix is not None+ dev = residual.device++ res_out = torch.empty_like(residual)+ post_out = torch.empty(t, n, dtype=torch.float32, device=dev)+ comb_out = torch.empty(t, n, n, dtype=torch.float32, device=dev)+ li_out = torch.empty(t, h, dtype=residual.dtype, device=dev)++ block_h = min(512, triton.next_power_of_2(h))+ # NS feeds a tl.arange in stage 2 -> keep it a power of two.+ ns = 1+ while ns * 2 <= min(16, h // block_h):+ ns *= 2+ split = triton.cdiv(triton.cdiv(h, ns), block_h) * block_h+ ns = triton.cdiv(h, split)+ blk_mix = triton.next_power_of_2(mix)+ sq_part = torch.empty(t, ns, dtype=torch.float32, device=dev)+ mix_part = torch.empty(t, ns, blk_mix, dtype=torch.float32, device=dev)+ pre_out = torch.empty(t, n, dtype=torch.float32, device=dev)++ _mhc_stage1_kernel[(t, ns)](+ x.contiguous() if has_post else residual, # dummy ptr when unused+ residual,+ post_mix.contiguous().view(t, n) if has_post else post_out,+ comb_mix.contiguous() if has_post else comb_out,+ fn, res_out, sq_part, mix_part,+ H=h, N=n, MIX=mix, BLK_MIX=blk_mix,+ SPLIT=split, BLOCK_H=block_h, NS=ns,+ HAS_POST=has_post,+ num_warps=4, num_stages=2,+ )+ _mhc_stage2_kernel[(t,)](+ sq_part, mix_part, hc_scale, hc_base,+ post_out, comb_out, pre_out,+ rms_eps, hc_eps, post_mult,+ SINKHORN=sinkhorn_repeat,+ H=h, N=n, MIX=mix, BLK_MIX=blk_mix, NS=ns,+ num_warps=1,+ )+ _mhc_stage3_kernel[(t, triton.cdiv(h, 1024))](+ res_out, pre_out, li_out,+ H=h, N=n, BLOCK_H=min(1024, triton.next_power_of_2(h)),+ num_warps=4,+ )+ return res_out, post_out.view(t, n, 1), comb_out, li_out+++__all__ = ["mhc_fused_post_pre_triton"]diff --git a/python/freetoken/kvcache/__init__.py b/python/freetoken/kvcache/__init__.pyindex c6c0f1bb..41b82f1d 100644--- a/python/freetoken/kvcache/__init__.py+++ b/python/freetoken/kvcache/__init__.py@@ -40,7 +40,8 @@ def resolve_pool_class(model_config: ModelConfig) -> type[BaseKVCachePool]:from .mha_pool import MHAKVCachereturn MHAKVCache- types = {spec.attn_type for spec in specs_fn()}+ specs = list(specs_fn())+ types = {spec.attn_type for spec in specs}if AttnType.DSV4 in types:from .dsv4_paged_pool import DSV4PagedKVCache@@ -50,6 +51,11 @@ def resolve_pool_class(model_config: ModelConfig) -> type[BaseKVCachePool]:return HybridSWAKVCacheif AttnType.DSA in types:+ # kpool-compressed indexer (glm5_next): shadow slab + tail rings.+ if any(s.attn_type == AttnType.DSA and s.index_ratio > 1 for s in specs):+ from .dsa_pool import KpoolDSAKVCache++ return KpoolDSAKVCachefrom .dsa_pool import DSAKVCachereturn DSAKVCache@@ -138,36 +144,31 @@ def create_kvcache_pool(from .mha_pool import MHAKVCache- # Hybrid linear-attention models (e.g. Qwen3.5 GatedDeltaNet) only store paged KV- # for their full-attention layers; the linear layers keep a separate recurrent- # state. Back just those layers and remap their global ids to dense storage slots- # so we don't over-allocate slabs for the (majority) linear layers.+ # Hybrid linear-attention models only store paged KV for their non-linear+ # layers; the linear layers keep a separate recurrent state (LinearStatePool).+ # The linear group emits no paged spec, so the remaining paged spec(s) drive+ # the dispatch below; the paged pool backs JUST those layers via a global-id+ # -> dense-slot remap (layer_ids) so the (majority) linear layers cost no+ # slabs.layer_ids: tuple[int, ...] | None = None- num_kv_heads = model_config.num_kv_heads- head_dim = model_config.head_dim+ kv_specs = [s for s in model_config.kv_cache_group_specs() if s.num_layers > 0]if model_config.has_linear_attention:- specs = [s for s in model_config.kv_cache_group_specs() if s.num_layers > 0]- assert len(specs) == 1, f"expected one paged-KV group, got {[s.name for s in specs]}"- spec = specs[0]- layer_ids = spec.layer_ids- num_kv_heads = spec.num_kv_heads- head_dim = spec.head_dim-- # Latent-KV MLA models declare it on their single full-attention group: they get- # the latent pool (one slab, V aliases K), plus the DSA index-key slab when the- # spec carries indexer dims. The same spec fields drive the KV cost model, so the- # factory and the budget can never disagree.- kv_specs = model_config.kv_cache_group_specs()-- # GQA block-sparse (MiniMax-M3): one full-attention group carrying the index dims- # with mla=False -> the MHA pool plus the index-key slab. The same spec fields- # drive the KV cost model, so the factory and the budget can never disagree.+ assert len(kv_specs) == 1, (+ f"hybrid-linear models support one paged-KV group, got "+ f"{[s.name for s in kv_specs]}"+ )+ layer_ids = kv_specs[0].layer_ids++ # Latent-KV MLA / GQA block-sparse models declare their geometry on the single+ # paged spec; the same spec fields drive the KV cost model, so the factory and+ # the budget can never disagree.from freetoken.attention import AttnType as _AttnTypeif len(kv_specs) == 1 and kv_specs[0].attn_type == _AttnType.BSA:from .bsa_pool import BSAKVCachespec = kv_specs[0]+ assert layer_ids is None, "hybrid-linear x BSA has no pool support yet"return BSAKVCache(num_kv_heads=spec.num_kv_heads,num_layers=model_config.num_layers,@@ -206,35 +207,51 @@ def create_kvcache_pool()if len(kv_specs) == 1 and kv_specs[0].mla:- from .dsa_pool import DSAKVCache, MLAKVCache+ from .dsa_pool import DSAKVCache, KpoolDSAKVCache, MLAKVCachespec = kv_specs[0]+ # With a layer remap the pool allocates len(layer_ids) slabs; without one+ # it backs every model layer (all-MLA models, GLM-5.2).+ num_layers = model_config.num_layers if layer_ids is None else len(layer_ids)if spec.index_head_dim > 0 and spec.num_index_layers > 0:- return DSAKVCache(+ common = dict(latent_dim=spec.head_dim,- num_layers=model_config.num_layers,+ num_layers=num_layers,num_pages=num_pages,page_size=page_size,dtype=dtype,device=device,index_head_dim=spec.index_head_dim,num_index_layers=spec.num_index_layers,+ layer_ids=layer_ids,)+ if spec.index_ratio > 1:+ # kpool tail rings are keyed by Req.table_idx; + 1 covers the dummy request row.+ if num_req_slots is None:+ raise ValueError("kpool pools need num_req_slots (max_running_req + 1)")+ return KpoolDSAKVCache(+ **common,+ index_ratio=spec.index_ratio,+ num_req_slots=num_req_slots,+ )+ return DSAKVCache(**common)return MLAKVCache(latent_dim=spec.head_dim,- num_layers=model_config.num_layers,+ num_layers=num_layers,num_pages=num_pages,page_size=page_size,dtype=dtype,device=device,+ layer_ids=layer_ids,)+ spec = kv_specs[0] if len(kv_specs) == 1 else Nonereturn MHAKVCache(- num_kv_heads=num_kv_heads,+ num_kv_heads=spec.num_kv_heads if spec is not None else model_config.num_kv_heads,num_pages=num_pages,page_size=page_size,num_layers=model_config.num_layers,- head_dim=head_dim,+ head_dim=spec.head_dim if spec is not None else model_config.head_dim,device=device,dtype=dtype,layer_ids=layer_ids,diff --git a/python/freetoken/kvcache/dsa_pool.py b/python/freetoken/kvcache/dsa_pool.pyindex e6a51ac9..13225cec 100644--- a/python/freetoken/kvcache/dsa_pool.py+++ b/python/freetoken/kvcache/dsa_pool.py@@ -5,9 +5,10 @@ per-token latent ``ckv (kv_lora_rank) | kpe (qk_rope_head_dim)`` -- there is noseparate V (``v_cache`` aliases ``k_cache``, same convention as dsv4_paged_pool'ssingle-latent tiers). ``DSAKVCache`` extends it with the DeepSeek-Sparse-Attentionindex-key slab: one ``index_head_dim``-wide bf16 row per token per full-indexer-layer, addressed by the SAME physical rows as the latent slab (page_size == 1), and-``rebuild`` resizes BOTH slabs atomically so the allocator can never hand out a slot-one slab has and the other lacks.+layer, addressed by the SAME physical rows as the latent slab (GLM-5.2, page 1;+``KpoolDSAKVCache`` overrides the geometry to a 1/ratio shadow). ``rebuild``+resizes ALL slabs atomically so the allocator can never hand out a slot one slab+has and the other lacks.Storage lives here -- not in the attention backend -- so the engine's rebuild path(``MHAKVCache.rebuild``-shaped: fresh allocation, object identity preserved, views@@ -27,6 +28,10 @@ class MLAKVCache(BaseKVCachePool):The leading singleton keeps the buffer shape-compatible with MHAKVCache's(tokens = shape[2] * shape[3]).++ ``layer_ids`` backs only a SUBSET of the model's layers (hybrid linear x+ MLA/DSA) while callers keep addressing by GLOBAL layer id -- the same remap+ contract as MHAKVCache; None keeps the identity mapping."""def __init__(@@ -37,14 +42,23 @@ class MLAKVCache(BaseKVCachePool):page_size: int,dtype: torch.dtype,device: torch.device,+ layer_ids: "tuple[int, ...] | None" = None,) -> None:self._latent_dim = latent_dim- self._num_layers = num_layers+ if layer_ids is None:+ self._num_layers = num_layers+ self._layer_index: dict[int, int] | None = None+ else:+ self._num_layers = len(layer_ids)+ self._layer_index = {int(g): i for i, g in enumerate(layer_ids)}self._page_size = page_sizeself._dtype = dtypeself._device = deviceself._alloc(num_pages)+ def _local_layer(self, layer_id: int) -> int:+ return layer_id if self._layer_index is None else self._layer_index[layer_id]+def _alloc(self, num_pages: int) -> None:self._num_pages = num_pagesself._kv_buffer = torch.empty(@@ -53,10 +67,12 @@ class MLAKVCache(BaseKVCachePool):dtype=self._dtype,)- # -- views ------------------------------------------------------------------+ # -- views (addressed by GLOBAL layer id; remapped when layer_ids was given) --def k_cache(self, layer_id: int) -> torch.Tensor:"""Paged latent view ``[num_pages, page_size, latent_dim]``."""- return self._kv_buffer[0, layer_id].view(self._num_pages, self._page_size, -1)+ return self._kv_buffer[0, self._local_layer(layer_id)].view(+ self._num_pages, self._page_size, -1+ )def v_cache(self, layer_id: int) -> torch.Tensor:# MLA: K == V (single latent); same buffer, dsv4_paged_pool precedent.@@ -64,7 +80,7 @@ class MLAKVCache(BaseKVCachePool):def latent_rows(self, layer_id: int) -> torch.Tensor:"""Row-flat latent view ``[num_pages * page_size, latent_dim]``."""- return self._kv_buffer[0, layer_id].view(-1, self._latent_dim)+ return self._kv_buffer[0, self._local_layer(layer_id)].view(-1, self._latent_dim)# -- writes -----------------------------------------------------------------def store_kv(@@ -77,8 +93,8 @@ class MLAKVCache(BaseKVCachePool):"""Scatter this forward's latent rows: ``c_kv`` [T, kv_lora_rank] and``k_rope`` [T, qk_rope_head_dim] land in the row's two halves.- v0: two narrow ``index_put_`` scatters. TODO: generalize kernel/csrc- store.cu to a two-width fused store and route this through it.+ Two narrow ``index_put_`` scatters. TODO: fuse into kernel/csrc+ store.cu (two-width store)."""rows = self.latent_rows(layer_id)split = rows.shape[1] - k_rope.shape[-1]@@ -141,20 +157,29 @@ class DSAKVCache(MLAKVCache):device: torch.device,index_head_dim: int,num_index_layers: int,+ layer_ids: "tuple[int, ...] | None" = None,) -> None:self._index_head_dim = index_head_dimself._num_index_layers = num_index_layers- super().__init__(latent_dim, num_layers, num_pages, page_size, dtype, device)+ super().__init__(+ latent_dim, num_layers, num_pages, page_size, dtype, device,+ layer_ids=layer_ids,+ )++ def _index_rows(self, num_pages: int) -> int:+ """Index-slab row count: one row per token (KpoolDSAKVCache overrides to the+ 1/ratio shadow + scratch layout)."""+ return num_pages * self._page_sizedef _alloc(self, num_pages: int) -> None:- # Both slabs in one allocation step: rebuild can never leave the pool with a- # grown latent slab and a stale index slab (the OOB class this type exists for).+ # Both slabs in one allocation step so rebuild can never leave one grown+ # and the other stale.super()._alloc(num_pages)# bf16 == the 2 bytes/token/layer the KV cost model budgets for this slab# (cache_status._kv_cost_model); keep the two in lockstep.self._index_k_buffer = torch.zeros(self._num_index_layers,- num_pages * self._page_size,+ self._index_rows(num_pages),self._index_head_dim,dtype=torch.bfloat16,device=self._device,@@ -180,4 +205,56 @@ class DSAKVCache(MLAKVCache):self._index_k_buffer[slot][out_loc] = k-__all__ = ["MLAKVCache", "DSAKVCache"]+class KpoolDSAKVCache(DSAKVCache):+ """DSAKVCache with the index slab as a 1/ratio SHADOW of the KV pages, plus+ per-request scratch rows and tail rings.++ A pooled entry lives at ``token_slot // index_ratio`` (``page_size %+ index_ratio == 0``), so compressed rows follow page sharing/eviction for+ free. Rows whose pool does not close write to the request's scratch row+ instead (scoring never reads it). The tail rings hold the in-progress+ pool's raw K + gate at ``pos % index_ratio`` for pools straddling two+ forwards; never cleared -- prefix-cache resume points are page-aligned,+ so a stale ring is never read.+ """++ def __init__(self, *args, num_req_slots: int, index_ratio: int, **kwargs) -> None:+ self._num_req_slots = num_req_slots+ self._index_ratio = index_ratio+ super().__init__(*args, **kwargs)+ assert self._page_size % index_ratio == 0, (+ f"kpool needs page_size ({self._page_size}) divisible by "+ f"index_ratio ({index_ratio})"+ )++ def _index_rows(self, num_pages: int) -> int:+ # 1/ratio shadow of every token slot + one scratch row per request slot.+ return num_pages * self._page_size // self._index_ratio + self._num_req_slots++ @property+ def cmp_scratch_base(self) -> int:+ """First scratch row (== shadow row count); request ``table_idx`` offsets it."""+ return self._num_pages * self._page_size // self._index_ratio++ def _alloc(self, num_pages: int) -> None:+ super()._alloc(num_pages)+ self._tail_k = torch.zeros(+ self._num_index_layers, self._num_req_slots, self._index_ratio,+ self._index_head_dim, dtype=torch.bfloat16, device=self._device,+ )+ self._tail_gate = torch.zeros_like(self._tail_k)++ def rebuild(self, num_pages: int) -> None:+ self._tail_k = None+ self._tail_gate = None+ super().rebuild(num_pages)++ def tail_k(self, slot: int) -> torch.Tensor:+ """Tail raw keys for an indexer layer slot: ``[num_req_slots, ratio, head_dim]``."""+ return self._tail_k[slot]++ def tail_gate(self, slot: int) -> torch.Tensor:+ return self._tail_gate[slot]+++__all__ = ["MLAKVCache", "DSAKVCache", "KpoolDSAKVCache"]diff --git a/python/freetoken/layers/__init__.py b/python/freetoken/layers/__init__.pyindex 2dbd6373..31c19bf9 100644--- a/python/freetoken/layers/__init__.py+++ b/python/freetoken/layers/__init__.py@@ -1,4 +1,10 @@-from .activation import gelu_and_mul, gelu_tanh_and_mul, silu_and_mul, swigluoai_and_mul+from .activation import (+ gelu_and_mul,+ gelu_tanh_and_mul,+ silu_and_mul,+ swiglu_clamp_and_mul,+ swigluoai_and_mul,+)from .base import BaseOP, OPList, StateLessOPfrom .embedding import ParallelLMHead, VocabParallelEmbeddingfrom .linear import (@@ -23,6 +29,7 @@ __all__ = ["gelu_and_mul","gelu_tanh_and_mul","swigluoai_and_mul",+ "swiglu_clamp_and_mul","BaseOP","StateLessOP","OPList",diff --git a/python/freetoken/layers/activation.py b/python/freetoken/layers/activation.pyindex 93602b6c..ae1d9881 100644--- a/python/freetoken/layers/activation.py+++ b/python/freetoken/layers/activation.py@@ -56,4 +56,19 @@ def swigluoai_and_mul(return swigluoai_and_mul(x, out=out, alpha=alpha, limit=limit)-__all__ = ["silu_and_mul", "gelu_and_mul", "gelu_tanh_and_mul", "swigluoai_and_mul"]+def swiglu_clamp_and_mul(+ x, out=None, *, alpha: float = 1.0, limit: float = 10.0+):+ """GLM-5.3 clamped SwiGLU over UNINTERLEAVED halves: ``clamp(gate, max=limit) * sigmoid(alpha * gate) * clamp(up, +-limit)``."""+ from freetoken.kernel.triton.activation import swiglu_clamp_and_mul++ return swiglu_clamp_and_mul(x, out=out, alpha=alpha, limit=limit)+++__all__ = [+ "silu_and_mul",+ "gelu_and_mul",+ "gelu_tanh_and_mul",+ "swigluoai_and_mul",+ "swiglu_clamp_and_mul",+]diff --git a/python/freetoken/layers/mhc.py b/python/freetoken/layers/mhc.pynew file mode 100644index 00000000..1cd6459d--- /dev/null+++ b/python/freetoken/layers/mhc.py@@ -0,0 +1,150 @@+"""mHC -- Manifold-Constrained Hyper-Connections (GLM-5.3-Flash; arXiv 2512.24880).++The residual stream is widened to ``hc_mult`` (n) parallel streams. Around every+sublayer the streams are mixed by three learned, token-dependent maps computed+from ONE fp32 GEMM over the flattened streams (``fn [2n+n^2, n*hidden]``),+RMS-normalized over the full ``n*hidden`` vector and split into:++* ``pre_mix [n]`` sigmoid gates: the sublayer input is ``sum_i pre_i * res_i``+* ``post_mix [n]`` sigmoid*mult gates: how much sublayer output enters each stream+* ``comb_mix [n, n]`` softmax + Sinkhorn-projected (approximately doubly-stochastic)+ stream-mixing matrix -- the "manifold constraint": mixing+ neither amplifies nor loses residual mass.++``mhc_post`` then rebuilds the streams: ``out_j = sum_i comb_ij * res_i + post_j * x``.++Semantics match vLLM's reference (``model_executor/kernels/mhc/torch.py``,+PR #53906) bit-for-bit in fp32; the per-layer weights are ``hc_{attn,ffn}_fn`` /+``_scale`` / ``_base`` from the checkpoint. This torch implementation is the+correctness baseline; the fused triton kernel (``kernel/triton/mhc.py``,+dispatched by ``mhc_fused_post_pre`` below) replaces it on CUDA and is+validated against this file.+"""++from __future__ import annotations++import torch+++def hc_expand(x: torch.Tensor, n: int) -> torch.Tensor:+ """[T, hidden] -> [T, n, hidden] by replication (model entry)."""+ return x.unsqueeze(1).expand(-1, n, -1).contiguous()+++def hc_contract(x: torch.Tensor) -> torch.Tensor:+ """[T, n, hidden] -> [T, hidden] by averaging (model exit)."""+ return x.mean(dim=1)+++def mhc_pre(+ residual: torch.Tensor, # [T, n, hidden] bf16+ fn: torch.Tensor, # [2n + n^2, n*hidden] fp32+ hc_scale: torch.Tensor, # [3] fp32+ hc_base: torch.Tensor, # [2n + n^2] fp32+ rms_eps: float,+ hc_eps: float,+ post_mult: float,+ sinkhorn_repeat: int,+) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor]:+ """Returns (post_mix [T, n, 1] fp32, comb_mix [T, n, n] fp32,+ layer_input [T, hidden] bf16)."""+ n, hidden = residual.shape[-2], residual.shape[-1]+ t = residual.shape[0]++ x = residual.reshape(t, n * hidden).to(torch.float32)+ mixes = x @ fn.t()+ # RMS over the FULL flattened n*hidden vector (not per stream).+ mixes = mixes * torch.rsqrt(x.square().sum(-1, keepdim=True) / (n * hidden) + rms_eps)++ pre_mix = torch.sigmoid(mixes[:, :n] * hc_scale[0] + hc_base[:n]) + hc_eps+ post_mix = torch.sigmoid(mixes[:, n : 2 * n] * hc_scale[1] + hc_base[n : 2 * n])+ post_mix = post_mix * post_mult++ comb = mixes[:, 2 * n :].view(t, n, n) * hc_scale[2] + hc_base[2 * n :].view(1, n, n)+ comb = torch.softmax(comb, dim=-1) + hc_eps+ # Sinkhorn-Knopp projection toward the doubly-stochastic manifold: alternate+ # column / row normalization, ``sinkhorn_repeat`` column steps in total.+ comb = comb / (comb.sum(dim=-2, keepdim=True) + hc_eps)+ for _ in range(sinkhorn_repeat - 1):+ comb = comb / (comb.sum(dim=-1, keepdim=True) + hc_eps)+ comb = comb / (comb.sum(dim=-2, keepdim=True) + hc_eps)++ layer_input = (+ (pre_mix.unsqueeze(-1) * residual.to(torch.float32)).sum(dim=1).to(residual.dtype)+ )+ return post_mix.view(t, n, 1), comb, layer_input+++def mhc_post(+ x: torch.Tensor, # [T, hidden] sublayer output+ residual: torch.Tensor, # [T, n, hidden]+ post_mix: torch.Tensor, # [T, n, 1] fp32+ comb_mix: torch.Tensor, # [T, n, n] fp32+) -> torch.Tensor:+ """out_j = sum_i comb_ij * residual_i + post_j * x; returns [T, n, hidden]."""+ mixed = torch.einsum(+ "tij,tih->tjh", comb_mix.to(torch.float32), residual.to(torch.float32)+ )+ post = post_mix.to(torch.float32) * x.unsqueeze(-2).to(torch.float32)+ return (mixed + post).to(residual.dtype)+++def mhc_fused_post_pre_torch(+ x: torch.Tensor,+ residual: torch.Tensor,+ post_mix: torch.Tensor,+ comb_mix: torch.Tensor,+ fn: torch.Tensor,+ hc_scale: torch.Tensor,+ hc_base: torch.Tensor,+ rms_eps: float,+ hc_eps: float,+ post_mult: float,+ sinkhorn_repeat: int,+) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor]:+ """Decomposed reference: hc_post then hc_pre; the fused triton kernel+ replaces it on CUDA and is validated against this function."""+ residual_new = mhc_post(x, residual, post_mix, comb_mix)+ post_new, comb_new, layer_input = mhc_pre(+ residual_new, fn, hc_scale, hc_base, rms_eps, hc_eps, post_mult, sinkhorn_repeat+ )+ return residual_new, post_new, comb_new, layer_input+++def mhc_fused_post_pre(+ x: torch.Tensor,+ residual: torch.Tensor,+ post_mix: torch.Tensor,+ comb_mix: torch.Tensor,+ fn: torch.Tensor,+ hc_scale: torch.Tensor,+ hc_base: torch.Tensor,+ rms_eps: float,+ hc_eps: float,+ post_mult: float,+ sinkhorn_repeat: int,+) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor]:+ """Apply the previous sublayer's hc_post, then this sublayer's hc_pre on the+ updated streams. The fused triton kernel serves every batch size on CUDA+ (deliberately no T threshold); the decomposed torch path serves CPU/tests."""+ if residual.is_cuda:+ from freetoken.kernel.triton.mhc import mhc_fused_post_pre_triton++ return mhc_fused_post_pre_triton(+ x, residual, post_mix, comb_mix, fn, hc_scale, hc_base,+ rms_eps, hc_eps, post_mult, sinkhorn_repeat,+ )+ return mhc_fused_post_pre_torch(+ x, residual, post_mix, comb_mix, fn, hc_scale, hc_base,+ rms_eps, hc_eps, post_mult, sinkhorn_repeat,+ )+++__all__ = [+ "hc_expand",+ "hc_contract",+ "mhc_pre",+ "mhc_post",+ "mhc_fused_post_pre",+ "mhc_fused_post_pre_torch",+]diff --git a/python/freetoken/models/config.py b/python/freetoken/models/config.pyindex 229cce81..8ce69d65 100644--- a/python/freetoken/models/config.py+++ b/python/freetoken/models/config.py@@ -20,18 +20,24 @@ def vision_load_enabled() -> bool:def detect_expert_quant(hf_config: Any) -> str:"""Routed-expert quantization from a checkpoint's ``quantization_config``: ``"nvfp4"`` for- a ModelOpt FP4 build, else the lowercased algo string (``"none"`` when unquantized). Models- with mixed-precision configs (e.g. qwen3_5_moe) need their own detector."""+ a ModelOpt FP4 build (``quant_algo: NVFP4``) OR an llm-compressor NVFP4 export+ (``quant_method: compressed-tensors`` + ``format: nvfp4-pack-quantized``, e.g.+ RedHatAI/GLM-5.3-Flash-NVFP4), else the lowercased algo string (``"none"`` when+ unquantized). Models with mixed-precision configs (e.g. qwen3_5_moe) need their+ own detector."""quant = getattr(hf_config, "quantization_config", None)if quant is None:return "none"- if isinstance(quant, dict):- algo = quant.get("quant_algo") or quant.get("quant_method")- else:- algo = getattr(quant, "quant_algo", None) or getattr(quant, "quant_method", None)+ get = quant.get if isinstance(quant, dict) else (lambda k, d=None: getattr(quant, k, d))+ algo = get("quant_algo") or get("quant_method")if algo is None:return "none"- return "nvfp4" if "fp4" in str(algo).lower() else str(algo).lower()+ if "fp4" in str(algo).lower():+ return "nvfp4"+ # exact "nvfp4" (not the "fp4" substring) so MXFP4 exports don't misroute+ if "nvfp4" in str(get("format") or "").lower():+ return "nvfp4"+ return str(algo).lower()def detect_compressed_tensors_nvfp4(hf_config: Any) -> bool:@@ -97,8 +103,9 @@ class KVCacheGroupSpec:mla: bool = Falseindex_head_dim: int = 0num_index_layers: int = 0- # QSA compression: one index-key row per index_ratio tokens (1 keeps the BSA/DSA- # per-token slab). The pool factory and the cost model divide by the same value.+ # Grouped index-key compression: one index-key row per ``index_ratio`` tokens+ # (QSA groups, glm5_next kpool pools; 1 keeps the per-token BSA/DSA slab). The+ # pool factory and the KV cost model divide by the same value.index_ratio: int = 1# Attention-type taxonomy value for this group; drives the backend capability# matrix and (with the pool factory) selects the KV pool family.@@ -137,8 +144,9 @@ class FullAttentionGroupConfig(BaseAttentionGroupConfig):mla: bool = Falseindex_head_dim: int = 0num_index_layers: int = 0- # QSA compression ratio (> 1 -> AttnType.QSA): index-key rows are per token group,- # so the index slab costs index_head_dim * num_index_layers * 2 // index_ratio per token.+ # Grouped index-key compression ratio (see KVCacheGroupSpec.index_ratio).+ # GQA + ratio > 1 -> AttnType.QSA (Qwen3.8); MLA + ratio > 1 -> the glm5_next+ # kpool DSA layout (attn type stays DSA; the pool factory branches on mla).index_ratio: int = 1@@ -165,6 +173,9 @@ class LinearGatedDeltaGroupConfig(BaseAttentionGroupConfig):conv_kernel_dim: int# Output-gate activation name ("silu", "sigmoid"), forwarded to rms_norm_gated.output_gate: str+ # "gdn" and "kda" share the same state geometry (one LinearStatePool serves+ # both); the variant selects the kernels.+ variant: Literal["gdn", "kda"] = "gdn"@dataclass(frozen=True)@@ -307,6 +318,10 @@ class ModelConfig:# DSA indexer geometry the model module needs. Opaque to model-agnostic engine code;# None for every other model.glm_dsa_args: Any | None = None+ # GLM-5.3-Flash (glm5_next) payload (Glm5NextArgs): NoPE-MLA dims, the kpool indexer+ # geometry, the KDA head config, and the mHC knobs. Opaque to model-agnostic engine+ # code; None for every other model.+ glm5_args: Any | None = None# MiniMax-M3 (minimax_m3) payload (MiniMaxM3Args): the block-sparse indexer geometry# (index heads/dim, top-k blocks, init/local blocks, sparse layer set) plus the# swigluoai/dense-MLP scalars the model module needs. Opaque to model-agnostic enginediff --git a/python/freetoken/models/glm5_next/__init__.py b/python/freetoken/models/glm5_next/__init__.pynew file mode 100644index 00000000..c1c05140--- /dev/null+++ b/python/freetoken/models/glm5_next/__init__.py@@ -0,0 +1,10 @@+from .config import parse_config+from .model import Glm5NextForCausalLM+from .weight import iter_weights, load_nvfp4_expert_sources++__all__ = [+ "Glm5NextForCausalLM",+ "parse_config",+ "iter_weights",+ "load_nvfp4_expert_sources",+]diff --git a/python/freetoken/models/glm5_next/args.py b/python/freetoken/models/glm5_next/args.pynew file mode 100644index 00000000..c25fef48--- /dev/null+++ b/python/freetoken/models/glm5_next/args.py@@ -0,0 +1,225 @@+"""GLM-5.3-Flash (``glm5_next``) hyperparameters.++GLM-5.3-Flash is the first hybrid GLM: 34 of 45 decoder layers run KDA linear+attention (Kimi-Delta-Attention: per-channel gated delta rule with separate q/k/v+short convolutions) and the remaining 11 run DeepSeek-sparse attention -- MLA+latent KV with a Lightning indexer whose K cache is *pool-compressed*+(``index_kpool`` tokens fold into one stored entry). The main attention is NoPE+(``qk_rope_head_dim == 0``, ``mla_use_nope``): no rotary embedding on Q/K at all;+positional information enters only through the indexer's pool-compression APE.+The residual stream is widened by mHC (Manifold-Constrained Hyper-Connections,+``hc_mult`` streams mixed by a Sinkhorn-projected matrix around every sublayer).++This payload carries everything the model module needs beyond the generic+``ModelConfig`` fields; it is stashed on ``ModelConfig.glm5_args`` (opaque to the+engine). Field spellings follow the checkpoint's ``config.json`` (transformers+5.16 ``Glm5NextTextConfig``); ``load_args`` folds the checkpoint aliases+(``hc_mult``, ``hc_sinkhorn_iters``, ``mla_use_nope``, the nested+``linear_attn_config`` dict) the same way vLLM's config class does, so a future+flattened checkpoint keeps loading.+"""++from __future__ import annotations++from dataclasses import dataclass+from typing import Any, Tuple++# Layer-type strings used by the checkpoint's ``layer_types`` field.+KDA_LAYER = "linear_attention"+DSA_LAYER = "deepseek_sparse_attention"+++@dataclass(frozen=True)+class Glm5NextArgs:+ hidden_size: int+ num_heads: int+ # ---- MLA (the 11 "deepseek_sparse_attention" layers) ----+ q_lora_rank: int+ kv_lora_rank: int+ qk_nope_head_dim: int+ qk_rope_head_dim: int # 0: NoPE -- main attention carries no rotary dims+ v_head_dim: int+ mla_nope: bool+ norm_eps: float+ max_position: int+ # ---- DSA indexer ----+ index_n_heads: int+ index_head_dim: int+ index_topk: int+ indexer_types: Tuple[str, ...]+ indexer_rope_interleave: bool+ # kpool compression: every ``index_kpool`` indexer-K entries pool into one+ # stored entry (softmax(gate + APE)-weighted sum); top-k selects pools+ # (select_k = index_topk // index_kpool) and ``always_select_tail``+ # force-includes the in-progress tail pool.+ index_kpool: int+ index_kpool_compress: bool+ index_kpool_always_select_tail: bool+ # ---- KDA linear attention (the 34 "linear_attention" layers) ----+ linear_num_heads: int+ linear_head_dim: int+ linear_conv_kernel_dim: int+ linear_lower_bound: float+ # ---- per-layer layout ----+ layer_types: Tuple[str, ...]+ mlp_layer_types: Tuple[str, ...]+ # ---- mHC (Manifold-Constrained Hyper-Connections) ----+ mhc: bool+ mhc_num_residual_streams: int+ hc_eps: float+ mhc_sinkhorn_iterations: int+ mhc_tau: float+ mhc_post_mult_value: float+ mhc_no_norm_weight: bool+ # ---- misc ----+ swiglu_limit: float | None+ rope_theta: float # indexer-side only; the main attention is NoPE++ @property+ def qk_head_dim(self) -> int:+ return self.qk_nope_head_dim + self.qk_rope_head_dim++ @property+ def latent_dim(self) -> int:+ """Width of one MLA latent row in the paged pool: ckv | kpe."""+ return self.kv_lora_rank + self.qk_rope_head_dim++ @property+ def kda_layer_ids(self) -> Tuple[int, ...]:+ return tuple(i for i, t in enumerate(self.layer_types) if t == KDA_LAYER)++ @property+ def dsa_layer_ids(self) -> Tuple[int, ...]:+ return tuple(i for i, t in enumerate(self.layer_types) if t == DSA_LAYER)++ def is_kda_layer(self, layer_id: int) -> bool:+ return self.layer_types[layer_id] == KDA_LAYER+++def _require_mhc(mhc: bool) -> bool:+ """The decoder layer wires the hyper-connection tensors unconditionally+ (model.py touches hc_attn_fn on every forward), so an mhc=False checkpoint+ would die with an AttributeError mid-forward -- refuse it at parse until a+ real one exists to implement against."""+ if not mhc:+ raise NotImplementedError(+ "glm5_next requires mhc=True (manifold-constrained hyper-connections); "+ "an mhc=False checkpoint has no implementation yet."+ )+ return mhc+++def _get(cfg: Any, name: str, default: Any = None) -> Any:+ return getattr(cfg, name, default)+++def load_args(hf_config: Any) -> Glm5NextArgs:+ """Build ``Glm5NextArgs`` from the checkpoint config (nested ``text_config`` or a+ flat text-only config). Raw checkpoint spellings win; the vLLM-normalized names+ are accepted as fallbacks."""+ text = _get(hf_config, "text_config", hf_config)++ num_layers = int(text.num_hidden_layers)+ layer_types = tuple(_get(text, "layer_types", ()) or ())+ if not layer_types:+ raise ValueError("glm5_next config is missing layer_types")+ if len(layer_types) != num_layers:+ raise ValueError(+ f"layer_types has {len(layer_types)} entries for {num_layers} layers"+ )+ unknown = sorted(set(layer_types) - {KDA_LAYER, DSA_LAYER})+ if unknown:+ raise ValueError(f"unsupported layer_types entries: {unknown}")++ mlp_layer_types = tuple(_get(text, "mlp_layer_types", ()) or ())+ if not mlp_layer_types:+ # Older-schema fallback (mirrors vLLM): derive from first_k_dense_replace.+ first_dense = int(_get(text, "first_k_dense_replace", 0) or 0)+ mlp_layer_types = ("dense",) * first_dense + ("sparse",) * (+ num_layers - first_dense+ )++ # Checkpoint alias folding (transformers 5.16 spellings first).+ mla_nope = _get(text, "mla_use_nope", _get(text, "mla_nope", False))+ hc_mult = _get(text, "hc_mult", _get(text, "mhc_num_residual_streams", 4))+ hc_sinkhorn = _get(+ text, "hc_sinkhorn_iters", _get(text, "mhc_sinkhorn_iterations", 20)+ )++ # KDA head geometry ships as the nested ``linear_attn_config`` dict; a future+ # flattened schema would carry vLLM-style ``linear_*`` top-level fields.+ linear_cfg = _get(text, "linear_attn_config", None)+ if linear_cfg is not None and not isinstance(linear_cfg, dict):+ linear_cfg = {+ k: getattr(linear_cfg, k)+ for k in (+ "num_heads",+ "head_dim",+ "short_conv_kernel_size",+ "gate_lower_bound",+ )+ if hasattr(linear_cfg, k)+ }+ linear_cfg = linear_cfg or {}+ linear_num_heads = int(+ linear_cfg.get("num_heads", _get(text, "linear_num_heads", 0))+ )+ linear_head_dim = int(linear_cfg.get("head_dim", _get(text, "linear_head_dim", 0)))+ linear_conv = int(+ linear_cfg.get(+ "short_conv_kernel_size", _get(text, "linear_conv_kernel_dim", 4)+ )+ )+ linear_lower_bound = float(+ linear_cfg.get("gate_lower_bound", _get(text, "linear_lower_bound", -5.0))+ )+ if KDA_LAYER in layer_types and (linear_num_heads <= 0 or linear_head_dim <= 0):+ raise ValueError("glm5_next config is missing the KDA linear_attn_config dims")++ # The main attention is NoPE; rope exists only on the indexer side. The+ # checkpoint ships no rope_theta -- fall back to the transformers default.+ rope = _get(text, "rope_parameters", None) or {}+ rope_theta = float(rope.get("rope_theta", _get(text, "rope_theta", 10000.0)))++ swiglu_limit = _get(text, "swiglu_limit", None)++ return Glm5NextArgs(+ hidden_size=int(text.hidden_size),+ num_heads=int(text.num_attention_heads),+ q_lora_rank=int(text.q_lora_rank),+ kv_lora_rank=int(text.kv_lora_rank),+ qk_nope_head_dim=int(text.qk_nope_head_dim),+ qk_rope_head_dim=int(text.qk_rope_head_dim),+ v_head_dim=int(text.v_head_dim),+ mla_nope=bool(mla_nope),+ norm_eps=float(text.rms_norm_eps),+ max_position=int(text.max_position_embeddings),+ index_n_heads=int(_get(text, "index_n_heads", 0) or 0),+ index_head_dim=int(_get(text, "index_head_dim", 0) or 0),+ index_topk=int(_get(text, "index_topk", 0) or 0),+ indexer_types=tuple(_get(text, "indexer_types", ()) or ()),+ indexer_rope_interleave=bool(_get(text, "indexer_rope_interleave", False)),+ index_kpool=int(_get(text, "index_kpool", 1) or 1),+ index_kpool_compress=bool(_get(text, "index_kpool_compress", False)),+ index_kpool_always_select_tail=bool(+ _get(text, "index_kpool_always_select_tail", False)+ ),+ linear_num_heads=linear_num_heads,+ linear_head_dim=linear_head_dim,+ linear_conv_kernel_dim=linear_conv,+ linear_lower_bound=linear_lower_bound,+ layer_types=layer_types,+ mlp_layer_types=mlp_layer_types,+ mhc=_require_mhc(bool(_get(text, "mhc", False))),+ mhc_num_residual_streams=int(hc_mult),+ hc_eps=float(_get(text, "hc_eps", 1e-6)),+ mhc_sinkhorn_iterations=int(hc_sinkhorn),+ mhc_tau=float(_get(text, "mhc_tau", 0.05)),+ mhc_post_mult_value=float(_get(text, "mhc_post_mult_value", 2.0)),+ mhc_no_norm_weight=bool(_get(text, "mhc_no_norm_weight", False)),+ swiglu_limit=(None if swiglu_limit is None else float(swiglu_limit)),+ rope_theta=rope_theta,+ )+++__all__ = ["Glm5NextArgs", "load_args", "KDA_LAYER", "DSA_LAYER"]diff --git a/python/freetoken/models/glm5_next/attention.py b/python/freetoken/models/glm5_next/attention.pynew file mode 100644index 00000000..1ab6c340--- /dev/null+++ b/python/freetoken/models/glm5_next/attention.py@@ -0,0 +1,184 @@+"""GLM-5.3-Flash NoPE Multi-head Latent Attention with a kpool DSA indexer.++MLA weight-absorption as in glm_moe_dsa (kv_b absorbed into Q and onto the+output; the paged pool stores one latent row per token) with two GLM-5.3+differences:++* **NoPE** (``mla_use_nope``, ``qk_rope_head_dim == 0``): no rotary embedding+ anywhere in the main attention -- Q is all-nope [T, H, 256], the latent row is+ bare ckv (512, no kpe half). All rope plumbing degenerates to zero-width+ tensors, which the DSA backend's cat/scatter handle natively. Positional+ information enters ONLY through the indexer's pool-compression APE.+* **kpool indexer**: every DSA layer owns its indexer (no IndexShare). The+ indexer K cache stores one entry per ``index_kpool`` (4) tokens: a per-channel+ ``softmax(gate + ape)``-weighted sum of the raw keys. Scoring runs at pool+ granularity (select_k = index_topk / kpool) and the selected pools expand back+ to token rows, with the in-progress tail pool force-included+ (``index_kpool_always_select_tail``). This module owns the PROJECTIONS (wq_b /+ wk+k_norm / weights_proj / compress gate + APE); pooling, scoring, selection+ and the tail buffer live in the backend (attention/dsa_indexer_kpool.py).++Faithfulness note: the reference stack stores pooled entries as+Hadamard-rotated fp8 (a quantization device; the rotation cancels in the dot+product). FreeToken stores pooled entries in bf16 -- mathematically the same+score with strictly less quantization error -- matching the GLM-5.2 precedent+of bf16 indexer keys.+TODO: fp8 index slab (+ Hadamard rotation, upstream parity) to halve the slab bytes.+"""++from __future__ import annotations++from typing import TYPE_CHECKING++import torch+from freetoken.core import get_global_ctx+from freetoken.layers import BaseOP, LinearReplicated, RMSNorm+# Shared with GLM-5.2 (weight.py imports privately from the same package).+from freetoken.models.glm_moe_dsa.attention import _IdxLayerNorm, _make_proj+from freetoken.utils import nvtx_annotate++if TYPE_CHECKING:+ from freetoken.models.config import ModelConfig+++class Glm5NextIndexer(BaseOP):+ """kpool DSA indexer projections (every DSA layer owns one; no rope -- the+ checkpoint's NoPE geometry leaves ``qk_rope_head_dim == 0`` so position enters+ only via the compression APE).++ Kept bf16 in every quant mode: small (~17 MB/layer) and the top-k boundary is+ precision-sensitive (same reasoning as glm_moe_dsa's indexer).+ """++ def __init__(self, config: ModelConfig, layer_id: int):+ args = config.glm5_args+ self.n_heads = args.index_n_heads+ self.head_dim = args.index_head_dim+ self.kpool = args.index_kpool+ self.wq_b = LinearReplicated(+ args.q_lora_rank, self.n_heads * self.head_dim, has_bias=False+ )+ self.wk = LinearReplicated(args.hidden_size, self.head_dim, has_bias=False)+ self.k_norm = _IdxLayerNorm(self.head_dim, eps=1e-6)+ self.weights_proj = LinearReplicated(+ args.hidden_size, self.n_heads, has_bias=False+ )+ # Pool-compression parameters (checkpoint names, no ".weight" suffix on+ # the gate: it is stored as a bare [head_dim, hidden] tensor).+ self.index_kpool_compress_gate = torch.empty(+ self.head_dim, args.hidden_size+ )+ # Per-pool-slot position bias, fp32 (models/weight.py exempts it from the+ # model-dtype downcast alongside A_log/dt_bias).+ self.index_kpool_compress_ape = torch.empty(+ self.kpool, self.head_dim, dtype=torch.float32+ )++ def compute(self, x: torch.Tensor, q_resid: torch.Tensor) -> "DSAIndexerInputs":+ """Per-token indexer projections as a typed inputs object: q [T, Hi, Di],+ k [T, Di], weights [T, Hi] fp32, plus the kpool gate scores [T, Di] and+ the [kpool, Di] APE parameter (passed per call; the backend keeps no copy)."""+ from freetoken.attention.dsa import DSAIndexerInputs++ t = x.shape[0]+ q = self.wq_b.forward(q_resid).view(t, self.n_heads, self.head_dim)+ k = self.k_norm.forward(self.wk.forward(x))+ w = self.weights_proj.forward(x).float() * (self.n_heads**-0.5)+ gate = torch.nn.functional.linear(x, self.index_kpool_compress_gate)+ return DSAIndexerInputs(+ q=q, k=k, w=w, gate=gate, ape=self.index_kpool_compress_ape+ )+++class Glm5NextAttention(BaseOP):+ def __init__(self, config: ModelConfig, layer_id: int):+ args = config.glm5_args+ self.layer_id = layer_id+ self.indexer = Glm5NextIndexer(config, layer_id)+ self.num_heads = args.num_heads+ self.qk_nope_head_dim = args.qk_nope_head_dim+ self.qk_rope_head_dim = args.qk_rope_head_dim # 0 (NoPE)+ self.qk_head_dim = args.qk_head_dim+ self.v_head_dim = args.v_head_dim+ self.kv_lora_rank = args.kv_lora_rank+ assert args.qk_rope_head_dim == 0 and args.mla_nope, (+ "glm5_next attention implements the NoPE geometry; a roped variant "+ "would need the glm_moe_dsa rope plumbing back"+ )++ quant = config.attn_quant+ self.q_a_proj = _make_proj(quant, args.hidden_size, args.q_lora_rank)+ self.q_a_layernorm = RMSNorm(args.q_lora_rank, eps=args.norm_eps)+ self.q_b_proj = _make_proj(+ quant, args.q_lora_rank, self.num_heads * self.qk_head_dim+ )+ # NoPE: kv_a projects to bare ckv (no +qk_rope_head_dim rows).+ self.kv_a_proj_with_mqa = _make_proj(+ quant, args.hidden_size, self.kv_lora_rank+ )+ self.kv_a_layernorm = RMSNorm(self.kv_lora_rank, eps=args.norm_eps)+ # kv_b stays bf16 in every mode (bmm absorption operand, not a Linear).+ self.kv_b_proj = LinearReplicated(+ self.kv_lora_rank,+ self.num_heads * (self.qk_nope_head_dim + self.v_head_dim),+ has_bias=False,+ )+ self.o_proj = _make_proj(+ quant, self.num_heads * self.v_head_dim, args.hidden_size+ )+ self._w_uk: torch.Tensor | None = None+ self._w_uv: torch.Tensor | None = None++ def _kv_b(self) -> tuple[torch.Tensor, torch.Tensor]:+ """Per-head kv_b split in bmm-ready bf16 layout (same contract and+ prepare_for_runtime budgeting as glm_moe_dsa; see that module)."""+ if self._w_uk is None:+ w = self.kv_b_proj.weight.view(+ self.num_heads,+ self.qk_nope_head_dim + self.v_head_dim,+ self.kv_lora_rank,+ )+ self._w_uk = w[:, : self.qk_nope_head_dim, :].contiguous()+ self._w_uv = w[:, self.qk_nope_head_dim :, :].transpose(1, 2).contiguous()+ return self._w_uk, self._w_uv++ def prepare_for_runtime(self) -> None:+ self._kv_b()+ self.kv_b_proj.weight = None # checkpoint layout freed; repacked forms serve++ @nvtx_annotate("MLA")+ def forward(self, x: torch.Tensor) -> torch.Tensor:+ ctx = get_global_ctx()+ t = x.shape[0]+ w_uk, w_uv = self._kv_b()++ q_a_resid = self.q_a_layernorm.forward(self.q_a_proj.forward(x))+ q = self.q_b_proj.forward(q_a_resid)+ # NoPE: the whole head is the nope part; no rope split, no rope kernel.+ q_nope = q.view(t, self.num_heads, self.qk_head_dim)++ c_kv = self.kv_a_layernorm.forward(self.kv_a_proj_with_mqa.forward(x))++ # Absorb kv_b's k-part into the query: q_nope[H,T,nope] @ W_uk[H,nope,lora].+ q_absorbed = torch.bmm(q_nope.transpose(0, 1).contiguous(), w_uk).transpose(0, 1)++ indexer_inputs = (+ self.indexer.compute(x, q_a_resid)+ if getattr(ctx.attn_backend, "dsa_enabled", False)+ else None+ )++ # Zero-width rope halves: cat/scatter no-ops in the backend.+ q_pe = q.new_empty(t, self.num_heads, 0)+ k_rope = q.new_empty(t, 0)+ o_latent = ctx.attn_backend.mla_forward(+ q_absorbed.contiguous(), q_pe, c_kv.contiguous(), k_rope,+ self.layer_id, ctx.batch, indexer_inputs=indexer_inputs,+ ) # [T, H, kv_lora_rank]++ # Absorb kv_b's v-part onto the output: o_latent[H,T,lora] @ W_uv_t[H,lora,v].+ o = torch.bmm(o_latent.transpose(0, 1).contiguous(), w_uv).transpose(0, 1)+ return self.o_proj.forward(o.reshape(t, self.num_heads * self.v_head_dim))+++__all__ = ["Glm5NextAttention", "Glm5NextIndexer"]diff --git a/python/freetoken/models/glm5_next/config.py b/python/freetoken/models/glm5_next/config.pynew file mode 100644index 00000000..294ef579--- /dev/null+++ b/python/freetoken/models/glm5_next/config.py@@ -0,0 +1,198 @@+"""Engine-facing config for GLM-5.3-Flash (``glm5_next``).++Two attention groups, ordered by first layer id:++* ``linear`` -- 34 KDA layers (``LinearGatedDeltaGroupConfig`` with+ ``variant="kda"``). KDA's state geometry coincides with GDN's (conv width+ ``2*K*dk + V*dv`` == q|k|v at ``H*d`` each; recurrent ``[H, d, d]``), so the+ existing ``LinearStatePool`` shapes serve it unchanged.+* ``full`` -- 11 DSA layers (``FullAttentionGroupConfig`` with ``mla=True`` and+ the indexer dims). The MLA is NoPE (``qk_rope_head_dim == 0``): the latent row+ is bare ckv (512), and the indexer K slab is kpool-compressed+ (``index_kpool=4`` -> tokens/4 stored entries; the spec's ``index_ratio``+ drives both the pool factory and the KV cost model).++MoE routing mirrors glm_moe_dsa (sigmoid ``noaux_tc``, shared expert, routed+scaling 2.5) with 288 routed experts and a clamped SwiGLU+(``swiglu_limit=10``). Everything model-specific beyond ``ModelConfig`` rides in+``glm5_args`` (``Glm5NextArgs``).++Resident-weight quantization (attn/dense/lm_head fp8-at-load, the GLM-5.2+bandwidth trick) is OFF by default: the NVFP4 exports deliberately quantize only+the routed experts (attention / shared expert / lm_head ship bf16 behind the+quantization_config ignore list), and a serving engine must not override the+checkpoint author's precision decision silently -- the checkpoint is served+as-is, like vLLM/sglang. FREETOKEN_GLM5_ATTN_FP8=1 / FREETOKEN_GLM5_MLP_FP8=1+opt into the W8A16 requantization (measured decode 36 -> 45.5 tok/s on the+hybrid reference setup; the win needs CUDA graphs -- launch-bound eager decode+gets slower).+"""++from __future__ import annotations++import os+from typing import Any++from freetoken.models.config import (+ FullAttentionGroupConfig,+ LinearGatedDeltaGroupConfig,+ ModelConfig,+ RotaryConfig,+ detect_expert_quant,+)++from .args import load_args++# Load-time W8A16 fp8 for resident weights: default OFF, env opt-in (see module+# docstring for the rationale and measured numbers). attn covers the KDA+# in_proj_qkv/o_proj + DSA projections (the precision-sensitive b|f_a|g_a gate+# slice stays bf16, see kda.py); mlp covers dense MLPs, shared expert, lm_head.+_ATTN_FP8 = os.getenv("FREETOKEN_GLM5_ATTN_FP8", "0") != "0"+_MLP_FP8 = os.getenv("FREETOKEN_GLM5_MLP_FP8", "0") != "0"+++def _dsa_on(args, dsa_layer_ids) -> bool:+ """DSA serving switch, resolved ONCE into the attention-group spec (the pool+ factory, KV cost model, and backend read the spec, never the env)."""+ return (+ len(dsa_layer_ids) > 0+ and args.index_topk > 0+ and args.index_head_dim > 0+ and os.getenv("FREETOKEN_GLM5_DSA", "1") != "0"+ )+++def parse_config(hf_config: Any) -> ModelConfig:+ args = load_args(hf_config)+ text = getattr(hf_config, "text_config", hf_config)++ num_layers = len(args.layer_types)+ # Dev/testing only: cap the layer count so the forward path / KV / offload cache+ # can be exercised without the full ~175 GB of experts. Unset in normal use.+ _cap = os.environ.get("FREETOKEN_GLM5_MAX_LAYERS")+ if _cap:+ num_layers = min(num_layers, int(_cap))++ kda_ids = tuple(i for i in args.kda_layer_ids if i < num_layers)+ dsa_ids = tuple(i for i in args.dsa_layer_ids if i < num_layers)++ # NoPE: the main attention carries no rotary dims (rotary_dim == 0); rope+ # survives only in the indexer geometry (args.rope_theta / interleave).+ rotary_config = RotaryConfig(+ head_dim=args.qk_head_dim,+ rotary_dim=args.qk_rope_head_dim,+ max_position=args.max_position,+ base=args.rope_theta,+ scaling=None,+ )+ latent_dim = args.latent_dim # 512: bare ckv, no kpe rows++ dsa_on = _dsa_on(args, dsa_ids)+ # Each DSA layer owns its indexer ("full" in indexer_types); count only the+ # layers that actually exist under a dev layer cap.+ num_index_layers = (+ sum(+ 1+ for i in dsa_ids+ if i < len(args.indexer_types) and args.indexer_types[i] == "full"+ )+ if dsa_on+ else 0+ )++ linear_group = LinearGatedDeltaGroupConfig(+ name="linear",+ layer_ids=kda_ids,+ # KDA: one qk-sized and one v-sized head set (H=64, d=128 each). Mapping onto+ # the GDN field names keeps LinearStatePool's conv-width formula exact:+ # 2*K*dk + V*dv = (q|k) + v = 3 * H * d.+ num_key_heads=args.linear_num_heads,+ num_value_heads=args.linear_num_heads,+ key_head_dim=args.linear_head_dim,+ value_head_dim=args.linear_head_dim,+ conv_kernel_dim=args.linear_conv_kernel_dim,+ output_gate="sigmoid", # KDA o_norm gates with sigmoid (kda.py)+ variant="kda",+ )+ full_group = FullAttentionGroupConfig(+ name="full",+ layer_ids=dsa_ids,+ num_kv_heads=1, # single shared MLA latent+ head_dim=latent_dim,+ rotary_config=rotary_config,+ mla=True,+ index_head_dim=args.index_head_dim if dsa_on else 0,+ num_index_layers=num_index_layers,+ index_ratio=(args.index_kpool if dsa_on and args.index_kpool_compress else 1),+ )+ groups = tuple(+ sorted(+ (linear_group, full_group),+ key=lambda g: g.layer_ids[0] if g.layer_ids else 1 << 30,+ )+ )++ # The MLP layout is a dense prefix + sparse tail; ModelConfig models exactly that+ # via first_k_dense_replace, so assert the checkpoint matches before collapsing.+ mlp_types = args.mlp_layer_types[:num_layers]+ first_dense = next(+ (i for i, t in enumerate(mlp_types) if t == "sparse"), len(mlp_types)+ )+ assert all(t == "dense" for t in mlp_types[:first_dense]) and all(+ t == "sparse" for t in mlp_types[first_dense:]+ ), f"mlp_layer_types is not a dense-prefix layout: {mlp_types}"++ return ModelConfig(+ num_layers=num_layers,+ num_qo_heads=args.num_heads,+ num_kv_heads=1,+ head_dim=latent_dim,+ hidden_size=args.hidden_size,+ vocab_size=text.vocab_size,+ intermediate_size=text.intermediate_size,+ # hidden_act stands proxy for the EXPERT activation everywhere the engine+ # gates on it (NVFP4 backend selection, CPU-executor capability): GLM-5.3+ # experts and dense MLPs both run clamped SwiGLU when swiglu_limit is set,+ # even though the HF config still says "silu". Passing "silu" through would+ # let auto-selection repack experts into the marlin/b12x silu-only epilogue.+ hidden_act=(+ "swiglu_clamp" if args.swiglu_limit is not None else text.hidden_act+ ),+ rms_norm_eps=args.norm_eps,+ tie_word_embeddings=bool(getattr(text, "tie_word_embeddings", False)),+ rotary_config=rotary_config,+ attention_groups=groups,+ num_experts=(+ getattr(text, "n_routed_experts", None) or getattr(text, "num_experts", 0)+ ),+ num_experts_per_tok=(+ getattr(text, "num_experts_per_tok", None)+ or getattr(text, "num_experts_per_token", 0)+ ),+ moe_intermediate_size=getattr(text, "moe_intermediate_size", 0)+ or text.intermediate_size,+ norm_topk_prob=bool(getattr(text, "norm_topk_prob", True)),+ model_type=getattr(hf_config, "model_type", "glm5_next"),+ architectures=getattr(+ hf_config, "architectures", ["Glm5NextForConditionalGeneration"]+ ),+ moe_enabled=True,+ expert_quant=detect_expert_quant(hf_config),+ first_k_dense_replace=first_dense,+ n_shared_experts=int(getattr(text, "n_shared_experts", 0) or 0),+ routed_scaling_factor=float(getattr(text, "routed_scaling_factor", 1.0)),+ n_group=int(getattr(text, "n_group", 1) or 1),+ topk_group=int(getattr(text, "topk_group", 1) or 1),+ attn_sm_scale=args.qk_head_dim**-0.5,+ has_attn_bias=bool(getattr(text, "attention_bias", False)),+ swiglu_limit=args.swiglu_limit,+ attn_quant="fp8_pertensor" if _ATTN_FP8 else "none",+ dense_quant="fp8_pertensor" if _MLP_FP8 else "none",+ lm_head_quant="fp8_pertensor" if _MLP_FP8 else "none",+ vision_config=None, # text-only milestone; model.visual.* weights are dropped+ image_token_id=getattr(hf_config, "image_token_id", None),+ glm5_args=args,+ )+++__all__ = ["parse_config"]diff --git a/python/freetoken/models/glm5_next/kda.py b/python/freetoken/models/glm5_next/kda.pynew file mode 100644index 00000000..1d44e3a4--- /dev/null+++ b/python/freetoken/models/glm5_next/kda.py@@ -0,0 +1,252 @@+"""GLM-5.3-Flash KDA (Kimi Delta Attention) op.++Per-channel gated delta rule over H=64 heads of D=128, with SEPARATE q/k/v short+convolutions (vs GDN's one fused conv), a low-rank forget gate (``f_a`` ->+``f_b`` -> raw per-channel logits; the bounded safe gate ``lower_bound *+sigmoid(exp(A_log) * (g + dt_bias))`` is computed inside the kernels), a+per-head sigmoid beta (``b_proj``), and a sigmoid-gated output RMSNorm+(``o_norm`` gated by ``g_b(g_a(x))``).++State lives in ``ctx.linear_state_pool`` exactly like GDN: the conv state is the+merged q|k|v stream (width 3*H*D == the pool's ``2*K*dk + V*dv``) and the+recurrent state is one [D, D] matrix per head, stored in the KERNEL's [V, K]+layout (coincides with the pool's [K, V] declaration because D_k == D_v; same+convention as GDN, see qwen3_5_moe/gdn.py).++Kernels: ``fused_recurrent_kda`` decodes with in-kernel gate/beta/l2norm and+per-slot state read/write (slot 0 is its NULL sentinel == the pool's padding+slot); ``chunk_kda_with_fused_gate`` prefills from an explicitly gathered+initial state and returns the final state, which this op scatters back (the+kernel CLOBBERS its v buffer -- v here is an ephemeral conv output, so that is+free). Hybrid-radix track snapshots ride the per-chunk h (``return_h``).+"""++from __future__ import annotations++from typing import TYPE_CHECKING++import torch+from freetoken.core import get_global_ctx+from freetoken.kernel.causal_conv1d import causal_conv1d_decode, causal_conv1d_varlen+from freetoken.layers import BaseOP, LinearColParallelMerged, LinearReplicated+from freetoken.utils import nvtx_annotate++if TYPE_CHECKING:+ from freetoken.models.config import ModelConfig+++class _DepthwiseConv1d(BaseOP):+ """Merged q|k|v depthwise conv weight ``[3*H*D, 1, K]`` (key ``conv1d.weight``;+ the loader concatenates the checkpoint's q/k/v_conv1d along channels)."""++ def __init__(self, conv_dim: int, kernel: int):+ self.weight = torch.empty(conv_dim, 1, kernel)+++class _GatedRMSNormSigmoid(BaseOP):+ """RMSNorm(x) * sigmoid(z), fused (KDA's o_norm; GDN's variant gates with silu)."""++ def __init__(self, dim: int, eps: float):+ self.weight = torch.empty(dim)+ self.eps = eps++ def forward(self, x: torch.Tensor, z: torch.Tensor) -> torch.Tensor:+ from freetoken.kernel.fla import rms_norm_gated++ return rms_norm_gated(+ x=x, weight=self.weight, bias=None, z=z, eps=self.eps,+ is_rms_norm=True, norm_before_gate=True, activation="sigmoid",+ )+++class Glm5NextKDA(BaseOP):+ """KDA op; state is held in ``ctx.linear_state_pool`` keyed by the request's+ linear slot (``FLAMetadata.cache_indices``). Parameter names follow the+ checkpoint modulo two load-time fusions (see weight.py): ``in_proj`` is+ q|k|v|b|f_a|g_a concatenated, ``conv1d`` is q|k|v conv concatenated."""++ def __init__(self, config: ModelConfig, layer_id: int):+ args = config.glm5_args+ self.layer_id = layer_id+ self.num_heads = args.linear_num_heads+ self.head_dim = args.linear_head_dim+ self.proj_size = self.num_heads * self.head_dim # H * D+ self.conv_dim = 3 * self.proj_size # merged q|k|v stream+ self.conv_kernel_size = args.linear_conv_kernel_dim+ self.lower_bound = args.linear_lower_bound+ self.scale = self.head_dim**-0.5++ p, h, d = self.proj_size, self.num_heads, self.head_dim+ # q|k|v dominate the resident read (3 * 8192 x 4096 = 201 MB/layer bf16 --+ # the single biggest dense-weight stream in the model); under the fp8+ # resident mode they split into their own W8A16 GEMM while the small,+ # precision-sensitive gate projections (b|f_a|g_a) stay bf16 (the GDN+ # qkvz/ba split precedent). BF16 mode keeps the single fused GEMM.+ self._fp8 = config.attn_quant == "fp8_pertensor"+ self._bfg_split = [h, d, d]+ if self._fp8:+ from freetoken.kernel.triton.fp8_pertensor_linear import Fp8PerTensorColMerged++ self.in_proj_qkv = Fp8PerTensorColMerged(+ args.hidden_size, [p, p, p], has_bias=False+ )+ self.in_proj_bfg = LinearColParallelMerged(+ args.hidden_size, self._bfg_split, has_bias=False+ )+ else:+ self._in_proj_split = [p, p, p, h, d, d]+ self.in_proj = LinearColParallelMerged(+ args.hidden_size, self._in_proj_split, has_bias=False+ )+ # Low-rank gate up-projections (128 -> 8192): forget gate and output gate.+ self.f_b_proj = LinearReplicated(d, p, has_bias=False)+ self.g_b_proj = LinearReplicated(d, p, has_bias=False)+ self.conv1d = _DepthwiseConv1d(self.conv_dim, self.conv_kernel_size)+ # Gate params stay fp32 (exp/sigmoid precision; the kernels read fp32).+ # models/weight.py exempts *.A_log / *.dt_bias from the model-dtype downcast.+ self.A_log = torch.empty(h, dtype=torch.float32)+ self.dt_bias = torch.empty(p, dtype=torch.float32)+ self.o_norm = _GatedRMSNormSigmoid(d, eps=args.norm_eps)+ # o_proj follows the resident quant mode (rationale: the split comment+ # at _fp8 above); f_b/g_b stay bf16 alongside the gate slice.+ from .attention import _make_proj++ self.o_proj = _make_proj(config.attn_quant, p, args.hidden_size)++ def _conv_weight(self) -> torch.Tensor:+ return self.conv1d.weight.squeeze(1) # [conv_dim, kernel]++ def _write_track_snapshot(self, pool, li, conv_in, h, fla) -> None:+ """Hybrid-radix: snapshot recurrent + conv state at the chunk-aligned track+ boundary into a donatable pool slot (same contract as GDN, see+ qwen3_5_moe/gdn.py). h rows are the kernel's per-chunk [V, K] states --+ a direct copy into the pool's [K, V] slots (D_k == D_v)."""+ rec = pool.recurrent_states[li]+ rec.index_copy_(0, fla.track_dst, h[0, fla.track_h_row].to(rec.dtype))+ cv = pool.conv_states[li]+ conv_win = conv_in[fla.track_conv_src].transpose(-1, -2).contiguous()+ cv.index_copy_(0, fla.track_dst, conv_win.to(cv.dtype))++ @nvtx_annotate("KDA")+ def forward(self, hidden_states: torch.Tensor) -> torch.Tensor:+ ctx = get_global_ctx()+ batch = ctx.batch+ pool = ctx.linear_state_pool+ total = hidden_states.shape[0]+ dtype = hidden_states.dtype+ h, d, p = self.num_heads, self.head_dim, self.proj_size++ fla = batch.fla_metadata+ if fla is None:+ from freetoken.attention.linear import build_fla_metadata++ fla = build_fla_metadata(batch, hidden_states.device)+ batch.fla_metadata = fla++ if self._fp8:+ conv_in = self.in_proj_qkv.forward(hidden_states)+ b, f_a, g_a = torch.split(+ self.in_proj_bfg.forward(hidden_states), self._bfg_split, dim=-1+ )+ else:+ proj = self.in_proj.forward(hidden_states)+ conv_in, b, f_a, g_a = torch.split(+ proj, [self.conv_dim, h, d, d], dim=-1+ )+ g1 = self.f_b_proj.forward(f_a) # raw forget-gate logits [T, H*D]+ g2 = self.g_b_proj.forward(g_a) # output-gate logits [T, H*D]+ li = pool.local_index(self.layer_id)++ if batch.is_decode:+ mixed = causal_conv1d_decode(+ conv_in, pool.conv_states[li], self._conv_weight(), fla.cache_indices+ )+ bsz = mixed.shape[0]+ q, k, v = (+ t.reshape(1, bsz, h, d).to(dtype)+ for t in torch.split(mixed, [p, p, p], dim=-1)+ )+ core_out, _ = _fused_recurrent(+ q, k, v,+ g=g1.view(1, bsz, h, d),+ beta=b.view(1, bsz, h),+ state_pool=pool.recurrent_states[li],+ indices=fla.cache_indices,+ cu_seqlens=fla.cu_seqlens,+ a_log=self.A_log,+ dt_bias=self.dt_bias,+ lower_bound=self.lower_bound,+ scale=self.scale,+ )+ else:+ x = conv_in.transpose(0, 1).contiguous() # [conv_dim, total]+ mixed = causal_conv1d_varlen(+ x, self._conv_weight(), pool.conv_states[li],+ fla.cu_seqlens, fla.cache_indices, fla.has_initial_state,+ ).transpose(0, 1)+ q, k, v = (+ t.reshape(1, total, h, d).to(dtype)+ for t in torch.split(mixed, [p, p, p], dim=-1)+ )+ # Fresh sequences start from a zeroed slot; then gather every request's+ # initial state (the chunk kernel takes it dense, [N, H, D, D]).+ rec = pool.recurrent_states[li]+ if fla.fresh_state_indices is not None:+ rec.index_fill_(0, fla.fresh_state_indices, 0.0)+ slot_ids = fla.cache_indices.long()+ initial = rec.index_select(0, slot_ids)++ from freetoken.kernel.fla import chunk_kda_with_fused_gate++ track = fla.track_dst is not None+ result = chunk_kda_with_fused_gate(+ q=q, k=k, v=v, # NOTE: v (ephemeral conv output) is clobbered+ raw_g=g1.view(1, total, h, d),+ beta=b.float().sigmoid().view(1, total, h),+ A_log=self.A_log,+ g_bias=self.dt_bias,+ scale=self.scale,+ initial_state=initial,+ output_final_state=True,+ use_qk_l2norm_in_kernel=True,+ cu_seqlens=fla.cu_seqlens,+ safe_gate=True,+ lower_bound=self.lower_bound,+ return_h=track,+ )+ if track:+ core_out, final_state, chunk_h = result+ self._write_track_snapshot(pool, li, conv_in, chunk_h, fla)+ else:+ core_out, final_state = result+ rec.index_copy_(0, slot_ids, final_state.to(rec.dtype))++ core_out = core_out.reshape(-1, d)+ out = self.o_norm.forward(core_out, g2.reshape(-1, d)).reshape(total, -1)+ return self.o_proj.forward(out.to(dtype))+++def _fused_recurrent(+ q, k, v, g, beta, state_pool, indices, cu_seqlens,+ a_log, dt_bias, lower_bound, scale,+):+ """Decode via the vendored recurrent kernel: gate + beta-sigmoid + q/k l2norm+ in-kernel, state read/written in place at ``indices`` (int32, 1 token/req)."""+ from freetoken.kernel.fla import fused_recurrent_kda++ return fused_recurrent_kda(+ q=q, k=k, v=v, g=g, beta=beta,+ scale=scale,+ initial_state=state_pool,+ use_qk_l2norm_in_kernel=True,+ cu_seqlens=cu_seqlens,+ ssm_state_indices=indices,+ sigmoid_beta=True,+ a_log=a_log,+ g_bias=dt_bias,+ compute_gate=True,+ lower_bound=lower_bound,+ )+++__all__ = ["Glm5NextKDA"]diff --git a/python/freetoken/models/glm5_next/mlp.py b/python/freetoken/models/glm5_next/mlp.pynew file mode 100644index 00000000..1ee508bf--- /dev/null+++ b/python/freetoken/models/glm5_next/mlp.py@@ -0,0 +1,47 @@+"""Clamped-SwiGLU MLP for GLM-5.3-Flash's leading dense layers and shared experts.++Same shape as glm_moe_dsa's GlmDsaGatedMLP (bf16 in the NVFP4 checkpoint;+optional W8A16 fp8-at-load via ``ModelConfig.dense_quant``), but the activation+is the GLM-5.3 clamped SwiGLU (``swiglu_limit``):+``clamp(gate, max=L) * sigmoid(gate_clamped) * clamp(up, +-L)``.+"""++from __future__ import annotations+++import torch+from freetoken.layers import BaseOP, swiglu_clamp_and_mul+from freetoken.utils import nvtx_annotate++from .attention import _make_proj+++class Glm5NextGatedMLP(BaseOP):+ def __init__(+ self,+ hidden_size: int,+ intermediate_size: int,+ quant: str = "none",+ swiglu_limit: float | None = None,+ ):+ self.gate_proj = _make_proj(quant, hidden_size, intermediate_size)+ self.up_proj = _make_proj(quant, hidden_size, intermediate_size)+ self.down_proj = _make_proj(quant, intermediate_size, hidden_size)+ self.swiglu_limit = swiglu_limit++ @nvtx_annotate("MLP")+ def forward(self, x: torch.Tensor) -> torch.Tensor:+ gate = self.gate_proj.forward(x)+ up = self.up_proj.forward(x)+ del x+ if self.swiglu_limit is None:+ import torch.nn.functional as F++ return self.down_proj.forward(F.silu(gate) * up)+ gated = swiglu_clamp_and_mul(+ torch.cat([gate, up], dim=-1), alpha=1.0, limit=self.swiglu_limit+ )+ return self.down_proj.forward(gated)+++__all__ = ["Glm5NextGatedMLP"]diff --git a/python/freetoken/models/glm5_next/model.py b/python/freetoken/models/glm5_next/model.pynew file mode 100644index 00000000..4c011dd6--- /dev/null+++ b/python/freetoken/models/glm5_next/model.py@@ -0,0 +1,207 @@+"""GLM-5.3-Flash (glm5_next) model: hybrid KDA/DSA decoder with mHC residual streams.++Layer layout comes from the checkpoint's ``layer_types`` (34 KDA linear-attention+layers, 11 NoPE-MLA/DSA layers at 3:1) and ``mlp_layer_types`` (3 dense + 42 MoE).+The residual stream is mHC-widened to ``hc_mult`` (4) parallel streams:++ layer 0: residual = hc_expand(x); (post, comb, x) = mhc_pre(residual, hc_attn_*)+ each sublayer boundary fuses the previous hc_post with the next hc_pre+ (mhc_fused_post_pre), and the sublayer input is RMS-normed AFTER the mix+ (the reference fuses the norm into its hc kernels; decomposed here, same math).+ last layer: x = mhc_post(...); x = hc_contract(x) -> final norm -> lm_head.++The deferred (post, comb) pair threads through the layer loop exactly like+glm_moe_dsa's (x, residual) pair. lm_head quant mirrors glm_moe_dsa (optional+W8A16 fp8 at load; the ~1.2 GiB bf16 head is read every decode step).+"""++from __future__ import annotations++from typing import TYPE_CHECKING, Tuple++import torch+from freetoken.core import get_global_ctx+from freetoken.layers import (+ BaseOP,+ OPList,+ ParallelLMHead,+ RMSNorm,+ VocabParallelEmbedding,+)+from freetoken.layers.mhc import hc_contract, hc_expand, mhc_fused_post_pre, mhc_post, mhc_pre+from freetoken.models.blocks import BaseLLMModel+from freetoken.utils import nvtx_annotate++from .attention import Glm5NextAttention+from .kda import Glm5NextKDA+from .mlp import Glm5NextGatedMLP+from .moe import Glm5NextSparseBlock++if TYPE_CHECKING:+ from freetoken.models.config import ModelConfig+++class Glm5Fp8LMHead(ParallelLMHead):+ """W8A16 lm_head (fp8-e4m3 weight + per-row scale, quantized at load); the+ full-vocab decode GEMV reads the whole head every step -- fp8 halves it.+ Same contract as glm_moe_dsa's GlmFp8LMHead."""++ def __init__(self, num_embeddings: int, embedding_dim: int):+ super().__init__(num_embeddings, embedding_dim, tie_word_embeddings=False)+ self.weight = torch.empty(num_embeddings, embedding_dim, dtype=torch.float8_e4m3fn)+ self.weight_scale = torch.empty(num_embeddings, dtype=torch.float32)++ def forward(self, x: torch.Tensor) -> torch.Tensor:+ from freetoken.kernel.triton.fp8_pertensor_linear import fp8_pertensor_linear++ batch = get_global_ctx().batch+ if batch.is_prefill:+ indices = batch.attn_metadata.get_last_indices(batch.size)+ x = x[indices].contiguous()+ return fp8_pertensor_linear(x, self.weight, self.weight_scale)+++class Glm5NextDecoderLayer(BaseOP):+ def __init__(self, config: ModelConfig, layer_id: int):+ args = config.glm5_args+ self._layer_id = layer_id+ self._is_last = layer_id == config.num_layers - 1+ self.mhc = args.mhc+ self._n = args.mhc_num_residual_streams+ self._hc_eps = args.hc_eps+ self._rms_eps = args.norm_eps+ self._post_mult = args.mhc_post_mult_value+ self._sinkhorn = args.mhc_sinkhorn_iterations++ if args.is_kda_layer(layer_id):+ self.self_attn: BaseOP = Glm5NextKDA(config, layer_id)+ else:+ self.self_attn = Glm5NextAttention(config, layer_id)+ if layer_id >= config.first_k_dense_replace:+ self.mlp: BaseOP = Glm5NextSparseBlock(config, layer_id)+ else:+ self.mlp = Glm5NextGatedMLP(+ config.hidden_size, config.intermediate_size,+ quant=config.dense_quant, swiglu_limit=config.swiglu_limit,+ )+ self.input_layernorm = RMSNorm(size=config.hidden_size, eps=args.norm_eps)+ self.post_attention_layernorm = RMSNorm(size=config.hidden_size, eps=args.norm_eps)++ if self.mhc:+ n, hidden = self._n, config.hidden_size+ mix = 2 * n + n * n+ # fp32 mHC weights (models/weight.py exempts hc_* from the dtype downcast).+ self.hc_attn_fn = torch.empty(mix, n * hidden, dtype=torch.float32)+ self.hc_attn_base = torch.empty(mix, dtype=torch.float32)+ self.hc_attn_scale = torch.empty(3, dtype=torch.float32)+ self.hc_ffn_fn = torch.empty(mix, n * hidden, dtype=torch.float32)+ self.hc_ffn_base = torch.empty(mix, dtype=torch.float32)+ self.hc_ffn_scale = torch.empty(3, dtype=torch.float32)++ def _pre(self, residual, fn, scale, base):+ # Layer 0's standalone pre rides the fused kernel too (HAS_POST=False+ # path; x/post/comb are the no-post sentinels) -- same dispatch, same+ # numerics, and the kernel wins at every batch size (see layers/mhc.py).+ if residual.is_cuda:+ _, post, comb, x = mhc_fused_post_pre(+ residual.new_empty(residual.shape[0], residual.shape[-1]),+ residual, None, None, fn, scale, base,+ self._rms_eps, self._hc_eps, self._post_mult, self._sinkhorn,+ )+ return post, comb, x+ return mhc_pre(+ residual, fn, scale, base,+ self._rms_eps, self._hc_eps, self._post_mult, self._sinkhorn,+ )++ def _fused(self, x, residual, post, comb, fn, scale, base):+ return mhc_fused_post_pre(+ x, residual, post, comb, fn, scale, base,+ self._rms_eps, self._hc_eps, self._post_mult, self._sinkhorn,+ )++ @nvtx_annotate("Layer_{}", layer_id_field="_layer_id")+ def forward(+ self,+ x: torch.Tensor,+ residual: torch.Tensor | None,+ post: torch.Tensor | None,+ comb: torch.Tensor | None,+ ) -> Tuple[torch.Tensor, torch.Tensor | None, torch.Tensor | None, torch.Tensor | None]:+ if post is None:+ if residual is None:+ residual = hc_expand(x, self._n)+ post, comb, x = self._pre(+ residual, self.hc_attn_fn, self.hc_attn_scale, self.hc_attn_base+ )+ else:+ residual, post, comb, x = self._fused(+ x, residual, post, comb,+ self.hc_attn_fn, self.hc_attn_scale, self.hc_attn_base,+ )+ x = self.input_layernorm.forward(x)+ x = self.self_attn.forward(x)++ residual, post, comb, x = self._fused(+ x, residual, post, comb,+ self.hc_ffn_fn, self.hc_ffn_scale, self.hc_ffn_base,+ )+ x = self.post_attention_layernorm.forward(x)+ x = self.mlp.forward(x)++ if self._is_last:+ x = mhc_post(x, residual, post, comb)+ return hc_contract(x), None, None, None+ return x, residual, post, comb+++class Glm5NextModel(BaseOP):+ def __init__(self, config: ModelConfig):+ self.embed_tokens = VocabParallelEmbedding(+ num_embeddings=config.vocab_size,+ embedding_dim=config.hidden_size,+ )+ self.layers = OPList(+ [Glm5NextDecoderLayer(config, i) for i in range(config.num_layers)]+ )+ self.norm = RMSNorm(size=config.hidden_size, eps=config.rms_norm_eps)++ def forward(self, input_ids: torch.Tensor) -> torch.Tensor:+ x = self.embed_tokens.forward(input_ids)+ residual = post = comb = None+ for layer in self.layers.op_list:+ x, residual, post, comb = layer.forward(x, residual, post, comb)+ return self.norm.forward(x)+++class Glm5NextForCausalLM(BaseLLMModel):+ def __init__(self, config: ModelConfig):+ self._config = config+ self.model = Glm5NextModel(config)+ if config.lm_head_quant == "fp8_pertensor" and not config.tie_word_embeddings:+ self.lm_head: BaseOP = Glm5Fp8LMHead(+ num_embeddings=config.vocab_size, embedding_dim=config.hidden_size+ )+ else:+ self.lm_head = ParallelLMHead(+ num_embeddings=config.vocab_size,+ embedding_dim=config.hidden_size,+ tie_word_embeddings=config.tie_word_embeddings,+ tied_embedding=self.model.embed_tokens if config.tie_word_embeddings else None,+ )++ def prepare_for_runtime(self) -> None:+ """Post-load, pre-KV-sizing hook: materialize the DSA layers' bmm-ready+ kv_b splits and free the checkpoint-layout originals (glm_moe_dsa+ precedent)."""+ for layer in self.model.layers.op_list:+ if isinstance(layer.self_attn, Glm5NextAttention):+ layer.self_attn.prepare_for_runtime()+ torch.cuda.empty_cache()++ def forward(self) -> torch.Tensor:+ output = self.model.forward(get_global_ctx().batch.input_ids)+ return self.lm_head.forward(output)+++__all__ = ["Glm5NextForCausalLM"]diff --git a/python/freetoken/models/glm5_next/moe.py b/python/freetoken/models/glm5_next/moe.pynew file mode 100644index 00000000..038ffc0d--- /dev/null+++ b/python/freetoken/models/glm5_next/moe.py@@ -0,0 +1,92 @@+"""GLM-5.3-Flash sparse MoE block.++Routing is identical to glm_moe_dsa (sigmoid scores + selection-only+``e_score_correction_bias``, optional group-limited top-k, gather unbiased+scores, renormalize, scale by ``routed_scaling_factor``); the deltas are the+expert count (288, top-8) and the clamped-SwiGLU activation (``swiglu_limit`` =+10), which rides ``make_moe_layer``'s ``extra_attrs`` into the offload kernels+(triton NVFP4 in-GPU, generic-epilogue CPU GEMV). The marlin/b12x borrowed+kernels hard-code silu; the backend selector already falls back to triton for+non-silu experts.+"""++from __future__ import annotations++from typing import TYPE_CHECKING, Tuple++import torch+import torch.nn.functional as F+from freetoken.layers import BaseOP, LinearReplicated, make_moe_layer++from .mlp import Glm5NextGatedMLP++if TYPE_CHECKING:+ from freetoken.models.config import ModelConfig++TopK = Tuple[torch.Tensor, torch.Tensor]+++class Glm5NextSparseBlock(BaseOP):+ def __init__(self, config: ModelConfig, layer_id: int):+ self.top_k = config.num_experts_per_tok+ self.num_experts = config.num_experts+ self.norm_topk_prob = config.norm_topk_prob+ self.routed_scaling_factor = config.routed_scaling_factor+ self.n_group = config.n_group+ self.topk_group = config.topk_group++ self.gate = LinearReplicated(config.hidden_size, config.num_experts, has_bias=False)+ self.e_score_correction_bias = torch.empty(config.num_experts, dtype=torch.float32)++ # The offload cache indexes experts by MoE layer (global minus dense prefix).+ self.experts = make_moe_layer(+ config,+ layer_id=layer_id - config.first_k_dense_replace,+ activation="swiglu_clamp" if config.swiglu_limit is not None else "silu",+ renormalize=config.norm_topk_prob,+ extra_attrs={+ "swiglu_limit": config.swiglu_limit,+ "hidden_act_alpha": 1.0, # plain sigmoid inside the clamped swiglu+ },+ )+ self.shared_experts = Glm5NextGatedMLP(+ config.hidden_size,+ config.moe_intermediate_size * max(1, config.n_shared_experts),+ quant=config.dense_quant,+ swiglu_limit=config.swiglu_limit,+ )++ def _group_limited(self, scores_for_choice: torch.Tensor) -> torch.Tensor:+ m = scores_for_choice.shape[0]+ e, g = self.num_experts, self.n_group+ group_scores = scores_for_choice.view(m, g, e // g).topk(2, dim=-1)[0].sum(dim=-1)+ group_idx = torch.topk(group_scores, self.topk_group, dim=-1, sorted=False)[1]+ group_mask = torch.zeros_like(group_scores)+ group_mask.scatter_(1, group_idx, 1.0)+ score_mask = group_mask.unsqueeze(-1).expand(m, g, e // g).reshape(m, e)+ return scores_for_choice.masked_fill(~score_mask.bool(), float("-inf"))++ def _route(self, hidden_states: torch.Tensor) -> TopK:+ # HF computes router logits in fp32 (moe_router_dtype: float32); match exactly.+ logits = F.linear(hidden_states.float(), self.gate.weight.float())+ scores = logits.sigmoid()+ scores_for_choice = scores + self.e_score_correction_bias.float()+ if self.n_group > 1:+ scores_for_choice = self._group_limited(scores_for_choice)+ _, topk_ids = torch.topk(scores_for_choice, self.top_k, dim=-1)+ topk_weights = scores.gather(-1, topk_ids)+ if self.norm_topk_prob:+ topk_weights = topk_weights / (topk_weights.sum(dim=-1, keepdim=True) + 1e-20)+ topk_weights = topk_weights * self.routed_scaling_factor+ return topk_weights.to(torch.float32).contiguous(), topk_ids.to(torch.int32).contiguous()++ def forward(self, hidden_states: torch.Tensor) -> torch.Tensor:+ num_tokens, hidden_dim = hidden_states.shape+ hidden_states = hidden_states.view(-1, hidden_dim)+ topk_weights, topk_ids = self._route(hidden_states)+ out = self.experts.routed_forward(hidden_states, topk_weights, topk_ids)+ out = out + self.shared_experts.forward(hidden_states)+ return out.view(num_tokens, hidden_dim)+++__all__ = ["Glm5NextSparseBlock"]diff --git a/python/freetoken/models/glm5_next/weight.py b/python/freetoken/models/glm5_next/weight.pynew file mode 100644index 00000000..fda85225--- /dev/null+++ b/python/freetoken/models/glm5_next/weight.py@@ -0,0 +1,275 @@+"""Weight loading for GLM-5.3-Flash (``glm5_next``).++Supported checkpoints: NVFP4 exports of GLM-5.3-Flash in the multimodal-wrapper+layout (``model.language_model.*``) -- ModelOpt tensor kinds (LibertAIDAI) or+compressed-tensors kinds (RedHatAI), selected by ``quantization_config``. Not+supported: bf16-expert originals (zai-org), text-only key layouts, TP > 1.++Routed experts go to the offload cache via ``load_nvfp4_expert_sources``;+everything else loads bf16 with keys renamed ``model.language_model.X`` ->+``model.X``. ``model.visual.*`` and the trailing MTP layer are never read.++Load-time fusions (must mirror the module split orders):++* KDA ``in_proj`` = q|k|v|b|f_a|g_a projections concatenated on the output axis+* KDA ``conv1d`` = q|k|v depthwise conv weights concatenated on the channel axis++fp32-kept tensors: ``A_log`` / ``dt_bias``, the mHC ``hc_*`` tensors, the indexer+APE, and the router ``e_score_correction_bias``. Optional W8A16 fp8-at-load+follows ``ModelConfig.attn_quant`` / ``dense_quant`` / ``lm_head_quant``+(defaults and env opt-ins: see config.py).+"""++from __future__ import annotations++import json+import os+import re+from typing import Iterator++import torch+from freetoken.distributed import get_tp_info+from freetoken.models.glm_moe_dsa.weight import _ShardReader, _quant_fp8_per_row+from freetoken.models.loader import drop_page_cache+from freetoken.models.nvfp4_banks import (+ Nvfp4ExpertSourceSpec,+ load_nvfp4_expert_source_banks,+)+from freetoken.utils import cached_load_hf_config, download_hf_weight+from tqdm import tqdm++from .args import Glm5NextArgs+from .config import parse_config++# Checkpoint prefix (multimodal wrapper) -> model prefix.+_CKPT = "model.language_model"+_MODEL = "model"++# MTP-layer experts (layer == num_layers under the full checkpoint) map to None+# alongside the dense prefix; the bank loader skips them.+def _layer_to_bank(layer, config):+ return (+ None+ if layer < config.first_k_dense_replace or layer >= config.num_layers+ else layer - config.first_k_dense_replace+ )+++# ModelOpt export (LibertAIDAI/GLM-5.3-Flash-NVFP4): weight | weight_scale |+# weight_scale_2 (dequant-side global).+_NVFP4_SOURCE_SPEC = Nvfp4ExpertSourceSpec(+ key_pattern=re.compile(+ r"^model\.language_model\.layers\.(?P<layer>\d+)\.mlp\.experts\.(?P<expert>\d+)\."+ r"(?P<proj>gate_proj|up_proj|down_proj)\.(?P<kind>weight|weight_scale|weight_scale_2)$"+ ),+ proj_to_role={"gate_proj": "gate", "up_proj": "up", "down_proj": "down"},+ layer_to_bank=_layer_to_bank,+ desc="GLM-5.3 NVFP4 experts",+)++# llm-compressor export (RedHatAI/GLM-5.3-Flash-NVFP4): weight_packed |+# weight_scale | weight_global_scale (quant-side global -> reciprocal at ingest).+# ``input_global_scale`` (the calibrated W4A4 activation scale) deliberately does+# not match: our routed-expert paths are W4A16 and never quantize activations.+_NVFP4_CT_SOURCE_SPEC = Nvfp4ExpertSourceSpec(+ key_pattern=re.compile(+ r"^model\.language_model\.layers\.(?P<layer>\d+)\.mlp\.experts\.(?P<expert>\d+)\."+ r"(?P<proj>gate_proj|up_proj|down_proj)\."+ r"(?P<kind>weight_packed|weight_global_scale|weight_scale)$"+ ),+ proj_to_role={"gate_proj": "gate", "up_proj": "up", "down_proj": "down"},+ layer_to_bank=_layer_to_bank,+ desc="GLM-5.3 NVFP4 experts (compressed-tensors)",+ kind_map={"weight_packed": "weight", "weight_global_scale": "weight_scale_2"},+ global_reciprocal=True,+)+++def _select_expert_source_spec(model_path: str) -> Nvfp4ExpertSourceSpec:+ quant = getattr(cached_load_hf_config(model_path), "quantization_config", None) or {}+ get = quant.get if isinstance(quant, dict) else (lambda k, d=None: getattr(quant, k, d))+ method = str(get("quant_method") or "").lower()+ return _NVFP4_CT_SOURCE_SPEC if method == "compressed-tensors" else _NVFP4_SOURCE_SPEC++# KDA in_proj fusion order; MUST match Glm5NextKDA._in_proj_split.+_KDA_IN_PROJ = ("q_proj", "k_proj", "v_proj", "b_proj", "f_a_proj", "g_a_proj")+++def load_nvfp4_expert_sources(model_path: str, config, layer_sink=None):+ return load_nvfp4_expert_source_banks(+ model_path,+ config,+ _select_expert_source_spec(model_path),+ drop_page_cache=drop_page_cache,+ primary=get_tp_info().is_primary(),+ layer_sink=layer_sink,+ )+++def _maybe_fp8(key: str, w: torch.Tensor, fp8: bool):+ if fp8:+ q, scale = _quant_fp8_per_row(w)+ yield f"{key}.weight", q+ yield f"{key}.weight_scale", scale+ else:+ yield f"{key}.weight", w.to(torch.bfloat16)+++def _iter_kda_layer(reader, layer: int, attn_fp8: bool) -> Iterator[tuple[str, torch.Tensor]]:+ src = f"{_CKPT}.layers.{layer}.self_attn"+ dst = f"{_MODEL}.layers.{layer}.self_attn"+ if attn_fp8:+ # fp8 resident: q|k|v (the 201 MB/layer read) as one W8A16 GEMM with+ # per-row scales; the small gate projections b|f_a|g_a stay bf16.+ qkv = torch.cat(+ [reader.get(f"{src}.{p}.weight").to(torch.bfloat16) for p in ("q_proj", "k_proj", "v_proj")],+ dim=0,+ )+ q, scale = _quant_fp8_per_row(qkv)+ yield f"{dst}.in_proj_qkv.weight", q+ yield f"{dst}.in_proj_qkv.weight_scale", scale+ bfg = torch.cat(+ [reader.get(f"{src}.{p}.weight").to(torch.bfloat16) for p in ("b_proj", "f_a_proj", "g_a_proj")],+ dim=0,+ )+ yield f"{dst}.in_proj_bfg.weight", bfg+ else:+ # One fused input GEMM: q|k|v|b|f_a|g_a (output-axis concat).+ fused = torch.cat(+ [reader.get(f"{src}.{p}.weight").to(torch.bfloat16) for p in _KDA_IN_PROJ], dim=0+ )+ yield f"{dst}.in_proj.weight", fused+ # One merged depthwise conv over the q|k|v stream (channel-axis concat).+ conv = torch.cat(+ [reader.get(f"{src}.{p}_conv1d.weight").to(torch.bfloat16) for p in ("q", "k", "v")],+ dim=0,+ )+ yield f"{dst}.conv1d.weight", conv+ for p in ("f_b_proj", "g_b_proj"):+ yield f"{dst}.{p}.weight", reader.get(f"{src}.{p}.weight").to(torch.bfloat16)+ yield from _maybe_fp8(f"{dst}.o_proj", reader.get(f"{src}.o_proj.weight"), attn_fp8)+ # Gate params stay fp32 (the recurrent kernels read them as fp32).+ yield f"{dst}.A_log", reader.get(f"{src}.A_log").to(torch.float32)+ yield f"{dst}.dt_bias", reader.get(f"{src}.dt_bias").to(torch.float32)+ yield f"{dst}.o_norm.weight", reader.get(f"{src}.o_norm.weight").to(torch.bfloat16)+++def _iter_dsa_layer(reader, layer: int, attn_fp8: bool) -> Iterator[tuple[str, torch.Tensor]]:+ src = f"{_CKPT}.layers.{layer}.self_attn"+ dst = f"{_MODEL}.layers.{layer}.self_attn"+ fp8_projs = ("q_a_proj", "q_b_proj", "kv_a_proj_with_mqa", "o_proj") if attn_fp8 else ()+ for proj in ("q_a_proj", "q_b_proj", "kv_a_proj_with_mqa", "kv_b_proj", "o_proj"):+ w = reader.get(f"{src}.{proj}.weight")+ yield from _maybe_fp8(f"{dst}.{proj}", w, proj in fp8_projs)+ for norm in ("q_a_layernorm", "kv_a_layernorm"):+ yield f"{dst}.{norm}.weight", reader.get(f"{src}.{norm}.weight").to(torch.bfloat16)+ # kpool indexer (every DSA layer owns one). Kept bf16; the APE is fp32.+ for proj in ("wq_b", "wk", "weights_proj"):+ yield f"{dst}.indexer.{proj}.weight", reader.get(+ f"{src}.indexer.{proj}.weight"+ ).to(torch.bfloat16)+ for part, dtype in (+ ("k_norm.weight", torch.bfloat16),+ ("k_norm.bias", torch.bfloat16),+ ("index_kpool_compress_gate", torch.bfloat16),+ ("index_kpool_compress_ape", torch.float32),+ ):+ yield f"{dst}.indexer.{part}", reader.get(f"{src}.indexer.{part}").to(dtype)+++def iter_weights(+ model_path: str,+ device: torch.device,+ *,+ include_moe_experts: bool,+ include_non_moe: bool,+) -> Iterator[tuple[str, torch.Tensor]]:+ assert not include_moe_experts, (+ "GLM-5.3 stores routed experts as NVFP4 and only supports the offload backend; "+ "experts are loaded into the offload cache via load_nvfp4_expert_sources()."+ )+ assert include_non_moe+ if get_tp_info().size > 1:+ # The loader emits full fused KDA/DSA tensors; TP sharding (per-head q|k|v|b+ # splits, replicated f_a|g_a, row-parallel o_proj) is not implemented yet --+ # same status as every other linear-hybrid / offload-family model in tree.+ raise NotImplementedError("glm5_next weight loading currently supports TP=1 only")+ config = parse_config(cached_load_hf_config(model_path))+ args: Glm5NextArgs = config.glm5_args+ folder = download_hf_weight(model_path)+ with open(os.path.join(folder, "model.safetensors.index.json")) as f:+ weight_map = json.load(f)["weight_map"]+ reader = _ShardReader(folder, weight_map, device)+ primary = get_tp_info().is_primary()+ attn_fp8 = config.attn_quant == "fp8_pertensor"+ mlp_fp8 = config.dense_quant == "fp8_pertensor"+ head_fp8 = config.lm_head_quant == "fp8_pertensor"+ if primary:+ from freetoken.utils import init_logger++ init_logger(__name__).info(+ f"GLM-5.3 resident quant: attn={config.attn_quant} dense={config.dense_quant} "+ f"lm_head={config.lm_head_quant} (FREETOKEN_GLM5_ATTN_FP8/FREETOKEN_GLM5_MLP_FP8; "+ "an FTW conversion records these choices implicitly -- serve with the same flags)"+ )+ try:+ for layer in tqdm(+ range(config.num_layers),+ desc="Loading GLM-5.3 dense weights",+ disable=not primary,+ ):+ src = f"{_CKPT}.layers.{layer}"+ dst = f"{_MODEL}.layers.{layer}"+ if args.is_kda_layer(layer):+ yield from _iter_kda_layer(reader, layer, attn_fp8)+ else:+ yield from _iter_dsa_layer(reader, layer, attn_fp8)++ # mHC mixing tensors, fp32 on every layer.+ for hc in ("hc_attn_fn", "hc_attn_base", "hc_attn_scale",+ "hc_ffn_fn", "hc_ffn_base", "hc_ffn_scale"):+ yield f"{dst}.{hc}", reader.get(f"{src}.{hc}").to(torch.float32)++ for norm in ("input_layernorm", "post_attention_layernorm"):+ yield f"{dst}.{norm}.weight", reader.get(f"{src}.{norm}.weight").to(+ torch.bfloat16+ )++ if layer < config.first_k_dense_replace:+ for proj in ("gate_proj", "up_proj", "down_proj"):+ yield from _maybe_fp8(+ f"{dst}.mlp.{proj}", reader.get(f"{src}.mlp.{proj}.weight"), mlp_fp8+ )+ else:+ yield f"{dst}.mlp.gate.weight", reader.get(f"{src}.mlp.gate.weight").to(+ torch.bfloat16+ )+ yield (+ f"{dst}.mlp.e_score_correction_bias",+ # fp32 like HF's router math (the module declares fp32; a bf16+ # cast would perturb top-8 selection on fp32-bias checkpoints).+ reader.get(f"{src}.mlp.gate.e_score_correction_bias").to(torch.float32),+ )+ for proj in ("gate_proj", "up_proj", "down_proj"):+ yield from _maybe_fp8(+ f"{dst}.mlp.shared_experts.{proj}",+ reader.get(f"{src}.mlp.shared_experts.{proj}.weight"),+ mlp_fp8,+ )++ yield f"{_MODEL}.embed_tokens.weight", reader.get(+ f"{_CKPT}.embed_tokens.weight"+ ).to(torch.bfloat16)+ yield f"{_MODEL}.norm.weight", reader.get(f"{_CKPT}.norm.weight").to(torch.bfloat16)+ head = reader.get("lm_head.weight")+ if head_fp8 and not config.tie_word_embeddings:+ q, scale = _quant_fp8_per_row(head)+ yield "lm_head.weight", q+ yield "lm_head.weight_scale", scale+ else:+ yield "lm_head.weight", head.to(torch.bfloat16)+ finally:+ reader.close()+++__all__ = ["iter_weights", "load_nvfp4_expert_sources"]diff --git a/python/freetoken/models/glm_moe_dsa/attention.py b/python/freetoken/models/glm_moe_dsa/attention.pyindex 14b1c100..e25d961f 100644--- a/python/freetoken/models/glm_moe_dsa/attention.py+++ b/python/freetoken/models/glm_moe_dsa/attention.py@@ -93,14 +93,16 @@ class GlmDsaIndexer(BaseOP):def compute(self, x: torch.Tensor, q_resid: torch.Tensor, positions: torch.Tensor- ) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor]:- """Per-token indexer projections: (q [T, H, D], k [T, D], weights [T, H] fp32)."""+ ) -> "DSAIndexerInputs":+ """Per-token indexer projections: q [T, H, D], k [T, D], weights [T, H] fp32."""+ from freetoken.attention.dsa import DSAIndexerInputs+t = x.shape[0]q = self.wq_b.forward(q_resid).view(t, self.n_heads * self.head_dim)k = self.k_norm.forward(self.wk.forward(x)).view(t, self.head_dim)q, k = self._rope.forward(positions, q, k)w = self.weights_proj.forward(x).float() * (self.n_heads**-0.5)- return q.view(t, self.n_heads, self.head_dim), k, w+ return DSAIndexerInputs(q=q.view(t, self.n_heads, self.head_dim), k=k, w=w)class GlmMoeDsaAttention(BaseOP):@@ -213,7 +215,7 @@ class GlmMoeDsaAttention(BaseOP):# DSA: full layers hand the backend this token's indexer projections (the# backend caches the keys, scores the history, and selects top-k); shared# layers pass None and reuse their group leader's selection.- indexer_qkw = (+ indexer_inputs = (self.indexer.compute(x, q_a_resid, ctx.batch.positions)if self.indexer is not None and getattr(ctx.attn_backend, "dsa_enabled", False)else None@@ -223,7 +225,7 @@ class GlmMoeDsaAttention(BaseOP):# concatenated latent copy on the hot path.o_latent = ctx.attn_backend.mla_forward(q_absorbed.contiguous(), q_rope.contiguous(), c_kv.contiguous(),- k_rope.contiguous(), self.layer_id, ctx.batch, indexer_qkw=indexer_qkw,+ k_rope.contiguous(), self.layer_id, ctx.batch, indexer_inputs=indexer_inputs,) # [T, H, kv_lora_rank]# Absorb kv_b's v-part onto the output: o_latent[H,T,lora] @ W_uv_t[H,lora,v].diff --git a/python/freetoken/models/nvfp4_banks.py b/python/freetoken/models/nvfp4_banks.pyindex 0e3ab6a5..6b933ff1 100644--- a/python/freetoken/models/nvfp4_banks.py+++ b/python/freetoken/models/nvfp4_banks.py@@ -22,6 +22,22 @@ class Nvfp4ExpertSourceSpec:proj_to_role: dict[str, str]layer_to_bank: LayerToBankdesc: str+ # Maps checkpoint tensor-kind names onto the canonical (modelopt) kinds, e.g.+ # compressed-tensors' weight_packed -> weight, weight_global_scale -> weight_scale_2.+ kind_map: dict[str, str] | None = None+ # The checkpoint stores the QUANT-side global scale (local fp8 scales were+ # multiplied by it before the cast); the banks keep its reciprocal.+ global_reciprocal: bool = False+++def _canon_kind(spec: "Nvfp4ExpertSourceSpec", kind: str) -> str:+ return spec.kind_map.get(kind, kind) if spec.kind_map else kind+++def _ingest_global(spec: "Nvfp4ExpertSourceSpec", tensor: torch.Tensor) -> torch.Tensor:+ if spec.global_reciprocal:+ tensor = 1.0 / tensor.float()+ return tensor.to(torch.float16)def _num_moe_layers(config) -> int:@@ -112,7 +128,7 @@ def load_nvfp4_expert_source_banks(proj = match.group("proj")if proj not in spec.proj_to_role:raise ValueError(f"{spec.desc}: unknown NVFP4 expert projection {proj!r}")- kind = match.group("kind")+ kind = _canon_kind(spec, match.group("kind"))if kind == "weight_scale_2":global_shards[shard].append((name, match, bank_layer))elif kind in {"weight", "weight_scale"}:@@ -130,7 +146,7 @@ def load_nvfp4_expert_source_banks(int(match.group("expert")),match.group("proj"),)- globals_map[key] = f.get_tensor(name).to(torch.float16)+ globals_map[key] = _ingest_global(spec, f.get_tensor(name))drop_page_cache(path)_hb = _alloc_nvfp4_host_banks(num_layers, E, H, I) # unpinned; pinned after fill@@ -154,7 +170,7 @@ def load_nvfp4_expert_source_banks(expert = int(match.group("expert"))proj = match.group("proj")role = spec.proj_to_role[proj]- kind = match.group("kind")+ kind = _canon_kind(spec, match.group("kind"))tensor = f.get_tensor(name)if kind == "weight":if role == "gate":@@ -236,7 +252,7 @@ def load_nvfp4_expert_source_banks_parallel(bank_layer = _bank_layer(spec, int(match.group("layer")), config)if bank_layer is None:continue- kind = match.group("kind")+ kind = _canon_kind(spec, match.group("kind"))if kind == "weight_scale_2":global_names_by_shard[shard].append(name)elif kind in {"weight", "weight_scale"}:@@ -253,7 +269,7 @@ def load_nvfp4_expert_source_banks_parallel(for name in global_names_by_shard[shard]:m = spec.key_pattern.match(name)globals_map[(int(m.group("layer")), int(m.group("expert")), m.group("proj"))] = (- f.get_tensor(name).to(torch.float16)+ _ingest_global(spec, f.get_tensor(name)))drop_page_cache(path)@@ -279,7 +295,7 @@ def load_nvfp4_expert_source_banks_parallel(expert = int(match.group("expert"))proj = match.group("proj")role = spec.proj_to_role[proj]- kind = match.group("kind")+ kind = _canon_kind(spec, match.group("kind"))if kind == "weight":if role == "gate":gate_up_packed[bank_layer_id][expert, :I] = tensordiff --git a/python/freetoken/models/register.py b/python/freetoken/models/register.pyindex b94d8291..b8c17322 100644--- a/python/freetoken/models/register.py+++ b/python/freetoken/models/register.py@@ -130,6 +130,19 @@ _MODEL_REGISTRY: dict[str, ModelSpec] = {"freetoken.models.glm_moe_dsa","GlmMoeDsaForCausalLM",),+ # GLM-5.3-Flash (model_type glm5_next): hybrid KDA linear attention (34/45 layers)+ # + NoPE-MLA/DSA with a kpool-compressed indexer (11/45), mHC x4 residual streams,+ # 288-expert sigmoid/noaux_tc MoE; natively-multimodal wrapper config (text tower+ # in text_config, weights under model.language_model.), served text-only.+ "Glm5NextForConditionalGeneration": ModelSpec(+ "freetoken.models.glm5_next",+ "Glm5NextForCausalLM",+ ),+ # Text-only sibling (the text_config's own architectures entry).+ "Glm5NextForCausalLM": ModelSpec(+ "freetoken.models.glm5_next",+ "Glm5NextForCausalLM",+ ),}diff --git a/python/freetoken/moe/benchbw.py b/python/freetoken/moe/benchbw.pyindex f3e5359a..83b3d196 100644--- a/python/freetoken/moe/benchbw.py+++ b/python/freetoken/moe/benchbw.py@@ -122,6 +122,10 @@ WORKLOADS: dict[str, Workload] = {activation="gpt_oss_swiglu", swiglu_limit=7.0),"dsv4": Workload("dsv4", 4096, 2048, 256, 6, ("ds_fp4",), swiglu_limit=7.0),"glm4.7-nvfp4": Workload("glm4.7-nvfp4", 5120, 1536, 160, 8, ("nvfp4",)),+ "glm5.3-flash-nvfp4": Workload(+ "glm5.3-flash-nvfp4", 4096, 2048, 288, 8, ("nvfp4",),+ activation="swiglu_clamp", swiglu_alpha=1.0, swiglu_limit=10.0,+ ),"minimax-m2.5": Workload("minimax-m2.5", 3072, 1536, 256, 8, ("nvfp4",)),}diff --git a/python/freetoken/moe/cpu_executor.py b/python/freetoken/moe/cpu_executor.pyindex b96205aa..d9cb43c1 100644--- a/python/freetoken/moe/cpu_executor.py+++ b/python/freetoken/moe/cpu_executor.py@@ -65,6 +65,7 @@ _ACT_IDS = {"gelu_pytorch_tanh": 2,"gpt_oss_swiglu": 3,"swigluoai": 3,+ "swiglu_clamp": 4,}# Weight-format ids must match WFmt in csrc/cpu_moe/cpu_moe_ext.cpp.diff --git a/python/freetoken/moe/fused_nvfp4.py b/python/freetoken/moe/fused_nvfp4.pyindex 29a561e6..f736b80c 100644--- a/python/freetoken/moe/fused_nvfp4.py+++ b/python/freetoken/moe/fused_nvfp4.py@@ -25,6 +25,7 @@ from freetoken.layers import (gelu_and_mul,gelu_tanh_and_mul,silu_and_mul,+ swiglu_clamp_and_mul,swigluoai_and_mul,)from freetoken.moe.fused import moe_align_block_size@@ -40,12 +41,16 @@ def _run_act(act_limit: float,) -> None:"""gemm1 -> gemm2 activation dispatch. ``swigluoai`` (MiniMax-M3, clamped- gpt-oss swiglu over the banks' uninterleaved [gate; up] halves) carries the+ gpt-oss swiglu over the banks' uninterleaved [gate; up] halves) and+ ``swiglu_clamp`` (GLM-5.3, same clamp without the +1 up bias) carry theper-model ``act_alpha``/``act_limit`` scalars; the plain *_and_mul kindsignore them."""if activation == "swigluoai":swigluoai_and_mul(gate_up, out, alpha=act_alpha, limit=act_limit)return+ if activation == "swiglu_clamp":+ swiglu_clamp_and_mul(gate_up, out, alpha=act_alpha, limit=act_limit)+ return_ACT[activation](gate_up, out)# Decode is captured into a CUDA graph, so the config must be fixed (no triton.autotune,diff --git a/tests/attention/test_dsa_kpool.py b/tests/attention/test_dsa_kpool.pynew file mode 100644index 00000000..fb856e17--- /dev/null+++ b/tests/attention/test_dsa_kpool.py@@ -0,0 +1,369 @@+"""Glm5NextDSABackend (kpool indexer) vs an eager reference.++Exercises the full path -- raw K/gate stores, pool-completion compression,+pool-granular scoring, top-k + expansion + tail, gathered sparse MLA -- on a+single request at toy dims (Hi=16 index heads, Di=64, kpool=4, topk=32):++* kv_len <= index_topk: the identity/dense path must equal full softmax MLA.+* kv_len > index_topk: prefill queries must match a subset-softmax reference+ built from an independently computed pooled-score top-k (+ tail).+* decode: pooled entries appear exactly at pool-completion steps and match the+ softmax(gate+APE) reference; decode outputs match the same subset reference.+"""++from __future__ import annotations++from types import SimpleNamespace++import pytest+import torch++pytestmark = pytest.mark.skipif(not torch.cuda.is_available(), reason="needs CUDA")++H, LATENT = 2, 64 # MLA heads, kv_lora_rank (== latent width: NoPE has no kpe half)+HI, DI = 16, 64 # index heads (kernel needs >= 16), index head dim (pow2)+KPOOL, TOPK = 4, 32+SM_SCALE = 0.125+DEV = "cuda"+++def _args(num_layers=1):+ from freetoken.models.glm5_next.args import Glm5NextArgs++ return Glm5NextArgs(+ hidden_size=32, num_heads=H,+ q_lora_rank=16, kv_lora_rank=LATENT, qk_nope_head_dim=LATENT,+ qk_rope_head_dim=0, v_head_dim=LATENT, mla_nope=True, norm_eps=1e-5,+ max_position=4096,+ index_n_heads=HI, index_head_dim=DI, index_topk=TOPK,+ indexer_types=("full",) * num_layers, indexer_rope_interleave=True,+ index_kpool=KPOOL, index_kpool_compress=True,+ index_kpool_always_select_tail=True,+ linear_num_heads=0, linear_head_dim=0, linear_conv_kernel_dim=4,+ linear_lower_bound=-5.0,+ layer_types=("deepseek_sparse_attention",) * num_layers,+ mlp_layer_types=("dense",) * num_layers,+ mhc=False, mhc_num_residual_streams=1, hc_eps=1e-6,+ mhc_sinkhorn_iterations=0, mhc_tau=0.05, mhc_post_mult_value=2.0,+ mhc_no_norm_weight=False, swiglu_limit=None, rope_theta=10000.0,+ )+++def harness(monkeypatch):+ from freetoken.attention.dsa_indexer_kpool import Glm5NextDSABackend+ from freetoken.kvcache.dsa_pool import KpoolDSAKVCache++ pool = KpoolDSAKVCache(+ latent_dim=LATENT, num_layers=1, num_pages=8, page_size=64,+ dtype=torch.bfloat16, device=torch.device(DEV),+ index_head_dim=DI, num_index_layers=1,+ index_ratio=KPOOL, num_req_slots=4,+ )+ page_table = torch.full((4, 512), -1, dtype=torch.int32, device=DEV)+ page_table[0, :512] = torch.arange(512, dtype=torch.int32, device=DEV)+ ctx = SimpleNamespace(kv_cache=pool, page_table=page_table)+ monkeypatch.setattr("freetoken.attention.dsa.get_global_ctx", lambda: ctx)++ config = SimpleNamespace(+ glm5_args=_args(), glm_dsa_args=None, num_qo_heads=H,+ attn_sm_scale=SM_SCALE, num_layers=1,+ )+ backend = Glm5NextDSABackend(config)+ torch.manual_seed(0)+ # The APE is a MODEL parameter, passed per call via DSAIndexerInputs.+ ape = torch.randn(KPOOL, DI, device=DEV, dtype=torch.float32) * 0.3+ return backend, pool, ape+++def _req(device_len, cached_len=0):+ return SimpleNamespace(+ table_idx=0, device_len=device_len, extend_len=device_len - cached_len,+ cached_len=cached_len, linear_slot_idx=None,+ )+++def _prefill_batch(t0, t1):+ return SimpleNamespace(+ phase="prefill", padded_reqs=[_req(t1, t0)], reqs=[_req(t1, t0)],+ positions=torch.arange(t0, t1, device=DEV),+ out_loc=torch.arange(t0, t1, device=DEV),+ active_table_idx=None,+ )+++def _decode_batch(pos):+ return SimpleNamespace(+ phase="decode", padded_reqs=[_req(pos + 1, pos)], reqs=[_req(pos + 1, pos)],+ positions=torch.tensor([pos], device=DEV),+ out_loc=torch.tensor([pos], device=DEV),+ active_table_idx=torch.tensor([0], device=DEV),+ )+++def _rand_seq(total, seed=1):+ torch.manual_seed(seed)+ mk = lambda *s: torch.randn(*s, device=DEV, dtype=torch.bfloat16)+ return dict(+ q_nope=mk(total, H, LATENT), c_kv=mk(total, LATENT),+ qi=mk(total, HI, DI), ki=mk(total, DI),+ wi=(torch.randn(total, HI, device=DEV).float() * 0.5),+ gate=mk(total, DI),+ )+++def _run(backend, batch, d, sl, ape):+ from freetoken.attention.dsa import DSAIndexerInputs++ t = batch.positions.shape[0]+ backend.prepare_metadata(batch)+ return backend.mla_forward(+ d["q_nope"][sl], d["q_nope"].new_empty(t, H, 0),+ d["c_kv"][sl], d["c_kv"].new_empty(t, 0),+ 0, batch,+ indexer_inputs=DSAIndexerInputs(+ q=d["qi"][sl], k=d["ki"][sl], w=d["wi"][sl],+ gate=d["gate"][sl], ape=ape,+ ),+ )+++# ---- eager references -----------------------------------------------------------------+++def _ref_pooled(d, ape, n_pools):+ """[n_pools, DI] softmax(gate+ape)-weighted pooled keys (fp32 -> bf16)."""+ k = d["ki"][: n_pools * KPOOL].view(n_pools, KPOOL, DI).float()+ g = d["gate"][: n_pools * KPOOL].view(n_pools, KPOOL, DI).float()+ w = torch.softmax(g + ape, dim=1)+ return (w * k).sum(1).to(torch.bfloat16)+++def _ref_scores(d, ape, q_idx_t, w_t, n_pools):+ """Pool scores for one query: sum_h w_h * relu(q_h . k_pool) * DI**-0.5."""+ kp = _ref_pooled(d, ape, n_pools).float()+ s = torch.relu(q_idx_t.float() @ kp.T) # [HI, n_pools]+ return ((w_t * DI**-0.5).unsqueeze(1) * s).sum(0)+++def _ref_attend(d, q_t, positions):+ """Full softmax MLA over latent rows at ``positions`` for one query [H, LATENT]."""+ lat = d["c_kv"][positions].float() # [n, LATENT]+ logits = q_t.float() @ lat.T * SM_SCALE # [H, n]+ p = torch.softmax(logits, dim=-1)+ return (p @ lat).to(torch.bfloat16)+++def _ref_selected_positions(d, ape, q_idx_t, w_t, pos):+ """Reference kpool selection for a query at ``pos``: top-k complete pools+ expanded to tokens, plus the tail [n_pools*KPOOL, pos]."""+ n_pools = (pos + 1) // KPOOL+ sel_pools = min(TOPK // KPOOL, n_pools)+ picked = torch.topk(+ _ref_scores(d, ape, q_idx_t, w_t, n_pools), sel_pools+ ).indices.tolist()+ positions = [p * KPOOL + o for p in picked for o in range(KPOOL)]+ positions += list(range(n_pools * KPOOL, pos + 1))+ return sorted(set(positions))+++def test_dense_path_matches_full_softmax(harness):+ backend, pool, ape = harness+ total = 20 # < TOPK -> identity/dense path+ d = _rand_seq(total)+ out = _run(backend, _prefill_batch(0, total), d, slice(0, total), ape)+ for t in range(total):+ ref = _ref_attend(d, d["q_nope"][t], list(range(t + 1)))+ err = (out[t].float() - ref.float()).abs().max().item()+ assert err < 2e-2, f"dense query {t}: err {err}"+++def test_sparse_prefill_matches_reference(harness):+ backend, pool, ape = harness+ total = 60 # > TOPK -> kpool scoring+ d = _rand_seq(total, seed=2)+ out = _run(backend, _prefill_batch(0, total), d, slice(0, total), ape)++ # Pooled entries in the slab match the compression reference.+ n_pools = total // KPOOL+ # Shadow slab: pool p lives at token_slot // KPOOL == p (identity page table).+ slab = pool.index_k_cache(0)[torch.arange(n_pools, device=DEV)]+ ref_pool = _ref_pooled(d, ape, n_pools)+ assert (slab.float() - ref_pool.float()).abs().max().item() < 2e-2++ for t in (35, 47, 59): # queries past TOPK (sparse regime)+ sel = _ref_selected_positions(d, ape, d["qi"][t], d["wi"][t], t)+ ref = _ref_attend(d, d["q_nope"][t], sel)+ err = (out[t].float() - ref.float()).abs().max().item()+ assert err < 3e-2, f"sparse query {t}: err {err}"+++def test_decode_completion_and_selection(harness):+ backend, pool, ape = harness+ total, extra = 60, 6 # decode positions 60..65; completion at 63+ d = _rand_seq(total + extra, seed=3)+ _run(backend, _prefill_batch(0, total), d, slice(0, total), ape)++ for pos in range(total, total + extra):+ out = _run(backend, _decode_batch(pos), d, slice(pos, pos + 1), ape)+ sel = _ref_selected_positions(d, ape, d["qi"][pos], d["wi"][pos], pos)+ ref = _ref_attend(d, d["q_nope"][pos], sel)+ err = (out[0].float() - ref.float()).abs().max().item()+ assert err < 3e-2, f"decode pos {pos}: err {err}"++ if pos % KPOOL == KPOOL - 1: # a pool completed this step+ n_pools = (pos + 1) // KPOOL+ row = pos // KPOOL # the pool's shadow row+ got = pool.index_k_cache(0)[row]+ want = _ref_pooled(d, ape, n_pools)[-1]+ assert (got.float() - want.float()).abs().max().item() < 2e-2+++def test_sparse_batch_with_sub_pool_request(harness):+ """A sparse prefill batch may carry a request shorter than one pool: it has nothing+ to score and must come out as the dense (tail-only) attention over its own rows."""+ from freetoken.attention import dsa as dsa_mod++ backend, pool, ape = harness+ dsa_mod.get_global_ctx().page_table[1, :64] = torch.arange(+ 448, 512, dtype=torch.int32, device=DEV+ )+ ta = 60 # > TOPK -> the batch takes the sparse path+ for tb in (1, 2, 3):+ d = _rand_seq(ta + tb, seed=6)+ req_b = _req(tb)+ req_b.table_idx = 1+ reqs = [_req(ta), req_b]+ batch = SimpleNamespace(+ phase="prefill", padded_reqs=reqs, reqs=reqs,+ positions=torch.cat([torch.arange(ta), torch.arange(tb)]).to(DEV),+ out_loc=torch.cat([torch.arange(ta), torch.arange(448, 448 + tb)]).to(DEV),+ active_table_idx=None,+ )+ out = _run(backend, batch, d, slice(0, ta + tb), ape)+ for j in range(tb):+ t = ta + j+ ref = _ref_attend(d, d["q_nope"][t], list(range(ta, t + 1)))+ err = (out[t].float() - ref.float()).abs().max().item()+ assert err < 3e-2, f"sub-pool request len {tb}, query {j}: err {err}"+ t = ta - 1+ sel = _ref_selected_positions(d, ape, d["qi"][t], d["wi"][t], t)+ ref = _ref_attend(d, d["q_nope"][t], sel)+ assert (out[t].float() - ref.float()).abs().max().item() < 3e-2+++def test_chunked_prefill_mid_pool_start(harness):+ """A chunk may start MID-POOL (main's soft prefill_chunk_align keeps an+ unaligned end when the budget cannot fill a page): the straddling pool's+ older members come from the tail ring and the pooled slab + outputs must+ match the single-shot run."""+ backend, pool, ape = harness+ total, split = 60, 30 # split % KPOOL == 2 -> pool 7 straddles the chunks+ d = _rand_seq(total, seed=4)+ out1 = _run(backend, _prefill_batch(0, split), d, slice(0, split), ape)+ out2 = _run(backend, _prefill_batch(split, total), d, slice(split, total), ape)++ n_pools = total // KPOOL+ slab = pool.index_k_cache(0)[torch.arange(n_pools, device=DEV)]+ ref_pool = _ref_pooled(d, ape, n_pools)+ assert (slab.float() - ref_pool.float()).abs().max().item() < 2e-2++ for t in (35, 47, 59):+ sel = _ref_selected_positions(d, ape, d["qi"][t], d["wi"][t], t)+ ref = _ref_attend(d, d["q_nope"][t], sel)+ err = (out2[t - split].float() - ref.float()).abs().max().item()+ assert err < 3e-2, f"mid-pool chunked query {t}: err {err}"+++def test_interleaved_decode_requests_do_not_pollute_rings(harness):+ """Two requests decoding in alternation: tail rings and shadow rows are keyed+ by table_idx, so neither request's pools may absorb the other's raw K/gate."""+ from freetoken.attention import dsa as dsa_mod++ backend, pool, ape = harness+ dsa_mod.get_global_ctx().page_table[1, :128] = torch.arange(+ 256, 384, dtype=torch.int32, device=DEV+ )+ total, extra = 60, 6+ streams = { # table_idx -> (data, physical row base)+ 0: (_rand_seq(total + extra, seed=7), 0),+ 1: (_rand_seq(total + extra, seed=8), 256),+ }++ def _batch_for(table, phase, t0, t1):+ base = streams[table][1]+ req = _req(t1, t0)+ req.table_idx = table+ return SimpleNamespace(+ phase=phase, padded_reqs=[req], reqs=[req],+ positions=torch.arange(t0, t1, device=DEV),+ out_loc=torch.arange(base + t0, base + t1, device=DEV),+ active_table_idx=(+ torch.tensor([table], device=DEV) if phase == "decode" else None+ ),+ )++ for table in (0, 1):+ d = streams[table][0]+ _run(backend, _batch_for(table, "prefill", 0, total), d, slice(0, total), ape)++ for pos in range(total, total + extra):+ for table in (0, 1): # alternate every step+ d, base = streams[table]+ out = _run(+ backend, _batch_for(table, "decode", pos, pos + 1), d,+ slice(pos, pos + 1), ape,+ )+ sel = _ref_selected_positions(d, ape, d["qi"][pos], d["wi"][pos], pos)+ ref = _ref_attend(d, d["q_nope"][pos], sel)+ err = (out[0].float() - ref.float()).abs().max().item()+ assert err < 3e-2, f"table {table} decode pos {pos}: err {err}"++ if pos % KPOOL == KPOOL - 1:+ n_pools = (pos + 1) // KPOOL+ row = (base + pos) // KPOOL+ got = pool.index_k_cache(0)[row]+ want = _ref_pooled(d, ape, n_pools)[-1]+ err = (got.float() - want.float()).abs().max().item()+ assert err < 2e-2, f"table {table} pool at pos {pos}: err {err}"+++def test_padding_and_empty_batch_leave_shadow_rows_clean():+ """The compression kernel writes every row somewhere, but masked-off rows+ (padding request == -1, or a pool that cannot close yet) must land only on+ their designated scratch rows -- the shadow region and the rings stay clean.+ An empty batch is a no-op."""+ from freetoken.kernel.triton.kpool_compress import kpool_compress_store++ shadow_n, n_req = 8, 2+ slab = torch.full((shadow_n + n_req, DI), 7.0, dtype=torch.bfloat16, device=DEV)+ ring_k = torch.full((n_req * KPOOL, DI), 3.0, dtype=torch.bfloat16, device=DEV)+ ring_g = ring_k.clone()+ ape = torch.randn(KPOOL, DI, dtype=torch.float32, device=DEV)+ k = torch.randn(2, DI, dtype=torch.bfloat16, device=DEV)+ gate = torch.randn(2, DI, dtype=torch.bfloat16, device=DEV)++ kpool_compress_store(+ k, gate, ring_k, ring_g, ape,+ ring_slots=torch.tensor([0, 1], dtype=torch.int32, device=DEV),+ token_to_req=torch.tensor([0, -1], dtype=torch.int32, device=DEV),+ cu_seqlens=torch.tensor([0, 1, 1], dtype=torch.int32, device=DEV),+ positions=torch.tensor([0, 0], device=DEV), # pos 0: no pool can close+ slab=slab, cmp_rows=torch.tensor([8, 9], dtype=torch.int32, device=DEV),+ ratio=KPOOL,+ )+ assert torch.equal(slab[:shadow_n], torch.full_like(slab[:shadow_n], 7.0))+ assert torch.equal(ring_k, torch.full_like(ring_k, 3.0))+ assert torch.equal(ring_g, ring_k)++ before = slab.clone()+ kpool_compress_store(+ k[:0], gate[:0], ring_k, ring_g, ape,+ ring_slots=torch.tensor([0], dtype=torch.int32, device=DEV),+ token_to_req=torch.empty(0, dtype=torch.int32, device=DEV),+ cu_seqlens=torch.tensor([0, 0], dtype=torch.int32, device=DEV),+ positions=torch.empty(0, dtype=torch.int64, device=DEV),+ slab=slab, cmp_rows=torch.empty(0, dtype=torch.int32, device=DEV),+ ratio=KPOOL,+ )+ assert torch.equal(slab, before)diff --git a/tests/e2e/test_aime.py b/tests/e2e/test_aime.pyindex 0a4c3a67..f2803539 100644--- a/tests/e2e/test_aime.py+++ b/tests/e2e/test_aime.py@@ -18,6 +18,10 @@ Env knobs:FREETOKEN_AIME_SAMPLES pass@N sample count when sampling (default 3)FREETOKEN_AIME_MIN_FREE_GIB free-GPU-memory gate (default 70; raise for a big resident model)FREETOKEN_TEST_MOE_CACHE_SIZE >0 switches to the offload MoE backend (fp8/GLM/MiniMax)+ FREETOKEN_TEST_MOE_BACKEND offload-family backend override (default offload; e.g. hybrid)+ FREETOKEN_TEST_MOE_CPU_THREADS CPU MoE worker threads (0 = physical cores)+ FREETOKEN_TEST_MOE_CACHE_AUTO 1 sizes the MoE cache from free VRAM instead of MOE_CACHE_SIZE+ FREETOKEN_TEST_KV_RESERVE KV token floor reserved before auto-cache fills expertsFREETOKEN_TEST_MEM_RATIO offload memory ratio (default 0.9)fp8 / offload recipe: point FREETOKEN_TEST_MODEL at the fp8 checkpoint dir and set@@ -164,10 +168,14 @@ def build_llm(model_path: Path) -> LLM:max_extend_tokens=8192,)cache_size = int(os.environ.get("FREETOKEN_TEST_MOE_CACHE_SIZE", "0"))- if cache_size > 0:+ cache_auto = os.environ.get("FREETOKEN_TEST_MOE_CACHE_AUTO") == "1"+ if cache_size > 0 or cache_auto:kwargs.update(- moe_backend="offload",+ moe_backend=os.environ.get("FREETOKEN_TEST_MOE_BACKEND", "offload"),+ moe_cpu_threads=int(os.environ.get("FREETOKEN_TEST_MOE_CPU_THREADS", "0")),+ moe_cache_auto=cache_auto,moe_cache_size=cache_size,+ kv_reserve_tokens=int(os.environ.get("FREETOKEN_TEST_KV_RESERVE", "8192")),moe_cache_policy="lru",memory_ratio=float(os.environ.get("FREETOKEN_TEST_MEM_RATIO", "0.9")),max_seq_len_override=max_tokens() + 2048,diff --git a/tests/engine/test_attention_backend_matrix.py b/tests/engine/test_attention_backend_matrix.pyindex fc9abfbe..ef7dc7ff 100644--- a/tests/engine/test_attention_backend_matrix.py+++ b/tests/engine/test_attention_backend_matrix.py@@ -259,13 +259,15 @@ def test_legal_explicit_combinations_pass(monkeypatch, kind, backend):@pytest.mark.parametrize("kind", ["mla", "dsa"])-def test_mla_requires_page_size_one(monkeypatch, kind):+def test_mla_auto_adjusts_page_size(monkeypatch, kind):+ """Any --page-size is auto-adjusted (with a warning) to the layout's+ required size: 1 for plain latent-KV, 64 for the kpool variant."""from freetoken.engine.engine import _adjust_config_patch_env(monkeypatch)config = _config(kind, attention_backend="auto", page_size=16)- with pytest.raises(ValueError, match="page-size 1"):- _adjust_config(config)+ _adjust_config(config)+ assert config.page_size == 1def test_trtllm_page_size_coercion_is_part_aware(monkeypatch):diff --git a/tests/kernels/test_kda.py b/tests/kernels/test_kda.pynew file mode 100644index 00000000..b217ee32--- /dev/null+++ b/tests/kernels/test_kda.py@@ -0,0 +1,106 @@+"""Pins for OUR divergences from the upstream-vendored KDA kernels.++The vendored kernels (freetoken/kernel/fla) are tested upstream and not re-tested+here. The eager reference replicates their exact math (safe gate ``gk =+lower_bound * sigmoid(exp(A_log) * (g_raw + dt_bias))``, ``beta =+sigmoid(beta_raw)``, in-loop q/k l2norm, per-channel-decayed delta rule on a+[V, K] state) so a divergence pin can assert numerics, not just reachability.+"""++from __future__ import annotations++import pytest+import torch++pytestmark = pytest.mark.skipif(not torch.cuda.is_available(), reason="needs CUDA")++H, D = 4, 128 # head count trimmed; head_dim matches GLM-5.3 (kernel specializes on D)+LOWER_BOUND = -5.0+SCALE = D**-0.5+++def _l2norm(x: torch.Tensor) -> torch.Tensor:+ return x / torch.sqrt((x * x).sum(-1, keepdim=True) + 1e-6)+++def _reference(+ q: torch.Tensor, # [T, H, D] bf16+ k: torch.Tensor,+ v: torch.Tensor,+ g_raw: torch.Tensor, # [T, H, D] bf16+ beta_raw: torch.Tensor, # [T, H] bf16+ a_log: torch.Tensor, # [H] fp32+ dt_bias: torch.Tensor, # [H*D] fp32+ h0: torch.Tensor | None = None, # [H, V, K] fp32+) -> tuple[torch.Tensor, torch.Tensor]:+ T = q.shape[0]+ h = (+ h0.clone().float()+ if h0 is not None+ else torch.zeros(H, D, D, dtype=torch.float32, device=q.device)+ )+ amp = a_log.float().exp().view(H, 1)+ bias = dt_bias.float().view(H, D)+ outs = []+ for t in range(T):+ gk = LOWER_BOUND * torch.sigmoid(amp * (g_raw[t].float() + bias)) # [H, K]+ h = h * gk.exp().unsqueeze(1) # decay per k-channel: [H, V, K] * [H, 1, K]+ kt = _l2norm(k[t].float())+ v_err = v[t].float() - torch.einsum("hvk,hk->hv", h, kt)+ v_err = v_err * torch.sigmoid(beta_raw[t].float()).unsqueeze(-1)+ h = h + torch.einsum("hv,hk->hvk", v_err, kt)+ qt = _l2norm(q[t].float()) * SCALE+ outs.append(torch.einsum("hvk,hk->hv", h, qt))+ return torch.stack(outs), h+++def _rand_inputs(T: int, seed: int = 0, device="cuda"):+ torch.manual_seed(seed)+ mk = lambda *s: torch.randn(*s, device=device, dtype=torch.bfloat16)+ q, k, v, g_raw = mk(T, H, D), mk(T, H, D), mk(T, H, D), mk(T, H, D)+ beta_raw = mk(T, H)+ a_log = torch.randn(H, device=device, dtype=torch.float32) * 0.5+ dt_bias = torch.randn(H * D, device=device, dtype=torch.float32) * 0.5+ return q, k, v, g_raw, beta_raw, a_log, dt_bias+++def _assert_close(ours, ref, tag, atol=2e-2, rtol=2e-2):+ ours, ref = ours.float(), ref.float()+ err = (ours - ref).abs().max().item()+ rel = err / (ref.abs().max().item() + 1e-8)+ assert torch.allclose(ours, ref, atol=atol, rtol=rtol), (+ f"{tag}: max abs err {err:.5f}, rel {rel:.5f}"+ )+++def test_fused_recurrent_serves_slot_zero():+ """--cache-type naive keys state by raw table_idx, so a real request can sit+ on slot 0. Upstream vLLM's kernel treats 0 as its NULL_BLOCK_ID sentinel and+ silently skips it (state frozen, garbage output); our vendored copy diverges+ to accept every non-negative slot (GDN-kernel parity). Same math as the+ parametrized reference test, just on slot 0."""+ from freetoken.kernel.fla import fused_recurrent_kda++ T = 7+ q, k, v, g_raw, beta_raw, a_log, dt_bias = _rand_inputs(T)+ ref_o, ref_h = _reference(q, k, v, g_raw, beta_raw, a_log, dt_bias)++ pool = torch.zeros(2, H, D, D, dtype=torch.float32, device="cuda")+ indices = torch.zeros((1, T), dtype=torch.int64, device="cuda") # slot 0+ cu = torch.tensor([0, T], dtype=torch.int32, device="cuda")+ o, _ = fused_recurrent_kda(+ q=q.unsqueeze(0), k=k.unsqueeze(0), v=v.unsqueeze(0),+ g=g_raw.unsqueeze(0), beta=beta_raw.unsqueeze(0),+ initial_state=pool,+ use_qk_l2norm_in_kernel=True,+ cu_seqlens=cu,+ ssm_state_indices=indices,+ sigmoid_beta=True,+ a_log=a_log,+ g_bias=dt_bias,+ compute_gate=True,+ lower_bound=LOWER_BOUND,+ )+ _assert_close(o[0], ref_o, "slot-0 output")+ _assert_close(pool[0], ref_h, "slot-0 final state")+ assert pool[1].abs().max().item() == 0.0 # only slot 0 was toucheddiff --git a/tests/kernels/test_swiglu_clamp.py b/tests/kernels/test_swiglu_clamp.pynew file mode 100644index 00000000..2515bcdd--- /dev/null+++ b/tests/kernels/test_swiglu_clamp.py@@ -0,0 +1,43 @@+"""swiglu_clamp (GLM-5.3 ``swiglu_limit``) activation parity.++Reference: vLLM's SiluAndMulWithClamp with alpha=1, beta=0 --+``clamp(gate, max=L) * sigmoid(gate_clamped) * clamp(up, +-L)``. Checks the+Triton kernel, its distinction from swigluoai (the +1 up bias), and that the+compiled CPU MoE extension advertises the new generic act id.+"""++from __future__ import annotations++import pytest+import torch++LIMIT = 10.0+++def _ref(x: torch.Tensor, limit: float = LIMIT) -> torch.Tensor:+ d = x.shape[-1] // 2+ gate = x[..., :d].float().clamp(max=limit)+ up = x[..., d:].float().clamp(min=-limit, max=limit)+ return (gate * torch.sigmoid(gate) * up).to(x.dtype)++[email protected](not torch.cuda.is_available(), reason="needs CUDA")+def test_triton_matches_reference():+ from freetoken.layers import swiglu_clamp_and_mul++ torch.manual_seed(0)+ # Scale up so the clamp actually engages on a good fraction of elements.+ x = torch.randn(129, 2 * 512, device="cuda", dtype=torch.bfloat16) * 8.0+ out = swiglu_clamp_and_mul(x, alpha=1.0, limit=LIMIT)+ ref = _ref(x)+ assert (out.float() - ref.float()).abs().max().item() < 2e-2+ assert (x[..., :512].float() > LIMIT).any(), "test data never hit the clamp"+++def test_cpu_extension_supports_swiglu_clamp():+ from freetoken.moe.cpu_executor import compiled_extension_supports++ assert compiled_extension_supports("swiglu_clamp"), (+ "compiled _cpu_moe extension is stale -- rebuild with ACT_SWIGLU_CLAMP "+ "(python setup.py build_ext --inplace)"+ )diff --git a/tests/kvcache/test_hybrid_linear_paged_pools.py b/tests/kvcache/test_hybrid_linear_paged_pools.pynew file mode 100644index 00000000..8b60801c--- /dev/null+++ b/tests/kvcache/test_hybrid_linear_paged_pools.py@@ -0,0 +1,117 @@+"""Pool-factory generalization: hybrid linear x ANY paged family.++The old factory hard-required "linear + one GQA group" (Qwen3.5 GDN shape);+glm5_next is linear (KDA) x DSA. Checks the factory dispatch, the MLA/DSA+layer-id remap (34 KDA layers cost no latent slabs), the kpool pool selection+(gate slab), and the KV cost model's kpool double-count of the index slabs.+"""++from __future__ import annotations++import pytest+import torch++from freetoken.kvcache import create_kvcache_pool, resolve_pool_class+from freetoken.kvcache.dsa_pool import DSAKVCache, KpoolDSAKVCache+from freetoken.models.config import (+ FullAttentionGroupConfig,+ LinearGatedDeltaGroupConfig,+ ModelConfig,+ RotaryConfig,+)++[email protected](autouse=True)+def _single_rank_tp():+ from freetoken.distributed import set_tp_info, try_get_tp_info++ if try_get_tp_info() is None:+ set_tp_info(rank=0, size=1)+++def _glm5_like_config(index_kpool=4, index_head_dim=128):+ n_layers = 12+ dsa_ids = tuple(range(3, n_layers, 4)) # 3, 7, 11+ kda_ids = tuple(i for i in range(n_layers) if i not in dsa_ids)+ rotary = RotaryConfig(head_dim=256, rotary_dim=0, max_position=4096, base=1e4, scaling=None)+ groups = (+ LinearGatedDeltaGroupConfig(+ name="linear", layer_ids=kda_ids,+ num_key_heads=4, num_value_heads=4, key_head_dim=128, value_head_dim=128,+ conv_kernel_dim=4, output_gate="sigmoid", variant="kda",+ ),+ FullAttentionGroupConfig(+ name="full", layer_ids=dsa_ids, num_kv_heads=1, head_dim=512,+ rotary_config=rotary, mla=True,+ index_head_dim=index_head_dim, num_index_layers=len(dsa_ids),+ index_ratio=index_kpool,+ ),+ )+ return ModelConfig(+ num_layers=n_layers, num_qo_heads=4, num_kv_heads=1, head_dim=512,+ hidden_size=256, vocab_size=1000, intermediate_size=512,+ rms_norm_eps=1e-5, rotary_config=rotary, hidden_act="silu",+ tie_word_embeddings=False, num_experts=8, num_experts_per_tok=2,+ moe_intermediate_size=64, norm_topk_prob=True, model_type="glm5_next",+ architectures=["Glm5NextForCausalLM"], moe_enabled=True,+ attention_groups=groups,+ )+++def test_factory_builds_kpool_pool_with_layer_remap():+ cfg = _glm5_like_config()+ assert resolve_pool_class(cfg) is KpoolDSAKVCache++ pool = create_kvcache_pool(+ model_config=cfg, num_pages=4, page_size=64,+ dtype=torch.bfloat16, device=torch.device("cpu"), num_req_slots=5,+ )+ assert isinstance(pool, KpoolDSAKVCache)+ # Latent slabs back ONLY the 3 DSA layers (34-of-45 economy at real scale).+ assert pool._kv_buffer.shape[1] == 3+ # Global layer-id addressing: DSA layers resolve, KDA layers have no slab.+ for lid in (3, 7, 11):+ assert pool.latent_rows(lid).shape == (256, 512)+ with pytest.raises(KeyError):+ pool.latent_rows(0) # a KDA layer+ # Shadow index slab: tokens/ratio rows + one scratch row per request slot.+ assert pool.index_k_cache(0).shape == (256 // 4 + 5, 128)+ assert pool.cmp_scratch_base == 256 // 4+ # kpool tail rings exist at [num_req_slots, ratio, head_dim] per indexer layer.+ assert pool.tail_k(0).shape == pool.tail_gate(0).shape+ assert pool.tail_k(0).shape == (5, 4, 128)+++def test_factory_kpool1_builds_plain_dsa_pool():+ cfg = _glm5_like_config(index_kpool=1)+ assert resolve_pool_class(cfg) is DSAKVCache+ pool = create_kvcache_pool(+ model_config=cfg, num_pages=16, page_size=1,+ dtype=torch.bfloat16, device=torch.device("cpu"),+ )+ assert type(pool) is DSAKVCache+++def test_cost_model_kpool_shadow_slab_quarter_cost():+ """The shadow slab stores one row per index_ratio tokens: the kpool spec's+ index bytes are 1/ratio of the plain DSA slab (rings/scratch are per-request+ and not part of the per-token price)."""+ from types import SimpleNamespace++ from freetoken.kvcache.base import spec_kv_bytes_per_token++ tp = SimpleNamespace(size=1)+ econf = SimpleNamespace(tp_info=tp, dtype=torch.bfloat16)+ (spec,) = [+ s for s in _glm5_like_config().kv_cache_group_specs() if s.num_layers > 0+ ]+ (spec1,) = [+ s+ for s in _glm5_like_config(index_kpool=1).kv_cache_group_specs()+ if s.num_layers > 0+ ]+ index_full = spec.index_head_dim * spec.num_index_layers * 2+ assert (+ spec_kv_bytes_per_token(spec1, econf) - spec_kv_bytes_per_token(spec, econf)+ == index_full - index_full // 4+ )\ No newline at end of filediff --git a/tests/layers/test_mhc.py b/tests/layers/test_mhc.pynew file mode 100644index 00000000..4cf144e7--- /dev/null+++ b/tests/layers/test_mhc.py@@ -0,0 +1,178 @@+"""mHC (Manifold-Constrained Hyper-Connections) unit tests.++Checks the algebraic contracts of layers/mhc.py at GLM-5.3 geometry (n=4):+Sinkhorn projection yields an (approximately) doubly-stochastic comb matrix,+pre/post mixing matches naive per-token einsums, and identity-ish weights give+the classic single-stream residual behaviour.+"""++from __future__ import annotations++import pytest+import torch++from freetoken.layers.mhc import (+ hc_contract,+ hc_expand,+ mhc_fused_post_pre,+ mhc_post,+ mhc_pre,+)++N, HIDDEN, T = 4, 64, 9+MIX = 2 * N + N * N+EPS = 1e-6+RMS_EPS = 1e-5+POST_MULT = 2.0+SINKHORN = 20+++def _weights(seed=0, device="cpu"):+ torch.manual_seed(seed)+ fn = torch.randn(MIX, N * HIDDEN, dtype=torch.float32, device=device) * 0.05+ scale = torch.randn(3, dtype=torch.float32, device=device).abs() + 0.5+ base = torch.randn(MIX, dtype=torch.float32, device=device) * 0.3+ return fn, scale, base+++def _residual(seed=1, device="cpu"):+ torch.manual_seed(seed)+ return torch.randn(T, N, HIDDEN, dtype=torch.bfloat16, device=device)+++def test_comb_is_doubly_stochastic():+ fn, scale, base = _weights(seed=2)+ res = _residual(seed=3)+ _, comb, _ = mhc_pre(res, fn, scale, base, RMS_EPS, EPS, POST_MULT, SINKHORN)+ rows = comb.sum(dim=-1)+ cols = comb.sum(dim=-2)+ assert torch.allclose(rows, torch.ones_like(rows), atol=1e-3)+ assert torch.allclose(cols, torch.ones_like(cols), atol=1e-3)+ assert (comb > 0).all()+++def test_post_matches_naive():+ res = _residual(seed=4)+ x = torch.randn(T, HIDDEN, dtype=torch.bfloat16)+ post = torch.rand(T, N, 1, dtype=torch.float32) * POST_MULT+ comb = torch.softmax(torch.randn(T, N, N), dim=-1)+ out = mhc_post(x, res, post, comb)+ assert out.shape == (T, N, HIDDEN)++ ref = torch.zeros(T, N, HIDDEN, dtype=torch.float32)+ for t in range(T):+ for j in range(N):+ acc = post[t, j, 0] * x[t].float()+ for i in range(N):+ acc = acc + comb[t, i, j] * res[t, i].float()+ ref[t, j] = acc+ assert torch.allclose(out.float(), ref.to(torch.bfloat16).float())+++def test_fused_equals_decomposed():+ fn, scale, base = _weights(seed=7)+ res = _residual(seed=8)+ x = torch.randn(T, HIDDEN, dtype=torch.bfloat16)+ post0, comb0, _ = mhc_pre(res, fn, scale, base, RMS_EPS, EPS, POST_MULT, SINKHORN)++ r1, p1, c1, li1 = mhc_fused_post_pre(+ x, res, post0, comb0, fn, scale, base, RMS_EPS, EPS, POST_MULT, SINKHORN+ )+ r_ref = mhc_post(x, res, post0, comb0)+ p_ref, c_ref, li_ref = mhc_pre(+ r_ref, fn, scale, base, RMS_EPS, EPS, POST_MULT, SINKHORN+ )+ assert torch.equal(r1, r_ref)+ assert torch.equal(p1, p_ref)+ assert torch.equal(c1, c_ref)+ assert torch.equal(li1, li_ref)++[email protected](not torch.cuda.is_available(), reason="needs CUDA")[email protected]("t,hidden", [(1, 64), (9, 64), (3, 4096)])+def test_triton_fused_matches_torch(t, hidden):+ """The fused triton kernel must reproduce the decomposed torch reference+ (hc_post -> hc_pre) on every output, including GLM-5.3's real hidden size."""+ from freetoken.layers.mhc import mhc_fused_post_pre_torch+ from freetoken.kernel.triton.mhc import mhc_fused_post_pre_triton++ torch.manual_seed(11)+ mix = 2 * N + N * N+ fn = torch.randn(mix, N * hidden, dtype=torch.float32, device="cuda") * 0.05+ scale = torch.rand(3, dtype=torch.float32, device="cuda") + 0.5+ base = torch.randn(mix, dtype=torch.float32, device="cuda") * 0.3+ res = torch.randn(t, N, hidden, dtype=torch.bfloat16, device="cuda")+ x = torch.randn(t, hidden, dtype=torch.bfloat16, device="cuda")+ post0 = torch.rand(t, N, 1, dtype=torch.float32, device="cuda") * POST_MULT+ comb0 = torch.softmax(torch.randn(t, N, N, device="cuda"), dim=-1)++ ref = mhc_fused_post_pre_torch(+ x, res, post0, comb0, fn, scale, base, RMS_EPS, EPS, POST_MULT, SINKHORN+ )+ got = mhc_fused_post_pre_triton(+ x, res, post0, comb0, fn, scale, base, RMS_EPS, EPS, POST_MULT, SINKHORN+ )+ names = ("residual", "post", "comb", "layer_input")+ tols = (2e-2, 2e-3, 2e-3, 2e-2)+ for name, r, g, tol in zip(names, ref, got, tols):+ assert g.shape == r.shape, (name, g.shape, r.shape)+ err = (g.float() - r.float()).abs().max().item()+ assert err < tol, f"{name}: max abs err {err}"++[email protected](not torch.cuda.is_available(), reason="needs CUDA")[email protected]("dtype,tol", [(torch.float16, 2e-2), (torch.float32, 1e-4)])+def test_triton_fused_respects_input_dtype(dtype, tol):+ """--dtype float16/float32 must not be silently bf16-rounded: the kernel+ stores in the OUTPUT tensor's dtype (regression for the hard-coded+ tl.bfloat16 stores; fp32's tolerance is far below bf16's 2^-8 grid)."""+ from freetoken.layers.mhc import mhc_fused_post_pre_torch+ from freetoken.kernel.triton.mhc import mhc_fused_post_pre_triton++ torch.manual_seed(13)+ t, hidden = 4, 4096+ mix = 2 * N + N * N+ fn = torch.randn(mix, N * hidden, dtype=torch.float32, device="cuda") * 0.05+ scale = torch.rand(3, dtype=torch.float32, device="cuda") + 0.5+ base = torch.randn(mix, dtype=torch.float32, device="cuda") * 0.3+ res = torch.randn(t, N, hidden, dtype=dtype, device="cuda")+ x = torch.randn(t, hidden, dtype=dtype, device="cuda")+ post0 = torch.rand(t, N, 1, dtype=torch.float32, device="cuda") * POST_MULT+ comb0 = torch.softmax(torch.randn(t, N, N, device="cuda"), dim=-1)++ ref = mhc_fused_post_pre_torch(+ x, res, post0, comb0, fn, scale, base, RMS_EPS, EPS, POST_MULT, SINKHORN+ )+ got = mhc_fused_post_pre_triton(+ x, res, post0, comb0, fn, scale, base, RMS_EPS, EPS, POST_MULT, SINKHORN+ )+ assert got[0].dtype == dtype and got[3].dtype == dtype+ for name, r, g in zip(("residual", "layer_input"), (ref[0], ref[3]), (got[0], got[3])):+ err = (g.float() - r.float()).abs().max().item()+ assert err < tol, f"{name} [{dtype}]: max abs err {err}"++[email protected](not torch.cuda.is_available(), reason="needs CUDA")+def test_triton_pre_only_matches_torch():+ """HAS_POST=False path (layer 0's standalone hc_pre through the fused kernel)."""+ from freetoken.kernel.triton.mhc import mhc_fused_post_pre_triton++ torch.manual_seed(12)+ t, hidden = 5, 128+ mix = 2 * N + N * N+ fn = torch.randn(mix, N * hidden, dtype=torch.float32, device="cuda") * 0.05+ scale = torch.rand(3, dtype=torch.float32, device="cuda") + 0.5+ base = torch.randn(mix, dtype=torch.float32, device="cuda") * 0.3+ res = torch.randn(t, N, hidden, dtype=torch.bfloat16, device="cuda")++ ref_post, ref_comb, ref_li = mhc_pre(+ res, fn, scale, base, RMS_EPS, EPS, POST_MULT, SINKHORN+ )+ got_res, got_post, got_comb, got_li = mhc_fused_post_pre_triton(+ res.new_empty(t, hidden), res, None, None, fn, scale, base,+ RMS_EPS, EPS, POST_MULT, SINKHORN,+ )+ assert torch.equal(got_res, res) # pass-through when no post+ assert (got_post.float() - ref_post.float()).abs().max().item() < 2e-3+ assert (got_comb.float() - ref_comb.float()).abs().max().item() < 2e-3+ assert (got_li.float() - ref_li.float()).abs().max().item() < 2e-2diff --git a/tests/models/test_glm5_next_config.py b/tests/models/test_glm5_next_config.pynew file mode 100644index 00000000..ae4daf5b--- /dev/null+++ b/tests/models/test_glm5_next_config.py@@ -0,0 +1,281 @@+"""glm5_next (GLM-5.3-Flash) config parsing: attention groups, kpool spec, aliases.++Runs off a trimmed copy of the real checkpoint config.json via RawConfigShim (the+exact object cached_load_hf_config falls back to when transformers doesn't know+``glm5_next`` yet), so the parse path under test is the one production hits.+"""++from __future__ import annotations++import pytest++from freetoken.attention.base import AttnType+from freetoken.models.config import (+ FullAttentionGroupConfig,+ LinearGatedDeltaGroupConfig,+)+from freetoken.models.glm5_next.args import load_args+from freetoken.models.glm5_next.config import parse_config+from freetoken.utils.hf import RawConfigShim++_NUM_LAYERS = 45+_DSA_IDS = tuple(range(3, _NUM_LAYERS, 4)) # 3, 7, ..., 43+_KDA_IDS = tuple(i for i in range(_NUM_LAYERS) if i not in _DSA_IDS)+++def _layer_types() -> list[str]:+ return [+ "deepseek_sparse_attention" if i in _DSA_IDS else "linear_attention"+ for i in range(_NUM_LAYERS)+ ]+++def _text_config() -> dict:+ # Trimmed from zai-org/GLM-5.3-Flash config.json (text_config).+ return {+ "hidden_size": 4096,+ "intermediate_size": 12288,+ "num_hidden_layers": _NUM_LAYERS,+ "num_attention_heads": 64,+ "num_key_value_heads": 64,+ "vocab_size": 154880,+ "hidden_act": "silu",+ "rms_norm_eps": 1e-5,+ "max_position_embeddings": 1048576,+ "tie_word_embeddings": False,+ # MLA (NoPE)+ "q_lora_rank": 1536,+ "kv_lora_rank": 512,+ "qk_nope_head_dim": 256,+ "qk_rope_head_dim": 0,+ "qk_head_dim": 256,+ "v_head_dim": 256,+ "mla_use_nope": True,+ # DSA indexer + kpool+ "index_n_heads": 32,+ "index_head_dim": 128,+ "index_topk": 2048,+ "indexer_types": ["full"] * _NUM_LAYERS,+ "indexer_rope_interleave": True,+ "index_kpool": 4,+ "index_kpool_compress": True,+ "index_kpool_always_select_tail": True,+ # KDA+ "linear_attn_config": {+ "num_heads": 64,+ "head_dim": 128,+ "short_conv_kernel_size": 4,+ "gate_lower_bound": -5.0,+ "kda_layers": list(_KDA_IDS),+ "full_attn_layers": list(_DSA_IDS),+ },+ # layout+ "layer_types": _layer_types(),+ "mlp_layer_types": ["dense"] * 3 + ["sparse"] * (_NUM_LAYERS - 3),+ "first_k_dense_replace": 3,+ # mHC (checkpoint spellings)+ "mhc": True,+ "hc_mult": 4,+ "hc_eps": 1e-6,+ "hc_sinkhorn_iters": 20,+ # MoE+ "n_routed_experts": 288,+ "num_experts_per_tok": 8,+ "n_shared_experts": 1,+ "moe_intermediate_size": 2048,+ "norm_topk_prob": True,+ "routed_scaling_factor": 2.5,+ "scoring_func": "sigmoid",+ "topk_method": "noaux_tc",+ "n_group": 1,+ "topk_group": 1,+ "swiglu_limit": 10.0,+ "num_nextn_predict_layers": 1,+ "attention_bias": False,+ "model_type": "glm5_next_text",+ }+++def _hf_config(quantization_config: dict | None = None) -> RawConfigShim:+ data: dict = {+ "architectures": ["Glm5NextForConditionalGeneration"],+ "model_type": "glm5_next",+ "text_config": _text_config(),+ "vision_config": {"model_type": "glm5_next_vision", "depth": 24},+ "image_token_id": 154854,+ }+ if quantization_config is not None:+ data["quantization_config"] = quantization_config+ return RawConfigShim(data)+++_CT_NVFP4_QUANT = {+ # From RedHatAI/GLM-5.3-Flash-NVFP4 (llm-compressor, experts-only calibrated NVFP4).+ "quant_method": "compressed-tensors",+ "format": "nvfp4-pack-quantized",+ "config_groups": {+ "group_0": {+ "targets": ["re:.*mlp\\.experts\\..*(gate|up|down)_proj$"],+ "weights": {"num_bits": 4, "type": "float", "group_size": 16},+ }+ },+}++_NVFP4_QUANT = {+ # From LibertAIDAI/GLM-5.3-Flash-NVFP4 (ModelOpt weight-only NVFP4).+ "quant_algo": "NVFP4",+ "quant_method": "modelopt",+ "config_groups": {+ "group_0": {+ "targets": ["Linear"],+ "weights": {"num_bits": 4, "type": "float", "group_size": 16},+ }+ },+ "ignore": ["lm_head", "model.visual.*"],+}+++def test_attention_groups():+ cfg = parse_config(_hf_config())++ assert cfg.num_layers == _NUM_LAYERS+ assert len(cfg.attention_groups) == 2+ linear, full = cfg.attention_groups # ordered by first layer id (0 < 3)++ assert isinstance(linear, LinearGatedDeltaGroupConfig)+ assert linear.variant == "kda"+ assert linear.layer_ids == _KDA_IDS+ assert (linear.num_key_heads, linear.key_head_dim) == (64, 128)+ assert (linear.num_value_heads, linear.value_head_dim) == (64, 128)+ assert linear.conv_kernel_dim == 4++ assert isinstance(full, FullAttentionGroupConfig)+ assert full.layer_ids == _DSA_IDS+ assert full.mla is True+ assert full.head_dim == 512 # bare ckv latent: kv_lora_rank + 0 rope dims+ assert full.num_kv_heads == 1+ assert full.index_head_dim == 128+ assert full.num_index_layers == len(_DSA_IDS)+ assert full.index_ratio == 4++ assert cfg.has_linear_attention and cfg.has_hybrid_attention+ assert cfg.attn_type_for_layer(0) == AttnType.LINEAR+ assert cfg.attn_type_for_layer(3) == AttnType.DSA+++def test_kv_cache_group_specs_skip_linear_and_carry_kpool():+ cfg = parse_config(_hf_config())+ specs = [s for s in cfg.kv_cache_group_specs() if s.num_layers > 0]+ # The linear group keeps recurrent state (no paged KV); exactly one paged spec.+ assert len(specs) == 1+ (spec,) = specs+ assert spec.attn_type == AttnType.DSA+ assert spec.layer_ids == _DSA_IDS+ assert (spec.mla, spec.head_dim, spec.index_head_dim) == (True, 512, 128)+ assert spec.index_ratio == 4+ assert spec.num_index_layers == len(_DSA_IDS)+++def test_moe_and_scalars():+ cfg = parse_config(_hf_config(_NVFP4_QUANT))+ assert cfg.expert_quant == "nvfp4"+ # Only DERIVED facts are pinned here; fixture echoes (num_experts == 288+ # and friends) assert nothing the parse could get wrong.+ assert cfg.attn_sm_scale == pytest.approx(256**-0.5)+ assert cfg.num_moe_layers == _NUM_LAYERS - 3+ assert cfg.is_moe+ # Checkpoint-faithful default; the FREETOKEN_GLM5_*_FP8 env flags opt into+ # the W8A16 fp8 load.+ assert (cfg.attn_quant, cfg.dense_quant, cfg.lm_head_quant) == ("none",) * 3+ # Text-only serving: the vision tower is never built.+ assert cfg.vision_config is None+++def test_args_alias_folding_and_nope():+ args = load_args(_hf_config())+ # Checkpoint spellings fold into the canonical fields.+ assert args.mla_nope is True # from mla_use_nope+ assert args.mhc_num_residual_streams == 4 # from hc_mult+ assert args.mhc_sinkhorn_iterations == 20 # from hc_sinkhorn_iters+ assert args.linear_num_heads == 64 # from nested linear_attn_config+ assert args.linear_lower_bound == -5.0+ # NoPE geometry.+ assert args.qk_rope_head_dim == 0+ assert args.qk_head_dim == 256+ assert args.latent_dim == 512+ assert args.kda_layer_ids == _KDA_IDS+ assert args.dsa_layer_ids == _DSA_IDS+ # Defaults for fields the checkpoint doesn't ship.+ assert args.rope_theta == 10000.0+ assert args.mhc_tau == 0.05+ assert args.mhc_post_mult_value == 2.0+++def test_registry_resolves_glm5_next():+ from freetoken.models.register import get_model_spec++ spec = get_model_spec("Glm5NextForConditionalGeneration")+ assert spec.module == "freetoken.models.glm5_next"+ assert spec.model_cls == "Glm5NextForCausalLM"+ assert get_model_spec("Glm5NextForCausalLM").module == spec.module+++def test_dev_layer_cap(monkeypatch):+ monkeypatch.setenv("FREETOKEN_GLM5_MAX_LAYERS", "5")+ cfg = parse_config(_hf_config())+ assert cfg.num_layers == 5+ linear, full = cfg.attention_groups+ assert linear.layer_ids == (0, 1, 2, 4)+ assert full.layer_ids == (3,)+ assert full.num_index_layers == 1+ # 3 dense + 2 sparse under the cap.+ assert cfg.first_k_dense_replace == 3+++def test_rejects_unknown_layer_types():+ data = _hf_config().to_dict()+ data["text_config"]["layer_types"][0] = "full_attention"+ with pytest.raises(ValueError, match="unsupported layer_types"):+ load_args(RawConfigShim(data))+++def test_compressed_tensors_nvfp4_detected():+ """RedHatAI/GLM-5.3-Flash-NVFP4 (llm-compressor): expert_quant resolves to+ nvfp4 off the ``format`` field (quant_algo is absent for compressed-tensors)."""+ cfg = parse_config(_hf_config(_CT_NVFP4_QUANT))+ assert cfg.expert_quant == "nvfp4"+++def test_expert_source_spec_selection():+ """quant_method picks the bank source spec: compressed-tensors maps+ weight_packed/weight_global_scale onto the canonical kinds with a reciprocal+ global; modelopt stays identity."""+ from freetoken.models.glm5_next.weight import (+ _NVFP4_CT_SOURCE_SPEC,+ _NVFP4_SOURCE_SPEC,+ )++ ct = _NVFP4_CT_SOURCE_SPEC+ m = ct.key_pattern.match(+ "model.language_model.layers.5.mlp.experts.7.gate_proj.weight_packed"+ )+ assert m and m.group("kind") == "weight_packed"+ assert ct.kind_map["weight_packed"] == "weight"+ assert ct.kind_map["weight_global_scale"] == "weight_scale_2"+ assert ct.global_reciprocal+ # W4A16 serving never consumes the calibrated activation scale.+ assert ct.key_pattern.match(+ "model.language_model.layers.5.mlp.experts.7.gate_proj.input_global_scale"+ ) is None+ assert _NVFP4_SOURCE_SPEC.kind_map is None+ assert not _NVFP4_SOURCE_SPEC.global_reciprocal+++def test_ingest_global_reciprocal():+ import torch+ from freetoken.models.glm5_next.weight import _NVFP4_CT_SOURCE_SPEC, _NVFP4_SOURCE_SPEC+ from freetoken.models.nvfp4_banks import _ingest_global++ g = torch.tensor(4.0)+ assert _ingest_global(_NVFP4_CT_SOURCE_SPEC, g).item() == 0.25+ assert _ingest_global(_NVFP4_SOURCE_SPEC, g).item() == 4.0diff --git a/tests/models/test_glm5_next_kda_op.py b/tests/models/test_glm5_next_kda_op.pynew file mode 100644index 00000000..11adbf11--- /dev/null+++ b/tests/models/test_glm5_next_kda_op.py@@ -0,0 +1,258 @@+"""Glm5NextKDA op vs an eager reference (projection/conv/gate/norm wiring).++The kernel math itself is validated in tests/kernels/test_kda.py; this test checks+the OP-level wiring: the fused in_proj split (q|k|v|b|f_a|g_a), the merged q|k|v+depthwise causal conv (+silu) against the state pool, the low-rank f/g gates, the+sigmoid-gated output RMSNorm, and prefill -> decode state continuity through+``LinearStatePool``.+"""++from __future__ import annotations++from types import SimpleNamespace++import pytest+import torch++pytestmark = pytest.mark.skipif(not torch.cuda.is_available(), reason="needs CUDA")++[email protected](autouse=True)+def _single_rank_tp():+ from freetoken.distributed import set_tp_info, try_get_tp_info++ if try_get_tp_info() is None:+ set_tp_info(rank=0, size=1)++HIDDEN, H, D, KERNEL = 256, 4, 128, 4+P = H * D+LOWER_BOUND = -5.0+++def _make_args():+ from freetoken.models.glm5_next.args import Glm5NextArgs++ n_layers = 2+ return Glm5NextArgs(+ hidden_size=HIDDEN, num_heads=8,+ q_lora_rank=64, kv_lora_rank=32, qk_nope_head_dim=32, qk_rope_head_dim=0,+ v_head_dim=32, mla_nope=True, norm_eps=1e-5, max_position=4096,+ index_n_heads=0, index_head_dim=0, index_topk=0, indexer_types=(),+ indexer_rope_interleave=True, index_kpool=1, index_kpool_compress=False,+ index_kpool_always_select_tail=False,+ linear_num_heads=H, linear_head_dim=D, linear_conv_kernel_dim=KERNEL,+ linear_lower_bound=LOWER_BOUND,+ layer_types=("linear_attention",) * n_layers,+ mlp_layer_types=("dense",) * n_layers,+ mhc=False, mhc_num_residual_streams=1, hc_eps=1e-6,+ mhc_sinkhorn_iterations=0, mhc_tau=0.05, mhc_post_mult_value=2.0,+ mhc_no_norm_weight=False, swiglu_limit=None, rope_theta=10000.0,+ )+++def _make_op(seed=0):+ from freetoken.models.glm5_next.kda import Glm5NextKDA++ cfg = SimpleNamespace(glm5_args=_make_args(), attn_quant="none")+ op = Glm5NextKDA(cfg, layer_id=0)+ torch.manual_seed(seed)+ dev, dt = "cuda", torch.bfloat16+ op.in_proj.weight = torch.randn(3 * P + H + 2 * D, HIDDEN, device=dev, dtype=dt) * 0.05+ op.f_b_proj.weight = torch.randn(P, D, device=dev, dtype=dt) * 0.05+ op.g_b_proj.weight = torch.randn(P, D, device=dev, dtype=dt) * 0.05+ op.conv1d.weight = torch.randn(3 * P, 1, KERNEL, device=dev, dtype=dt) * 0.2+ op.A_log = torch.randn(H, device=dev, dtype=torch.float32) * 0.5+ op.dt_bias = torch.randn(P, device=dev, dtype=torch.float32) * 0.5+ op.o_norm.weight = torch.randn(D, device=dev, dtype=dt) * 0.1 + 1.0+ op.o_proj.weight = torch.randn(HIDDEN, P, device=dev, dtype=dt) * 0.05+ return op+++def _make_pool(num_slots=4):+ from freetoken.kvcache.linear_state_pool import LinearStatePool+ from freetoken.models.config import LinearGatedDeltaGroupConfig++ group = LinearGatedDeltaGroupConfig(+ name="linear", layer_ids=(0,),+ num_key_heads=H, num_value_heads=H, key_head_dim=D, value_head_dim=D,+ conv_kernel_dim=KERNEL, output_gate=True, variant="kda",+ )+ return LinearStatePool(+ group, num_slots, dtype=torch.bfloat16,+ device=torch.device("cuda"), tp_size=1,+ )+++def _patch_ctx(monkeypatch, pool, batch):+ ctx = SimpleNamespace(batch=batch, linear_state_pool=pool)+ monkeypatch.setattr(+ "freetoken.models.glm5_next.kda.get_global_ctx", lambda: ctx+ )+++def _l2norm(x):+ return x / torch.sqrt((x * x).sum(-1, keepdim=True) + 1e-6)+++def _reference_forward(op, x_seq, conv_ctx=None, h0=None):+ """Eager op reference for one sequence [T, HIDDEN] (fp32 where the kernels are+ fp32). Returns (out [T, HIDDEN], conv_tail [3P, KERNEL-1], state [H, D, D])."""+ T = x_seq.shape[0]+ proj = x_seq.to(torch.bfloat16) @ op.in_proj.weight.T+ conv_in, b, f_a, g_a = torch.split(proj, [3 * P, H, D, D], dim=-1)+ g1 = (f_a @ op.f_b_proj.weight.T).float()+ g2 = g_a @ op.g_b_proj.weight.T++ # depthwise causal conv + silu over the merged q|k|v stream, with optional+ # left-context from a previous chunk (conv state semantics).+ w = op.conv1d.weight.squeeze(1).float() # [3P, KERNEL]+ stream = conv_in.T.float() # [3P, T]+ left = (+ conv_ctx.float()+ if conv_ctx is not None+ else torch.zeros(3 * P, KERNEL - 1, device=x_seq.device)+ )+ padded = torch.cat([left, stream], dim=1) # [3P, KERNEL-1+T]+ conv = torch.stack(+ [(padded[:, t : t + KERNEL] * w).sum(-1) for t in range(T)], dim=1+ )+ mixed = torch.nn.functional.silu(conv).T # [T, 3P]+ conv_tail = padded[:, -(KERNEL - 1):]++ q, k, v = (t.reshape(T, H, D) for t in torch.split(mixed, [P, P, P], dim=-1))+ # bf16 round-trip like the op (kernel inputs are bf16)+ q, k, v = q.to(torch.bfloat16).float(), k.to(torch.bfloat16).float(), v.to(torch.bfloat16).float()++ h = h0.clone() if h0 is not None else torch.zeros(H, D, D, device=x_seq.device)+ amp = op.A_log.exp().view(H, 1)+ bias = op.dt_bias.view(H, D)+ core = []+ for t in range(T):+ gk = LOWER_BOUND * torch.sigmoid(amp * (g1[t].view(H, D) + bias))+ h = h * gk.exp().unsqueeze(1)+ kt = _l2norm(k[t])+ v_err = (v[t] - torch.einsum("hvk,hk->hv", h, kt)) * torch.sigmoid(+ b[t].float()+ ).unsqueeze(-1)+ h = h + torch.einsum("hv,hk->hvk", v_err, kt)+ core.append(torch.einsum("hvk,hk->hv", h, _l2norm(q[t]) * D**-0.5))+ core = torch.stack(core) # [T, H, D]++ xn = core.reshape(-1, D)+ rms = xn * torch.rsqrt(xn.pow(2).mean(-1, keepdim=True) + op.o_norm.eps)+ gated = rms * op.o_norm.weight.float() * torch.sigmoid(g2.reshape(-1, D).float())+ out = gated.reshape(T, P).to(torch.bfloat16) @ op.o_proj.weight.T+ return out.float(), conv_tail, h+++def _fla(cu, indices, has_init=None, fresh=None):+ from freetoken.attention.linear import FLAMetadata++ return FLAMetadata(+ cu_seqlens=cu, cache_indices=indices,+ has_initial_state=has_init, fresh_state_indices=fresh,+ )+++def _assert_close(ours, ref, tag, atol=3e-2):+ err = (ours.float() - ref.float()).abs().max().item()+ scale = ref.float().abs().max().item() + 1e-8+ assert err / scale < atol, f"{tag}: max abs err {err:.5f} (ref scale {scale:.3f})"+++def test_prefill_matches_reference(monkeypatch):+ op = _make_op()+ pool = _make_pool()+ lens = [33, 70]+ total = sum(lens)+ torch.manual_seed(10)+ x = torch.randn(total, HIDDEN, device="cuda", dtype=torch.bfloat16)++ cu = torch.tensor([0, *torch.tensor(lens).cumsum(0).tolist()], dtype=torch.int32, device="cuda")+ indices = torch.tensor([1, 2], dtype=torch.int32, device="cuda")+ has_init = torch.tensor([False, False], device="cuda")+ fresh = torch.tensor([1, 2], dtype=torch.int64, device="cuda")+ batch = SimpleNamespace(is_decode=False, fla_metadata=_fla(cu, indices, has_init, fresh))+ _patch_ctx(monkeypatch, pool, batch)++ out = op.forward(x)++ start = 0+ for i, ln in enumerate(lens):+ sl = slice(start, start + ln)+ ref_out, ref_conv, ref_h = _reference_forward(op, x[sl].float())+ _assert_close(out[sl], ref_out, f"seq{i} prefill out")+ _assert_close(pool.recurrent_states[0, i + 1], ref_h, f"seq{i} state")+ _assert_close(pool.conv_states[0, i + 1], ref_conv, f"seq{i} conv state")+ start += ln+++def test_prefill_then_decode_continuity(monkeypatch):+ op = _make_op(seed=1)+ pool = _make_pool()+ T0, T1 = 40, 3+ torch.manual_seed(11)+ x = torch.randn(T0 + T1, HIDDEN, device="cuda", dtype=torch.bfloat16)+ ref_out, _, _ = _reference_forward(op, x.float())++ cu = torch.tensor([0, T0], dtype=torch.int32, device="cuda")+ indices = torch.tensor([1], dtype=torch.int32, device="cuda")+ batch = SimpleNamespace(+ is_decode=False,+ fla_metadata=_fla(+ cu, indices,+ torch.tensor([False], device="cuda"),+ torch.tensor([1], dtype=torch.int64, device="cuda"),+ ),+ )+ _patch_ctx(monkeypatch, pool, batch)+ out0 = op.forward(x[:T0])+ _assert_close(out0, ref_out[:T0], "prefill out")++ for t in range(T0, T0 + T1):+ batch = SimpleNamespace(+ is_decode=True,+ fla_metadata=_fla(+ torch.tensor([0, 1], dtype=torch.int32, device="cuda"), indices+ ),+ )+ _patch_ctx(monkeypatch, pool, batch)+ out_t = op.forward(x[t : t + 1])+ _assert_close(out_t[0], ref_out[t], f"decode token {t}")+++def test_chunked_prefill_continuation(monkeypatch):+ """Second prefill chunk with has_initial_state=True must continue conv AND+ recurrent state exactly (the chunked-prefill path)."""+ op = _make_op(seed=2)+ pool = _make_pool()+ T0, T1 = 64, 30+ torch.manual_seed(12)+ x = torch.randn(T0 + T1, HIDDEN, device="cuda", dtype=torch.bfloat16)+ ref_out, ref_conv, ref_h = _reference_forward(op, x.float())++ indices = torch.tensor([1], dtype=torch.int32, device="cuda")+ batch = SimpleNamespace(+ is_decode=False,+ fla_metadata=_fla(+ torch.tensor([0, T0], dtype=torch.int32, device="cuda"), indices,+ torch.tensor([False], device="cuda"),+ torch.tensor([1], dtype=torch.int64, device="cuda"),+ ),+ )+ _patch_ctx(monkeypatch, pool, batch)+ out0 = op.forward(x[:T0])+ _assert_close(out0, ref_out[:T0], "chunk0 out")++ batch = SimpleNamespace(+ is_decode=False,+ fla_metadata=_fla(+ torch.tensor([0, T1], dtype=torch.int32, device="cuda"), indices,+ torch.tensor([True], device="cuda"), None,+ ),+ )+ _patch_ctx(monkeypatch, pool, batch)+ out1 = op.forward(x[T0:])+ _assert_close(out1, ref_out[T0:], "chunk1 out")+ _assert_close(pool.recurrent_states[0, 1], ref_h, "final state")+ _assert_close(pool.conv_states[0, 1], ref_conv, "final conv state")diff --git a/tests/models/test_glm5_next_kda_snapshot.py b/tests/models/test_glm5_next_kda_snapshot.pynew file mode 100644index 00000000..e040b925--- /dev/null+++ b/tests/models/test_glm5_next_kda_snapshot.py@@ -0,0 +1,98 @@+"""KDA hybrid-radix track-snapshot contract (prefix caching for glm5_next).++The scheduler snapshots each request's linear state at the deepest chunk-aligned+(x64) boundary of a prefill into a donatable pool slot (FLAMetadata.track_*; the+op writes it from the chunk kernel's per-chunk h + the raw conv window). A later+request restores by copying that slot and continuing with+``has_initial_state=True``. Checks, at the KDA-op level:++* the snapshot equals the TRUE state after exactly 64 tokens (independent run)+* restore + continuation reproduces the uninterrupted run's outputs+* the 64-boundary is kpool-aligned by construction (64 % 4 == 0)+"""++from __future__ import annotations++from types import SimpleNamespace++import pytest+import torch++from tests.models.test_glm5_next_kda_op import ( # reuse the op harness+ _make_op,+ _make_pool,+ _patch_ctx,+ _fla,+ _assert_close,+ HIDDEN,+)++pytestmark = pytest.mark.skipif(not torch.cuda.is_available(), reason="needs CUDA")++CHUNK = 64++[email protected](autouse=True)+def _single_rank_tp():+ from freetoken.distributed import set_tp_info, try_get_tp_info++ if try_get_tp_info() is None:+ set_tp_info(rank=0, size=1)+++def _prefill(op, pool, monkeypatch, x, slot, t0=0, has_init=False, track=None):+ t = x.shape[0]+ fla = _fla(+ torch.tensor([0, t], dtype=torch.int32, device="cuda"),+ torch.tensor([slot], dtype=torch.int32, device="cuda"),+ torch.tensor([has_init], device="cuda"),+ None if has_init else torch.tensor([slot], dtype=torch.int64, device="cuda"),+ )+ if track is not None:+ fla.track_dst, fla.track_h_row, fla.track_conv_src = track+ batch = SimpleNamespace(is_decode=False, fla_metadata=fla)+ _patch_ctx(monkeypatch, pool, batch)+ return op.forward(x)+++def test_snapshot_restore_roundtrip(monkeypatch):+ from freetoken.kernel.fla.index import prepare_chunk_offsets++ op = _make_op(seed=5)+ pool = _make_pool(num_slots=6)+ total = 100 # crosses one x64 boundary; tail 36 tokens+ torch.manual_seed(20)+ x = torch.randn(total, HIDDEN, device="cuda", dtype=torch.bfloat16)++ # --- ground truth: state after exactly CHUNK tokens (independent run, slot 3)+ _prefill(op, pool, monkeypatch, x[:CHUNK], slot=3)+ true_rec = pool.recurrent_states[0, 3].clone()+ true_conv = pool.conv_states[0, 3].clone()++ # --- tracked run (slot 1, snapshot into slot 2), as _build_track_metadata would+ km1 = pool.conv_states.shape[-1]+ cu_host = torch.tensor([0, total], dtype=torch.int64)+ boh = prepare_chunk_offsets(cu_host, CHUNK).tolist()+ c = (total - 1) // CHUNK # deepest mid-chunk boundary: 1 -> position 64+ track = (+ torch.tensor([2], dtype=torch.int64, device="cuda"),+ torch.tensor([boh[0] + c], dtype=torch.int64, device="cuda"),+ torch.tensor([[c * CHUNK - km1 + j for j in range(km1)]], dtype=torch.int64, device="cuda"),+ )+ out_full = _prefill(op, pool, monkeypatch, x, slot=1, track=track)++ _assert_close(pool.recurrent_states[0, 2], true_rec, "snapshot recurrent state")+ _assert_close(pool.conv_states[0, 2], true_conv, "snapshot conv state")++ # --- restore: copy snapshot -> fresh slot 4, continue [64, 100)+ pool.copy_from(2, 4)+ out_cont = _prefill(+ op, pool, monkeypatch, x[CHUNK:], slot=4, t0=CHUNK, has_init=True+ )+ _assert_close(out_cont, out_full[CHUNK:], "restored continuation outputs")+ _assert_close(+ pool.recurrent_states[0, 4], pool.recurrent_states[0, 1], "final states agree"+ )++ # kpool alignment is subsumed by the x64 snapshot boundary.+ assert CHUNK % 4 == 0\ No newline at end of filediff --git a/tests/models/test_glm5_next_model.py b/tests/models/test_glm5_next_model.pynew file mode 100644index 00000000..9a499fba--- /dev/null+++ b/tests/models/test_glm5_next_model.py@@ -0,0 +1,210 @@+"""Glm5NextForCausalLM wiring smoke test (tiny random model, dense MLPs).++The per-op math is covered elsewhere (KDA kernels/op, kpool backend, mHC); this+test checks the ASSEMBLY: a 2-layer hybrid (KDA + DSA) model with mHC threading+runs prefill and decode through the real backends/pools, and the strongest+cache invariant holds -- decoding token T after prefilling [0, T) produces the+same logits as prefilling [0, T] outright (state handoff across the KDA+recurrent pool, the MLA latent pool, and the kpool indexer cache).+"""++from __future__ import annotations++from types import SimpleNamespace++import pytest+import torch++pytestmark = pytest.mark.skipif(not torch.cuda.is_available(), reason="needs CUDA")++HIDDEN, VOCAB = 64, 128+KDA_H, KDA_D = 2, 128 # KDA kernels specialize on D=128+IDX_H, IDX_D = 16, 64+LATENT = 32+DEV = "cuda"+++def _hf_config():+ from freetoken.utils.hf import RawConfigShim++ text = {+ "hidden_size": HIDDEN, "intermediate_size": 96, "num_hidden_layers": 2,+ "num_attention_heads": 2, "vocab_size": VOCAB, "hidden_act": "silu",+ "rms_norm_eps": 1e-5, "max_position_embeddings": 4096,+ "tie_word_embeddings": False,+ "q_lora_rank": 48, "kv_lora_rank": LATENT, "qk_nope_head_dim": 32,+ "qk_rope_head_dim": 0, "v_head_dim": 32, "mla_use_nope": True,+ "index_n_heads": IDX_H, "index_head_dim": IDX_D, "index_topk": 32,+ "indexer_types": ["full", "full"], "indexer_rope_interleave": True,+ "index_kpool": 4, "index_kpool_compress": True,+ "index_kpool_always_select_tail": True,+ "linear_attn_config": {+ "num_heads": KDA_H, "head_dim": KDA_D,+ "short_conv_kernel_size": 4, "gate_lower_bound": -5.0,+ },+ "layer_types": ["linear_attention", "deepseek_sparse_attention"],+ "mlp_layer_types": ["dense", "dense"], # no MoE machinery in this test+ "first_k_dense_replace": 2,+ "mhc": True, "hc_mult": 4, "hc_eps": 1e-6, "hc_sinkhorn_iters": 20,+ "n_routed_experts": 8, "num_experts_per_tok": 2, "n_shared_experts": 1,+ "moe_intermediate_size": 32, "norm_topk_prob": True,+ "routed_scaling_factor": 2.5, "scoring_func": "sigmoid",+ "n_group": 1, "topk_group": 1, "swiglu_limit": 10.0,+ "attention_bias": False, "model_type": "glm5_next_text",+ }+ return RawConfigShim({+ "architectures": ["Glm5NextForConditionalGeneration"],+ "model_type": "glm5_next", "text_config": text,+ })+++def rig(monkeypatch):+ from freetoken.attention.dsa_indexer_kpool import Glm5NextDSABackend+ from freetoken.distributed import set_tp_info, try_get_tp_info+ from freetoken.kvcache.dsa_pool import KpoolDSAKVCache+ from freetoken.kvcache.linear_state_pool import LinearStatePool+ from freetoken.models.glm5_next.config import parse_config+ from freetoken.models.glm5_next.model import Glm5NextForCausalLM++ if try_get_tp_info() is None:+ set_tp_info(rank=0, size=1)+ config = parse_config(_hf_config())++ prev_dtype = torch.get_default_dtype()+ torch.set_default_dtype(torch.bfloat16)+ prev_dev = torch.get_default_device()+ torch.set_default_device(DEV)+ try:+ model = Glm5NextForCausalLM(config)+ finally:+ torch.set_default_dtype(prev_dtype)+ torch.set_default_device(prev_dev)++ # Random weights via the state-dict round trip (keeps shapes/dtypes honest).+ torch.manual_seed(0)+ sd = model.state_dict()+ rand = {}+ for k, v in sd.items():+ t = torch.randn(v.shape, dtype=torch.float32, device=DEV) * 0.05+ if k.endswith("norm.weight") or ".o_norm.weight" in k:+ t = t.abs() + 0.5+ rand[k] = t.to(v.dtype)+ model.load_state_dict(rand)++ kv = KpoolDSAKVCache(+ latent_dim=LATENT, num_layers=2, num_pages=4, page_size=64,+ dtype=torch.bfloat16, device=torch.device(DEV),+ index_head_dim=IDX_D, num_index_layers=1,+ index_ratio=4, num_req_slots=4,+ )+ page_table = torch.full((2, 256), -1, dtype=torch.int32, device=DEV)+ page_table[0] = torch.arange(256, dtype=torch.int32, device=DEV)+ linear_pool = LinearStatePool(+ config.linear_attention_group(), num_slots=4,+ dtype=torch.bfloat16, device=torch.device(DEV), tp_size=1,+ )++ ctx = SimpleNamespace(+ kv_cache=kv, page_table=page_table, linear_state_pool=linear_pool,+ attn_backend=None, batch=None,+ )+ for mod in (+ "freetoken.attention.dsa.get_global_ctx",+ "freetoken.models.glm5_next.kda.get_global_ctx",+ "freetoken.models.glm5_next.attention.get_global_ctx",+ "freetoken.models.glm5_next.model.get_global_ctx",+ "freetoken.layers.embedding.get_global_ctx",+ ):+ monkeypatch.setattr(mod, lambda: ctx)+ ctx.attn_backend = Glm5NextDSABackend(config)+ return model, ctx+++def _req(device_len, cached_len):+ return SimpleNamespace(+ table_idx=0, device_len=device_len, extend_len=device_len - cached_len,+ cached_len=cached_len, linear_slot_idx=1, mamba_ping_pong=None,+ )+++def _batch(ctx, ids, t0, phase):+ from freetoken.attention.linear import FLAMetadata++ t1 = t0 + len(ids)+ is_decode = phase == "decode"+ batch = SimpleNamespace(+ phase=phase,+ is_prefill=not is_decode, is_decode=is_decode, size=1,+ reqs=[_req(t1, t0)], padded_reqs=[_req(t1, t0)],+ input_ids=torch.tensor(ids, device=DEV),+ positions=torch.arange(t0, t1, device=DEV),+ out_loc=torch.arange(t0, t1, device=DEV),+ active_table_idx=torch.tensor([0], device=DEV) if is_decode else None,+ fla_metadata=FLAMetadata(+ cu_seqlens=torch.tensor([0, len(ids)], dtype=torch.int32, device=DEV),+ cache_indices=torch.tensor([1], dtype=torch.int32, device=DEV),+ has_initial_state=None if is_decode else torch.tensor([t0 > 0], device=DEV),+ fresh_state_indices=(+ None if (is_decode or t0 > 0)+ else torch.tensor([1], dtype=torch.int64, device=DEV)+ ),+ ),+ mm_embeds=None,+ )+ ctx.batch = batch+ ctx.attn_backend.prepare_metadata(batch)+ return batch+++def _reset(ctx):+ ctx.linear_state_pool.reset(1)+ ctx.kv_cache._kv_buffer.zero_()+ ctx.kv_cache._index_k_buffer.zero_()+ ctx.kv_cache._tail_k.zero_()+ ctx.kv_cache._tail_gate.zero_()+++def test_prefill_decode_consistency(rig):+ model, ctx = rig+ torch.manual_seed(1)+ total = 24+ ids = torch.randint(0, VOCAB, (total,)).tolist()++ # One-shot prefill over the full sequence: last-token logits per position+ # are only produced for the final token, so run it twice at different splits.+ _reset(ctx)+ _batch(ctx, ids, 0, "prefill")+ full_logits = model.forward() # [1, VOCAB] logits of the last position+ assert full_logits.shape == (1, VOCAB)+ assert torch.isfinite(full_logits.float()).all()++ # Prefill [0, total-1) then decode the last token: must match the one-shot run.+ _reset(ctx)+ _batch(ctx, ids[:-1], 0, "prefill")+ model.forward()+ _batch(ctx, ids[-1:], total - 1, "decode")+ dec_logits = model.forward()+ err = (dec_logits.float() - full_logits.float()).abs().max().item()+ scale = full_logits.float().abs().max().item() + 1e-8+ assert err / scale < 3e-2, f"decode/prefill divergence: {err} (scale {scale})"+++def test_chunked_prefill_consistency(rig):+ model, ctx = rig+ torch.manual_seed(2)+ total = 28 # split 16 + 12; chunk boundary pool-aligned (16 % 4 == 0)+ ids = torch.randint(0, VOCAB, (total,)).tolist()++ _reset(ctx)+ _batch(ctx, ids, 0, "prefill")+ full_logits = model.forward()++ _reset(ctx)+ _batch(ctx, ids[:16], 0, "prefill")+ model.forward()+ _batch(ctx, ids[16:], 16, "prefill")+ chunk_logits = model.forward()+ err = (chunk_logits.float() - full_logits.float()).abs().max().item()+ scale = full_logits.float().abs().max().item() + 1e-8+ assert err / scale < 3e-2, f"chunked/one-shot divergence: {err} (scale {scale})"diff --git a/tests/models/test_glm_dsa.py b/tests/models/test_glm_dsa.pyindex 84d6c1b3..0dceabad 100644--- a/tests/models/test_glm_dsa.py+++ b/tests/models/test_glm_dsa.py@@ -298,11 +298,15 @@ def test_backend_ragged_prefill_identity_and_selection():q_pe = torch.randn(t, h, dr, device="cuda", dtype=torch.bfloat16)c_kv = torch.randn(t, dv, device="cuda", dtype=torch.bfloat16)k_rope = torch.randn(t, dr, device="cuda", dtype=torch.bfloat16)- qkw = (torch.randn(t, idx_h, idx_d, device="cuda", dtype=torch.bfloat16),- torch.randn(t, idx_d, device="cuda", dtype=torch.bfloat16),- torch.randn(t, idx_h, device="cuda").abs())+ from freetoken.attention.dsa import DSAIndexerInputs- o0 = backend.mla_forward(q_nope, q_pe, c_kv, k_rope, 0, batch, indexer_qkw=qkw)+ qkw = DSAIndexerInputs(+ q=torch.randn(t, idx_h, idx_d, device="cuda", dtype=torch.bfloat16),+ k=torch.randn(t, idx_d, device="cuda", dtype=torch.bfloat16),+ w=torch.randn(t, idx_h, device="cuda").abs(),+ )++ o0 = backend.mla_forward(q_nope, q_pe, c_kv, k_rope, 0, batch, indexer_inputs=qkw)# request A (kv <= topk): selection covers all live -> equals dense referenceq_cat = torch.cat([q_nope, q_pe], -1)@@ -313,7 +317,7 @@ def test_backend_ragged_prefill_identity_and_selection():assert (o0[j].float() - ref).abs().max().item() < 3e-2, f"A q{j}"# request B (kv > topk): causal top-k reference from the same scoring math- q_idx, k_idx, w = qkw+ q_idx, k_idx, w = qkw.q, qkw.k, qkw.wfor j in (0, 11): # first and last of B's queriesrow = 8 + jpos = 88 + j@@ -327,7 +331,7 @@ def test_backend_ragged_prefill_identity_and_selection():# leader/follower: layer 1 (shared, no indexer) reuses layer 0's selection; with# identical latent content its output must match layer 0's- o1 = backend.mla_forward(q_nope, q_pe, c_kv, k_rope, 1, batch, indexer_qkw=None)+ o1 = backend.mla_forward(q_nope, q_pe, c_kv, k_rope, 1, batch, indexer_inputs=None)assert (o0.float() - o1.float()).abs().max().item() < 3e-2# identity wiring (dense ablation): same batch through an MLAKVCache backend@@ -338,7 +342,7 @@ def test_backend_ragged_prefill_identity_and_selection():batch_d = SimpleNamespace(reqs=reqs, positions=positions, out_loc=out_loc,active_table_idx=None, attn_metadata=None)backend_d.prepare_metadata(batch_d)- od = backend_d.mla_forward(q_nope, q_pe, c_kv, k_rope, 0, batch_d, indexer_qkw=None)+ od = backend_d.mla_forward(q_nope, q_pe, c_kv, k_rope, 0, batch_d, indexer_inputs=None)slab_d = ctx_d.kv_cache.latent_rows(0)for j in range(8):live = ctx_d.page_table[0, : 33 + j]