FreeToken brings datacenter-scale model serving to your desktop. Run massive models locally, fast and efficiently.
perf(attention): drop redundant activation copies in glm5 dsa prefill (#795)
Mukkesh Ganesh0c50eb9d829e
diff --git a/python/freetoken/attention/dsa.py b/python/freetoken/attention/dsa.pyindex 398d1700..e8c7b3c2 100644--- a/python/freetoken/attention/dsa.py+++ b/python/freetoken/attention/dsa.py@@ -193,14 +193,17 @@ class DSAAttnBackend(DSAIndexerMixin, BaseAttnBackend):# ----- attention ------------------------------------------------------------------def _attend(- self, q_cat: torch.Tensor, layer_id: int, sel: torch.Tensor, cnt: torch.Tensor+ self, q_cat: torch.Tensor, layer_id: int, sel: torch.Tensor, cnt: torch.Tensor,+ out: torch.Tensor | None = None,) -> torch.Tensor:- """Gathered-KV MLA over latent rows: q [b, m, H, 576] -> [b, m, H, 512]."""+ """Gathered-KV MLA over latent rows: q [b, m, H, 576] -> [b, m, H, 512].+ ``q_cat`` keeps its strides (a transposed bmm output needs no copy) and+ ``out`` receives the result in place."""from freetoken.kernel.triton.glm_dsa_sparse import glm_dsa_sparse_attnreturn glm_dsa_sparse_attn(q_cat, self.kvcache.latent_rows(layer_id), sel, self.sm_scale,- counts=cnt, d_v=self.kv_lora_rank,+ counts=cnt, d_v=self.kv_lora_rank, out=out,)def mla_forward(@@ -253,7 +256,10 @@ class DSAAttnBackend(DSAIndexerMixin, BaseAttnBackend):md.sel.clear()md.sel[layer_id] = (sel, cnt)sel, cnt = md.sel[self._leader[layer_id]]- q_cat = torch.cat([q_nope, q_pe], dim=-1).view(bs, 1, self.num_heads, self.latent_dim)+ if q_pe.shape[-1] == 0: # NoPE: the cat would be a full-width copy+ q_cat = q_nope.unsqueeze(1)+ else:+ q_cat = torch.cat([q_nope, q_pe], dim=-1).view(bs, 1, self.num_heads, self.latent_dim)o = self._attend(q_cat, layer_id, sel, cnt)return o.view(bs, self.num_heads, self.kv_lora_rank)@@ -287,7 +293,12 @@ class DSAAttnBackend(DSAIndexerMixin, BaseAttnBackend):def _prefill(self, md, layer_id, q_nope, q_pe, batch, inputs) -> torch.Tensor:t = q_nope.shape[0]- q_cat = torch.cat([q_nope, q_pe], dim=-1) # [T, H, 576]+ # NoPE (zero rope width): the cat would copy the whole [T, H, lora] q for+ # nothing -- pass it through and let the kernel read its strides.+ if q_pe.shape[-1] == 0:+ q_cat = q_nope+ else:+ q_cat = torch.cat([q_nope, q_pe], dim=-1) # [T, H, 576]reqs = batch.padded_reqs if hasattr(batch, "padded_reqs") else batch.reqspage_table = get_global_ctx().page_tableqo = md.qo_indptr_cpu.tolist()@@ -320,10 +331,13 @@ class DSAAttnBackend(DSAIndexerMixin, BaseAttnBackend):# through per-query counts.sel = page_table[r.table_idx, : r.device_len].view(1, 1, -1).to(torch.int32)cnt = (batch.positions[qo[i] : qo[i + 1]] + 1).to(torch.int32).view(1, m)- o[qo[i] : qo[i + 1]] = self._attend(+ # The kernel writes straight into the output slice (out=): no per-request+ # attention output next to o at the peak.+ self._attend(q_cat[qo[i] : qo[i + 1]].view(1, m, self.num_heads, self.latent_dim),layer_id, sel, cnt,- ).view(m, self.num_heads, self.kv_lora_rank)+ out=o[qo[i] : qo[i + 1]].view(1, m, self.num_heads, self.kv_lora_rank),+ )return o# ----- CUDA graph (decode) ----------------------------------------------------------diff --git a/python/freetoken/attention/dsa_indexer_kpool.py b/python/freetoken/attention/dsa_indexer_kpool.pyindex 6743a143..2b2492aa 100644--- a/python/freetoken/attention/dsa_indexer_kpool.py+++ b/python/freetoken/attention/dsa_indexer_kpool.py@@ -243,7 +243,10 @@ class Glm5NextDSABackend(DSAAttnBackend):md.sel.clear()md.sel[layer_id] = (sel, cnt)sel, cnt = md.sel[self._leader[layer_id]]- q_cat = torch.cat([q_nope, q_pe], dim=-1).view(bs, 1, self.num_heads, self.latent_dim)+ if q_pe.shape[-1] == 0: # NoPE: the cat would be a full-width copy+ q_cat = q_nope.unsqueeze(1)+ else:+ q_cat = torch.cat([q_nope, q_pe], dim=-1).view(bs, 1, self.num_heads, self.latent_dim)o = self._attend(q_cat, layer_id, sel, cnt)return o.view(bs, self.num_heads, self.kv_lora_rank)diff --git a/python/freetoken/kernel/triton/glm_dsa_sparse.py b/python/freetoken/kernel/triton/glm_dsa_sparse.pyindex 8aaeeb9b..0f67aaa7 100644--- a/python/freetoken/kernel/triton/glm_dsa_sparse.py+++ b/python/freetoken/kernel/triton/glm_dsa_sparse.py@@ -345,13 +345,14 @@ def _split_count(b: int, m: int, h: int, topk: int, device) -> int:def glm_dsa_sparse_attn(- q: torch.Tensor, # [b, m, h, d_v + d_r] (ckv-absorbed | rope)+ q: torch.Tensor, # [b, m, h, d_v + d_r] (ckv-absorbed | rope), any stridespool: torch.Tensor, # [rows, d_v + d_r] GLOBAL latent pool slab (this layer)topk_idxs: torch.Tensor, # [b, m|1, topk] int32 global rows, -1 maskedsoftmax_scale: float,counts: torch.Tensor | None = None, # [b, m] int32 live columns per query (device-read)d_v: int = 512,force_splits: int | None = None, # tests only: 0 = single-program, N = split-k N+ out: torch.Tensor | None = None, # [b, m, h, d_v] to write into, any strides) -> torch.Tensor:"""Sparse MLA attention over gathered latent rows; returns ``[b, m, h, d_v]``.@@ -359,18 +360,29 @@ def glm_dsa_sparse_attn(list across all queries with STRIDE 0 -- the identity-selection dense path: everyquery reads the same position-ordered row list, causally bounded by its own``counts[q] = position + 1``, with zero per-query index materialization.++ ``q`` and ``out`` are consumed with their given strides (a transposed bmm output+ works without a contiguous copy); ``out`` lets a preallocated workspace receive+ the result instead of a fresh allocation."""b, m, h, d = q.shaped_r = d - d_vtopk = topk_idxs.shape[-1]assert pool.shape[-1] == d, (pool.shape, d)- q = q.contiguous()pool_2d = pool.reshape(-1, d)assert pool_2d.stride(-1) == 1idx = topk_idxs.contiguous().to(torch.int32)broadcast_m = idx.shape[1] == 1 and m > 1assert broadcast_m or idx.shape[1] == m, (idx.shape, m)- o = q.new_empty(b, m, h, d_v)+ if out is None:+ o = q.new_empty(b, m, h, d_v)+ else:+ o = out+ assert o.shape == (b, m, h, d_v), (o.shape, (b, m, h, d_v))+ assert o.dtype == q.dtype and o.device == q.device, (o.dtype, o.device, q.dtype, q.device)+ # The kernels address q and o with int32 offsets.+ reach = max(sum((n - 1) * st for n, st in zip(t.shape, t.stride())) for t in (q, o))+ assert reach < 2**31, f"q/o span {reach} elements, past int32 addressing; lower --max-prefill-length"has_counts = counts is not Noneif has_counts:diff --git a/python/freetoken/models/glm5_next/attention.py b/python/freetoken/models/glm5_next/attention.pyindex 0814a687..b208433a 100644--- a/python/freetoken/models/glm5_next/attention.py+++ b/python/freetoken/models/glm5_next/attention.py@@ -167,26 +167,31 @@ class Glm5NextAttention(BaseOP):c_kv = self.kv_a_layernorm.forward(self.kv_a_proj_with_mqa.forward(x))- # Absorb kv_b's k-part into the query: q_nope[H,T,nope] @ W_uk[H,nope,lora].- q_absorbed = torch.bmm(q_nope.transpose(0, 1).contiguous(), w_uk).transpose(0, 1)-indexer_inputs = (self.indexer.compute(x, q_a_resid)if getattr(ctx.attn_backend, "dsa_enabled", False)else None)+ # Absorb kv_b's k-part into the query: q_nope[H,T,nope] @ W_uk[H,nope,lora].+ # The [H,T,lora] output serves downstream as a transposed view; the dels free+ # the q_b_proj output, the next-widest transient, before attention runs.+ q_absorbed = torch.bmm(q_nope.transpose(0, 1), w_uk)+ del q, q_nope, q_a_resid+# Zero-width rope halves: cat/scatter no-ops in the backend.- q_pe = q.new_empty(t, self.num_heads, 0)- k_rope = q.new_empty(t, 0)+ q_pe = q_absorbed.new_empty(t, self.num_heads, 0)+ k_rope = q_absorbed.new_empty(t, 0)o_latent = ctx.attn_backend.mla_forward(- q_absorbed.contiguous(), q_pe, c_kv.contiguous(), k_rope,+ q_absorbed.transpose(0, 1), q_pe, c_kv, k_rope,self.layer_id, ctx.batch, indexer_inputs=indexer_inputs,) # [T, H, kv_lora_rank]+ del q_absorbed# Absorb kv_b's v-part onto the output: o_latent[H,T,lora] @ W_uv_t[H,lora,v].- o = torch.bmm(o_latent.transpose(0, 1).contiguous(), w_uv).transpose(0, 1)- return self.o_proj.forward(o.reshape(t, self.num_heads * self.v_head_dim))+ o = torch.bmm(o_latent.transpose(0, 1), w_uv)+ del o_latent+ return self.o_proj.forward(o.transpose(0, 1).reshape(t, self.num_heads * self.v_head_dim))__all__ = ["Glm5NextAttention", "Glm5NextIndexer"]diff --git a/tests/models/test_glm_dsa.py b/tests/models/test_glm_dsa.pyindex 7a82894d..4b21ec5c 100644--- a/tests/models/test_glm_dsa.py+++ b/tests/models/test_glm_dsa.py@@ -227,6 +227,72 @@ def test_splitk_matches_single_program():assert (o_single.float() - o_split.float()).abs().max().item() < 1e-2[email protected]("dr", [64, 0])+def test_strided_q_and_out_slice_write(dr):+ """The glm5_next NoPE plumbing hands the kernel a TRANSPOSED bmm output view as+ q (no [T, H, d] contiguous copy) and writes the result into a slice of the+ [T, H, d_v] prefill workspace (out=). Both must reproduce the contiguous-q /+ fresh-allocation path exactly: same kernel, same inputs, only strides differ.+ dr = 0 pins the NoPE compile variant glm5_next runs; dr = 64 the roped one."""+ from freetoken.kernel.triton.glm_dsa_sparse import glm_dsa_sparse_attn++ torch.manual_seed(4)+ h, dv, n, m = 8, 64, 300, 6+ pool = torch.randn(n + 20, dv + dr, device="cuda", dtype=torch.bfloat16) * 0.5+ rows = torch.randperm(n + 20, device="cuda")[:n].to(torch.int32)+ q_mat = torch.randn(h, m, dv + dr, device="cuda", dtype=torch.bfloat16) # bmm-style [H, m, d]+ q_strided = q_mat.transpose(0, 1) # [m, h, d] view -- what mla_forward receives++ idx = torch.full((1, 1, n), -1, device="cuda", dtype=torch.int32)+ idx[0, 0] = rows+ pos = torch.arange(n - m, n, device="cuda")+ cnt = (pos + 1).to(torch.int32).view(1, m)+ scale = (dv + dr) ** -0.5++ o_ref = glm_dsa_sparse_attn(+ q_strided.contiguous().unsqueeze(0), pool, idx, scale, counts=cnt, d_v=dv,+ )++ ws = torch.full((64, h, dv), float("nan"), device="cuda", dtype=torch.bfloat16)+ glm_dsa_sparse_attn(+ q_strided.unsqueeze(0), pool, idx, scale, counts=cnt, d_v=dv,+ out=ws[10 : 10 + m].view(1, m, h, dv),+ )+ assert torch.equal(ws[10 : 10 + m], o_ref[0]), "strided q / slice out != contiguous path"+ assert torch.isnan(ws[:10]).all() and torch.isnan(ws[10 + m :]).all(), \+ "kernel wrote outside the out= slice"++[email protected]("dr", [64, 0])+def test_splitk_writes_out_slice(dr):+ """An m == 1 prefill (a radix-hit prompt) auto-selects split-k; the merge stage+ must write the merged result into an arbitrary out= slice exactly like the+ single-program kernel does."""+ from freetoken.kernel.triton.glm_dsa_sparse import glm_dsa_sparse_attn++ torch.manual_seed(6)+ h, dv, n = 8, 64, 300+ pool = torch.randn(n + 20, dv + dr, device="cuda", dtype=torch.bfloat16) * 0.5+ rows = torch.randperm(n + 20, device="cuda")[:n].to(torch.int32)+ q_mat = torch.randn(h, 1, dv + dr, device="cuda", dtype=torch.bfloat16)+ q = q_mat.transpose(0, 1).unsqueeze(0) # [1, 1, h, d] strided view+ idx = rows.view(1, 1, n)+ cnt = torch.tensor([[n]], device="cuda", dtype=torch.int32)+ scale = (dv + dr) ** -0.5++ o_ref = glm_dsa_sparse_attn(+ q.contiguous(), pool, idx, scale, counts=cnt, d_v=dv, force_splits=4,+ )+ ws = torch.full((4, h, dv), float("nan"), device="cuda", dtype=torch.bfloat16)+ glm_dsa_sparse_attn(+ q, pool, idx, scale, counts=cnt, d_v=dv, force_splits=4,+ out=ws[2:3].view(1, 1, h, dv),+ )+ assert torch.equal(ws[2], o_ref[0, 0]), "split-k merge into out= slice != reference"+ assert torch.isnan(ws[:2]).all() and torch.isnan(ws[3:]).all(), \+ "split-k wrote outside the out= slice"++def _make_backend(dsa: bool, latent=80, dv=64, idx_dim=32, idx_heads=16, topk=64, pages=400):"""Minimal ctx + pool + DSAAttnBackend (no engine)."""from types import SimpleNamespace