Skip to content

FreeToken brings datacenter-scale model serving to your desktop. Run massive models locally, fast and efficiently.

perf(attention): drop redundant activation copies in glm5 dsa prefill (#795)

Mukkesh Ganesh0c50eb9d829e
diff --git a/python/freetoken/attention/dsa.py b/python/freetoken/attention/dsa.py
index 398d1700..e8c7b3c2 100644
--- a/python/freetoken/attention/dsa.py
+++ b/python/freetoken/attention/dsa.py
@@ -193,14 +193,17 @@ class DSAAttnBackend(DSAIndexerMixin, BaseAttnBackend):
# ----- attention ------------------------------------------------------------------
def _attend(
- self, q_cat: torch.Tensor, layer_id: int, sel: torch.Tensor, cnt: torch.Tensor
+ self, q_cat: torch.Tensor, layer_id: int, sel: torch.Tensor, cnt: torch.Tensor,
+ out: torch.Tensor | None = None,
) -> torch.Tensor:
- """Gathered-KV MLA over latent rows: q [b, m, H, 576] -> [b, m, H, 512]."""
+ """Gathered-KV MLA over latent rows: q [b, m, H, 576] -> [b, m, H, 512].
+ ``q_cat`` keeps its strides (a transposed bmm output needs no copy) and
+ ``out`` receives the result in place."""
from freetoken.kernel.triton.glm_dsa_sparse import glm_dsa_sparse_attn
return glm_dsa_sparse_attn(
q_cat, self.kvcache.latent_rows(layer_id), sel, self.sm_scale,
- counts=cnt, d_v=self.kv_lora_rank,
+ counts=cnt, d_v=self.kv_lora_rank, out=out,
)
def mla_forward(
@@ -253,7 +256,10 @@ class DSAAttnBackend(DSAIndexerMixin, BaseAttnBackend):
md.sel.clear()
md.sel[layer_id] = (sel, cnt)
sel, cnt = md.sel[self._leader[layer_id]]
- q_cat = torch.cat([q_nope, q_pe], dim=-1).view(bs, 1, self.num_heads, self.latent_dim)
+ if q_pe.shape[-1] == 0: # NoPE: the cat would be a full-width copy
+ q_cat = q_nope.unsqueeze(1)
+ else:
+ q_cat = torch.cat([q_nope, q_pe], dim=-1).view(bs, 1, self.num_heads, self.latent_dim)
o = self._attend(q_cat, layer_id, sel, cnt)
return o.view(bs, self.num_heads, self.kv_lora_rank)
@@ -287,7 +293,12 @@ class DSAAttnBackend(DSAIndexerMixin, BaseAttnBackend):
def _prefill(self, md, layer_id, q_nope, q_pe, batch, inputs) -> torch.Tensor:
t = q_nope.shape[0]
- q_cat = torch.cat([q_nope, q_pe], dim=-1) # [T, H, 576]
+ # NoPE (zero rope width): the cat would copy the whole [T, H, lora] q for
+ # nothing -- pass it through and let the kernel read its strides.
+ if q_pe.shape[-1] == 0:
+ q_cat = q_nope
+ else:
+ q_cat = torch.cat([q_nope, q_pe], dim=-1) # [T, H, 576]
reqs = batch.padded_reqs if hasattr(batch, "padded_reqs") else batch.reqs
page_table = get_global_ctx().page_table
qo = md.qo_indptr_cpu.tolist()
@@ -320,10 +331,13 @@ class DSAAttnBackend(DSAIndexerMixin, BaseAttnBackend):
# through per-query counts.
sel = page_table[r.table_idx, : r.device_len].view(1, 1, -1).to(torch.int32)
cnt = (batch.positions[qo[i] : qo[i + 1]] + 1).to(torch.int32).view(1, m)
- o[qo[i] : qo[i + 1]] = self._attend(
+ # The kernel writes straight into the output slice (out=): no per-request
+ # attention output next to o at the peak.
+ self._attend(
q_cat[qo[i] : qo[i + 1]].view(1, m, self.num_heads, self.latent_dim),
layer_id, sel, cnt,
- ).view(m, self.num_heads, self.kv_lora_rank)
+ out=o[qo[i] : qo[i + 1]].view(1, m, self.num_heads, self.kv_lora_rank),
+ )
return o
# ----- CUDA graph (decode) ----------------------------------------------------------
diff --git a/python/freetoken/attention/dsa_indexer_kpool.py b/python/freetoken/attention/dsa_indexer_kpool.py
index 6743a143..2b2492aa 100644
--- a/python/freetoken/attention/dsa_indexer_kpool.py
+++ b/python/freetoken/attention/dsa_indexer_kpool.py
@@ -243,7 +243,10 @@ class Glm5NextDSABackend(DSAAttnBackend):
md.sel.clear()
md.sel[layer_id] = (sel, cnt)
sel, cnt = md.sel[self._leader[layer_id]]
- q_cat = torch.cat([q_nope, q_pe], dim=-1).view(bs, 1, self.num_heads, self.latent_dim)
+ if q_pe.shape[-1] == 0: # NoPE: the cat would be a full-width copy
+ q_cat = q_nope.unsqueeze(1)
+ else:
+ q_cat = torch.cat([q_nope, q_pe], dim=-1).view(bs, 1, self.num_heads, self.latent_dim)
o = self._attend(q_cat, layer_id, sel, cnt)
return o.view(bs, self.num_heads, self.kv_lora_rank)
diff --git a/python/freetoken/kernel/triton/glm_dsa_sparse.py b/python/freetoken/kernel/triton/glm_dsa_sparse.py
index 8aaeeb9b..0f67aaa7 100644
--- a/python/freetoken/kernel/triton/glm_dsa_sparse.py
+++ b/python/freetoken/kernel/triton/glm_dsa_sparse.py
@@ -345,13 +345,14 @@ def _split_count(b: int, m: int, h: int, topk: int, device) -> int:
def glm_dsa_sparse_attn(
- q: torch.Tensor, # [b, m, h, d_v + d_r] (ckv-absorbed | rope)
+ q: torch.Tensor, # [b, m, h, d_v + d_r] (ckv-absorbed | rope), any strides
pool: torch.Tensor, # [rows, d_v + d_r] GLOBAL latent pool slab (this layer)
topk_idxs: torch.Tensor, # [b, m|1, topk] int32 global rows, -1 masked
softmax_scale: float,
counts: torch.Tensor | None = None, # [b, m] int32 live columns per query (device-read)
d_v: int = 512,
force_splits: int | None = None, # tests only: 0 = single-program, N = split-k N
+ out: torch.Tensor | None = None, # [b, m, h, d_v] to write into, any strides
) -> torch.Tensor:
"""Sparse MLA attention over gathered latent rows; returns ``[b, m, h, d_v]``.
@@ -359,18 +360,29 @@ def glm_dsa_sparse_attn(
list across all queries with STRIDE 0 -- the identity-selection dense path: every
query reads the same position-ordered row list, causally bounded by its own
``counts[q] = position + 1``, with zero per-query index materialization.
+
+ ``q`` and ``out`` are consumed with their given strides (a transposed bmm output
+ works without a contiguous copy); ``out`` lets a preallocated workspace receive
+ the result instead of a fresh allocation.
"""
b, m, h, d = q.shape
d_r = d - d_v
topk = topk_idxs.shape[-1]
assert pool.shape[-1] == d, (pool.shape, d)
- q = q.contiguous()
pool_2d = pool.reshape(-1, d)
assert pool_2d.stride(-1) == 1
idx = topk_idxs.contiguous().to(torch.int32)
broadcast_m = idx.shape[1] == 1 and m > 1
assert broadcast_m or idx.shape[1] == m, (idx.shape, m)
- o = q.new_empty(b, m, h, d_v)
+ if out is None:
+ o = q.new_empty(b, m, h, d_v)
+ else:
+ o = out
+ assert o.shape == (b, m, h, d_v), (o.shape, (b, m, h, d_v))
+ assert o.dtype == q.dtype and o.device == q.device, (o.dtype, o.device, q.dtype, q.device)
+ # The kernels address q and o with int32 offsets.
+ reach = max(sum((n - 1) * st for n, st in zip(t.shape, t.stride())) for t in (q, o))
+ assert reach < 2**31, f"q/o span {reach} elements, past int32 addressing; lower --max-prefill-length"
has_counts = counts is not None
if has_counts:
diff --git a/python/freetoken/models/glm5_next/attention.py b/python/freetoken/models/glm5_next/attention.py
index 0814a687..b208433a 100644
--- a/python/freetoken/models/glm5_next/attention.py
+++ b/python/freetoken/models/glm5_next/attention.py
@@ -167,26 +167,31 @@ class Glm5NextAttention(BaseOP):
c_kv = self.kv_a_layernorm.forward(self.kv_a_proj_with_mqa.forward(x))
- # Absorb kv_b's k-part into the query: q_nope[H,T,nope] @ W_uk[H,nope,lora].
- q_absorbed = torch.bmm(q_nope.transpose(0, 1).contiguous(), w_uk).transpose(0, 1)
-
indexer_inputs = (
self.indexer.compute(x, q_a_resid)
if getattr(ctx.attn_backend, "dsa_enabled", False)
else None
)
+ # Absorb kv_b's k-part into the query: q_nope[H,T,nope] @ W_uk[H,nope,lora].
+ # The [H,T,lora] output serves downstream as a transposed view; the dels free
+ # the q_b_proj output, the next-widest transient, before attention runs.
+ q_absorbed = torch.bmm(q_nope.transpose(0, 1), w_uk)
+ del q, q_nope, q_a_resid
+
# Zero-width rope halves: cat/scatter no-ops in the backend.
- q_pe = q.new_empty(t, self.num_heads, 0)
- k_rope = q.new_empty(t, 0)
+ q_pe = q_absorbed.new_empty(t, self.num_heads, 0)
+ k_rope = q_absorbed.new_empty(t, 0)
o_latent = ctx.attn_backend.mla_forward(
- q_absorbed.contiguous(), q_pe, c_kv.contiguous(), k_rope,
+ q_absorbed.transpose(0, 1), q_pe, c_kv, k_rope,
self.layer_id, ctx.batch, indexer_inputs=indexer_inputs,
) # [T, H, kv_lora_rank]
+ del q_absorbed
# Absorb kv_b's v-part onto the output: o_latent[H,T,lora] @ W_uv_t[H,lora,v].
- o = torch.bmm(o_latent.transpose(0, 1).contiguous(), w_uv).transpose(0, 1)
- return self.o_proj.forward(o.reshape(t, self.num_heads * self.v_head_dim))
+ o = torch.bmm(o_latent.transpose(0, 1), w_uv)
+ del o_latent
+ return self.o_proj.forward(o.transpose(0, 1).reshape(t, self.num_heads * self.v_head_dim))
__all__ = ["Glm5NextAttention", "Glm5NextIndexer"]
diff --git a/tests/models/test_glm_dsa.py b/tests/models/test_glm_dsa.py
index 7a82894d..4b21ec5c 100644
--- a/tests/models/test_glm_dsa.py
+++ b/tests/models/test_glm_dsa.py
@@ -227,6 +227,72 @@ def test_splitk_matches_single_program():
assert (o_single.float() - o_split.float()).abs().max().item() < 1e-2
[email protected]("dr", [64, 0])
+def test_strided_q_and_out_slice_write(dr):
+ """The glm5_next NoPE plumbing hands the kernel a TRANSPOSED bmm output view as
+ q (no [T, H, d] contiguous copy) and writes the result into a slice of the
+ [T, H, d_v] prefill workspace (out=). Both must reproduce the contiguous-q /
+ fresh-allocation path exactly: same kernel, same inputs, only strides differ.
+ dr = 0 pins the NoPE compile variant glm5_next runs; dr = 64 the roped one."""
+ from freetoken.kernel.triton.glm_dsa_sparse import glm_dsa_sparse_attn
+
+ torch.manual_seed(4)
+ h, dv, n, m = 8, 64, 300, 6
+ pool = torch.randn(n + 20, dv + dr, device="cuda", dtype=torch.bfloat16) * 0.5
+ rows = torch.randperm(n + 20, device="cuda")[:n].to(torch.int32)
+ q_mat = torch.randn(h, m, dv + dr, device="cuda", dtype=torch.bfloat16) # bmm-style [H, m, d]
+ q_strided = q_mat.transpose(0, 1) # [m, h, d] view -- what mla_forward receives
+
+ idx = torch.full((1, 1, n), -1, device="cuda", dtype=torch.int32)
+ idx[0, 0] = rows
+ pos = torch.arange(n - m, n, device="cuda")
+ cnt = (pos + 1).to(torch.int32).view(1, m)
+ scale = (dv + dr) ** -0.5
+
+ o_ref = glm_dsa_sparse_attn(
+ q_strided.contiguous().unsqueeze(0), pool, idx, scale, counts=cnt, d_v=dv,
+ )
+
+ ws = torch.full((64, h, dv), float("nan"), device="cuda", dtype=torch.bfloat16)
+ glm_dsa_sparse_attn(
+ q_strided.unsqueeze(0), pool, idx, scale, counts=cnt, d_v=dv,
+ out=ws[10 : 10 + m].view(1, m, h, dv),
+ )
+ assert torch.equal(ws[10 : 10 + m], o_ref[0]), "strided q / slice out != contiguous path"
+ assert torch.isnan(ws[:10]).all() and torch.isnan(ws[10 + m :]).all(), \
+ "kernel wrote outside the out= slice"
+
+
[email protected]("dr", [64, 0])
+def test_splitk_writes_out_slice(dr):
+ """An m == 1 prefill (a radix-hit prompt) auto-selects split-k; the merge stage
+ must write the merged result into an arbitrary out= slice exactly like the
+ single-program kernel does."""
+ from freetoken.kernel.triton.glm_dsa_sparse import glm_dsa_sparse_attn
+
+ torch.manual_seed(6)
+ h, dv, n = 8, 64, 300
+ pool = torch.randn(n + 20, dv + dr, device="cuda", dtype=torch.bfloat16) * 0.5
+ rows = torch.randperm(n + 20, device="cuda")[:n].to(torch.int32)
+ q_mat = torch.randn(h, 1, dv + dr, device="cuda", dtype=torch.bfloat16)
+ q = q_mat.transpose(0, 1).unsqueeze(0) # [1, 1, h, d] strided view
+ idx = rows.view(1, 1, n)
+ cnt = torch.tensor([[n]], device="cuda", dtype=torch.int32)
+ scale = (dv + dr) ** -0.5
+
+ o_ref = glm_dsa_sparse_attn(
+ q.contiguous(), pool, idx, scale, counts=cnt, d_v=dv, force_splits=4,
+ )
+ ws = torch.full((4, h, dv), float("nan"), device="cuda", dtype=torch.bfloat16)
+ glm_dsa_sparse_attn(
+ q, pool, idx, scale, counts=cnt, d_v=dv, force_splits=4,
+ out=ws[2:3].view(1, 1, h, dv),
+ )
+ assert torch.equal(ws[2], o_ref[0, 0]), "split-k merge into out= slice != reference"
+ assert torch.isnan(ws[:2]).all() and torch.isnan(ws[3:]).all(), \
+ "split-k wrote outside the out= slice"
+
+
def _make_backend(dsa: bool, latent=80, dv=64, idx_dim=32, idx_heads=16, topk=64, pages=400):
"""Minimal ctx + pool + DSAAttnBackend (no engine)."""
from types import SimpleNamespace