FreeToken brings datacenter-scale model serving to your desktop. Run massive models locally, fast and efficiently.
"""glm5_next (GLM-5.3-Flash) kpool DSA backend: pooled-indexer addressing.
Extends the GLM-5.2 DSA backend with the kpool compression scheme: the indexer K
cache is scored at POOL granularity -- every ``index_kpool`` (4) consecutive
tokens fold into one entry, a per-channel ``softmax(gate + APE)``-weighted sum of
their raw keys -- so indexer compute and top-k shrink by 4x. Selection picks
``index_topk // kpool`` pools, each pool expands back to its constituent token
rows, and the request's trailing incomplete pool ("tail", up to kpool-1 tokens)
is force-included (``index_kpool_always_select_tail``). Selection widths are
therefore ``(kpool - 1) + select_k * kpool`` with ``-1`` gather-only sentinels;
the sparse-MLA kernel masks them.
Slab convention (KpoolDSAKVCache): the index slab is a 1/kpool SHADOW of the KV
pages -- a pool's entry lives at row ``token_slot // kpool`` (well-defined because
the engine pins ``page_size % kpool == 0``, so a pool never straddles a page).
Raw keys + gate scores of the in-progress pool live in per-request tail rings
written at ``pos % kpool``; a pool that closes reads its older members from the
ring (so a chunk may start mid-pool), and a decode step whose pool does NOT close
scatters its (garbage) pooled candidate into the request's scratch row -- an
unconditional, CUDA-graph-safe write that scoring never reads (QSA precedent).
Faithfulness: pooled entries are stored bf16 (the reference Hadamard-rotates and
fp8-quantizes them -- a memory device whose rotation cancels in the score); the
pooling softmax runs fp32, matching the reference kernel. Selection is a plain
top-k over pool scores plus the tail: the reference does NOT force-include the
query's own (complete) pool -- the model is trained with that scheme, so neither
do we. Sharing note: IndexShare does not exist here; every DSA layer owns its
indexer slot and is its own leader.
"""
from __future__ import annotations
from typing import TYPE_CHECKING, Tuple
import torch
from .dsa import DSAAttnBackend, DSAMetadata, KpoolPlan
if TYPE_CHECKING:
from freetoken.core import Batch
from freetoken.models import ModelConfig
class Glm5NextDSABackend(DSAAttnBackend):
def __init__(self, config: "ModelConfig") -> None:
super().__init__(config)
self._slots_buf: torch.Tensor | None = None # static graph buffer for ring_slots
if self.dsa_enabled:
from freetoken.kvcache.dsa_pool import KpoolDSAKVCache
assert isinstance(self.kvcache, KpoolDSAKVCache), (
"glm5_next kpool indexer needs the KpoolDSAKVCache (tail rings); "
f"the pool factory built {type(self.kvcache).__name__}"
)
# ----- CUDA-graph decode staging ------------------------------------------------------
# The tail rings are keyed by Req.table_idx; a captured decode step must read it
# from a static buffer restaged per replay (rows/kvlen precedent in the parent).
def init_capture_graph(self, max_seq_len: int, bs_list, rows_per_req: int = 1) -> None:
super().init_capture_graph(max_seq_len, bs_list, rows_per_req)
self._slots_buf = torch.zeros(max(bs_list), dtype=torch.int64, device=self.device)
def _stage_decode(self, batch: "Batch", bs: int, table_idx: torch.Tensor) -> None:
super()._stage_decode(batch, bs, table_idx)
self._slots_buf[:bs].copy_(table_idx)
batch.attn_metadata.ring_slots = self._slots_buf[:bs]
def reset_capture(self) -> None:
super().reset_capture()
self._slots_buf = None
# ----- model-family hooks -----------------------------------------------------------
def _model_args(self, config: "ModelConfig"):
args = config.glm5_args
assert args is not None, "kpool dsa backend needs ModelConfig.glm5_args"
self.kpool = args.index_kpool
assert self.kpool > 1 and args.index_kpool_compress, (
"Glm5NextDSABackend serves the kpool-compressed indexer; a kpool=1 "
"checkpoint should run the plain DSA backend"
)
assert args.index_topk % self.kpool == 0
self.select_k = args.index_topk // self.kpool
assert args.index_kpool_always_select_tail, (
"tail force-inclusion is baked into the selection layout"
)
return args
def _build_index_slots(self, args, config: "ModelConfig") -> None:
# No IndexShare: every DSA layer owns its indexer and is its own leader.
for lid in args.dsa_layer_ids:
if lid >= config.num_layers:
continue
self._idx_slot[lid] = len(self._idx_slot)
self._leader[lid] = lid
# ----- store: fused single path (prefill AND decode, CUDA-graph capturable) ----------
def _plan_kpool_writes(self, md, batch: "Batch", slot: int):
"""Per-token slab/ring routing for this forward; layer-invariant, so the
first DSA layer computes it and the rest reuse it (QSA _plan_index_writes
shape). Pure device arithmetic: no host sync, graph-capturable.
Rebuilt at the FIRST indexer slot of every forward, never trusted across
forwards: a capture batch runs its warmup and its capture through ONE
metadata object, and a cached plan would bake the warmup's (non-graph-pool)
tensor addresses into the graph (QSA precedent)."""
if slot != 0 and md.kpool_plan is not None:
return md.kpool_plan
kp = self.kpool
out_loc = batch.out_loc.to(torch.int64)
positions = batch.positions.to(torch.int64)
t = out_loc.numel()
if md.is_decode:
# One token per request. ring_slots is the backend's STATIC buffer under
# graphs (restaged per replay in _stage_decode); eager reads the
# scheduler-staged active_table_idx. arange shapes are fixed per capture.
ring_slots = (
md.ring_slots if md.ring_slots is not None else batch.active_table_idx
).to(torch.int64)
token_to_req = torch.arange(t, device=self.device, dtype=torch.int32)
cu_seqlens = torch.arange(t + 1, device=self.device, dtype=torch.int32)
else:
reqs = batch.padded_reqs if hasattr(batch, "padded_reqs") else batch.reqs
ring_slots = torch.tensor(
[r.table_idx for r in reqs], dtype=torch.int64, pin_memory=True
).to(self.device, non_blocking=True)
cu_cpu = md.qo_indptr_cpu
cu_seqlens = cu_cpu.to(self.device, non_blocking=True)
token_to_req = torch.repeat_interleave(
torch.arange(len(reqs), dtype=torch.int32),
(cu_cpu[1:] - cu_cpu[:-1]).to(torch.int64),
).pin_memory().to(self.device, non_blocking=True)
slots = ring_slots.index_select(0, token_to_req.to(torch.int64))
# page_size % kpool == 0, so out_loc % kp == position % kp: a group closes
# exactly on position % kp == kp - 1. Non-closing rows land in the request's
# scratch row (never scored).
closing = positions % kp == kp - 1
cmp_rows = torch.where(
closing, out_loc // kp, self.kvcache.cmp_scratch_base + slots
).to(torch.int32)
# Ring refresh: only each request's last kp rows survive to the next forward
# (one keeper per pos%kp residue -- deterministic, no write races).
rows = torch.arange(t, device=self.device, dtype=torch.int64)
ends = cu_seqlens.to(torch.int64).index_select(0, token_to_req.to(torch.int64) + 1)
keep = rows >= ends - kp
ring_row = slots * kp + positions % kp
ring_rows = torch.where(keep, ring_row, torch.full_like(ring_row, -1)).to(
torch.int32
)
md.kpool_plan = KpoolPlan(cmp_rows, ring_rows, ring_slots, token_to_req, cu_seqlens)
return md.kpool_plan
def _store_index(self, inputs, batch: "Batch", layer_id: int) -> None:
"""One fused kernel serves prefill and decode (layout and member
resolution: module docstring); the ring refresh follows the read."""
from freetoken.kernel.triton.kpool_compress import kpool_compress_store
from freetoken.kernel.triton.qsa import qsa_store_rows
md = batch.attn_metadata
assert isinstance(md, DSAMetadata)
assert inputs.gate is not None and inputs.ape is not None, (
"kpool store needs DSAIndexerInputs.gate/ape from the model's indexer"
)
k, gate = inputs.k, inputs.gate.to(inputs.k.dtype)
slot = self._idx_slot[layer_id]
tail_k, tail_g = self.kvcache.tail_k(slot), self.kvcache.tail_gate(slot)
plan = self._plan_kpool_writes(md, batch, slot)
kpool_compress_store(
k, gate,
tail_k.view(-1, k.shape[-1]), tail_g.view(-1, k.shape[-1]),
inputs.ape,
plan.ring_slots, plan.token_to_req, plan.cu_seqlens, batch.positions,
self.kvcache.index_k_cache(slot), plan.cmp_rows,
self.kpool,
)
# After the compression read: the ring rows this forward overwrites are
# exactly the ones a straddling group just consumed.
qsa_store_rows(tail_k, plan.ring_rows, k)
qsa_store_rows(tail_g, plan.ring_rows, gate)
# ----- selection: pools -> token rows + tail ------------------------------------------
def _expand_and_tail(
self,
picks: torch.Tensor, # [B, m, select_k] pool ids, -1 sentinel
rows: torch.Tensor, # [B, W] or [B, m, W] position-ordered physical rows
q_pos: torch.Tensor, # [B, m] query positions (token-granular)
) -> Tuple[torch.Tensor, torch.Tensor]:
"""Selected pools -> token rows, tail tokens appended FIRST (fixed kpool-1
slots, -1 padded). Returns (sel [B, m, (kpool-1) + select_k*kpool] int32,
cnt [B, m] int32)."""
kp = self.kpool
b, m, k_sel = picks.shape
device = picks.device
offs = torch.arange(kp, device=device)
# History: pool pick p -> token positions p*kp + [0, kp).
hist_pos = picks.unsqueeze(-1) * kp + offs # [B, m, k_sel, kp]
hist_pos = torch.where(
picks.unsqueeze(-1) < 0, hist_pos.new_full((), -1), hist_pos
).view(b, m, k_sel * kp)
# Tail: positions [n_pools*kp, q_pos] of each query's own request/step.
n_pools = (q_pos + 1) // kp # complete pools at this query
tail_start = n_pools * kp
toffs = torch.arange(kp - 1, device=device)
tail_pos = tail_start.unsqueeze(-1) + toffs # [B, m, kp-1]
tail_pos = torch.where(tail_pos <= q_pos.unsqueeze(-1), tail_pos, tail_pos.new_full((), -1))
pos = torch.cat([tail_pos, hist_pos], dim=-1) # [B, m, width]
rows_b = rows if rows.dim() == 3 else rows.unsqueeze(1).expand(b, m, -1)
sel = rows_b.gather(-1, pos.clamp_min(0).long()).to(torch.int32)
sel = torch.where(pos < 0, sel.new_full((), -1), sel)
# Valid entries: the fixed tail slots + all expanded picked pools. -1
# sentinels inside the bound are masked by the sparse kernel.
# clamp_max (not minimum(new_tensor)): no H2D copy, CUDA-graph safe.
cnt = ((kp - 1) + n_pools.clamp_max(k_sel) * kp).to(torch.int32)
return sel, cnt
def _decode(self, md, layer_id, q_nope, q_pe, inputs) -> torch.Tensor:
bs = q_nope.shape[0]
rows, kvlen = md.rows, md.kvlen
if not self.dsa_enabled:
return super()._decode(md, layer_id, q_nope, q_pe, inputs)
if inputs is not None:
q_idx, w = inputs.q, inputs.w
kp = self.kpool
n_pools = (kvlen // kp).to(torch.int32)
# Pool p's shadow row = any member's token slot // kp (pools never
# straddle pages): stride the position-ordered row snapshot down to
# pool granularity, then divide into the shadow slab.
pool_rows = (rows[:, kp - 1 :: kp] // kp).contiguous()
s = self.dsa_decode_scores(q_idx, w, self._idx_slot[layer_id], pool_rows, n_pools)
k_sel = min(self.select_k, s.shape[-1])
picks = self.indexer_select_decode(
s.view(bs, 1, -1), valid=n_pools, topk=k_sel, offset=0
) # [bs, 1, k_sel] pool ids
sel, cnt = self._expand_and_tail(
picks.transpose(0, 1), # [1, bs, k_sel]
rows.unsqueeze(0), # [1, bs, W]
(kvlen - 1).view(1, bs),
)
sel = sel.transpose(0, 1) # [bs, 1, width]
cnt = cnt.view(bs, 1)
md.sel.clear()
md.sel[layer_id] = (sel, cnt)
sel, cnt = md.sel[self._leader[layer_id]]
if q_pe.shape[-1] == 0: # NoPE: the cat would be a full-width copy
q_cat = q_nope.unsqueeze(1)
else:
q_cat = torch.cat([q_nope, q_pe], dim=-1).view(bs, 1, self.num_heads, self.latent_dim)
o = self._attend(q_cat, layer_id, sel, cnt)
return o.view(bs, self.num_heads, self.kv_lora_rank)
def _select_prefill(
self, slot: int, q_idx: torch.Tensor, w: torch.Tensor,
rows: torch.Tensor, positions: torch.Tensor, start_pos: int,
) -> Tuple[torch.Tensor, torch.Tensor]:
"""Per-request causal top-k at pool granularity, expanded to token rows.
``start_pos`` is the request's cached_len (host int, no device sync)."""
kp = self.kpool
kv_len = rows.numel()
n_pools_total = kv_len // kp
pool_rows = rows[kp - 1 :: kp] // kp # shadow rows (token slot // kpool)
k_pool = self.kvcache.index_k_cache(slot).index_select(0, pool_rows.long())
k_sel = min(self.select_k, max(n_pools_total, 1))
m = q_idx.shape[0]
width = (kp - 1) + k_sel * kp
if n_pools_total == 0:
# No complete pool to score (a sub-kpool request riding a sparse batch):
# the selection is the tail alone (select_prefill would return zero pick
# columns and break the fixed width below).
picks = torch.full((1, m, k_sel), -1, dtype=torch.int32, device=self.device)
return self._expand_and_tail(picks, rows.view(1, -1), positions.view(1, -1))
sel = torch.empty(m, width, dtype=torch.int32, device=self.device)
cnt = torch.empty(m, dtype=torch.int32, device=self.device)
chunk = 512
for s0 in range(0, m, chunk):
s1 = min(s0 + chunk, m)
scores = self.dsa_prefill_logits(q_idx[s0:s1], k_pool, w[s0:s1])
picks = self.indexer_select_prefill(
scores.unsqueeze(0), start_pos=start_pos + s0, seqlen=s1 - s0,
ratio=kp, topk=k_sel, offset=0,
) # [1, s1-s0, k_sel] pool ids
sel_c, cnt_c = self._expand_and_tail(
picks, rows.view(1, -1), positions[s0:s1].view(1, -1)
)
sel[s0:s1] = sel_c[0]
cnt[s0:s1] = cnt_c[0]
return sel.view(1, m, width), cnt.view(1, m)
__all__ = ["Glm5NextDSABackend"]