Skip to content

FreeToken brings datacenter-scale model serving to your desktop. Run massive models locally, fast and efficiently.

9.2 KB
Raw Blame
"""GLM-5.3-Flash NoPE Multi-head Latent Attention with a kpool DSA indexer.

MLA weight-absorption as in glm_moe_dsa (kv_b absorbed into Q and onto the
output; the paged pool stores one latent row per token) with two GLM-5.3
differences:

* **NoPE** (``mla_use_nope``, ``qk_rope_head_dim == 0``): no rotary embedding
  anywhere in the main attention -- Q is all-nope [T, H, 256], the latent row is
  bare ckv (512, no kpe half). All rope plumbing degenerates to zero-width
  tensors, which the DSA backend's cat/scatter handle natively. Positional
  information enters ONLY through the indexer's pool-compression APE.
* **kpool indexer**: every DSA layer owns its indexer (no IndexShare). The
  indexer K cache stores one entry per ``index_kpool`` (4) tokens: a per-channel
  ``softmax(gate + ape)``-weighted sum of the raw keys. Scoring runs at pool
  granularity (select_k = index_topk / kpool) and the selected pools expand back
  to token rows, with the in-progress tail pool force-included
  (``index_kpool_always_select_tail``). This module owns the PROJECTIONS (wq_b /
  wk+k_norm / weights_proj / compress gate + APE); pooling, scoring, selection
  and the tail buffer live in the backend (attention/dsa_indexer_kpool.py).

Faithfulness note: the reference stack stores pooled entries as
Hadamard-rotated fp8 (a quantization device; the rotation cancels in the dot
product). FreeToken stores pooled entries in bf16 -- mathematically the same
score with strictly less quantization error -- matching the GLM-5.2 precedent
of bf16 indexer keys.
TODO: fp8 index slab (+ Hadamard rotation, upstream parity) to halve the slab bytes.
"""

from __future__ import annotations

from typing import TYPE_CHECKING

import torch
from freetoken.layers.quantization import QuantKind
from freetoken.core import get_global_ctx
from freetoken.layers import BaseOP, LinearReplicated, RMSNorm
# Shared with GLM-5.2 (weight.py imports privately from the same package).
from freetoken.models.glm_moe_dsa.attention import _IdxLayerNorm
from freetoken.utils import nvtx_annotate

if TYPE_CHECKING:
    from freetoken.models.config import ModelConfig


class Glm5NextIndexer(BaseOP):
    """kpool DSA indexer projections (every DSA layer owns one; no rope -- the
    checkpoint's NoPE geometry leaves ``qk_rope_head_dim == 0`` so position enters
    only via the compression APE).

    ``wq_b`` follows the checkpoint's quant config; ``wk`` and ``weights_proj`` stay bf16 (same reasoning as glm_moe_dsa's indexer).
    """

    def __init__(self, config: ModelConfig, layer_id: int, *, prefix: str = ""):
        args = config.glm5_args
        self.n_heads = args.index_n_heads
        self.head_dim = args.index_head_dim
        self.kpool = args.index_kpool
        self.wq_b = LinearReplicated(
            args.q_lora_rank, self.n_heads * self.head_dim, has_bias=False,
            quant_config=config.quant, prefix=f"{prefix}.wq_b",
        )
        self.wk = LinearReplicated(args.hidden_size, self.head_dim, has_bias=False)
        self.k_norm = _IdxLayerNorm(self.head_dim, eps=1e-6)
        self.weights_proj = LinearReplicated(
            args.hidden_size, self.n_heads, has_bias=False
        )
        # Pool-compression parameters (checkpoint names, no ".weight" suffix on
        # the gate: it is stored as a bare [head_dim, hidden] tensor).
        self.index_kpool_compress_gate = torch.empty(
            self.head_dim, args.hidden_size
        )
        # Per-pool-slot position bias, fp32 (models/weight.py exempts it from the
        # model-dtype downcast alongside A_log/dt_bias).
        self.index_kpool_compress_ape = torch.empty(
            self.kpool, self.head_dim, dtype=torch.float32
        )

    def compute(self, x: torch.Tensor, q_resid: torch.Tensor) -> "DSAIndexerInputs":
        """Per-token indexer projections as a typed inputs object: q [T, Hi, Di],
        k [T, Di], weights [T, Hi] fp32, plus the kpool gate scores [T, Di] and
        the [kpool, Di] APE parameter (passed per call; the backend keeps no copy)."""
        from freetoken.attention.dsa import DSAIndexerInputs

        t = x.shape[0]
        q = self.wq_b.forward(q_resid).view(t, self.n_heads, self.head_dim)
        k = self.k_norm.forward(self.wk.forward(x))
        w = self.weights_proj.forward(x).float() * (self.n_heads**-0.5)
        gate = torch.nn.functional.linear(x, self.index_kpool_compress_gate)
        return DSAIndexerInputs(
            q=q, k=k, w=w, gate=gate, ape=self.index_kpool_compress_ape
        )


class Glm5NextAttention(BaseOP):
    def __init__(self, config: ModelConfig, layer_id: int, *, prefix: str = ""):
        args = config.glm5_args
        self.layer_id = layer_id
        self.indexer = Glm5NextIndexer(config, layer_id, prefix=f"{prefix}.indexer")
        self.num_heads = args.num_heads
        self.qk_nope_head_dim = args.qk_nope_head_dim
        self.qk_rope_head_dim = args.qk_rope_head_dim  # 0 (NoPE)
        self.qk_head_dim = args.qk_head_dim
        self.v_head_dim = args.v_head_dim
        self.kv_lora_rank = args.kv_lora_rank
        assert args.qk_rope_head_dim == 0 and args.mla_nope, (
            "glm5_next attention implements the NoPE geometry; a roped variant "
            "would need the glm_moe_dsa rope plumbing back"
        )

        self.q_a_proj = LinearReplicated(
            args.hidden_size, args.q_lora_rank, has_bias=False,
            quant_config=config.quant, prefix=f"{prefix}.q_a_proj",
        )
        self.q_a_layernorm = RMSNorm(args.q_lora_rank, eps=args.norm_eps)
        self.q_b_proj = LinearReplicated(
            args.q_lora_rank, self.num_heads * self.qk_head_dim, has_bias=False,
            quant_config=config.quant, prefix=f"{prefix}.q_b_proj",
        )
        # NoPE: kv_a projects to bare ckv (no +qk_rope_head_dim rows).
        self.kv_a_proj_with_mqa = LinearReplicated(
            args.hidden_size, self.kv_lora_rank, has_bias=False,
            quant_config=config.quant, prefix=f"{prefix}.kv_a_proj_with_mqa",
        )
        self.kv_a_layernorm = RMSNorm(self.kv_lora_rank, eps=args.norm_eps)
        # the MLA absorption reads kv_b_proj.weight as bmm operands, so only an unquantized scheme is served
        self.kv_b_proj = LinearReplicated(
            self.kv_lora_rank,
            self.num_heads * (self.qk_nope_head_dim + self.v_head_dim),
            has_bias=False,
            quant_config=config.quant, prefix=f"{prefix}.kv_b_proj",
        )
        assert self.kv_b_proj.quant_method.kind is QuantKind.NONE, f"{prefix}.kv_b_proj: a quantized kv_b_proj needs a dequantized copy for the MLA absorption"
        self.o_proj = LinearReplicated(
            self.num_heads * self.v_head_dim, args.hidden_size, has_bias=False,
            quant_config=config.quant, prefix=f"{prefix}.o_proj",
        )
        self._w_uk: torch.Tensor | None = None
        self._w_uv: torch.Tensor | None = None

    def _kv_b(self) -> tuple[torch.Tensor, torch.Tensor]:
        """Per-head kv_b split in bmm-ready bf16 layout (same contract and
        prepare_for_runtime budgeting as glm_moe_dsa; see that module)."""
        if self._w_uk is None:
            w = self.kv_b_proj.weight.view(
                self.num_heads,
                self.qk_nope_head_dim + self.v_head_dim,
                self.kv_lora_rank,
            )
            self._w_uk = w[:, : self.qk_nope_head_dim, :].contiguous()
            self._w_uv = w[:, self.qk_nope_head_dim :, :].transpose(1, 2).contiguous()
        return self._w_uk, self._w_uv

    def prepare_for_runtime(self) -> None:
        self._kv_b()
        self.kv_b_proj.weight = None  # checkpoint layout freed; repacked forms serve

    @nvtx_annotate("MLA")
    def forward(self, x: torch.Tensor) -> torch.Tensor:
        ctx = get_global_ctx()
        t = x.shape[0]
        w_uk, w_uv = self._kv_b()

        q_a_resid = self.q_a_layernorm.forward(self.q_a_proj.forward(x))
        q = self.q_b_proj.forward(q_a_resid)
        # NoPE: the whole head is the nope part; no rope split, no rope kernel.
        q_nope = q.view(t, self.num_heads, self.qk_head_dim)

        c_kv = self.kv_a_layernorm.forward(self.kv_a_proj_with_mqa.forward(x))

        indexer_inputs = (
            self.indexer.compute(x, q_a_resid)
            if getattr(ctx.attn_backend, "dsa_enabled", False)
            else None
        )

        # Absorb kv_b's k-part into the query: q_nope[H,T,nope] @ W_uk[H,nope,lora].
        # The [H,T,lora] output serves downstream as a transposed view; the dels free
        # the q_b_proj output, the next-widest transient, before attention runs.
        q_absorbed = torch.bmm(q_nope.transpose(0, 1), w_uk)
        del q, q_nope, q_a_resid

        # Zero-width rope halves: cat/scatter no-ops in the backend.
        q_pe = q_absorbed.new_empty(t, self.num_heads, 0)
        k_rope = q_absorbed.new_empty(t, 0)
        o_latent = ctx.attn_backend.mla_forward(
            q_absorbed.transpose(0, 1), q_pe, c_kv, k_rope,
            self.layer_id, ctx.batch, indexer_inputs=indexer_inputs,
        )  # [T, H, kv_lora_rank]
        del q_absorbed

        # Absorb kv_b's v-part onto the output: o_latent[H,T,lora] @ W_uv_t[H,lora,v].
        o = torch.bmm(o_latent.transpose(0, 1), w_uv)
        del o_latent
        return self.o_proj.forward(o.transpose(0, 1).reshape(t, self.num_heads * self.v_head_dim))


__all__ = ["Glm5NextAttention", "Glm5NextIndexer"]