Skip to content

FreeToken brings datacenter-scale model serving to your desktop. Run massive models locally, fast and efficiently.

4.4 KB
Raw Blame
from __future__ import annotations

from typing import TYPE_CHECKING

import torch
from freetoken.attention import AttentionSpec
from freetoken.core import get_global_ctx
from freetoken.layers import BaseOP, GemmaRMSNorm, LinearColParallelMerged, LinearReplicated
from freetoken.layers.rotary import get_rope
from freetoken.models.config import SWAAttentionGroupConfig
from freetoken.utils import nvtx_annotate


if TYPE_CHECKING:
    from freetoken.models.config import ModelConfig


class MuseGlimmerAttention(BaseOP):
    """Gated GQA for one sliding-window or full-attention (NoPE) layer.

        q, k, v = split(qkv_proj(x))
        q = qk_norm(q); k = qk_norm(k)          # weightless per-head RMSNorm
        q, k = rope(q, k)                        # sliding layers only; full layers are NoPE
        attn = paged_attention(q, k, v)          # sm_scale carries qk_scale_factor/sqrt(d)
        out = o_proj(attn * sigmoid(gate_proj(x)))

    The reference multiplies q by ``qk_scale_factor`` before rope; rope is a rotation, so
    the factor commutes and is folded into ``AttentionSpec.sm_scale`` instead (parse_config
    sets ``attn_sm_scale = qk_scale_factor / sqrt(head_dim)``). The gate stays its own
    linear: the NVFP4 release quantizes it with a different global scale than q/k/v.
    """

    def __init__(self, config: ModelConfig, layer_id: int, *, prefix: str = ""):
        self.layer_id = layer_id
        group = config.attention_group_for_layer(layer_id)
        self.is_swa = isinstance(group, SWAAttentionGroupConfig)
        head_dim = group.head_dim
        self.head_dim = head_dim
        self.num_q = config.num_qo_heads
        self.num_kv = group.num_kv_heads
        self.qo_attn_dim = self.num_q * head_dim
        self.kv_attn_dim = self.num_kv * head_dim

        self._qkv_split = [self.qo_attn_dim, self.kv_attn_dim, self.kv_attn_dim]
        self.qkv_proj = LinearColParallelMerged(
            config.hidden_size, self._qkv_split, has_bias=False,
            quant_config=config.quant, prefix=f"{prefix}.qkv_proj",
        )
        self.gate_proj = LinearColParallelMerged(
            config.hidden_size, [self.qo_attn_dim], has_bias=False,
            quant_config=config.quant, prefix=f"{prefix}.gate_proj",
        )
        self.o_proj = LinearReplicated(
            self.qo_attn_dim, config.hidden_size, has_bias=False,
            quant_config=config.quant, prefix=f"{prefix}.o_proj",
        )
        # Weightless: the checkpoint carries no q/k norm weights; the runtime ones vector
        # is intentionally not part of state_dict.
        self.qk_norm = GemmaRMSNorm(head_dim, eps=config.rms_norm_eps, with_scale=False)
        self.attn_spec = AttentionSpec(
            sliding_window=group.sliding_window if self.is_swa else None,
            sm_scale=config.attn_sm_scale,
        )
        # base == 0.0 marks the NoPE full-attention layers; RotaryEmbedding must never
        # see it (0 ** x in the inv_freq computation).
        rotary_config = group.rotary_config
        self.rotary = (
            get_rope(
                head_dim=head_dim,
                rotary_dim=rotary_config.rotary_dim,
                max_position=rotary_config.max_position,
                base=rotary_config.base,
            )
            if rotary_config.base > 0.0
            else None
        )

    @nvtx_annotate("MHA")
    def forward(self, x: torch.Tensor) -> torch.Tensor:
        ctx = get_global_ctx()
        T = x.shape[0]

        gate = self.gate_proj.forward(x)
        qkv = self.qkv_proj.forward(x)
        q, k, v = torch.split(qkv, self._qkv_split, dim=-1)
        del qkv
        q = self.qk_norm.forward(q.contiguous().view(T, self.num_q, self.head_dim))
        k = self.qk_norm.forward(k.contiguous().view(T, self.num_kv, self.head_dim))
        v = v.contiguous()  # the split view has the qkv row stride; the KV store needs contiguous

        q = q.reshape(T, self.qo_attn_dim)
        k = k.reshape(T, self.kv_attn_dim)
        if self.rotary is not None:
            q, k = self.rotary.forward(ctx.batch.positions, q, k)

        o = ctx.attn_backend.forward(
            q.view(T, self.num_q, self.head_dim),
            k,
            v,
            self.layer_id,
            ctx.batch,
            attn_spec=self.attn_spec,
        )
        gated = o.reshape(T, self.qo_attn_dim) * torch.sigmoid(gate)
        return self.o_proj.forward(gated)


__all__ = ["MuseGlimmerAttention"]