Skip to content

FreeToken brings datacenter-scale model serving to your desktop. Run massive models locally, fast and efficiently.

8.2 KB
Raw Blame
"""GLM-5.3-Flash (glm5_next) model: hybrid KDA/DSA decoder with mHC residual streams.

Layer layout comes from the checkpoint's ``layer_types`` (34 KDA linear-attention
layers, 11 NoPE-MLA/DSA layers at 3:1) and ``mlp_layer_types`` (3 dense + 42 MoE).
The residual stream is mHC-widened to ``hc_mult`` (4) parallel streams:

    layer 0:  residual = hc_expand(x);  (post, comb, x) = mhc_pre(residual, hc_attn_*)
    each sublayer boundary fuses the previous hc_post with the next hc_pre
    (mhc_fused_post_pre), and the sublayer input is RMS-normed AFTER the mix
    (the reference fuses the norm into its hc kernels; decomposed here, same math).
    last layer: x = mhc_post(...); x = hc_contract(x)  -> final norm -> lm_head.

The deferred (post, comb) pair threads through the layer loop exactly like
glm_moe_dsa's (x, residual) pair. lm_head quant mirrors glm_moe_dsa (optional
W8A16 fp8 at load; the ~1.2 GiB bf16 head is read every decode step).
"""

from __future__ import annotations

from typing import TYPE_CHECKING, Tuple

import torch
from freetoken.core import get_global_ctx
from freetoken.layers import (
    BaseOP,
    OPList,
    ParallelLMHead,
    RMSNorm,
    VocabParallelEmbedding,
)
from freetoken.layers.mhc import hc_contract, hc_expand, mhc_fused_post_pre, mhc_post, mhc_pre
from freetoken.models.blocks import BaseLLMModel, embed_input_ids
from freetoken.utils import nvtx_annotate

from .attention import Glm5NextAttention
from .kda import Glm5NextKDA
from .mlp import Glm5NextGatedMLP
from .moe import Glm5NextSparseBlock
from .vision import Glm5NextVisionModel

if TYPE_CHECKING:
    from freetoken.message import MMItem
    from freetoken.models.config import ModelConfig


class Glm5NextDecoderLayer(BaseOP):
    def __init__(self, config: ModelConfig, layer_id: int, *, prefix: str = ""):
        args = config.glm5_args
        self._layer_id = layer_id
        self._is_last = layer_id == config.num_layers - 1
        self.mhc = args.mhc
        self._n = args.mhc_num_residual_streams
        self._hc_eps = args.hc_eps
        self._rms_eps = args.norm_eps
        self._post_mult = args.mhc_post_mult_value
        self._sinkhorn = args.mhc_sinkhorn_iterations

        if args.is_kda_layer(layer_id):
            self.self_attn: BaseOP = Glm5NextKDA(config, layer_id, prefix=f"{prefix}.self_attn")
        else:
            self.self_attn = Glm5NextAttention(config, layer_id, prefix=f"{prefix}.self_attn")
        if layer_id >= config.first_k_dense_replace:
            self.mlp: BaseOP = Glm5NextSparseBlock(config, layer_id, prefix=f"{prefix}.mlp")
        else:
            self.mlp = Glm5NextGatedMLP(
                config.hidden_size, config.intermediate_size, swiglu_limit=config.swiglu_limit,
                quant_config=config.quant, prefix=f"{prefix}.mlp",
            )
        self.input_layernorm = RMSNorm(size=config.hidden_size, eps=args.norm_eps)
        self.post_attention_layernorm = RMSNorm(size=config.hidden_size, eps=args.norm_eps)

        if self.mhc:
            n, hidden = self._n, config.hidden_size
            mix = 2 * n + n * n
            # fp32 mHC weights (models/weight.py exempts hc_* from the dtype downcast).
            self.hc_attn_fn = torch.empty(mix, n * hidden, dtype=torch.float32)
            self.hc_attn_base = torch.empty(mix, dtype=torch.float32)
            self.hc_attn_scale = torch.empty(3, dtype=torch.float32)
            self.hc_ffn_fn = torch.empty(mix, n * hidden, dtype=torch.float32)
            self.hc_ffn_base = torch.empty(mix, dtype=torch.float32)
            self.hc_ffn_scale = torch.empty(3, dtype=torch.float32)

    def _pre(self, residual, fn, scale, base):
        # Layer 0's standalone pre rides the fused kernel too (HAS_POST=False
        # path; x/post/comb are the no-post sentinels) -- same dispatch, same
        # numerics, and the kernel wins at every batch size (see layers/mhc.py).
        if residual.is_cuda:
            _, post, comb, x = mhc_fused_post_pre(
                residual.new_empty(residual.shape[0], residual.shape[-1]),
                residual, None, None, fn, scale, base,
                self._rms_eps, self._hc_eps, self._post_mult, self._sinkhorn,
            )
            return post, comb, x
        return mhc_pre(
            residual, fn, scale, base,
            self._rms_eps, self._hc_eps, self._post_mult, self._sinkhorn,
        )

    def _fused(self, x, residual, post, comb, fn, scale, base):
        return mhc_fused_post_pre(
            x, residual, post, comb, fn, scale, base,
            self._rms_eps, self._hc_eps, self._post_mult, self._sinkhorn,
        )

    @nvtx_annotate("Layer_{}", layer_id_field="_layer_id")
    def forward(
        self,
        x: torch.Tensor,
        residual: torch.Tensor | None,
        post: torch.Tensor | None,
        comb: torch.Tensor | None,
    ) -> Tuple[torch.Tensor, torch.Tensor | None, torch.Tensor | None, torch.Tensor | None]:
        if post is None:
            if residual is None:
                residual = hc_expand(x, self._n)
            post, comb, x = self._pre(
                residual, self.hc_attn_fn, self.hc_attn_scale, self.hc_attn_base
            )
        else:
            residual, post, comb, x = self._fused(
                x, residual, post, comb,
                self.hc_attn_fn, self.hc_attn_scale, self.hc_attn_base,
            )
        x = self.input_layernorm.forward(x)
        x = self.self_attn.forward(x)

        residual, post, comb, x = self._fused(
            x, residual, post, comb,
            self.hc_ffn_fn, self.hc_ffn_scale, self.hc_ffn_base,
        )
        x = self.post_attention_layernorm.forward(x)
        x = self.mlp.forward(x)

        if self._is_last:
            x = mhc_post(x, residual, post, comb)
            return hc_contract(x), None, None, None
        return x, residual, post, comb


class Glm5NextModel(BaseOP):
    def __init__(self, config: ModelConfig, *, prefix: str = "model"):
        self.embed_tokens = VocabParallelEmbedding(
            num_embeddings=config.vocab_size,
            embedding_dim=config.hidden_size,
            quant_config=config.quant,
            prefix=f"{prefix}.embed_tokens",
        )
        self.layers = OPList(
            [Glm5NextDecoderLayer(config, i, prefix=f"{prefix}.layers.{i}") for i in range(config.num_layers)]
        )
        self.norm = RMSNorm(size=config.hidden_size, eps=config.rms_norm_eps)

    def forward(self, input_ids: torch.Tensor) -> torch.Tensor:
        x = embed_input_ids(self.embed_tokens, input_ids, get_global_ctx().batch)
        residual = post = comb = None
        for layer in self.layers.op_list:
            x, residual, post, comb = layer.forward(x, residual, post, comb)
        return self.norm.forward(x)


class Glm5NextForCausalLM(BaseLLMModel):
    def __init__(self, config: ModelConfig):
        self._config = config
        self.model = Glm5NextModel(config)
        self.lm_head = ParallelLMHead(
            num_embeddings=config.vocab_size,
            embedding_dim=config.hidden_size,
            tie_word_embeddings=config.tie_word_embeddings,
            tied_embedding=self.model.embed_tokens if config.tie_word_embeddings else None,
            quant_config=config.quant,
            prefix="lm_head",
        )

    def prepare_for_runtime(self) -> None:
        """Post-load, pre-KV-sizing hook: materialize the DSA layers' bmm-ready
        kv_b splits and free the checkpoint-layout originals (glm_moe_dsa
        precedent)."""
        for layer in self.model.layers.op_list:
            if isinstance(layer.self_attn, Glm5NextAttention):
                layer.self_attn.prepare_for_runtime()
        torch.cuda.empty_cache()

    def forward(self) -> torch.Tensor:
        output = self.model.forward(get_global_ctx().batch.input_ids)
        return self.lm_head.forward(output)


class Glm5NextForConditionalGeneration(Glm5NextForCausalLM):
    def __init__(self, config: ModelConfig):
        super().__init__(config)
        if config.is_multimodal:
            self.visual = Glm5NextVisionModel(config.vision_config)

    def place_encoder_weights(self, mode: str) -> None:
        self.visual.place_weights(mode)

    def encode(self, item: MMItem) -> torch.Tensor:
        return self.visual.forward(item.feature, [item.grid_thw])


__all__ = ["Glm5NextForCausalLM", "Glm5NextForConditionalGeneration"]