Skip to content

FreeToken brings datacenter-scale model serving to your desktop. Run massive models locally, fast and efficiently.

9.3 KB
Raw Blame
"""GLM-5.3-Flash (``glm5_next``) hyperparameters.

GLM-5.3-Flash is the first hybrid GLM: 34 of 45 decoder layers run KDA linear
attention (Kimi-Delta-Attention: per-channel gated delta rule with separate q/k/v
short convolutions) and the remaining 11 run DeepSeek-sparse attention -- MLA
latent KV with a Lightning indexer whose K cache is *pool-compressed*
(``index_kpool`` tokens fold into one stored entry). The main attention is NoPE
(``qk_rope_head_dim == 0``, ``mla_use_nope``): no rotary embedding on Q/K at all;
positional information enters only through the indexer's pool-compression APE.
The residual stream is widened by mHC (Manifold-Constrained Hyper-Connections,
``hc_mult`` streams mixed by a Sinkhorn-projected matrix around every sublayer).

This payload carries everything the model module needs beyond the generic
``ModelConfig`` fields; it is stashed on ``ModelConfig.glm5_args`` (opaque to the
engine). Field spellings follow the checkpoint's ``config.json`` (transformers
5.16 ``Glm5NextTextConfig``); ``load_args`` folds the checkpoint aliases
(``hc_mult``, ``hc_sinkhorn_iters``, ``mla_use_nope``, the nested
``linear_attn_config`` dict) the same way vLLM's config class does, so a future
flattened checkpoint keeps loading.
"""

from __future__ import annotations

from dataclasses import dataclass
from typing import Any, Tuple

# Layer-type strings after transformers' legacy remap, which renames the checkpoint's deepseek_sparse_attention.
KDA_LAYER = "linear_attention"
DSA_LAYER = "indexed_attention"


@dataclass(frozen=True)
class Glm5NextArgs:
    hidden_size: int
    num_heads: int
    # ---- MLA (the 11 "indexed_attention" layers) ----
    q_lora_rank: int
    kv_lora_rank: int
    qk_nope_head_dim: int
    qk_rope_head_dim: int  # 0: NoPE -- main attention carries no rotary dims
    v_head_dim: int
    mla_nope: bool
    norm_eps: float
    max_position: int
    # ---- DSA indexer ----
    index_n_heads: int
    index_head_dim: int
    index_topk: int
    indexer_types: Tuple[str, ...]
    indexer_rope_interleave: bool
    # kpool compression: every ``index_kpool`` indexer-K entries pool into one
    # stored entry (softmax(gate + APE)-weighted sum); top-k selects pools
    # (select_k = index_topk // index_kpool) and ``always_select_tail``
    # force-includes the in-progress tail pool.
    index_kpool: int
    index_kpool_compress: bool
    index_kpool_always_select_tail: bool
    # ---- KDA linear attention (the 34 "linear_attention" layers) ----
    linear_num_heads: int
    linear_head_dim: int
    linear_conv_kernel_dim: int
    linear_lower_bound: float
    # ---- per-layer layout ----
    layer_types: Tuple[str, ...]
    mlp_layer_types: Tuple[str, ...]
    # ---- mHC (Manifold-Constrained Hyper-Connections) ----
    mhc: bool
    mhc_num_residual_streams: int
    hc_eps: float
    mhc_sinkhorn_iterations: int
    mhc_tau: float
    mhc_post_mult_value: float
    mhc_no_norm_weight: bool
    # ---- misc ----
    swiglu_limit: float | None
    rope_theta: float  # indexer-side only; the main attention is NoPE

    @property
    def qk_head_dim(self) -> int:
        return self.qk_nope_head_dim + self.qk_rope_head_dim

    @property
    def latent_dim(self) -> int:
        """Width of one MLA latent row in the paged pool: ckv | kpe."""
        return self.kv_lora_rank + self.qk_rope_head_dim

    @property
    def kda_layer_ids(self) -> Tuple[int, ...]:
        return tuple(i for i, t in enumerate(self.layer_types) if t == KDA_LAYER)

    @property
    def dsa_layer_ids(self) -> Tuple[int, ...]:
        return tuple(i for i, t in enumerate(self.layer_types) if t == DSA_LAYER)

    def is_kda_layer(self, layer_id: int) -> bool:
        return self.layer_types[layer_id] == KDA_LAYER


def _require_mhc(mhc: bool) -> bool:
    """The decoder layer wires the hyper-connection tensors unconditionally
    (model.py touches hc_attn_fn on every forward), so an mhc=False checkpoint
    would die with an AttributeError mid-forward -- refuse it at parse until a
    real one exists to implement against."""
    if not mhc:
        raise NotImplementedError(
            "glm5_next requires mhc=True (manifold-constrained hyper-connections); "
            "an mhc=False checkpoint has no implementation yet."
        )
    return mhc


def _get(cfg: Any, name: str, default: Any = None) -> Any:
    return getattr(cfg, name, default)


def load_args(hf_config: Any) -> Glm5NextArgs:
    """Build ``Glm5NextArgs`` from the checkpoint config (nested ``text_config`` or a
    flat text-only config). Raw checkpoint spellings win; the vLLM-normalized names
    are accepted as fallbacks."""
    text = _get(hf_config, "text_config", hf_config)

    from transformers.configuration_utils import remap_legacy_layer_types

    num_layers = int(text.num_hidden_layers)
    layer_types = tuple(remap_legacy_layer_types(list(_get(text, "layer_types", ()) or ())))
    if not layer_types:
        raise ValueError("glm5_next config is missing layer_types")
    if len(layer_types) != num_layers:
        raise ValueError(
            f"layer_types has {len(layer_types)} entries for {num_layers} layers"
        )
    unknown = sorted(set(layer_types) - {KDA_LAYER, DSA_LAYER})
    if unknown:
        raise ValueError(f"unsupported layer_types entries: {unknown}")

    mlp_layer_types = tuple(_get(text, "mlp_layer_types", ()) or ())
    if not mlp_layer_types:
        # Older-schema fallback (mirrors vLLM): derive from first_k_dense_replace.
        first_dense = int(_get(text, "first_k_dense_replace", 0) or 0)
        mlp_layer_types = ("dense",) * first_dense + ("sparse",) * (
            num_layers - first_dense
        )

    # Checkpoint alias folding (transformers 5.16 spellings first).
    mla_nope = _get(text, "mla_use_nope", _get(text, "mla_nope", False))
    hc_mult = _get(text, "hc_mult", _get(text, "mhc_num_residual_streams", 4))
    hc_sinkhorn = _get(
        text, "hc_sinkhorn_iters", _get(text, "mhc_sinkhorn_iterations", 20)
    )

    # KDA head geometry ships as the nested ``linear_attn_config`` dict; a future
    # flattened schema would carry vLLM-style ``linear_*`` top-level fields.
    linear_cfg = _get(text, "linear_attn_config", None)
    if linear_cfg is not None and not isinstance(linear_cfg, dict):
        linear_cfg = {
            k: getattr(linear_cfg, k)
            for k in (
                "num_heads",
                "head_dim",
                "short_conv_kernel_size",
                "gate_lower_bound",
            )
            if hasattr(linear_cfg, k)
        }
    linear_cfg = linear_cfg or {}
    linear_num_heads = int(
        linear_cfg.get("num_heads", _get(text, "linear_num_heads", 0))
    )
    linear_head_dim = int(linear_cfg.get("head_dim", _get(text, "linear_head_dim", 0)))
    linear_conv = int(
        linear_cfg.get(
            "short_conv_kernel_size", _get(text, "linear_conv_kernel_dim", 4)
        )
    )
    linear_lower_bound = float(
        linear_cfg.get("gate_lower_bound", _get(text, "linear_lower_bound", -5.0))
    )
    if KDA_LAYER in layer_types and (linear_num_heads <= 0 or linear_head_dim <= 0):
        raise ValueError("glm5_next config is missing the KDA linear_attn_config dims")

    # The main attention is NoPE; rope exists only on the indexer side. The
    # checkpoint ships no rope_theta -- fall back to the transformers default.
    rope = _get(text, "rope_parameters", None) or {}
    rope_theta = float(rope.get("rope_theta", _get(text, "rope_theta", 10000.0)))

    swiglu_limit = _get(text, "swiglu_limit", None)

    return Glm5NextArgs(
        hidden_size=int(text.hidden_size),
        num_heads=int(text.num_attention_heads),
        q_lora_rank=int(text.q_lora_rank),
        kv_lora_rank=int(text.kv_lora_rank),
        qk_nope_head_dim=int(text.qk_nope_head_dim),
        qk_rope_head_dim=int(text.qk_rope_head_dim),
        v_head_dim=int(text.v_head_dim),
        mla_nope=bool(mla_nope),
        norm_eps=float(text.rms_norm_eps),
        max_position=int(text.max_position_embeddings),
        index_n_heads=int(_get(text, "index_n_heads", 0) or 0),
        index_head_dim=int(_get(text, "index_head_dim", 0) or 0),
        index_topk=int(_get(text, "index_topk", 0) or 0),
        indexer_types=tuple(_get(text, "indexer_types", ()) or ()),
        indexer_rope_interleave=bool(_get(text, "indexer_rope_interleave", False)),
        index_kpool=int(_get(text, "index_kpool", 1) or 1),
        index_kpool_compress=bool(_get(text, "index_kpool_compress", False)),
        index_kpool_always_select_tail=bool(
            _get(text, "index_kpool_always_select_tail", False)
        ),
        linear_num_heads=linear_num_heads,
        linear_head_dim=linear_head_dim,
        linear_conv_kernel_dim=linear_conv,
        linear_lower_bound=linear_lower_bound,
        layer_types=layer_types,
        mlp_layer_types=mlp_layer_types,
        mhc=_require_mhc(bool(_get(text, "mhc", False))),
        mhc_num_residual_streams=int(hc_mult),
        hc_eps=float(_get(text, "hc_eps", 1e-6)),
        mhc_sinkhorn_iterations=int(hc_sinkhorn),
        mhc_tau=float(_get(text, "mhc_tau", 0.05)),
        mhc_post_mult_value=float(_get(text, "mhc_post_mult_value", 2.0)),
        mhc_no_norm_weight=bool(_get(text, "mhc_no_norm_weight", False)),
        swiglu_limit=(None if swiglu_limit is None else float(swiglu_limit)),
        rope_theta=rope_theta,
    )


__all__ = ["Glm5NextArgs", "load_args", "KDA_LAYER", "DSA_LAYER"]