Skip to content

FreeToken brings datacenter-scale model serving to your desktop. Run massive models locally, fast and efficiently.

9.8 KB
Raw Blame
"""Engine-facing config for GLM-5.3-Flash (``glm5_next``).

Two attention groups, ordered by first layer id:

* ``linear`` -- 34 KDA layers (``LinearGatedDeltaGroupConfig`` with
  ``variant="kda"``). KDA's state geometry coincides with GDN's (conv width
  ``2*K*dk + V*dv`` == q|k|v at ``H*d`` each; recurrent ``[H, d, d]``), so the
  existing ``LinearStatePool`` shapes serve it unchanged.
* ``full`` -- 11 DSA layers (``FullAttentionGroupConfig`` with ``mla=True`` and
  the indexer dims). The MLA is NoPE (``qk_rope_head_dim == 0``): the latent row
  is bare ckv (512), and the indexer K slab is kpool-compressed
  (``index_kpool=4`` -> tokens/4 stored entries; the spec's ``index_ratio``
  drives both the pool factory and the KV cost model).

MoE routing mirrors glm_moe_dsa (sigmoid ``noaux_tc``, shared expert, routed
scaling 2.5) with 288 routed experts and a clamped SwiGLU
(``swiglu_limit=10``). Everything model-specific beyond ``ModelConfig`` rides in
``glm5_args`` (``Glm5NextArgs``).

The checkpoint is served in the precision its quantization_config declares.
"""

from __future__ import annotations

import os
from dataclasses import dataclass
from typing import Any

from freetoken.models.config import (
    FullAttentionGroupConfig,
    LinearGatedDeltaGroupConfig,
    ModelConfig,
    RotaryConfig,
    detect_expert_quant,
)

from .args import load_args


@dataclass(frozen=True)
class VisionConfig:
    hidden_size: int
    depth: int
    num_heads: int
    intermediate_size: int
    projection_intermediate_size: int
    out_hidden_size: int
    in_channels: int
    patch_size: int
    temporal_patch_size: int
    spatial_merge_size: int
    rms_norm_eps: float
    swiglu_limit: float
    attention_bias: bool


def parse_vision_config(hf_config: Any) -> VisionConfig | None:
    """None when the config carries no vision section, which is how a text-only engine asks for no tower."""
    vc = getattr(hf_config, "vision_config", None)
    if vc is None:
        return None
    if vc.hidden_act != "silu":
        raise NotImplementedError(f"glm5_next vision tower activation {vc.hidden_act!r}; only silu is implemented")
    return VisionConfig(
        hidden_size=vc.hidden_size,
        depth=vc.depth,
        num_heads=vc.num_heads,
        intermediate_size=vc.intermediate_size,
        projection_intermediate_size=vc.projection_intermediate_size,
        out_hidden_size=vc.out_hidden_size,
        in_channels=vc.in_channels,
        patch_size=vc.patch_size,
        temporal_patch_size=vc.temporal_patch_size,
        spatial_merge_size=vc.spatial_merge_size,
        rms_norm_eps=vc.rms_norm_eps,
        swiglu_limit=vc.swiglu_limit,
        attention_bias=bool(vc.attention_bias),
    )


def _dsa_on(args, dsa_layer_ids) -> bool:
    """DSA serving switch, resolved ONCE into the attention-group spec (the pool
    factory, KV cost model, and backend read the spec, never the env)."""
    return (
        len(dsa_layer_ids) > 0
        and args.index_topk > 0
        and args.index_head_dim > 0
        and os.getenv("FREETOKEN_GLM5_DSA", "1") != "0"
    )


def _quant_accessor(hf_config: Any):
    """A ``get(key, default=None)`` accessor over the HF ``quantization_config`` (dict or
    object), or ``None`` when the model has no quant config."""
    quant = getattr(hf_config, "quantization_config", None)
    if quant is None:
        return None
    return quant.get if isinstance(quant, dict) else (lambda k, d=None: getattr(quant, k, d))


def _fp8_block_quant(hf_config: Any) -> tuple[str, tuple[int, int] | None]:
    """Detect DeepSeek-V3-style 128x128 block-fp8 from HF ``quantization_config``.

    Returns ``("fp8_block", (block_n, block_k))`` for a block-fp8 checkpoint (weights
    fp8-e4m3 + per-block ``weight_scale_inv``, dynamic activation), else ``("none", None)``.
    The quantization_config sits on the top-level hf_config (not ``text_config``).
    """
    get = _quant_accessor(hf_config)
    if get is None:
        return "none", None
    method = str(get("quant_method") or get("quant_algo") or "").lower()
    block = get("weight_block_size")
    if method == "fp8" and block:
        bs = tuple(int(x) for x in block)
        assert bs == (128, 128), f"only 128x128 block-fp8 is supported, got {bs}"
        return "fp8_block", bs
    return "none", None


def parse_config(hf_config: Any) -> ModelConfig:
    args = load_args(hf_config)
    text = getattr(hf_config, "text_config", hf_config)

    num_layers = len(args.layer_types)
    # Dev/testing only: cap the layer count so the forward path / KV / offload cache
    # can be exercised without the full ~175 GB of experts. Unset in normal use.
    _cap = os.environ.get("FREETOKEN_GLM5_MAX_LAYERS")
    if _cap:
        num_layers = min(num_layers, int(_cap))

    kda_ids = tuple(i for i in args.kda_layer_ids if i < num_layers)
    dsa_ids = tuple(i for i in args.dsa_layer_ids if i < num_layers)

    # NoPE: the main attention carries no rotary dims (rotary_dim == 0); rope
    # survives only in the indexer geometry (args.rope_theta / interleave).
    rotary_config = RotaryConfig(
        head_dim=args.qk_head_dim,
        rotary_dim=args.qk_rope_head_dim,
        max_position=args.max_position,
        base=args.rope_theta,
        scaling=None,
    )
    latent_dim = args.latent_dim  # 512: bare ckv, no kpe rows

    dsa_on = _dsa_on(args, dsa_ids)
    # Each DSA layer owns its indexer ("full" in indexer_types); count only the
    # layers that actually exist under a dev layer cap.
    num_index_layers = (
        sum(
            1
            for i in dsa_ids
            if i < len(args.indexer_types) and args.indexer_types[i] == "full"
        )
        if dsa_on
        else 0
    )

    linear_group = LinearGatedDeltaGroupConfig(
        name="linear",
        layer_ids=kda_ids,
        # KDA: one qk-sized and one v-sized head set (H=64, d=128 each). Mapping onto
        # the GDN field names keeps LinearStatePool's conv-width formula exact:
        # 2*K*dk + V*dv = (q|k) + v = 3 * H * d.
        num_key_heads=args.linear_num_heads,
        num_value_heads=args.linear_num_heads,
        key_head_dim=args.linear_head_dim,
        value_head_dim=args.linear_head_dim,
        conv_kernel_dim=args.linear_conv_kernel_dim,
        output_gate="sigmoid",  # KDA o_norm gates with sigmoid (kda.py)
        variant="kda",
    )
    full_group = FullAttentionGroupConfig(
        name="full",
        layer_ids=dsa_ids,
        num_kv_heads=1,  # single shared MLA latent
        head_dim=latent_dim,
        rotary_config=rotary_config,
        mla=True,
        index_head_dim=args.index_head_dim if dsa_on else 0,
        num_index_layers=num_index_layers,
        index_ratio=(args.index_kpool if dsa_on and args.index_kpool_compress else 1),
    )
    groups = tuple(
        sorted(
            (linear_group, full_group),
            key=lambda g: g.layer_ids[0] if g.layer_ids else 1 << 30,
        )
    )

    # The MLP layout is a dense prefix + sparse tail; ModelConfig models exactly that
    # via first_k_dense_replace, so assert the checkpoint matches before collapsing.
    mlp_types = args.mlp_layer_types[:num_layers]
    first_dense = next(
        (i for i, t in enumerate(mlp_types) if t == "sparse"), len(mlp_types)
    )
    assert all(t == "dense" for t in mlp_types[:first_dense]) and all(
        t == "sparse" for t in mlp_types[first_dense:]
    ), f"mlp_layer_types is not a dense-prefix layout: {mlp_types}"

    expert_quant, weight_block_size = _fp8_block_quant(hf_config)
    if expert_quant == "none":
        expert_quant = detect_expert_quant(hf_config)
    return ModelConfig(
        num_layers=num_layers,
        num_qo_heads=args.num_heads,
        num_kv_heads=1,
        head_dim=latent_dim,
        hidden_size=args.hidden_size,
        vocab_size=text.vocab_size,
        intermediate_size=text.intermediate_size,
        # hidden_act stands proxy for the EXPERT activation everywhere the engine
        # gates on it (NVFP4 backend selection, CPU-executor capability): GLM-5.3
        # experts and dense MLPs both run clamped SwiGLU when swiglu_limit is set,
        # even though the HF config still says "silu". Passing "silu" through would
        # let auto-selection repack experts into the marlin/b12x silu-only epilogue.
        hidden_act=(
            "swiglu_clamp" if args.swiglu_limit is not None else text.hidden_act
        ),
        rms_norm_eps=args.norm_eps,
        tie_word_embeddings=bool(getattr(text, "tie_word_embeddings", False)),
        rotary_config=rotary_config,
        attention_groups=groups,
        num_experts=(
            getattr(text, "n_routed_experts", None) or getattr(text, "num_experts", 0)
        ),
        num_experts_per_tok=(
            getattr(text, "num_experts_per_tok", None)
            or getattr(text, "num_experts_per_token", 0)
        ),
        moe_intermediate_size=getattr(text, "moe_intermediate_size", 0)
        or text.intermediate_size,
        norm_topk_prob=bool(getattr(text, "norm_topk_prob", True)),
        model_type=getattr(hf_config, "model_type", "glm5_next"),
        architectures=getattr(
            hf_config, "architectures", ["Glm5NextForConditionalGeneration"]
        ),
        moe_enabled=True,
        expert_quant=expert_quant,
        weight_block_size=weight_block_size,
        first_k_dense_replace=first_dense,
        n_shared_experts=int(getattr(text, "n_shared_experts", 0) or 0),
        routed_scaling_factor=float(getattr(text, "routed_scaling_factor", 1.0)),
        n_group=int(getattr(text, "n_group", 1) or 1),
        topk_group=int(getattr(text, "topk_group", 1) or 1),
        attn_sm_scale=args.qk_head_dim**-0.5,
        has_attn_bias=bool(getattr(text, "attention_bias", False)),
        swiglu_limit=args.swiglu_limit,
        vision_config=parse_vision_config(hf_config),
        image_token_id=getattr(hf_config, "image_token_id", None),
        glm5_args=args,
    )


__all__ = ["VisionConfig", "parse_config", "parse_vision_config"]