FreeToken brings datacenter-scale model serving to your desktop. Run massive models locally, fast and efficiently.
"""Engine-facing config for GLM-5.3-Flash (``glm5_next``).
Two attention groups, ordered by first layer id:
* ``linear`` -- 34 KDA layers (``LinearGatedDeltaGroupConfig`` with
``variant="kda"``). KDA's state geometry coincides with GDN's (conv width
``2*K*dk + V*dv`` == q|k|v at ``H*d`` each; recurrent ``[H, d, d]``), so the
existing ``LinearStatePool`` shapes serve it unchanged.
* ``full`` -- 11 DSA layers (``FullAttentionGroupConfig`` with ``mla=True`` and
the indexer dims). The MLA is NoPE (``qk_rope_head_dim == 0``): the latent row
is bare ckv (512), and the indexer K slab is kpool-compressed
(``index_kpool=4`` -> tokens/4 stored entries; the spec's ``index_ratio``
drives both the pool factory and the KV cost model).
MoE routing mirrors glm_moe_dsa (sigmoid ``noaux_tc``, shared expert, routed
scaling 2.5) with 288 routed experts and a clamped SwiGLU
(``swiglu_limit=10``). Everything model-specific beyond ``ModelConfig`` rides in
``glm5_args`` (``Glm5NextArgs``).
The checkpoint is served in the precision its quantization_config declares.
"""
from __future__ import annotations
import os
from dataclasses import dataclass
from typing import Any
from freetoken.models.config import (
FullAttentionGroupConfig,
LinearGatedDeltaGroupConfig,
ModelConfig,
RotaryConfig,
detect_expert_quant,
)
from .args import load_args
@dataclass(frozen=True)
class VisionConfig:
hidden_size: int
depth: int
num_heads: int
intermediate_size: int
projection_intermediate_size: int
out_hidden_size: int
in_channels: int
patch_size: int
temporal_patch_size: int
spatial_merge_size: int
rms_norm_eps: float
swiglu_limit: float
attention_bias: bool
def parse_vision_config(hf_config: Any) -> VisionConfig | None:
"""None when the config carries no vision section, which is how a text-only engine asks for no tower."""
vc = getattr(hf_config, "vision_config", None)
if vc is None:
return None
if vc.hidden_act != "silu":
raise NotImplementedError(f"glm5_next vision tower activation {vc.hidden_act!r}; only silu is implemented")
return VisionConfig(
hidden_size=vc.hidden_size,
depth=vc.depth,
num_heads=vc.num_heads,
intermediate_size=vc.intermediate_size,
projection_intermediate_size=vc.projection_intermediate_size,
out_hidden_size=vc.out_hidden_size,
in_channels=vc.in_channels,
patch_size=vc.patch_size,
temporal_patch_size=vc.temporal_patch_size,
spatial_merge_size=vc.spatial_merge_size,
rms_norm_eps=vc.rms_norm_eps,
swiglu_limit=vc.swiglu_limit,
attention_bias=bool(vc.attention_bias),
)
def _dsa_on(args, dsa_layer_ids) -> bool:
"""DSA serving switch, resolved ONCE into the attention-group spec (the pool
factory, KV cost model, and backend read the spec, never the env)."""
return (
len(dsa_layer_ids) > 0
and args.index_topk > 0
and args.index_head_dim > 0
and os.getenv("FREETOKEN_GLM5_DSA", "1") != "0"
)
def _quant_accessor(hf_config: Any):
"""A ``get(key, default=None)`` accessor over the HF ``quantization_config`` (dict or
object), or ``None`` when the model has no quant config."""
quant = getattr(hf_config, "quantization_config", None)
if quant is None:
return None
return quant.get if isinstance(quant, dict) else (lambda k, d=None: getattr(quant, k, d))
def _fp8_block_quant(hf_config: Any) -> tuple[str, tuple[int, int] | None]:
"""Detect DeepSeek-V3-style 128x128 block-fp8 from HF ``quantization_config``.
Returns ``("fp8_block", (block_n, block_k))`` for a block-fp8 checkpoint (weights
fp8-e4m3 + per-block ``weight_scale_inv``, dynamic activation), else ``("none", None)``.
The quantization_config sits on the top-level hf_config (not ``text_config``).
"""
get = _quant_accessor(hf_config)
if get is None:
return "none", None
method = str(get("quant_method") or get("quant_algo") or "").lower()
block = get("weight_block_size")
if method == "fp8" and block:
bs = tuple(int(x) for x in block)
assert bs == (128, 128), f"only 128x128 block-fp8 is supported, got {bs}"
return "fp8_block", bs
return "none", None
def parse_config(hf_config: Any) -> ModelConfig:
args = load_args(hf_config)
text = getattr(hf_config, "text_config", hf_config)
num_layers = len(args.layer_types)
# Dev/testing only: cap the layer count so the forward path / KV / offload cache
# can be exercised without the full ~175 GB of experts. Unset in normal use.
_cap = os.environ.get("FREETOKEN_GLM5_MAX_LAYERS")
if _cap:
num_layers = min(num_layers, int(_cap))
kda_ids = tuple(i for i in args.kda_layer_ids if i < num_layers)
dsa_ids = tuple(i for i in args.dsa_layer_ids if i < num_layers)
# NoPE: the main attention carries no rotary dims (rotary_dim == 0); rope
# survives only in the indexer geometry (args.rope_theta / interleave).
rotary_config = RotaryConfig(
head_dim=args.qk_head_dim,
rotary_dim=args.qk_rope_head_dim,
max_position=args.max_position,
base=args.rope_theta,
scaling=None,
)
latent_dim = args.latent_dim # 512: bare ckv, no kpe rows
dsa_on = _dsa_on(args, dsa_ids)
# Each DSA layer owns its indexer ("full" in indexer_types); count only the
# layers that actually exist under a dev layer cap.
num_index_layers = (
sum(
1
for i in dsa_ids
if i < len(args.indexer_types) and args.indexer_types[i] == "full"
)
if dsa_on
else 0
)
linear_group = LinearGatedDeltaGroupConfig(
name="linear",
layer_ids=kda_ids,
# KDA: one qk-sized and one v-sized head set (H=64, d=128 each). Mapping onto
# the GDN field names keeps LinearStatePool's conv-width formula exact:
# 2*K*dk + V*dv = (q|k) + v = 3 * H * d.
num_key_heads=args.linear_num_heads,
num_value_heads=args.linear_num_heads,
key_head_dim=args.linear_head_dim,
value_head_dim=args.linear_head_dim,
conv_kernel_dim=args.linear_conv_kernel_dim,
output_gate="sigmoid", # KDA o_norm gates with sigmoid (kda.py)
variant="kda",
)
full_group = FullAttentionGroupConfig(
name="full",
layer_ids=dsa_ids,
num_kv_heads=1, # single shared MLA latent
head_dim=latent_dim,
rotary_config=rotary_config,
mla=True,
index_head_dim=args.index_head_dim if dsa_on else 0,
num_index_layers=num_index_layers,
index_ratio=(args.index_kpool if dsa_on and args.index_kpool_compress else 1),
)
groups = tuple(
sorted(
(linear_group, full_group),
key=lambda g: g.layer_ids[0] if g.layer_ids else 1 << 30,
)
)
# The MLP layout is a dense prefix + sparse tail; ModelConfig models exactly that
# via first_k_dense_replace, so assert the checkpoint matches before collapsing.
mlp_types = args.mlp_layer_types[:num_layers]
first_dense = next(
(i for i, t in enumerate(mlp_types) if t == "sparse"), len(mlp_types)
)
assert all(t == "dense" for t in mlp_types[:first_dense]) and all(
t == "sparse" for t in mlp_types[first_dense:]
), f"mlp_layer_types is not a dense-prefix layout: {mlp_types}"
expert_quant, weight_block_size = _fp8_block_quant(hf_config)
if expert_quant == "none":
expert_quant = detect_expert_quant(hf_config)
return ModelConfig(
num_layers=num_layers,
num_qo_heads=args.num_heads,
num_kv_heads=1,
head_dim=latent_dim,
hidden_size=args.hidden_size,
vocab_size=text.vocab_size,
intermediate_size=text.intermediate_size,
# hidden_act stands proxy for the EXPERT activation everywhere the engine
# gates on it (NVFP4 backend selection, CPU-executor capability): GLM-5.3
# experts and dense MLPs both run clamped SwiGLU when swiglu_limit is set,
# even though the HF config still says "silu". Passing "silu" through would
# let auto-selection repack experts into the marlin/b12x silu-only epilogue.
hidden_act=(
"swiglu_clamp" if args.swiglu_limit is not None else text.hidden_act
),
rms_norm_eps=args.norm_eps,
tie_word_embeddings=bool(getattr(text, "tie_word_embeddings", False)),
rotary_config=rotary_config,
attention_groups=groups,
num_experts=(
getattr(text, "n_routed_experts", None) or getattr(text, "num_experts", 0)
),
num_experts_per_tok=(
getattr(text, "num_experts_per_tok", None)
or getattr(text, "num_experts_per_token", 0)
),
moe_intermediate_size=getattr(text, "moe_intermediate_size", 0)
or text.intermediate_size,
norm_topk_prob=bool(getattr(text, "norm_topk_prob", True)),
model_type=getattr(hf_config, "model_type", "glm5_next"),
architectures=getattr(
hf_config, "architectures", ["Glm5NextForConditionalGeneration"]
),
moe_enabled=True,
expert_quant=expert_quant,
weight_block_size=weight_block_size,
first_k_dense_replace=first_dense,
n_shared_experts=int(getattr(text, "n_shared_experts", 0) or 0),
routed_scaling_factor=float(getattr(text, "routed_scaling_factor", 1.0)),
n_group=int(getattr(text, "n_group", 1) or 1),
topk_group=int(getattr(text, "topk_group", 1) or 1),
attn_sm_scale=args.qk_head_dim**-0.5,
has_attn_bias=bool(getattr(text, "attention_bias", False)),
swiglu_limit=args.swiglu_limit,
vision_config=parse_vision_config(hf_config),
image_token_id=getattr(hf_config, "image_token_id", None),
glm5_args=args,
)
__all__ = ["VisionConfig", "parse_config", "parse_vision_config"]