FreeToken brings datacenter-scale model serving to your desktop. Run massive models locally, fast and efficiently.
"""glm5_next (GLM-5.3-Flash) config parsing: attention groups, kpool spec, aliases.
Runs off a trimmed copy of the real checkpoint config.json via RawConfigShim (the
exact object cached_load_hf_config falls back to when transformers doesn't know
``glm5_next`` yet), so the parse path under test is the one production hits.
"""
from __future__ import annotations
import pytest
from freetoken.attention.base import AttnType
from freetoken.models.config import (
FullAttentionGroupConfig,
LinearGatedDeltaGroupConfig,
)
from freetoken.models.glm5_next.args import load_args
from freetoken.models.glm5_next.config import parse_config
from freetoken.utils.hf import RawConfigShim
_NUM_LAYERS = 45
_DSA_IDS = tuple(range(3, _NUM_LAYERS, 4)) # 3, 7, ..., 43
_KDA_IDS = tuple(i for i in range(_NUM_LAYERS) if i not in _DSA_IDS)
def _layer_types() -> list[str]:
return [
"deepseek_sparse_attention" if i in _DSA_IDS else "linear_attention"
for i in range(_NUM_LAYERS)
]
def _text_config() -> dict:
# Trimmed from zai-org/GLM-5.3-Flash config.json (text_config).
return {
"hidden_size": 4096,
"intermediate_size": 12288,
"num_hidden_layers": _NUM_LAYERS,
"num_attention_heads": 64,
"num_key_value_heads": 64,
"vocab_size": 154880,
"hidden_act": "silu",
"rms_norm_eps": 1e-5,
"max_position_embeddings": 1048576,
"tie_word_embeddings": False,
# MLA (NoPE)
"q_lora_rank": 1536,
"kv_lora_rank": 512,
"qk_nope_head_dim": 256,
"qk_rope_head_dim": 0,
"qk_head_dim": 256,
"v_head_dim": 256,
"mla_use_nope": True,
# DSA indexer + kpool
"index_n_heads": 32,
"index_head_dim": 128,
"index_topk": 2048,
"indexer_types": ["full"] * _NUM_LAYERS,
"indexer_rope_interleave": True,
"index_kpool": 4,
"index_kpool_compress": True,
"index_kpool_always_select_tail": True,
# KDA
"linear_attn_config": {
"num_heads": 64,
"head_dim": 128,
"short_conv_kernel_size": 4,
"gate_lower_bound": -5.0,
"kda_layers": list(_KDA_IDS),
"full_attn_layers": list(_DSA_IDS),
},
# layout
"layer_types": _layer_types(),
"mlp_layer_types": ["dense"] * 3 + ["sparse"] * (_NUM_LAYERS - 3),
"first_k_dense_replace": 3,
# mHC (checkpoint spellings)
"mhc": True,
"hc_mult": 4,
"hc_eps": 1e-6,
"hc_sinkhorn_iters": 20,
# MoE
"n_routed_experts": 288,
"num_experts_per_tok": 8,
"n_shared_experts": 1,
"moe_intermediate_size": 2048,
"norm_topk_prob": True,
"routed_scaling_factor": 2.5,
"scoring_func": "sigmoid",
"topk_method": "noaux_tc",
"n_group": 1,
"topk_group": 1,
"swiglu_limit": 10.0,
"num_nextn_predict_layers": 1,
"attention_bias": False,
"model_type": "glm5_next_text",
}
def _hf_config(quantization_config: dict | None = None) -> RawConfigShim:
data: dict = {
"architectures": ["Glm5NextForConditionalGeneration"],
"model_type": "glm5_next",
"text_config": _text_config(),
"vision_config": {
"model_type": "glm5_next_vision", "depth": 24, "hidden_size": 1024, "num_heads": 16, "intermediate_size": 4096,
"projection_intermediate_size": 10240, "out_hidden_size": 4096, "in_channels": 3, "patch_size": 14,
"temporal_patch_size": 2, "spatial_merge_size": 2, "rms_norm_eps": 1e-5, "swiglu_limit": 10.0,
"attention_bias": True, "hidden_act": "silu",
},
"image_token_id": 154854,
}
if quantization_config is not None:
data["quantization_config"] = quantization_config
return RawConfigShim(data)
_CT_NVFP4_QUANT = {
# From RedHatAI/GLM-5.3-Flash-NVFP4 (llm-compressor, experts-only calibrated NVFP4).
"quant_method": "compressed-tensors",
"format": "nvfp4-pack-quantized",
"config_groups": {
"group_0": {
"targets": ["re:.*mlp\\.experts\\..*(gate|up|down)_proj$"],
"weights": {"num_bits": 4, "type": "float", "group_size": 16},
}
},
}
_CT_MIXED_QUANT = {
# From RedHatAI/GLM-5.3-Flash-NVFP4 as published: nvfp4 routed experts plus fp8 experts on the MTP layer, so the top-level format is "mixed-precision".
"quant_method": "compressed-tensors",
"format": "mixed-precision",
"config_groups": {
"group_0": {
"targets": ["re:.*\\.layers\\.(?:[3-9]|[1-3][0-9]|4[0-4])\\.mlp\\.experts\\..*(gate|up|down)_proj$"],
"weights": {"num_bits": 4, "type": "float", "group_size": 16, "strategy": "tensor_group"},
"format": "nvfp4-pack-quantized",
},
"group_1": {
"targets": ["re:.*\\.layers\\.45\\.mlp\\.experts\\.\\d+\\.(gate_proj|up_proj|down_proj)$"],
"weights": {"num_bits": 8, "type": "float", "strategy": "block"},
"format": "float-quantized",
},
},
}
_NVFP4_QUANT = {
# From LibertAIDAI/GLM-5.3-Flash-NVFP4 (ModelOpt weight-only NVFP4).
"quant_algo": "NVFP4",
"quant_method": "modelopt",
"config_groups": {
"group_0": {
"targets": ["Linear"],
"weights": {"num_bits": 4, "type": "float", "group_size": 16},
}
},
"ignore": ["lm_head", "model.visual.*"],
}
def test_attention_groups():
cfg = parse_config(_hf_config())
assert cfg.num_layers == _NUM_LAYERS
assert len(cfg.attention_groups) == 2
linear, full = cfg.attention_groups # ordered by first layer id (0 < 3)
assert isinstance(linear, LinearGatedDeltaGroupConfig)
assert linear.variant == "kda"
assert linear.layer_ids == _KDA_IDS
assert (linear.num_key_heads, linear.key_head_dim) == (64, 128)
assert (linear.num_value_heads, linear.value_head_dim) == (64, 128)
assert linear.conv_kernel_dim == 4
assert isinstance(full, FullAttentionGroupConfig)
assert full.layer_ids == _DSA_IDS
assert full.mla is True
assert full.head_dim == 512 # bare ckv latent: kv_lora_rank + 0 rope dims
assert full.num_kv_heads == 1
assert full.index_head_dim == 128
assert full.num_index_layers == len(_DSA_IDS)
assert full.index_ratio == 4
assert cfg.has_linear_attention and cfg.has_hybrid_attention
assert cfg.attn_type_for_layer(0) == AttnType.LINEAR
assert cfg.attn_type_for_layer(3) == AttnType.DSA
def test_kv_cache_group_specs_skip_linear_and_carry_kpool():
cfg = parse_config(_hf_config())
specs = [s for s in cfg.kv_cache_group_specs() if s.num_layers > 0]
# The linear group keeps recurrent state (no paged KV); exactly one paged spec.
assert len(specs) == 1
(spec,) = specs
assert spec.attn_type == AttnType.DSA
assert spec.layer_ids == _DSA_IDS
assert (spec.mla, spec.head_dim, spec.index_head_dim) == (True, 512, 128)
assert spec.index_ratio == 4
assert spec.num_index_layers == len(_DSA_IDS)
def test_moe_and_scalars():
cfg = parse_config(_hf_config(_NVFP4_QUANT))
assert cfg.expert_quant == "nvfp4"
# Only DERIVED facts are pinned here; fixture echoes (num_experts == 288
# and friends) assert nothing the parse could get wrong.
assert cfg.attn_sm_scale == pytest.approx(256**-0.5)
assert cfg.num_moe_layers == _NUM_LAYERS - 3
assert cfg.is_moe
# checkpoint-faithful: the resident projections follow the quantization_config, never a switch
assert (cfg.attn_quant, cfg.dense_quant, cfg.lm_head_quant) == ("none",) * 3
def test_vision_section_parses_into_the_tower_config():
cfg = parse_config(_hf_config())
# the merger lands in the text width, so the soft tokens scatter into the embeddings unprojected
assert cfg.vision_config.out_hidden_size == cfg.hidden_size
# a text-only engine hands the parser a config without the section
assert parse_config(RawConfigShim({**_hf_config().to_dict(), "vision_config": None})).vision_config is None
# the tower implements the clamped SwiGLU only
data = _hf_config().to_dict()
data["vision_config"]["hidden_act"] = "gelu"
with pytest.raises(NotImplementedError, match="silu"):
parse_config(RawConfigShim(data))
def test_args_alias_folding_and_nope():
args = load_args(_hf_config())
# Checkpoint spellings fold into the canonical fields.
assert args.mla_nope is True # from mla_use_nope
assert args.mhc_num_residual_streams == 4 # from hc_mult
assert args.mhc_sinkhorn_iterations == 20 # from hc_sinkhorn_iters
assert args.linear_num_heads == 64 # from nested linear_attn_config
assert args.linear_lower_bound == -5.0
# NoPE geometry.
assert args.qk_rope_head_dim == 0
assert args.qk_head_dim == 256
assert args.latent_dim == 512
assert args.kda_layer_ids == _KDA_IDS
assert args.dsa_layer_ids == _DSA_IDS
# Defaults for fields the checkpoint doesn't ship.
assert args.rope_theta == 10000.0
assert args.mhc_tau == 0.05
assert args.mhc_post_mult_value == 2.0
def test_registry_resolves_glm5_next():
from freetoken.models.register import get_model_spec
spec = get_model_spec("Glm5NextForConditionalGeneration")
assert spec.module == "freetoken.models.glm5_next"
assert spec.model_cls == "Glm5NextForConditionalGeneration"
assert get_model_spec("Glm5NextForCausalLM").module == spec.module
def test_dev_layer_cap(monkeypatch):
monkeypatch.setenv("FREETOKEN_GLM5_MAX_LAYERS", "5")
cfg = parse_config(_hf_config())
assert cfg.num_layers == 5
linear, full = cfg.attention_groups
assert linear.layer_ids == (0, 1, 2, 4)
assert full.layer_ids == (3,)
assert full.num_index_layers == 1
# 3 dense + 2 sparse under the cap.
assert cfg.first_k_dense_replace == 3
def test_rejects_unknown_layer_types():
data = _hf_config().to_dict()
data["text_config"]["layer_types"][0] = "full_attention"
with pytest.raises(ValueError, match="unsupported layer_types"):
load_args(RawConfigShim(data))
def test_compressed_tensors_nvfp4_detected():
"""RedHatAI/GLM-5.3-Flash-NVFP4 (llm-compressor): expert_quant resolves to
nvfp4 off the ``format`` field (quant_algo is absent for compressed-tensors)."""
cfg = parse_config(_hf_config(_CT_NVFP4_QUANT))
assert cfg.expert_quant == "nvfp4"
def test_compressed_tensors_mixed_precision_detected():
"""The published RedHatAI export says ``format: mixed-precision`` at the top and
``nvfp4-pack-quantized`` only inside the routed-expert group; expert_quant must
still resolve to nvfp4, not to the raw quant_method."""
cfg = parse_config(_hf_config(_CT_MIXED_QUANT))
assert cfg.expert_quant == "nvfp4"
def test_compressed_tensors_mixed_precision_reads_the_expert_group():
"""A mixed export with nvfp4 dense layers but fp8 experts must not report nvfp4
experts: the group that targets the experts decides."""
quant = {
"quant_method": "compressed-tensors",
"format": "mixed-precision",
"config_groups": {
"group_0": {
"targets": ["re:.*self_attn.*_proj$"],
"weights": {"num_bits": 4, "type": "float", "group_size": 16, "strategy": "tensor_group"},
"format": "nvfp4-pack-quantized",
},
"group_1": {
"targets": ["re:.*mlp\\.experts\\..*(gate|up|down)_proj$"],
"weights": {"num_bits": 8, "type": "float", "strategy": "block"},
"format": "float-quantized",
},
},
}
cfg = parse_config(_hf_config(quant))
assert cfg.expert_quant == "compressed-tensors"
def test_expert_source_spec_selection():
"""quant_method picks the bank source spec: compressed-tensors maps
weight_packed/weight_global_scale onto the canonical kinds with a reciprocal
global; modelopt stays identity."""
from freetoken.models.glm5_next.weight import (
_NVFP4_CT_SOURCE_SPEC,
_NVFP4_SOURCE_SPEC,
)
ct = _NVFP4_CT_SOURCE_SPEC
m = ct.key_pattern.match(
"model.language_model.layers.5.mlp.experts.7.gate_proj.weight_packed"
)
assert m and m.group("kind") == "weight_packed"
assert ct.kind_map["weight_packed"] == "weight"
assert ct.kind_map["weight_global_scale"] == "weight_scale_2"
assert ct.global_reciprocal
# W4A16 serving never consumes the calibrated activation scale.
assert ct.key_pattern.match(
"model.language_model.layers.5.mlp.experts.7.gate_proj.input_global_scale"
) is None
assert _NVFP4_SOURCE_SPEC.kind_map is None
assert not _NVFP4_SOURCE_SPEC.global_reciprocal
def test_ingest_global_reciprocal():
import torch
from freetoken.models.glm5_next.weight import _NVFP4_CT_SOURCE_SPEC, _NVFP4_SOURCE_SPEC
from freetoken.models.nvfp4_banks import _ingest_global
g = torch.tensor(4.0)
assert _ingest_global(_NVFP4_CT_SOURCE_SPEC, g).item() == 0.25
assert _ingest_global(_NVFP4_SOURCE_SPEC, g).item() == 4.0