FreeToken brings datacenter-scale model serving to your desktop. Run massive models locally, fast and efficiently.
"""qwen4_exp.parse_config against synthetic configs shaped like the released checkpoints."""
from types import SimpleNamespace
import pytest
from freetoken.attention import AttnType
from freetoken.models.config import (
FullAttentionGroupConfig,
LinearGatedDeltaGroupConfig,
)
from freetoken.models.qwen4_exp.config import parse_config
from .common import LOVEDHEART_NVFP4_FP8, NVIDIA_NVFP4, QWEN_FP8, RADIXARK_NVFP4
def _text_config():
return SimpleNamespace(
num_hidden_layers=48,
hidden_size=2560,
vocab_size=248320,
head_dim=256,
num_attention_heads=24,
num_key_value_heads=2,
layer_types=[
"full_attention" if (i + 1) % 4 == 0 else "linear_attention" for i in range(48)
],
rope_parameters={
"rope_type": "default",
"rope_theta": 10000000,
"partial_rotary_factor": 0.25,
"mrope_interleaved": True,
"mrope_section": [11, 11, 10],
},
max_position_embeddings=262144,
rms_norm_eps=1e-6,
hidden_act="silu",
tie_word_embeddings=False,
num_experts=512,
num_experts_per_tok=10,
moe_intermediate_size=640,
shared_expert_intermediate_size=640,
linear_num_key_heads=16,
linear_num_value_heads=48,
linear_key_head_dim=128,
linear_value_head_dim=128,
linear_conv_kernel_dim=4,
output_gate_type="sigmoid",
indexer_n_heads=4,
indexer_kv_heads=1,
indexer_head_dim=128,
indexer_budget=2048,
indexer_compress_ratio=4,
hc_count=4,
hc_lowrank=320,
ple_layer_ids=[2],
ple_embed_dim=2560,
ple_conv_kernel_size=4,
ngram_size=3,
heads_per_ngram=8,
ngram_vocab_size_base=20000000,
make_ngram_vocab_size_divisible_by=128,
split_ngram_parts=128,
bos_token_id=248044,
eos_token_id=248044,
)
def _hf_config(quantization_config=RADIXARK_NVFP4):
return SimpleNamespace(
model_type="qwen4_exp",
architectures=["Qwen4ExpForConditionalGeneration"],
image_token_id=248056,
text_config=_text_config(),
quantization_config=quantization_config,
)
def test_groups_and_layer_split():
cfg = parse_config(_hf_config())
full = [g for g in cfg.attention_groups if isinstance(g, FullAttentionGroupConfig)]
linear = [g for g in cfg.attention_groups if isinstance(g, LinearGatedDeltaGroupConfig)]
assert len(full) == 1 and len(linear) == 1
assert full[0].layer_ids == tuple(range(3, 48, 4))
assert len(linear[0].layer_ids) == 36
assert full[0].index_head_dim == 128
assert full[0].num_index_layers == 12
assert full[0].index_ratio == 4
assert full[0].rotary_config.rotary_dim == 64
assert full[0].rotary_config.scaling is None
assert linear[0].output_gate == "sigmoid"
assert linear[0].num_key_heads == 16 and linear[0].num_value_heads == 48
def test_kv_specs_resolve_qsa():
cfg = parse_config(_hf_config())
specs = {s.name: s for s in cfg.kv_cache_group_specs()}
assert specs["full"].attn_type is AttnType.QSA
assert specs["full"].index_ratio == 4
assert cfg.attn_type_for_layer(3) is AttnType.QSA
assert cfg.attn_type_for_layer(0) is AttnType.LINEAR
assert cfg.has_linear_attention
def test_moe_dims():
cfg = parse_config(_hf_config())
assert cfg.num_experts == 512
assert cfg.num_experts_per_tok == 10
assert cfg.norm_topk_prob is True
assert cfg.moe_enabled
@pytest.mark.parametrize(
"quantization_config, expert_quant",
[
pytest.param(RADIXARK_NVFP4, "nvfp4", id="RadixArk/Qwen3.8-Flash-Next-NVFP4"),
pytest.param(NVIDIA_NVFP4, "nvfp4", id="nvidia/Qwen3.8-Flash-Next-NVFP4"),
pytest.param(QWEN_FP8, "fp8_block", id="Qwen/Qwen3.8-Flash-Next-FP8"),
pytest.param(None, "none", id="Qwen/Qwen3.8-Flash-Next"),
],
)
def test_released_checkpoints_expert_quant(quantization_config, expert_quant):
assert parse_config(_hf_config(quantization_config)).expert_quant == expert_quant
def test_qwen4_args_payload():
args = parse_config(_hf_config()).qwen4_args
assert args.ple_layer_ids == (1,)
assert args.hc_count == 4 and args.hc_lowrank == 320
assert args.index_topk_blocks == 512
assert args.num_ngram_heads == 16
assert args.ngram_head_dim == 160
assert args.ple_conv_state_len == 9
assert args.ple_state_width == 10240
assert args.ngram_boundary_token_id == 248044
def test_ple_on_full_attention_layer_rejected():
hf = _hf_config()
hf.text_config.ple_layer_ids = [4] # one-indexed 4 == zero-based 3, a full_attention layer
with pytest.raises(ValueError, match="linear_attention"):
parse_config(hf)
def test_output_gate_null_falls_back_to_hidden_act():
hf = _hf_config()
hf.text_config.output_gate_type = None
linear = [
g
for g in parse_config(hf).attention_groups
if isinstance(g, LinearGatedDeltaGroupConfig)
]
assert linear[0].output_gate == "silu"
def test_eos_token_id_list_uses_the_first_entry():
base = parse_config(_hf_config()).qwen4_args.ngram_boundary_token_id
hf = _hf_config()
hf.text_config.eos_token_id = [base, base + 1]
assert parse_config(hf).qwen4_args.ngram_boundary_token_id == base
def _vision_config():
return SimpleNamespace(
hidden_size=1152, depth=27, num_heads=16, intermediate_size=4304, patch_size=16, temporal_patch_size=2,
spatial_merge_size=2, num_position_embeddings=2304, out_hidden_size=2560, in_channels=3,
deepstack_visual_indexes=[],
)
def test_vision_turns_on_mrope_and_the_tower():
hf = _hf_config()
hf.vision_config = _vision_config()
config = parse_config(hf)
assert config.is_multimodal and config.model_is_mrope
assert config.rotary_config.mrope_section == [11, 11, 10]
assert config.rotary_config.mrope_layout == "interleaved" and config.rotary_config.rotary_dim == 64
assert config.vision_config.out_hidden_size == 2560 and config.vision_config.deepstack_visual_indexes == ()
assert config.qwen4_args.image_token_id == 248056
def test_text_only_keeps_the_1d_rope():
config = parse_config(_hf_config())
assert not config.is_multimodal and not config.model_is_mrope and config.rotary_config.mrope_section is None
# the merged-projection prefixes the model asks the QuantConfig about (attention.py / layers/linear_attn/gdn.py)
DENSE_PREFIXES = (
"model.layers.3.self_attn.qkv_proj", "model.layers.3.self_attn.o_proj",
"model.layers.0.linear_attn.in_proj_qkvz", "model.layers.0.linear_attn.out_proj",
)
BF16_PREFIXES = (
"model.layers.0.linear_attn.in_proj_ba", "model.layers.0.mlp.shared_expert.gate_up_proj",
"model.layers.3.self_attn.indexer.index_qk_proj", "lm_head",
)
def _quant(hf, tmp_path):
from freetoken.models.register import checkpoint_quant_config, get_model_spec
return checkpoint_quant_config(str(tmp_path), hf, get_model_spec(hf.architectures[0]))
def test_block_fp8_dense_schemes(tmp_path):
quant = _quant(_hf_config(LOVEDHEART_NVFP4_FP8), tmp_path)
for prefix in DENSE_PREFIXES:
scheme = quant.scheme_for(prefix)
assert str(scheme.kind) == "fp8_block" and scheme.has("weight_scale_inv"), prefix
for prefix in BF16_PREFIXES:
assert quant.scheme_for(prefix) is None, prefix
assert str(quant.scheme_for("model.layers.0.mlp.experts").kind) == "nvfp4"
assert parse_config(_hf_config(LOVEDHEART_NVFP4_FP8)).expert_quant == "nvfp4"
@pytest.mark.parametrize("quantization_config", [RADIXARK_NVFP4, NVIDIA_NVFP4, None], ids=["RadixArk", "nvidia", "bf16"])
def test_released_checkpoints_keep_the_dense_projections_bf16(quantization_config, tmp_path):
quant = _quant(_hf_config(quantization_config), tmp_path)
for prefix in DENSE_PREFIXES + BF16_PREFIXES:
assert quant.scheme_for(prefix) is None, prefix
# Qwen3.8-Flash-Next model card, "Processing Ultra-Long Texts": the 1M-token override.
MODEL_CARD_YARN = {
"mrope_interleaved": True,
"mrope_section": [11, 11, 10],
"rope_type": "yarn",
"rope_theta": 10000000,
"partial_rotary_factor": 0.25,
"factor": 4.0,
"original_max_position_embeddings": 262144,
}
def _write_checkpoint_config(path, **fields) -> None:
import json
text = {k: v for k, v in vars(_text_config()).items()}
config = {
"model_type": "qwen4_exp",
"architectures": ["Qwen4ExpForConditionalGeneration"],
"image_token_id": 248056,
"text_config": text,
"quantization_config": RADIXARK_NVFP4,
**fields,
}
(path / "config.json").write_text(json.dumps(config))
def test_the_engine_serves_the_model_card_1m_recipe(tmp_path):
import torch
from freetoken.distributed import DistributedInfo
from freetoken.engine.config import EngineConfig
from freetoken.mm.config import ENCODER_KINDS, MultimodalConfig
_write_checkpoint_config(tmp_path)
config = EngineConfig(
model_path=str(tmp_path),
tp_info=DistributedInfo(rank=0, size=1),
dtype=torch.bfloat16,
hf_overrides={"text_config": {"rope_parameters": MODEL_CARD_YARN}},
mm=MultimodalConfig(disabled_encoders=frozenset(ENCODER_KINDS)),
)
rotary = config.model_config.rotary_config
assert rotary.scaling["rope_type"] == "yarn" and rotary.scaling["factor"] == 4.0 and rotary.mrope_section is None
assert rotary.max_position == 262144 and rotary.rotary_dim == 64 and rotary.base == 10000000
assert config.max_seq_len == 4 * 262144
plain = EngineConfig(model_path=str(tmp_path), tp_info=DistributedInfo(rank=0, size=1), dtype=torch.bfloat16,
mm=MultimodalConfig(disabled_encoders=frozenset(ENCODER_KINDS)))
assert plain.model_config.rotary_config.scaling is None and plain.max_seq_len == 262144