Skip to content

FreeToken brings datacenter-scale model serving to your desktop. Run massive models locally, fast and efficiently.

9.8 KB
Raw Blame
"""qwen4_exp.parse_config against synthetic configs shaped like the released checkpoints."""

from types import SimpleNamespace

import pytest

from freetoken.attention import AttnType
from freetoken.models.config import (
    FullAttentionGroupConfig,
    LinearGatedDeltaGroupConfig,
)
from freetoken.models.qwen4_exp.config import parse_config

from .common import LOVEDHEART_NVFP4_FP8, NVIDIA_NVFP4, QWEN_FP8, RADIXARK_NVFP4


def _text_config():
    return SimpleNamespace(
        num_hidden_layers=48,
        hidden_size=2560,
        vocab_size=248320,
        head_dim=256,
        num_attention_heads=24,
        num_key_value_heads=2,
        layer_types=[
            "full_attention" if (i + 1) % 4 == 0 else "linear_attention" for i in range(48)
        ],
        rope_parameters={
            "rope_type": "default",
            "rope_theta": 10000000,
            "partial_rotary_factor": 0.25,
            "mrope_interleaved": True,
            "mrope_section": [11, 11, 10],
        },
        max_position_embeddings=262144,
        rms_norm_eps=1e-6,
        hidden_act="silu",
        tie_word_embeddings=False,
        num_experts=512,
        num_experts_per_tok=10,
        moe_intermediate_size=640,
        shared_expert_intermediate_size=640,
        linear_num_key_heads=16,
        linear_num_value_heads=48,
        linear_key_head_dim=128,
        linear_value_head_dim=128,
        linear_conv_kernel_dim=4,
        output_gate_type="sigmoid",
        indexer_n_heads=4,
        indexer_kv_heads=1,
        indexer_head_dim=128,
        indexer_budget=2048,
        indexer_compress_ratio=4,
        hc_count=4,
        hc_lowrank=320,
        ple_layer_ids=[2],
        ple_embed_dim=2560,
        ple_conv_kernel_size=4,
        ngram_size=3,
        heads_per_ngram=8,
        ngram_vocab_size_base=20000000,
        make_ngram_vocab_size_divisible_by=128,
        split_ngram_parts=128,
        bos_token_id=248044,
        eos_token_id=248044,
    )


def _hf_config(quantization_config=RADIXARK_NVFP4):
    return SimpleNamespace(
        model_type="qwen4_exp",
        architectures=["Qwen4ExpForConditionalGeneration"],
        image_token_id=248056,
        text_config=_text_config(),
        quantization_config=quantization_config,
    )


def test_groups_and_layer_split():
    cfg = parse_config(_hf_config())
    full = [g for g in cfg.attention_groups if isinstance(g, FullAttentionGroupConfig)]
    linear = [g for g in cfg.attention_groups if isinstance(g, LinearGatedDeltaGroupConfig)]
    assert len(full) == 1 and len(linear) == 1
    assert full[0].layer_ids == tuple(range(3, 48, 4))
    assert len(linear[0].layer_ids) == 36
    assert full[0].index_head_dim == 128
    assert full[0].num_index_layers == 12
    assert full[0].index_ratio == 4
    assert full[0].rotary_config.rotary_dim == 64
    assert full[0].rotary_config.scaling is None
    assert linear[0].output_gate == "sigmoid"
    assert linear[0].num_key_heads == 16 and linear[0].num_value_heads == 48


def test_kv_specs_resolve_qsa():
    cfg = parse_config(_hf_config())
    specs = {s.name: s for s in cfg.kv_cache_group_specs()}
    assert specs["full"].attn_type is AttnType.QSA
    assert specs["full"].index_ratio == 4
    assert cfg.attn_type_for_layer(3) is AttnType.QSA
    assert cfg.attn_type_for_layer(0) is AttnType.LINEAR
    assert cfg.has_linear_attention


def test_moe_dims():
    cfg = parse_config(_hf_config())
    assert cfg.num_experts == 512
    assert cfg.num_experts_per_tok == 10
    assert cfg.norm_topk_prob is True
    assert cfg.moe_enabled


@pytest.mark.parametrize(
    "quantization_config, expert_quant",
    [
        pytest.param(RADIXARK_NVFP4, "nvfp4", id="RadixArk/Qwen3.8-Flash-Next-NVFP4"),
        pytest.param(NVIDIA_NVFP4, "nvfp4", id="nvidia/Qwen3.8-Flash-Next-NVFP4"),
        pytest.param(QWEN_FP8, "fp8_block", id="Qwen/Qwen3.8-Flash-Next-FP8"),
        pytest.param(None, "none", id="Qwen/Qwen3.8-Flash-Next"),
    ],
)
def test_released_checkpoints_expert_quant(quantization_config, expert_quant):
    assert parse_config(_hf_config(quantization_config)).expert_quant == expert_quant


def test_qwen4_args_payload():
    args = parse_config(_hf_config()).qwen4_args
    assert args.ple_layer_ids == (1,)
    assert args.hc_count == 4 and args.hc_lowrank == 320
    assert args.index_topk_blocks == 512
    assert args.num_ngram_heads == 16
    assert args.ngram_head_dim == 160
    assert args.ple_conv_state_len == 9
    assert args.ple_state_width == 10240
    assert args.ngram_boundary_token_id == 248044


def test_ple_on_full_attention_layer_rejected():
    hf = _hf_config()
    hf.text_config.ple_layer_ids = [4]  # one-indexed 4 == zero-based 3, a full_attention layer
    with pytest.raises(ValueError, match="linear_attention"):
        parse_config(hf)


def test_output_gate_null_falls_back_to_hidden_act():
    hf = _hf_config()
    hf.text_config.output_gate_type = None
    linear = [
        g
        for g in parse_config(hf).attention_groups
        if isinstance(g, LinearGatedDeltaGroupConfig)
    ]
    assert linear[0].output_gate == "silu"


def test_eos_token_id_list_uses_the_first_entry():
    base = parse_config(_hf_config()).qwen4_args.ngram_boundary_token_id
    hf = _hf_config()
    hf.text_config.eos_token_id = [base, base + 1]
    assert parse_config(hf).qwen4_args.ngram_boundary_token_id == base


def _vision_config():
    return SimpleNamespace(
        hidden_size=1152, depth=27, num_heads=16, intermediate_size=4304, patch_size=16, temporal_patch_size=2,
        spatial_merge_size=2, num_position_embeddings=2304, out_hidden_size=2560, in_channels=3,
        deepstack_visual_indexes=[],
    )


def test_vision_turns_on_mrope_and_the_tower():
    hf = _hf_config()
    hf.vision_config = _vision_config()
    config = parse_config(hf)
    assert config.is_multimodal and config.model_is_mrope
    assert config.rotary_config.mrope_section == [11, 11, 10]
    assert config.rotary_config.mrope_layout == "interleaved" and config.rotary_config.rotary_dim == 64
    assert config.vision_config.out_hidden_size == 2560 and config.vision_config.deepstack_visual_indexes == ()
    assert config.qwen4_args.image_token_id == 248056


def test_text_only_keeps_the_1d_rope():
    config = parse_config(_hf_config())
    assert not config.is_multimodal and not config.model_is_mrope and config.rotary_config.mrope_section is None
# the merged-projection prefixes the model asks the QuantConfig about (attention.py / layers/linear_attn/gdn.py)
DENSE_PREFIXES = (
    "model.layers.3.self_attn.qkv_proj", "model.layers.3.self_attn.o_proj",
    "model.layers.0.linear_attn.in_proj_qkvz", "model.layers.0.linear_attn.out_proj",
)
BF16_PREFIXES = (
    "model.layers.0.linear_attn.in_proj_ba", "model.layers.0.mlp.shared_expert.gate_up_proj",
    "model.layers.3.self_attn.indexer.index_qk_proj", "lm_head",
)


def _quant(hf, tmp_path):
    from freetoken.models.register import checkpoint_quant_config, get_model_spec

    return checkpoint_quant_config(str(tmp_path), hf, get_model_spec(hf.architectures[0]))


def test_block_fp8_dense_schemes(tmp_path):
    quant = _quant(_hf_config(LOVEDHEART_NVFP4_FP8), tmp_path)
    for prefix in DENSE_PREFIXES:
        scheme = quant.scheme_for(prefix)
        assert str(scheme.kind) == "fp8_block" and scheme.has("weight_scale_inv"), prefix
    for prefix in BF16_PREFIXES:
        assert quant.scheme_for(prefix) is None, prefix
    assert str(quant.scheme_for("model.layers.0.mlp.experts").kind) == "nvfp4"
    assert parse_config(_hf_config(LOVEDHEART_NVFP4_FP8)).expert_quant == "nvfp4"


@pytest.mark.parametrize("quantization_config", [RADIXARK_NVFP4, NVIDIA_NVFP4, None], ids=["RadixArk", "nvidia", "bf16"])
def test_released_checkpoints_keep_the_dense_projections_bf16(quantization_config, tmp_path):
    quant = _quant(_hf_config(quantization_config), tmp_path)
    for prefix in DENSE_PREFIXES + BF16_PREFIXES:
        assert quant.scheme_for(prefix) is None, prefix


# Qwen3.8-Flash-Next model card, "Processing Ultra-Long Texts": the 1M-token override.
MODEL_CARD_YARN = {
    "mrope_interleaved": True,
    "mrope_section": [11, 11, 10],
    "rope_type": "yarn",
    "rope_theta": 10000000,
    "partial_rotary_factor": 0.25,
    "factor": 4.0,
    "original_max_position_embeddings": 262144,
}


def _write_checkpoint_config(path, **fields) -> None:
    import json

    text = {k: v for k, v in vars(_text_config()).items()}
    config = {
        "model_type": "qwen4_exp",
        "architectures": ["Qwen4ExpForConditionalGeneration"],
        "image_token_id": 248056,
        "text_config": text,
        "quantization_config": RADIXARK_NVFP4,
        **fields,
    }
    (path / "config.json").write_text(json.dumps(config))


def test_the_engine_serves_the_model_card_1m_recipe(tmp_path):
    import torch

    from freetoken.distributed import DistributedInfo
    from freetoken.engine.config import EngineConfig
    from freetoken.mm.config import ENCODER_KINDS, MultimodalConfig

    _write_checkpoint_config(tmp_path)
    config = EngineConfig(
        model_path=str(tmp_path),
        tp_info=DistributedInfo(rank=0, size=1),
        dtype=torch.bfloat16,
        hf_overrides={"text_config": {"rope_parameters": MODEL_CARD_YARN}},
        mm=MultimodalConfig(disabled_encoders=frozenset(ENCODER_KINDS)),
    )
    rotary = config.model_config.rotary_config
    assert rotary.scaling["rope_type"] == "yarn" and rotary.scaling["factor"] == 4.0 and rotary.mrope_section is None
    assert rotary.max_position == 262144 and rotary.rotary_dim == 64 and rotary.base == 10000000
    assert config.max_seq_len == 4 * 262144
    plain = EngineConfig(model_path=str(tmp_path), tp_info=DistributedInfo(rank=0, size=1), dtype=torch.bfloat16,
                         mm=MultimodalConfig(disabled_encoders=frozenset(ENCODER_KINDS)))
    assert plain.model_config.rotary_config.scaling is None and plain.max_seq_len == 262144