FreeToken brings datacenter-scale model serving to your desktop. Run massive models locally, fast and efficiently.
"""Shared fixtures for the qwen4_exp tests: toy configs, hash constants, the QSA pool and backend.
The geometry is the shipping one everywhere it matters for QSA (head_dim 256, index head_dim
128 with a 64-wide partial rope, index_ratio 4, budget 2048 -> 512 blocks -> 2051 selected
tokens, page_size 64); only the head counts, the hidden size and the layer count are scaled
down so a test fits on a shared GPU. Holds no tests itself.
"""
from __future__ import annotations
from types import SimpleNamespace
import pytest
import torch
from freetoken.distributed import set_tp_info, try_get_tp_info
EOS = 7
VOCAB = 512
requires_cuda = pytest.mark.skipif(not torch.cuda.is_available(), reason="needs a GPU")
def hf_config(
num_layers: int = 4,
head_dim: int = 256,
num_q: int = 4,
num_kv: int = 2,
index_head_dim: int = 128,
index_heads: int = 4,
budget: int = 2048,
ratio: int = 4,
hidden: int = 256,
max_position: int = 1 << 16,
rope_theta: float = 10000000.0,
**text_overrides,
) -> SimpleNamespace:
text = SimpleNamespace(
num_hidden_layers=num_layers,
hidden_size=hidden,
vocab_size=VOCAB,
head_dim=head_dim,
num_attention_heads=num_q,
num_key_value_heads=num_kv,
layer_types=[
"full_attention" if (i + 1) % 4 == 0 else "linear_attention"
for i in range(num_layers)
],
rope_parameters={
"rope_type": "default",
"rope_theta": rope_theta,
"partial_rotary_factor": 0.25,
},
max_position_embeddings=max_position,
rms_norm_eps=1e-6,
hidden_act="silu",
tie_word_embeddings=False,
num_experts=8,
num_experts_per_tok=2,
moe_intermediate_size=64,
shared_expert_intermediate_size=64,
linear_num_key_heads=2,
linear_num_value_heads=6,
linear_key_head_dim=32,
linear_value_head_dim=32,
linear_conv_kernel_dim=4,
output_gate_type="sigmoid",
indexer_n_heads=index_heads,
indexer_kv_heads=1,
indexer_head_dim=index_head_dim,
indexer_budget=budget,
indexer_compress_ratio=ratio,
hc_count=4,
hc_lowrank=16,
ple_layer_ids=[2],
ple_embed_dim=64,
ple_conv_kernel_size=4,
ngram_size=3,
heads_per_ngram=2,
ngram_vocab_size_base=1000,
make_ngram_vocab_size_divisible_by=8,
split_ngram_parts=4,
eos_token_id=EOS,
)
for name, value in text_overrides.items():
setattr(text, name, value)
return SimpleNamespace(
model_type="qwen4_exp",
architectures=["Qwen4ExpForConditionalGeneration"],
text_config=text,
quantization_config=None,
)
def toy_hf_config(num_layers: int = 4, **text_overrides) -> SimpleNamespace:
"""The small-geometry config the PLE/skeleton tests share (hidden 128, head_dim 64)."""
return hf_config(
num_layers=num_layers, head_dim=64, num_kv=1, index_head_dim=64, index_heads=2,
budget=16, hidden=128, max_position=4096, rope_theta=10000.0, **text_overrides,
)
def hash_constants(args):
"""Checkpoint-shape int64 hash tensors, derived like the dummy-weight path."""
from freetoken.models.qwen4_exp.ple import derive_ngram_hash_constants
multipliers, sizes, offsets = derive_ngram_hash_constants(
vocab_size=VOCAB,
ngram_size=args.ngram_size,
num_ngram_heads=args.num_ngram_heads,
ngram_vocab_size_base=args.ngram_vocab_size_base,
ple_layer_index=0,
)
return [torch.tensor(v, dtype=torch.int64) for v in (multipliers, sizes, offsets)]
def parsed_config(**kwargs):
from freetoken.models.qwen4_exp.config import parse_config
if try_get_tp_info() is None:
set_tp_info(rank=0, size=1)
return parse_config(hf_config(**kwargs))
def fresh_ctx(page_size: int = 64, **fields):
import freetoken.core as core
from freetoken.core import Context, set_global_ctx
core._GLOBAL_CTX = None # test-only: each scenario builds its own ctx
ctx = Context(page_size=page_size)
for name, value in fields.items():
setattr(ctx, name, value)
set_global_ctx(ctx)
return ctx
def fill_weights(op, seed: int, device: torch.device, scale: float = 0.05) -> None:
gen = torch.Generator(device=device).manual_seed(seed)
for tensor in op.state_dict().values():
if tensor.is_floating_point():
tensor.normal_(0.0, scale, generator=gen)
else:
tensor.zero_()
class Fixture:
"""QSA pool + page table + the sparse backend, with a first-fit page allocator."""
def __init__(
self,
config,
num_pages: int,
max_running_req: int = 8,
device: str = "cuda",
dtype: torch.dtype = torch.bfloat16,
page_size: int = 64,
) -> None:
from freetoken.attention.qsa_sparse import QSASparseAttnBackend
from freetoken.kvcache import create_kvcache_pool
self.config = config
self.device = torch.device(device)
self.dtype = dtype
self.page_size = page_size
self.num_req_slots = max_running_req + 1
self.pool = create_kvcache_pool(
model_config=config,
num_pages=num_pages + 1, # + 1 for the dummy page, as create_kv_pool does
page_size=page_size,
dtype=dtype,
device=self.device,
num_req_slots=self.num_req_slots,
)
self.page_table = torch.zeros(
(self.num_req_slots, num_pages * page_size), dtype=torch.int32, device=self.device
)
self.page_table[max_running_req].fill_(num_pages * page_size) # dummy page
self.ctx = fresh_ctx(
page_size=page_size, page_table=self.page_table, kv_cache=self.pool
)
self.backend = QSASparseAttnBackend(config)
self.ctx.attn_backend = self.backend
self._free = list(range(num_pages))
def layer(self, layer_id: int, seed: int = 1):
from freetoken.models.qwen4_exp.attention import Qwen4ExpAttention
from freetoken.utils.torch_utils import torch_dtype
with torch.device(self.device), torch_dtype(self.dtype):
attn = Qwen4ExpAttention(self.config, layer_id=layer_id)
fill_weights(attn, seed, self.device)
return attn
def allocate(self, table_idx: int, cached_len: int, device_len: int) -> None:
for page in range(-(-cached_len // self.page_size), -(-device_len // self.page_size)):
base = self._free.pop(0) * self.page_size
columns = slice(page * self.page_size, (page + 1) * self.page_size)
self.page_table[table_idx, columns] = torch.arange(
base, base + self.page_size, dtype=torch.int32, device=self.device
)
def req(self, table_idx: int, cached_len: int, device_len: int) -> SimpleNamespace:
self.allocate(table_idx, cached_len, device_len)
return SimpleNamespace(
table_idx=table_idx,
cached_len=cached_len,
device_len=device_len,
extend_len=device_len - cached_len,
)
def step(self, req: SimpleNamespace) -> None:
self.allocate(req.table_idx, req.device_len, req.device_len + 1)
req.cached_len, req.device_len, req.extend_len = req.device_len, req.device_len + 1, 1
def batch(self, reqs, phase: str) -> SimpleNamespace:
positions = torch.cat(
[
torch.arange(r.cached_len, r.device_len, dtype=torch.int32, device=self.device)
for r in reqs
]
)
out_loc = torch.cat(
[self.page_table[r.table_idx, r.cached_len : r.device_len] for r in reqs]
).contiguous()
batch = SimpleNamespace(
reqs=reqs,
padded_reqs=reqs,
phase=phase,
size=len(reqs),
padded_size=len(reqs),
is_prefill=phase == "prefill",
is_decode=phase == "decode",
positions=positions,
get_attn_positions=lambda: positions,
mm_embeds=None,
out_loc=out_loc,
attn_metadata=None,
active_table_idx=torch.tensor(
[r.table_idx for r in reqs], dtype=torch.int32, device=self.device
),
)
self.backend.prepare_metadata(batch)
return batch
def selection_spy(monkeypatch, backend) -> dict:
"""Record the expanded token selection of every ``_select`` call."""
from freetoken.attention.qsa_sparse import QSASparseAttnBackend
seen: dict[str, torch.Tensor] = {}
original = QSASparseAttnBackend._select
def spy(self, index, md, slot):
indices = original(self, index, md, slot)
seen["indices"] = indices.clone()
return indices
monkeypatch.setattr(QSASparseAttnBackend, "_select", spy)
return seen
LM = "model.language_model"
# quantization_config of each released checkpoint, trimmed to the entries parse_config and the reader look at
# RadixArk/Qwen3.8-Flash-Next-NVFP4: modelopt NVFP4 everywhere except the ignore list
RADIXARK_NVFP4 = {
"quant_algo": "NVFP4",
"quant_method": "modelopt",
"ignore": [
"model.embed_tokens",
"mtp.*",
"model.mtp.*",
"*.self_attn.*",
"*.linear_attn.*",
"*.mlp.gate*",
"*.mlp.shared_expert.*",
"*.mlp.shared_expert_gate*",
"*hyper_connection*",
"*.ple.*",
"model.visual.*",
"model.language_model.embed_tokens",
"lm_head",
],
}
# nvidia/Qwen3.8-Flash-Next-NVFP4: modelopt MIXED_PRECISION, the per-module algo sits in quantized_layers
NVIDIA_NVFP4 = {
"quant_algo": "MIXED_PRECISION",
"quant_method": "modelopt",
"quantized_layers": {
**{f"model.language_model.layers.{i}.mlp.experts": {"quant_algo": "NVFP4", "group_size": 16} for i in range(48)},
"model.language_model.layers.1.ple.ple_embedding.ngram_embedding": {"quant_algo": "FP8"},
"mtp.layers.0.mlp.experts": {"quant_algo": "FP8_PB_WO", "group_size": 128},
},
"ignore": ["lm_head", "model.language_model.embed_tokens", "model.language_model.layers.0.mlp.shared_expert*", "model.visual*"],
}
# Qwen/Qwen3.8-Flash-Next-FP8: 128x128 block-fp8 experts, everything else listed in modules_to_not_convert
QWEN_FP8 = {
"quant_method": "fp8",
"activation_scheme": "dynamic",
"weight_per_tensor": False,
"act_per_tensor": False,
"weight_block_size": [128, 128],
"modules_to_not_convert": [
"lm_head",
"model.language_model.embed_tokens",
"model.language_model.hyper_connection_mixer.input_mix_weight_up",
"model.language_model.layers.0.linear_attn.in_proj_qkv",
"model.language_model.layers.3.self_attn.q_proj",
"model.language_model.layers.3.mlp.gate",
"model.language_model.layers.3.mlp.shared_expert.gate_proj",
],
"modules_to_convert": ["ple.ple_embedding.ngram_embedding"],
}
def mixed_precision_quant(gdn_layers, attn_layers, moe_layers) -> dict:
"""modelopt MIXED_PRECISION with NVFP4 routed experts and FP8_PB_WO attention / GDN projections, ignore list as in lovedheart/Qwen3.8-Flash-Next-NVFP4-FP8."""
return {
"quant_method": "modelopt",
"quant_algo": "MIXED_PRECISION",
"quantized_layers": {
**{f"{LM}.layers.{i}.mlp.experts": {"quant_algo": "NVFP4", "group_size": 16} for i in moe_layers},
**{f"{LM}.layers.{i}.linear_attn.{p}": {"quant_algo": "FP8_PB_WO", "group_size": 128}
for i in gdn_layers for p in ("in_proj_qkv", "in_proj_z", "out_proj")},
**{f"{LM}.layers.{i}.self_attn.{p}_proj": {"quant_algo": "FP8_PB_WO", "group_size": 128}
for i in attn_layers for p in "qkvo"},
},
"ignore": [
"model.embed_tokens", "mtp.*", "model.mtp.*", "*.mlp.gate*", "*.mlp.shared_expert.*",
"*.mlp.shared_expert_gate*", "*hyper_connection*", "*.ple.*", "model.visual.*",
"model.language_model.embed_tokens", "lm_head", "*.self_attn.indexer*",
],
}
# lovedheart/Qwen3.8-Flash-Next-NVFP4-FP8, trimmed to layers 0 (GDN) and 3 (attention)
LOVEDHEART_NVFP4_FP8 = mixed_precision_quant(gdn_layers=(0,), attn_layers=(3,), moe_layers=(0, 3))
def install_quant_config(model_path: str) -> None:
"""Install ``model_path``'s QuantConfig process-wide, as EngineConfig does before the reader runs."""
from freetoken.layers.quantization import set_quant_config
from freetoken.models.register import checkpoint_quant_config, get_model_spec
from freetoken.utils import cached_load_hf_config
hf = cached_load_hf_config(model_path)
set_quant_config(checkpoint_quant_config(model_path, hf, get_model_spec(hf.architectures[0])))
def meta_state_dict(model_path: str) -> dict[str, torch.Tensor]:
"""State dict of the model the engine builds for ``model_path`` (experts offloaded), on the meta device."""
from freetoken.engine.config import EngineConfig
from freetoken.engine.engine import _decode_target
from freetoken.layers import rotary
from freetoken.mm.config import ENCODER_KINDS, MultimodalConfig
from freetoken.models import create_model
from freetoken.utils.torch_utils import torch_dtype
if try_get_tp_info() is None:
set_tp_info(rank=0, size=1)
config = EngineConfig(model_path=model_path, tp_info=try_get_tp_info(), dtype=torch.bfloat16, moe_strategy="offload",
mm=MultimodalConfig(disabled_encoders=frozenset(ENCODER_KINDS)))
object.__setattr__(config.model_config, "moe_strategy", "offload")
object.__setattr__(config.model_config, "decode_target", _decode_target(config))
saved = rotary._ROPE_DEVICE
rotary.set_rope_device(torch.device("cpu")) # get_rope refuses to build on meta
rotary.get_rope.cache_clear()
try:
with torch.device("meta"), torch_dtype(torch.bfloat16):
return create_model(config.model_config).state_dict()
finally:
rotary.set_rope_device(saved)
rotary.get_rope.cache_clear() # the cpu rope must not leak into the GPU tests' cache