Skip to content

FreeToken brings datacenter-scale model serving to your desktop. Run massive models locally, fast and efficiently.

9.3 KB
Raw Blame
"""Shared fixtures for the deepseek_v41 tests: a tiny V4.1-shaped config (2 window layers, a ratio-2
encoder group with one Full + one Reuse layer, a ratio-1 decoder group with a Full candidate source and
a Reindex layer) and synthetic bf16 and FP8/MXFP4 checkpoints in the real tensor naming. Holds no tests itself."""

from __future__ import annotations

import json
import os
from types import SimpleNamespace

import pytest
import torch

requires_cuda = pytest.mark.skipif(not torch.cuda.is_available(), reason="needs a GPU")

VOCAB = 512
DIM = 256
N_LAYERS = 6
RATIOS = [0, 0, 2, 2, 1, 1, 0]  # + one trailing MTP layer the model ignores
KV_SOURCES = [2, 4]
INDEX_SOURCES = [2, 4, 5]
ENGRAM_LAYERS = [1]
ENGRAM_VOCAB = 500  # bucket prime start; 3 orders x 2 heads ~= 3000 rows
ENGRAM_ROWS = 4096


def tiny_text_config(**over) -> dict:
    text = dict(
        model_type="deepseek_v41_text",
        vocab_size=VOCAB, hidden_size=DIM, moe_intermediate_size=64, num_hidden_layers=N_LAYERS, num_attention_heads=4,
        num_key_value_heads=1, head_dim=64, qk_rope_head_dim=16, q_lora_rank=64, o_lora_rank=32, o_groups=2,
        hidden_act="silu", swiglu_limit=10.0, rms_norm_eps=1e-20, max_position_embeddings=16384,
        rope_theta=10000, rope_scaling=dict(rope_type="yarn", factor=16, beta_fast=32, beta_slow=1, original_max_position_embeddings=1024),
        n_routed_experts=8, n_shared_experts=1, num_experts_per_tok=2, scoring_func="sqrtsoftplus", topk_method="noaux_tc",
        norm_topk_prob=True, routed_scaling_factor=1.5, sliding_window=128,
        compress_ratios=list(RATIOS), compress_rope_theta=160000, kv_source_layer_ids=list(KV_SOURCES),
        index_source_layer_ids=list(INDEX_SOURCES), index_n_heads=2, index_head_dim=32, index_topk=8,
        candidate_source_layer_id=4, candidate_topk_blocks=2, candidate_block_size=4,
        hc_mult=4, hc_sinkhorn_iters=20, hc_eps=1e-6,
        engram_layer_ids=list(ENGRAM_LAYERS), engram_num_embeddings=[ENGRAM_ROWS], engram_max_ngram_size=4,
        engram_vocab_size=ENGRAM_VOCAB, engram_n_heads=2, engram_head_dim=32, engram_pad_token_id=2,
        engram_compressed_vocab_size=VOCAB,  # the synthetic tokenizer maps every id to itself
        num_nextn_predict_layers=1, dspark_block_size=0, dspark_target_layer_ids=[],
    )
    text.update(over)
    return text


def tiny_hf_config(**over) -> SimpleNamespace:
    return SimpleNamespace(architectures=["DeepseekV41ForCausalLM"], model_type="deepseek_v41", text_config=tiny_text_config(**over))


def checkpoint_tensors(text: dict, seed: int = 0) -> dict[str, torch.Tensor]:
    """Random bf16 weights under the checkpoint's names (the reference ``convert.py`` dialect without
    quantization: no ``.scale`` tensors). hc / sink / router-bias / engram gates in fp32 as shipped."""
    g = torch.Generator().manual_seed(seed)

    def rnd(*shape, scale=0.02, dtype=torch.bfloat16):
        return (torch.randn(*shape, generator=g) * scale).to(dtype)

    dim, heads, hd, rd = text["hidden_size"], text["num_attention_heads"], text["head_dim"], text["qk_rope_head_dim"]
    qr, ol, og = text["q_lora_rank"], text["o_lora_rank"], text["o_groups"]
    inter, E, hc = text["moe_intermediate_size"], text["n_routed_experts"], text["hc_mult"]
    ih, ihd = text["index_n_heads"], text["index_head_dim"]
    mix = (2 + hc) * hc
    t: dict[str, torch.Tensor] = {
        "embed.weight": rnd(text["vocab_size"], dim, scale=1.0),
        "norm.weight": torch.ones(dim, dtype=torch.bfloat16),
        "head.weight": rnd(text["vocab_size"], dim),
    }
    for L in range(text["num_hidden_layers"]):
        a = f"layers.{L}.attn"
        t[f"{a}.wq_a.weight"] = rnd(qr, dim)
        t[f"{a}.q_norm.weight"] = torch.ones(qr, dtype=torch.bfloat16)
        t[f"{a}.wq_b.weight"] = rnd(heads * hd, qr)
        t[f"{a}.wkv.weight"] = rnd(hd, dim)
        t[f"{a}.kv_norm.weight"] = torch.ones(hd, dtype=torch.bfloat16)
        t[f"{a}.wo_a.weight"] = rnd(og * ol, heads * hd // og)
        t[f"{a}.wo_b.weight"] = rnd(dim, og * ol)
        t[f"{a}.attn_sink"] = rnd(heads, scale=0.5, dtype=torch.float32)
        ratio = text["compress_ratios"][L]
        if L in text["kv_source_layer_ids"]:
            t[f"{a}.compressor.wkv.weight"] = rnd(hd, dim)
            if ratio > 1:
                t[f"{a}.compressor.wgate.weight"] = rnd(hd, dim)
            t[f"{a}.compressor.norm.weight"] = torch.ones(hd, dtype=torch.bfloat16)
            t[f"{a}.indexer.wk.weight"] = rnd(ihd, hd)
            t[f"{a}.indexer.k_norm.weight"] = torch.ones(ihd, dtype=torch.bfloat16)
        if L in text["index_source_layer_ids"]:
            t[f"{a}.indexer.wq_b.weight"] = rnd(ih * ihd, qr)
            t[f"{a}.indexer.weights_proj.weight"] = rnd(ih, dim)
        p = f"layers.{L}"
        t[f"{p}.attn_norm.weight"] = torch.ones(dim, dtype=torch.bfloat16)
        t[f"{p}.ffn_norm.weight"] = torch.ones(dim, dtype=torch.bfloat16)
        t[f"{p}.ffn.gate.weight"] = rnd(E, dim)
        t[f"{p}.ffn.gate.bias"] = rnd(E, scale=0.1, dtype=torch.float32)
        t[f"{p}.ffn.gate.bias_vl"] = rnd(E, scale=0.1, dtype=torch.float32)
        for w, shape in (("w1", (inter, dim)), ("w2", (dim, inter)), ("w3", (inter, dim))):
            t[f"{p}.ffn.shared_experts.{w}.weight"] = rnd(*shape)
            for e in range(E):
                t[f"{p}.ffn.experts.{e}.{w}.weight"] = rnd(*shape)
        for nm in ("hc_attn_fn", "hc_ffn_fn"):
            t[f"{p}.{nm}"] = rnd(mix, hc * dim, scale=0.05, dtype=torch.float32)
        for nm in ("hc_attn_base", "hc_ffn_base"):
            t[f"{p}.{nm}"] = rnd(mix, scale=0.3, dtype=torch.float32)
        for nm in ("hc_attn_scale", "hc_ffn_scale"):
            t[f"{p}.{nm}"] = (torch.rand(3, generator=g) + 0.5).to(torch.float32)
        if L in text["engram_layer_ids"]:
            e = f"{p}.engram"
            cols = (text["engram_max_ngram_size"] - 1) * text["engram_n_heads"]
            t[f"{e}.wkv.weight"] = rnd(dim * (hc + 1), cols * text["engram_head_dim"])
            t[f"{e}.q_weight"] = torch.ones(hc, dim, dtype=torch.bfloat16)
            t[f"{e}.k_weight"] = torch.ones(hc, dim, dtype=torch.bfloat16)
            rows = text["engram_num_embeddings"][text["engram_layer_ids"].index(L)]
            t[f"{e}.embed.weight"] = rnd(rows, text["engram_head_dim"], scale=1.0).to(torch.float8_e4m3fn)
            t[f"{e}.embed.scale"] = torch.full((rows, text["engram_head_dim"] // 32), 127, dtype=torch.uint8).view(torch.float8_e8m0fnu)
    # the DSpark stage and the vision tower: present in a real checkpoint, dropped by the loader
    t["mtp.0.attn_norm.weight"] = torch.ones(dim, dtype=torch.bfloat16)
    t["vision.norm.weight"] = torch.ones(8, dtype=torch.bfloat16)
    return t


def write_tiny_checkpoint(folder: str, seed: int = 0, *, quantized: bool = False, **over) -> dict[str, torch.Tensor]:
    """A loadable checkpoint directory: HF-style config.json + one safetensors shard + index."""
    from safetensors.torch import save_file

    if quantized:
        over.setdefault("moe_intermediate_size", 256)
    text = tiny_text_config(**over)
    os.makedirs(folder, exist_ok=True)
    with open(os.path.join(folder, "config.json"), "w") as f:
        config = {"architectures": ["DeepseekV41ForCausalLM"], "model_type": "deepseek_v41", "text_config": text}
        if quantized:
            config["quantization_config"] = {
                "quant_method": "fp8", "activation_scheme": "dynamic",
                "weight_block_size": [32, 32], "scale_fmt": "ue8m0", "expert_dtype": "fp4",
            }
        json.dump(config, f)
    tensors = checkpoint_tensors(text, seed)
    if quantized:
        tensors = _quantize_checkpoint(tensors)
    save_file(tensors, os.path.join(folder, "model-00001-of-00001.safetensors"))
    with open(os.path.join(folder, "model.safetensors.index.json"), "w") as f:
        json.dump({"metadata": {}, "weight_map": {k: "model-00001-of-00001.safetensors" for k in tensors}}, f)
    return tensors


def _quantize_checkpoint(tensors):
    from .reference.kernel import _round_fp4

    out = dict(tensors)
    fp8_suffixes = (
        ".attn.wq_a", ".attn.wq_b", ".attn.wkv", ".attn.wo_a", ".attn.wo_b",
        ".indexer.wq_b", ".shared_experts.w1", ".shared_experts.w2", ".shared_experts.w3", ".engram.wkv",
    )
    grid = torch.tensor([0., .5, 1., 1.5, 2., 3., 4., 6.])
    for name, weight in tensors.items():
        if not name.endswith(".weight"):
            continue
        stem = name.removesuffix(".weight")
        if ".ffn.experts." in name:
            groups = weight.float().unflatten(-1, (-1, 32))
            scale = torch.exp2(torch.ceil(torch.log2(groups.abs().amax(-1).clamp_min(1e-12) / 6)))
            q = _round_fp4(groups / scale.unsqueeze(-1)).flatten(-2)
            codes = (q.abs().unsqueeze(-1) == grid).to(torch.int32).argmax(-1).to(torch.uint8)
            codes |= torch.signbit(q).to(torch.uint8) * 8
            out[name] = codes[:, 0::2] | (codes[:, 1::2] << 4)
            out[stem + ".scale"] = scale.to(torch.float8_e8m0fnu)
        elif stem.endswith(fp8_suffixes):
            n, k = weight.shape
            blocks = weight.float().reshape(n // 32, 32, k // 32, 32).permute(0, 2, 1, 3)
            scale = torch.exp2(torch.ceil(torch.log2(blocks.abs().amax((-1, -2)).clamp_min(1e-4) / 448)))
            expanded = scale.repeat_interleave(32, 0).repeat_interleave(32, 1)
            out[name] = (weight.float() / expanded).to(torch.float8_e4m3fn)
            out[stem + ".scale"] = scale.to(torch.float8_e8m0fnu)
    return out