Skip to content

FreeToken brings datacenter-scale model serving to your desktop. Run massive models locally, fast and efficiently.

11.5 KB
Raw Blame
"""Weight loading for DeepSeek-V4.1-Flash.

* :func:`iter_weights` streams the resident (non-expert) tensors keyed by the model's attribute
  paths: checkpoint ``layers.N.*`` -> ``model.layers.N.*``, ``embed`` / ``norm`` -> ``model.*``,
  ``head`` -> ``head``. fp8 linears keep their e4m3 payload and e8m0 ``scale`` (declared by the quant
  method as ``weight_scale_inv``); ``wo_a`` keeps its e4m3 payload and ``wo_a_scale`` too (the layer
  dequantizes in-kernel for the reference's bf16 grouped einsum). The engine casts everything else to
  the declared parameter dtype (fp32 compressor weights, hc gates, engram gate weights).
* :func:`iter_expert_pieces` streams the routed MXFP4 experts (e2m1 pairs + e8m0 per-32 scales) per
  expert for the offload banks.

Dropped: the DSpark ``mtp.*`` stage; vision weights and ``gate.bias_vl`` when serving text-only.
The Engram tables (``layers.{1,14}.engram.embed.*``) are
not state-dict tensors: the disk row store maps them in place (``engram_table.py``); an FTW conversion
carries them as side files (:func:`ftw_side_files`).
"""

from __future__ import annotations

from contextlib import ExitStack
import json
import os
import re

import safetensors
import torch
from freetoken.distributed import get_tp_info
from freetoken.layers.quantization import QuantKind
from freetoken.models.loader import drop_page_cache
from tqdm import tqdm

from .args import DeepseekV41Args, Mode


class _ShardReader:
    def __init__(self, folder: str, weight_map: dict, device):
        self._folder = folder
        self._weight_map = weight_map
        self._device = str(device)
        self._handles: dict[str, object] = {}
        self._stack = ExitStack()

    def has(self, name: str) -> bool:
        return name in self._weight_map

    def get(self, name: str) -> torch.Tensor:
        shard = self._weight_map[name]
        handle = self._handles.get(shard)
        if handle is None:
            handle = self._stack.enter_context(safetensors.safe_open(os.path.join(self._folder, shard), framework="pt", device=self._device))
            self._handles[shard] = handle
        return handle.get_tensor(name)

    def close(self) -> None:
        try:
            self._stack.close()
        finally:
            for shard in self._handles:
                drop_page_cache(os.path.join(self._folder, shard))
            self._handles.clear()


def weight_map(model_path: str) -> dict:
    with open(os.path.join(model_path, "model.safetensors.index.json")) as f:
        return json.load(f)["weight_map"]


def load_args(model_path: str) -> DeepseekV41Args:
    from freetoken.utils.hf import cached_load_hf_config

    return DeepseekV41Args.from_hf(cached_load_hf_config(model_path))


def dequant_fp8_block(weight: torch.Tensor, scale: torch.Tensor) -> torch.Tensor:
    """Square block-scaled e4m3 -> bf16; ``scale`` holds e8m0 codes (``2^(code-127)``), the block
    edge follows from the shapes."""
    n, k = weight.shape
    block = n // scale.shape[0]
    assert (n // block, k // block) == tuple(scale.shape), (weight.shape, scale.shape)
    s = torch.exp2(scale.view(torch.uint8).to(torch.float32) - 127.0)
    s = s.repeat_interleave(block, dim=0).repeat_interleave(block, dim=1)
    return (weight.to(torch.float32) * s).to(torch.bfloat16)


def iter_weights(model_path: str, device, *, include_moe_experts: bool = True, include_non_moe: bool = True, include_vision: bool = True):
    args = load_args(model_path)
    reader = _ShardReader(model_path, weight_map(model_path), device)
    get = reader.get
    def linear(src: str, dst: str):
        yield f"{dst}.weight", get(f"{src}.weight")
        if reader.has(f"{src}.scale"):
            yield f"{dst}.weight_scale_inv", get(f"{src}.scale")

    try:
        if include_moe_experts:
            # Resident experts support unquantized checkpoints; FP4 uses the offload cache.
            if reader.has("layers.0.ffn.experts.0.w1.scale"):
                raise ValueError(
                    "DeepSeek-V4.1's fp4 routed experts are served from the offload cache; run with "
                    "--moe-strategy offload (include_moe_experts must be False)."
                )
            for L in range(args.n_layers):
                p = f"layers.{L}.ffn.experts"
                gate_up = torch.stack([torch.cat([get(f"{p}.{e}.w1.weight"), get(f"{p}.{e}.w3.weight")]) for e in range(args.n_routed_experts)])
                down = torch.stack([get(f"{p}.{e}.w2.weight") for e in range(args.n_routed_experts)])
                yield f"model.layers.{L}.ffn.experts.gate_up_proj", gate_up
                yield f"model.layers.{L}.ffn.experts.down_proj", down
        if not include_non_moe:
            return
        if include_vision and reader.has("vision.patch_embed.proj.weight"):
            yield from iter_vision_weights(model_path, device)
        yield "model.embed.weight", get("embed.weight")
        yield "model.norm.weight", get("norm.weight")
        yield "head.weight", get("head.weight")
        for L in range(args.n_layers):
            role = args.roles[L]
            a, m = f"layers.{L}.attn", f"model.layers.{L}.attn"
            yield from linear(f"{a}.wq_a", f"{m}.wq_a")
            yield f"{m}.q_norm.weight", get(f"{a}.q_norm.weight")
            yield from linear(f"{a}.wq_b", f"{m}.wq_b")
            yield from linear(f"{a}.wkv", f"{m}.wkv")
            yield f"{m}.kv_norm.weight", get(f"{a}.kv_norm.weight")
            yield f"{m}.wo_a", get(f"{a}.wo_a.weight")
            if reader.has(f"{a}.wo_a.scale"):  # the fp8 payload + scale; the layer dequantizes in-kernel
                yield f"{m}.wo_a_scale", get(f"{a}.wo_a.scale")
            yield from linear(f"{a}.wo_b", f"{m}.wo_b")
            yield f"{m}.attn_sink", get(f"{a}.attn_sink")
            if role.mode is Mode.FULL:
                c = f"{a}.compressor"
                yield f"{m}.compressor.norm.weight", get(f"{c}.norm.weight")
                if role.ratio > 1:
                    yield f"{m}.compressor.wkv", get(f"{c}.wkv.weight")
                    yield f"{m}.compressor.wgate", get(f"{c}.wgate.weight")
                else:
                    yield from linear(f"{c}.wkv", f"{m}.compressor.wkv")
                yield from linear(f"{a}.indexer.wk", f"{m}.indexer.wk")
                yield f"{m}.indexer.k_norm.weight", get(f"{a}.indexer.k_norm.weight")
            if role.has_indexer:
                yield from linear(f"{a}.indexer.wq_b", f"{m}.indexer.wq_b")
                yield from linear(f"{a}.indexer.weights_proj", f"{m}.indexer.weights_proj")

            p = f"layers.{L}"
            yield f"model.{p}.attn_norm.weight", get(f"{p}.attn_norm.weight")
            yield f"model.{p}.ffn_norm.weight", get(f"{p}.ffn_norm.weight")
            yield f"model.{p}.ffn.gate.weight", get(f"{p}.ffn.gate.weight")
            yield f"model.{p}.ffn.gate.bias", get(f"{p}.ffn.gate.bias")
            for proj in ("w1", "w2", "w3"):
                yield from linear(f"{p}.ffn.shared_experts.{proj}", f"model.{p}.ffn.shared_experts.{proj}")
            for nm in ("hc_attn_fn", "hc_ffn_fn", "hc_attn_base", "hc_ffn_base", "hc_attn_scale", "hc_ffn_scale"):
                yield f"model.{p}.{nm}", get(f"{p}.{nm}")
            if L in args.engram_layer_ids:
                e = f"{p}.engram"
                yield from linear(f"{e}.wkv", f"model.{e}.wkv")
                yield f"model.{e}.q_weight", get(f"{e}.q_weight")
                yield f"model.{e}.k_weight", get(f"{e}.k_weight")
    finally:
        reader.close()


def iter_vision_weights(model_path: str, device):
    names = weight_map(model_path)
    reader = _ShardReader(model_path, names, device)
    try:
        for name in names:
            if name.startswith(("vision.", "aligner.")) or name in ("image_start", "image_end", "image_newline"):
                yield f"visual.{name}", reader.get(name)
        args = load_args(model_path)
        yield "visual.routing_bias", torch.stack([reader.get(f"layers.{i}.ffn.gate.bias_vl") for i in range(args.n_layers)])
    finally:
        reader.close()


# ----- FTW side files: the Engram tables ----------------------------------------------------------
def engram_table_names(layer_id: int) -> tuple[str, str]:
    return f"layers.{layer_id}.engram.embed.weight", f"layers.{layer_id}.engram.embed.scale"


def ftw_side_files(model_path: str, out_dir: str) -> list[str]:
    """Copy the checkpoint shards holding the Engram tables (fp8 rows + ue8m0 scales) next to an FTW
    checkpoint. The tables are never FTW entries: ``load_host_tables`` maps them in place from whatever
    safetensors file holds them (``engram_table.engram_row_source`` reads the headers), and the FTW
    loader never reads ``.safetensors`` as weights, so the shard's other tensors ride along unused."""
    import shutil

    from freetoken.models.loader import safetensors_weight_map
    from freetoken.utils import download_hf_weight

    folder = download_hf_weight(model_path)
    args = load_args(folder)
    shard_of = safetensors_weight_map(folder)
    shards = sorted({shard_of[n] for layer_id in args.engram_layer_ids if layer_id < args.n_layers for n in engram_table_names(layer_id)})
    for shard in shards:
        shutil.copyfile(os.path.join(folder, shard), os.path.join(out_dir, shard))
    return shards


# ----- routed MXFP4 expert pieces ---------------------------------------------------------------
_EXPERT_RE = re.compile(r"^layers\.(?P<layer>\d+)\.ffn\.experts\.(?P<expert>\d+)\.(?P<proj>w1|w2|w3)\.(?P<kind>weight|scale)$")
_PROJ_ROLE = {"w1": "gate", "w3": "up", "w2": "down"}
_KIND_SUFFIX = {"weight": "", "scale": "_scale"}


def iter_expert_pieces(model_path: str, config, kind: QuantKind, *, layers=None, parallel: bool | None = False, workers: int = 8, chunk: int = 8 << 20):
    """One piece per expert: ``{gate, up, down}`` e2m1 pairs and their e8m0 ``_scale`` companions.
    The DSpark stage's experts (``mtp.*``) never match."""
    if kind is not QuantKind.MXFP4:
        return None
    if get_tp_info().size > 1:
        raise NotImplementedError("DeepSeek-V4.1 expert banks support TP=1 only")
    from freetoken.models.weight import iter_expert_tensors_parallel
    from freetoken.moe.expert_pieces import per_expert_pieces

    args = load_args(model_path)
    L, E = args.n_layers, args.n_routed_experts
    keep = None if layers is None else set(layers)

    def locate(raw_name: str):
        m = _EXPERT_RE.match(raw_name)
        if m is None or int(m["layer"]) >= L or (keep is not None and int(m["layer"]) not in keep):
            return None
        return int(m["layer"]), int(m["expert"]), _PROJ_ROLE[m["proj"]] + _KIND_SUFFIX[m["kind"]]

    if parallel:
        tensors = iter_expert_tensors_parallel(model_path, lambda n: locate(n) is not None, workers=workers, chunk=chunk)
        return per_expert_pieces(tensors, locate, tensors_per_expert=6)

    def _serial():
        reader = _ShardReader(model_path, weight_map(model_path), torch.device("cpu"))
        try:
            for li in tqdm(range(L) if keep is None else sorted(keep), desc="Loading DeepSeek-V4.1 experts (serial)", disable=not get_tp_info().is_primary()):
                for e in range(E):
                    base = f"layers.{li}.ffn.experts.{e}"
                    for proj in ("w1", "w3", "w2"):
                        for kind_ in ("weight", "scale"):
                            name = f"{base}.{proj}.{kind_}"
                            yield name, reader.get(name)
        finally:
            reader.close()

    return per_expert_pieces(_serial(), locate, tensors_per_expert=6)


__all__ = ["iter_weights", "iter_vision_weights", "iter_expert_pieces", "ftw_side_files", "load_args", "weight_map", "dequant_fp8_block"]