Skip to content

FreeToken brings datacenter-scale model serving to your desktop. Run massive models locally, fast and efficiently.

10.9 KB
Raw Blame
"""Weight loading for DeepSeek-V4-Flash (engine path).

  - :func:`iter_weights` streams resident (non-expert) tensors keyed by the model's
    attribute paths (``model.`` + the checkpoint name). ``wo_a`` dequantized to bf16 to match
    the reference bf16 einsum.
  - :func:`iter_expert_pieces` streams the routed MXFP4 experts (e2m1 pairs + e8m0 per-32
    scales, no global) as per-expert pieces for the expert quant method's banks.
"""

from __future__ import annotations

import json
import os
import re

import safetensors
import torch
from tqdm import tqdm

from freetoken.layers.quantization import QuantKind
from freetoken.distributed import get_tp_info
from freetoken.models.loader import drop_page_cache

from .args import DeepseekV4Args, load_args


class _ShardReader:
    def __init__(self, folder: str, weight_map: dict, device):
        self._folder = folder
        self._weight_map = weight_map
        self._device = str(device)
        self._handles: dict[str, object] = {}

    def has(self, name: str) -> bool:
        return name in self._weight_map

    def get(self, name: str) -> torch.Tensor:
        shard = self._weight_map[name]
        handle = self._handles.get(shard)
        if handle is None:
            handle = safetensors.safe_open(
                os.path.join(self._folder, shard), framework="pt", device=self._device
            ).__enter__()
            self._handles[shard] = handle
        return handle.get_tensor(name)

    def close(self) -> None:
        for shard, handle in self._handles.items():
            try:
                handle.__exit__(None, None, None)
            except Exception:
                pass
            drop_page_cache(os.path.join(self._folder, shard))
        self._handles.clear()


def _weight_map(model_path: str) -> dict:
    with open(os.path.join(model_path, "model.safetensors.index.json")) as f:
        return json.load(f)["weight_map"]


def _dequant_fp8_block(weight: torch.Tensor, scale: torch.Tensor, block: int = 128) -> torch.Tensor:
    """Dequantize 128x128 block-scaled FP8 (e4m3) to bf16.

    scale is e8m0 exponent codes, ``value = 2^(code-127)`` (Triton FP8 GEMM convention).
    Used for ``wo_a`` to match the reference's bf16 einsum.
    """
    n, k = weight.shape
    codes = scale.view(torch.uint8).to(torch.float32)
    s = torch.exp2(codes - 127.0)
    s = s.repeat_interleave(block, dim=0).repeat_interleave(block, dim=1)[:n, :k]
    return (weight.to(torch.float32) * s).to(torch.bfloat16)


def iter_weights(
    model_path: str,
    device,
    *,
    include_moe_experts: bool = True,
    include_non_moe: bool = True,
    include_vision: bool = True,
):
    """Stream resident (non-expert) weights as ``(name, tensor)`` keyed to engine params.

    Routed MXFP4 experts come from the offload cache, so ``include_moe_experts`` must be
    False (DeepSeek-V4 only runs ``--moe-strategy offload``). Tensors yielded in checkpoint
    dtype (fp8 + e8m0 preserved); ``wo_a`` dequantized to bf16 to match the reference einsum.
    ``include_vision`` (the engine's active-encoder set) drops the ``visual.*`` tower
    tensors; the router biases stay in either way, since the gate builds
    them on any vision checkpoint.
    """
    if include_moe_experts:
        raise ValueError(
            "DeepSeek-V4 routed experts are served from the offload cache; "
            "run with --moe-strategy offload (include_moe_experts must be False)."
        )
    if not include_non_moe:
        return

    args = load_args(model_path, max_batch_size=1)
    reader = _ShardReader(model_path, _weight_map(model_path), device)

    def get(name: str) -> torch.Tensor:
        return reader.get(name)

    def linear(src: str, dst: str):
        yield f"{dst}.weight", get(f"{src}.weight")
        # fp8 linears declare the e8m0 block scale under the quant method's role name
        if reader.has(f"{src}.scale"):
            yield f"{dst}.weight_scale_inv", get(f"{src}.scale")

    try:
        yield "model.embed.weight", get("embed.weight")
        yield "model.norm.weight", get("norm.weight")
        yield "model.head.weight", get("head.weight")
        for nm in ("hc_head_fn", "hc_head_base", "hc_head_scale"):
            yield f"model.{nm}", get(nm)

        if include_vision:
            yield from _iter_vision(reader, args)

        for L in range(args.n_layers):
            a = f"layers.{L}.attn"
            m = f"model.{a}"
            yield from linear(f"{a}.wq_a", f"{m}.wq_a")
            yield f"{m}.q_norm.weight", get(f"{a}.q_norm.weight")
            yield from linear(f"{a}.wq_b", f"{m}.wq_b")
            yield from linear(f"{a}.wkv", f"{m}.wkv")
            yield f"{m}.kv_norm.weight", get(f"{a}.kv_norm.weight")
            # wo_a: FP8 in the checkpoint, dequantized to bf16 (reference bf16 einsum).
            yield f"{m}.wo_a", _dequant_fp8_block(
                get(f"{a}.wo_a.weight"), get(f"{a}.wo_a.scale")
            )
            yield from linear(f"{a}.wo_b", f"{m}.wo_b")
            yield f"{m}.attn_sink", get(f"{a}.attn_sink")

            ratio = args.compress_ratios[L]
            if ratio:
                c = f"{a}.compressor"
                for nm in ("ape", "wkv.weight", "wgate.weight", "norm.weight"):
                    yield f"model.{c}.{nm}", get(f"{c}.{nm}")
                if ratio == 4:
                    idx = f"{a}.indexer"
                    yield from linear(f"{idx}.wq_b", f"model.{idx}.wq_b")
                    yield f"model.{idx}.weights_proj.weight", get(f"{idx}.weights_proj.weight")
                    ic = f"{idx}.compressor"
                    for nm in ("ape", "wkv.weight", "wgate.weight", "norm.weight"):
                        yield f"model.{ic}.{nm}", get(f"{ic}.{nm}")

            yield f"model.layers.{L}.attn_norm.weight", get(f"layers.{L}.attn_norm.weight")
            yield f"model.layers.{L}.ffn_norm.weight", get(f"layers.{L}.ffn_norm.weight")

            g = f"layers.{L}.ffn.gate"
            yield f"model.{g}.weight", get(f"{g}.weight")
            if L < args.n_hash_layers:
                yield f"model.{g}.tid2eid", get(f"{g}.tid2eid")
                if args.vision_n_layers > 0:
                    # a vision checkpoint gives its hash layers a text bias too
                    yield f"model.{g}.bias", get(f"{g}.bias")
            else:
                yield f"model.{g}.bias", get(f"{g}.bias")
            if args.vision_n_layers > 0:
                yield f"model.{g}.bias_vl", get(f"{g}.bias_vl")
            for proj in ("w1", "w2", "w3"):
                src = f"layers.{L}.ffn.shared_experts.{proj}"
                yield from linear(src, f"model.{src}")

            for nm in (
                "hc_attn_fn", "hc_ffn_fn", "hc_attn_base",
                "hc_ffn_base", "hc_attn_scale", "hc_ffn_scale",
            ):
                yield f"model.layers.{L}.{nm}", get(f"layers.{L}.{nm}")
    finally:
        reader.close()


# --------------------------------------------------------------------------------------
# Vision tower.
# --------------------------------------------------------------------------------------
# The wrapper mounts the tower, the aligner and the sentinels under one ``visual``
# container, so every engine-side name carries the ``visual.`` prefix the shared
# VISION_KEY_PREFIXES filter matches; the checkpoint's unprefixed names
# (``vision.*``, ``aligner.*``, ``image_*``) are renamed here on load.
_VISION_BLOCK_SUFFIXES = (
    "norm1.weight",
    "attn.wqkv.weight",
    "attn.wqkv.bias",
    "attn.wo.weight",
    "attn.wo.bias",
    "norm2.weight",
    "mlp.w1.weight",
    "mlp.w2.weight",
)
_VISION_TAIL = (
    "vision.patch_embed.proj.weight",
    "vision.patch_embed.proj.bias",
    "vision.norm.weight",
    "aligner.w1.weight",
    "aligner.w1.bias",
    "aligner.w2.weight",
    "aligner.w2.bias",
    "image_start",
    "image_end",
    "image_newline",
    "image_pad",
)


def _iter_vision(reader, args: DeepseekV4Args):
    if args.vision_n_layers <= 0:
        return
    for L in range(args.vision_n_layers):
        for suffix in _VISION_BLOCK_SUFFIXES:
            yield f"visual.vision.blocks.{L}.{suffix}", reader.get(f"vision.blocks.{L}.{suffix}")
    for name in _VISION_TAIL:
        yield f"visual.{name}", reader.get(name)


def iter_vision_weights(model_path: str, device):
    """The vision tower alone, for the engine's encoder-only load and the FTW converter."""
    args = load_args(model_path, max_batch_size=1)
    reader = _ShardReader(model_path, _weight_map(model_path), device)
    try:
        yield from _iter_vision(reader, args)
    finally:
        reader.close()


# --------------------------------------------------------------------------------------
# Routed MXFP4 expert pieces.
# --------------------------------------------------------------------------------------
_EXPERT_RE = re.compile(
    r"^layers\.(?P<layer>\d+)\.ffn\.experts\.(?P<expert>\d+)\."
    r"(?P<proj>w1|w2|w3)\.(?P<kind>weight|scale)$"
)
_PROJ_ROLE = {"w1": "gate", "w3": "up", "w2": "down"}
_KIND_SUFFIX = {"weight": "", "scale": "_scale"}


def iter_expert_pieces(model_path: str, config, kind: QuantKind, *, layers=None, parallel: bool | None = False, workers: int = 8, chunk: int = 8 << 20):
    """Routed experts, one piece per expert: ``{gate, up, down}`` e2m1 pairs and their e8m0
    ``_scale`` companions (``w1`` / ``w3`` / ``w2``). The MTP layer's experts are skipped."""
    if kind is not QuantKind.MXFP4:
        return None
    if get_tp_info().size > 1:
        raise NotImplementedError("DeepSeek-V4 expert banks support TP=1 only")
    from freetoken.models.weight import iter_expert_tensors_parallel
    from freetoken.moe.expert_pieces import per_expert_pieces

    args = load_args(model_path, max_batch_size=1)
    L, E = args.n_layers, args.n_routed_experts
    keep = None if layers is None else set(layers)

    def locate(raw_name: str):
        m = _EXPERT_RE.match(raw_name)
        if m is None or int(m["layer"]) >= L or (keep is not None and int(m["layer"]) not in keep):
            return None
        return int(m["layer"]), int(m["expert"]), _PROJ_ROLE[m["proj"]] + _KIND_SUFFIX[m["kind"]]

    if parallel:
        tensors = iter_expert_tensors_parallel(model_path, lambda n: locate(n) is not None, workers=workers, chunk=chunk)
        return per_expert_pieces(tensors, locate, tensors_per_expert=6)

    def _serial():
        reader = _ShardReader(model_path, _weight_map(model_path), torch.device("cpu"))
        try:
            for li in tqdm(range(L) if keep is None else sorted(keep), desc="Loading DSV4 experts (serial)", disable=not get_tp_info().is_primary()):
                for e in range(E):
                    base = f"layers.{li}.ffn.experts.{e}"
                    for proj in ("w1", "w3", "w2"):
                        for kind_ in ("weight", "scale"):
                            name = f"{base}.{proj}.{kind_}"
                            yield name, reader.get(name)
        finally:
            reader.close()

    return per_expert_pieces(_serial(), locate, tensors_per_expert=6)


__all__ = ["iter_weights", "iter_expert_pieces", "iter_vision_weights"]