Skip to content

FreeToken brings datacenter-scale model serving to your desktop. Run massive models locally, fast and efficiently.

7.6 KB
Raw Blame
from __future__ import annotations

from typing import Iterator

import safetensors
import torch
from freetoken.distributed import get_tp_info
from freetoken.models.config import detect_compressed_tensors_nvfp4
from freetoken.models.loader import (
    CT_SCALE_SUFFIXES,
    ShardReader,
    ct_bf16_fuse,
    ct_nvfp4_fuse,
    iter_weight_files,
    nvfp4_parts_ct,
)
from freetoken.utils import cached_load_hf_config
from tqdm import tqdm

# The image path loads under the wrapper's vision_tower, adapter and projection included, so one prefix covers it.
_VISION_RENAMES = (
    ("model.vision_tower.", "vision_tower."),
    ("model.vision_adapter.", "vision_tower.adapter."),
    ("model.vision_projection.", "vision_tower.projection."),
)
_VISION_QKV_PARTS = (".attn.q_proj", ".attn.k_proj", ".attn.v_proj")

# Fused projections, concatenated on the output dim in this exact order to match the
# model's merged-linear splits. The full suffix keeps the standalone attention gate
# ``.self_attn.gate_proj`` out of the SwiGLU ``.mlp.gate_proj`` fusion.
_FUSIONS: dict[str, tuple[str, ...]] = {
    ".self_attn.qkv_proj": (".self_attn.q_proj", ".self_attn.k_proj", ".self_attn.v_proj"),
    ".mlp.gate_up_proj": (".mlp.gate_proj", ".mlp.up_proj"),
}


def _rename(raw_name: str) -> str:
    """HF text key -> FreeToken state-dict key."""
    if raw_name.startswith("model.language_model."):
        return "model." + raw_name[len("model.language_model.") :]
    if raw_name.startswith("language_model."):
        return "model." + raw_name[len("language_model.") :]
    return raw_name  # lm_head.weight


def _vision_name(raw_name: str) -> str | None:
    """FreeToken state-dict key of an image-path tensor, None for text keys."""
    for hf_prefix, prefix in _VISION_RENAMES:
        if raw_name.startswith(hf_prefix):
            return prefix + raw_name[len(hf_prefix) :]
    return None


def _vision_tensors(name: str, tensor: torch.Tensor, buf: dict) -> list[tuple[str, torch.Tensor]]:
    """What one image-path tensor yields: q/k/v weights and biases wait in buf and leave as one fused attn.qkv."""
    stem, _, leaf = name.rpartition(".")
    for idx, part in enumerate(_VISION_QKV_PARTS):
        if stem.endswith(part):
            key = f"{stem[: -len(part)]}.attn.qkv.{leaf}"
            slots = buf.setdefault(key, {})
            slots[idx] = tensor
            if len(slots) < len(_VISION_QKV_PARTS):
                return []
            del buf[key]
            return [(key, torch.cat([slots[i] for i in range(len(_VISION_QKV_PARTS))], dim=0))]
    return [(name, tensor)]


def iter_weights(
    model_path: str,
    device: torch.device,
    *,
    include_moe_experts: bool,
    include_non_moe: bool,
    include_vision: bool = True,
) -> Iterator[tuple[str, torch.Tensor]]:
    if not include_non_moe:
        return  # dense model: there is no experts-only pass
    if get_tp_info().size > 1:
        raise NotImplementedError("muse_glimmer weight loading currently supports TP=1 only")

    if detect_compressed_tensors_nvfp4(cached_load_hf_config(model_path)):
        yield from _iter_weights_compressed_tensors(model_path, device, include_vision)
        return

    fuse_buf: dict = {}
    vision_buf: dict = {}
    for file in tqdm(
        iter_weight_files(model_path),
        desc="Loading weights",
        disable=not get_tp_info().is_primary(),
    ):
        with safetensors.safe_open(file, framework="pt", device=str(device)) as f:
            for raw_name in f.keys():
                vision = _vision_name(raw_name)
                if vision is not None:
                    if include_vision:
                        yield from _vision_tensors(vision, f.get_tensor(raw_name), vision_buf)
                    continue
                name = _rename(raw_name)
                tensor = f.get_tensor(raw_name)
                if name.endswith(".weight"):
                    emit = ct_bf16_fuse(name[: -len(".weight")], tensor, fuse_buf, _FUSIONS)
                    if emit is not None:
                        yield from emit
                        continue
                # All norms pass through raw: the decoder's centered (1+w) norms apply the
                # +1 at runtime (GemmaPlusOneRMSNorm), the final norm / qk norms need none.
                yield name, tensor

    assert not fuse_buf, f"Incomplete projection fusions: {list(fuse_buf.keys())}"
    assert not vision_buf, f"Incomplete vision qkv fusions: {list(vision_buf.keys())}"


def iter_vision_weights(model_path: str, device: torch.device) -> Iterator[tuple[str, torch.Tensor]]:
    """The vision tower alone, named as iter_weights names it."""
    vision_buf: dict = {}
    for file in iter_weight_files(model_path):
        with safetensors.safe_open(file, framework="pt", device=str(device)) as f:
            for raw_name in f.keys():
                vision = _vision_name(raw_name)
                if vision is not None:
                    yield from _vision_tensors(vision, f.get_tensor(raw_name), vision_buf)
    assert not vision_buf, f"Incomplete vision qkv fusions: {list(vision_buf.keys())}"


def _iter_weights_compressed_tensors(
    model_path: str, device: torch.device, include_vision: bool
) -> Iterator[tuple[str, torch.Tensor]]:
    """Dense pass for the compressed-tensors NVFP4 checkpoint.

    Every text Linear (q/k/v/o, the attention gate, the MLP) is kept native NVFP4
    (W4A16): ``.weight`` (uint8 packed) + ``.weight_scale`` (fp8 block) + ``.weight_global``
    (fp32 per-row, the reciprocal of the stored quant-side global). q/k/v fuse into
    ``qkv_proj`` and gate/up into ``gate_up_proj`` on the output dim, each part keeping
    its own scales, so the fused FP4 weights are exact. Embeddings, norms, lm_head and the
    vision tower are bf16 (the checkpoint's ignore list). Scale lookups go through the
    shard-map reader: they can land in a different shard than their weight_packed."""
    nvfp4_buf: dict = {}
    vision_buf: dict = {}
    reader = ShardReader(model_path, device)
    try:
        for file in tqdm(
            reader.files(),
            desc="Loading compressed-tensors weights",
            disable=not get_tp_info().is_primary(),
        ):
            for raw_name in reader.names_in(file):
                if raw_name.endswith(CT_SCALE_SUFFIXES):
                    continue  # consumed with their weight_packed
                vision = _vision_name(raw_name)
                if vision is not None:
                    if include_vision:
                        yield from _vision_tensors(vision, reader.get_tensor(raw_name), vision_buf)
                    continue
                name = _rename(raw_name)
                if raw_name.endswith(".weight_packed"):
                    base = name[: -len(".weight_packed")]
                    parts = nvfp4_parts_ct(reader, raw_name[: -len(".weight_packed")])
                    emit = ct_nvfp4_fuse(base, parts, nvfp4_buf, _FUSIONS)
                    if emit is not None:
                        yield from emit
                    else:  # standalone: o_proj, down_proj
                        w, s, g, a = parts
                        yield base + ".weight", w
                        yield base + ".weight_scale", s
                        yield base + ".weight_global", g
                        if a is not None:
                            yield base + ".input_scale", a
                    continue
                yield name, reader.get_tensor(raw_name)
    finally:
        reader.close()

    assert not nvfp4_buf, f"Incomplete NVFP4 fusions: {list(nvfp4_buf.keys())}"
    assert not vision_buf, f"Incomplete vision qkv fusions: {list(vision_buf.keys())}"


__all__ = ["iter_weights"]