Skip to content

FreeToken brings datacenter-scale model serving to your desktop. Run massive models locally, fast and efficiently.

12.4 KB
Raw Blame
from __future__ import annotations

import glob
import json
import os
import re
import struct
from dataclasses import dataclass
from typing import Iterable, Iterator

import torch
from freetoken.utils import div_ceil, download_hf_weight

SPLIT_DIM_0 = (".q_proj", ".k_proj", ".v_proj", ".gate_proj", ".up_proj")
SPLIT_DIM_1 = (".o_proj", ".down_proj")


@dataclass(frozen=True)
class MergeRule:
    fused_suffix: str
    slot: str
    slots: tuple[str, ...]


def shard_tensor(
    key: str,
    value: torch.Tensor,
    *,
    rank: int,
    world_size: int,
    num_kv_heads: int,
) -> torch.Tensor:
    if any(key.count(sub) for sub in SPLIT_DIM_0):
        is_kv_proj = any(key.count(sub) for sub in (".k_proj", ".v_proj"))
        if is_kv_proj and num_kv_heads is not None and num_kv_heads < world_size:
            head_dim = value.shape[0] // num_kv_heads
            head_idx = rank * num_kv_heads // world_size
            return value[head_idx * head_dim : (head_idx + 1) * head_dim].clone()
        return value.chunk(world_size, dim=0)[rank].clone()
    if any(key.count(sub) for sub in SPLIT_DIM_1):
        return value.chunk(world_size, dim=1)[rank].clone()
    if key.count("lm_head") or key.count("embed_tokens"):
        num_embeddings = value.shape[0]
        num_embeddings_per_partition = div_ceil(num_embeddings, world_size)
        vocab_start_idx = rank * num_embeddings_per_partition
        vocab_end_idx = min((rank + 1) * num_embeddings_per_partition, num_embeddings)
        return value[vocab_start_idx:vocab_end_idx, :].clone()
    return value


def iter_weight_files(model_path: str) -> list[str]:
    model_folder = download_hf_weight(model_path)
    # The index is authoritative: download_hf_weight only fetches the shards it names, and it
    # covers subdirectory layouts (nvidia's FP4-Local releases ship tensors/model-*.safetensors).
    index = os.path.join(model_folder, "model.safetensors.index.json")
    if os.path.exists(index):
        with open(index, encoding="utf-8") as f:
            shards = sorted(set(json.load(f)["weight_map"].values()))
        return [os.path.join(model_folder, shard) for shard in shards]
    files = glob.glob(f"{model_folder}/*.safetensors")
    return [f for f in files if not f.endswith("consolidated.safetensors")] or files


def safetensors_weight_map(folder: str) -> dict[str, str]:
    """Tensor name -> shard basename, from the index or from each shard's header when the checkpoint ships none."""
    index = os.path.join(folder, "model.safetensors.index.json")
    if os.path.exists(index):
        with open(index, encoding="utf-8") as f:
            return json.load(f)["weight_map"]
    weight_map: dict[str, str] = {}
    for path in sorted(iter_weight_files(folder)):
        with open(path, "rb") as fh:
            n = struct.unpack("<Q", fh.read(8))[0]
            header = json.loads(fh.read(n))
        for name in header:
            if name != "__metadata__":
                weight_map[name] = os.path.basename(path)
    return weight_map


def drop_page_cache(path: str) -> None:
    """drop a file's page cache: banks + full checkpoint cache don't both fit in host RAM (OOM)."""
    try:
        fd = os.open(path, os.O_RDONLY)
        try:
            os.posix_fadvise(fd, 0, 0, os.POSIX_FADV_DONTNEED)
        finally:
            os.close(fd)
    except OSError:
        pass


def iter_root_safetensor_files_from_index(
    model_path: str,
    *,
    index_file: str = "model.safetensors.index.json",
) -> list[str]:
    model_folder = download_hf_weight(model_path)
    if os.path.basename(os.path.normpath(model_folder)) in {"metal", "original"}:
        raise ValueError("GPT-OSS loading requires the root GPT-OSS model directory")

    root_files = sorted(glob.glob(os.path.join(model_folder, "*.safetensors")))
    index_path = os.path.join(model_folder, index_file)
    if not os.path.isfile(index_path):
        files = [f for f in root_files if not f.endswith("consolidated.safetensors")]
        if not files:
            raise ValueError("No root GPT-OSS safetensors shards found")
        return files

    with open(index_path, encoding="utf-8") as f:
        weight_map = json.load(f)["weight_map"]

    indexed_files = []
    for filename in dict.fromkeys(weight_map.values()):
        if os.path.dirname(filename):
            continue
        path = os.path.join(model_folder, filename)
        if path in root_files:
            indexed_files.append(path)

    if not indexed_files:
        raise ValueError(
            "No root GPT-OSS safetensors shards found from model.safetensors.index.json"
        )
    return sorted(indexed_files)


def _merge_info(key: str, rules: dict[str, MergeRule]) -> tuple[str, MergeRule] | None:
    for suffix, rule in rules.items():
        if key.endswith(suffix + ".weight") or key.endswith(suffix) or key.count(suffix):
            return key.replace(suffix, rule.fused_suffix), rule
    return None


def iter_merged_tensors(
    tensors: Iterable[tuple[str, torch.Tensor]],
    rules: dict[str, MergeRule],
    *,
    model_name: str,
) -> Iterator[tuple[str, torch.Tensor]]:
    merge_buf: dict[str, dict[str, torch.Tensor]] = {}
    for name, tensor in tensors:
        info = _merge_info(name, rules)
        if info is None:
            yield name, tensor
            continue
        merged_key, rule = info
        slots = merge_buf.setdefault(merged_key, {})
        slots[rule.slot] = tensor
        if not all(slot in slots for slot in rule.slots):
            continue
        parts = [slots[slot] for slot in rule.slots]
        del merge_buf[merged_key]
        yield merged_key, torch.cat(parts, dim=0)

    assert not merge_buf, (
        f"{model_name}: Incomplete merge groups in checkpoint: {list(merge_buf.keys())}"
    )


# ---------------------------------------------------------------------------------
# compressed-tensors NVFP4 (llm-compressor) dense-weight helpers, shared by the
# models that serve such checkpoints natively (muse_glimmer). Storage:
# ``weight_packed`` (uint8 [O, IN//2]) + ``weight_scale`` (fp8-e4m3 block [O, IN//16])
# + a scalar ``weight_global_scale``. The stored global is the *quant-side* scale, so
# the dequant/native global is its reciprocal (vLLM inverts it identically).
# ---------------------------------------------------------------------------------

# Quant scales consumed with their ``weight_packed``.
CT_SCALE_SUFFIXES = (
    ".weight_scale", ".weight_global_scale", ".input_global_scale", ".input_scale",
)


class ShardReader:
    """Serves tensors by name across safetensors shards (handles opened lazily).

    The quant scales of a ``weight_packed`` can land in a DIFFERENT shard than the
    packed weight itself (Muse-Glimmer-30B-NVFP4 splits layer 49's down_proj across
    the shard boundary), so sibling lookups must go through the index's weight_map
    rather than the shard the packed weight came from."""

    def __init__(self, model_path: str, device: torch.device):
        folder = download_hf_weight(model_path)
        self._map = {name: os.path.join(folder, shard) for name, shard in safetensors_weight_map(folder).items()}
        self._device = str(device)
        self._handles: dict[str, object] = {}

    def files(self) -> list[str]:
        return sorted(set(self._map.values()))

    def has(self, name: str) -> bool:
        return name in self._map

    def names_in(self, file: str) -> list[str]:
        return [name for name, shard in self._map.items() if shard == file]

    def get_tensor(self, name: str) -> torch.Tensor:
        import safetensors

        file = self._map[name]
        h = self._handles.get(file)
        if h is None:
            h = safetensors.safe_open(file, framework="pt", device=self._device).__enter__()
            self._handles[file] = h
        return h.get_tensor(name)

    def close(self) -> None:
        for h in self._handles.values():
            try:
                h.__exit__(None, None, None)
            except Exception:  # noqa: BLE001 -- best-effort handle cleanup
                pass
        self._handles.clear()


def nvfp4_parts_ct(f, raw_base: str) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor | None]:
    """compressed-tensors NVFP4 -> ``(packed uint8 [O, IN//2], block scale fp8 [O, IN//16], per-output-row global fp32 [O], dequant-side input_scale fp32 scalar or None)`` for the NVFP4 linear method."""
    w = f.get_tensor(raw_base + ".weight_packed")
    s = f.get_tensor(raw_base + ".weight_scale")
    wg = f.get_tensor(raw_base + ".weight_global_scale").reshape(1).to(torch.float32)
    g = (1.0 / wg).expand(w.shape[0]).contiguous()
    a = None
    if f.has(raw_base + ".input_global_scale"):
        a = (1.0 / f.get_tensor(raw_base + ".input_global_scale").reshape(()).to(torch.float32))
    return w, s, g, a


def ct_nvfp4_fuse(base: str, parts_tuple: tuple, buf: dict, groups: dict[str, tuple[str, ...]]):
    """Buffer a native NVFP4 fusion part ``(w, s, g, a)``; once complete, emit the concatenated ``.weight`` / ``.weight_scale`` / ``.weight_global`` (output-dim concat, each part keeps its own scales, so the fused FP4 weight is exact) plus the largest ``.input_scale`` when every part has one.
    ``[]`` while incomplete, ``None`` if ``base`` is not a fusion part of any group in ``groups``."""
    for fused_suffix, parts in groups.items():
        for idx, part in enumerate(parts):
            if base.endswith(part):
                key = base[: -len(part)] + fused_suffix
                slots = buf.setdefault(key, {})
                slots[idx] = parts_tuple
                if len(slots) < len(parts):
                    return []
                del buf[key]
                ws = [slots[i][0] for i in range(len(parts))]
                ss = [slots[i][1] for i in range(len(parts))]
                gs = [slots[i][2] for i in range(len(parts))]
                acts = [slots[i][3] for i in range(len(parts))]
                out = [
                    (key + ".weight", torch.cat(ws, dim=0)),
                    (key + ".weight_scale", torch.cat(ss, dim=0)),
                    (key + ".weight_global", torch.cat(gs, dim=0)),
                ]
                if all(a is not None for a in acts):
                    out.append((key + ".input_scale", torch.stack(acts).max()))
                return out
    return None


def ct_bf16_fuse(base: str, tensor: torch.Tensor, buf: dict, groups: dict[str, tuple[str, ...]]):
    """Buffer a bf16 fusion part; emit the concatenated ``.weight`` once complete, ``[]``
    while incomplete, ``None`` if ``base`` is not a part of any group in ``groups``."""
    for fused_suffix, parts in groups.items():
        for idx, part in enumerate(parts):
            if base.endswith(part):
                key = base[: -len(part)] + fused_suffix
                slots = buf.setdefault(key, {})
                slots[idx] = tensor
                if len(slots) < len(parts):
                    return []
                del buf[key]
                return [(key + ".weight", torch.cat([slots[i] for i in range(len(parts))], dim=0))]
    return None


def _expert_stack_info(key: str, expert_pattern: re.Pattern[str]) -> tuple[str, int] | None:
    match = expert_pattern.match(key)
    if match is None:
        return None
    packed_name = match.group("name")
    if packed_name.endswith(".weight"):
        packed_name = packed_name.removesuffix(".weight")
    return f"{match.group('prefix')}.{packed_name}", int(match.group("idx"))


def iter_stacked_experts(
    tensors: Iterable[tuple[str, torch.Tensor]],
    *,
    num_experts: int,
    model_name: str,
    expert_pattern: re.Pattern[str],
) -> Iterator[tuple[str, torch.Tensor]]:
    expert_buf: dict[str, dict[int, torch.Tensor]] = {}
    for name, tensor in tensors:
        expert_info = _expert_stack_info(name, expert_pattern)
        if expert_info is None:
            yield name, tensor
            continue
        packed_key, expert_idx = expert_info
        slots = expert_buf.setdefault(packed_key, {})
        slots[expert_idx] = tensor
        if len(slots) != num_experts:
            continue
        experts = [slots[idx] for idx in range(num_experts)]
        del expert_buf[packed_key]
        yield packed_key, torch.stack(experts, dim=0)

    assert not expert_buf, (
        f"{model_name}: Incomplete expert tensors in checkpoint: {list(expert_buf.keys())}"
    )


__all__ = [
    "MergeRule",
    "iter_root_safetensor_files_from_index",
    "iter_weight_files",
    "safetensors_weight_map",
    "iter_merged_tensors",
    "iter_stacked_experts",
    "shard_tensor",
]