FreeToken brings datacenter-scale model serving to your desktop. Run massive models locally, fast and efficiently.
from __future__ import annotations
import glob
import json
import os
import re
import struct
from dataclasses import dataclass
from typing import Iterable, Iterator
import torch
from freetoken.utils import div_ceil, download_hf_weight
SPLIT_DIM_0 = (".q_proj", ".k_proj", ".v_proj", ".gate_proj", ".up_proj")
SPLIT_DIM_1 = (".o_proj", ".down_proj")
@dataclass(frozen=True)
class MergeRule:
fused_suffix: str
slot: str
slots: tuple[str, ...]
def shard_tensor(
key: str,
value: torch.Tensor,
*,
rank: int,
world_size: int,
num_kv_heads: int,
) -> torch.Tensor:
if any(key.count(sub) for sub in SPLIT_DIM_0):
is_kv_proj = any(key.count(sub) for sub in (".k_proj", ".v_proj"))
if is_kv_proj and num_kv_heads is not None and num_kv_heads < world_size:
head_dim = value.shape[0] // num_kv_heads
head_idx = rank * num_kv_heads // world_size
return value[head_idx * head_dim : (head_idx + 1) * head_dim].clone()
return value.chunk(world_size, dim=0)[rank].clone()
if any(key.count(sub) for sub in SPLIT_DIM_1):
return value.chunk(world_size, dim=1)[rank].clone()
if key.count("lm_head") or key.count("embed_tokens"):
num_embeddings = value.shape[0]
num_embeddings_per_partition = div_ceil(num_embeddings, world_size)
vocab_start_idx = rank * num_embeddings_per_partition
vocab_end_idx = min((rank + 1) * num_embeddings_per_partition, num_embeddings)
return value[vocab_start_idx:vocab_end_idx, :].clone()
return value
def iter_weight_files(model_path: str) -> list[str]:
model_folder = download_hf_weight(model_path)
# The index is authoritative: download_hf_weight only fetches the shards it names, and it
# covers subdirectory layouts (nvidia's FP4-Local releases ship tensors/model-*.safetensors).
index = os.path.join(model_folder, "model.safetensors.index.json")
if os.path.exists(index):
with open(index, encoding="utf-8") as f:
shards = sorted(set(json.load(f)["weight_map"].values()))
return [os.path.join(model_folder, shard) for shard in shards]
files = glob.glob(f"{model_folder}/*.safetensors")
return [f for f in files if not f.endswith("consolidated.safetensors")] or files
def safetensors_weight_map(folder: str) -> dict[str, str]:
"""Tensor name -> shard basename, from the index or from each shard's header when the checkpoint ships none."""
index = os.path.join(folder, "model.safetensors.index.json")
if os.path.exists(index):
with open(index, encoding="utf-8") as f:
return json.load(f)["weight_map"]
weight_map: dict[str, str] = {}
for path in sorted(iter_weight_files(folder)):
with open(path, "rb") as fh:
n = struct.unpack("<Q", fh.read(8))[0]
header = json.loads(fh.read(n))
for name in header:
if name != "__metadata__":
weight_map[name] = os.path.basename(path)
return weight_map
def drop_page_cache(path: str) -> None:
"""drop a file's page cache: banks + full checkpoint cache don't both fit in host RAM (OOM)."""
try:
fd = os.open(path, os.O_RDONLY)
try:
os.posix_fadvise(fd, 0, 0, os.POSIX_FADV_DONTNEED)
finally:
os.close(fd)
except OSError:
pass
def iter_root_safetensor_files_from_index(
model_path: str,
*,
index_file: str = "model.safetensors.index.json",
) -> list[str]:
model_folder = download_hf_weight(model_path)
if os.path.basename(os.path.normpath(model_folder)) in {"metal", "original"}:
raise ValueError("GPT-OSS loading requires the root GPT-OSS model directory")
root_files = sorted(glob.glob(os.path.join(model_folder, "*.safetensors")))
index_path = os.path.join(model_folder, index_file)
if not os.path.isfile(index_path):
files = [f for f in root_files if not f.endswith("consolidated.safetensors")]
if not files:
raise ValueError("No root GPT-OSS safetensors shards found")
return files
with open(index_path, encoding="utf-8") as f:
weight_map = json.load(f)["weight_map"]
indexed_files = []
for filename in dict.fromkeys(weight_map.values()):
if os.path.dirname(filename):
continue
path = os.path.join(model_folder, filename)
if path in root_files:
indexed_files.append(path)
if not indexed_files:
raise ValueError(
"No root GPT-OSS safetensors shards found from model.safetensors.index.json"
)
return sorted(indexed_files)
def _merge_info(key: str, rules: dict[str, MergeRule]) -> tuple[str, MergeRule] | None:
for suffix, rule in rules.items():
if key.endswith(suffix + ".weight") or key.endswith(suffix) or key.count(suffix):
return key.replace(suffix, rule.fused_suffix), rule
return None
def iter_merged_tensors(
tensors: Iterable[tuple[str, torch.Tensor]],
rules: dict[str, MergeRule],
*,
model_name: str,
) -> Iterator[tuple[str, torch.Tensor]]:
merge_buf: dict[str, dict[str, torch.Tensor]] = {}
for name, tensor in tensors:
info = _merge_info(name, rules)
if info is None:
yield name, tensor
continue
merged_key, rule = info
slots = merge_buf.setdefault(merged_key, {})
slots[rule.slot] = tensor
if not all(slot in slots for slot in rule.slots):
continue
parts = [slots[slot] for slot in rule.slots]
del merge_buf[merged_key]
yield merged_key, torch.cat(parts, dim=0)
assert not merge_buf, (
f"{model_name}: Incomplete merge groups in checkpoint: {list(merge_buf.keys())}"
)
# ---------------------------------------------------------------------------------
# compressed-tensors NVFP4 (llm-compressor) dense-weight helpers, shared by the
# models that serve such checkpoints natively (muse_glimmer). Storage:
# ``weight_packed`` (uint8 [O, IN//2]) + ``weight_scale`` (fp8-e4m3 block [O, IN//16])
# + a scalar ``weight_global_scale``. The stored global is the *quant-side* scale, so
# the dequant/native global is its reciprocal (vLLM inverts it identically).
# ---------------------------------------------------------------------------------
# Quant scales consumed with their ``weight_packed``.
CT_SCALE_SUFFIXES = (
".weight_scale", ".weight_global_scale", ".input_global_scale", ".input_scale",
)
class ShardReader:
"""Serves tensors by name across safetensors shards (handles opened lazily).
The quant scales of a ``weight_packed`` can land in a DIFFERENT shard than the
packed weight itself (Muse-Glimmer-30B-NVFP4 splits layer 49's down_proj across
the shard boundary), so sibling lookups must go through the index's weight_map
rather than the shard the packed weight came from."""
def __init__(self, model_path: str, device: torch.device):
folder = download_hf_weight(model_path)
self._map = {name: os.path.join(folder, shard) for name, shard in safetensors_weight_map(folder).items()}
self._device = str(device)
self._handles: dict[str, object] = {}
def files(self) -> list[str]:
return sorted(set(self._map.values()))
def has(self, name: str) -> bool:
return name in self._map
def names_in(self, file: str) -> list[str]:
return [name for name, shard in self._map.items() if shard == file]
def get_tensor(self, name: str) -> torch.Tensor:
import safetensors
file = self._map[name]
h = self._handles.get(file)
if h is None:
h = safetensors.safe_open(file, framework="pt", device=self._device).__enter__()
self._handles[file] = h
return h.get_tensor(name)
def close(self) -> None:
for h in self._handles.values():
try:
h.__exit__(None, None, None)
except Exception: # noqa: BLE001 -- best-effort handle cleanup
pass
self._handles.clear()
def nvfp4_parts_ct(f, raw_base: str) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor | None]:
"""compressed-tensors NVFP4 -> ``(packed uint8 [O, IN//2], block scale fp8 [O, IN//16], per-output-row global fp32 [O], dequant-side input_scale fp32 scalar or None)`` for the NVFP4 linear method."""
w = f.get_tensor(raw_base + ".weight_packed")
s = f.get_tensor(raw_base + ".weight_scale")
wg = f.get_tensor(raw_base + ".weight_global_scale").reshape(1).to(torch.float32)
g = (1.0 / wg).expand(w.shape[0]).contiguous()
a = None
if f.has(raw_base + ".input_global_scale"):
a = (1.0 / f.get_tensor(raw_base + ".input_global_scale").reshape(()).to(torch.float32))
return w, s, g, a
def ct_nvfp4_fuse(base: str, parts_tuple: tuple, buf: dict, groups: dict[str, tuple[str, ...]]):
"""Buffer a native NVFP4 fusion part ``(w, s, g, a)``; once complete, emit the concatenated ``.weight`` / ``.weight_scale`` / ``.weight_global`` (output-dim concat, each part keeps its own scales, so the fused FP4 weight is exact) plus the largest ``.input_scale`` when every part has one.
``[]`` while incomplete, ``None`` if ``base`` is not a fusion part of any group in ``groups``."""
for fused_suffix, parts in groups.items():
for idx, part in enumerate(parts):
if base.endswith(part):
key = base[: -len(part)] + fused_suffix
slots = buf.setdefault(key, {})
slots[idx] = parts_tuple
if len(slots) < len(parts):
return []
del buf[key]
ws = [slots[i][0] for i in range(len(parts))]
ss = [slots[i][1] for i in range(len(parts))]
gs = [slots[i][2] for i in range(len(parts))]
acts = [slots[i][3] for i in range(len(parts))]
out = [
(key + ".weight", torch.cat(ws, dim=0)),
(key + ".weight_scale", torch.cat(ss, dim=0)),
(key + ".weight_global", torch.cat(gs, dim=0)),
]
if all(a is not None for a in acts):
out.append((key + ".input_scale", torch.stack(acts).max()))
return out
return None
def ct_bf16_fuse(base: str, tensor: torch.Tensor, buf: dict, groups: dict[str, tuple[str, ...]]):
"""Buffer a bf16 fusion part; emit the concatenated ``.weight`` once complete, ``[]``
while incomplete, ``None`` if ``base`` is not a part of any group in ``groups``."""
for fused_suffix, parts in groups.items():
for idx, part in enumerate(parts):
if base.endswith(part):
key = base[: -len(part)] + fused_suffix
slots = buf.setdefault(key, {})
slots[idx] = tensor
if len(slots) < len(parts):
return []
del buf[key]
return [(key + ".weight", torch.cat([slots[i] for i in range(len(parts))], dim=0))]
return None
def _expert_stack_info(key: str, expert_pattern: re.Pattern[str]) -> tuple[str, int] | None:
match = expert_pattern.match(key)
if match is None:
return None
packed_name = match.group("name")
if packed_name.endswith(".weight"):
packed_name = packed_name.removesuffix(".weight")
return f"{match.group('prefix')}.{packed_name}", int(match.group("idx"))
def iter_stacked_experts(
tensors: Iterable[tuple[str, torch.Tensor]],
*,
num_experts: int,
model_name: str,
expert_pattern: re.Pattern[str],
) -> Iterator[tuple[str, torch.Tensor]]:
expert_buf: dict[str, dict[int, torch.Tensor]] = {}
for name, tensor in tensors:
expert_info = _expert_stack_info(name, expert_pattern)
if expert_info is None:
yield name, tensor
continue
packed_key, expert_idx = expert_info
slots = expert_buf.setdefault(packed_key, {})
slots[expert_idx] = tensor
if len(slots) != num_experts:
continue
experts = [slots[idx] for idx in range(num_experts)]
del expert_buf[packed_key]
yield packed_key, torch.stack(experts, dim=0)
assert not expert_buf, (
f"{model_name}: Incomplete expert tensors in checkpoint: {list(expert_buf.keys())}"
)
__all__ = [
"MergeRule",
"iter_root_safetensor_files_from_index",
"iter_weight_files",
"safetensors_weight_map",
"iter_merged_tensors",
"iter_stacked_experts",
"shard_tensor",
]