FreeToken brings datacenter-scale model serving to your desktop. Run massive models locally, fast and efficiently.
"""Expert banks for the offload MoE cache: load, pack and pin the routed experts.
The expert kernel (``QuantMethod.kernel``) owns the bank layout and the pack step; the
checkpoint side delivers pieces (``moe.expert_pieces``) and this module fills the pinned host
banks from them (``build_expert_banks``). GGUF experts keep their packed ggml rows
(``models.gguf.experts``).
"""
from __future__ import annotations
import math
import os
from dataclasses import dataclass, field
import torch
from freetoken.layers.quantization import QuantKind
from freetoken.utils import init_logger
from .host_banks import HostBank
from .offload_cache import _BANK_BYTES_PER_EXPERT, placeholder_bank
logger = init_logger(__name__)
# the parallel expert-bank reader needs POSIX O_DIRECT + preadv; without them the serial (safetensors/mmap) build is the only option
_PARALLEL_READER_SUPPORTED = hasattr(os, "O_DIRECT") and hasattr(os, "preadv")
@dataclass(frozen=True)
class ExpertBanks:
"""Loaded expert banks, normalized for ``OffloadMoeCache`` wiring."""
quant_format: str # _BANK_SCHEMAS key
# Pinned host banks, keyed by the format's schema: one [num_experts, ...]
# tensor per layer (independent allocations -> per-layer host attributes).
sources: dict[str, list[torch.Tensor]]
# marlin/b12x per-expert global scales ([L*E]); None for formats without them
gate_up_alpha: torch.Tensor | None = field(default=None)
down_alpha: torch.Tensor | None = field(default=None)
# per-layer HostResidency values actually applied by the loader; None -> all pinned (also the degrade signal when a request was not honored)
layer_residency: list[str] | None = field(default=None)
# True iff the ``layer_sink`` passed to the loader was actually engaged (each layer
# streamed straight to its sink instead of staying materialized here) -- set by
# convert.py's per-format streaming gate; ``sources`` may hold released tensors.
streamed: bool = False
# the expert (kind, kernel) the banks were packed for; None for the legacy providers
kind: QuantKind | None = None
kernel: str | None = None
layout: dict | None = None
# per-layer quant tag (e.g. the gate_up and down GGUF types), for layers whose banks match in dtype and row shape but not in quant; None = one quant
layer_tags: list | None = None
# per bank, per layer: bytes past an expert matrix's last row that the kernel reads and that must read as zeros (ggml MMQ); None = no such reads
row_tail_bytes: dict[str, list[int]] | None = None
# per-layer quant_format of experts in several formats (``quant_format``/``kind``/``kernel`` then name the most common one); None = every layer is ``quant_format``
layer_formats: list[str] | None = None
# the residency plan holding DEFERRED layers' banks, for host_banks.probe_pin; None when nothing was deferred
deferred_plan: object | None = None
def _dummy_fill(role: str, tensor: torch.Tensor) -> None:
"""Random but finite bank contents for --use-dummy-weight."""
if role.endswith("_scale"):
if tensor.dtype is torch.uint8:
tensor.fill_(127) # e8m0 exponent code for 1.0
else:
tensor.fill_(1.0)
elif role.endswith("_global"):
tensor.fill_(0.01)
elif tensor.dtype in (torch.uint8, torch.int32):
tensor.view(torch.uint8).random_(0, 256)
elif tensor.dtype is torch.float8_e4m3fn:
tensor.view(torch.uint8).random_(0, 16) # small codes, no NaN / inf
else:
tensor.normal_()
def build_expert_banks(
method,
num_layers: int,
pieces,
*,
device: torch.device,
layer_sink=None,
dummy: bool = False,
resident: bool = False,
layers: list[int] | None = None,
) -> ExpertBanks:
"""Fill host banks in the kernel's layout from a stream of expert pieces.
``pieces`` yields ``(layer_id, e0, e1, {role: tensor[e1 - e0, ...]})`` in any order;
each batch is packed in place into rows ``e0:e1`` of that layer's banks. A layer is
complete once its ``num_experts`` rows have arrived: with ``layer_sink=None`` its banks
are pinned in the background, otherwise the sink receives them (converter). ``dummy``
skips the pieces and fills the banks with finite random contents.
``resident`` packs into banks on ``device`` instead of host banks (the resident 'fused' experts).
``layers`` fills only those bank layers (the ones ``method`` serves, for experts of several
formats): pieces of other layers are skipped and the other layers' sources are None.
"""
from freetoken.moe.host_banks import LayerCompletionTracker, PinPipeline, pin_banks
from freetoken.moe.legacy_format import legacy_format_for
kernel = method.kernel
layout = method.layout()
E = method.cfg.num_experts
ids = list(range(num_layers)) if layers is None else sorted(layers)
filled = set(ids)
specs = {role: ((E, *spec.shape), spec.dtype) for role, spec in layout.items() if not spec.resident}
if resident:
assert layer_sink is None
hb = None
banks = {role: {l: torch.empty(shape, dtype=dtype, device=device) for l in ids} for role, (shape, dtype) in specs.items()}
else:
hb = {role: {l: HostBank(shape, dtype) for l in ids} for role, (shape, dtype) in specs.items()}
banks = {role: {l: b.tensor for l, b in per_layer.items()} for role, per_layer in hb.items()}
alphas = {
role: torch.empty(num_layers * E, dtype=spec.dtype, device=device)
for role, spec in layout.items() if spec.resident
}
def _sources() -> dict[str, list]:
return {role: [per_layer.get(l) for l in range(num_layers)] for role, per_layer in banks.items()}
if dummy:
for role, per_layer in banks.items():
for tensor in per_layer.values():
_dummy_fill(role, tensor)
for alpha in alphas.values():
alpha.fill_(1.0)
if hb is not None and torch.cuda.is_available():
pin_banks(hb)
return ExpertBanks(
legacy_format_for(method.kind, kernel.name), _sources(),
gate_up_alpha=alphas.get("gate_up_alpha"), down_alpha=alphas.get("down_alpha"),
kind=method.kind, kernel=kernel.name, layout=layout,
)
def _fill(sink) -> None:
tracker = LayerCompletionTracker(E, hb, sink) if sink is not None else None
# a reader that skips a layer or mislabels a piece must fail here, not serve uninitialized rows
written = torch.zeros(num_layers, E, dtype=torch.int32)
for layer_id, e0, e1, piece in pieces:
if not (0 <= layer_id < num_layers and 0 <= e0 < e1 <= E):
raise ValueError(f"expert piece out of range: layer {layer_id}, experts {e0}:{e1} of {num_layers} x {E}")
if layer_id not in filled:
continue
# refuse before writing: a duplicate row would also complete the layer early and hand the sink a half-filled bank
if written[layer_id, e0:e1].any():
raise ValueError(f"expert rows written more than once: layer {layer_id}, experts {e0}:{e1}")
written[layer_id, e0:e1] = 1
out = {role: banks[role][layer_id][e0:e1] for role in specs}
got = method.pack(piece, out)
for role, values in got.items():
alphas[role][layer_id * E + e0 : layer_id * E + e1] = values.to(alphas[role].dtype)
if tracker is not None:
for _ in range(e1 - e0):
tracker.note(layer_id)
missing = (written[ids] == 0).nonzero().tolist()
if missing:
missing = [[ids[l], e] for l, e in missing]
raise ValueError(f"expert banks were not filled: {len(missing)} (layer, expert) rows missing (first {missing[:4]})")
if layer_sink is not None:
_fill(layer_sink)
elif torch.cuda.is_available() and not resident:
with PinPipeline() as pins:
_fill(pins)
else:
_fill(None)
return ExpertBanks(
legacy_format_for(method.kind, kernel.name), _sources(),
gate_up_alpha=alphas.get("gate_up_alpha"), down_alpha=alphas.get("down_alpha"),
streamed=layer_sink is not None, kind=method.kind, kernel=kernel.name, layout=layout,
)
def method_groups(layer_methods) -> list[tuple]:
"""``[(method, [bank layers])]`` of the layers' expert methods, one entry per (kind, kernel), most layers first."""
groups: dict = {}
for layer, method in enumerate(layer_methods):
groups.setdefault((method.kind, method.kernel.name), (method, []))[1].append(layer)
return sorted(groups.values(), key=lambda g: (-len(g[1]), g[1][0]))
def is_mixed(layer_methods) -> bool:
return layer_methods is not None and len(method_groups(layer_methods)) > 1
def merge_expert_banks(parts: list[ExpertBanks], layer_methods, num_experts: int) -> ExpertBanks:
"""One ``ExpertBanks`` over the per-format ``parts`` (each filled only on its layers): the banks are the
union of the roles, a layer without a role gets a zero-byte placeholder (see ``OffloadMoeCache.layer_layouts``)."""
from freetoken.moe.legacy_format import legacy_format_for
num_layers = len(layer_methods)
sources: dict[str, list] = {}
for part in parts:
if part.gate_up_alpha is not None or part.down_alpha is not None:
raise ValueError(f"experts of several formats cannot use the {part.kernel} kernel, which keeps GPU-resident alphas")
for role, per_layer in part.sources.items():
merged = sources.setdefault(role, [None] * num_layers)
for layer, tensor in enumerate(per_layer):
if tensor is not None:
assert merged[layer] is None, (role, layer)
merged[layer] = tensor
for per_layer in sources.values():
for layer, tensor in enumerate(per_layer):
if tensor is None:
per_layer[layer] = placeholder_bank(num_experts)
formats = [legacy_format_for(m.kind, m.kernel.name) for m in layer_methods]
primary = parts[0]
return ExpertBanks(
primary.quant_format, sources, streamed=all(p.streamed for p in parts),
kind=primary.kind, kernel=primary.kernel, layer_tags=formats, layer_formats=formats,
)
def _mixed_expert_banks(model_path, model_config, layer_methods, device, dummy, parallel, workers, chunk, layer_sink=None) -> ExpertBanks:
from freetoken.moe.expert_pieces import iter_expert_pieces
num_layers = model_config.num_moe_layers
assert len(layer_methods) == num_layers, (len(layer_methods), num_layers)
groups = method_groups(layer_methods)
# resolve every format's reader before any bank exists, so a missing parallel reader falls back to serial with nothing allocated
readers = [
None if dummy else iter_expert_pieces(
model_path, model_config, method.kind, layers=layers, parallel=parallel, workers=workers, chunk=chunk
)
for method, layers in groups
]
parts = []
for (method, layers), pieces in zip(groups, readers):
parts.append(build_expert_banks(
method, num_layers, pieces, device=device, layer_sink=layer_sink, dummy=dummy, layers=layers,
))
logger.info_rank0(f"expert banks: {method.kind} / {method.kernel.name} on {len(layers)} of {num_layers} MoE layers")
return merge_expert_banks(parts, layer_methods, model_config.num_experts)
_PARALLEL_CHUNK = 8 << 20 # default O_DIRECT chunk for the parallel reader
def _method_expert_banks(model_path, model_config, method, device, dummy, parallel, workers, chunk, layer_sink=None, resident=False) -> ExpertBanks:
from freetoken.moe.expert_pieces import iter_expert_pieces
num_layers = model_config.num_moe_layers
if dummy:
return build_expert_banks(method, num_layers, None, device=device, dummy=True, resident=resident)
pieces = iter_expert_pieces(
model_path, model_config, method.kind, parallel=parallel, workers=workers, chunk=chunk
)
return build_expert_banks(method, num_layers, pieces, device=device, layer_sink=layer_sink, resident=resident)
def _host_ram_fits_parallel(model_path: str) -> bool:
"""Best-effort: can free host RAM hold the expert banks plus the parallel reader's one
extra (non-reclaimable) whole-shard buffer? Unknown (non-local path / no /proc) -> True,
i.e. keep the fast path. Banks ~= checkpoint size (experts dominate); transient ~= the
largest shard. Uses MemAvailable (counts reclaimable cache) -- the OOM-relevant figure."""
avail = None
try:
with open("/proc/meminfo") as f:
for line in f:
if line.startswith("MemAvailable:"):
avail = int(line.split()[1]) * 1024
break
except OSError:
pass
if avail is None:
return True
try: # the loader's index-first shard list also covers subdirectory layouts (a root glob does not)
from freetoken.models.loader import iter_weight_files
sizes = [os.path.getsize(p) for p in iter_weight_files(model_path)]
except Exception:
return True
if not sizes:
return True
return avail > sum(sizes) + max(sizes)
def ftw_bank_bytes(model_path: str) -> int | None:
"""Total expert-bank bytes of an FTW checkpoint, from its metadata (no bank IO).
``None`` when the checkpoint is not FTW -- callers that size things pre-load (auto split residency) then leave the load unchanged."""
import json
meta = os.path.join(model_path, "freetoken_weight.json")
if not os.path.isfile(meta):
return None
with open(meta, encoding="utf-8") as f:
tensors = json.load(f).get("tensors", [])
return sum(t["nbytes"] for t in tensors if t.get("kind") == "experts_bank")
def _method_expert_bytes(method) -> int:
return method.cfg.num_experts * sum(
math.prod(spec.shape) * torch.empty((), dtype=spec.dtype).element_size()
for spec in method.layout().values() if not spec.resident
)
def bank_bytes_estimate(model_config, method=None, layer_methods=None) -> int | None:
"""Estimated total expert-bank bytes of a raw checkpoint before loading it.
With a bound expert ``method`` the kernel's layout gives the exact host bytes (``layer_methods``,
one per MoE layer, for experts of several formats); otherwise the format-tag table sizes the
format. ``None`` for unknown formats or missing dims (callers then skip the pre-load sizing)."""
layers = getattr(model_config, "num_moe_layers", None)
if layer_methods is not None:
return sum(_method_expert_bytes(m) for m in layer_methods)
if method is not None and layers:
return layers * _method_expert_bytes(method)
expert_quant = getattr(model_config, "expert_quant", "none")
fmt = expert_quant if expert_quant != "none" else (
getattr(model_config, "moe_weight_format", None) or "bf16"
)
per_expert = _BANK_BYTES_PER_EXPERT.get(fmt)
layers = getattr(model_config, "num_moe_layers", None)
experts = getattr(model_config, "num_experts", None)
hidden = getattr(model_config, "hidden_size", None)
inter = getattr(model_config, "moe_intermediate_size", None)
if per_expert is None or not all((layers, experts, hidden, inter)):
return None
return layers * experts * per_expert(hidden, inter)
def load_expert_banks(
model_path: str,
model_config,
*,
method=None,
device: torch.device,
dummy: bool = False,
parallel: bool | None = None,
workers: int = 8,
chunk: int = _PARALLEL_CHUNK,
layer_sink=None,
layer_residency: list[str] | None = None,
resident: bool = False,
layer_methods: list | None = None,
) -> ExpertBanks:
"""Load (or fabricate, with ``dummy=True``) the expert banks. Two paths, both returning
the same normalized ``ExpertBanks`` and both pinning after fill:
* **Fast path (FTW)**: if ``model_path`` is a converted FTW checkpoint, read its
repacked banks directly (contiguous chunked O_DIRECT). No auto-conversion.
* **Slow path** (the original checkpoint): auto-pick **parallel** (the common parallel chunked
O_DIRECT reader) when experts are stored as many small tensors -- the serial read is
slow there -- else the **serial baseline** (packed experts: serial already saturates,
parallel only adds read amplification). parallel unavailable for a quant falls back to serial.
``parallel`` overrides the slow-path auto-pick: ``None`` = auto (production), ``True`` /
``False`` = force parallel / serial (used by the loader benchmark and the converter).
``layer_sink`` (the converter only): forwarded to whichever provider is picked; a
provider only engages it (and reports ``ExpertBanks.streamed=True``) for its own
streamable formats, so callers must check ``streamed`` rather than assume it fired.
``method`` (the bound expert quant method of the model's offload layers) selects
the generic path: the family's pieces packed by the method's kernel. Without it only an
FTW checkpoint loads.
``layer_residency``: per-layer ``HostResidency`` labels applied at settle time -- explicitly on the FTW fast path, ambiently (``requested_residency``) in the slow-path providers.
Applied labels are echoed on ``ExpertBanks.layer_residency``; a loader that settles some other way leaves it ``None`` (CPU-layer decode still works on pinned banks, it just saves no pin quota).
``resident`` (with ``method``): the banks land on ``device`` for the resident 'fused' experts; no host bank outlives the load.
``layer_methods`` (one bound method per MoE layer) serves experts of several formats: each
format fills its own layers and the banks merge over the union of roles (``merge_expert_banks``).
"""
from freetoken.checkpoint.ftw import is_ftw_checkpoint, load_ftw_banks, load_ftw_banks_to_device
mixed = is_mixed(layer_methods)
if mixed and (resident or layer_residency is not None):
raise ValueError("experts of several formats are served from the offload slot cache only (no resident experts, no --moe-cpu-layers)")
if resident:
assert method is not None and layer_sink is None and layer_residency is None
if model_path and is_ftw_checkpoint(model_path) and not dummy:
if resident:
banks = load_ftw_banks_to_device(model_path, num_layers=model_config.num_moe_layers, device=device, workers=workers, chunk=chunk)
else:
banks = load_ftw_banks(
model_path, num_layers=model_config.num_moe_layers, workers=workers, chunk=chunk,
layer_residency=layer_residency,
)
if banks is not None:
logger.info_rank0(f"expert banks: FTW fast path (FTW checkpoint {model_path})")
return banks
if parallel and not _PARALLEL_READER_SUPPORTED:
logger.warning_rank0(
"expert banks: parallel O_DIRECT reader unsupported on this platform "
"(no os.O_DIRECT/preadv) -> serial build"
)
parallel = False
auto = parallel is None
if auto:
from freetoken.models.weight import experts_scattered
parallel = _PARALLEL_READER_SUPPORTED and not dummy and experts_scattered(model_path)
# Low-RAM fallback: the parallel reader holds whole-shard ANONYMOUS buffers
# (non-reclaimable) on top of the ~bank-sized resident set, so on a memory-tight box
# it OOMs where the serial path (reclaimable file mmap) survives. Drop to serial when
# free RAM can't cover the banks + one shard's transient. (--expert-load serial/parallel
# bypass this by forcing ``parallel`` explicitly.)
if parallel and not _host_ram_fits_parallel(model_path):
logger.warning_rank0(
"expert banks: low free RAM -> serial build (avoids parallel-reader OOM; "
"override with --expert-load parallel)"
)
parallel = False
logger.info_rank0(f"expert banks: slow path ({'parallel' if parallel else 'serial'} build)")
# parallel's reader resolves hub ids + handles single-file/no-index checkpoints, so it won't
# OSError on those (which would leak the banks it pre-allocated, since host banks live for
# the process). Only NotImplementedError (quant has no parallel reader; raised before any
# allocation) falls back to serial.
from freetoken.moe.host_banks import requested_residency
def _build(par: bool) -> ExpertBanks:
if mixed:
return _mixed_expert_banks(model_path, model_config, layer_methods, device, dummy, par, workers, chunk, layer_sink)
if method is not None and method.kind is QuantKind.GGUF:
from freetoken.models.gguf.experts import load_gguf_expert_banks
# each layer keeps its own ggml types, so the banks come from the file, not from one method's layout
return load_gguf_expert_banks(model_path, model_config.num_moe_layers, first_layer=model_config.first_k_dense_replace, dummy=dummy, device=device if resident else None)
if method is None:
raise ValueError("the experts of a raw checkpoint are packed by the offload layers' MoE quant method; pass method=")
return _method_expert_banks(model_path, model_config, method, device, dummy, par, workers, chunk, layer_sink, resident)
with requested_residency(layer_residency) as residency_plan:
try:
banks = _build(parallel)
except NotImplementedError as exc:
if not parallel:
raise
logger.warning_rank0(f"parallel reader unavailable ({exc}); falling back to serial build")
banks = _build(False)
return _echo_residency(banks, layer_residency, residency_plan)
def pin_probe_enabled(config, mixed_formats: bool) -> bool:
"""Whether ``--moe-cpu-layers auto`` finds the pin limit by pinning until refused.
Only where pinning is known to be capped; FREETOKEN_PIN_BUDGET_GB fixes the limit instead (then a failed pin is an error)."""
from freetoken.checkpoint.ftw import is_ftw_checkpoint
from freetoken.moe.host_banks import pin_capped
return (
(config.moe_cpu_layers or "").strip() == "auto"
and pin_capped()
and not os.environ.get("FREETOKEN_PIN_BUDGET_GB")
and config.moe_strategy in ("offload", "hybrid")
and not mixed_formats
and not config.use_dummy_weight
and not (config.model_path and is_ftw_checkpoint(config.model_path)) # FTW settles its banks itself
)
def probe_banks(banks: ExpertBanks, num_layers: int) -> tuple[ExpertBanks, frozenset[int]]:
"""Settle DEFERRED banks with host_banks.probe_pin; returns the banks and the OS-locked layers."""
import dataclasses
import time
from freetoken.moe.host_banks import HostResidency, probe_pin
plan = banks.deferred_plan
if plan is None:
return banks, frozenset() # the loader settled the banks itself (pinned)
t0 = time.perf_counter()
labels = probe_pin(plan, num_layers)
locked = frozenset(i for i, r in enumerate(labels) if r != HostResidency.PINNED.value)
pinned_bytes = sum(b.nbytes for l, bs in plan.deferred.items() if l not in locked for b in bs)
logger.info_rank0(
f"--moe-cpu-layers auto (pin probe): pinned {num_layers - len(locked)} MoE layers ({pinned_bytes / 2**30:.2f} GiB) "
f"in {time.perf_counter() - t0:.1f}s, locked {len(locked)} {sorted(locked)}"
)
return dataclasses.replace(banks, layer_residency=labels, deferred_plan=None), locked
def _echo_residency(banks: ExpertBanks, requested, plan) -> ExpertBanks:
"""Stamp an honored residency request onto the ExpertBanks; keep None (and warn) when no settle point consulted the plan."""
if requested is None or banks.layer_residency is not None:
return banks
if plan is not None and plan.applied:
import dataclasses
if plan.deferred:
return dataclasses.replace(banks, layer_residency=list(requested), deferred_plan=plan)
labels = [plan.actual.get(i, r) for i, r in enumerate(requested)]
downgraded = [i for i, r in enumerate(requested) if labels[i] != r]
if downgraded:
logger.warning_rank0(
f"--moe-cpu-layers: layers {downgraded} settled pageable instead of "
f"OS-locked (lock failed); they still decode on the CPU executor but "
f"may swap under memory pressure"
)
return dataclasses.replace(banks, layer_residency=labels)
from freetoken.moe.host_banks import HostResidency
if any(r != HostResidency.PINNED.value for r in requested):
logger.warning_rank0(
"--moe-cpu-layers: this checkpoint's bank loader settles banks without "
"per-layer residency (pre-pins everything); CPU-layer decode still works "
"but saves no pinned quota"
)
return banks
def attach_resident_banks(layers, banks: ExpertBanks) -> None:
"""Give each resident MoE layer its rows of the device ``banks``; bank layer ``i`` goes to the layer whose ``layer_id`` is ``i``."""
ids = sorted(layer.layer_id for layer in layers if layer.layer_id is not None)
num_layers = len(next(iter(banks.sources.values())))
if ids != list(range(num_layers)) or len(ids) != len(layers):
raise ValueError(f"resident MoE layer ids {ids} do not cover the {num_layers} expert bank layers")
for layer in layers:
i, e = layer.layer_id, layer.num_experts
layout = layer.quant_method.layout()
rows = {role: (e, *spec.shape) for role, spec in layout.items() if not spec.resident}
if set(banks.sources) != set(rows):
raise ValueError(f"expert banks hold {sorted(banks.sources)}, the {layer.quant_method.kernel.name} kernel reads {sorted(rows)}")
for role, shape in rows.items():
bank = banks.sources[role][i]
if tuple(bank.shape) != shape or bank.dtype != layout[role].dtype:
raise ValueError(
f"bank {role!r} of layer {i} is {tuple(bank.shape)} {bank.dtype} but the {layer.quant_method.kernel.name} kernel "
f"wants {shape} {layout[role].dtype}; the banks were packed for another kernel or model"
)
alphas = []
# apply() reads view.alphas as (gate_up, down)
for role in (role for role in ("gate_up_alpha", "down_alpha") if role in layout):
a = getattr(banks, role)
if a is None or a.numel() != num_layers * e or a.dtype != layout[role].dtype:
raise ValueError(f"the {layer.quant_method.kernel.name} kernel needs {num_layers * e} {layout[role].dtype} {role} values, the expert banks carry {None if a is None else (a.numel(), a.dtype)}")
alphas.append(a)
layer._expert_banks = {role: per_layer[i] for role, per_layer in banks.sources.items()}
layer._expert_alphas = tuple(a[i * e : (i + 1) * e] for a in alphas) or None