Skip to content

FreeToken brings datacenter-scale model serving to your desktop. Run massive models locally, fast and efficiently.

feat(moe): serve experts of several formats through per-format slot pools (#700)

Xiaoze Fane8c6bf11235d
diff --git a/python/freetoken/checkpoint/convert.py b/python/freetoken/checkpoint/convert.py
index 054fade4..5551759f 100644
--- a/python/freetoken/checkpoint/convert.py
+++ b/python/freetoken/checkpoint/convert.py
@@ -203,14 +203,14 @@ def convert_checkpoint(
# every MoE strategy reads the routed experts from the banks, fused included
offload = getattr(mc, "is_moe", False)
include_moe_experts = not offload
- method = None
+ method = layer_methods = None
if offload:
- from freetoken.engine.engine import offload_expert_method
+ from freetoken.engine.engine import offload_expert_methods
from freetoken.layers import set_rope_device
set_rope_device(dev)
object.__setattr__(cfg, "moe_strategy", "offload")
- method = offload_expert_method(cfg)
+ method, layer_methods = offload_expert_methods(cfg)
from freetoken.utils.progress import byte_bar, count_bar
@@ -239,6 +239,7 @@ def convert_checkpoint(
# 2) offload expert banks (post-repack) + alpha scales (slow path auto-picks parallel/serial)
quant_format = None
+ layer_quant_formats = None
num_layers = None
if offload:
# every method-packed format streams each layer to its own FTW entry as it completes (via the sink); the GGUF provider reports through ExpertBanks.streamed whether it engaged the sink or materialized the whole bank set first
@@ -247,9 +248,11 @@ def convert_checkpoint(
names = legacy_bank_names(legacy_format_for(method.kind, method.kernel.name)) if method is not None else {}
sink = _ConvertSink(writer, names=names)
banks = load_expert_banks(
- model_path, mc, method=method, device=dev, layer_sink=sink
+ model_path, mc, method=method, device=dev, layer_sink=sink, layer_methods=layer_methods
)
quant_format = banks.quant_format
+ layer_quant_formats = banks.layer_formats
+ assert layer_quant_formats is None or banks.streamed, "experts of several formats always stream to the sink"
if banks.streamed:
sink.close()
num_layers = sink.num_layers # however many distinct layers the sink actually saw
@@ -313,6 +316,9 @@ def convert_checkpoint(
# read back at load (ftw.load_ftw_banks). dtype/moe_backend were dropped: each
# tensor already carries its own dtype, and nothing reads a model-level backend.
"quant_format": quant_format,
+ # Experts of several formats: each MoE layer's quant_format (quant_format above is the most common). A layer only
+ # stores the banks its kernel reads; the reader gives it zero-byte placeholders for the others.
+ "layer_quant_formats": layer_quant_formats,
# The reader takes num_layers from the model config (copied into this
# checkpoint); recording it here too gives load_ftw_banks a cross-check that
# the banks match the config they ship with. None for non-offload checkpoints.
diff --git a/python/freetoken/checkpoint/ftw.py b/python/freetoken/checkpoint/ftw.py
index e31e9d82..ae97a98d 100644
--- a/python/freetoken/checkpoint/ftw.py
+++ b/python/freetoken/checkpoint/ftw.py
@@ -461,11 +461,18 @@ def _split_bank_entries(reader: FTWReader, path: str, num_layers: int):
mixed = {e["name"] for e in flat_entries} & per_layer_groups.keys()
assert not mixed, f"FTW bank(s) mix flat and per-layer row layouts: {sorted(mixed)}"
+ # experts of several formats: a layer stores only the banks its kernel reads
+ several = reader.meta("layer_quant_formats") is not None
+ assert not (several and flat_entries), "experts of several formats are stored per layer"
for base, by_layer in per_layer_groups.items():
- assert sorted(by_layer) == list(range(num_layers)), (
- f"FTW bank {base!r} has per-layer entries for layers {sorted(by_layer)}, "
+ layers = sorted(by_layer)
+ assert (set(layers) <= set(range(num_layers))) if several else layers == list(range(num_layers)), (
+ f"FTW bank {base!r} has per-layer entries for layers {layers}, "
f"expected exactly range({num_layers})"
)
+ if several:
+ covered = set().union(*(by_layer.keys() for by_layer in per_layer_groups.values()))
+ assert covered == set(range(num_layers)), f"FTW expert banks have no entries for layers {sorted(set(range(num_layers)) - covered)}"
return bank_entries, alpha_entries, flat_entries, per_layer_groups
@@ -562,11 +569,16 @@ def load_ftw_banks(
row_view_args[name].append((head_pad, layer_bytes, num_experts, tuple(row_shape), dtype))
row_jobs.append((name, bank, win_off, win_end - win_off, layer_bytes, layer_id))
+ num_experts = next((e["shape"][0] for by_layer in per_layer_groups.values() for e in by_layer.values()), None)
for base, by_layer in per_layer_groups.items():
row_hb[base] = []
row_view_args[base] = []
for layer_id in range(num_layers):
- e = by_layer[layer_id]
+ e = by_layer.get(layer_id)
+ if e is None: # experts of several formats: this layer's kernel reads no such bank
+ row_hb[base].append(None)
+ row_view_args[base].append("placeholder")
+ continue
assert e["global_off"] % ALIGN == 0, (base, layer_id, e["global_off"]) # writer invariant
bank = HostBank(tuple(e["shape"]), _dtype_of(e["dtype"]), backing=_backing(layer_id))
row_hb[base].append(bank)
@@ -611,10 +623,15 @@ def load_ftw_banks(
bar.close()
reader.close()
+ from freetoken.moe.offload_cache import placeholder_bank
+
sources: dict[str, list] = {}
for name, banks in row_hb.items():
views = []
for bank, view_args in zip(banks, row_view_args[name]):
+ if view_args == "placeholder":
+ views.append(placeholder_bank(num_experts))
+ continue
if view_args is None: # per-layer entry: already shaped [num_experts, ...]
views.append(bank.tensor)
continue
@@ -635,6 +652,8 @@ def load_ftw_banks(
applied = list(residency)
for banks in row_hb.values():
for layer_id, bank in enumerate(banks):
+ if bank is None:
+ continue
if (applied[layer_id] == HostResidency.LOCKED.value
and bank.residency is not HostResidency.LOCKED):
applied[layer_id] = HostResidency.PAGEABLE.value
@@ -643,7 +662,7 @@ def load_ftw_banks(
by_layer = [0] * num_layers
for name, banks in row_hb.items():
for layer_id, bank in enumerate(banks):
- by_layer[layer_id] += bank.nbytes
+ by_layer[layer_id] += bank.nbytes if bank is not None else 0
locked = [i for i in unpinned if applied[i] == HostResidency.LOCKED.value]
pageable = [i for i in unpinned if i not in set(locked)]
pinned_b = sum(b for i, b in enumerate(by_layer) if i not in set(unpinned))
@@ -666,9 +685,11 @@ def load_ftw_banks(
# alphas are the small per-expert scale vectors, distinguished by their reserved names
# (not a separate kind); everything else under experts_bank is a weight source.
alpha_kw = {n: alpha_hb[n].tensor for n in alpha_hb}
+ layer_formats = reader.meta("layer_quant_formats")
return ExpertBanks(
quant_format, sources, **alpha_kw,
layer_residency=applied, kind=kind, kernel=kernel,
+ layer_tags=layer_formats, layer_formats=layer_formats,
)
@@ -683,6 +704,8 @@ def load_ftw_banks_to_device(path: str, *, num_layers: int, device: torch.device
split = _split_bank_entries(reader, path, num_layers)
if split is None:
return None
+ if reader.meta("layer_quant_formats") is not None:
+ raise ValueError(f"{path!r} holds experts of several formats, which are served from the offload slot cache only")
bank_entries, alpha_entries, flat_entries, per_layer_groups = split
# (name, layer_id or None for an alpha, read offset, read length, head pad, shape, dtype)
diff --git a/python/freetoken/engine/engine.py b/python/freetoken/engine/engine.py
index 81f42dba..980bb043 100644
--- a/python/freetoken/engine/engine.py
+++ b/python/freetoken/engine/engine.py
@@ -606,6 +606,7 @@ class Engine:
dummy=config.use_dummy_weight,
parallel={"serial": False, "parallel": True}.get(config.expert_load, None),
resident=True,
+ layer_methods=[layer.quant_method for layer in sorted(layers, key=lambda layer: layer.layer_id)],
)
attach_resident_banks(layers, banks)
@@ -670,10 +671,21 @@ class Engine:
)
def _init_offload_moe_cache(self, config: EngineConfig) -> OffloadMoeCache:
+ from freetoken.moe.expert_banks import is_mixed
+
method = shared_offload_method(self.model)
num_moe_layers = config.model_config.num_moe_layers
+ # experts of several formats: one method per MoE layer, one slot pool per format
+ layer_methods = offload_layer_methods(self.model)
+ if not is_mixed(layer_methods):
+ layer_methods = None
+ elif config.moe_strategy == "cpu" or config.moe_cpu_layers:
+ raise ValueError(
+ "this model's experts come in several formats, which are served from per-format GPU slot pools; "
+ "they do not combine with --moe-strategy cpu or --moe-cpu-layers (use offload or hybrid)"
+ )
cpu_layer_ids = _resolve_cpu_layers(config, num_moe_layers, reserved=self._host_tables_bytes, method=method)
- _check_pin_budget(config, reserved=self._host_tables_bytes, method=method)
+ _check_pin_budget(config, reserved=self._host_tables_bytes, method=method, layer_methods=layer_methods)
# the kernels were picked for model_config.decode_target; --moe-cpu-layers auto may still find that every bank fits the pin budget
decode_target = config.model_config.decode_target
if decode_target == "cpu" and not cpu_layer_ids:
@@ -737,6 +749,7 @@ class Engine:
dummy=config.use_dummy_weight,
parallel=expert_parallel,
layer_residency=requested_residency,
+ layer_methods=layer_methods,
)
except PinFailed as exc:
raise RuntimeError(f"{exc}; {_pin_hint(self._host_tables_bytes)}") from exc
@@ -755,7 +768,7 @@ class Engine:
f"--moe-cache-split {config.moe_cache_split} is ignored: all MoE layers share one slot pool "
f"(--moe-cache-scope per-layer gives each layer its own)"
)
- self._moe_slot_limit = method.slot_limit() if method is not None else None
+ self._moe_slot_limit = _slot_limit(layer_methods) if layer_methods is not None else (method.slot_limit() if method is not None else None)
# with GPU decode, a pool below num_experts slots bounds the batch; the startup split keeps room for one token per step
pools = self._moe_pools(1)
size_cap = sum(p.max_num_slots for p in pools)
@@ -798,8 +811,17 @@ class Engine:
)
if decode_target == "gpu":
_clamp_decode_batch_to_pools(config, pool_sizes)
- layout = max_slots = None
- if method is not None:
+ layout = layer_layouts = max_slots = None
+ if layer_methods is not None:
+ _check_layer_formats(banks, layer_methods)
+ layer_layouts = [m.layout() for m in layer_methods]
+ max_slots = self._moe_slot_limit
+ elif method is not None:
+ if banks.layer_formats is not None:
+ raise ValueError(
+ f"expert banks hold experts of several formats ({sorted(set(banks.layer_formats))}) but the model "
+ f"binds {method.kind} / {method.kernel.name} on every MoE layer; reconvert the checkpoint"
+ )
if banks.kind is not None and (banks.kind, banks.kernel) != (method.kind, method.kernel.name):
raise ValueError(
f"expert banks were packed for {banks.kind} / {banks.kernel} but the model "
@@ -822,6 +844,7 @@ class Engine:
decode_target=decode_target,
hybrid_max_fetch=config.moe_hybrid_max_fetch,
layout=layout,
+ layer_layouts=layer_layouts,
max_slots=max_slots,
pool_of_layer=pool_of_layer(pools, config.model_config.num_moe_layers),
pool_sizes=pool_sizes,
@@ -903,10 +926,8 @@ class Engine:
# Decode batches never exceed max_running_req, but CUDA-graph padding can
# round a batch up to the largest captured size; cover both.
max_tokens = max(config.max_running_req, config.cuda_graph_max_bs or 0, *(config.cuda_graph_bs or []), 1)
- method = sample.quant_method
- formats = {layer.quant_method.cpu_format for layer in layers}
- if len(formats) > 1:
- raise NotImplementedError(f"CPU MoE backend needs one expert format across the MoE layers, got {sorted(map(str, formats))}")
+ by_id = sorted(layers, key=lambda layer: layer.layer_id)
+ layer_formats = [layer.quant_method.cpu_spec() for layer in by_id]
executor = CpuMoeExecutor(
cache,
top_k=sample.top_k,
@@ -917,8 +938,7 @@ class Engine:
device=self.device,
swiglu_alpha=float(sample.alpha),
swiglu_limit=sample.limit,
- fmt=method.cpu_format,
- act_block=method.cfg.act_block,
+ layer_formats=layer_formats,
)
cache.set_cpu_executor(executor)
self.cpu_moe_executor = executor
@@ -1475,10 +1495,10 @@ def _pin_budget_bytes(reserved: int = 0) -> int | None:
return max(0, cap - reserved)
-def _bank_bytes(config: EngineConfig, method=None) -> int | None:
+def _bank_bytes(config: EngineConfig, method=None, layer_methods=None) -> int | None:
from freetoken.moe.expert_banks import bank_bytes_estimate, ftw_bank_bytes
- return ftw_bank_bytes(config.model_path) or bank_bytes_estimate(config.model_config, method)
+ return ftw_bank_bytes(config.model_path) or bank_bytes_estimate(config.model_config, method, layer_methods)
def _pin_hint(reserved: int) -> str:
@@ -1490,12 +1510,12 @@ def _pin_hint(reserved: int) -> str:
)
-def _check_pin_budget(config: EngineConfig, *, reserved: int, method=None) -> None:
+def _check_pin_budget(config: EngineConfig, *, reserved: int, method=None, layer_methods=None) -> None:
"""Stop a plain offload boot whose banks exceed a known pin budget before any bank is read."""
if config.moe_cpu_layers or config.moe_strategy not in ("offload", "hybrid"):
return
budget = _pin_budget_bytes(reserved)
- bank_bytes = _bank_bytes(config, method) if budget is not None else None
+ bank_bytes = _bank_bytes(config, method, layer_methods) if budget is not None else None
if bank_bytes and bank_bytes > budget:
raise ValueError(
f"expert banks need {bank_bytes / 2**30:.1f} GiB of pinned host RAM but the pin budget is "
@@ -1574,27 +1594,58 @@ def _adjust_ftw_quant_backend(model_path: str, quant_backend: QuantBackend) -> Q
return QuantBackend(quant_backend.items + (((LayerKind.MOE, kind), kernel),))
-def shared_offload_method(model):
- """The expert method every offload MoE layer of ``model`` uses, or None for models whose MoE layers carry none (GGUF).
-
- The offload cache holds one bank layout, so the layers must agree on (kind, kernel)."""
+def offload_layer_methods(model) -> list | None:
+ """The expert method of each offload MoE layer of ``model``, by MoE-layer id; None for models whose MoE layers carry none (GGUF)."""
layers = [l for l in iter_offload_moe_layers(model) if getattr(l, "quant_method", None) is not None]
if not layers:
return None
- keys = {(layer.quant_method.kind, layer.quant_method.kernel.name) for layer in layers}
- if len(keys) != 1:
- found = sorted(f"{kind} / {kernel}" for kind, kernel in keys)
- raise ValueError(f"expert layers disagree on format / kernel: {found}; per-layer mixed expert formats are not supported")
- kind, kernel = keys.pop()
- logger.info_rank0(f"MoE experts: {kind} via {kernel}")
- return layers[0].quant_method
+ layers.sort(key=lambda layer: layer.layer_id)
+ assert [layer.layer_id for layer in layers] == list(range(len(layers))), [layer.layer_id for layer in layers]
+ return [layer.quant_method for layer in layers]
-def offload_expert_method(config: EngineConfig):
- """The offload expert method of ``config``'s model, from a meta-device build.
+def shared_offload_method(model):
+ """The expert method the offload MoE layers of ``model`` use, or None for models whose MoE layers carry none (GGUF).
+
+ With experts of several formats (a checkpoint that keeps some layers unquantized) this is the format of the most
+ layers; the cache then takes every layer's method (``offload_layer_methods``) and keeps one slot pool per format."""
+ from freetoken.moe.expert_banks import method_groups
+
+ methods = offload_layer_methods(model)
+ if methods is None:
+ return None
+ groups = method_groups(methods)
+ found = ", ".join(f"{m.kind} via {m.kernel.name}" + (f" on {len(ids)} layers" if len(groups) > 1 else "") for m, ids in groups)
+ logger.info_rank0(f"MoE experts: {found}")
+ return groups[0][0]
+
+
+def _slot_limit(layer_methods) -> int | None:
+ """The tightest slot cap of the layers' expert kernels (a pool never mixes kernels, but the limit is checked per pool against the largest)."""
+ limits = [limit for limit in (m.slot_limit() for m in layer_methods) if limit is not None]
+ return min(limits) if limits else None
+
+
+def _check_layer_formats(banks, layer_methods) -> None:
+ from freetoken.moe.legacy_format import legacy_format_for
+
+ want = [legacy_format_for(m.kind, m.kernel.name) for m in layer_methods]
+ have = banks.layer_formats or [banks.quant_format] * len(want)
+ bad = [layer for layer, (w, h) in enumerate(zip(want, have)) if w != h]
+ if bad:
+ raise ValueError(
+ f"expert banks of MoE layers {bad[:8]} were packed as {[have[l] for l in bad[:8]]} but the model binds "
+ f"{[want[l] for l in bad[:8]]}; reconvert the checkpoint or select those kernels"
+ )
+
+
+def offload_expert_methods(config: EngineConfig) -> tuple:
+ """``(method, layer_methods)`` of ``config``'s model, from a meta-device build: the most common expert method and,
+ for experts of several formats, one per MoE layer (else None).
For tools that load expert banks without an engine (the FTW converter); they pack for GPU decode."""
from freetoken.models import create_model
+ from freetoken.moe.expert_banks import is_mixed
from freetoken.utils.torch_utils import torch_dtype
set_quant_backend(_adjust_ftw_quant_backend(config.model_path, QuantBackend.parse(config.quant_backend)))
@@ -1602,7 +1653,13 @@ def offload_expert_method(config: EngineConfig):
object.__setattr__(config.model_config, "decode_target", "gpu")
with torch.device("meta"), torch_dtype(config.dtype):
model = create_model(config.model_config)
- return shared_offload_method(model)
+ layer_methods = offload_layer_methods(model)
+ return shared_offload_method(model), (layer_methods if is_mixed(layer_methods) else None)
+
+
+def offload_expert_method(config: EngineConfig):
+ """The (most common) offload expert method of ``config``'s model; see ``offload_expert_methods``."""
+ return offload_expert_methods(config)[0]
def _adjust_config(config: EngineConfig):
diff --git a/python/freetoken/kernel/csrc/cpu_moe/cpu_moe_ext.cpp b/python/freetoken/kernel/csrc/cpu_moe/cpu_moe_ext.cpp
index c8b4d1b4..6236816e 100644
--- a/python/freetoken/kernel/csrc/cpu_moe/cpu_moe_ext.cpp
+++ b/python/freetoken/kernel/csrc/cpu_moe/cpu_moe_ext.cpp
@@ -21,6 +21,7 @@
#include <stdexcept>
#include <string>
#include <thread>
+#include <tuple>
#include <utility>
#include <vector>
@@ -103,6 +104,9 @@ std::unique_ptr<MoeFormat> make_format(const std::string& name, const MoeParams&
throw std::invalid_argument("unknown CPU MoE weight format " + name);
}
+// One format instance as cpu_executor.py passes it: name, bank tables by role, integer options.
+using FormatSpec = std::tuple<std::string, std::map<std::string, uint64_t>, std::map<std::string, int64_t>>;
+
struct CpuMoeExecutor {
// A host-function callback gets one pointer, so it carries the executor with the task.
struct HostTask {
@@ -112,10 +116,9 @@ struct CpuMoeExecutor {
const int num_layers, num_experts;
const bool apply_on_input;
- MoeParams params; // the format keeps a reference; declared before fmt
- std::unique_ptr<MoeFormat> fmt;
- bool preps_inter = false;
- bool rounds_routes = false;
+ MoeParams params; // the formats keep a reference; declared before fmts
+ std::vector<std::unique_ptr<MoeFormat>> fmts;
+ std::vector<int> layer_fmt; // [num_layers], index into fmts
int n_iblk = 0, n_hblk = 0; // kRowBlock blocks per intermediate row and per output row
// Pinned IO shared by every layer and batch size (one task in flight): [max_tokens, H] x and y, [max_tokens, top_k] ids and w.
@@ -167,9 +170,8 @@ struct CpuMoeExecutor {
CpuMoeExecutor(int num_threads_, int num_layers_, int num_experts_, int top_k_,
int hidden_size, int inter_size, int max_tokens_, int activation_id,
- int apply_router_weight_on_input, const std::string& weight_format,
- std::map<std::string, uint64_t> banks, std::map<std::string, int64_t> options,
- double swiglu_alpha_, double swiglu_limit_,
+ int apply_router_weight_on_input, std::vector<FormatSpec> formats,
+ std::vector<int> layer_formats, double swiglu_alpha_, double swiglu_limit_,
std::vector<int> core_ids_, uintptr_t x_ptr, uintptr_t ids_ptr, uintptr_t w_ptr,
uintptr_t y_ptr)
: num_layers(num_layers_),
@@ -184,9 +186,13 @@ struct CpuMoeExecutor {
num_threads(num_threads_ > 0 ? num_threads_ : 1),
core_ids(std::move(core_ids_)) {
if (params.top_k > kMaxTopK) throw std::invalid_argument("CPU MoE supports top_k <= " + std::to_string(kMaxTopK));
- fmt = make_format(weight_format, params, FormatArgs{weight_format, std::move(banks), std::move(options)});
- preps_inter = fmt->preps_inter();
- rounds_routes = fmt->rounds_routes();
+ for (auto& [name, banks, options] : formats)
+ fmts.push_back(make_format(name, params, FormatArgs{name, std::move(banks), std::move(options)}));
+ if (static_cast<int>(layer_formats.size()) != num_layers)
+ throw std::invalid_argument("CPU MoE needs the format of every layer");
+ for (const int f : layer_formats)
+ if (f < 0 || f >= static_cast<int>(fmts.size())) throw std::out_of_range("CPU MoE layer format index out of range");
+ layer_fmt = std::move(layer_formats);
n_iblk = (params.I + kRowBlock - 1) / kRowBlock;
n_hblk = (params.H + kRowBlock - 1) / kRowBlock;
g_scratch.assign(static_cast<size_t>(params.max_tokens) * params.top_k * params.I, 0);
@@ -217,7 +223,16 @@ struct CpuMoeExecutor {
return MoeTask{static_cast<uint64_t>(layer_id), static_cast<uint64_t>(num_tokens)};
}
- std::string isa_name() const { return fmt->isa_name(); }
+ std::string isa_name() const {
+ std::vector<std::string> names;
+ for (const auto& f : fmts)
+ if (const std::string n = f->isa_name(); std::find(names.begin(), names.end(), n) == names.end()) names.push_back(n);
+ std::string out;
+ for (const auto& n : names) out += (out.empty() ? "" : ",") + n;
+ return out;
+ }
+
+ const MoeFormat& format_of(const MoeTask& t) const { return *fmts[layer_fmt[t.layer]]; }
// Whether any route of the task names an expert; do_pass1 and do_pass2 skip the others.
bool has_routes(const MoeTask& t) const {
@@ -249,8 +264,8 @@ struct CpuMoeExecutor {
const float w_in = apply_on_input ? io_w[static_cast<size_t>(tok) * top_k + k] : 1.0f;
const int i0 = static_cast<int>(ib) * kRowBlock;
const int i1 = std::min(I, i0 + kRowBlock);
- fmt->gate_up(static_cast<int>(t.layer), e, tok, i0, i1, io_x + (size_t)tok * H, w_in,
- g_scratch.data() + ((size_t)tok * top_k + k) * I);
+ format_of(t).gate_up(static_cast<int>(t.layer), e, tok, i0, i1, io_x + (size_t)tok * H, w_in,
+ g_scratch.data() + ((size_t)tok * top_k + k) * I);
}
// Pass 2 item p = one kRowBlock block of one token's output row, summed over its routes.
@@ -272,7 +287,9 @@ struct CpuMoeExecutor {
++n;
}
float part[kMaxTopK * kRowBlock];
- if (n > 0) fmt->down(static_cast<int>(t.layer), h0, h1, routes, n, part);
+ const MoeFormat& f = format_of(t);
+ if (n > 0) f.down(static_cast<int>(t.layer), h0, h1, routes, n, part);
+ const bool rounds_routes = f.rounds_routes();
bf16_t* y_row = io_y + (size_t)tok * H;
for (int h = h0; h < h1; ++h) {
float acc = 0.0f;
@@ -294,11 +311,12 @@ struct CpuMoeExecutor {
barrier(local_sense);
// Prepare the intermediate rows (per token,route) for the down GEMV, see prep_inter.
// Needs all of pass1 done (a full row spans every iblk).
- if (preps_inter) {
+ if (prt_total > 0) {
+ MoeFormat& f = *fmts[layer_fmt[t.layer]];
for (;;) {
int64_t r = prt_next.fetch_add(1, std::memory_order_relaxed);
if (r >= prt_total) break;
- fmt->prep_inter(r, g_scratch.data() + (size_t)r * params.I);
+ f.prep_inter(r, g_scratch.data() + (size_t)r * params.I);
}
barrier(local_sense);
}
@@ -348,9 +366,10 @@ struct CpuMoeExecutor {
void submit(MoeTask t) {
const int num_tokens = static_cast<int>(t.num_tokens);
+ MoeFormat& fmt = *fmts[layer_fmt[t.layer]];
p1_total = static_cast<int64_t>(num_tokens) * params.top_k * n_iblk;
p2_total = static_cast<int64_t>(num_tokens) * n_hblk;
- prt_total = preps_inter ? static_cast<int64_t>(num_tokens) * params.top_k : 0;
+ prt_total = fmt.preps_inter() ? static_cast<int64_t>(num_tokens) * params.top_k : 0;
p1_next.store(0, std::memory_order_relaxed);
p2_next.store(0, std::memory_order_relaxed);
prt_next.store(0, std::memory_order_relaxed);
@@ -359,7 +378,7 @@ struct CpuMoeExecutor {
bar_sense.store(0, std::memory_order_relaxed);
// Prepare the per-token input once, up front.
// Single-threaded: tiny for decode, and done before the workers are woken below.
- for (int tok = 0; tok < num_tokens; ++tok) fmt->prep_input(tok, io_x + (size_t)tok * params.H);
+ for (int tok = 0; tok < num_tokens; ++tok) fmt.prep_input(tok, io_x + (size_t)tok * params.H);
{
std::lock_guard<std::mutex> lk(task_mtx);
cur_task = t;
@@ -510,14 +529,14 @@ PYBIND11_MODULE(TORCH_EXTENSION_NAME, m) {
using freetoken::cpu_moe::CpuMoeExecutor;
using freetoken::cpu_moe::GpuSync;
py::class_<CpuMoeExecutor>(m, "CpuMoeExecutor")
- .def(py::init<int, int, int, int, int, int, int, int, int, std::string,
- std::map<std::string, uint64_t>, std::map<std::string, int64_t>,
- double, double, std::vector<int>, uintptr_t, uintptr_t, uintptr_t, uintptr_t>(),
+ .def(py::init<int, int, int, int, int, int, int, int, int, std::vector<freetoken::cpu_moe::FormatSpec>,
+ std::vector<int>, double, double, std::vector<int>, uintptr_t, uintptr_t, uintptr_t,
+ uintptr_t>(),
py::arg("num_threads"), py::arg("num_layers"), py::arg("num_experts"),
py::arg("top_k"), py::arg("hidden_size"), py::arg("inter_size"),
py::arg("max_tokens"), py::arg("activation_id"),
- py::arg("apply_router_weight_on_input"), py::arg("weight_format"),
- py::arg("banks"), py::arg("options"), py::arg("swiglu_alpha"), py::arg("swiglu_limit"),
+ py::arg("apply_router_weight_on_input"), py::arg("formats"), py::arg("layer_formats"),
+ py::arg("swiglu_alpha"), py::arg("swiglu_limit"),
py::arg("core_ids"), py::arg("x_ptr"), py::arg("ids_ptr"), py::arg("w_ptr"),
py::arg("y_ptr"))
.def("submit_with_cuda_stream", &CpuMoeExecutor::submit_with_cuda_stream,
diff --git a/python/freetoken/layers/quantization/moe/base.py b/python/freetoken/layers/quantization/moe/base.py
index de1ce004..5ae89c97 100644
--- a/python/freetoken/layers/quantization/moe/base.py
+++ b/python/freetoken/layers/quantization/moe/base.py
@@ -151,6 +151,10 @@ class MoEKernel(ABC):
def worth_it(self, cfg: MoEConfig) -> bool:
return True
+ def cpu_options(self, cfg: MoEConfig) -> dict[str, int]:
+ """Integer options of ``cpu_format`` for these experts."""
+ return {}
+
def slot_limit(self, cfg: MoEConfig) -> int | None:
"""Most GPU cache slots the kernel can address for ``cfg``; None for no limit."""
return self.max_slots
@@ -196,6 +200,10 @@ class MoEMethod(QuantMethod):
def cpu_format(self) -> str | None:
return self.kernel.cpu_format
+ def cpu_spec(self) -> tuple[str | None, dict[str, int]]:
+ """The CPU executor format and options of this layer's experts."""
+ return self.cpu_format, self.kernel.cpu_options(self.cfg)
+
def create_weights(self, layer: Any) -> None:
# resident experts stay out of the state dict; attach_resident_banks fills these after the dense weights load
layer._expert_banks = None
diff --git a/python/freetoken/layers/quantization/moe/mxfp4.py b/python/freetoken/layers/quantization/moe/mxfp4.py
index 1af6d9f4..a17b5bbe 100644
--- a/python/freetoken/layers/quantization/moe/mxfp4.py
+++ b/python/freetoken/layers/quantization/moe/mxfp4.py
@@ -28,6 +28,9 @@ class TritonMxfp4MoEKernel(MoEKernel):
return "W4A8 ds_fp4 experts need the fp8 activation quant block"
return self._common_reject(cfg, tp_ok=False, cpu_ok=True, plain_silu_only=False)
+ def cpu_options(self, cfg: MoEConfig) -> dict[str, int]:
+ return {"act_block": int(cfg.act_block)}
+
def layout(self, cfg: MoEConfig) -> dict[str, BankSpec]:
i, h = cfg.intermediate, cfg.hidden
return {
diff --git a/python/freetoken/models/deepseek_v4/weight.py b/python/freetoken/models/deepseek_v4/weight.py
index fbcddb1c..8bc4c1f0 100644
--- a/python/freetoken/models/deepseek_v4/weight.py
+++ b/python/freetoken/models/deepseek_v4/weight.py
@@ -237,7 +237,7 @@ _PROJ_ROLE = {"w1": "gate", "w3": "up", "w2": "down"}
_KIND_SUFFIX = {"weight": "", "scale": "_scale"}
-def iter_expert_pieces(model_path: str, config, kind: QuantKind, *, parallel: bool | None = False, workers: int = 8, chunk: int = 8 << 20):
+def iter_expert_pieces(model_path: str, config, kind: QuantKind, *, layers=None, parallel: bool | None = False, workers: int = 8, chunk: int = 8 << 20):
"""Routed experts, one piece per expert: ``{gate, up, down}`` e2m1 pairs and their e8m0
``_scale`` companions (``w1`` / ``w3`` / ``w2``). The MTP layer's experts are skipped."""
if kind is not QuantKind.MXFP4:
@@ -249,10 +249,11 @@ def iter_expert_pieces(model_path: str, config, kind: QuantKind, *, parallel: bo
args = load_args(model_path, max_batch_size=1)
L, E = args.n_layers, args.n_routed_experts
+ keep = None if layers is None else set(layers)
def locate(raw_name: str):
m = _EXPERT_RE.match(raw_name)
- if m is None or int(m["layer"]) >= L:
+ if m is None or int(m["layer"]) >= L or (keep is not None and int(m["layer"]) not in keep):
return None
return int(m["layer"]), int(m["expert"]), _PROJ_ROLE[m["proj"]] + _KIND_SUFFIX[m["kind"]]
@@ -263,7 +264,7 @@ def iter_expert_pieces(model_path: str, config, kind: QuantKind, *, parallel: bo
def _serial():
reader = _ShardReader(model_path, _weight_map(model_path), torch.device("cpu"))
try:
- for li in tqdm(range(L), desc="Loading DSV4 experts (serial)", disable=not get_tp_info().is_primary()):
+ for li in tqdm(range(L) if keep is None else sorted(keep), desc="Loading DSV4 experts (serial)", disable=not get_tp_info().is_primary()):
for e in range(E):
base = f"layers.{li}.ffn.experts.{e}"
for proj in ("w1", "w3", "w2"):
diff --git a/python/freetoken/models/deepseek_v41/weight.py b/python/freetoken/models/deepseek_v41/weight.py
index 1ccd23ea..962bd195 100644
--- a/python/freetoken/models/deepseek_v41/weight.py
+++ b/python/freetoken/models/deepseek_v41/weight.py
@@ -200,7 +200,7 @@ _PROJ_ROLE = {"w1": "gate", "w3": "up", "w2": "down"}
_KIND_SUFFIX = {"weight": "", "scale": "_scale"}
-def iter_expert_pieces(model_path: str, config, kind: QuantKind, *, parallel: bool | None = False, workers: int = 8, chunk: int = 8 << 20):
+def iter_expert_pieces(model_path: str, config, kind: QuantKind, *, layers=None, parallel: bool | None = False, workers: int = 8, chunk: int = 8 << 20):
"""One piece per expert: ``{gate, up, down}`` e2m1 pairs and their e8m0 ``_scale`` companions.
The DSpark stage's experts (``mtp.*``) never match."""
if kind is not QuantKind.MXFP4:
@@ -212,10 +212,11 @@ def iter_expert_pieces(model_path: str, config, kind: QuantKind, *, parallel: bo
args = load_args(model_path)
L, E = args.n_layers, args.n_routed_experts
+ keep = None if layers is None else set(layers)
def locate(raw_name: str):
m = _EXPERT_RE.match(raw_name)
- if m is None or int(m["layer"]) >= L:
+ if m is None or int(m["layer"]) >= L or (keep is not None and int(m["layer"]) not in keep):
return None
return int(m["layer"]), int(m["expert"]), _PROJ_ROLE[m["proj"]] + _KIND_SUFFIX[m["kind"]]
@@ -226,7 +227,7 @@ def iter_expert_pieces(model_path: str, config, kind: QuantKind, *, parallel: bo
def _serial():
reader = _ShardReader(model_path, weight_map(model_path), torch.device("cpu"))
try:
- for li in tqdm(range(L), desc="Loading DeepSeek-V4.1 experts (serial)", disable=not get_tp_info().is_primary()):
+ for li in tqdm(range(L) if keep is None else sorted(keep), desc="Loading DeepSeek-V4.1 experts (serial)", disable=not get_tp_info().is_primary()):
for e in range(E):
base = f"layers.{li}.ffn.experts.{e}"
for proj in ("w1", "w3", "w2"):
diff --git a/python/freetoken/models/glm5_next/weight.py b/python/freetoken/models/glm5_next/weight.py
index 6f967dd8..09f91694 100644
--- a/python/freetoken/models/glm5_next/weight.py
+++ b/python/freetoken/models/glm5_next/weight.py
@@ -250,7 +250,7 @@ def iter_vision_weights(model_path: str, device: torch.device) -> Iterator[tuple
reader.close()
-def iter_expert_pieces(model_path, config, kind: QuantKind, *, parallel: bool | None = False, workers: int = 8, chunk: int = 8 << 20):
+def iter_expert_pieces(model_path, config, kind: QuantKind, *, layers=None, parallel: bool | None = False, workers: int = 8, chunk: int = 8 << 20):
"""Block-fp8 routed experts, one piece per expert: ``{gate, up, down}`` fp8 codes and their ``_scale`` companions; other kinds use the generic readers."""
if kind is not QuantKind.FP8_BLOCK:
return None
@@ -260,13 +260,14 @@ def iter_expert_pieces(model_path, config, kind: QuantKind, *, parallel: bool |
from freetoken.moe.expert_pieces import per_expert_pieces
suffix = {"weight": "", "weight_scale_inv": "_scale"}
+ keep = None if layers is None else set(layers)
def locate(raw_name: str):
m = _FP8_EXPERT_RE.match(raw_name)
if m is None:
return None
bank = _layer_to_bank(int(m["layer"]), config)
- if bank is None:
+ if bank is None or (keep is not None and bank not in keep):
return None
return bank, int(m["expert"]), m["proj"] + suffix[m["kind"]]
@@ -284,8 +285,11 @@ def iter_expert_pieces(model_path, config, kind: QuantKind, *, parallel: bool |
weight_map = json.load(f)["weight_map"]
reader = _ShardReader(folder, weight_map, torch.device("cpu"))
try:
- layers = range(config.first_k_dense_replace, config.num_layers)
- for layer in tqdm(layers, desc="Loading GLM-5.3 fp8 experts (serial)", disable=not get_tp_info().is_primary()):
+ ckpt_layers = [
+ layer for layer in range(config.first_k_dense_replace, config.num_layers)
+ if keep is None or _layer_to_bank(layer, config) in keep
+ ]
+ for layer in tqdm(ckpt_layers, desc="Loading GLM-5.3 fp8 experts (serial)", disable=not get_tp_info().is_primary()):
for e in range(config.num_experts):
base = f"{_CKPT}.layers.{layer}.mlp.experts.{e}"
for proj in ("gate", "up", "down"):
diff --git a/python/freetoken/models/gpt_oss/weight.py b/python/freetoken/models/gpt_oss/weight.py
index 686f0246..cb5830c3 100644
--- a/python/freetoken/models/gpt_oss/weight.py
+++ b/python/freetoken/models/gpt_oss/weight.py
@@ -288,7 +288,7 @@ def _source_slices(config, tp_info) -> dict[str, tuple[str, tuple[slice, ...]]]:
}
-def iter_expert_pieces(model_path: str, config, kind: QuantKind, *, parallel: bool = False, workers: int = 8, chunk: int = 8 << 20):
+def iter_expert_pieces(model_path: str, config, kind: QuantKind, *, layers=None, parallel: bool = False, workers: int = 8, chunk: int = 8 << 20):
"""gpt-oss experts as HF ships them, one piece per layer: stacked ``[E, ...]`` mxfp4
``_blocks`` / ``_scales`` / ``_bias`` tensors sliced to this rank's intermediate range."""
if kind is not QuantKind.MXFP4:
@@ -298,10 +298,11 @@ def iter_expert_pieces(model_path: str, config, kind: QuantKind, *, parallel: bo
tp_info = get_tp_info()
slices = _source_slices(config, tp_info)
num_layers, num_experts = config.num_layers, config.num_experts
+ keep = None if layers is None else set(layers)
def _is_expert(name: str) -> bool:
info = _expert_layer_and_name(name)
- return info is not None and 0 <= info[0] < num_layers
+ return info is not None and 0 <= info[0] < num_layers and (keep is None or info[0] in keep)
def _tensors():
if parallel:
@@ -320,6 +321,8 @@ def iter_expert_pieces(model_path: str, config, kind: QuantKind, *, parallel: bo
layer_id, source = info
if not 0 <= layer_id < num_layers:
raise ValueError(f"Unexpected GPT-OSS expert layer in checkpoint: {name}")
+ if keep is not None and layer_id not in keep:
+ continue
if source not in slices:
raise ValueError(f"Unexpected GPT-OSS expert source: {name}")
yield name, _read_safetensor_slice(f, name, slices[source][1])
diff --git a/python/freetoken/models/nvfp4_banks.py b/python/freetoken/models/nvfp4_banks.py
index 2dfdaaa3..04da0509 100644
--- a/python/freetoken/models/nvfp4_banks.py
+++ b/python/freetoken/models/nvfp4_banks.py
@@ -4,7 +4,7 @@ import collections
import os
import re
from dataclasses import dataclass
-from typing import Callable
+from typing import Callable, Collection
import safetensors
import torch
@@ -68,6 +68,7 @@ def iter_nvfp4_expert_pieces(
config,
spec: Nvfp4ExpertSourceSpec,
*,
+ layers: Collection[int] | None = None,
parallel: bool = False,
workers: int = 8,
chunk: int = 8 << 20,
@@ -80,6 +81,7 @@ def iter_nvfp4_expert_pieces(
Serial reads walk the shards in order; ``parallel`` uses the chunked O_DIRECT reader. Either
way tensors of one expert may span shards, so they are grouped by (layer, expert) as they land.
+ ``layers`` (bank layer ids, default all) reads only those layers' experts.
"""
from freetoken.models.loader import drop_page_cache as _drop
from freetoken.models.loader import safetensors_weight_map
@@ -89,13 +91,14 @@ def iter_nvfp4_expert_pieces(
folder = download_hf_weight(model_path)
weight_map = safetensors_weight_map(folder)
+ keep = None if layers is None else set(layers)
wanted: dict[str, tuple[int, int, str]] = {}
for name in weight_map:
match = spec.key_pattern.match(name)
if match is None:
continue
bank_layer = _bank_layer(spec, int(match.group("layer")), config)
- if bank_layer is None:
+ if bank_layer is None or (keep is not None and bank_layer not in keep):
continue
proj = match.group("proj")
if proj not in spec.proj_to_role:
@@ -104,7 +107,7 @@ def iter_nvfp4_expert_pieces(
if kind not in ("weight", "weight_scale", "weight_scale_2"):
raise ValueError(f"{spec.desc}: unknown NVFP4 expert tensor kind {kind!r}")
wanted[name] = (bank_layer, int(match.group("expert")), spec.proj_to_role[proj] + _kind_suffix(kind))
- expected = _num_moe_layers(config) * config.num_experts * 9
+ expected = (_num_moe_layers(config) if keep is None else len(keep)) * config.num_experts * 9
if len(wanted) != expected:
raise ValueError(f"{spec.desc}: found {len(wanted)} expert tensors, expected {expected}")
diff --git a/python/freetoken/models/qwen3_5_moe/weight.py b/python/freetoken/models/qwen3_5_moe/weight.py
index de5b9e89..724ad153 100644
--- a/python/freetoken/models/qwen3_5_moe/weight.py
+++ b/python/freetoken/models/qwen3_5_moe/weight.py
@@ -344,7 +344,7 @@ def _moe_dims(model_config):
)
-def iter_expert_pieces(model_path, config, kind: QuantKind, *, parallel: bool | None = False, workers: int = 8, chunk: int = 8 << 20):
+def iter_expert_pieces(model_path, config, kind: QuantKind, *, layers=None, parallel: bool | None = False, workers: int = 8, chunk: int = 8 << 20):
"""Block-fp8 routed experts, one piece per expert: ``{gate, up, down}`` fp8 codes and their
``_scale`` (block scale) companions, named as the checkpoint's dialect stores them. Other expert kinds use the generic readers."""
if kind is not QuantKind.FP8_BLOCK:
@@ -355,6 +355,7 @@ def iter_expert_pieces(model_path, config, kind: QuantKind, *, parallel: bool |
from freetoken.moe.expert_pieces import per_expert_pieces
L, E, H, I, dense = _moe_dims(config)
+ keep = None if layers is None else set(layers)
scale = get_quant_config().stored_tensors(QuantKind.FP8_BLOCK)["weight_scale_inv"].name
key_re = re.compile(_FP8_EXPERT_KEY_RE.format(scale=re.escape(scale)))
suffix = {"weight": "", scale: "_scale"}
@@ -366,20 +367,22 @@ def iter_expert_pieces(model_path, config, kind: QuantKind, *, parallel: bool |
li = int(m["layer"]) - dense
if not 0 <= li < L:
raise ValueError(f"unexpected routed-expert layer in {raw_name}")
+ if keep is not None and li not in keep:
+ return None
return li, int(m["expert"]), m["proj"] + suffix[m["kind"]]
if parallel is None:
parallel = experts_scattered(model_path)
if parallel:
tensors = iter_expert_tensors_parallel(
- model_path, lambda n: key_re.match(n) is not None, workers=workers, chunk=chunk
+ model_path, lambda n: locate(n) is not None, workers=workers, chunk=chunk
)
return per_expert_pieces(tensors, locate, tensors_per_expert=6)
def _serial():
reader = ShardReader(model_path, torch.device("cpu"))
try:
- for li in tqdm(range(L), desc="Loading fp8 experts (serial)", disable=not get_tp_info().is_primary()):
+ for li in tqdm(range(L) if keep is None else sorted(keep), desc="Loading fp8 experts (serial)", disable=not get_tp_info().is_primary()):
for e in range(E):
base = f"model.language_model.layers.{dense + li}.mlp.experts.{e}"
for proj in ("gate", "up", "down"):
diff --git a/python/freetoken/moe/benchbw.py b/python/freetoken/moe/benchbw.py
index cd260cb2..86a03f41 100644
--- a/python/freetoken/moe/benchbw.py
+++ b/python/freetoken/moe/benchbw.py
@@ -476,7 +476,7 @@ def _build_cpu_moe_executor(fmt: str, wl: Workload, banks: dict, num_threads: in
cache, top_k=wl.top_k, activation=wl.activation,
apply_router_weight_on_input=False, num_threads=num_threads, max_tokens=1,
device=torch.device("cuda"), swiglu_alpha=wl.swiglu_alpha,
- swiglu_limit=wl.swiglu_limit, act_block=wl.act_block,
+ swiglu_limit=wl.swiglu_limit, layer_formats=[(fmt, {"act_block": wl.act_block} if wl.act_block is not None else {})] * cache.num_layers,
)
diff --git a/python/freetoken/moe/cpu_executor.py b/python/freetoken/moe/cpu_executor.py
index bb9ee563..5d9bd3c9 100644
--- a/python/freetoken/moe/cpu_executor.py
+++ b/python/freetoken/moe/cpu_executor.py
@@ -163,28 +163,34 @@ class CpuMoeExecutor:
device: torch.device,
swiglu_alpha: float = 1.702,
swiglu_limit: float | None = None,
- fmt: str | None = None,
- act_block: int | None = None,
+ layer_formats: list[tuple[str, dict[str, int]]] | None = None,
) -> None:
from freetoken.kernel import _cpu_moe
from freetoken.moe.legacy_format import canonical_role
- fmt = fmt or cache.quant_format
- if fmt not in _cpu_moe.weight_formats():
- raise NotImplementedError(
- f"--moe-strategy cpu/hybrid computes experts on the CPU and supports "
- f"{sorted(_cpu_moe.weight_formats())} formats, but this checkpoint's experts are "
- f"{fmt!r}; use --moe-strategy offload (GPU-side dequant) instead."
- )
+ # layer_formats[l] = (format, options) of layer l; None runs every layer in cache.quant_format
+ if layer_formats is None:
+ layer_formats = [(cache.quant_format, {})] * int(cache.num_layers)
+ assert len(layer_formats) == cache.num_layers, (len(layer_formats), cache.num_layers)
+ keys = [(name, tuple(sorted(opts.items()))) for name, opts in layer_formats]
+ specs = list(dict.fromkeys(keys))
+ names = sorted({name for name, _ in specs})
+ for f in names:
+ if f not in _cpu_moe.weight_formats():
+ raise NotImplementedError(
+ f"--moe-strategy cpu/hybrid computes experts on the CPU and supports "
+ f"{sorted(_cpu_moe.weight_formats())} formats, but this checkpoint's experts are "
+ f"{f!r}; use --moe-strategy offload (GPU-side dequant) instead."
+ )
if activation not in _ACT_IDS:
raise NotImplementedError(f"CPU MoE backend: unsupported activation {activation!r}")
- if apply_router_weight_on_input and fmt in ("mxfp4_triton", "ds_fp4"):
- raise NotImplementedError(f"CPU MoE backend: {fmt} experts apply the router weight on the output only")
+ if apply_router_weight_on_input and (output_only := {"mxfp4_triton", "ds_fp4"} & set(names)):
+ raise NotImplementedError(f"CPU MoE backend: {sorted(output_only)} experts apply the router weight on the output only")
# ABI probe: a stale prebuilt _cpu_moe.so accepts newer act ids without
# error and silently computes the wrong activation in the generic
# epilogue -- fail loudly with the rebuild instruction instead. (mxfp4
# handles its act inside the kernel and predates the marker.)
- if _ACT_IDS[activation] >= 3 and fmt != "mxfp4_triton":
+ if _ACT_IDS[activation] >= 3 and names != ["mxfp4_triton"]:
supported = getattr(_cpu_moe, "max_generic_act_id", lambda: 2)()
if _ACT_IDS[activation] > supported:
raise RuntimeError(
@@ -197,16 +203,24 @@ class CpuMoeExecutor:
self.num_layers = int(cache.num_layers)
self.num_experts = int(cache.num_experts)
self.top_k = int(top_k)
- self.quant_format = fmt
+ self.quant_format = ",".join(names)
self.device = device
self.max_tokens = int(max_tokens)
self.apply_router_weight_on_input = bool(apply_router_weight_on_input)
# The per-layer tensors and their pointer tables must outlive the executor
# (C++ holds raw addresses into both).
self._banks: list[torch.Tensor] = []
- banks, options, (self.H, self.I) = self._resolve_banks(
- {canonical_role(name): per_layer for name, per_layer in cache.bank_sources.items()}, fmt
- )
+ by_role = {canonical_role(name): per_layer for name, per_layer in cache.bank_sources.items()}
+ # every format's tables address every layer, but a format reads only its own layers' rows
+ formats, shapes = [], set()
+ for name, opts in specs:
+ banks, options, shape = self._resolve_banks(by_role, name, keys.index((name, opts)), dict(opts))
+ formats.append((name, banks, {**dict(opts), **options}))
+ shapes.add(shape)
+ if len(shapes) != 1:
+ raise NotImplementedError(f"CPU MoE formats disagree on the expert shape (hidden, intermediate): {sorted(shapes)}")
+ (self.H, self.I), = shapes
+ layer_counts = {spec: keys.count(spec) for spec in specs}
# ds_fp4: the reference FP8 activation round-trip is a scalar per-element chain
# that the C++ side runs single-threaded on the CUDA host-callback thread --
@@ -215,12 +229,15 @@ class CpuMoeExecutor:
# round-trip as a captured GPU kernel BEFORE the D2H (see decode_submit) and tell
# the C++ side to skip its own. Measured on DeepSeek-V4-Flash bs=1 decode:
# 12.85 -> 15.65 tok/s, output bit-identical (tests/moe/test_dsfp4_prequant.py).
- self._gpu_prequant = fmt == "ds_fp4" and device.type == "cuda" and _DSFP4_GPU_PREQUANT
# the W4A8 activation round-trip block follows the checkpoint's fp8 block (128 on V4, 32 on V4.1)
- self._act_block = act_block
- if fmt == "ds_fp4":
- assert act_block is not None, "ds_fp4 experts need act_block, the checkpoint's fp8 activation quant block"
- options.update(act_block=int(act_block), input_prequant=int(self._gpu_prequant))
+ blocks = {options.get("act_block") for name, _, options in formats if name == "ds_fp4"}
+ assert None not in blocks, "ds_fp4 experts need the act_block option, the checkpoint's fp8 activation quant block"
+ # the GPU round-trip rewrites the whole input, so only when every layer is ds_fp4 with one block
+ self._gpu_prequant = names == ["ds_fp4"] and len(blocks) == 1 and device.type == "cuda" and _DSFP4_GPU_PREQUANT
+ self._act_block = next(iter(blocks)) if self._gpu_prequant else None
+ for name, _, options in formats:
+ if name == "ds_fp4":
+ options["input_prequant"] = int(self._gpu_prequant)
if self._gpu_prequant:
logger.info_rank0(
"ds_fp4: input FP8 round-trip moved to the GPU "
@@ -271,9 +288,8 @@ class CpuMoeExecutor:
max_tokens=self.max_tokens,
activation_id=_ACT_IDS[activation],
apply_router_weight_on_input=1 if apply_router_weight_on_input else 0,
- weight_format=fmt,
- banks=banks,
- options=options,
+ formats=formats,
+ layer_formats=[specs.index(k) for k in keys],
swiglu_alpha=float(swiglu_alpha),
swiglu_limit=float(swiglu_limit) if swiglu_limit is not None else float("inf"),
core_ids=core_ids,
@@ -332,9 +348,11 @@ class CpuMoeExecutor:
self._watchdog.start()
+ fmt_label = "; ".join(f"{name}{dict(opts) if opts else ''}: {n} layers" for (name, opts), n in layer_counts.items())
logger.info_rank0(
f"CPU MoE executor ready: threads={nthreads} (pinned to cores "
- f"{core_ids[0]}..{core_ids[-1]}) isa={self.isa} fmt={fmt} "
+ f"{core_ids[0]}..{core_ids[-1]}) isa={self.isa} "
+ f"fmt={fmt_label} "
f"H={self.H} I={self.I} experts={self.num_experts} layers={self.num_layers} "
f"top_k={self.top_k} act={activation} max_tokens={self.max_tokens} "
f"spin_wait={self.spin_wait} "
@@ -360,23 +378,23 @@ class CpuMoeExecutor:
"""Per-layer table addresses keyed by canonical bank role."""
return {role: self._make_table(layers).data_ptr() for role, layers in roles.items()}
- def _resolve_fp8_block_banks(self, banks: dict) -> tuple[dict, dict, tuple[int, int]]:
+ def _resolve_fp8_block_banks(self, banks: dict, rep: int = 0) -> tuple[dict, dict, tuple[int, int]]:
"""Block-FP8 schema: row-major e4m3 + one bf16 scale per 128x128 block; the scale banks pad their last dim, so C++ gets the real row stride."""
gu, gus = banks["gate_up"], banks["gate_up_scale"]
dn, dns = banks["down"], banks["down_scale"]
- assert gu[0].element_size() == 1 and dn[0].element_size() == 1, (gu[0].dtype, dn[0].dtype)
- assert gus[0].dtype == torch.bfloat16 and dns[0].dtype == torch.bfloat16, (gus[0].dtype, dns[0].dtype)
- I = int(gu[0].shape[1] // 2)
- H = int(gu[0].shape[2])
- assert gu[0].shape[1] == 2 * I
- assert tuple(dn[0].shape[1:]) == (H, I), (dn[0].shape, H, I)
- assert gus[0].shape[1] == (2 * I + 127) // 128 and gus[0].shape[2] >= (H + 127) // 128, (gus[0].shape, H, I)
- assert dns[0].shape[1] == (H + 127) // 128 and dns[0].shape[2] >= (I + 127) // 128, (dns[0].shape, H, I)
+ assert gu[rep].element_size() == 1 and dn[rep].element_size() == 1, (gu[rep].dtype, dn[rep].dtype)
+ assert gus[rep].dtype == torch.bfloat16 and dns[rep].dtype == torch.bfloat16, (gus[rep].dtype, dns[rep].dtype)
+ I = int(gu[rep].shape[1] // 2)
+ H = int(gu[rep].shape[2])
+ assert gu[rep].shape[1] == 2 * I
+ assert tuple(dn[rep].shape[1:]) == (H, I), (dn[rep].shape, H, I)
+ assert gus[rep].shape[1] == (2 * I + 127) // 128 and gus[rep].shape[2] >= (H + 127) // 128, (gus[rep].shape, H, I)
+ assert dns[rep].shape[1] == (H + 127) // 128 and dns[rep].shape[2] >= (I + 127) // 128, (dns[rep].shape, H, I)
banks = self._tables(gate_up=gu, down=dn, gate_up_scale=gus, down_scale=dns)
- options = dict(gu_scale_stride=int(gus[0].stride(1)), dn_scale_stride=int(dns[0].stride(1)))
+ options = dict(gu_scale_stride=int(gus[rep].stride(1)), dn_scale_stride=int(dns[rep].stride(1)))
return banks, options, (H, I)
- def _resolve_banks(self, banks: dict, fmt: str) -> tuple[dict, dict, tuple[int, int]]:
+ def _resolve_banks(self, banks: dict, fmt: str, rep: int = 0, opts: dict | None = None) -> tuple[dict, dict, tuple[int, int]]:
"""Return (banks, options, (H, I)) for the C++ ctor: banks maps each canonical role
the format reads to its per-layer table's address, options holds its integer settings.
@@ -389,102 +407,102 @@ class CpuMoeExecutor:
if fmt == "bf16":
gate_up = banks["gate_up"]
down = banks["down"]
- if gate_up[0].dtype != torch.bfloat16 or down[0].dtype != torch.bfloat16:
+ if gate_up[rep].dtype != torch.bfloat16 or down[rep].dtype != torch.bfloat16:
raise NotImplementedError(
- f"bf16 CPU MoE requires bf16 banks, got {gate_up[0].dtype}/{down[0].dtype}"
+ f"bf16 CPU MoE requires bf16 banks, got {gate_up[rep].dtype}/{down[rep].dtype}"
)
- H = int(gate_up[0].shape[2])
- I = int(gate_up[0].shape[1] // 2)
- assert gate_up[0].shape[1] == 2 * I
- assert tuple(down[0].shape[1:]) == (H, I), (down[0].shape, H, I)
+ H = int(gate_up[rep].shape[2])
+ I = int(gate_up[rep].shape[1] // 2)
+ assert gate_up[rep].shape[1] == 2 * I
+ assert tuple(down[rep].shape[1:]) == (H, I), (down[rep].shape, H, I)
banks = self._tables(gate_up=gate_up, down=down)
return banks, {}, (H, I)
if fmt == "q4_0":
- return self._resolve_q4_0_banks(banks)
+ return self._resolve_q4_0_banks(banks, rep)
if fmt == "mxfp4_triton":
- return self._resolve_mxfp4_banks(banks)
+ return self._resolve_mxfp4_banks(banks, rep)
if fmt == "ds_fp4":
- return self._resolve_dsfp4_banks(banks)
+ return self._resolve_dsfp4_banks(banks, rep)
if fmt == "fp8_block":
- return self._resolve_fp8_block_banks(banks)
+ return self._resolve_fp8_block_banks(banks, rep)
# nvfp4: packed e2m1 (2/byte) + fp8-e4m3 per-16 block scales + fp16 row globals.
gup, gus, gug = banks["gate_up"], banks["gate_up_scale"], banks["gate_up_global"]
dnp, dns, dng = banks["down"], banks["down_scale"], banks["down_global"]
- assert gup[0].dtype == torch.uint8 and dnp[0].dtype == torch.uint8, (gup[0].dtype, dnp[0].dtype)
- assert gus[0].element_size() == 1 and dns[0].element_size() == 1, "block scales must be 1 byte"
- assert gug[0].dtype == torch.float16 and dng[0].dtype == torch.float16, (gug[0].dtype, dng[0].dtype)
- I = int(gup[0].shape[1] // 2)
- H = int(gup[0].shape[2] * 2)
- assert gup[0].shape[1] == 2 * I
+ assert gup[rep].dtype == torch.uint8 and dnp[rep].dtype == torch.uint8, (gup[rep].dtype, dnp[rep].dtype)
+ assert gus[rep].element_size() == 1 and dns[rep].element_size() == 1, "block scales must be 1 byte"
+ assert gug[rep].dtype == torch.float16 and dng[rep].dtype == torch.float16, (gug[rep].dtype, dng[rep].dtype)
+ I = int(gup[rep].shape[1] // 2)
+ H = int(gup[rep].shape[2] * 2)
+ assert gup[rep].shape[1] == 2 * I
assert H % 16 == 0 and I % 16 == 0, (H, I)
- assert tuple(dnp[0].shape[1:]) == (H, I // 2), (dnp[0].shape, H, I)
- assert tuple(gus[0].shape[1:]) == (2 * I, H // 16), (gus[0].shape, I, H)
- assert tuple(dns[0].shape[1:]) == (H, I // 16), (dns[0].shape, H, I)
- assert tuple(gug[0].shape[1:]) == (2 * I,) and tuple(dng[0].shape[1:]) == (H,)
+ assert tuple(dnp[rep].shape[1:]) == (H, I // 2), (dnp[rep].shape, H, I)
+ assert tuple(gus[rep].shape[1:]) == (2 * I, H // 16), (gus[rep].shape, I, H)
+ assert tuple(dns[rep].shape[1:]) == (H, I // 16), (dns[rep].shape, H, I)
+ assert tuple(gug[rep].shape[1:]) == (2 * I,) and tuple(dng[rep].shape[1:]) == (H,)
banks = self._tables(gate_up=gup, down=dnp, gate_up_scale=gus, gate_up_global=gug, down_scale=dns, down_global=dng)
return banks, {}, (H, I)
- def _resolve_q4_0_banks(self, banks: dict) -> tuple[dict, dict, tuple[int, int]]:
+ def _resolve_q4_0_banks(self, banks: dict, rep: int = 0) -> tuple[dict, dict, tuple[int, int]]:
"""Native GGUF Q4_0 schema (gemma4 GGUF): per-32 blocks (fp16 scale + 16 nibble
bytes), row-major over K -- the *same* packed banks the GPU offload path streams.
gate_up is [S, 2I, H//32*18], down is [S, H, I//32*18]; the C++ W4A16 GEMV reads a
row in place (18 bytes / 32 K) and dequantizes weights inside the K-loop."""
gate_up, down = banks["gate_up"], banks["down"]
- assert gate_up[0].dtype == torch.uint8 and down[0].dtype == torch.uint8, (
- gate_up[0].dtype, down[0].dtype,
+ assert gate_up[rep].dtype == torch.uint8 and down[rep].dtype == torch.uint8, (
+ gate_up[rep].dtype, down[rep].dtype,
)
- I = int(gate_up[0].shape[1] // 2)
- H = int(down[0].shape[1])
- assert gate_up[0].shape[1] == 2 * I
+ I = int(gate_up[rep].shape[1] // 2)
+ H = int(down[rep].shape[1])
+ assert gate_up[rep].shape[1] == 2 * I
assert H % 32 == 0 and I % 32 == 0, (H, I)
- assert int(gate_up[0].shape[2]) == (H // 32) * 18, (gate_up[0].shape, H)
- assert int(down[0].shape[2]) == (I // 32) * 18, (down[0].shape, I)
+ assert int(gate_up[rep].shape[2]) == (H // 32) * 18, (gate_up[rep].shape, H)
+ assert int(down[rep].shape[2]) == (I // 32) * 18, (down[rep].shape, I)
banks = self._tables(gate_up=gate_up, down=down)
return banks, {}, (H, I)
- def _resolve_mxfp4_banks(self, banks: dict) -> tuple[dict, dict, tuple[int, int]]:
+ def _resolve_mxfp4_banks(self, banks: dict, rep: int = 0) -> tuple[dict, dict, tuple[int, int]]:
"""gpt-oss mxfp4 ``mxfp4_triton`` schema: transposed split-K blocks/scales
(N innermost) + per-output-row biases. The C++ kernel streams K and
accumulates a contiguous N-block, so the GPU-tiled layout is read in place
(no repack, no extra host memory). Block scales are e8m0 (1 byte / 32 K)."""
gub, gus, gob = banks["gate_up"], banks["gate_up_scale"], banks["gate_up_bias"]
dnb, dns, dob = banks["down"], banks["down_scale"], banks["down_bias"]
- assert gub[0].dtype == torch.uint8 and dnb[0].dtype == torch.uint8, (gub[0].dtype, dnb[0].dtype)
- assert gus[0].dtype == torch.uint8 and dns[0].dtype == torch.uint8, (gus[0].dtype, dns[0].dtype)
- assert gob[0].dtype == torch.bfloat16 and dob[0].dtype == torch.bfloat16, (gob[0].dtype, dob[0].dtype)
+ assert gub[rep].dtype == torch.uint8 and dnb[rep].dtype == torch.uint8, (gub[rep].dtype, dnb[rep].dtype)
+ assert gus[rep].dtype == torch.uint8 and dns[rep].dtype == torch.uint8, (gus[rep].dtype, dns[rep].dtype)
+ assert gob[rep].dtype == torch.bfloat16 and dob[rep].dtype == torch.bfloat16, (gob[rep].dtype, dob[rep].dtype)
# gate_up_blocks [E, H//2, 2I]; down_blocks [E, I//2, H]
- H = int(gub[0].shape[1] * 2)
- I = int(gub[0].shape[2] // 2)
- assert gub[0].shape[2] == 2 * I
+ H = int(gub[rep].shape[1] * 2)
+ I = int(gub[rep].shape[2] // 2)
+ assert gub[rep].shape[2] == 2 * I
assert H % 32 == 0 and I % 32 == 0, (H, I)
- assert tuple(dnb[0].shape[1:]) == (I // 2, H), (dnb[0].shape, H, I)
- assert tuple(gus[0].shape[1:]) == (H // 32, 2 * I), (gus[0].shape, H, I)
- assert tuple(dns[0].shape[1:]) == (I // 32, H), (dns[0].shape, H, I)
- assert tuple(gob[0].shape[1:]) == (2 * I,) and tuple(dob[0].shape[1:]) == (H,)
+ assert tuple(dnb[rep].shape[1:]) == (I // 2, H), (dnb[rep].shape, H, I)
+ assert tuple(gus[rep].shape[1:]) == (H // 32, 2 * I), (gus[rep].shape, H, I)
+ assert tuple(dns[rep].shape[1:]) == (I // 32, H), (dns[rep].shape, H, I)
+ assert tuple(gob[rep].shape[1:]) == (2 * I,) and tuple(dob[rep].shape[1:]) == (H,)
banks = self._tables(gate_up=gub, down=dnb, gate_up_scale=gus, down_scale=dns, gate_up_bias=gob, down_bias=dob)
return banks, {}, (H, I)
- def _resolve_dsfp4_banks(self, banks: dict) -> tuple[dict, dict, tuple[int, int]]:
+ def _resolve_dsfp4_banks(self, banks: dict, rep: int = 0) -> tuple[dict, dict, tuple[int, int]]:
"""DeepSeek-V4 ``ds_fp4`` schema: row-major e2m1 (2/byte) + e8m0 per-32 block
scales, no global, no bias. Layout matches nvfp4 (K contiguous per output row),
so the C++ GEMV reads it in place. The kernel additionally FP8-round-trips the
activations (the checkpoint's fp8 block) to match the DeepSeek W4A8 reference."""
gup, gus = banks["gate_up"], banks["gate_up_scale"]
dnp, dns = banks["down"], banks["down_scale"]
- assert gup[0].dtype == torch.uint8 and dnp[0].dtype == torch.uint8, (gup[0].dtype, dnp[0].dtype)
- assert gus[0].element_size() == 1 and dns[0].element_size() == 1, "block scales must be 1 byte"
- I = int(gup[0].shape[1] // 2)
- H = int(gup[0].shape[2] * 2)
- assert gup[0].shape[1] == 2 * I
+ assert gup[rep].dtype == torch.uint8 and dnp[rep].dtype == torch.uint8, (gup[rep].dtype, dnp[rep].dtype)
+ assert gus[rep].element_size() == 1 and dns[rep].element_size() == 1, "block scales must be 1 byte"
+ I = int(gup[rep].shape[1] // 2)
+ H = int(gup[rep].shape[2] * 2)
+ assert gup[rep].shape[1] == 2 * I
assert H % 32 == 0 and I % 32 == 0, (H, I) # e8m0 scales per 32 along K
- assert tuple(dnp[0].shape[1:]) == (H, I // 2), (dnp[0].shape, H, I)
- assert tuple(gus[0].shape[1:]) == (2 * I, H // 32), (gus[0].shape, I, H)
- assert tuple(dns[0].shape[1:]) == (H, I // 32), (dns[0].shape, H, I)
+ assert tuple(dnp[rep].shape[1:]) == (H, I // 2), (dnp[rep].shape, H, I)
+ assert tuple(gus[rep].shape[1:]) == (2 * I, H // 32), (gus[rep].shape, I, H)
+ assert tuple(dns[rep].shape[1:]) == (H, I // 32), (dns[rep].shape, H, I)
banks = self._tables(gate_up=gup, down=dnp, gate_up_scale=gus, down_scale=dns)
return banks, {}, (H, I)
diff --git a/python/freetoken/moe/expert_banks.py b/python/freetoken/moe/expert_banks.py
index 03ce4b75..7e25e58f 100644
--- a/python/freetoken/moe/expert_banks.py
+++ b/python/freetoken/moe/expert_banks.py
@@ -18,8 +18,8 @@ import torch
from freetoken.layers.quantization import QuantKind
from freetoken.utils import init_logger
-from .host_banks import alloc_layer_banks
-from .offload_cache import _BANK_BYTES_PER_EXPERT
+from .host_banks import HostBank
+from .offload_cache import _BANK_BYTES_PER_EXPERT, placeholder_bank
logger = init_logger(__name__)
@@ -52,6 +52,8 @@ class ExpertBanks:
layer_tags: list | None = None
# per bank, per layer: bytes past an expert matrix's last row that the kernel reads and that must read as zeros (ggml MMQ); None = no such reads
row_tail_bytes: dict[str, list[int]] | None = None
+ # per-layer quant_format of experts in several formats (``quant_format``/``kind``/``kernel`` then name the most common one); None = every layer is ``quant_format``
+ layer_formats: list[str] | None = None
def _dummy_fill(role: str, tensor: torch.Tensor) -> None:
@@ -80,6 +82,7 @@ def build_expert_banks(
layer_sink=None,
dummy: bool = False,
resident: bool = False,
+ layers: list[int] | None = None,
) -> ExpertBanks:
"""Fill host banks in the kernel's layout from a stream of expert pieces.
@@ -89,6 +92,8 @@ def build_expert_banks(
are pinned in the background, otherwise the sink receives them (converter). ``dummy``
skips the pieces and fills the banks with finite random contents.
``resident`` packs into banks on ``device`` instead of host banks (the resident 'fused' experts).
+ ``layers`` fills only those bank layers (the ones ``method`` serves, for experts of several
+ formats): pieces of other layers are skipped and the other layers' sources are None.
"""
from freetoken.moe.host_banks import LayerCompletionTracker, PinPipeline, pin_banks
from freetoken.moe.legacy_format import legacy_format_for
@@ -96,29 +101,34 @@ def build_expert_banks(
kernel = method.kernel
layout = method.layout()
E = method.cfg.num_experts
+ ids = list(range(num_layers)) if layers is None else sorted(layers)
+ filled = set(ids)
specs = {role: ((E, *spec.shape), spec.dtype) for role, spec in layout.items() if not spec.resident}
if resident:
assert layer_sink is None
hb = None
- banks = {role: [torch.empty(shape, dtype=dtype, device=device) for _ in range(num_layers)] for role, (shape, dtype) in specs.items()}
+ banks = {role: {l: torch.empty(shape, dtype=dtype, device=device) for l in ids} for role, (shape, dtype) in specs.items()}
else:
- hb = alloc_layer_banks(specs, num_layers)
- banks = {role: [b.tensor for b in hb[role]] for role in specs}
+ hb = {role: {l: HostBank(shape, dtype) for l in ids} for role, (shape, dtype) in specs.items()}
+ banks = {role: {l: b.tensor for l, b in per_layer.items()} for role, per_layer in hb.items()}
alphas = {
role: torch.empty(num_layers * E, dtype=spec.dtype, device=device)
for role, spec in layout.items() if spec.resident
}
+ def _sources() -> dict[str, list]:
+ return {role: [per_layer.get(l) for l in range(num_layers)] for role, per_layer in banks.items()}
+
if dummy:
for role, per_layer in banks.items():
- for tensor in per_layer:
+ for tensor in per_layer.values():
_dummy_fill(role, tensor)
for alpha in alphas.values():
alpha.fill_(1.0)
if hb is not None and torch.cuda.is_available():
pin_banks(hb)
return ExpertBanks(
- legacy_format_for(method.kind, kernel.name), banks,
+ legacy_format_for(method.kind, kernel.name), _sources(),
gate_up_alpha=alphas.get("gate_up_alpha"), down_alpha=alphas.get("down_alpha"),
kind=method.kind, kernel=kernel.name, layout=layout,
)
@@ -130,6 +140,8 @@ def build_expert_banks(
for layer_id, e0, e1, piece in pieces:
if not (0 <= layer_id < num_layers and 0 <= e0 < e1 <= E):
raise ValueError(f"expert piece out of range: layer {layer_id}, experts {e0}:{e1} of {num_layers} x {E}")
+ if layer_id not in filled:
+ continue
# refuse before writing: a duplicate row would also complete the layer early and hand the sink a half-filled bank
if written[layer_id, e0:e1].any():
raise ValueError(f"expert rows written more than once: layer {layer_id}, experts {e0}:{e1}")
@@ -141,8 +153,9 @@ def build_expert_banks(
if tracker is not None:
for _ in range(e1 - e0):
tracker.note(layer_id)
- missing = (written == 0).nonzero().tolist()
+ missing = (written[ids] == 0).nonzero().tolist()
if missing:
+ missing = [[ids[l], e] for l, e in missing]
raise ValueError(f"expert banks were not filled: {len(missing)} (layer, expert) rows missing (first {missing[:4]})")
if layer_sink is not None:
@@ -154,12 +167,74 @@ def build_expert_banks(
_fill(None)
return ExpertBanks(
- legacy_format_for(method.kind, kernel.name), banks,
+ legacy_format_for(method.kind, kernel.name), _sources(),
gate_up_alpha=alphas.get("gate_up_alpha"), down_alpha=alphas.get("down_alpha"),
streamed=layer_sink is not None, kind=method.kind, kernel=kernel.name, layout=layout,
)
+def method_groups(layer_methods) -> list[tuple]:
+ """``[(method, [bank layers])]`` of the layers' expert methods, one entry per (kind, kernel), most layers first."""
+ groups: dict = {}
+ for layer, method in enumerate(layer_methods):
+ groups.setdefault((method.kind, method.kernel.name), (method, []))[1].append(layer)
+ return sorted(groups.values(), key=lambda g: (-len(g[1]), g[1][0]))
+
+
+def is_mixed(layer_methods) -> bool:
+ return layer_methods is not None and len(method_groups(layer_methods)) > 1
+
+
+def merge_expert_banks(parts: list[ExpertBanks], layer_methods, num_experts: int) -> ExpertBanks:
+ """One ``ExpertBanks`` over the per-format ``parts`` (each filled only on its layers): the banks are the
+ union of the roles, a layer without a role gets a zero-byte placeholder (see ``OffloadMoeCache.layer_layouts``)."""
+ from freetoken.moe.legacy_format import legacy_format_for
+
+ num_layers = len(layer_methods)
+ sources: dict[str, list] = {}
+ for part in parts:
+ if part.gate_up_alpha is not None or part.down_alpha is not None:
+ raise ValueError(f"experts of several formats cannot use the {part.kernel} kernel, which keeps GPU-resident alphas")
+ for role, per_layer in part.sources.items():
+ merged = sources.setdefault(role, [None] * num_layers)
+ for layer, tensor in enumerate(per_layer):
+ if tensor is not None:
+ assert merged[layer] is None, (role, layer)
+ merged[layer] = tensor
+ for per_layer in sources.values():
+ for layer, tensor in enumerate(per_layer):
+ if tensor is None:
+ per_layer[layer] = placeholder_bank(num_experts)
+ formats = [legacy_format_for(m.kind, m.kernel.name) for m in layer_methods]
+ primary = parts[0]
+ return ExpertBanks(
+ primary.quant_format, sources, streamed=all(p.streamed for p in parts),
+ kind=primary.kind, kernel=primary.kernel, layer_tags=formats, layer_formats=formats,
+ )
+
+
+def _mixed_expert_banks(model_path, model_config, layer_methods, device, dummy, parallel, workers, chunk, layer_sink=None) -> ExpertBanks:
+ from freetoken.moe.expert_pieces import iter_expert_pieces
+
+ num_layers = model_config.num_moe_layers
+ assert len(layer_methods) == num_layers, (len(layer_methods), num_layers)
+ groups = method_groups(layer_methods)
+ # resolve every format's reader before any bank exists, so a missing parallel reader falls back to serial with nothing allocated
+ readers = [
+ None if dummy else iter_expert_pieces(
+ model_path, model_config, method.kind, layers=layers, parallel=parallel, workers=workers, chunk=chunk
+ )
+ for method, layers in groups
+ ]
+ parts = []
+ for (method, layers), pieces in zip(groups, readers):
+ parts.append(build_expert_banks(
+ method, num_layers, pieces, device=device, layer_sink=layer_sink, dummy=dummy, layers=layers,
+ ))
+ logger.info_rank0(f"expert banks: {method.kind} / {method.kernel.name} on {len(layers)} of {num_layers} MoE layers")
+ return merge_expert_banks(parts, layer_methods, model_config.num_experts)
+
+
_PARALLEL_CHUNK = 8 << 20 # default O_DIRECT chunk for the parallel reader
@@ -216,19 +291,24 @@ def ftw_bank_bytes(model_path: str) -> int | None:
return sum(t["nbytes"] for t in tensors if t.get("kind") == "experts_bank")
-def bank_bytes_estimate(model_config, method=None) -> int | None:
+def _method_expert_bytes(method) -> int:
+ return method.cfg.num_experts * sum(
+ math.prod(spec.shape) * torch.empty((), dtype=spec.dtype).element_size()
+ for spec in method.layout().values() if not spec.resident
+ )
+
+
+def bank_bytes_estimate(model_config, method=None, layer_methods=None) -> int | None:
"""Estimated total expert-bank bytes of a raw checkpoint before loading it.
- With a bound expert ``method`` the kernel's layout gives the exact host bytes; otherwise the
- format-tag table sizes the format. ``None`` for unknown formats or missing dims
- (callers then skip the pre-load sizing)."""
+ With a bound expert ``method`` the kernel's layout gives the exact host bytes (``layer_methods``,
+ one per MoE layer, for experts of several formats); otherwise the format-tag table sizes the
+ format. ``None`` for unknown formats or missing dims (callers then skip the pre-load sizing)."""
layers = getattr(model_config, "num_moe_layers", None)
+ if layer_methods is not None:
+ return sum(_method_expert_bytes(m) for m in layer_methods)
if method is not None and layers:
- per_expert = sum(
- math.prod(spec.shape) * torch.empty((), dtype=spec.dtype).element_size()
- for spec in method.layout().values() if not spec.resident
- )
- return layers * method.cfg.num_experts * per_expert
+ return layers * _method_expert_bytes(method)
expert_quant = getattr(model_config, "expert_quant", "none")
fmt = expert_quant if expert_quant != "none" else (
getattr(model_config, "moe_weight_format", None) or "bf16"
@@ -256,6 +336,7 @@ def load_expert_banks(
layer_sink=None,
layer_residency: list[str] | None = None,
resident: bool = False,
+ layer_methods: list | None = None,
) -> ExpertBanks:
"""Load (or fabricate, with ``dummy=True``) the expert banks. Two paths, both returning
the same normalized ``ExpertBanks`` and both pinning after fill:
@@ -282,9 +363,15 @@ def load_expert_banks(
Applied labels are echoed on ``ExpertBanks.layer_residency``; a loader that settles some other way leaves it ``None`` (CPU-layer decode still works on pinned banks, it just saves no pin quota).
``resident`` (with ``method``): the banks land on ``device`` for the resident 'fused' experts; no host bank outlives the load.
+
+ ``layer_methods`` (one bound method per MoE layer) serves experts of several formats: each
+ format fills its own layers and the banks merge over the union of roles (``merge_expert_banks``).
"""
from freetoken.checkpoint.ftw import is_ftw_checkpoint, load_ftw_banks, load_ftw_banks_to_device
+ mixed = is_mixed(layer_methods)
+ if mixed and (resident or layer_residency is not None):
+ raise ValueError("experts of several formats are served from the offload slot cache only (no resident experts, no --moe-cpu-layers)")
if resident:
assert method is not None and layer_sink is None and layer_residency is None
if model_path and is_ftw_checkpoint(model_path) and not dummy:
@@ -330,6 +417,8 @@ def load_expert_banks(
from freetoken.moe.host_banks import requested_residency
def _build(par: bool) -> ExpertBanks:
+ if mixed:
+ return _mixed_expert_banks(model_path, model_config, layer_methods, device, dummy, par, workers, chunk, layer_sink)
if method is not None and method.kind is QuantKind.GGUF:
from freetoken.models.gguf.experts import load_gguf_expert_banks
diff --git a/python/freetoken/moe/expert_pieces.py b/python/freetoken/moe/expert_pieces.py
index 79f655a9..917d84bb 100644
--- a/python/freetoken/moe/expert_pieces.py
+++ b/python/freetoken/moe/expert_pieces.py
@@ -8,7 +8,7 @@ their ``_scale`` / ``_global`` companions, or an already fused ``gate_up``). The
from __future__ import annotations
-from typing import Callable, Iterable, Iterator
+from typing import Callable, Collection, Iterable, Iterator
import torch
from freetoken.layers.quantization import QuantKind
@@ -38,24 +38,34 @@ def _model_hook(spec, name: str):
def iter_expert_pieces(
- model_path: str, config, kind: QuantKind, *, parallel: bool = False, workers: int = 8, chunk: int = 8 << 20
+ model_path: str,
+ config,
+ kind: QuantKind,
+ *,
+ layers: Collection[int] | None = None,
+ parallel: bool = False,
+ workers: int = 8,
+ chunk: int = 8 << 20,
) -> Iterator[Piece]:
"""The pieces of ``model_path``'s routed experts, stored as ``kind``.
- A family may ship its own ``iter_expert_pieces(model_path, config, kind, *, parallel, workers,
- chunk)`` for the storage forms only it uses (it returns None for the rest); otherwise bf16
+ A family may ship its own ``iter_expert_pieces(model_path, config, kind, *, layers, parallel,
+ workers, chunk)`` for the storage forms only it uses (it returns None for the rest); otherwise bf16
experts come from the family's stacked ``iter_weights`` and NVFP4 experts from its
``nvfp4_expert_spec``. The reader is resolved here, before any bank is allocated, so a
missing parallel reader raises ``NotImplementedError`` while a serial fallback is still cheap.
+
+ ``layers`` (bank layer ids, default all) limits the pieces to those layers, for experts of several
+ formats: a reader skips the other layers' tensors and checks only these layers for completeness.
"""
spec = get_model_spec(config.architectures[0])
hook = _model_hook(spec, "iter_expert_pieces")
if hook is not None:
- pieces = hook(model_path, config, kind, parallel=parallel, workers=workers, chunk=chunk)
+ pieces = hook(model_path, config, kind, layers=layers, parallel=parallel, workers=workers, chunk=chunk)
if pieces is not None:
return pieces
if kind is QuantKind.NONE:
- return _bf16_pieces(model_path, config, spec, parallel=parallel, workers=workers, chunk=chunk)
+ return _bf16_pieces(model_path, config, spec, layers=layers, parallel=parallel, workers=workers, chunk=chunk)
if kind is QuantKind.NVFP4:
spec_hook = _model_hook(spec, "nvfp4_expert_spec")
if spec_hook is None:
@@ -63,7 +73,7 @@ def iter_expert_pieces(
from freetoken.models.nvfp4_banks import iter_nvfp4_expert_pieces
return iter_nvfp4_expert_pieces(
- model_path, config, spec_hook(model_path, config), parallel=parallel, workers=workers, chunk=chunk
+ model_path, config, spec_hook(model_path, config), layers=layers, parallel=parallel, workers=workers, chunk=chunk
)
raise NotImplementedError(f"{spec.module} provides no expert reader for {kind!r} experts")
@@ -81,18 +91,31 @@ def packed_expert_source_info(key: str) -> tuple[int, str] | None:
return None
-def stacked_expert_pieces(tensors: Iterable[tuple[str, torch.Tensor]], config) -> Iterator[Piece]:
- """Whole-layer ``experts.gate_up_proj`` / ``experts.down_proj`` tensors -> one piece per layer."""
+def _source_layer(key: str) -> int | None:
+ """Checkpoint layer of a ``model.layers.N....`` key."""
+ parts = key.split(".")
+ return int(parts[2]) if len(parts) > 2 and parts[:2] == ["model", "layers"] and parts[2].isdigit() else None
+
+
+def stacked_expert_pieces(tensors: Iterable[tuple[str, torch.Tensor]], config, *, layers: Collection[int] | None = None) -> Iterator[Piece]:
+ """Whole-layer ``experts.gate_up_proj`` / ``experts.down_proj`` tensors -> one piece per layer, for ``layers`` (bank layer ids, default all)."""
+ keep = None if layers is None else set(layers)
pending: dict[int, dict[str, torch.Tensor]] = {}
num_experts = config.num_experts
for name, tensor in tensors:
info = packed_expert_source_info(name)
if info is None:
+ # another format's layer stores its experts in another form
+ layer = _source_layer(name)
+ if keep is not None and layer is not None and bank_layer_of(config, layer) not in keep:
+ continue
raise ValueError(f"Unexpected expert weight key: {name}")
layer, packed_name = info
bank_layer = bank_layer_of(config, layer)
if bank_layer is None:
raise ValueError(f"Unexpected MoE expert layer {layer}; expected a routed-expert layer")
+ if keep is not None and bank_layer not in keep:
+ continue
if tensor.size(0) != num_experts:
raise ValueError(f"Unexpected {packed_name} expert count {tensor.size(0)}; expected {num_experts}")
piece = pending.setdefault(bank_layer, {})
@@ -104,7 +127,7 @@ def stacked_expert_pieces(tensors: Iterable[tuple[str, torch.Tensor]], config) -
raise ValueError(f"Missing MoE expert source layers: {sorted(pending)}")
-def _bf16_pieces(model_path: str, config, spec, *, parallel: bool, workers: int, chunk: int) -> Iterator[Piece]:
+def _bf16_pieces(model_path: str, config, spec, *, layers: Collection[int] | None, parallel: bool, workers: int, chunk: int) -> Iterator[Piece]:
device = torch.device("cpu")
if parallel:
iter_weights = _model_hook(spec, "iter_weights_parallel")
@@ -116,7 +139,7 @@ def _bf16_pieces(model_path: str, config, spec, *, parallel: bool, workers: int,
else:
iter_weights = _load_attr(spec.module, spec.iter_weights)
tensors = iter_weights(model_path, device, include_moe_experts=True, include_non_moe=False)
- return stacked_expert_pieces(tensors, config)
+ return stacked_expert_pieces(tensors, config, layers=layers)
def per_expert_pieces(
diff --git a/python/freetoken/moe/host_banks.py b/python/freetoken/moe/host_banks.py
index 91e26487..95562a88 100644
--- a/python/freetoken/moe/host_banks.py
+++ b/python/freetoken/moe/host_banks.py
@@ -204,18 +204,6 @@ def alloc_banks(specs: dict[str, tuple[tuple[int, ...], torch.dtype]]) -> dict[s
return {name: HostBank(shape, dtype) for name, (shape, dtype) in specs.items()}
-def alloc_layer_banks(
- specs: dict[str, tuple[tuple[int, ...], torch.dtype]], num_layers: int
-) -> dict[str, list[HostBank]]:
- """Allocate per-layer host banks: ``{name: ([num_experts, ...] row shape, dtype)}``
- -> one independently allocated (page-aligned, independently pin/lock-able)
- ``HostBank`` per layer per name."""
- return {
- name: [HostBank(shape, dtype) for _ in range(num_layers)]
- for name, (shape, dtype) in specs.items()
- }
-
-
class _ResidencyPlan:
"""Per-layer ``HostResidency`` labels, ambiently visible to the bank settle points.
@@ -265,13 +253,13 @@ def _settle(bank: HostBank, residency: str) -> None:
bank.lock()
-def pin_banks(banks: dict[str, HostBank | list[HostBank]]) -> None:
+def pin_banks(banks: dict[str, HostBank | list[HostBank] | dict[int, HostBank]]) -> None:
"""Settle every bank after it has been filled -- pin-after-fill by default.
- List-valued entries are per-layer and honor the ambient :func:`requested_residency` plan; scalar banks always pin."""
+ List-valued entries (indexed by layer id) and dict-valued ones (layer id -> bank) are per-layer and honor the ambient :func:`requested_residency` plan; scalar banks always pin."""
plan = _requested_residency
for bank in banks.values():
- if isinstance(bank, list):
- for layer_id, layer_bank in enumerate(bank):
+ if isinstance(bank, (list, dict)):
+ for layer_id, layer_bank in bank.items() if isinstance(bank, dict) else enumerate(bank):
residency = (
HostResidency.PINNED.value if plan is None
else plan.residency_for(layer_id)
@@ -479,7 +467,6 @@ __all__ = [
"LayerCompletionTracker",
"PinPipeline",
"alloc_banks",
- "alloc_layer_banks",
"born_pinned_default",
"pin_banks",
"read_file_into",
diff --git a/python/freetoken/moe/offload_cache.py b/python/freetoken/moe/offload_cache.py
index a209f9a9..23ba3251 100644
--- a/python/freetoken/moe/offload_cache.py
+++ b/python/freetoken/moe/offload_cache.py
@@ -102,6 +102,11 @@ _BANK_BYTES_PER_EXPERT = {
MARLIN_MAX_CACHE_SIZE = 992
+def placeholder_bank(num_experts: int) -> torch.Tensor:
+ """The source of a bank whose role a layer's expert kernel does not read (experts of several formats): zero bytes per row."""
+ return torch.empty((num_experts, 0), dtype=torch.uint8)
+
+
def bank_layouts(sources: dict[str, list[torch.Tensor]], layer_tags: list | None = None) -> list[tuple]:
"""Each layer's layout key: its quant tag and every bank's (dtype, row shape). A slot pool holds layers of one key."""
num_layers = len(next(iter(sources.values())))
@@ -154,6 +159,9 @@ class OffloadMoeCache:
# bank layout from the expert kernel (a BankSpec per role); when given it replaces the _BANK_SCHEMAS lookup and the slot cap comes from max_slots
layout: dict | None = None
max_slots: int | None = None
+ # Per-layer kernel layouts for experts of several formats (replaces ``layout``). The banks are the union of their roles;
+ # a layer whose kernel reads no such role carries a zero-byte [num_experts, 0] uint8 placeholder (see placeholder_bank).
+ layer_layouts: list[dict] | None = None
# Each layer belongs to one pool: a contiguous slot range with its own LRU scope; pool p owns slots [sum(pool_sizes[:p]), +pool_sizes[p]). Slot ids in slot_for_id are local to the pool. None = one pool for all layers.
pool_of_layer: list[int] | None = None
pool_sizes: list[int] | None = None
@@ -162,7 +170,12 @@ class OffloadMoeCache:
policy_ids = {"lru": 0}
assert self.cache_policy in policy_ids
assert self.decode_target in ("gpu", "cpu", "hybrid"), self.decode_target
- if self.layout is None:
+ if self.layer_layouts is not None:
+ assert self.layout is None, "pass layout or layer_layouts, not both"
+ assert len(self.layer_layouts) == self.num_layers, (len(self.layer_layouts), self.num_layers)
+ if any(spec.resident for layout in self.layer_layouts for spec in layout.values()):
+ raise ValueError("experts of several formats cannot use a kernel with GPU-resident alphas (marlin, b12x)")
+ elif self.layout is None:
assert self.quant_format in _BANK_SCHEMAS, f"unknown quant_format {self.quant_format!r}"
if self.pool_sizes is None:
self.pool_of_layer = [0] * self.num_layers
@@ -231,7 +244,11 @@ class OffloadMoeCache:
# carry independent host attributes -- see layer_residency) and their GPU
# slot caches, keyed by the format's bank schema (attached by
# set_bank_sources). The GPU slot cache stays one unified pool per bank.
- if self.layout is not None:
+ if self.layer_layouts is not None:
+ self.bank_schema = tuple(dict.fromkeys(
+ role for layout in self.layer_layouts for role, spec in layout.items() if not spec.resident
+ ))
+ elif self.layout is not None:
self.bank_schema = tuple(role for role, spec in self.layout.items() if not spec.resident)
else:
self.bank_schema = _BANK_SCHEMAS[self.quant_format]
@@ -371,9 +388,9 @@ class OffloadMoeCache:
for s, r, d in zip(self.pool_sizes[:-1], row, pad):
offsets[-1].append(offsets[-1][-1] + s * r + d)
self._pool_offsets = offsets
- # local slots below this index overlap the prefill staging area in some bank
+ # local slots below this index overlap the prefill staging area in some bank; a pool's zero-byte rows (a role its kernel does not read) overlap nothing
self._pool_stable_from = [
- max(min(max(-(-(2 * stage[k] - offsets[k][p]) // rows[k][p]), 0), s) for k in range(len(rows)))
+ max((min(max(-(-(2 * stage[k] - offsets[k][p]) // rows[k][p]), 0), s) for k in range(len(rows)) if rows[k][p]), default=0)
for p, s in enumerate(self.pool_sizes)
]
self._inval = []
@@ -382,6 +399,8 @@ class OffloadMoeCache:
for k in range(len(rows)):
lo, hi = b * stage[k], (b + 1) * stage[k]
for p, (base, s) in enumerate(spans):
+ if not rows[k][p]:
+ continue
first = max((lo - offsets[k][p]) // rows[k][p], 0)
end = min(-(-(hi - offsets[k][p]) // rows[k][p]), s)
mask[base + first : base + max(first, end)] = True
@@ -488,8 +507,12 @@ class OffloadMoeCache:
for layer_id, source in enumerate(per_layer):
assert source.is_contiguous(), f"bank {name!r} layer {layer_id} must be contiguous"
assert source.size(0) == self.num_experts, (name, layer_id, source.shape)
- if self.layout is not None:
- spec = self.layout[name]
+ spec = self.layout[name] if self.layout is not None else None
+ if self.layer_layouts is not None:
+ spec = self.layer_layouts[layer_id].get(name)
+ if spec is None and source.numel() != 0:
+ raise ValueError(f"layer {layer_id}'s expert kernel reads no {name!r} bank, so it takes a zero-byte placeholder")
+ if spec is not None:
if tuple(source.shape[1:]) != tuple(spec.shape) or source.dtype != spec.dtype:
raise ValueError(
f"bank {name!r} rows are {tuple(source.shape[1:])} {source.dtype} but the expert kernel's layout "
@@ -586,6 +609,9 @@ class OffloadMoeCache:
# a 0 placeholder keeps the descriptor shape
layer_src_ptrs[layer_id].append(0)
continue
+ if source.numel() == 0:
+ layer_src_ptrs[layer_id].append(0) # placeholder bank: zero bytes per row, never dereferenced
+ continue
# The kernel dereferences these on the GPU, so store each host bank's
# device alias (== data_ptr() under UVA identity; differs on
# Windows/WDDM).
@@ -980,6 +1006,8 @@ class OffloadMoeCache:
dst, src, nbytes = [], [], []
for b, feat in enumerate(self._row_bytes[layout]):
stage = self._copy_dst_ptrs_host[b] + buffer_id * self._stage_bytes[b]
+ if feat == 0:
+ continue
if feat < _SMALL_BANK_FEAT_BYTES:
# Whole layer as one entry, EVEN with zero misses: it keeps every
# batch entry above the driver's async floor and covers the hit
@@ -1229,6 +1257,8 @@ class OffloadMoeCache:
views = self.stage_views(layer_id, 0) if whole else self.pool_bank_views(layer_id)
for (per_layer, _), cache in zip(self.banks, views):
+ if cache.numel() == 0:
+ continue
fast_index_copy_jit(
cache,
self.evict_slots,
diff --git a/tests/models/test_models_loader.py b/tests/models/test_models_loader.py
index b0ac2b13..4f634c4c 100644
--- a/tests/models/test_models_loader.py
+++ b/tests/models/test_models_loader.py
@@ -201,3 +201,22 @@ def test_stacked_expert_pieces_pair_each_layer_in_arrival_order():
assert torch.equal(pieces[1][3]["gate_up"], torch.full((2, 3, 4), 2.0))
with pytest.raises(ValueError, match="Missing MoE expert source layers"):
list(stacked_expert_pieces(tensors[:3], config))
+
+
+def test_stacked_expert_pieces_skip_the_layers_of_other_formats():
+ from freetoken.moe.expert_pieces import stacked_expert_pieces
+
+ config = SimpleNamespace(num_layers=3, num_experts=2)
+ tensors = [
+ ("model.layers.0.mlp.experts.gate_up_proj", torch.full((2, 3, 4), 2.0)),
+ ("model.layers.1.mlp.experts.0.gate_proj.weight_scale", torch.zeros(3, 1)),
+ ("model.layers.0.mlp.experts.down_proj", torch.full((2, 4, 3), 10.0)),
+ ("model.layers.2.mlp.experts.gate_up_proj", torch.full((2, 3, 4), 3.0)),
+ ("model.layers.2.mlp.experts.down_proj", torch.full((2, 4, 3), 11.0)),
+ ]
+
+ pieces = list(stacked_expert_pieces(tensors, config, layers=[0]))
+
+ assert [(layer, e0, e1) for layer, e0, e1, _ in pieces] == [(0, 0, 2)]
+ with pytest.raises(ValueError, match="Unexpected expert weight key"):
+ list(stacked_expert_pieces(tensors, config))
diff --git a/tests/models/test_qwen3_5_moe_weight.py b/tests/models/test_qwen3_5_moe_weight.py
index 158f1f45..16f439a9 100644
--- a/tests/models/test_qwen3_5_moe_weight.py
+++ b/tests/models/test_qwen3_5_moe_weight.py
@@ -507,6 +507,8 @@ def test_nvfp4_expert_pieces_read_either_dialect_from_a_single_file(checkpoint):
spec = nvfp4_expert_spec(folder, config)
pieces = list(iter_nvfp4_expert_pieces(folder, config, spec))
assert len(pieces) == 2 * E
+ only = list(iter_nvfp4_expert_pieces(folder, config, spec, layers=[1]))
+ assert len(only) == E and {p[0] for p in only} == {1}
layer, e0, e1, piece = next(p for p in pieces if p[0] == 1 and p[1] == 2)
assert (e0, e1) == (2, 3)
base = f"{LM}.layers.1.mlp.experts.2.gate_proj"
@@ -529,6 +531,8 @@ def test_block_fp8_expert_pieces_read_either_dialect(checkpoint, parallel):
config = parse_config(cached_load_hf_config(folder))
pieces = list(iter_expert_pieces(folder, config, QuantKind.FP8_BLOCK, parallel=parallel))
assert len(pieces) == 2 * E
+ only = list(iter_expert_pieces(folder, config, QuantKind.FP8_BLOCK, layers=[1], parallel=parallel))
+ assert len(only) == E and {p[0] for p in only} == {1}
layer, e0, e1, piece = next(p for p in pieces if p[0] == 1 and p[1] == 2)
base = f"{LM}.layers.1.mlp.experts.2"
scale = "weight_scale_inv" if name == "fp8_block" else "weight_scale"
diff --git a/tests/moe/test_cpu_moe.py b/tests/moe/test_cpu_moe.py
index 38e79b27..56c6c18a 100644
--- a/tests/moe/test_cpu_moe.py
+++ b/tests/moe/test_cpu_moe.py
@@ -391,7 +391,7 @@ def test_cpu_decode_dsfp4_matches_gpu(bs):
max_tokens=bs,
device=dev,
swiglu_limit=swiglu_limit,
- act_block=128,
+ layer_formats=[("ds_fp4", {"act_block": 128})] * cache.num_layers,
)
hidden = torch.randn(bs, H, device=dev, dtype=torch.bfloat16)
@@ -689,7 +689,7 @@ def test_cpu_moe_decode_cuda_graph_replay_dsfp4():
ex = CpuMoeExecutor(
cache, top_k=top_k, activation="silu",
apply_router_weight_on_input=False, num_threads=8, max_tokens=bs, device=dev,
- swiglu_limit=limit, act_block=128,
+ swiglu_limit=limit, layer_formats=[("ds_fp4", {"act_block": 128})] * cache.num_layers,
)
b = cache.bank_sources
layer_banks = [b[n][layer].to(dev) for n in (
diff --git a/tests/moe/test_dsfp4_prequant.py b/tests/moe/test_dsfp4_prequant.py
index 8187402d..1212c166 100644
--- a/tests/moe/test_dsfp4_prequant.py
+++ b/tests/moe/test_dsfp4_prequant.py
@@ -49,7 +49,7 @@ def test_dsfp4_gpu_prequant_bit_parity(monkeypatch):
monkeypatch.setattr(cpu_executor, "_DSFP4_GPU_PREQUANT", not cpu_roundtrip)
ex = CpuMoeExecutor(
cache, top_k=TOPK, activation="silu", apply_router_weight_on_input=False,
- num_threads=4, max_tokens=2, device=dev, swiglu_limit=10.0, act_block=128,
+ num_threads=4, max_tokens=2, device=dev, swiglu_limit=10.0, layer_formats=[("ds_fp4", {"act_block": 128})] * cache.num_layers,
)
assert ex._gpu_prequant != cpu_roundtrip, "ds_fp4 on CUDA should enable GPU prequant"
out = ex.decode(0, x, wts, ids.clone()).clone()
diff --git a/tests/moe/test_dsfp4_reference_semantics.py b/tests/moe/test_dsfp4_reference_semantics.py
index 2f89cf33..f537b248 100644
--- a/tests/moe/test_dsfp4_reference_semantics.py
+++ b/tests/moe/test_dsfp4_reference_semantics.py
@@ -105,7 +105,7 @@ def test_cpu_executor_matches_the_transcription(block):
cache = SimpleNamespace(quant_format="ds_fp4", bank_sources={k: [v] for k, v in pinned.items()}, num_layers=1, num_experts=E)
dev = torch.device("cuda", 0)
x, slots, w = _inputs(2, dev)
- ex = CpuMoeExecutor(cache, top_k=TOP_K, activation="silu", apply_router_weight_on_input=False, num_threads=4, max_tokens=2, device=dev, swiglu_limit=LIMIT, act_block=block)
+ ex = CpuMoeExecutor(cache, top_k=TOP_K, activation="silu", apply_router_weight_on_input=False, num_threads=4, max_tokens=2, device=dev, swiglu_limit=LIMIT, layer_formats=[("ds_fp4", {"act_block": block})] * cache.num_layers)
got = ex.decode(0, x, w, slots.clone()).clone().float()
torch.cuda.synchronize()
del ex
@@ -162,7 +162,7 @@ def test_inactive_routes_contribute_zero_without_reading_their_slots(block):
buf.copy_(t.cpu())
pinned[name] = buf
cache = SimpleNamespace(quant_format="ds_fp4", bank_sources={k: [v] for k, v in pinned.items()}, num_layers=1, num_experts=E)
- ex = CpuMoeExecutor(cache, top_k=TOP_K, activation="silu", apply_router_weight_on_input=False, num_threads=4, max_tokens=2, device=torch.device("cuda", 0), swiglu_limit=LIMIT, act_block=block)
+ ex = CpuMoeExecutor(cache, top_k=TOP_K, activation="silu", apply_router_weight_on_input=False, num_threads=4, max_tokens=2, device=torch.device("cuda", 0), swiglu_limit=LIMIT, layer_formats=[("ds_fp4", {"act_block": block})] * cache.num_layers)
cpu_ids = torch.where(on_gpu, -1, slots)
cpu_out = ex.decode(0, x, w, cpu_ids.clone()).clone()
torch.cuda.synchronize()
diff --git a/tests/moe/test_mixed_expert_formats.py b/tests/moe/test_mixed_expert_formats.py
new file mode 100644
index 00000000..c9301dc1
--- /dev/null
+++ b/tests/moe/test_mixed_expert_formats.py
@@ -0,0 +1,288 @@
+"""Experts of several formats in one model: bf16 layers next to NVFP4 layers.
+
+A checkpoint may leave some MoE layers unquantized (Muse Spark keeps its first and last layer's
+experts bf16). Each format fills its own layers' banks, the banks merge over the union of roles with
+zero-byte placeholders, the slot cache keeps one pool per format, the FTW stores each layer's own
+banks, and the CPU executor computes each layer in its format.
+"""
+
+from __future__ import annotations
+
+from types import SimpleNamespace
+
+import pytest
+import torch
+
+from freetoken.distributed import set_tp_info, try_get_tp_info
+
+cuda = pytest.mark.skipif(not torch.cuda.is_available(), reason="needs CUDA")
+
+E, H, I, TOPK = 8, 256, 128, 2
+FORMATS = ["bf16", "nvfp4", "nvfp4", "nvfp4", "bf16"]
+L = len(FORMATS)
+
+_E2M1 = torch.tensor([0.0, 0.5, 1.0, 1.5, 2.0, 3.0, 4.0, 6.0, -0.0, -0.5, -1.0, -1.5, -2.0, -3.0, -4.0, -6.0])
+
+
+def _init_tp():
+ if try_get_tp_info() is None:
+ set_tp_info(rank=0, size=1)
+
+
+def _layer(layer_id: int, fmt: str):
+ from freetoken.layers.moe import OffloadMoELayer
+ from freetoken.layers.quantization import NoQuantConfig, QuantBackend, QuantConfig, set_quant_backend
+
+ _init_tp()
+ if fmt == "bf16":
+ quant = NoQuantConfig()
+ else:
+ set_quant_backend(QuantBackend.parse("moe.nvfp4=triton"))
+ quant = QuantConfig.from_hf({"quantization_config": {"quant_method": "modelopt", "quant_algo": "NVFP4", "ignore": ["lm_head"]}})
+ return OffloadMoELayer(layer_id, E, TOPK, H, I, quant_config=quant, prefix=f"model.layers.{layer_id}.mlp.experts")
+
+
+def _dequant(packed, scale, row_global):
+ n, k2 = packed.shape
+ codes = torch.stack([packed & 0xF, packed >> 4], dim=-1).view(n, 2 * k2).long()
+ return _E2M1.to(packed.device)[codes] * scale.float().repeat_interleave(16, dim=1) * row_global.float().unsqueeze(1)
+
+
+def _pieces(seed: int = 0):
+ """The checkpoint side: per layer, bf16 or ModelOpt NVFP4 pieces in the layer's format."""
+ g = torch.Generator().manual_seed(seed)
+ pieces = {}
+ for layer, fmt in enumerate(FORMATS):
+ if fmt == "bf16":
+ pieces[layer] = {
+ "gate_up": (torch.randn(E, 2 * I, H, generator=g) / 16).to(torch.bfloat16),
+ "down": (torch.randn(E, H, I, generator=g) / 16).to(torch.bfloat16),
+ }
+ else:
+ pieces[layer] = {
+ "gate_up": torch.randint(0, 256, (E, 2 * I, H // 2), dtype=torch.uint8, generator=g),
+ "gate_up_scale": (torch.rand(E, 2 * I, H // 16, generator=g) + 0.25).to(torch.float8_e4m3fn),
+ "gate_up_global": torch.full((E, 2 * I), 0.02, dtype=torch.float16),
+ "down": torch.randint(0, 256, (E, H, I // 2), dtype=torch.uint8, generator=g),
+ "down_scale": (torch.rand(E, H, I // 16, generator=g) + 0.25).to(torch.float8_e4m3fn),
+ "down_global": torch.full((E, H), 0.02, dtype=torch.float16),
+ }
+ return pieces
+
+
+def _reader(pieces, kind):
+ """A family reader for one storage form: it yields only the layers stored that way."""
+ from freetoken.layers.quantization import QuantKind
+
+ want = "bf16" if kind is QuantKind.NONE else "nvfp4"
+ for layer, fmt in enumerate(FORMATS):
+ if fmt == want:
+ yield layer, 0, E, pieces[layer]
+
+
+def _weights(pieces, layer):
+ """Dense fp32 (gate_up [E, 2I, H], down [E, H, I]) of one layer, the reference."""
+ p = pieces[layer]
+ if FORMATS[layer] == "bf16":
+ return p["gate_up"].float(), p["down"].float()
+ gu = torch.stack([_dequant(p["gate_up"][e], p["gate_up_scale"][e], p["gate_up_global"][e]) for e in range(E)])
+ dn = torch.stack([_dequant(p["down"][e], p["down_scale"][e], p["down_global"][e]) for e in range(E)])
+ return gu, dn
+
+
+def _reference(pieces, layer, hidden, weights, ids):
+ gu, dn = _weights(pieces, layer)
+ out = torch.zeros(hidden.shape, dtype=torch.float32)
+ x = hidden.float().cpu()
+ for t in range(hidden.shape[0]):
+ for k in range(ids.shape[1]):
+ e = int(ids[t, k])
+ h = gu[e] @ x[t]
+ out[t] += float(weights[t, k]) * (dn[e] @ (torch.nn.functional.silu(h[:I]) * h[I:]))
+ return out
+
+
+def _assert_close(out, ref):
+ torch.testing.assert_close(out.float().cpu(), ref, rtol=3e-2, atol=0.03 * float(ref.abs().max()))
+
+
+def _mixed_banks(pieces, device, layer_sink=None):
+ from freetoken.moe.expert_banks import build_expert_banks, merge_expert_banks, method_groups
+
+ methods = [_layer(layer, fmt).quant_method for layer, fmt in enumerate(FORMATS)]
+ parts = [
+ build_expert_banks(method, L, _reader(pieces, method.kind), device=device, layer_sink=layer_sink, layers=layers)
+ for method, layers in method_groups(methods)
+ ]
+ return merge_expert_banks(parts, methods, E), methods
+
+
+def _mixed_cache(device, pieces, *, decode_target="gpu", prefill_overlap=False):
+ from freetoken.moe.offload_cache import OffloadMoeCache, bank_layouts
+ from freetoken.moe.pool_policy import group_layers, make_pools, pool_of_layer, split_slots
+ from freetoken.engine.cache_budget import layer_slot_bytes
+
+ banks, methods = _mixed_banks(pieces, device)
+ slot_bytes = layer_slot_bytes(banks.sources)
+ groups = group_layers("unified", bank_layouts(banks.sources, banks.layer_tags), slot_bytes)
+ pools = make_pools(groups, E, TOPK, None)
+ sizes = split_slots(4 * E, pools)
+ cache = OffloadMoeCache(
+ num_layers=L, num_experts=E, cache_size=sum(sizes), device=device, quant_format=banks.quant_format,
+ decode_target=decode_target, hybrid_max_fetch=1, prefill_overlap=prefill_overlap,
+ layer_layouts=[m.layout() for m in methods], pool_of_layer=pool_of_layer(pools, L), pool_sizes=sizes,
+ )
+ cache.set_bank_sources(banks.sources, layer_tags=banks.layer_tags)
+ cache.reset()
+ layers = [_layer(layer, fmt) for layer, fmt in enumerate(FORMATS)]
+ for layer in layers:
+ layer.offload_cache = cache
+ return cache, layers, banks
+
+
+def test_engine_binds_the_most_common_format_and_checks_every_layer():
+ from freetoken.engine.engine import _check_layer_formats, offload_layer_methods, shared_offload_method
+ from freetoken.layers.base import BaseOP
+ from freetoken.layers.quantization import QuantKind
+ from freetoken.moe.expert_banks import ExpertBanks
+
+ class Model(BaseOP):
+ def __init__(self):
+ self.layers = [_layer(layer, fmt) for layer, fmt in reversed(list(enumerate(FORMATS)))]
+
+ model = Model()
+ methods = offload_layer_methods(model)
+ assert [m.kind for m in methods] == [QuantKind.NONE if f == "bf16" else QuantKind.NVFP4 for f in FORMATS]
+ assert shared_offload_method(model).kind is QuantKind.NVFP4
+ _check_layer_formats(ExpertBanks("nvfp4", {}, layer_formats=list(FORMATS)), methods)
+ with pytest.raises(ValueError, match=r"MoE layers \[0, 4\]"):
+ _check_layer_formats(ExpertBanks("nvfp4", {}), methods)
+
+
+def test_mixed_dummy_banks_merge_the_roles_with_placeholders():
+ from freetoken.moe.expert_banks import load_expert_banks
+
+ methods = [_layer(layer, fmt).quant_method for layer, fmt in enumerate(FORMATS)]
+ config = SimpleNamespace(num_moe_layers=L, num_experts=E)
+ banks = load_expert_banks("", config, method=methods[1], device=torch.device("cpu"), dummy=True, layer_methods=methods)
+ assert banks.layer_formats == FORMATS and banks.quant_format == "nvfp4"
+ assert set(banks.sources) == {"gate_up", "gate_up_scale", "gate_up_global", "down", "down_scale", "down_global"}
+ for layer, fmt in enumerate(FORMATS):
+ if fmt == "bf16":
+ assert banks.sources["gate_up"][layer].shape == (E, 2 * I, H) and banks.sources["gate_up"][layer].dtype == torch.bfloat16
+ assert banks.sources["down_scale"][layer].shape == (E, 0)
+ else:
+ assert banks.sources["gate_up"][layer].shape == (E, 2 * I, H // 2) and banks.sources["gate_up"][layer].dtype == torch.uint8
+ with pytest.raises(ValueError, match="several formats"):
+ load_expert_banks("", config, method=methods[1], device=torch.device("cpu"), dummy=True,
+ layer_methods=methods, layer_residency=["pinned"] * L)
+
+
+@cuda
+def test_mixed_banks_give_each_reader_its_layers_and_resolve_every_reader_first(monkeypatch):
+ from freetoken.moe import expert_pieces
+ from freetoken.moe.expert_banks import _mixed_expert_banks
+
+ pieces = _pieces(seed=2)
+ methods = [_layer(layer, fmt).quant_method for layer, fmt in enumerate(FORMATS)]
+ events = []
+
+ def fake_reader(model_path, config, kind, *, layers=None, parallel=False, workers=8, chunk=0):
+ events.append(("resolve", sorted(layers)))
+
+ def read():
+ events.append(("read", sorted(layers)))
+ for layer in layers:
+ yield layer, 0, E, pieces[layer]
+
+ return read()
+
+ monkeypatch.setattr(expert_pieces, "iter_expert_pieces", fake_reader)
+ config = SimpleNamespace(num_moe_layers=L, num_experts=E)
+ banks = _mixed_expert_banks("", config, methods, torch.device("cpu"), False, False, 8, 0)
+ assert sorted(layers for step, layers in events if step == "resolve") == [[0, 4], [1, 2, 3]]
+ assert [step for step, _ in events[:2]] == ["resolve", "resolve"]
+ assert banks.layer_formats == FORMATS
+
+
+@cuda
+def test_mixed_layers_decode_and_prefill_through_their_pools():
+ torch.manual_seed(0)
+ device = torch.device("cuda")
+ pieces = _pieces()
+ cache, layers, banks = _mixed_cache(device, pieces, prefill_overlap=True)
+ assert len(cache.pool_sizes) == 2 and cache._raw_banks
+ # a staging buffer holds one bf16 layer (the widest rows); two of them still fit the 13 + 19 slots
+ assert cache.prefill_overlap
+ hidden = torch.randn(1, H, dtype=torch.bfloat16, device=device) / 4
+ weights = torch.rand(1, TOPK, device=device)
+ for step in range(6):
+ for layer in layers:
+ ids = torch.randperm(E)[:TOPK].to(torch.int32).view(1, TOPK)
+ out = layer._decode_routed(hidden, weights, ids.clone().to(device))
+ _assert_close(out, _reference(pieces, layer.layer_id, hidden, weights, ids))
+
+ M = 16
+ hidden = torch.randn(M, H, dtype=torch.bfloat16, device=device) / 4
+ ids = torch.stack([torch.randperm(E)[:TOPK] for _ in range(M)]).to(torch.int32)
+ weights = torch.rand(M, TOPK, device=device)
+ for layer in layers:
+ # the bf16 prefill kernel overwrites its input
+ out = layer._prefill_routed(hidden.clone(), weights, ids.to(device))
+ _assert_close(out, _reference(pieces, layer.layer_id, hidden, weights, ids))
+ # decode after the prefill stomped the staging slots must reload, not read another format's bytes
+ for layer in layers:
+ ids = torch.randperm(E)[:TOPK].to(torch.int32).view(1, TOPK)
+ out = layer._decode_routed(hidden[:1], weights[:1], ids.clone().to(device))
+ _assert_close(out, _reference(pieces, layer.layer_id, hidden[:1], weights[:1], ids))
+
+
+@cuda
+def test_cpu_executor_computes_each_layer_in_its_format():
+ pytest.importorskip("freetoken.kernel._cpu_moe")
+ from freetoken.moe.cpu_executor import CpuMoeExecutor
+
+ torch.manual_seed(1)
+ device = torch.device("cuda")
+ pieces = _pieces(seed=1)
+ cache, layers, _ = _mixed_cache(device, pieces, decode_target="hybrid")
+ ex = CpuMoeExecutor(
+ cache, top_k=TOPK, activation="silu", apply_router_weight_on_input=False, num_threads=2,
+ max_tokens=4, device=device, layer_formats=[(f, {}) for f in FORMATS],
+ )
+ cache.set_cpu_executor(ex)
+ bs = 3
+ hidden = torch.randn(bs, H, dtype=torch.bfloat16, device=device) / 4
+ weights = torch.rand(bs, TOPK, device=device)
+ for layer in range(L):
+ ids = torch.stack([torch.randperm(E)[:TOPK] for _ in range(bs)]).to(torch.int32)
+ _assert_close(ex.decode(layer, hidden, weights, ids.to(device)), _reference(pieces, layer, hidden, weights, ids))
+ # hybrid: capped PCIe fetch on the GPU, the overflow misses on the CPU, per layer format
+ for _ in range(3):
+ for layer in layers:
+ ids = torch.stack([torch.randperm(E)[:TOPK] for _ in range(bs)]).to(torch.int32)
+ out = layer._decode_routed(hidden, weights, ids.clone().to(device))
+ _assert_close(out, _reference(pieces, layer.layer_id, hidden, weights, ids))
+
+
+@cuda
+def test_ftw_stores_each_layers_own_banks(tmp_path):
+ from freetoken.checkpoint.convert import _ConvertSink
+ from freetoken.checkpoint.ftw import FTWWriter, load_ftw_banks
+
+ pieces = _pieces(seed=2)
+ reference, _ = _mixed_banks(pieces, torch.device("cuda"))
+ writer = FTWWriter(str(tmp_path))
+ sink = _ConvertSink(writer, names={"gate_up": "gate_up_packed", "down": "down_packed"})
+ banks, _ = _mixed_banks(pieces, torch.device("cuda"), layer_sink=sink)
+ assert banks.streamed and sink.num_layers == L
+ writer.finalize({"quant_format": banks.quant_format, "layer_quant_formats": banks.layer_formats, "expert_bank_num_layers": L})
+
+ loaded = load_ftw_banks(str(tmp_path), num_layers=L)
+ assert loaded.layer_formats == FORMATS and loaded.layer_tags == FORMATS and loaded.quant_format == "nvfp4"
+ assert set(loaded.sources) == set(reference.sources)
+ for role, per_layer in reference.sources.items():
+ for layer, tensor in enumerate(per_layer):
+ got = loaded.sources[role][layer]
+ assert got.shape == tensor.shape and got.dtype == tensor.dtype, (role, layer)
+ assert torch.equal(got.view(torch.uint8), tensor.view(torch.uint8)), (role, layer)